├── .gitignore ├── LICENSE ├── README.md ├── chidley.go ├── codegenTemplates.go ├── data ├── test.xml └── typeCollapsing.xml ├── decl.go ├── encoding_test.go ├── extractor.go ├── fieldTemplate.go ├── fqn.go ├── genericReader.go ├── go.mod ├── go.sum ├── javaTypes.go ├── jaxbTemplate.go ├── makereadme ├── chidley.md ├── main.go ├── readme.sh.old ├── template.go └── util.go ├── mvnTemplate.go ├── node.go ├── nodeTypeInfo.go ├── printGoStructVisitor.go ├── printJavaJaxbVisitor.go ├── source.go ├── stdoutWriter.go ├── test_xml.sh ├── tests_util.go ├── util.go ├── util_test.go ├── xml ├── ActorPreludeSample.xml ├── BrookeWestSample.xml ├── Fantasia_con_imitazione_BWV563.xml ├── MODIS-Imagery-OilSpill.kml ├── MODIS-Imagery-OilSpill.kml.gz ├── amazon_api.xml ├── arxiv_search_query.xml.gz ├── pubmed_xml_12750255.xml.bz2 ├── test1.xml ├── test2.xml ├── test2a.xml ├── test2b.xml ├── test2c.xml ├── test3a.xml ├── test3b.xml ├── test3c.xml ├── test4.xml ├── testSpacePreserve.xml └── testType.xml ├── xml_source_test.go └── xml_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | 25 | *~ 26 | codegen/ 27 | TAGS 28 | #*# 29 | 30 | chidley 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2014 Glen Newton 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /chidley.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Copyright 2014,2015,2016 Glen Newton 4 | // glen.newton@gmail.com 5 | 6 | import ( 7 | "bufio" 8 | "bytes" 9 | "errors" 10 | "flag" 11 | "fmt" 12 | "io" 13 | "log" 14 | "os" 15 | "path/filepath" 16 | "sort" 17 | 18 | "text/template" 19 | "time" 20 | ) 21 | 22 | func init() { 23 | 24 | flag.BoolVar(&DEBUG, "d", DEBUG, "Debug; prints out much information") 25 | flag.BoolVar(&addDbMetadata, "B", addDbMetadata, "Add database metadata to created Go structs") 26 | flag.BoolVar(&classicStructNamesWithUnderscores, "C", classicStructNamesWithUnderscores, "Structs have underscores instead of CamelCase; how chidley used to produce output; includes name spaces (see -n)") 27 | flag.BoolVar(&codeGenConvert, "W", codeGenConvert, "Generate Go code to convert XML to JSON or XML (latter useful for validation) and write it to stdout") 28 | flag.BoolVar(&flattenStrings, "F", flattenStrings, "Assume complete representative XML and collapse tags with only a single string and no attributes") 29 | flag.BoolVar(&ignoreXmlDecodingErrors, "I", ignoreXmlDecodingErrors, "If XML decoding error encountered, continue") 30 | flag.BoolVar(&nameSpaceInJsonName, "n", nameSpaceInJsonName, "Use the XML namespace prefix as prefix to JSON name") 31 | flag.BoolVar(&prettyPrint, "p", prettyPrint, "Pretty-print json in generated code (if applicable)") 32 | flag.BoolVar(&progress, "r", progress, "Progress: every 50000 input tags (elements)") 33 | flag.BoolVar(&readFromStandardIn, "c", readFromStandardIn, "Read XML from standard input") 34 | flag.BoolVar(&sortByXmlOrder, "X", sortByXmlOrder, "Sort output of structs in Go code by order encounered in source XML (default is alphabetical order)") 35 | flag.BoolVar(&structsToStdout, "G", structsToStdout, "Only write generated Go structs to stdout") 36 | flag.BoolVar(&url, "u", url, "Filename interpreted as an URL") 37 | flag.BoolVar(&useType, "t", useType, "Use type info obtained from XML (int, bool, etc); default is to assume everything is a string; better chance at working if XMl sample is not complete") 38 | flag.BoolVar(&writeJava, "J", writeJava, "Generated Java code for Java/JAXB") 39 | flag.BoolVar(&xmlName, "x", xmlName, "Add XMLName (Space, Local) for each XML element, to JSON") 40 | flag.BoolVar(&keepXmlFirstLetterCase, "K", keepXmlFirstLetterCase, "Do not change the case of the first letter of the XML tag names") 41 | flag.BoolVar(&validateFieldTemplate, "m", validateFieldTemplate, "Validate the field template. Useful to make sure the template defined with -T is valid") 42 | 43 | flag.BoolVar(&ignoreLowerCaseXmlTags, "L", ignoreLowerCaseXmlTags, "Ignore lower case XML tags") 44 | 45 | flag.StringVar(&attributePrefix, "a", attributePrefix, "Prefix to attribute names") 46 | flag.StringVar(&baseJavaDir, "D", baseJavaDir, "Base directory for generated Java code (root of maven project)") 47 | flag.StringVar(&cdataStringName, "M", cdataStringName, "Set name of CDATA string field") 48 | flag.StringVar(&fieldTemplateString, "T", fieldTemplateString, "Field template for the struct field definition. Can include annotations. Default is for XML and JSON") 49 | flag.StringVar(&javaAppName, "k", javaAppName, "App name for Java code (appended to ca.gnewton.chidley Java package name))") 50 | flag.StringVar(&lengthTagAttribute, "A", lengthTagAttribute, "The tag name attribute to use for the max length Go annotations") 51 | flag.StringVar(&lengthTagName, "N", lengthTagName, "The tag name to use for the max length Go annotations") 52 | flag.StringVar(&lengthTagSeparator, "S", lengthTagSeparator, "The tag name separator to use for the max length Go annotations") 53 | flag.StringVar(&namePrefix, "e", namePrefix, "Prefix to struct (element) names; must start with a capital") 54 | flag.StringVar(&userJavaPackageName, "P", userJavaPackageName, "Java package name (rightmost in full package name") 55 | 56 | flag.StringVar(&ignoredXmlTags, "h", ignoredXmlTags, "List of XML tags to ignore; comma separated") 57 | 58 | flag.Int64Var(&lengthTagPadding, "Z", lengthTagPadding, "The padding on the max length tag attribute") 59 | 60 | } 61 | 62 | func handleParameters() error { 63 | flag.Parse() 64 | 65 | if codeGenConvert || writeJava { 66 | structsToStdout = false 67 | } 68 | 69 | numBoolsSet := countNumberOfBoolsSet(outputs) 70 | if numBoolsSet > 1 { 71 | log.Print(" ERROR: Only one of -W -J -X -V -c can be set") 72 | } else if numBoolsSet == 0 { 73 | log.Print(" ERROR: At least one of -W -J -X -V -c must be set") 74 | } 75 | if sortByXmlOrder { 76 | structSort = printStructsByXml 77 | } 78 | 79 | var err error 80 | ignoredXmlTagsMap, err = extractExcludedTags(ignoredXmlTags) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | if lengthTagName == "" && lengthTagAttribute == "" || lengthTagName != "" && lengthTagAttribute != "" { 86 | return nil 87 | } 88 | 89 | return errors.New("Both lengthTagName and lengthTagAttribute must be set") 90 | } 91 | 92 | func main() { 93 | //log.Println(fieldTemplateString) 94 | 95 | //EXP() 96 | log.SetFlags(log.LstdFlags | log.Lshortfile) 97 | 98 | err := handleParameters() 99 | 100 | if err != nil { 101 | log.Println(err) 102 | flag.Usage() 103 | return 104 | } 105 | 106 | err = runValidateFieldTemplate(validateFieldTemplate) 107 | if err != nil { 108 | 109 | return 110 | } 111 | if validateFieldTemplate { 112 | return 113 | } 114 | 115 | if len(flag.Args()) == 0 && !readFromStandardIn { 116 | fmt.Println("chidley xmlFileName|url") 117 | fmt.Println("xmlFileName can be .gz or .bz2: uncompressed transparently") 118 | flag.Usage() 119 | return 120 | } 121 | 122 | var sourceNames []string 123 | 124 | if !readFromStandardIn { 125 | sourceNames = flag.Args() 126 | } 127 | if !url && !readFromStandardIn { 128 | for i, _ := range sourceNames { 129 | sourceNames[i], err = filepath.Abs(sourceNames[i]) 130 | if err != nil { 131 | log.Fatal("FATAL ERROR: " + err.Error()) 132 | } 133 | } 134 | } 135 | 136 | sources, err := makeSourceReaders(sourceNames, url, readFromStandardIn) 137 | if err != nil { 138 | log.Fatal("FATAL ERROR: " + err.Error()) 139 | } 140 | 141 | ex := Extractor{ 142 | namePrefix: namePrefix, 143 | nameSuffix: nameSuffix, 144 | useType: useType, 145 | progress: progress, 146 | ignoreXmlDecodingErrors: ignoreXmlDecodingErrors, 147 | initted: false, 148 | } 149 | 150 | if DEBUG { 151 | log.Print("extracting") 152 | } 153 | 154 | m := &ex 155 | m.init() 156 | 157 | for source := range sources { 158 | if DEBUG { 159 | log.Println("READER", source) 160 | } 161 | err = m.extract(source.getReader()) 162 | 163 | if err != nil { 164 | log.Println("ERROR: " + err.Error()) 165 | if !ignoreXmlDecodingErrors { 166 | log.Fatal("FATAL ERROR: " + err.Error()) 167 | } 168 | } 169 | if DEBUG { 170 | log.Println("DONE READER", source) 171 | } 172 | } 173 | 174 | ex.done() 175 | 176 | switch { 177 | case codeGenConvert: 178 | generateGoCode(os.Stdout, sourceNames, &ex) 179 | 180 | case structsToStdout: 181 | generateGoStructs(os.Stdout, sourceNames[0], &ex) 182 | 183 | case writeJava: 184 | if len(userJavaPackageName) > 0 { 185 | javaAppName = userJavaPackageName 186 | } 187 | javaPackage := javaBasePackage + "." + javaAppName 188 | javaDir := baseJavaDir + "/" + mavenJavaBase + "/" + javaBasePackagePath + "/" + javaAppName 189 | 190 | os.RemoveAll(baseJavaDir) 191 | os.MkdirAll(javaDir+"/xml", 0755) 192 | date := time.Now() 193 | printJavaJaxbVisitor := PrintJavaJaxbVisitor{ 194 | alreadyVisited: make(map[string]bool), 195 | globalTagAttributes: ex.globalTagAttributes, 196 | nameSpaceTagMap: ex.nameSpaceTagMap, 197 | useType: useType, 198 | javaDir: javaDir, 199 | javaPackage: javaPackage, 200 | namePrefix: namePrefix, 201 | Date: date, 202 | } 203 | 204 | var onlyChild *Node 205 | for _, child := range ex.root.children { 206 | printJavaJaxbVisitor.Visit(child) 207 | // Bad: assume only one base element 208 | onlyChild = child 209 | } 210 | fullPath, err := getFullPath(sourceNames[0]) 211 | if err != nil { 212 | log.Fatal(err) 213 | } 214 | printJavaJaxbMain(onlyChild.makeJavaType(namePrefix, ""), javaDir, javaPackage, fullPath, date) 215 | printPackageInfo(onlyChild, javaDir, javaPackage, ex.globalTagAttributes, ex.nameSpaceTagMap) 216 | 217 | printMavenPom(baseJavaDir+"/pom.xml", javaAppName) 218 | } 219 | 220 | } 221 | 222 | func printPackageInfo(node *Node, javaDir string, javaPackage string, globalTagAttributes map[string][]*FQN, nameSpaceTagMap map[string]string) { 223 | 224 | //log.Printf("%+v\n", node) 225 | 226 | if node.space != "" { 227 | _ = findNameSpaces(globalTagAttributes[nk(node)]) 228 | //attributes := findNameSpaces(globalTagAttributes[nk(node)]) 229 | 230 | t := template.Must(template.New("package-info").Parse(jaxbPackageInfoTemplage)) 231 | packageInfoPath := javaDir + "/xml/package-info.java" 232 | fi, err := os.Create(packageInfoPath) 233 | if err != nil { 234 | log.Print("Problem creating file: " + packageInfoPath) 235 | panic(err) 236 | } 237 | defer fi.Close() 238 | 239 | writer := bufio.NewWriter(fi) 240 | packageInfo := JaxbPackageInfo{ 241 | BaseNameSpace: node.space, 242 | //AdditionalNameSpace []*FQN 243 | PackageName: javaPackage + ".xml", 244 | } 245 | err = t.Execute(writer, packageInfo) 246 | if err != nil { 247 | log.Println("executing template:", err) 248 | } 249 | bufio.NewWriter(writer).Flush() 250 | } 251 | 252 | } 253 | 254 | const XMLNS = "xmlns" 255 | 256 | func findNameSpaces(attributes []*FQN) []*FQN { 257 | if attributes == nil || len(attributes) == 0 { 258 | return nil 259 | } 260 | xmlns := make([]*FQN, 0) 261 | return xmlns 262 | } 263 | 264 | func printMavenPom(pomPath string, javaAppName string) { 265 | t := template.Must(template.New("mavenPom").Parse(mavenPomTemplate)) 266 | fi, err := os.Create(pomPath) 267 | if err != nil { 268 | log.Print("Problem creating file: " + pomPath) 269 | panic(err) 270 | } 271 | defer fi.Close() 272 | 273 | writer := bufio.NewWriter(fi) 274 | maven := JaxbMavenPomInfo{ 275 | AppName: javaAppName, 276 | } 277 | err = t.Execute(writer, maven) 278 | if err != nil { 279 | log.Println("executing template:", err) 280 | } 281 | bufio.NewWriter(writer).Flush() 282 | } 283 | 284 | func printJavaJaxbMain(rootElementName string, javaDir string, javaPackage string, sourceXMLFilename string, date time.Time) { 285 | t := template.Must(template.New("chidleyJaxbGenClass").Parse(jaxbMainTemplate)) 286 | writer, f, err := javaClassWriter(javaDir, javaPackage, "Main") 287 | defer f.Close() 288 | 289 | classInfo := JaxbMainClassInfo{ 290 | PackageName: javaPackage, 291 | BaseXMLClassName: rootElementName, 292 | SourceXMLFilename: sourceXMLFilename, 293 | Date: date, 294 | } 295 | err = t.Execute(writer, classInfo) 296 | if err != nil { 297 | log.Println("executing template:", err) 298 | } 299 | bufio.NewWriter(writer).Flush() 300 | 301 | } 302 | 303 | //func makeSourceReaders(sourceNames []string, url bool, standardIn bool) ([]Source, error) { 304 | func makeSourceReaders(sourceNames []string, url bool, standardIn bool) (chan Source, error) { 305 | var err error 306 | //sources := make([]Source, len(sourceNames)) 307 | sources := make(chan Source, 1) 308 | 309 | go func() { 310 | var newSource Source 311 | for i, _ := range sourceNames { 312 | if url { 313 | newSource = new(UrlSource) 314 | if DEBUG { 315 | log.Print("Making UrlSource") 316 | } 317 | } else { 318 | if standardIn { 319 | newSource = new(StdinSource) 320 | if DEBUG { 321 | log.Print("Making StdinSource") 322 | } 323 | } else { 324 | newSource = new(FileSource) 325 | if DEBUG { 326 | //log.Print("Making FileSource") 327 | } 328 | } 329 | } 330 | 331 | err = newSource.newSource(sourceNames[i]) 332 | if err != nil { 333 | log.Fatal(err) 334 | } 335 | sources <- newSource 336 | if DEBUG { 337 | log.Print("Making Source:[" + sourceNames[i] + "]") 338 | } 339 | } 340 | close(sources) 341 | }() 342 | return sources, err 343 | } 344 | 345 | func attributes(atts map[string]bool) string { 346 | ret := ": " 347 | for k, _ := range atts { 348 | ret = ret + k + ", " 349 | } 350 | return ret 351 | } 352 | 353 | func indent(d int) string { 354 | indent := "" 355 | for i := 0; i < d; i++ { 356 | indent = indent + "\t" 357 | } 358 | return indent 359 | } 360 | 361 | func countNumberOfBoolsSet(a []*bool) int { 362 | counter := 0 363 | for i := 0; i < len(a); i++ { 364 | if *a[i] { 365 | counter += 1 366 | } 367 | } 368 | return counter 369 | } 370 | 371 | func makeOneLevelDown(node *Node, globalTagAttributes map[string]([]*FQN)) []*XMLType { 372 | var children []*XMLType 373 | 374 | for _, np := range node.children { 375 | if np == nil { 376 | continue 377 | } 378 | for _, n := range np.children { 379 | if n == nil { 380 | continue 381 | } 382 | if flattenStrings && isStringOnlyField(n, len(globalTagAttributes[nk(n)])) { 383 | continue 384 | } 385 | x := XMLType{NameType: n.makeType(namePrefix, nameSuffix), 386 | XMLName: n.name, 387 | XMLNameUpper: capitalizeFirstLetter(n.name), 388 | XMLSpace: n.space} 389 | children = append(children, &x) 390 | } 391 | } 392 | return children 393 | } 394 | func printChildrenChildren(node *Node) { 395 | for k, v := range node.children { 396 | log.Print(k) 397 | log.Printf("children: %+v\n", v.children) 398 | } 399 | } 400 | 401 | // Order Xml is encountered 402 | func printStructsByXml(v *PrintGoStructVisitor) error { 403 | orderNodes := make(map[int]*Node) 404 | var order []int 405 | 406 | for k := range v.alreadyVisitedNodes { 407 | nodeOrder := v.alreadyVisitedNodes[k].discoveredOrder 408 | orderNodes[nodeOrder] = v.alreadyVisitedNodes[k] 409 | order = append(order, nodeOrder) 410 | } 411 | sort.Ints(order) 412 | 413 | for o := range order { 414 | err := print(v, orderNodes[o]) 415 | if err != nil { 416 | return err 417 | } 418 | } 419 | return nil 420 | } 421 | 422 | // Alphabetical order 423 | func printStructsAlphabetical(v *PrintGoStructVisitor) error { 424 | var keys []string 425 | for k := range v.alreadyVisitedNodes { 426 | keys = append(keys, k) 427 | } 428 | sort.Strings(keys) 429 | 430 | for _, k := range keys { 431 | err := print(v, v.alreadyVisitedNodes[k]) 432 | if err != nil { 433 | return err 434 | } 435 | } 436 | return nil 437 | 438 | } 439 | 440 | func generateGoStructs(out io.Writer, sourceName string, ex *Extractor) { 441 | printGoStructVisitor := new(PrintGoStructVisitor) 442 | 443 | printGoStructVisitor.init(os.Stdout, 999, ex.globalTagAttributes, ex.nameSpaceTagMap, useType, nameSpaceInJsonName) 444 | printGoStructVisitor.Visit(ex.root) 445 | structSort(printGoStructVisitor) 446 | } 447 | 448 | //Writes structs to a string then uses this in a template to generate Go code 449 | func generateGoCode(out io.Writer, sourceNames []string, ex *Extractor) error { 450 | buf := bytes.NewBufferString("") 451 | printGoStructVisitor := new(PrintGoStructVisitor) 452 | printGoStructVisitor.init(buf, 9999, ex.globalTagAttributes, ex.nameSpaceTagMap, useType, nameSpaceInJsonName) 453 | printGoStructVisitor.Visit(ex.root) 454 | 455 | structSort(printGoStructVisitor) 456 | 457 | xt := XMLType{NameType: ex.firstNode.makeType(namePrefix, nameSuffix), 458 | XMLName: ex.firstNode.name, 459 | XMLNameUpper: capitalizeFirstLetter(ex.firstNode.name), 460 | XMLSpace: ex.firstNode.space, 461 | } 462 | 463 | fullPath, err := getFullPath(sourceNames[0]) 464 | if err != nil { 465 | return err 466 | } 467 | 468 | fullPaths, err := getFullPaths(sourceNames) 469 | if err != nil { 470 | return err 471 | } 472 | x := XmlInfo{ 473 | BaseXML: &xt, 474 | OneLevelDownXML: makeOneLevelDown(ex.root, ex.globalTagAttributes), 475 | Filenames: fullPaths, 476 | Filename: fullPath, 477 | Structs: buf.String(), 478 | } 479 | x.init() 480 | t := template.Must(template.New("chidleyGen").Parse(codeTemplate)) 481 | 482 | err = t.Execute(out, x) 483 | if err != nil { 484 | log.Println("executing template:", err) 485 | return err 486 | } 487 | return err 488 | } 489 | -------------------------------------------------------------------------------- /codegenTemplates.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type XmlInfo struct { 8 | BaseXML *XMLType 9 | OneLevelDownXML []*XMLType 10 | Structs string 11 | Filenames []string 12 | Filename string 13 | } 14 | 15 | type XMLType struct { 16 | NameType, XMLName, XMLNameUpper, XMLSpace string 17 | } 18 | 19 | // b := [5]int{1, 2, 3, 4, 5} 20 | func (xi *XmlInfo) init() { 21 | xi.Filename = "[" + strconv.Itoa(len(xi.Filenames)) + "]string{" 22 | for i, _ := range xi.Filenames { 23 | if i != 0 { 24 | xi.Filename = xi.Filename + "," 25 | } 26 | xi.Filename = xi.Filename + "\"" + xi.Filenames[i] + "\"" 27 | } 28 | xi.Filename = xi.Filename + "}" 29 | } 30 | 31 | const codeTemplate = ` 32 | 33 | package main 34 | 35 | ///////////////////////////////////////////////////////////////// 36 | //Code generated by chidley https://github.com/gnewton/chidley // 37 | ///////////////////////////////////////////////////////////////// 38 | 39 | import ( 40 | "bufio" 41 | "compress/bzip2" 42 | "compress/gzip" 43 | "encoding/json" 44 | "encoding/xml" 45 | "flag" 46 | "fmt" 47 | "io" 48 | "log" 49 | "os" 50 | "math" 51 | "strings" 52 | ) 53 | 54 | const ( 55 | JsonOut = iota 56 | XmlOut 57 | CountAll 58 | ) 59 | 60 | var toJson bool = false 61 | var toXml bool = false 62 | var oneLevelDown bool = false 63 | var countAll bool = false 64 | var musage bool = false 65 | 66 | var uniqueFlags = []*bool{ 67 | &toJson, 68 | &toXml, 69 | &countAll} 70 | 71 | var filenames = {{.Filename}} 72 | 73 | 74 | 75 | func init() { 76 | 77 | flag.BoolVar(&toJson, "j", toJson, "Convert to JSON") 78 | flag.BoolVar(&toXml, "x", toXml, "Convert to XML") 79 | flag.BoolVar(&countAll, "c", countAll, "Count each instance of XML tags") 80 | flag.BoolVar(&oneLevelDown, "s", oneLevelDown, "Stream XML by using XML elements one down from the root tag. Good for huge XML files (see http://blog.davidsingleton.org/parsing-huge-xml-files-with-go/") 81 | flag.BoolVar(&musage, "h", musage, "Usage") 82 | //flag.StringVar(&filename, "f", filename, "XML file or URL to read in") 83 | 84 | flag.Int64Var(&recordLimit, "n", recordLimit, "Limit # records output") 85 | } 86 | 87 | var out int = -1 88 | 89 | var counters map[string]*int 90 | 91 | var recordLimit int64 = int64(math.MaxInt64) 92 | var recordCounter = int64(0) 93 | 94 | func main() { 95 | flag.Parse() 96 | 97 | if musage { 98 | flag.Usage() 99 | return 100 | } 101 | 102 | numSetBools, outFlag := numberOfBoolsSet(uniqueFlags) 103 | if numSetBools == 0 { 104 | flag.Usage() 105 | return 106 | } 107 | 108 | if numSetBools != 1 { 109 | flag.Usage() 110 | log.Fatal("Only one of ", uniqueFlags, " can be set at once") 111 | } 112 | 113 | counter := 0 114 | stop := false 115 | counters = make(map[string]*int) 116 | for i, _ := range filenames { 117 | filename := filenames[i] 118 | reader, xmlFile, err := genericReader(filename) 119 | if err != nil { 120 | log.Fatal(err) 121 | return 122 | } 123 | 124 | decoder := xml.NewDecoder(reader) 125 | 126 | for { 127 | if stop { 128 | break 129 | } 130 | token, _ := decoder.Token() 131 | if token == nil { 132 | break 133 | } 134 | switch se := token.(type) { 135 | case xml.StartElement: 136 | counter++ 137 | handleFeed(se, decoder, outFlag) 138 | if recordCounter == recordLimit { 139 | stop = true 140 | } 141 | } 142 | 143 | } 144 | if stop { 145 | break 146 | } 147 | if xmlFile != nil { 148 | defer xmlFile.Close() 149 | } 150 | } 151 | if countAll { 152 | for k, v := range counters { 153 | fmt.Println(*v, k) 154 | } 155 | } 156 | } 157 | 158 | func handleFeed(se xml.StartElement, decoder *xml.Decoder, outFlag *bool) { 159 | if outFlag == &countAll { 160 | incrementCounter(se.Name.Space, se.Name.Local) 161 | } else { 162 | if !oneLevelDown{ 163 | if se.Name.Local == "{{.BaseXML.XMLName}}" && se.Name.Space == "{{.BaseXML.XMLSpace}}" { 164 | var item {{.BaseXML.NameType}} 165 | decoder.DecodeElement(&item, &se) 166 | switch outFlag { 167 | case &toJson: 168 | writeJson(item) 169 | case &toXml: 170 | writeXml(item) 171 | } 172 | } 173 | }else{ 174 | {{ range .OneLevelDownXML }} 175 | if se.Name.Local == "{{.XMLName}}" && se.Name.Space == "{{.XMLSpace}}" { 176 | recordCounter++ 177 | var item {{.NameType}} 178 | decoder.DecodeElement(&item, &se) 179 | switch outFlag { 180 | case &toJson: 181 | writeJson(item) 182 | case &toXml: 183 | writeXml(item) 184 | } 185 | } 186 | {{ end }} 187 | } 188 | } 189 | } 190 | 191 | func makeKey(space string, local string) string { 192 | if space == "" { 193 | space = "_" 194 | } 195 | return space + ":" + local 196 | } 197 | 198 | func incrementCounter(space string, local string) { 199 | key := makeKey(space, local) 200 | 201 | counter, ok := counters[key] 202 | if !ok { 203 | n := 1 204 | counters[key] = &n 205 | } else { 206 | newv := *counter + 1 207 | counters[key] = &newv 208 | } 209 | } 210 | 211 | func writeJson(item interface{}) { 212 | b, err := json.MarshalIndent(item, "", " ") 213 | if err != nil { 214 | log.Fatal(err) 215 | } 216 | fmt.Println(string(b)) 217 | } 218 | 219 | func writeXml(item interface{}) { 220 | output, err := xml.MarshalIndent(item, " ", " ") 221 | if err != nil { 222 | fmt.Printf("error: %v\n", err) 223 | } 224 | os.Stdout.Write(output) 225 | } 226 | 227 | func genericReader(filename string) (io.Reader, *os.File, error) { 228 | if filename == "" { 229 | return bufio.NewReader(os.Stdin), nil, nil 230 | } 231 | file, err := os.Open(filename) 232 | if err != nil { 233 | return nil, nil, err 234 | } 235 | if strings.HasSuffix(filename, "bz2") { 236 | return bufio.NewReader(bzip2.NewReader(bufio.NewReader(file))), file, err 237 | } 238 | 239 | if strings.HasSuffix(filename, "gz") { 240 | reader, err := gzip.NewReader(bufio.NewReader(file)) 241 | if err != nil { 242 | return nil, nil, err 243 | } 244 | return bufio.NewReader(reader), file, err 245 | } 246 | return bufio.NewReader(file), file, err 247 | } 248 | 249 | func numberOfBoolsSet(a []*bool) (int, *bool) { 250 | var setBool *bool 251 | counter := 0 252 | for i := 0; i < len(a); i++ { 253 | if *a[i] { 254 | counter += 1 255 | setBool = a[i] 256 | } 257 | } 258 | return counter, setBool 259 | } 260 | 261 | 262 | /////////////////////////// 263 | /// structs 264 | /////////////////////////// 265 | 266 | {{.Structs}} 267 | /////////////////////////// 268 | 269 | ` 270 | -------------------------------------------------------------------------------- /data/test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Dune 4 | 1966 5 | 6 | 7 | Frank 8 | Herbert 9 | 10 | 11 | 12 | 678 13 | 14 | 15 | 16 | 17 | 18 | Larry 19 | Niven 20 | 21 | 22 | Jerry 23 | Pournelle 24 | 25 | 26 | Footfall 27 | 1972 28 | 29 | 451 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /data/typeCollapsing.xml: -------------------------------------------------------------------------------- 1 | 2 | Bill Smith 3 | 42 4 | true 5 | 55.7 6 | 7 | -------------------------------------------------------------------------------- /decl.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | var DEBUG = false 8 | var attributePrefix = "Attr" 9 | var codeGenConvert = false 10 | var classicStructNamesWithUnderscores = false 11 | var nameSpaceInJsonName = false 12 | var prettyPrint = false 13 | var progress = false 14 | var readFromStandardIn = false 15 | var sortByXmlOrder = false 16 | var structsToStdout = true 17 | var validateFieldTemplate = false 18 | 19 | var ignoreLowerCaseXmlTags = false 20 | var ignoredXmlTags = "" 21 | var ignoredXmlTagsMap *map[string]struct{} 22 | 23 | var ignoreXmlDecodingErrors = false 24 | 25 | var codeGenDir = "codegen" 26 | var codeGenFilename = "CodeGenStructs.go" 27 | 28 | // Java out 29 | const javaBasePackage = "ca.gnewton.chidley" 30 | const mavenJavaBase = "src/main/java" 31 | 32 | var javaBasePackagePath = strings.Replace(javaBasePackage, ".", "/", -1) 33 | var javaAppName = "jaxb" 34 | var writeJava = false 35 | var baseJavaDir = "java" 36 | var userJavaPackageName = "" 37 | 38 | var namePrefix = "C" 39 | var nameSuffix = "" 40 | var xmlName = false 41 | var url = false 42 | var useType = false 43 | var addDbMetadata = false 44 | var flattenStrings = false 45 | 46 | //FIXXX: should not be global 47 | var keepXmlFirstLetterCase = true 48 | 49 | var lengthTagName = "" 50 | var lengthTagPadding int64 = 0 51 | var lengthTagAttribute = "" 52 | var lengthTagSeparator = ":" 53 | 54 | var cdataStringName = "Text" 55 | var cdataNumberName = "Number" 56 | var cdataBooleanName = "Flag" 57 | 58 | type structSortFunc func(v *PrintGoStructVisitor) 59 | 60 | var structSort = printStructsAlphabetical 61 | 62 | var outputs = []*bool{ 63 | &codeGenConvert, 64 | &structsToStdout, 65 | &writeJava, 66 | } 67 | -------------------------------------------------------------------------------- /encoding_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | const XML = ` 8 | 9 | 10 | 43 11 | 12 | ` 13 | 14 | func TestHasNonDefaultEncoding(t *testing.T) { 15 | 16 | } 17 | 18 | const XMLHasTagCalledRoot = ` 19 | 20 | 21 | 22 | 23 | ` 24 | 25 | func TestHasTagCalledRoot(t *testing.T) { 26 | 27 | } 28 | 29 | const XMLHasCapitalizationTagCollisions = ` 30 | 31 | 32 | 123 33 | 34 | 35 | add 36 | 37 | 38 | ` 39 | 40 | func TestHasCapitalizationTagCollisions(t *testing.T) { 41 | 42 | } 43 | -------------------------------------------------------------------------------- /extractor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/xml" 5 | "errors" 6 | "io" 7 | "log" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | const XML_NAMESPACE_ACRONYM = "xmlns" 13 | 14 | var nameMapper = map[string]string{ 15 | "-": "Hyphen", 16 | ".": "Dot", 17 | } 18 | 19 | var DiscoveredOrder = 0 20 | 21 | type Extractor struct { 22 | globalTagAttributes map[string]([]*FQN) 23 | globalTagAttributesMap map[string]bool 24 | globalNodeMap map[string]*Node 25 | namePrefix string 26 | nameSpaceTagMap map[string]string 27 | nameSuffix string 28 | root *Node 29 | firstNode *Node 30 | hasStartElements bool 31 | useType bool 32 | progress bool 33 | ignoreXmlDecodingErrors bool 34 | initted bool 35 | tokenChannel chan xml.Token 36 | handleTokensDoneChannel chan bool 37 | } 38 | 39 | const RootName = "ChidleyRoot314159" 40 | 41 | func (ex *Extractor) init() { 42 | ex.globalTagAttributes = make(map[string]([]*FQN)) 43 | ex.globalTagAttributesMap = make(map[string]bool) 44 | ex.nameSpaceTagMap = make(map[string]string) 45 | ex.globalNodeMap = make(map[string]*Node) 46 | ex.root = new(Node) 47 | //ex.root.initialize(RootName, "", "", nil) 48 | ex.root.initialize("", "", "", nil) 49 | ex.hasStartElements = false 50 | ex.initted = true 51 | ex.tokenChannel = make(chan xml.Token, 100) 52 | ex.handleTokensDoneChannel = make(chan bool) 53 | go handleTokens(ex) 54 | } 55 | 56 | func (ex *Extractor) done() { 57 | close(ex.tokenChannel) 58 | _ = <-ex.handleTokensDoneChannel 59 | } 60 | 61 | func (ex *Extractor) extract(reader io.Reader) error { 62 | if ex.initted == false { 63 | return errors.New("extractor not properly initted: must run extractor.init() first") 64 | } 65 | decoder := xml.NewDecoder(reader) 66 | 67 | for { 68 | token, err := decoder.Token() 69 | if err != nil { 70 | if err.Error() == "EOF" { 71 | // OK 72 | break 73 | } 74 | log.Println(err) 75 | if !ex.ignoreXmlDecodingErrors { 76 | return err 77 | } 78 | } 79 | if token == nil { 80 | log.Println("Empty token") 81 | break 82 | } 83 | ex.tokenChannel <- xml.CopyToken(token) 84 | } 85 | return nil 86 | } 87 | 88 | func handleTokens(ex *Extractor) { 89 | tChannel := ex.tokenChannel 90 | handleTokensDoneChannel := ex.handleTokensDoneChannel 91 | depth := 0 92 | thisNode := ex.root 93 | first := true 94 | var progressCounter int64 = 0 95 | 96 | for token := range tChannel { 97 | //log.Println(token) 98 | switch element := token.(type) { 99 | 100 | case xml.Comment: 101 | if DEBUG { 102 | //log.Print(thisNode.name) 103 | //log.Printf("Comment: %+v\n", string(element)) 104 | } 105 | 106 | case xml.ProcInst: 107 | if DEBUG { 108 | //log.Println("ProcInst: Target=" + element.Target + " Inst=[" + string(element.Inst) + "]") 109 | } 110 | 111 | case xml.Directive: 112 | if DEBUG { 113 | //log.Printf("Directive: %+v\n", string(element)) 114 | } 115 | 116 | case xml.StartElement: 117 | progressCounter += 1 118 | if DEBUG { 119 | //log.Printf("StartElement: %+v\n", element) 120 | } 121 | ex.hasStartElements = true 122 | 123 | if element.Name.Local == "" { 124 | continue 125 | } 126 | thisNode = ex.handleStartElement(element, thisNode) 127 | thisNode.tempCharData = "" 128 | thisNode.ignoredTag = isIgnoredTag(element.Name.Local) 129 | 130 | if first { 131 | first = false 132 | ex.firstNode = thisNode 133 | } 134 | depth += 1 135 | if progress { 136 | if progressCounter%50000 == 0 { 137 | log.Print(progressCounter) 138 | } 139 | } 140 | 141 | case xml.CharData: 142 | if DEBUG { 143 | //log.Print(thisNode.name) 144 | //log.Printf("CharData: [%+v]\n", string(element)) 145 | } 146 | 147 | //if !thisNode.hasCharData { 148 | charData := string(element) 149 | thisNode.tempCharData += charData //strings.TrimSpace(charData) 150 | thisNode.charDataCount += int64(len(charData)) 151 | //} 152 | 153 | case xml.EndElement: 154 | //if ignoredTag(element.Name.Local) { 155 | //continue 156 | //} 157 | 158 | thisNode.nodeTypeInfo.checkFieldType(thisNode.tempCharData) 159 | thisNode.nodeTypeInfo.addFieldLength(thisNode.charDataCount) 160 | thisNode.charDataCount = 0 161 | 162 | if DEBUG { 163 | //log.Printf("EndElement: %+v\n", element) 164 | //log.Printf("[[" + thisNode.tempCharData + "]]") 165 | //log.Println("Char is empty: ", isJustSpacesAndLinefeeds(thisNode.tempCharData)) 166 | } 167 | if !thisNode.hasCharData && !isJustSpacesAndLinefeeds(thisNode.tempCharData) { 168 | thisNode.hasCharData = true 169 | 170 | } else { 171 | 172 | } 173 | thisNode.tempCharData = "" 174 | depth -= 1 175 | 176 | for key, c := range thisNode.childCount { 177 | if c > 1 { 178 | thisNode.children[key].repeats = true 179 | } 180 | thisNode.childCount[key] = 0 181 | } 182 | if thisNode.peekParent() != nil { 183 | thisNode = thisNode.popParent() 184 | } 185 | } 186 | } 187 | handleTokensDoneChannel <- true 188 | close(handleTokensDoneChannel) 189 | } 190 | 191 | func space(n int) string { 192 | s := strconv.Itoa(n) + "Space" 193 | for i := 0; i < n; i++ { 194 | s += " " 195 | } 196 | return s 197 | } 198 | 199 | func (ex *Extractor) findNewNameSpaces(attrs []xml.Attr) { 200 | for _, attr := range attrs { 201 | if strings.HasPrefix(attr.Name.Space, XML_NAMESPACE_ACRONYM) { 202 | //log.Println("mmmmmmmmmmmmmmmmmmmmmmm", attr) 203 | //log.Println("+++++++++++++++++++++++++++", attr.Value, "|", attr.Name.Local, "|", attr.Name.Space) 204 | ex.nameSpaceTagMap[attr.Value] = attr.Name.Local 205 | } 206 | } 207 | } 208 | 209 | var full struct{} 210 | 211 | func (ex *Extractor) handleStartElement(startElement xml.StartElement, thisNode *Node) *Node { 212 | name := startElement.Name.Local 213 | space := startElement.Name.Space 214 | 215 | ex.findNewNameSpaces(startElement.Attr) 216 | 217 | var child *Node 218 | var attributes []*FQN 219 | key := nks(space, name) 220 | 221 | child, ok := thisNode.children[key] 222 | // Does thisNode node already exist as child 223 | //fmt.Println(space, name) 224 | if ok { 225 | thisNode.childCount[key] += 1 226 | attributes, ok = ex.globalTagAttributes[key] 227 | } else { 228 | // if thisNode node does not already exist as child, it may still exist as child on other node: 229 | child, ok = ex.globalNodeMap[key] 230 | if !ok { 231 | child = new(Node) 232 | DiscoveredOrder += 1 233 | child.discoveredOrder = DiscoveredOrder 234 | ex.globalNodeMap[key] = child 235 | spaceTag, _ := ex.nameSpaceTagMap[space] 236 | child.initialize(name, space, spaceTag, thisNode) 237 | thisNode.childCount[key] = 1 238 | 239 | attributes = make([]*FQN, 0, 2) 240 | ex.globalTagAttributes[key] = attributes 241 | } else { 242 | attributes = ex.globalTagAttributes[key] 243 | } 244 | thisNode.children[key] = child 245 | } 246 | child.pushParent(thisNode) 247 | 248 | // Extract attributes 249 | for _, attr := range startElement.Attr { 250 | bigKey := key + "_" + attr.Name.Space + "_" + attr.Name.Local 251 | _, ok := ex.globalTagAttributesMap[bigKey] 252 | if ok { 253 | fqn := findThisAttribute(attr.Name.Local, attr.Name.Space, ex.globalTagAttributes[key]) 254 | if fqn == nil { 255 | log.Println("This should not be happening: fqn is nil") 256 | continue 257 | } 258 | lenValue := len(attr.Value) 259 | if lenValue > fqn.maxLength { 260 | fqn.maxLength = lenValue 261 | } 262 | } else { 263 | fqn := new(FQN) 264 | fqn.name = attr.Name.Local 265 | fqn.space = attr.Name.Space 266 | fqn.maxLength = len(attr.Value) 267 | //log.Println(name, "|", fqn.name, "||", fqn.space) 268 | attributes = append(attributes, fqn) 269 | ex.globalTagAttributesMap[bigKey] = true 270 | } 271 | } 272 | ex.globalTagAttributes[key] = attributes 273 | return child 274 | } 275 | 276 | func isJustSpacesAndLinefeeds(s string) bool { 277 | s = strings.Replace(s, "\\n", "", -1) 278 | s = strings.Replace(s, "\n", "", -1) 279 | return len(strings.TrimSpace(s)) == 0 280 | } 281 | -------------------------------------------------------------------------------- /fieldTemplate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | //"fmt" 6 | "log" 7 | //"strconv" 8 | "text/template" 9 | ) 10 | 11 | type FieldDef struct { 12 | XMLName string 13 | XMLNameSpace string 14 | GoName string 15 | GoType string 16 | GoTypeArrayOrPointer string 17 | Length int64 18 | //Foo bool 19 | } 20 | 21 | var fieldTemplate *template.Template 22 | 23 | var fieldTemplateString = `{{.GoName}} {{.GoTypeArrayOrPointer}}{{.GoType}} ` + "`" + `xml:"{{if notEmpty .XMLNameSpace}}{{.XMLNameSpace}} {{end}}{{.XMLName}},omitempty" json:"{{.XMLName}},omitempty"` + "`" + `` 24 | 25 | func render(otd FieldDef) (string, error) { 26 | var err error 27 | // fieldTemplate = template.Must(template.New("fieldTemplate").Funcs(template.FuncMap{ 28 | fieldTemplate, err = template.New("fieldTemplate").Funcs(template.FuncMap{ 29 | "notEmpty": func(feature string) bool { 30 | return len(feature) > 0 31 | }, 32 | }).Parse(fieldTemplateString) 33 | 34 | if err != nil { 35 | return "", err 36 | } 37 | 38 | var buf bytes.Buffer 39 | 40 | //err := t.Execute(os.Stdout, ot) 41 | err = fieldTemplate.Execute(&buf, otd) 42 | if err != nil { 43 | return "", err 44 | } 45 | //fmt.Println(buf.String()) 46 | //return "\t" + buf.String() + " // ZZmaxLength=" + strconv.FormatInt(otd.Length, 10), nil 47 | return "\t" + buf.String(), nil 48 | } 49 | 50 | func runValidateFieldTemplate(printToLog bool) error { 51 | fieldDef := FieldDef{ 52 | XMLName: "xmlName", 53 | XMLNameSpace: "xmlNameSpace", 54 | GoName: "Foobar", 55 | GoType: "string", 56 | GoTypeArrayOrPointer: "[]", 57 | Length: 32, 58 | } 59 | 60 | string, err := render(fieldDef) 61 | if printToLog { 62 | log.Println("validateFieldTemplate") 63 | log.Println("Using template:", fieldTemplateString) 64 | log.Println(string) 65 | 66 | } 67 | if err != nil { 68 | log.Println("Error with field template:", err) 69 | } 70 | return err 71 | } 72 | -------------------------------------------------------------------------------- /fqn.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type FQN struct { 4 | space string 5 | name string 6 | maxLength int 7 | } 8 | 9 | type FQNAbbr struct { 10 | FQN 11 | abbr string 12 | } 13 | -------------------------------------------------------------------------------- /genericReader.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "compress/bzip2" 6 | "compress/gzip" 7 | "io" 8 | "log" 9 | "net/http" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | type GenericReader struct { 15 | io.Reader 16 | reader io.Reader 17 | } 18 | 19 | func (ge *GenericReader) Read(p []byte) (n int, err error) { 20 | return ge.reader.Read(p) 21 | } 22 | 23 | func (ge *GenericReader) Open(name string) error { 24 | if name == "" { 25 | return nil 26 | } 27 | if strings.HasPrefix(name, "http://") || strings.HasPrefix(name, "https://") { 28 | return ge.openUrl(name) 29 | } else { 30 | return ge.openLocalFile(name) 31 | } 32 | } 33 | 34 | func (ge *GenericReader) Close() error { 35 | return nil 36 | } 37 | 38 | func (ge *GenericReader) openLocalFile(filename string) error { 39 | file, err := openIfExistsIsFileIsReadable(filename) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | if strings.HasSuffix(filename, "bz2") { 45 | ge.reader = bufio.NewReader(bzip2.NewReader(bufio.NewReader(file))) 46 | } else { 47 | if strings.HasSuffix(filename, "gz") { 48 | reader, err := gzip.NewReader(bufio.NewReader(file)) 49 | if err != nil { 50 | return err 51 | } 52 | ge.reader = bufio.NewReader(reader) 53 | } else { 54 | ge.reader = bufio.NewReader(file) 55 | } 56 | } 57 | return nil 58 | } 59 | 60 | func (gr *GenericReader) openUrl(url string) error { 61 | res, err := http.Get(url) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | if res.StatusCode != 200 { 67 | log.Fatal("ERROR: bad http status code != 200: ", res.StatusCode, " ", url) 68 | return nil 69 | 70 | } 71 | gr.reader = res.Body 72 | return nil 73 | } 74 | 75 | func openIfExistsIsFileIsReadable(fileName string) (*os.File, error) { 76 | file, err := os.Open(fileName) // For read access. 77 | log.Print(fileName) 78 | if err != nil { 79 | log.Print("Problem opening file: [" + fileName + "]") 80 | log.Print(err) 81 | return nil, err 82 | } 83 | if _, err := os.Stat(fileName); os.IsNotExist(err) { 84 | log.Print("Problem stat'ing file: [" + fileName + "]") 85 | return nil, err 86 | } 87 | 88 | fi, err := file.Stat() 89 | if err != nil { 90 | log.Print("Problem stat'ing file: [" + fileName + "]") 91 | return nil, err 92 | } 93 | 94 | fm := fi.Mode() 95 | if !fm.IsRegular() { 96 | error := new(InternalError) 97 | error.ErrorString = "Is directory, needs to be file: " + fileName 98 | log.Print(error.ErrorString) 99 | return nil, error 100 | } 101 | 102 | log.Print(fm.Perm().String()) 103 | if fm.Perm().String()[7] != 'r' { 104 | error := new(InternalError) 105 | error.ErrorString = "Exists but unable to read: " + fileName 106 | log.Print(error.ErrorString) 107 | return nil, error 108 | } 109 | return file, nil 110 | } 111 | 112 | type InternalError struct { 113 | ErrorString string 114 | } 115 | 116 | func (ie *InternalError) Error() string { 117 | return "Error: " + ie.ErrorString 118 | } 119 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gnewton/chidley 2 | 3 | go 1.17 4 | 5 | require github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= 2 | github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= 3 | -------------------------------------------------------------------------------- /javaTypes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const JavaString = "String" 4 | const JavaBoolean = "boolean" 5 | const JavaShort = "short" 6 | const JavaFloat = "float" 7 | const JavaDouble = "double" 8 | const JavaInt = "int" 9 | const JavaLong = "long" 10 | 11 | const GoBool = "bool" 12 | 13 | const GoInt8 = "int8" 14 | const GoUint8 = "uint8" 15 | const GoInt16 = "int16" 16 | 17 | const GoUint16 = "uint16" 18 | const GoInt32 = "int32" 19 | 20 | const GoUint32 = "uint32" 21 | const GoInt64 = "int64" 22 | 23 | const GoFloat32 = "float32" 24 | 25 | const GoFloat64 = "float64" 26 | 27 | func findJavaType(nti *NodeTypeInfo, useType bool) string { 28 | if !useType { 29 | return JavaString 30 | } 31 | goType := findType(nti, true) 32 | 33 | switch goType { 34 | case GoBool: 35 | return JavaBoolean 36 | case GoInt8, GoUint8, GoInt16: 37 | return JavaShort 38 | case GoUint16, GoInt32: 39 | return JavaInt 40 | case GoUint32, GoInt64: 41 | return JavaLong 42 | case GoFloat32: 43 | return JavaFloat 44 | case GoFloat64: 45 | return JavaDouble 46 | } 47 | return JavaString 48 | } 49 | -------------------------------------------------------------------------------- /jaxbTemplate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type JaxbPackageInfo struct { 8 | BaseNameSpace string 9 | AdditionalNameSpace []*FQNAbbr 10 | PackageName string 11 | } 12 | 13 | type JaxbMainClassInfo struct { 14 | PackageName string 15 | BaseXMLClassName string 16 | SourceXMLFilename string 17 | Date time.Time 18 | } 19 | 20 | type JaxbClassInfo struct { 21 | Name string 22 | Root bool 23 | PackageName, ClassName string 24 | Attributes []*JaxbAttribute 25 | Fields []*JaxbField 26 | HasValue bool 27 | ValueType string 28 | Date time.Time 29 | } 30 | 31 | type JaxbAttribute struct { 32 | Name string 33 | NameUpper string 34 | NameLower string 35 | NameSpace string 36 | } 37 | type JaxbField struct { 38 | TypeName string 39 | Name string 40 | NameUpper string 41 | NameLower string 42 | NameSpace string 43 | Repeats bool 44 | } 45 | 46 | func (jb *JaxbClassInfo) init() { 47 | jb.Attributes = make([]*JaxbAttribute, 0) 48 | jb.Fields = make([]*JaxbField, 0) 49 | } 50 | 51 | const jaxbClassTemplate = ` 52 | // Generated by chidley https://github.com/gnewton/chidley 53 | // Date: {{.Date}} 54 | // 55 | package {{.PackageName}}.xml; 56 | 57 | import java.util.ArrayList; 58 | import javax.xml.bind.annotation.*; 59 | import com.google.gson.annotations.SerializedName; 60 | 61 | @XmlAccessorType(XmlAccessType.FIELD) 62 | @XmlRootElement(name="{{.Name}}") 63 | public class {{.ClassName}} { 64 | {{if .Attributes}} 65 | // Attributes{{end}} 66 | {{range .Attributes}} 67 | {{if .NameSpace}} 68 | @XmlAttribute(namespace = "{{.NameSpace}}"){{else}} @XmlAttribute(name="{{.Name}}"){{end}} 69 | @SerializedName("{{.Name}}") 70 | public String {{.NameLower}};{{end}} 71 | {{if .Fields}} 72 | // Fields{{end}}{{range .Fields}} 73 | @XmlElement(name="{{.Name}}") 74 | @SerializedName("{{.Name}}") 75 | {{if .Repeats}}public ArrayList<{{.TypeName}}> {{.NameLower}}{{else}}public {{.TypeName}} {{.NameLower}}{{end}}; 76 | {{end}} 77 | {{if .HasValue}} 78 | // Value 79 | @XmlValue 80 | public {{.ValueType}} tagValue;{{end}} 81 | } 82 | ` 83 | 84 | const jaxbMainTemplate = ` 85 | // Generated by chidley https://github.com/gnewton/chidley 86 | // Date: {{.Date}} 87 | // 88 | 89 | package {{.PackageName}}; 90 | 91 | import java.io.File; 92 | import javax.xml.bind.JAXBContext; 93 | import javax.xml.bind.JAXBException; 94 | import javax.xml.bind.Unmarshaller; 95 | import {{.PackageName}}.xml.{{.BaseXMLClassName}}; 96 | import com.google.gson.Gson; 97 | import com.google.gson.GsonBuilder; 98 | import java.net.URL; 99 | import java.net.URLClassLoader; 100 | 101 | 102 | 103 | public class Main { 104 | public static void main(String[] args) { 105 | try { 106 | //https://jaxp.java.net/1.5/JAXP1.5Guide.html 107 | // To fix error: Caused by: 108 | //org.xml.sax.SAXParseException; systemId: file:/home/gnewton/gocode/src/github.com/gnewton/chidley/xml/MozartTrio.xml; lineNumber: 2; columnNumber: 123; External DTD: Failed to read external DTD 'partwise.dtd', because 'http' access is not allowed due to restriction set by the accessExternalDTD property. 109 | System.setProperty("javax.xml.accessExternalSchema", "all"); 110 | System.setProperty("javax.xml.accessExternalDTD", "all"); 111 | System.setProperty("javax.xml.XMLConstants.ACCESS_EXTERNAL_STYLESHEET", "all"); 112 | 113 | // System.setProperty("http.agent", "Mozilla/4.76"); 114 | System.setProperty("http.agent", "Mozilla/5.0 (X11; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"); 115 | 116 | File file = new File("{{.SourceXMLFilename}}"); 117 | JAXBContext jaxbContext = JAXBContext.newInstance({{.BaseXMLClassName}}.class); 118 | 119 | Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); 120 | {{.BaseXMLClassName}} root = ({{.BaseXMLClassName}}) jaxbUnmarshaller.unmarshal(file); 121 | 122 | Gson gson = new GsonBuilder().setPrettyPrinting().create(); 123 | System.out.println(gson.toJson(root)); 124 | 125 | } catch (Throwable e) { 126 | e.printStackTrace(); 127 | // Print classpath 128 | System.err.println("CLASSPATH START"); 129 | ClassLoader cl = ClassLoader.getSystemClassLoader(); 130 | URL[] urls = ((URLClassLoader)cl).getURLs(); 131 | for(URL url: urls){ 132 | System.err.println("\n" + url.getFile()); 133 | } 134 | System.err.println("CLASSPATH END"); 135 | } 136 | } 137 | } 138 | ` 139 | 140 | const jaxbPackageInfoTemplage = ` 141 | @XmlSchema( 142 | namespace="{{.BaseNameSpace}}", 143 | elementFormDefault = XmlNsForm.QUALIFIED{{if .AdditionalNameSpace}}, 144 | xmlns={ 145 | {{range .AdditionalNameSpace}} 146 | @XmlNs(prefix="{{.abbr}}", namespaceURI="{{.space}}"), 147 | {{end}} 148 | } 149 | {{end}} 150 | ) 151 | package {{.PackageName}}; 152 | 153 | import javax.xml.bind.annotation.*; 154 | ` 155 | -------------------------------------------------------------------------------- /makereadme/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Copyright 2014,2015,2016 Glen Newton 4 | // glen.newton@gmail.com 5 | 6 | import ( 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "text/template" 11 | ) 12 | 13 | type ReadmeInfo struct { 14 | ChidleyUsage string 15 | GeneratedUsage string 16 | GeneratedXMLToJson string 17 | GeneratedXMLToXML string 18 | GeneratedCountElements string 19 | ChidleyOnlyStructOutput string 20 | SimpleExampleXMLFile string 21 | SimpleExampleXMLChidleyGoStructs string 22 | SimpleExampleXMLChidleyGoStructsCollapsed string 23 | SimpleExampleXMLChidleyGoStructsWithTypes string 24 | PubmedXMLFileName string 25 | 26 | PubmedExampleXMLChidleyGoStructsWithTypes string 27 | PubmedExampleXMLChidleyGoStructsWithTypeTiming string 28 | GeneratedPubmedCount string 29 | GeneratedPubmedNoStreaming string 30 | GeneratedPubmedStreaming string 31 | GeneratedPubmedXMLToJson string 32 | GeneratedPubmedXMLToXML string 33 | ChidleyGenerateJava string 34 | ChidleyGenerateJavaChangePackageName string 35 | ChidleyGenerateJavaMavenBuild string 36 | ChidleyGenerateJavaRun string 37 | } 38 | 39 | const empty = ` 40 | ***********EMPTY` 41 | 42 | func main() { 43 | log.SetFlags(log.LstdFlags | log.Lshortfile) 44 | var err error 45 | var tmp []byte 46 | 47 | ri := ReadmeInfo{ 48 | ChidleyUsage: empty, 49 | GeneratedUsage: empty, 50 | GeneratedXMLToJson: empty, 51 | GeneratedXMLToXML: empty, 52 | GeneratedCountElements: empty, 53 | ChidleyOnlyStructOutput: empty, 54 | SimpleExampleXMLFile: empty, 55 | SimpleExampleXMLChidleyGoStructs: empty, 56 | SimpleExampleXMLChidleyGoStructsCollapsed: empty, 57 | SimpleExampleXMLChidleyGoStructsWithTypes: empty, 58 | 59 | PubmedXMLFileName: empty, 60 | PubmedExampleXMLChidleyGoStructsWithTypes: empty, 61 | PubmedExampleXMLChidleyGoStructsWithTypeTiming: empty, 62 | GeneratedPubmedCount: empty, 63 | GeneratedPubmedNoStreaming: empty, 64 | GeneratedPubmedStreaming: empty, 65 | GeneratedPubmedXMLToJson: empty, 66 | GeneratedPubmedXMLToXML: empty, 67 | ChidleyGenerateJava: empty, 68 | ChidleyGenerateJavaChangePackageName: empty, 69 | ChidleyGenerateJavaMavenBuild: empty, 70 | ChidleyGenerateJavaRun: empty, 71 | } 72 | 73 | ri.ChidleyUsage, err = runCaptureStdout("..", "./chidley", "-n") 74 | 75 | if err != nil { 76 | log.Fatal(err) 77 | } 78 | 79 | tmp, err = ioutil.ReadFile("../data/test.xml") 80 | if err != nil { 81 | log.Fatal(err) 82 | } 83 | ri.SimpleExampleXMLFile = string(tmp) 84 | 85 | ri.SimpleExampleXMLChidleyGoStructs, err = runCaptureStdout("..", "./chidley", "-G", "data/test.xml") 86 | 87 | if err != nil { 88 | log.Fatal(err) 89 | } 90 | 91 | ri.SimpleExampleXMLChidleyGoStructsCollapsed = string(tmp) 92 | 93 | ri.SimpleExampleXMLChidleyGoStructsCollapsed, err = runCaptureStdout("..", "./chidley", "-G", "-F", "data/test.xml") 94 | if err != nil { 95 | log.Fatal(err) 96 | } 97 | 98 | ri.SimpleExampleXMLChidleyGoStructsWithTypes, err = runCaptureStdout("..", "./chidley", "-G", "-t", "data/test.xml") 99 | if err != nil { 100 | log.Fatal(err) 101 | } 102 | 103 | ri.PubmedXMLFileName = "xml/pubmed_xml_12750255.xml.bz2" 104 | ri.PubmedExampleXMLChidleyGoStructsWithTypes, err = runCaptureStdout("..", "./chidley", "-G", "-t", ri.PubmedXMLFileName) 105 | if err != nil { 106 | log.Fatal(err) 107 | } 108 | 109 | ri.PubmedExampleXMLChidleyGoStructsWithTypeTiming, err = runCaptureStderr("..", "/usr/bin/time", "-f", "\"Seconds: %E Resident size: %M\"", "./chidley", "-G", ri.PubmedXMLFileName) 110 | if err != nil { 111 | log.Fatal(err) 112 | } 113 | 114 | dirName := "example1" 115 | err = os.MkdirAll(dirName, 0700) 116 | if err != nil { 117 | log.Fatal(err) 118 | } 119 | var str string 120 | 121 | str, err = runCaptureStdout("..", "./chidley", "-W", ri.PubmedXMLFileName) 122 | if err != nil { 123 | log.Fatal(err) 124 | } 125 | 126 | err = ioutil.WriteFile(dirName+"/main.go", []byte(str), 0644) 127 | if err != nil { 128 | log.Fatal(err) 129 | } 130 | 131 | str, err = runCaptureStdout(dirName, "bash", "-c", "go build") 132 | if err != nil { 133 | log.Fatal(err) 134 | } 135 | 136 | ri.GeneratedPubmedCount, err = runCaptureStdout(dirName, "bash", "-c", "./"+dirName+" -c | sort -n") 137 | if err != nil { 138 | log.Fatal(err) 139 | } 140 | 141 | err = generateSimpleGoCode() 142 | 143 | if err != nil { 144 | log.Fatal(err) 145 | } 146 | 147 | // Build generated code 148 | var output string 149 | output, err = runCaptureStdout("gencode", "go", "build") 150 | if err != nil { 151 | log.Println(output) 152 | log.Fatal(err) 153 | } 154 | 155 | // Run generated code, usage 156 | ri.GeneratedUsage, err = runCaptureStdout("gencode", "./gencode", "-h") 157 | if err != nil { 158 | log.Fatal(err) 159 | } 160 | 161 | // Run generated code, convert XML to JSON 162 | ri.GeneratedXMLToJson, err = runCaptureStdout("gencode", "./gencode", "-j") 163 | if err != nil { 164 | log.Fatal(err) 165 | } 166 | 167 | // Run generated code, convert XML to XML 168 | ri.GeneratedXMLToXML, err = runCaptureStdout("gencode", "./gencode", "-x") 169 | if err != nil { 170 | log.Fatal(err) 171 | } 172 | 173 | // Run generated code, count XML tags 174 | ri.GeneratedCountElements, err = runCaptureStdout("gencode", "./gencode", "-c") 175 | if err != nil { 176 | log.Fatal(err) 177 | } 178 | 179 | //log.Println(out) 180 | 181 | if true { 182 | t := template.Must(template.New("readmeTemplate").Parse(readmeTemplate)) 183 | 184 | err = t.Execute(os.Stdout, ri) 185 | 186 | if err != nil { 187 | log.Println("executing template:", err) 188 | } 189 | } 190 | //os.Setenv("PATH", "/home/newtong/go/src/github.com/gnewton/chidley") 191 | 192 | } 193 | 194 | func generateSimpleGoCode() error { 195 | //tmp, err := runCaptureStdout(".", "mkdir", "gencode") 196 | _, _ = runCaptureStdout(".", "mkdir", "gencode") 197 | tmp, err := runCaptureStdout(".", "../chidley", "-W", "../data/test.xml") //> gencode/main.go) 198 | 199 | if err != nil { 200 | log.Println(err) 201 | return err 202 | } 203 | 204 | //file, err := os.Create("gencode/main.go") 205 | file, _ := os.Create("gencode/main.go") 206 | file.WriteString(tmp) 207 | file.Sync() 208 | file.Close() 209 | 210 | //_, _ = runCaptureStdout(".", "rmdir", "gencode") 211 | return err 212 | } 213 | -------------------------------------------------------------------------------- /makereadme/readme.sh.old: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | pubmed_filename="pubmedsample18n0001.xml.gz" 4 | pubmed_filename="pubmed18n0001.xml.gz" 5 | pubmed_url="ftp://ftp.ncbi.nlm.nih.gov/pubmed/baseline/" 6 | #pubmed_url="ftp://ftp.ncbi.nlm.nih.gov/pubmed/baseline-2018-sample/" 7 | pubmed_url+=$pubmed_filename 8 | 9 | test_filename="../data/test.xml" 10 | 11 | #echo "" > foo 12 | 13 | if [ ! -e "$pubmed_filename" ] 14 | then 15 | wget $pubmed_url 16 | fi 17 | 18 | function out(){ 19 | echo "$1" 20 | } 21 | 22 | function exe(){ 23 | out "\`\`\`" 24 | out "$($1 2>&1)" 25 | out "\`\`\`" 26 | } 27 | 28 | 29 | ############start 30 | out "# \`chidley\` converts *any* XML to Go structs (and therefor to JSON)" 31 | out "* By *any*, any XML that can be read by the Go [xml package](http://golang.org/pkg/encoding/xml/) decoder. " 32 | out "* Where *convert* means, generates Go code that when compiled, will convert the XML to JSON" 33 | out "* or will just generate the Go structs that represent the input XML" 34 | out "* or converts XML to XML (useful for validation) " 35 | out "" 36 | out "Author: Glen Newton" 37 | out "Language: Go" 38 | out "" 39 | out "## How does it work (with such a small memory footprint)" 40 | out "\`chidley\` uses the input XML to build a model of each XML element (aka tag)." 41 | out "It examines each instance of a tag, and builds a (single) prototypical representation; that is, the union of all the attributes and all of the child elements of all instances of the tag." 42 | out "So even if there are million instances of a specific tag, there is only one model tag representation." 43 | out "Note that a tag is unique by its namespace+tagname combination (in the Go xml package parlance, [\`space + local\`](http://golang.org/pkg/encoding/xml/#Name)." 44 | out "### Types" 45 | out "\`chidley\` by default makes all values (attributes, tag content) in the generated Go structs a string (which is always valid for XML attributes and content), but it has a flag (\`-t\`) where it will detect and use the most appropriate type. " 46 | out "\`chidley\` tries to fit the **smallest** Go type. " 47 | out "For example, if all instances of a tag contain a number, and all instances are -128 to 127, then it will use an \`int8\` in the Go struct." 48 | 49 | out "" 50 | out "## Usage" 51 | exe "../chidley -h" 52 | 53 | out "" 54 | out "### Specific Usages:" 55 | out "* \`chidley -W ...\`: writes Go code to standard out, so this output should be directed to a filename and subsequently be compiled. When compiled, the resulting binary will:" 56 | out " * convert the XML file to JSON" 57 | out " * or convert the XML file to XML (useful for validation)" 58 | out " * or count the # of elements (space, local) in the XML file" 59 | out "* \`chidley -G ...\`: writes just the Go structs that represent the input XML. For incorporation into the user's code base." 60 | 61 | out "### Example:" 62 | out "Using filename \`test.xml\`" 63 | exe "cat ../test.xml" 64 | cmd="../chidley ./${test_filename}" 65 | 66 | out $cmd 67 | out "### Generated Go structs" 68 | exe "$cmd" 69 | 70 | if [ ! -e "test1" ] 71 | then 72 | mkdir test1 73 | fi 74 | 75 | 76 | 77 | eval "../chidley -W ./${test_filename} > test1/t.go" 78 | 79 | cd test1 80 | go build 81 | 82 | 83 | out "### Usage -W" 84 | exe "./test1 -h" 85 | 86 | out "##### Generated code:: xml -> json" 87 | exe "./test1 -j -s" 88 | 89 | out "##### Generated code: xml -> xml" 90 | exe "./test1 -x -s" 91 | 92 | out "##### Generated code: Count elements -c" 93 | exe "./test1 -c" 94 | 95 | cd .. 96 | rm test1/* 97 | 98 | 99 | 100 | 101 | out "### Example chidley -G:" 102 | out "#### Default" 103 | 104 | out "#### Types turned on -t" 105 | exe "../chidley -t ./${test_filename}" 106 | 107 | out "Note the \`Number int16\` in \`Chiyear\`" 108 | 109 | out "## Larger more complex example" 110 | 111 | 112 | out "Using the large pubmed XML file, $pubmed_url " 113 | cmd="../chidley -t -F ./${pubmed_filename}" 114 | out "\`\$ ${cmd}" 115 | exe "$cmd" 116 | 117 | 118 | out "Timings" 119 | out "### Generate Go program: -W" 120 | out "#### Generated program: count tags" 121 | 122 | out "#### Generated program: convert XML to JSON" 123 | out "##### No streaming" 124 | out "##### With streaming -s" 125 | out "##### Sample of generated JSON" 126 | 127 | out "##### Sample of generated XML to XML" 128 | 129 | out "## Java/JAXB" 130 | out "### Usage" 131 | 132 | 133 | out "### New" 134 | 135 | out "#### Build Java package" 136 | out "#### Running" 137 | 138 | 139 | out "### Limitations" 140 | 141 | 142 | -------------------------------------------------------------------------------- /makereadme/template.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const endInline = ` 4 | 5 | ` + "`" + `` + "`" + `` + "`" 6 | 7 | const beginInline = ` 8 | 9 | ` + endInline 10 | 11 | const readmeTemplate = ` 12 | # ` + "`" + `chidley` + "`" + ` 13 | 14 | # NOTE: The below documentation is out of date with the most recent release. I will work to update these docs in the next week. 2018/10/17 15 | 16 | ############################################################################################################################################### 17 | 18 | ## ` + "`" + `chidley` + "`" + ` converts *any* XML to Go structs (and therefor to JSON) 19 | * By *any*, any XML that can be read by the Go [xml package](http://golang.org/pkg/encoding/xml/) decoder. 20 | * Where *convert* means, generates Go code that when compiled, will convert the XML to JSON 21 | * or will just generate the Go structs that represent the input XML 22 | * or converts XML to XML (useful for validation) 23 | 24 | Author: Glen Newton 25 | Language: Go 26 | 27 | ## New 28 | 29 | # 2016.08.14 30 | Added ability to sort structs into the same order the XML is encountered in the file. Useful for human readers comparing the Go structs to the original XML. 31 | Use flag ` + "`" + `-X` + "`" + ` to invoke. Overrides default of sorting by alphabetical sorting. 32 | 33 | # 2015.07.24 34 | ` + "`" + `chidley` + "`" + ` now supports the user naming of the resulting JAXB Java class package. 35 | Previously the package name could only be ` + "`" + `ca/gnewton/chidley/jaxb` + "`" + `. 36 | Now, using the ` + "`" + `-P name` + "`" + `, the ` + "`" + `jaxb` + "`" + ` default can be altered. 37 | 38 | So ` + "`" + `chidley -J -P "foobar" sample.xml` + "`" + ` will result in Java classes with package name: ` + "`" + `ca/gnewton/chidley/foobar` + "`" + `. 39 | 40 | 41 | ### Previous 42 | 43 | ` + "`" + `chidley` + "`" + ` now has support for Java/JAXB. It generates appropriate Java/JAXB classes and associated maven pom. 44 | 45 | See [Java/JAXB section below](#user-content-java) for usage. 46 | 47 | 48 | ## How does it work (with such a small memory footprint) 49 | ` + "`" + `chidley` + "`" + ` uses the input XML to build a model of each XML element (aka tag). 50 | It examines each instance of a tag, and builds a (single) prototypical representation; that is, the union of all the attributes and all of the child elements of all instances of the tag. 51 | So even if there are million instances of a specific tag, there is only one model tag representation. 52 | Note that a tag is unique by its namespace+tagname combination (in the Go xml package parlance, [` + "`" + `space + local` + "`" + `](http://golang.org/pkg/encoding/xml/#Name). 53 | ### Types 54 | ` + "`" + `chidley` + "`" + ` by default makes all values (attributes, tag content) in the generated Go structs a string (which is always valid for XML attributes and content), but it has a flag (` + "`" + `-t` + "`" + `) where it will detect and use the most appropriate type. 55 | ` + "`" + `chidley` + "`" + ` tries to fit the **smallest** Go type. 56 | For example, if all instances of a tag contain a number, and all instances are -128 to 127, then it will use an ` + "`" + `int8` + "`" + ` in the Go struct. 57 | 58 | ## ` + "`" + `chidley` + "`" + ` binary 59 | Compiled for 64bit Linux Fedora18, go version go1.3 linux/amd64 60 | 61 | ## Usage` + beginInline + ` 62 | $ chidley -h 63 | {{.ChidleyUsage}} 64 | $` + endInline + ` 65 | 66 | ### Specific Usages: 67 | * ` + "`" + `chidley -W ` + "`" + `: writes Go code to standard out, so this output should be directed to a filename and subsequently be compiled. When compiled, the resulting binary will: 68 | * convert the XML file to JSON 69 | * or convert the XML file to XML (useful for validation) 70 | * or count the # of elements (space, local) in the XML file 71 | * ` + "`" + `chidley -G ...` + "`" + `: writes just the Go structs that represent the input XML. For incorporation into the user's code base. 72 | 73 | 74 | ### Example 75 | #### ` + "`" + `data/test.xml` + "`" + `:` + beginInline + ` 76 | {{.SimpleExampleXMLFile}}` + endInline + ` 77 | 78 | #### Generated Go structs: 79 | ` + beginInline + ` 80 | {{.SimpleExampleXMLChidleyGoStructs}}` + endInline + ` 81 | 82 | Note that ` + "`" + `chidley` + "`" + `prepends a ` + "`" + `C` + "`" + `in front of every Go struct that corresponds to an XML tag in the original XML. It also does not alter the tag part of the Go struct name (except where noted below in "Name Changes"). 83 | The reason this is done is 1) The Go XML and JSON libraries only operate on public fields (name must start with a capital); and 2) To avoid name collisions. 84 | Here is a simple and contrived example of name collisions in XMK: 85 | ` + beginInline + ` 86 | 87 | Fred 88 | Gone with the Wind 89 | 90 | ` + endInline + ` 91 | In order to work` + "`" + `` + "`" + ` would have to be capitalized, causing a collision with ` + "`" + `` + "`" + ` 92 | Prefixing both with a capital c avoid this. 93 | 94 | This prefix can be changed with the ` + "`" + `-e` + "`" + ` flag 95 | 96 | 97 | 98 | Note that all XMl tags are converted to Go structs. 99 | However, for those that always correspond to a single element (no sub-tags & no XML attributes), like title:` + "`" + `Footfall` + "`" + ` this is a bit of a waste. 100 | It is possible to have ` + "`" + `chidley` + "`" + `collapse these into inline strings with the ` + "`" + `-F` + "`" + `flag. However, if your example XML is not canonical (i.e. it does not exhibit all uses of all XML tags), it may result in Go structs that do not capture everything that is needed. 101 | 102 | ## Name changes: XML vs. Go structs 103 | XML names can contain dots ` + "`" + `.` + "`" + ` and hyphens or dashes ` + "`" + `-` + "`" + `. These are not valid identifiers for Go structs or variables. These are mapped as: 104 | * ` + "`" + `"-": "_"` + "`" + ` 105 | * ` + "`" + `".": "_dot_"` + "`" + ` 106 | 107 | 108 | Example: 109 | ` + beginInline + ` 110 | ` + "`$" + `chidley -F data/test.xml` + "`" + `: 111 | {{.SimpleExampleXMLChidleyGoStructsCollapsed}}` + endInline + ` 112 | 113 | #### Generating code to read XML` + ` 114 | ` + "`chidley`" + ` can generate Go code that will read in the XML and output a number of things, including the equivalent JSON, XML (XML to XML useful for validation), and a count total for each XML tag in the source file 115 | 116 | Generating code: 117 | ` + beginInline + ` 118 | $ mkdir gencode 119 | $ chidley -W data/test.xml> gencode/main.go 120 | ` + endInline + ` 121 | 122 | #### Usage of generated code 123 | ` + beginInline + ` 124 | $ cd gencode 125 | $ go build 126 | $ ./gencode 127 | {{.GeneratedUsage}} 128 | ` + endInline + ` 129 | 130 | ##### Generated code: Convert XML to JSON ` + "`" + `-j` + "`" + beginInline + ` 131 | $ ./test1 -j -f ../../xml/test1.xml 132 | {{.GeneratedXMLToJson}}` + endInline + ` 133 | 134 | ##### Generated code: Convert XML to XML ` + "`" + `-x` + "`" + ` 135 | ` + beginInline + `$ ./test1 -x -f ../../xml/test1.xml 136 | {{.GeneratedXMLToXML}}` + endInline + ` 137 | 138 | ##### Generated code: Count elements ` + "`" + `-c` + "`" + ` 139 | XML elements (or tags) are counted in the source file (space,local) and are printed-out, unsorted 140 | ` + beginInline + `$ gencode -c 141 | {{.GeneratedCountElements}}` + endInline + ` 142 | 143 | **Note**: the underscore before the colon indicates there is no (or the default) namespace for the element. 144 | 145 | ## Type example 146 | ` + "`" + `` + "`" + `` + "`" + ` 147 | {{.SimpleExampleXMLFile}} 148 | 149 | ` + "`" + `` + "`" + `` + "`" + ` 150 | 151 | #### Default 152 | ` + "`" + `` + "`" + `` + "`" + ` 153 | $ ./chidley -G xml/testType.xml 154 | {{.SimpleExampleXMLChidleyGoStructs}} 155 | 156 | ` + "`" + `` + "`" + `` + "`" + ` 157 | 158 | #### Types turned on ` + "`" + `-t` + "`" + ` 159 | ` + "`" + `` + "`" + `` + "`" + ` 160 | $ ./chidley -G -t data/xml.test 161 | {{.SimpleExampleXMLChidleyGoStructsWithTypes}} 162 | 163 | ` + "`" + `` + "`" + `` + "`" + ` 164 | 165 | Notice: 166 | * ` + "`" + `Text int8` + "`" + ` in ` + "`" + `Chi_age` + "`" + ` 167 | * ` + "`" + `Text bool` + "`" + ` in ` + "`" + `Chi_married` + "`" + ` 168 | 169 | ## Go struct name prefix 170 | ` + "`" + `chidley` + "`" + ` by default prepends a prefix to Go struct type identifiers. The default is ` + "`" + `Chi` + "`" + ` but this can be changed with the ` + "`" + `-e` + "`" + ` flag. If changed from the default, the new prefix must start with a capital letter (for the XML annotation and decoder to work: the struct fields must be public). 171 | 172 | ## Warning 173 | If you are going to use the ` + "`" + `chidley` + "`" + ` generated Go structs on XML other than the input XML, you need to make sure the input XML has examples of all tags, and tag attribute and tag child tag combinations. 174 | 175 | If the input does not have all of these, and you use new XML that has tags not found in the input XML, attributes not seen in tags in the input XML, or child tags not encountered in the input XML, these will not be *seen* by the xml decoder, as they will not be in the Go structs used by the xml decoder. 176 | 177 | ## Limitations 178 | ` + "`" + `chidley` + "`" + ` is constrained by the underlying Go [xml package](http://golang.org/pkg/encoding/xml/) 179 | Some of these limitations include: 180 | * The default encoding supported by ` + "`" + `encoder/xml` + "`" + ` is UTF-8. Right now ` + "`" + `chidley` + "`" + ` does not support additional charsets. 181 | An xml decoder that handles charsets other than UTF-8 is possible (see example https://stackoverflow.com/questions/6002619/unmarshal-an-iso-8859-1-xml-input-in-go). 182 | It is possible that this method might be used in the future to extend ` + "`" + `chidley` + "`" + ` to include a small set of popular charsets. 183 | * For vanilla XML with no namespaces, there should be no problem using ` + "`" + `chidley` + "`" + ` 184 | 185 | ### Go ` + "`" + `xml` + "`" + ` package Namespace issues 186 | * There are a number of bugs open for the Go xml package that relate to XML namespaces: https://code.google.com/p/go/issues/list?can=2&q=xml+namespace If the XML you are using uses namespaces in certain ways, these bugs will impact whether ` + "`" + `chidley` + "`" + ` can create correct structs for your XML 187 | * For _most_ XML with namespaces, the JSON will be OK but if you convert XML to XML using the generated Go code, there will be a chance one of the above mentioned bugs may impact results. Here is an example I encountered: https://groups.google.com/d/msg/golang-nuts/drWStJSt0Pg/Z47JHeij7ToJ 188 | 189 | ## Name 190 | ` + "`" + `chidley` + "`" + ` is named after [Cape Chidley](https://en.wikipedia.org/wiki/Cape_Chidley), Canada 191 | 192 | ## Larger & more complex example 193 | Using the file ` + "`" + `{{.PubmedXMLFileName}}` + "`" + `. Generated from a query to pubmed (similar but much larger than [http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=20598978,444444,455555&retmode=xml](http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=20598978,444444,455555&retmode=xml)), returning a document in the [MEDLINE/Pubmed XML format](http://www.nlm.nih.gov/bsd/licensee/data_elements_doc.html). 194 | * Compressed size: 27M 195 | * Uncompressed size: 337M 196 | 197 | ### Generate Go structs from {{.PubmedXMLFileName}}: ` + "`" + `-G` + "`" + ` 198 | 199 | ` + "`" + `` + "`" + `` + "`" + ` 200 | 201 | {{.PubmedExampleXMLChidleyGoStructsWithTypes}} 202 | 203 | ` + "`" + `` + "`" + `` + "`" + ` 204 | ####Run times 205 | $ /usr/bin/time -f "%E %M" ./chidley -G xml/pubmed_xml_12750255.xml.bz2 206 | {{.PubmedExampleXMLChidleyGoStructsWithTypeTiming}} 207 | 208 | *Note:* All timings from Dell laptop 16GB, regular disk, 8 core i7-3720QM CPU @ 2.60GHz) 209 | ` + "`" + `Linux 3.11.10-100.fc18.x86_64 #1 SMP Mon Dec 2 20:28:38 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux` + "`" + ` 210 | 211 | 212 | 213 | ### Generate Go program: ` + "`" + `-W` + "`" + ` 214 | 215 | ` + "`" + `` + "`" + `` + "`" + ` 216 | $ ./chidley -W xml/pubmed_xml_12750255.xml.bz2 > examples/pubmed/ChiPubmed.go 217 | $ cd examples/pubmed/ 218 | $ go build 219 | $ 220 | ` + "`" + `` + "`" + `` + "`" + ` 221 | 222 | #### Generated program: count tags 223 | ` + "`" + `` + "`" + `` + "`" + ` 224 | $ /usr/bin/time -f "%E %M" ./pubmed -c | sort -n 225 | 0:36.58 10460 226 | {{.GeneratedPubmedCount}} 227 | 228 | 12121212 229 | 230 | $ 231 | ` + "`" + `` + "`" + `` + "`" + ` 232 | 233 | *Note:* The underscore before the colon indicates there is no (or the default) namespace for the element. 234 | 235 | 36 seconds for 337MB XML; resident size: 10.5MB 236 | 237 | #### Generated program: convert XML to JSON 238 | 239 | ##### No streaming 240 | ` + "`" + `` + "`" + `` + "`" + ` 241 | $ /usr/bin/time -f "%E %M" ./pubmed -j > /dev/null 242 | {{.GeneratedPubmedNoStreaming}} 243 | 0:57.26 2866408 244 | $ 245 | ` + "`" + `` + "`" + `` + "`" + ` 246 | 57 seconds for 337MB XML; resident size: 2.9GB 247 | 248 | ##### With streaming ` + "`" + `-s` + "`" + ` 249 | Streaming decodes using the XML elements that are one level down from the top level container element. 250 | ` + "`" + `` + "`" + `` + "`" + ` 251 | {{.GeneratedPubmedStreaming}} 252 | $ /usr/bin/time -f "%E %M" ./pubmed -j -s > /dev/null 253 | 0:58.72 15944 254 | ` + "`" + `` + "`" + `` + "`" + ` 255 | 59 seconds for 337MB XML; resident size: 16MB 256 | 257 | ##### Sample of generated JSON 258 | ` + "`" + `` + "`" + `` + "`" + ` 259 | $ /usr/bin/time -f "%E %M" ./pubmed -j -s |head -310 260 | {{.GeneratedPubmedXMLToJson}} 261 | { 262 | "MedlineCitation": { 263 | "Attr_Owner": "NLM", 264 | "Attr_Status": "MEDLINE", 265 | "Article": { 266 | "Attr_PubModel": "Print", 267 | "Abstract": { 268 | "AbstractText": [ 269 | { 270 | "Text": "A review on the operative methods for prophylaxis of urological complications (fistulas and strictures) due to radical hysterectomy with systemic dissection of lymph nodes is described. The authors recommend the method of T. H. Green as the most effective method. A new method for protection of the ureter with flaps, formed as a \"leg\" from omentum majus, is proposed. The modification has been used in 20 patients without postoperative complications. The method is recommended in cases, when postoperative stenosis or strictures of the ureters are expected as well as when postoperative irradiation is forthcoming." 271 | } 272 | ] 273 | }, 274 | "ArticleTitle": { 275 | "Text": "[A method for preventing the urologic complications connected with the surgical treatment of cancer of the cervix uteri]." 276 | }, 277 | "AuthorList": [ 278 | { 279 | "Attr_CompleteYN": "Y", 280 | "Author": [ 281 | { 282 | "Attr_ValidYN": "Y", 283 | "ForeName": { 284 | "Text": "T" 285 | }, 286 | "Initials": { 287 | "Text": "T" 288 | }, 289 | "LastName": { 290 | "Text": "Kŭrlov" 291 | } 292 | }, 293 | { 294 | "Attr_ValidYN": "Y", 295 | "ForeName": { 296 | "Text": "N" 297 | }, 298 | "Initials": { 299 | "Text": "N" 300 | }, 301 | "LastName": { 302 | "Text": "Vasilev" 303 | } 304 | } 305 | ] 306 | } 307 | ], 308 | "Journal": { 309 | "ISOAbbreviation": { 310 | "Text": "Akush Ginekol (Sofiia)" 311 | }, 312 | "ISSN": { 313 | "Attr_IssnType": "Print", 314 | "Text": "0324-0959" 315 | }, 316 | "JournalIssue": { 317 | "Attr_CitedMedium": "Print", 318 | "Issue": { 319 | "Text": "1" 320 | }, 321 | "PubDate": { 322 | "Year": { 323 | "Text": "1990" 324 | } 325 | }, 326 | "Volume": { 327 | "Text": "29" 328 | } 329 | }, 330 | "Title": { 331 | "Text": "Akusherstvo i ginekologii͡a" 332 | } 333 | }, 334 | "Language": [ 335 | { 336 | "Text": "bul" 337 | } 338 | ], 339 | "Pagination": { 340 | "MedlinePgn": { 341 | "Text": "55-7" 342 | } 343 | }, 344 | "PublicationTypeList": { 345 | "PublicationType": [ 346 | { 347 | "Text": "English Abstract" 348 | }, 349 | { 350 | "Text": "Journal Article" 351 | } 352 | ] 353 | }, 354 | "VernacularTitle": { 355 | "Text": "Metod za profilaktika na urologichnite uslozhneniia, svŭrzani s operativnoto lechenie na raka na matochnata shiĭka." 356 | } 357 | }, 358 | "CitationSubset": [ 359 | { 360 | "Text": "IM" 361 | } 362 | ], 363 | "DateCompleted": { 364 | "Day": { 365 | "Text": "22" 366 | }, 367 | "Month": { 368 | "Text": "08" 369 | }, 370 | "Year": { 371 | "Text": "1990" 372 | } 373 | }, 374 | "DateCreated": { 375 | "Day": { 376 | "Text": "22" 377 | }, 378 | "Month": { 379 | "Text": "08" 380 | }, 381 | "Year": { 382 | "Text": "1990" 383 | } 384 | }, 385 | "DateRevised": { 386 | "Day": { 387 | "Text": "15" 388 | }, 389 | "Month": { 390 | "Text": "11" 391 | }, 392 | "Year": { 393 | "Text": "2006" 394 | } 395 | }, 396 | "MedlineJournalInfo": { 397 | "Country": { 398 | "Text": "BULGARIA" 399 | }, 400 | "ISSNLinking": { 401 | "Text": "0324-0959" 402 | }, 403 | "MedlineTA": { 404 | "Text": "Akush Ginekol (Sofiia)" 405 | }, 406 | "NlmUniqueID": { 407 | "Text": "0370455" 408 | } 409 | }, 410 | "MeshHeadingList": { 411 | "MeshHeading": [ 412 | { 413 | "DescriptorName": { 414 | "Attr_MajorTopicYN": "N", 415 | "Text": "Female" 416 | } 417 | }, 418 | { 419 | "DescriptorName": { 420 | "Attr_MajorTopicYN": "N", 421 | "Text": "Humans" 422 | } 423 | }, 424 | { 425 | "DescriptorName": { 426 | "Attr_MajorTopicYN": "N", 427 | "Text": "Hysterectomy" 428 | }, 429 | "QualifierName": [ 430 | { 431 | "Attr_MajorTopicYN": "N", 432 | "Text": "methods" 433 | } 434 | ] 435 | }, 436 | { 437 | "DescriptorName": { 438 | "Attr_MajorTopicYN": "N", 439 | "Text": "Lymph Node Excision" 440 | }, 441 | "QualifierName": [ 442 | { 443 | "Attr_MajorTopicYN": "N", 444 | "Text": "methods" 445 | } 446 | ] 447 | }, 448 | { 449 | "DescriptorName": { 450 | "Attr_MajorTopicYN": "N", 451 | "Text": "Postoperative Complications" 452 | }, 453 | "QualifierName": [ 454 | { 455 | "Attr_MajorTopicYN": "N", 456 | "Text": "etiology" 457 | }, 458 | { 459 | "Attr_MajorTopicYN": "Y", 460 | "Text": "prevention \u0026 control" 461 | } 462 | ] 463 | }, 464 | { 465 | "DescriptorName": { 466 | "Attr_MajorTopicYN": "N", 467 | "Text": "Urologic Diseases" 468 | }, 469 | "QualifierName": [ 470 | { 471 | "Attr_MajorTopicYN": "N", 472 | "Text": "etiology" 473 | }, 474 | { 475 | "Attr_MajorTopicYN": "Y", 476 | "Text": "prevention \u0026 control" 477 | } 478 | ] 479 | }, 480 | { 481 | "DescriptorName": { 482 | "Attr_MajorTopicYN": "N", 483 | "Text": "Uterine Cervical Neoplasms" 484 | }, 485 | "QualifierName": [ 486 | { 487 | "Attr_MajorTopicYN": "N", 488 | "Text": "complications" 489 | }, 490 | { 491 | "Attr_MajorTopicYN": "Y", 492 | "Text": "surgery" 493 | } 494 | ] 495 | } 496 | ] 497 | }, 498 | "PMID": { 499 | "Attr_Version": "1", 500 | "Text": "2372101" 501 | } 502 | }, 503 | "PubmedData": { 504 | "ArticleIdList": { 505 | "ArticleId": [ 506 | { 507 | "Attr_IdType": "pubmed", 508 | "Text": "2372101" 509 | } 510 | ] 511 | }, 512 | "History": { 513 | "PubMedPubDate": [ 514 | { 515 | "Attr_PubStatus": "pubmed", 516 | "Day": { 517 | "Text": "1" 518 | }, 519 | "Month": { 520 | "Text": "1" 521 | }, 522 | "Year": { 523 | "Text": "1990" 524 | } 525 | }, 526 | { 527 | "Attr_PubStatus": "medline", 528 | "Day": { 529 | "Text": "1" 530 | }, 531 | "Hour": { 532 | "Text": "0" 533 | }, 534 | "Minute": { 535 | "Text": "1" 536 | }, 537 | "Month": { 538 | "Text": "1" 539 | }, 540 | "Year": { 541 | "Text": "1990" 542 | } 543 | }, 544 | { 545 | "Attr_PubStatus": "entrez", 546 | "Day": { 547 | "Text": "1" 548 | }, 549 | "Hour": { 550 | "Text": "0" 551 | }, 552 | "Minute": { 553 | "Text": "0" 554 | }, 555 | "Month": { 556 | "Text": "1" 557 | }, 558 | "Year": { 559 | "Text": "1990" 560 | } 561 | } 562 | ] 563 | }, 564 | "PublicationStatus": { 565 | "Text": "ppublish" 566 | } 567 | } 568 | } 569 | { 570 | "MedlineCitation": { 571 | ` + "`" + `` + "`" + `` + "`" + ` 572 | 573 | ##### Sample of generated XML to XML 574 | ` + "`" + `` + "`" + `` + "`" + ` 575 | $ ./pubmed -x -s |head -100 576 | {{.GeneratedPubmedXMLToXML}} 577 | 578 | 579 |
580 | 581 | A review on the operative methods for prophylaxis of urological complications (fistulas and strictures) due to radical hysterectomy with systemic dissection of lymph nodes is described. The authors recommend the method of T. H. Green as the most effective method. A new method for protection of the ureter with flaps, formed as a "leg" from omentum majus, is proposed. The modification has been used in 20 patients without postoperative complications. The method is recommended in cases, when postoperative stenosis or strictures of the ureters are expected as well as when postoperative irradiation is forthcoming. 582 | 583 | [A method for preventing the urologic complications connected with the surgical treatment of cancer of the cervix uteri]. 584 | 585 | 586 | T 587 | T 588 | Kŭrlov 589 | 590 | 591 | N 592 | N 593 | Vasilev 594 | 595 | 596 | 597 | Akush Ginekol (Sofiia) 598 | 0324-0959 599 | 600 | 1 601 | 602 | 1990 603 | 604 | 29 605 | 606 | Akusherstvo i ginekologii͡a 607 | 608 | bul 609 | 610 | 55-7 611 | 612 | 613 | English Abstract 614 | Journal Article 615 | 616 | Metod za profilaktika na urologichnite uslozhneniia, svŭrzani s operativnoto lechenie na raka na matochnata shiĭka. 617 |
618 | IM 619 | 620 | 22 621 | 08 622 | 1990 623 | 624 | 625 | 22 626 | 08 627 | 1990 628 | 629 | 630 | 15 631 | 11 632 | 2006 633 | 634 | 635 | BULGARIA 636 | 0324-0959 637 | Akush Ginekol (Sofiia) 638 | 0370455 639 | 640 | 641 | 642 | Female 643 | 644 | 645 | Humans 646 | 647 | 648 | Hysterectomy 649 | methods 650 | 651 | 652 | Lymph Node Excision 653 | methods 654 | 655 | 656 | Postoperative Complications 657 | etiology 658 | prevention & control 659 | 660 | 661 | Urologic Diseases 662 | etiology 663 | prevention & control 664 | 665 | 666 | Uterine Cervical Neoplasms 667 | complications 668 | surgery 669 | 670 | 671 | 2372101 672 | 673 | 674 | 675 | 2372101 676 | 677 | ` + "`" + `` + "`" + `` + "`" + ` 678 | 679 | ## Java/JAXB 680 | ` + "`" + `chidley` + "`" + ` now supports the production of Java/JAXB code. It generates a class-per-element, with classes mapping to the Go structs generated by the XML extraction. 681 | Its only dependency is Google [Gson](https://code.google.com/p/google-gson/), for JSON generation. 682 | 683 | 684 | ### Usage 685 | ` + "`" + `chidley` + "`" + ` creates a maven project in ` + "`" + `./java` + "`" + ` (settable using the ` + "`" + `-D` + "`" + ` flag) and creates Java JAXB files in ` + "`" + `src/main/java/ca/gnewton/chidley/jaxb/xml` + "`" + `. 686 | It creates a ` + "`" + `Main.java` + "`" + ` in ` + "`" + `src/main/java/ca/gnewton/chidley/jaxb` + "`" + ` 687 | ` + "`" + `` + "`" + `` + "`" + ` 688 | $ chidley -J xml/test1.xml 689 | {{.ChidleyGenerateJava}} 690 | 2014/09/02 10:22:27 printJavaJaxbVisitor.go:100: Writing java Class file: java/src/main/java/ca/gnewton/chidley/jaxb/xml/ChiDocs.java 691 | 2014/09/02 10:22:27 printJavaJaxbVisitor.go:100: Writing java Class file: java/src/main/java/ca/gnewton/chidley/jaxb/xml/ChiDoc.java 692 | 2014/09/02 10:22:27 printJavaJaxbVisitor.go:100: Writing java Class file: java/src/main/java/ca/gnewton/chidley/jaxb/xml/ChiTitle.java 693 | 2014/09/02 10:22:27 printJavaJaxbVisitor.go:100: Writing java Class file: java/src/main/java/ca/gnewton/chidley/jaxb/xml/ChiAuthor.java 694 | 2014/09/02 10:22:27 printJavaJaxbVisitor.go:100: Writing java Class file: java/src/main/java/ca/gnewton/chidley/jaxb/xml/ChiLast_name.java 695 | 2014/09/02 10:22:28 printJavaJaxbVisitor.go:100: Writing java Class file: java/src/main/java/ca/gnewton/chidley/jaxb/xml/ChiFirstName.java 696 | 2014/09/02 10:22:28 printJavaJaxbVisitor.go:100: Writing java Class file: java/src/main/java/ca/gnewton/chidley/jaxb/Main.java 697 | ` + "`" + `` + "`" + `` + "`" + ` 698 | 699 | ### New 700 | Changing the package with "-P": 701 | ` + "`" + `` + "`" + `` + "`" + ` 702 | $ chidley -J -P testFoo xml/test1.xml 703 | {{.ChidleyGenerateJavaChangePackageName}} 704 | 2015/07/24 15:56:52 printJavaJaxbVisitor.go:103: Writing java Class file: java/src/main/java/ca/gnewton/chidley/testFoo/xml/ChiDocs.java 705 | 2015/07/24 15:56:52 printJavaJaxbVisitor.go:103: Writing java Class file: java/src/main/java/ca/gnewton/chidley/testFoo/xml/ChiDoc.java 706 | 2015/07/24 15:56:52 printJavaJaxbVisitor.go:103: Writing java Class file: java/src/main/java/ca/gnewton/chidley/testFoo/xml/ChiTitle.java 707 | 2015/07/24 15:56:52 printJavaJaxbVisitor.go:103: Writing java Class file: java/src/main/java/ca/gnewton/chidley/testFoo/xml/ChiAuthor.java 708 | 2015/07/24 15:56:52 printJavaJaxbVisitor.go:103: Writing java Class file: java/src/main/java/ca/gnewton/chidley/testFoo/xml/ChiLast_name.java 709 | 2015/07/24 15:56:52 printJavaJaxbVisitor.go:103: Writing java Class file: java/src/main/java/ca/gnewton/chidley/testFoo/xml/ChiFirstName.java 710 | 2015/07/24 15:56:52 printJavaJaxbVisitor.go:103: Writing java Class file: java/src/main/java/ca/gnewton/chidley/testFoo/Main.java 711 | $ 712 | ` + "`" + `` + "`" + `` + "`" + ` 713 | 714 | #### Build Java package 715 | ` + "`" + `` + "`" + `` + "`" + ` 716 | $ cd java 717 | $ ls 718 | pom.xml src 719 | $ $ mvn package 720 | {{.ChidleyGenerateJavaMavenBuild}} 721 | [INFO] Scanning for projects... 722 | [INFO] 723 | [INFO] ------------------------------------------------------------------------ 724 | [INFO] Building chidley-jaxb 1.0-SNAPSHOT 725 | [INFO] ------------------------------------------------------------------------ 726 | [INFO] 727 | [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ chidley-jaxb --- 728 | [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! 729 | [INFO] skip non existing resourceDirectory /home/newtong/work/chidley/java/src/main/resources 730 | [INFO] 731 | [INFO] --- maven-compiler-plugin:2.5.1:compile (default-compile) @ chidley-jaxb --- 732 | [WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! 733 | [INFO] Compiling 7 source files to /home/newtong/work/chidley/java/target/classes 734 | [INFO] 735 | [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ chidley-jaxb --- 736 | [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! 737 | [INFO] skip non existing resourceDirectory /home/newtong/work/chidley/java/src/test/resources 738 | [INFO] 739 | [INFO] --- maven-compiler-plugin:2.5.1:testCompile (default-testCompile) @ chidley-jaxb --- 740 | [INFO] No sources to compile 741 | [INFO] 742 | [INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ chidley-jaxb --- 743 | [INFO] No tests to run. 744 | [INFO] 745 | [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ chidley-jaxb --- 746 | [INFO] Building jar: /home/newtong/work/chidley/java/target/chidley-jaxb-1.0-SNAPSHOT.jar 747 | [INFO] ------------------------------------------------------------------------ 748 | [INFO] BUILD SUCCESS 749 | [INFO] ------------------------------------------------------------------------ 750 | [INFO] Total time: 3.514s 751 | [INFO] Finished at: Tue Sep 02 10:29:18 EDT 2014 752 | [INFO] Final Memory: 11M/240M 753 | [INFO] ------------------------------------------------------------------------ 754 | $ 755 | ` + "`" + `` + "`" + `` + "`" + ` 756 | 757 | #### Running 758 | ` + "`" + `` + "`" + `` + "`" + ` 759 | $ export CLASSPATH=./target/chidley-jaxb-1.0-SNAPSHOT.jar:/home/myhome/.m2/repository/com/google/code/gson/gson/2.3/gson-2.3.jar:$CLASSPATH 760 | $ java ca.gnewton.chidley.jaxb.Main 761 | zzzzzz {{.ChidleyGenerateJavaRun}} mmmmm 762 | { 763 | "language": "eng", 764 | "doc": [ 765 | { 766 | "type": "book", 767 | "title": { 768 | "tagValue": "Dune" 769 | }, 770 | "author": { 771 | "last-name": { 772 | "tagValue": "Herbert" 773 | }, 774 | "firstName": { 775 | "tagValue": "Frank" 776 | } 777 | } 778 | }, 779 | { 780 | "type": "article", 781 | "title": { 782 | "tagValue": "Brave New Wold" 783 | }, 784 | "author": { 785 | "last-name": { 786 | "tagValue": "Huxley" 787 | }, 788 | "firstName": { 789 | "tagValue": "Aldous" 790 | } 791 | } 792 | } 793 | ] 794 | } 795 | $ 796 | ` + "`" + `` + "`" + `` + "`" + ` 797 | 798 | ### Limitations 799 | - Can handle vanilla XML (no namespaces) OK 800 | - Can handle top level namespaces OK 801 | ` + "`" + `` + "`" + `` + "`" + ` 802 | 803 | 807 | 808 | A book entry 809 | 810 | 811 | A article entry 812 | 813 | n 814 | ` + "`" + `` + "`" + `` + "`" + ` 815 | - *Cannot* handle element- or attribute-level namespaces (*soon*), like: 816 | ` + "`" + `` + "`" + `` + "`" + ` 817 | 818 | 819 | number one 820 | number two 821 | 822 | ` + "`" + `` + "`" + `` + "`" + ` 823 | - *Cannot* read ` + "`" + `gz` + "`" + ` or ` + "`" + `bz2` + "`" + ` compressed XML (soon) 824 | - *Cannot* do [XML streaming](https://stackoverflow.com/questions/1134189/can-jaxb-parse-large-xml-files-in-chunks) (thus limited to smaller XML files) (perhaps soon?) 825 | 826 | 827 | Copyright 2014,2015,2016 Glen Newton 828 | ` 829 | -------------------------------------------------------------------------------- /makereadme/util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os/exec" 6 | ) 7 | 8 | func runDiscardOutput(dir string, name string, arg ...string) error { 9 | 10 | //err, stderr, stdout := run(cmd, false, false, false) 11 | _, _, err := run(false, false, false, dir, name, arg...) 12 | return err 13 | } 14 | 15 | func run(stderr, stdout, seperate bool, dir, name string, arg ...string) (string, string, error) { 16 | //cmd, err := exec.Command(name, arg...).Output() 17 | //if dir != "" { 18 | //cmd.Dir = dir 19 | //} 20 | //err := cmd.Run() 21 | //log.Println("--------------", cmd.ProcessState) 22 | 23 | //str, _ := cmd.Output() 24 | 25 | //return string(cmd), "", err 26 | 27 | //cmd := exec.Command("sh", "-c", "echo stdout; echo 1>&2 stderr") 28 | 29 | cmd := exec.Command(name, arg...) 30 | if dir != "" { 31 | cmd.Dir = dir 32 | } 33 | stdoutStderr, err := cmd.CombinedOutput() 34 | if err != nil { 35 | log.Println("Error running:", name, arg) 36 | log.Println(err) 37 | return "", "", err 38 | } 39 | return string(stdoutStderr), "", err 40 | } 41 | 42 | func runCaptureStdout(dir string, name string, arg ...string) (string, error) { 43 | stdout, _, err := run(true, false, false, dir, name, arg...) 44 | return stdout, err 45 | } 46 | 47 | func runCaptureStderr(dir string, name string, arg ...string) (string, error) { 48 | _, stderr, err := run(false, true, false, dir, name, arg...) 49 | return stderr, err 50 | } 51 | 52 | // cmd := exec.Command("myCommand", "arg1", "arg2") 53 | // cmd.Dir = "/path/to/work/dir" 54 | // cmd.Run() 55 | 56 | func runCaptureAll(dir string, name string, arg ...string) (string, string, error) { 57 | stdout, stderr, err := run(true, true, false, dir, name, arg...) 58 | return stdout, stderr, err 59 | } 60 | -------------------------------------------------------------------------------- /mvnTemplate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type JaxbMavenPomInfo struct { 4 | AppName string 5 | } 6 | 7 | const mavenPomTemplate = ` 8 | 10 | 4.0.0 11 | ca.gnewton.chidley.{{.AppName}} 12 | {{.AppName}} 13 | jar 14 | 1.0-SNAPSHOT 15 | {{.AppName}} 16 | http://maven.apache.org 17 | 18 | 19 | junit 20 | junit 21 | 3.8.1 22 | test 23 | 24 | 25 | com.google.code.gson 26 | gson 27 | 2.3 28 | compile 29 | 30 | 31 | 32 | ` 33 | -------------------------------------------------------------------------------- /node.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | //"log" 6 | ) 7 | 8 | type Node struct { 9 | name string 10 | space string 11 | spaceTag string 12 | parent *Node 13 | parents []*Node 14 | children map[string]*Node 15 | childCount map[string]int 16 | repeats bool 17 | nodeTypeInfo *NodeTypeInfo 18 | hasCharData bool 19 | tempCharData string 20 | charDataCount int64 21 | discoveredOrder int 22 | ignoredTag bool 23 | } 24 | 25 | type NodeVisitor interface { 26 | Visit(n *Node) bool 27 | AlreadyVisited(n *Node) bool 28 | SetAlreadyVisited(n *Node) 29 | } 30 | 31 | func (n *Node) initialize(name string, space string, spaceTag string, parent *Node) { 32 | n.parent = parent 33 | n.parents = make([]*Node, 0, 0) 34 | n.pushParent(parent) 35 | n.name = name 36 | n.space = space 37 | n.spaceTag = spaceTag 38 | n.children = make(map[string]*Node) 39 | n.childCount = make(map[string]int) 40 | n.nodeTypeInfo = new(NodeTypeInfo) 41 | n.nodeTypeInfo.initialize() 42 | n.hasCharData = false 43 | n.ignoredTag = false 44 | } 45 | 46 | func (n *Node) makeName() string { 47 | spaceTag := "" 48 | if n.spaceTag != "" { 49 | spaceTag = "_" + n.spaceTag 50 | } 51 | //return capitalizeFirstLetter(cleanName(n.name)) + spaceTag 52 | return cleanName(n.name) + spaceTag 53 | } 54 | 55 | func (n *Node) makeType(prefix string, suffix string) string { 56 | return goVariableNameSanitize(capitalizeFirstLetter(makeTypeGeneric(n.name, n.spaceTag, prefix, suffix, !keepXmlFirstLetterCase)) + n.renderSpaceTag()) 57 | } 58 | 59 | func (n *Node) renderSpaceTag() string { 60 | if len(strings.TrimSpace(n.spaceTag)) == 0 { 61 | return "" 62 | } else { 63 | return "__" + n.spaceTag 64 | } 65 | } 66 | 67 | func (n *Node) makeJavaType(prefix string, suffix string) string { 68 | return capitalizeFirstLetter(makeTypeGeneric(n.name, n.spaceTag, prefix, suffix, !keepXmlFirstLetterCase)) 69 | } 70 | 71 | func (n *Node) peekParent() *Node { 72 | if len(n.parents) == 0 { 73 | return nil 74 | } 75 | a := n.parents 76 | return a[len(a)-1] 77 | } 78 | 79 | func (n *Node) pushParent(parent *Node) { 80 | n.parents = append(n.parents, parent) 81 | } 82 | 83 | func (n *Node) popParent() *Node { 84 | if len(n.parents) == 0 { 85 | return nil 86 | } 87 | var poppedNode *Node 88 | a := n.parents 89 | poppedNode, n.parents = a[len(a)-1], a[:len(a)-1] 90 | return poppedNode 91 | } 92 | 93 | func makeTypeGeneric(name string, space string, prefix string, suffix string, capitalizeName bool) string { 94 | spaceTag := "" 95 | 96 | if capitalizeName { 97 | name = capitalizeFirstLetter(name) 98 | } 99 | 100 | return prefix + spaceTag + cleanName(name) + suffix 101 | 102 | } 103 | -------------------------------------------------------------------------------- /nodeTypeInfo.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | ) 7 | 8 | type NodeTypeInfo struct { 9 | alwaysBool bool 10 | alwaysFloat32 bool 11 | alwaysFloat64 bool 12 | 13 | alwaysInt0 bool 14 | alwaysInt08 bool 15 | alwaysInt16 bool 16 | alwaysInt32 bool 17 | alwaysInt64 bool 18 | 19 | alwaysUint08 bool 20 | alwaysUint16 bool 21 | alwaysUint32 bool 22 | alwaysUint64 bool 23 | maxLength int64 24 | } 25 | 26 | func (nti *NodeTypeInfo) initialize() { 27 | nti.alwaysBool = true 28 | nti.alwaysFloat32 = true 29 | nti.alwaysFloat64 = true 30 | 31 | nti.alwaysInt0 = true 32 | nti.alwaysInt08 = true 33 | nti.alwaysInt16 = true 34 | nti.alwaysInt32 = true 35 | nti.alwaysInt64 = true 36 | 37 | nti.alwaysUint08 = true 38 | nti.alwaysUint16 = true 39 | nti.alwaysUint32 = true 40 | nti.alwaysUint64 = true 41 | nti.maxLength = 0 42 | } 43 | 44 | func (n *NodeTypeInfo) addFieldLength(l int64) { 45 | if n.maxLength < l { 46 | n.maxLength = l 47 | } 48 | } 49 | 50 | func (n *NodeTypeInfo) checkFieldType(v string) { 51 | v = strings.TrimSpace(v) 52 | 53 | if _, err := strconv.ParseBool(v); err != nil { 54 | n.alwaysBool = false 55 | } 56 | 57 | if _, err := strconv.ParseFloat(v, 32); err != nil { 58 | n.alwaysFloat32 = false 59 | } 60 | 61 | if _, err := strconv.ParseFloat(v, 64); err != nil { 62 | n.alwaysFloat64 = false 63 | } 64 | 65 | if _, err := strconv.ParseInt(v, 10, 0); err != nil { 66 | n.alwaysInt0 = false 67 | } 68 | 69 | if _, err := strconv.ParseInt(v, 10, 8); err != nil { 70 | n.alwaysInt08 = false 71 | } 72 | 73 | if _, err := strconv.ParseInt(v, 10, 16); err != nil { 74 | n.alwaysInt16 = false 75 | } 76 | 77 | if _, err := strconv.ParseInt(v, 10, 32); err != nil { 78 | n.alwaysInt32 = false 79 | } 80 | 81 | if _, err := strconv.ParseInt(v, 10, 64); err != nil { 82 | n.alwaysInt64 = false 83 | } 84 | 85 | if _, err := strconv.ParseUint(v, 10, 8); err != nil { 86 | n.alwaysUint08 = false 87 | } 88 | 89 | if _, err := strconv.ParseUint(v, 10, 16); err != nil { 90 | n.alwaysUint16 = false 91 | } 92 | 93 | if _, err := strconv.ParseUint(v, 10, 32); err != nil { 94 | n.alwaysUint32 = false 95 | } 96 | 97 | if _, err := strconv.ParseUint(v, 10, 64); err != nil { 98 | n.alwaysUint64 = false 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /printGoStructVisitor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sort" 7 | //"strconv" 8 | ) 9 | 10 | type PrintGoStructVisitor struct { 11 | alreadyVisited map[string]bool 12 | alreadyVisitedNodes map[string]*Node 13 | globalTagAttributes map[string]([]*FQN) 14 | 15 | maxDepth int 16 | depth int 17 | nameSpaceTagMap map[string]string 18 | useType bool 19 | nameSpaceInJsonName bool 20 | writer io.Writer 21 | } 22 | 23 | func (v *PrintGoStructVisitor) init(writer io.Writer, maxDepth int, globalTagAttributes map[string]([]*FQN), nameSpaceTagMap map[string]string, useType bool, nameSpaceInJsonName bool) { 24 | v.alreadyVisited = make(map[string]bool) 25 | v.alreadyVisitedNodes = make(map[string]*Node) 26 | v.globalTagAttributes = make(map[string]([]*FQN)) 27 | v.globalTagAttributes = globalTagAttributes 28 | v.writer = writer 29 | v.maxDepth = maxDepth 30 | v.depth = 0 31 | v.nameSpaceTagMap = nameSpaceTagMap 32 | v.useType = useType 33 | v.nameSpaceInJsonName = nameSpaceInJsonName 34 | } 35 | 36 | func (v *PrintGoStructVisitor) Visit(node *Node) bool { 37 | 38 | v.depth += 1 39 | 40 | if v.AlreadyVisited(node) || node.ignoredTag { 41 | v.depth += 1 42 | return false 43 | } 44 | v.SetAlreadyVisited(node) 45 | 46 | for _, child := range node.children { 47 | v.Visit(child) 48 | } 49 | v.depth += 1 50 | return true 51 | } 52 | 53 | func print(v *PrintGoStructVisitor, node *Node) error { 54 | if node.ignoredTag || node.name == "" { 55 | return nil 56 | } 57 | if flattenStrings && isStringOnlyField(node, len(v.globalTagAttributes[nk(node)])) { 58 | //v.lineChannel <- "//type " + node.makeType(namePrefix, nameSuffix) 59 | return nil 60 | } 61 | 62 | attributes := v.globalTagAttributes[nk(node)] 63 | //v.lineChannel <- "type " + node.makeType(namePrefix, nameSuffix) + " struct {" 64 | fmt.Fprintln(v.writer, "type "+node.makeType(namePrefix, nameSuffix)+" struct {") 65 | 66 | // fmt.Fprintln(v.writer, "\tXMLName xml.Name`"+makeXmlAnnotation(node.space, false, node.name)+" "+makeJsonAnnotation(node.spaceTag, false, node.name)+"`") 67 | 68 | fmt.Fprintln(v.writer, "\tXMLName xml.Name `"+makeAnnotation("xml", node.space, false, false, node.name)+" "+makeJsonAnnotation(node.spaceTag, false, node.name)+"`") 69 | 70 | //return makeAnnotation("xml", spaceTag, true, false, name) 71 | 72 | makeAttributes(v.writer, attributes, v.nameSpaceTagMap) 73 | 74 | err := v.printInternalFields(len(attributes), node) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | //v.lineChannel <- "}\n" 80 | fmt.Fprintln(v.writer, "}") 81 | fmt.Fprintln(v.writer, "") 82 | 83 | return nil 84 | } 85 | 86 | func (v *PrintGoStructVisitor) AlreadyVisited(n *Node) bool { 87 | _, ok := v.alreadyVisited[nk(n)] 88 | return ok 89 | } 90 | 91 | func (v *PrintGoStructVisitor) SetAlreadyVisited(n *Node) { 92 | v.alreadyVisited[nk(n)] = true 93 | v.alreadyVisitedNodes[nk(n)] = n 94 | } 95 | 96 | func (v *PrintGoStructVisitor) printInternalFields(nattributes int, n *Node) error { 97 | var fields []string 98 | 99 | // Fields in this struct 100 | for i, _ := range n.children { 101 | child := n.children[i] 102 | if child.ignoredTag { 103 | continue 104 | } 105 | var def FieldDef 106 | if flattenStrings && isStringOnlyField(child, len(v.globalTagAttributes[nk(child)])) { 107 | //field = "\t" + child.spaceTag + child.makeType(namePrefix, nameSuffix) + " string `" + makeXmlAnnotation(child.space, false, child.name) + "`" //+ " // ********* " + lengthTagName + ":\"" + lengthTagAttribute + lengthTagSeparator + strconv.FormatInt(child.nodeTypeInfo.maxLength+lengthTagPadding, 10) + "\"" 108 | def.GoName = child.makeType(namePrefix, nameSuffix) 109 | //def.GoType = "string" 110 | def.GoType = findType(child.nodeTypeInfo, useType) 111 | def.XMLName = child.name 112 | def.XMLNameSpace = child.space 113 | } else { 114 | 115 | // Field name and type are the same: i.e. Person *Person or Persons []Persons 116 | nameAndType := child.makeType(namePrefix, nameSuffix) 117 | 118 | def.GoName = nameAndType 119 | def.GoType = nameAndType 120 | def.XMLName = child.name 121 | def.XMLNameSpace = child.space 122 | 123 | if child.repeats { 124 | def.GoTypeArrayOrPointer = "[]*" 125 | } else { 126 | def.GoTypeArrayOrPointer = "*" 127 | } 128 | } 129 | if flattenStrings { 130 | def.Length = child.nodeTypeInfo.maxLength 131 | } 132 | fieldDefString, err := render(def) 133 | if err != nil { 134 | return err 135 | } 136 | fields = append(fields, fieldDefString) 137 | } 138 | 139 | // Is this chardata Field (string) 140 | if n.hasCharData { 141 | xmlString := " `xml:\",chardata\" " + makeJsonAnnotation("", false, "") + "`" 142 | thisType := findType(n.nodeTypeInfo, useType) 143 | thisVariableName := findFieldNameFromTypeInfo(thisType) 144 | 145 | charField := "\t" + thisVariableName + " " + thisType + xmlString 146 | 147 | if flattenStrings { 148 | //charField += "// maxLength=" + strconv.FormatInt(n.nodeTypeInfo.maxLength, 10) 149 | if len(n.children) == 0 && nattributes == 0 { 150 | charField += "// *******************" 151 | } 152 | } 153 | //GOOD 154 | //charField += " // maxLength=" + strconv.FormatInt(n.nodeTypeInfo.maxLength, 10) 155 | 156 | fields = append(fields, charField) 157 | } 158 | 159 | sort.Strings(fields) 160 | for i := 0; i < len(fields); i++ { 161 | //v.lineChannel <- fields[i] 162 | fmt.Fprintln(v.writer, fields[i]) 163 | } 164 | return nil 165 | } 166 | 167 | func makeJsonAnnotation(spaceTag string, useSpaceTagInName bool, name string) string { 168 | return makeAnnotation("json", spaceTag, false, useSpaceTagInName, name) 169 | } 170 | 171 | func makeXmlAnnotation(spaceTag string, useSpaceTag bool, name string) string { 172 | return makeAnnotation("xml", spaceTag, true, false, name) 173 | } 174 | 175 | func makeDbAnnotation(spaceTag string, useSpaceTag bool, name string) string { 176 | return makeAnnotation("db", spaceTag, true, false, name) 177 | } 178 | 179 | func makeAnnotation(annotationId string, spaceTag string, useSpaceTag bool, useSpaceTagInName bool, name string) (annotation string) { 180 | annotation = annotationId + ":\"" 181 | 182 | if useSpaceTag { 183 | annotation = annotation + spaceTag 184 | annotation = annotation + " " 185 | } 186 | 187 | if useSpaceTagInName { 188 | if spaceTag != "" { 189 | annotation = annotation + spaceTag 190 | } 191 | } 192 | 193 | annotation = annotation + name + ",omitempty\"" 194 | 195 | return annotation 196 | } 197 | -------------------------------------------------------------------------------- /printJavaJaxbVisitor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "log" 7 | "os" 8 | "text/template" 9 | "time" 10 | ) 11 | 12 | type PrintJavaJaxbVisitor struct { 13 | alreadyVisited map[string]bool 14 | globalTagAttributes map[string]([]*FQN) 15 | nameSpaceTagMap map[string]string 16 | useType bool 17 | javaDir string 18 | javaPackage string 19 | namePrefix string 20 | Date time.Time 21 | } 22 | 23 | func (v *PrintJavaJaxbVisitor) Visit(node *Node) bool { 24 | if v.AlreadyVisited(node) { 25 | return false 26 | } 27 | v.SetAlreadyVisited(node) 28 | 29 | attributes := v.globalTagAttributes[nk(node)] 30 | 31 | class := new(JaxbClassInfo) 32 | class.init() 33 | class.Date = v.Date 34 | class.PackageName = v.javaPackage 35 | class.ClassName = v.namePrefix + cleanName(capitalizeFirstLetter(node.name)) 36 | class.HasValue = node.hasCharData 37 | class.ValueType = findJavaType(node.nodeTypeInfo, v.useType) 38 | class.Name = node.name 39 | 40 | for _, fqn := range attributes { 41 | jat := new(JaxbAttribute) 42 | cleanName := cleanName(fqn.name) 43 | jat.Name = fqn.name 44 | jat.NameUpper = capitalizeFirstLetter(cleanName) 45 | if v.namePrefix != "" { 46 | jat.NameLower = lowerFirstLetter(v.namePrefix) + capitalizeFirstLetter(cleanName) 47 | } else { 48 | jat.NameLower = lowerFirstLetter(cleanName) 49 | } 50 | jat.NameSpace = fqn.space 51 | class.Attributes = append(class.Attributes, jat) 52 | } 53 | 54 | for _, child := range node.children { 55 | jaf := new(JaxbField) 56 | jaf.Name = child.name 57 | cleanName := cleanName(child.name) 58 | jaf.NameUpper = capitalizeFirstLetter(cleanName) 59 | if v.namePrefix != "" { 60 | jaf.NameLower = lowerFirstLetter(v.namePrefix) + capitalizeFirstLetter(cleanName) 61 | } else { 62 | jaf.NameLower = lowerFirstLetter(cleanName) 63 | } 64 | jaf.NameSpace = child.space 65 | jaf.Repeats = child.repeats 66 | jaf.TypeName = child.makeJavaType(v.namePrefix, "") 67 | class.Fields = append(class.Fields, jaf) 68 | 69 | } 70 | 71 | printJaxbClass(class, v.javaDir+"/xml") 72 | 73 | for _, child := range node.children { 74 | v.Visit(child) 75 | } 76 | 77 | return true 78 | } 79 | 80 | func (v *PrintJavaJaxbVisitor) AlreadyVisited(n *Node) bool { 81 | _, ok := v.alreadyVisited[nk(n)] 82 | return ok 83 | } 84 | 85 | func (v *PrintJavaJaxbVisitor) SetAlreadyVisited(n *Node) { 86 | v.alreadyVisited[nk(n)] = true 87 | } 88 | 89 | func printJaxbClass(class *JaxbClassInfo, dir string) { 90 | t := template.Must(template.New("chidleyJaxbGen").Parse(jaxbClassTemplate)) 91 | //err := t.Execute(os.Stdout, jb) 92 | writer, f, err := javaClassWriter(dir, class.PackageName+".xml", class.ClassName) 93 | defer f.Close() 94 | err = t.Execute(writer, class) 95 | if err != nil { 96 | log.Println("executing template:", err) 97 | } 98 | bufio.NewWriter(writer).Flush() 99 | } 100 | 101 | func javaClassWriter(dir string, packageName string, className string) (io.Writer, *os.File, error) { 102 | fullPath := dir + "/" + className + ".java" 103 | log.Print("Writing java Class file: " + fullPath) 104 | fi, err := os.Create(fullPath) 105 | if err != nil { 106 | log.Print("Problem creating file: " + fullPath) 107 | panic(err) 108 | } 109 | return bufio.NewWriter(fi), fi, nil 110 | } 111 | -------------------------------------------------------------------------------- /source.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "log" 7 | "net/http" 8 | "os" 9 | ) 10 | 11 | type Source interface { 12 | io.Closer 13 | newSource(name string) error 14 | getName() string 15 | getReader() io.Reader 16 | } 17 | 18 | type GenericSource struct { 19 | name string 20 | reader io.Reader 21 | } 22 | 23 | type FileSource struct { 24 | GenericSource 25 | file *os.File 26 | } 27 | 28 | type UrlSource struct { 29 | GenericSource 30 | } 31 | 32 | type StdinSource struct { 33 | GenericSource 34 | } 35 | 36 | //StdInSource impl 37 | func (us *StdinSource) copySource() (Source, error) { 38 | err := new(InternalError) 39 | //error.ErrorString = "copySource not supported" 40 | return nil, err 41 | } 42 | 43 | func (us *StdinSource) getName() string { 44 | return "" 45 | } 46 | 47 | func (us *StdinSource) newSource(name string) error { 48 | us.reader = bufio.NewReader(os.Stdin) 49 | return nil 50 | } 51 | 52 | func (us *StdinSource) Close() error { 53 | return nil 54 | } 55 | 56 | func (us *StdinSource) getReader() io.Reader { 57 | return us.reader 58 | } 59 | 60 | //UrlSource impl 61 | func (us *UrlSource) copySource() (Source, error) { 62 | copy := new(UrlSource) 63 | err := copy.newSource(us.name) 64 | return copy, err 65 | } 66 | 67 | func (us *UrlSource) getName() string { 68 | return us.name 69 | } 70 | 71 | func (us *UrlSource) newSource(name string) error { 72 | us.name = name 73 | var err error 74 | 75 | res, err := http.Get(name) 76 | if err != nil { 77 | log.Fatal(err) 78 | } 79 | 80 | if res.StatusCode != 200 { 81 | log.Fatal("ERROR: bad http status code != 200: ", res.StatusCode, " ", name) 82 | return nil 83 | 84 | } 85 | us.reader = res.Body 86 | 87 | return err 88 | } 89 | 90 | func (us UrlSource) Close() error { 91 | closer, ok := us.reader.(io.Closer) 92 | if ok { 93 | return closer.Close() 94 | } 95 | 96 | return nil 97 | } 98 | 99 | func (us *UrlSource) getReader() io.Reader { 100 | return us.reader 101 | } 102 | 103 | // FileSource impl 104 | func (fs *FileSource) copySource() (Source, error) { 105 | copy := new(FileSource) 106 | err := copy.newSource(fs.name) 107 | return copy, err 108 | } 109 | 110 | func (fs *FileSource) getName() string { 111 | return fs.name 112 | } 113 | 114 | func (fs *FileSource) newSource(name string) error { 115 | fs.name = name 116 | var err error 117 | fs.reader, fs.file, err = genericReader(name) 118 | 119 | return err 120 | } 121 | 122 | func (fs *FileSource) Close() error { 123 | return fs.file.Close() 124 | } 125 | 126 | func (fs FileSource) getReader() io.Reader { 127 | return fs.reader 128 | } 129 | -------------------------------------------------------------------------------- /stdoutWriter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type stdoutWriter struct { 8 | } 9 | 10 | var doneChannel chan bool 11 | 12 | func (w *stdoutWriter) open(s string, lineChannel chan string) error { 13 | doneChannel = make(chan bool) 14 | go w.writer(lineChannel, doneChannel) 15 | 16 | return nil 17 | } 18 | 19 | func (w *stdoutWriter) writer(lineChannel chan string, doneChannel chan bool) { 20 | for line := range lineChannel { 21 | fmt.Println(line) 22 | } 23 | doneChannel <- true 24 | } 25 | 26 | func (w *stdoutWriter) close() { 27 | _ = <-doneChannel 28 | } 29 | -------------------------------------------------------------------------------- /test_xml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | #FILES=`ls xml/*.xml xml/*.xml.bz2 xml/*.xml.gz` 5 | FILES=`ls xml/*.xml xml/*.xml.gz` 6 | 7 | for f in $FILES 8 | do 9 | echo "" 10 | echo "" 11 | echo "" 12 | echo "==================================================================" 13 | echo "Processing file $f" 14 | echo "" 15 | echo "Go code generation" 16 | /usr/bin/time -f "%E %M" ./chidley -W $f > test/Test.go 17 | cd test 18 | go build 19 | echo "Generated code: convert to JSON" 20 | /usr/bin/time -f "%E %M" ./test -j > /dev/null 21 | echo "Generated code: convert to JSON, streaming" 22 | /usr/bin/time -f "%E %M" ./test -j -s > /dev/null 23 | echo "Generated code: convert to XML" 24 | /usr/bin/time -f "%E %M" ./test -x > /dev/null 25 | echo "Generated code: convert to XML, streamingb" 26 | /usr/bin/time -f "%E %M" ./test -x -s > /dev/null 27 | cd .. 28 | echo "Java code generation" 29 | /usr/bin/time -f "%E %M" ./chidley -J $f 30 | cd java 31 | mvn package 32 | export CLASSPATH=target/jaxb-1.0-SNAPSHOT.jar:$CLASSPATH:/home/gnewton/.m2/repository 33 | echo "Running Java/JAXB XML -> JSON" 34 | /usr/bin/time -f "%E %M" java ca.gnewton.chidley.jaxb.Main > /dev/null 35 | cd .. 36 | done 37 | 38 | 39 | # From: http://www.ncbi.nlm.nih.gov/books/NBK25500/ and from openstreetmap.org 40 | declare -a URLS=('http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=science[journal]+AND+breast+cancer+AND+2008[pdat]' 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=science[journal]+AND+breast+cancer+AND+2008[pdat]&usehistory=y' 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=protein&id=6678417,9507199,28558982,28558984,28558988,28558990' 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=protein&db=protein&id=15718680&term=rat[orgn]+AND+srcdb+refseq[prop]&cmd=neighbor_history' 'http://eutils.ncbi.nlm.nih.gov/gquery?term=mouse[orgn]&retmode=xml' 'http://api06.dev.openstreetmap.org/api/capabilities' 'http://api.openstreetmap.org/api/0.6/trackpoints?bbox=0,51.5,0.25,51.75&page=0') 41 | 42 | # for u in "${URLS[@]}" 43 | # do 44 | # echo "#==================================================================" 45 | # echo "# $u " 46 | # ./chidley -V -u -s "" -p "T_" -a "Att_" "$u" 47 | # cd chidleyVerity 48 | # go build 49 | # ./chidleyVerity 50 | # cd .. 51 | # echo "#" 52 | # done 53 | 54 | 55 | -------------------------------------------------------------------------------- /tests_util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // "fmt" 5 | "bytes" 6 | "go/ast" 7 | "go/importer" 8 | "go/parser" 9 | "go/token" 10 | "go/types" 11 | "log" 12 | "strings" 13 | ) 14 | 15 | func extractor(xmlStrings []string) error { 16 | log.SetFlags(log.LstdFlags | log.Lshortfile) 17 | ex := Extractor{ 18 | namePrefix: namePrefix, 19 | nameSuffix: nameSuffix, 20 | useType: useType, 21 | progress: progress, 22 | ignoreXmlDecodingErrors: ignoreXmlDecodingErrors, 23 | } 24 | 25 | ex.init() 26 | 27 | for i, _ := range xmlStrings { 28 | //log.Println(xmlStrings[i]) 29 | err := ex.extract(strings.NewReader(xmlStrings[i])) 30 | 31 | if err != nil { 32 | return err 33 | } 34 | } 35 | 36 | ex.done() 37 | 38 | buf := bytes.NewBufferString("") 39 | fps := make([]string, 1) 40 | //fps[0] = "foo" 41 | generateGoCode(buf, fps, &ex) 42 | 43 | //log.Println(buf.String()) 44 | 45 | return parseAndType(buf.String()) 46 | } 47 | 48 | // Derived from example at https://github.com/golang/example/tree/master/gotypes#an-example 49 | func parseAndType(s string) error { 50 | fset := token.NewFileSet() 51 | //f, err := parser.ParseFile(fset, "hello.go", hello, 0) 52 | f, err := parser.ParseFile(fset, "hello.go", s, 0) 53 | if err != nil { 54 | //t.Error("Expected 1.5, got ", v) 55 | //log.Fatal(err) // parse error 56 | return err 57 | } 58 | 59 | conf := types.Config{Importer: importer.Default()} 60 | 61 | // Type-check the package containing only file f. 62 | // Check returns a *types.Package. 63 | _, err = conf.Check("cmd/hello", fset, []*ast.File{f}, nil) 64 | if err != nil { 65 | //log.Fatal(err) // type error 66 | return err 67 | } 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "compress/bzip2" 6 | "compress/gzip" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "log" 11 | "os" 12 | "sort" 13 | //"strconv" 14 | "strings" 15 | "unicode" 16 | 17 | "github.com/xi2/xz" 18 | ) 19 | 20 | func genericReader(filename string) (io.Reader, *os.File, error) { 21 | file, err := os.Open(filename) 22 | if err != nil { 23 | return nil, nil, err 24 | } 25 | if strings.HasSuffix(filename, "xz") { 26 | reader, err := xz.NewReader(bufio.NewReader(file), 0) 27 | if err != nil { 28 | return nil, nil, err 29 | } 30 | return bufio.NewReader(reader), file, err 31 | } 32 | 33 | if strings.HasSuffix(filename, "bz2") { 34 | return bufio.NewReader(bzip2.NewReader(bufio.NewReader(file))), file, err 35 | } 36 | 37 | if strings.HasSuffix(filename, "gz") { 38 | reader, err := gzip.NewReader(bufio.NewReader(file)) 39 | if err != nil { 40 | return nil, nil, err 41 | } 42 | return bufio.NewReader(reader), file, err 43 | } 44 | return bufio.NewReader(file), file, err 45 | } 46 | 47 | func cleanName(name string) string { 48 | for old, new := range nameMapper { 49 | parts := strings.Split(name, old) 50 | if len(parts) == 1 { 51 | continue 52 | } else { 53 | name := "" 54 | l := len(parts) 55 | for i := 0; i < l; i++ { 56 | if i == 0 { 57 | name += parts[i] 58 | } else { 59 | name += capitalizeFirstLetter(parts[i]) 60 | } 61 | if i+1 < l { 62 | name += new 63 | } 64 | } 65 | } 66 | } 67 | //return capitalizeFirstLetter(name) 68 | return name 69 | } 70 | 71 | const BoolType = "bool" 72 | const StringType = "string" 73 | const IntType = "int" 74 | const Int8Type = "int8" 75 | const Int16Type = "int16" 76 | const Int32Type = "int32" 77 | const Int64Type = "int64" 78 | const Float32Type = "float32" 79 | const Float64Type = "float64" 80 | 81 | func findType(nti *NodeTypeInfo, useType bool) string { 82 | if !useType { 83 | return StringType 84 | } 85 | 86 | if nti.alwaysBool { 87 | return BoolType 88 | } 89 | 90 | if nti.alwaysInt08 { 91 | return Int8Type 92 | } 93 | if nti.alwaysInt16 { 94 | return Int16Type 95 | } 96 | if nti.alwaysInt32 { 97 | return Int32Type 98 | } 99 | if nti.alwaysInt64 { 100 | return Int64Type 101 | } 102 | 103 | if nti.alwaysInt0 { 104 | return IntType 105 | } 106 | 107 | if nti.alwaysFloat32 { 108 | return Float32Type 109 | } 110 | if nti.alwaysFloat64 { 111 | return Float64Type 112 | } 113 | return StringType 114 | } 115 | 116 | type fqnSorter []*FQN 117 | 118 | func isStringOnlyField(n *Node, nattributes int) bool { 119 | return (len(n.children) == 0 && nattributes == 0) 120 | } 121 | 122 | func makeAttributes(writer io.Writer, attributes []*FQN, nameSpaceTagMap map[string]string) { 123 | sort.Sort(fqnSorter(attributes)) 124 | 125 | for _, fqn := range attributes { 126 | 127 | name := fqn.name 128 | nameSpace := fqn.space 129 | 130 | nameSpaceTag, ok := nameSpaceTagMap[nameSpace] 131 | if ok && nameSpaceTag != "" { 132 | nameSpaceTag = nameSpaceTag + "Space" 133 | } else { 134 | nameSpaceTag = nameSpace 135 | } 136 | 137 | nameSpaceTag = goVariableNameSanitize(nameSpaceTag) 138 | if len(nameSpace) > 0 { 139 | nameSpace = nameSpace + " " 140 | } 141 | 142 | //variableName := attributePrefix + capitalizeFirstLetter(nameSpaceTag) + cleanName(name) 143 | variableName := goVariableNameSanitize(attributePrefix + capitalizeFirstLetter(nameSpaceTag) + cleanName(name)) 144 | variableType := "string" 145 | 146 | //lineChannel <- "\t" + variableName + " " + variableType + "`xml:\"" + nameSpace + name + ",attr\" json:\",omitempty\"`" + " // maxLength=" + strconv.Itoa(fqn.maxLength) 147 | //fmt.Fprintln(writer, "\t"+variableName+" "+variableType+"`xml:\""+nameSpace+name+",attr\" json:\",omitempty\"`"+" // maxLength="+strconv.Itoa(fqn.maxLength)) 148 | fmt.Fprintln(writer, "\t"+variableName+" "+variableType+"`xml:\""+nameSpace+name+",attr\" json:\",omitempty\"`") 149 | } 150 | } 151 | 152 | func goVariableNameSanitize(s string) string { 153 | s = strings.Replace(s, ":", "_colon_", -1) 154 | s = strings.Replace(s, "/", "_slash_", -1) 155 | s = strings.Replace(s, ".", "_dot_", -1) 156 | s = strings.Replace(s, "-", "_dash_", -1) 157 | s = strings.Replace(s, " ", "_space_", -1) 158 | s = strings.Replace(s, "-", "_dash_", -1) 159 | 160 | return s 161 | } 162 | 163 | // Len is part of sort.Interface. 164 | func (s fqnSorter) Len() int { 165 | return len(s) 166 | } 167 | 168 | // Swap is part of sort.Interface. 169 | func (s fqnSorter) Swap(i, j int) { 170 | s[i], s[j] = s[j], s[i] 171 | } 172 | 173 | // Less is part of sort.Interface. It is implemented by calling the "by" closure in the sorter. 174 | func (s fqnSorter) Less(i, j int) bool { 175 | return strings.Compare(cleanName(s[i].name), cleanName(s[j].name)) < 0 176 | } 177 | 178 | // node key 179 | func nk(n *Node) string { 180 | return nks(n.space, n.name) 181 | } 182 | 183 | func nks(space, name string) string { 184 | return space + "NS" + name 185 | } 186 | 187 | func getFullPaths(filenames []string) ([]string, error) { 188 | fps := make([]string, len(filenames)) 189 | var err error 190 | for i, _ := range filenames { 191 | fps[i], err = getFullPath(filenames[i]) 192 | if err != nil { 193 | return nil, err 194 | } 195 | } 196 | 197 | return fps, nil 198 | } 199 | 200 | func getFullPath(filename string) (string, error) { 201 | if filename == "" { 202 | return "", nil 203 | } 204 | file, err := os.Open(filename) // For read access. 205 | if err != nil { 206 | log.Print("Error opening: " + filename) 207 | return "", err 208 | } 209 | return file.Name(), nil 210 | } 211 | 212 | type alterer func(string) string 213 | 214 | func alterFirstLetter(s string, f alterer) string { 215 | switch len(s) { 216 | case 0: 217 | return s 218 | case 1: 219 | v := f(s[0:1]) 220 | return v 221 | 222 | default: 223 | return f(s[0:1]) + s[1:] 224 | } 225 | } 226 | 227 | func capitalizeFirstLetter(s string) string { 228 | return alterFirstLetter(s, strings.ToUpper) 229 | } 230 | 231 | func lowerFirstLetter(s string) string { 232 | return alterFirstLetter(s, strings.ToLower) 233 | } 234 | 235 | func findThisAttribute(local, nameSpace string, attrs []*FQN) *FQN { 236 | for _, attr := range attrs { 237 | if attr.name == local && attr.space == nameSpace { 238 | return attr 239 | } 240 | } 241 | return nil 242 | } 243 | 244 | func makeAttributeName(key, namespace, local string) string { 245 | return key + "_" + local + "_" + namespace 246 | 247 | } 248 | 249 | func containsUnicodeSpace(s string) bool { 250 | if s == "" { 251 | return false 252 | } 253 | 254 | for _, rune := range s { 255 | //log.Printf("*** %#U ****", rune) 256 | if unicode.IsSpace(rune) { 257 | return true 258 | } 259 | } 260 | return false 261 | } 262 | 263 | func isIgnoredTag(tag string) bool { 264 | var ignored bool 265 | if ignoredXmlTagsMap == nil { 266 | return false 267 | } 268 | _, ignored = (*ignoredXmlTagsMap)[tag] 269 | 270 | if ignored || (ignoreLowerCaseXmlTags && tag == strings.ToLower(tag)) { 271 | return true 272 | } 273 | 274 | return false 275 | 276 | } 277 | 278 | func extractExcludedTags(tagsString string) (*map[string]struct{}, error) { 279 | ignoredMap := make(map[string]struct{}) 280 | if tagsString == "" { 281 | return &ignoredMap, nil 282 | } 283 | 284 | tags := strings.Split(tagsString, ",") 285 | 286 | for i, _ := range tags { 287 | tag := strings.TrimSpace(tags[i]) 288 | if containsUnicodeSpace(tag) { 289 | return nil, errors.New("Excluded tag contains space: [" + tag + "] in list of excluded tags:" + tagsString + "]") 290 | } 291 | ignoredMap[tag] = struct{}{} 292 | 293 | } 294 | return &ignoredMap, nil 295 | } 296 | 297 | func findFieldNameFromTypeInfo(t string) string { 298 | switch t { 299 | case IntType, Int8Type, Int16Type, Int32Type, Int64Type, Float32Type, Float64Type: 300 | return cdataNumberName 301 | case BoolType: 302 | return cdataBooleanName 303 | } 304 | return "string" 305 | 306 | } 307 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | //"log" 5 | "testing" 6 | // "unicode" 7 | ) 8 | 9 | func TestExtractExcludedTags_EmptyString(t *testing.T) { 10 | s := "" 11 | b := containsUnicodeSpace(s) 12 | if b != false { 13 | t.Error() 14 | } 15 | } 16 | 17 | func TestExtractExcludedTags_CorrectString(t *testing.T) { 18 | s := "a,b,c,d,e" 19 | b := containsUnicodeSpace(s) 20 | if b != false { 21 | t.Error() 22 | } 23 | } 24 | 25 | func TestExtractExcludedTags_StringWithSpaces(t *testing.T) { 26 | s := "a,b,c,d, e" 27 | b := containsUnicodeSpace(s) 28 | if b != true { 29 | t.Error() 30 | } 31 | } 32 | 33 | // Latin-1 spaces taken from https://golang.org/pkg/unicode/#IsSpace 34 | func TestExtractExcludedTags_StringWithAllLatin1Spaces(t *testing.T) { 35 | s := "\t\n\v\f\r U+0085 U+00A0" 36 | b := containsUnicodeSpace(s) 37 | if b != true { 38 | t.Error() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /xml/MODIS-Imagery-OilSpill.kml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | MODIS Imagery Overlays 5 | Deepwater Horizon Oil Spill 6 | 1 7 | Google Crisis Response 8 | 9 | This folder contains MODIS imagery of the oil spill in the gulf of Mexico in April 2010, caused by the sinking of the Deepwater Horizon oil rig.

Select dates in the Layers Panel on the left to see imagery from all the available dates.

Data courtesy of NASA
KML layer courtesy of Google Crisis Resposne

]]> 10 | 11 | 12 | 17 | 22 | #style_MainDoc 23 | 24 | -87.95 25 | 28.80 26 | 0 27 | 743000 28 | 0.00 29 | 0.00 30 | 31 | 32 | 2010-06-19 - MODIS 33 | #style_ModisOverlay 34 | 1 35 | 36 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-19/index.kml 37 | 38 | 39 | -88.25 40 | 29.94 41 | 2000000 42 | 43 | 44 | 45 | 2010-06-18 - MODIS 46 | #style_ModisOverlay 47 | 0 48 | 49 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-18/index.kml 50 | 51 | 52 | -88.25 53 | 29.94 54 | 2000000 55 | 56 | 57 | 58 | 2010-06-17 - MODIS 59 | #style_ModisOverlay 60 | 0 61 | 62 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-17/index.kml 63 | 64 | 65 | -88.25 66 | 29.94 67 | 2000000 68 | 69 | 70 | 71 | 2010-06-14 - MODIS 72 | #style_ModisOverlay 73 | 0 74 | 75 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-14/index.kml 76 | 77 | 78 | -88.25 79 | 29.94 80 | 2000000 81 | 82 | 83 | 84 | 2010-06-12 - MODIS 85 | #style_ModisOverlay 86 | 0 87 | 88 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-12/index.kml 89 | 90 | 91 | -88.25 92 | 29.94 93 | 2000000 94 | 95 | 96 | 97 | 2010-06-11 - MODIS 98 | #style_ModisOverlay 99 | 0 100 | 101 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-11/index.kml 102 | 103 | 104 | -88.25 105 | 29.94 106 | 2000000 107 | 108 | 109 | 110 | 2010-06-10 - MODIS 111 | #style_ModisOverlay 112 | 0 113 | 114 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-10/index.kml 115 | 116 | 117 | -88.25 118 | 29.94 119 | 2000000 120 | 121 | 122 | 123 | 2010-06-09 - MODIS 124 | #style_ModisOverlay 125 | 0 126 | 127 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-09/index.kml 128 | 129 | 130 | -88.25 131 | 29.94 132 | 2000000 133 | 134 | 135 | 136 | 2010-06-08 - MODIS 137 | #style_ModisOverlay 138 | 0 139 | 140 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-08/index.kml 141 | 142 | 143 | -88.25 144 | 29.94 145 | 2000000 146 | 147 | 148 | 149 | 2010-06-07 - MODIS 150 | #style_ModisOverlay 151 | 0 152 | 153 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_2010-06-07/index.kml 154 | 155 | 156 | -88.25 157 | 29.94 158 | 2000000 159 | 160 | 161 | 162 | 2010-05-27 - MODIS 163 | #style_ModisOverlay 164 | 0 165 | 166 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/modis_05_27_2010_aqua/doc.kml 167 | 168 | 169 | -88.25 170 | 29.94 171 | 2000000 172 | 173 | 174 | 175 | 2010-05-26 - MODIS 176 | #style_ModisOverlay 177 | 0 178 | 179 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_05-26-2010/doc.kml 180 | 181 | 182 | -88.25 183 | 29.94 184 | 2000000 185 | 186 | 187 | 188 | 2010-05-25 - MODIS 189 | #style_ModisOverlay 190 | 0 191 | 192 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/MODIS_0525_v13/doc.kml 193 | 194 | 195 | -88.25 196 | 29.94 197 | 2000000 198 | 199 | 200 | 201 | 2010-05-23 - MODIS 202 | #style_ModisOverlay 203 | 0 204 | 205 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/Modis_2010-05-23/index.kml 206 | 207 | 208 | -88.25 209 | 29.94 210 | 2000000 211 | 212 | 213 | 214 | 2010-05-22 - MODIS 215 | #style_ModisOverlay 216 | 0 217 | 218 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/Modis_2010-05-22/index.kml 219 | 220 | 221 | -88.25 222 | 29.94 223 | 2000000 224 | 225 | 226 | 227 | 2010-05-20 - MODIS 228 | #style_ModisOverlay 229 | 0 230 | 231 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-05-20/index.kml 232 | 233 | 234 | -88.25 235 | 29.94 236 | 2000000 237 | 238 | 239 | 240 | 2010-05-17 - MODIS 241 | #style_ModisOverlay 242 | 0 243 | 244 | http://mw1.google.com/mw-earth-vectordb/disaster/gulf_oil_spill/Modis_Oil_Spill-2010_05_17/index.kml 245 | 246 | 247 | -88.25 248 | 29.94 249 | 2000000 250 | 251 | 252 | 253 | 2010-05-11 - MODIS 254 | #style_ModisOverlay 255 | 0 256 | 257 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_250m_20100511/index.kml 258 | 259 | 260 | -88.25 261 | 29.94 262 | 2000000 263 | 264 | 265 | 266 | 2010-05-10 - MODIS 267 | #style_ModisOverlay 268 | 0 269 | 270 | http://mw1.google.com/mw-earth-vectordb/disaster/MODIS_2010-05-10_v2/MODIS_2010-05-10_v2.kml 271 | 272 | 273 | -88.25 274 | 29.94 275 | 2000000 276 | 277 | 278 | 279 | 2010-05-09 - MODIS 280 | #style_ModisOverlay 281 | 0 282 | 283 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/Modis_2010-05-09/index.kml 284 | 285 | 286 | -88.25 287 | 29.94 288 | 2000000 289 | 290 | 291 | 292 | 2010-05-08 - MODIS 293 | #style_ModisOverlay 294 | 0 295 | 296 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/Modis_2010-04-08/index.kml 297 | 298 | 299 | -88.25 300 | 29.94 301 | 2000000 302 | 303 | 304 | 305 | 2010-05-07 - MODIS 306 | #style_ModisOverlay 307 | 0 308 | 309 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Oil_Spill_2010/Modis_Oil_Spill-2010_05_07/doc.kml 310 | 311 | 312 | -88.25 313 | 29.94 314 | 2000000 315 | 316 | 317 | 318 | 2010-05-05 - MODIS 319 | #style_ModisOverlay 320 | 0 321 | 322 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_250m_20100505/index.kml 323 | 324 | 325 | -88.25 326 | 29.94 327 | 2000000 328 | 329 | 330 | 331 | 2010-05-04 - MODIS 332 | #style_ModisOverlay 333 | 0 334 | 335 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-05-04/index.kml 336 | 337 | 338 | -88.25 339 | 29.94 340 | 2000000 341 | 342 | 343 | 344 | 2010-04-29 - MODIS 345 | #style_ModisOverlay 346 | 0 347 | 348 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-04-29/index.kml 349 | 350 | 351 | -88.25 352 | 29.94 353 | 2000000 354 | 355 | 356 | 357 | 2010-04-28 - MODIS 358 | #style_ModisOverlay 359 | 0 360 | 361 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-04-28/index.kml 362 | 363 | 364 | -88.25 365 | 29.94 366 | 2000000 367 | 368 | 369 | 370 | 2010-04-27 - MODIS 371 | #style_ModisOverlay 372 | 0 373 | 374 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-04-27/index.kml 375 | 376 | 377 | -88.25 378 | 29.94 379 | 2000000 380 | 381 | 382 | 383 | 2010-04-26 - MODIS 384 | #style_ModisOverlay 385 | 0 386 | 387 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-04-26/index.kml 388 | 389 | 390 | -88.25 391 | 29.94 392 | 2000000 393 | 394 | 395 | 396 | 2010-04-25 - MODIS 397 | #style_ModisOverlay 398 | 0 399 | 400 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-04-25/index.kml 401 | 402 | 403 | -88.25 404 | 29.94 405 | 2000000 406 | 407 | 408 | 409 | 2010-04-22 - MODIS 410 | #style_ModisOverlay 411 | 0 412 | 413 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-04-22/index.kml 414 | 415 | 416 | -88.25 417 | 29.94 418 | 2000000 419 | 420 | 421 | 422 | 2010-04-21 - MODIS 423 | #style_ModisOverlay 424 | 0 425 | 426 | http://mw1.google.com/mw-earth-vectordb/disaster/Gulf_Spill_2010/MODIS_2010-04-21/index.kml 427 | 428 | 429 | -88.25 430 | 29.94 431 | 2000000 432 | 433 | 434 | 435 | 436 | 437 | -------------------------------------------------------------------------------- /xml/MODIS-Imagery-OilSpill.kml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gnewton/chidley/09b9269092fb8f0bf2548ba78529cfe9c9bd072e/xml/MODIS-Imagery-OilSpill.kml.gz -------------------------------------------------------------------------------- /xml/amazon_api.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | ID for the distribution 8 | Deployed | InProgress 9 | creation date and time in 10 | ISO 8601 format 11 | number of invalidation batches being 12 | processed for this distribution 13 | CloudFront domain name assigned to the 14 | distribution 15 | 16 | true | false 17 | number of unique trusted signers from 18 | all cache behaviors 19 | 20 | 21 | self | AWS account number 22 | 23 | number of active key pairs for 24 | AwsAccountNumber 25 | 26 | active key pair associated with 27 | AwsAccountNumber 28 | 29 | 30 | 31 | 32 | 33 | 34 | unique description for this 35 | distribution config 36 | 37 | number of CNAME aliases 38 | 39 | 40 | CNAME alias 41 | 42 | 43 | URL for default root object 44 | 45 | number of origins 46 | 47 | 48 | unique identifier for this origin 49 | domain name of origin 50 | 52 | 53 | origin-access-identity/ 54 | cloudfront/Id 55 | 56 | 58 | 59 | HTTP port that the custom origin 60 | listens on 61 | HTTPS port that the custom origin 62 | listens on 63 | http-only | 64 | match-viewer 65 | 66 | 67 | 68 | 69 | 70 | ID of the origin that the default cache behavior 71 | applies to 72 | 73 | true | false 74 | 75 | all | whitelist | none 76 | 77 | 78 | number of cookie names to 79 | forward to origin 80 | 81 | name of a cookie to forward to the origin 82 | 83 | 84 | 85 | 86 | number of headers to forward to origin 87 | 88 | header 89 | 90 | 91 | 92 | 93 | true | false 94 | number of trusted signers 95 | 96 | 97 | self | AWS account that can create 98 | signed URLs 99 | 100 | 101 | allow-all | 102 | redirect-to-https | https-only 103 | minimum TTL for objects in seconds 104 | 105 | 2 | 7 106 | 107 | 109 | GET 110 | HEAD 111 | 113 | DELETE 114 | GET 115 | HEAD 116 | OPTIONS 117 | PATCH 118 | POST 119 | PUT 120 | 121 | 122 | true | false 123 | 124 | 125 | number of cache behaviors 126 | 127 | 128 | 129 | pattern that specifies files that this 130 | cache behavior applies to 131 | ID of the origin that this cache behavior 132 | applies to 133 | 134 | true | false 135 | 136 | all | whitelist | none 137 | 139 | 140 | number of cookie names to 141 | forward to origin 142 | 143 | name of a cookie to forward to 144 | the origin 145 | 146 | 147 | 148 | 149 | number of headers to forward to origin 150 | 151 | header 152 | 153 | 154 | 155 | 156 | true | false 157 | number of trusted signers 158 | 159 | 160 | self | AWS account that can create 161 | signed URLs 162 | 163 | 164 | allow-all | 165 | redirect-to-https | https-only 166 | minimum TTL in seconds for files 167 | specified by PathPattern 168 | 169 | 2 | 7 170 | 171 | 174 | GET 175 | HEAD 176 | 178 | DELETE 179 | GET 180 | HEAD 181 | OPTIONS 182 | PATCH 183 | POST 184 | PUT 185 | 186 | 187 | true | false 188 | 189 | 190 | 191 | 192 | number of custom error responses 193 | 194 | 195 | HTTP status code for which you want to 196 | customize the response 197 | path to custom error page 198 | HTTP status code that you want CloudFront 199 | to return along with the custom error page 200 | minimum TTL for this 201 | ErrorCode 202 | 203 | 204 | 205 | 206 | 207 | blacklist | whitelist | none 208 | number of countries 209 | in the blacklist or whitelist 210 | 211 | 212 | two-letter country code in upper case 213 | 214 | 215 | 216 | comment about the distribution 217 | 218 | true | false 219 | true | false 220 | Amazon S3 bucket to save logs in 221 | prefix for log filenames 222 | 223 | 224 | IAM certificate ID 225 | true 226 | vip | sni-only 227 | 228 | maximum price class for the distribution 229 | true | false 230 | 231 | 232 | -------------------------------------------------------------------------------- /xml/arxiv_search_query.xml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gnewton/chidley/09b9269092fb8f0bf2548ba78529cfe9c9bd072e/xml/arxiv_search_query.xml.gz -------------------------------------------------------------------------------- /xml/pubmed_xml_12750255.xml.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gnewton/chidley/09b9269092fb8f0bf2548ba78529cfe9c9bd072e/xml/pubmed_xml_12750255.xml.bz2 -------------------------------------------------------------------------------- /xml/test1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Dune 5 | 6 | Herbert 7 | Frank 8 | 9 | 10 | 11 | Brave New Wold 12 | 13 | Huxley 14 | Aldous 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /xml/test2.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | number one 7 | number two 8 | 9 | -------------------------------------------------------------------------------- /xml/test2a.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | number one 7 | number two 8 | 9 | -------------------------------------------------------------------------------- /xml/test2b.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | number one 6 | number two 7 | 8 | -------------------------------------------------------------------------------- /xml/test2c.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | number one 4 | number two 5 | 6 | -------------------------------------------------------------------------------- /xml/test3a.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | number one 4 | 5 | -------------------------------------------------------------------------------- /xml/test3b.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | number two 7 | 8 | 9 | -------------------------------------------------------------------------------- /xml/test3c.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | number one 7 | 8 | -------------------------------------------------------------------------------- /xml/test4.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | A book entry 7 | 8 | 9 | A article entry 10 | 11 | 12 | -------------------------------------------------------------------------------- /xml/testSpacePreserve.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Alix 5 | Krakowski 6 | 7 | 8 | Roses are red, 9 | Violets are blue. 10 | -Alix 11 | 12 | 13 | -------------------------------------------------------------------------------- /xml/testType.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | bill 6 | 37 7 | 9999999999999 8 | true 9 | yes 10 | 11 | 12 | 13 | sarah 14 | 24 15 | 555555555555555 16 | false 17 | no 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /xml_source_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const sameNameDifferentNameSpaceXML = ` 4 | 5 | 6 | 7 | 1 8 | Google Crisis Response 9 | ogle Crisis Responsex 10 | 11 | 12 | ` 13 | 14 | //https://github.com/gnewton/chidley/issues/24 15 | const mixedCaseSameNameXML_Issue24 = ` 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | ` 25 | 26 | const tagsContainHyphens = ` 27 | 28 | 29 | Bill 30 | Smith 31 | 32 | ` 33 | 34 | //https://github.com/gnewton/chidley/issues/14 35 | const githubIssue14 = ` 36 | 37 | 38 | _ActionId-3525062221263473230--64db5972.154ce25275c.-7fc5 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | ` 47 | -------------------------------------------------------------------------------- /xml_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | //"log" 5 | "testing" 6 | ) 7 | 8 | func TestTagsContainHyphens(t *testing.T) { 9 | err := extractor([]string{tagsContainHyphens}) 10 | if err != nil { 11 | t.Error(err) 12 | } 13 | } 14 | 15 | func TestTagsWithSameNameDifferentNameSpaceXML(t *testing.T) { 16 | err := extractor([]string{sameNameDifferentNameSpaceXML}) 17 | if err != nil { 18 | t.Error(err) 19 | } 20 | } 21 | 22 | func TestMixedCaseSameNameTagsXML_Issue24(t *testing.T) { 23 | err := extractor([]string{mixedCaseSameNameXML_Issue24}) 24 | if err != nil { 25 | t.Error(err) 26 | } 27 | } 28 | 29 | //https://github.com/gnewton/chidley/issues/14 30 | func TestGithubIssue14(t *testing.T) { 31 | err := extractor([]string{githubIssue14}) 32 | if err != nil { 33 | t.Error(err) 34 | } 35 | } 36 | --------------------------------------------------------------------------------