├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── cmd └── flattablesc │ ├── README.md │ └── flattablesc.go ├── flattables.go ├── flattables_templates.go └── flattables_test.go /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | schema.template.20171127.0807 49 | schema.template.20171127.0955 50 | schema.template.20171127.0958 51 | FlatBuffersFromTableSet.template.20171128.1846 52 | FlatBuffersFromTableSet.template.20171128.1900 53 | FlatBuffersFromTableSet.template.20171130.1722 54 | schema.template.20171127.1012 55 | ln.sh 56 | junk 57 | make.sh 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Malcolm Gorman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | The irony of `FlatTables` is that the automatic code generation produces all your package-specific library code 2 | and godoc documentation, along with tests and a main that illustrates how to use your specific `FlatTables` 3 | package. Which means there is absolutely no (zero, zilch) godoc for `FlatTables` itself. Your invocation 4 | of the `flattablesc` CLI will generate your own godoc. Here's the 5 | [github.com/urban-wombat/flattables_sample godoc](https://godoc.org/github.com/urban-wombat/flattables_sample) 6 | to give you an idea of what your generated godoc will look like. 7 | 8 | # Getting started with `Google FlatBuffers` and `FlatTables` 9 | 10 | ## Install and Test 11 | 12 | If you hit a wall or feel that something is missing or unclear, email to: `urban.wombat.burrow@gmail.com` 13 | 14 | Don't be daunted by the large number of functions and methods that are generated by `flattablesc`. 15 | 16 | Look at the following two links to help narrow down your own path through the code:- 17 | * [https://github.com/urban-wombat/flattables_sample/blob/master/cmd/flattables_sample/flattables_sample_main.go](https://github.com/urban-wombat/flattables_sample/blob/master/cmd/flattables_sample/flattables_sample_main.go) 18 | which is a runnable main example. 19 | * [https://godoc.org/github.com/urban-wombat/flattables_sample#pkg-examples](https://godoc.org/github.com/urban-wombat/flattables_sample#pkg-examples) 20 | 21 | `flattables_sample` is nothing special. I just called it that (the package name) and ran `flattablesc` on this 22 | [tables.got](https://github.com/urban-wombat/flattables_sample/blob/master/tables.got) file. 23 | I did not hand-code any of it. `flattablesc` will generate all the same kinds of examples and useable code for you from your own set of tables. You will then see exactly how to write your own code to use `FlatBuffers`. The main restrictions are:- 24 | * `flattables` is tabular only 25 | * there are some table and columns names that are illegal (such as `for`) 26 | to not break generated Go code 27 | * some `FlatBuffers` data-type and style-guide constraints such as:- 28 | * fixed sizes (no int or uint) 29 | * lowercase column names and uppercase table names 30 | 31 | 1. Install FlatBuffers 32 | 33 | ``` 34 | go get -u github.com/google/flatbuffers 35 | ``` 36 | 37 | For more information: 38 | * [How-To: Install FlatBuffers](https://rwinslow.com/posts/how-to-install-flatbuffers) by [Robert Winslow](https://rwinslow.com) 39 | * [FlatBuffers Go Documentation](https://google.github.io/flatbuffers/flatbuffers_guide_use_go.html) 40 | 41 | 2. Install `gotables` and `FlatTables` 42 | 43 | ``` 44 | go get -u github.com/urban-wombat/gotables 45 | 46 | go get -u github.com/urban-wombat/flattables 47 | ``` 48 | 49 | Relationships between the packages: 50 | * `flattables` uses `gotables` 51 | * `flattablesc` uses `flattables` and `gotables` 52 | 53 | 3. Create your directory `my_package` 54 | 55 | ``` 56 | $ mkdir my_package 57 | ``` 58 | 59 | `my_package` (or whatever you decide to call it) will be your namespace and tail-end of your package name. 60 | 61 | 4. Create your `FlatTables` /data-file - this is a set of `gotables` tables, not a flatbuffers `.fbs` file. 62 | 63 | It doesn't matter where you create it or what you call it. But for simplicity, let's call it `tables.got` 64 | and create it in your newly-created directory `my_package`. 65 | 66 | The table names, column names and types are used to generate the `FlatBuffers` schema file `*.fbs` 67 | (the `*.fbs` file is auto-generated by flattablesc, not your problem). 68 | 69 | The data in the tables is used in the auto-generated bench tests. So add some dummy data for testing. 70 | 71 | You can come up with your own tables of data, or you may copy and paste the tables below into `tables.got` 72 | 73 | The `gotables` syntax is self-evident and most `Go` types are supported. 74 | 75 | Types **not** supported are:- 76 | * `int` and `uint` (their sizes are machine-dependent, and `FlatBuffers` allows fixed-sizes only) 77 | * `complex32` `complex64` (not supported by gotables) 78 | * `rune` (doesn't seem to be supported by `FlatBuffers`, perhaps because its size varies) 79 | 80 | If you just want to get started and not deal with creating your own `gotables` schema right now, just copy 81 | and paste the tables below into `tables.got` and proceed to the next step. You can go back later 82 | and whip up data that looks like your own. 83 | 84 | [package flattables_sample](https://godoc.org/github.com/urban-wombat/flattables_sample) is generated from 85 | [tables.got](https://github.com/urban-wombat/flattables_sample/blob/master/tables.got) which is end of week 86 | data from the Australian Stock Exchange (ASX) to give a sense of the kind of high volume data that can be 87 | written to Google `FlatBuffers` binary format. 88 | 89 | Below is a *different* example set of tables, different to what the current `flattables_sample` code is generated 90 | from, to give a sense of the flexibility of multiple tables. As many tables as you need to express your information 91 | payload. 92 | 93 | ``` 94 | [MyXyzTable] 95 | x y z 96 | int16 float32 float64 97 | 4 5 6 98 | 44 55 66 99 | 444 555 666 100 | 4444 5555 6666 101 | 16 32 64 102 | 103 | [StringsAndThings] 104 | flintstones nums spain female unsigned 105 | string int8 string bool uint32 106 | "Fred" 0 "The rain" false 0 107 | "Wilma" 1 "in Spain" true 11 108 | "Barney" 2 "stays mainly" false 22 109 | "Betty" 3 "in" true 33 110 | "Bam Bam" 4 "the" false 44 111 | "Pebbles" 5 "plain." true 55 112 | 113 | [Wombats] 114 | housingPolicy string = "burrow" 115 | topSpeedKmH int8 = 40 116 | species string = "Vombatus" 117 | class string = "Mammalia" 118 | wild bool = true 119 | ``` 120 | 121 | 122 | 5. Check its validity with `gotsyntax` 123 | 124 | ``` 125 | $ gotsyntax tables.got 126 | ``` 127 | 128 | The `FlatTables` utility `flattablesc` will also do a syntax check, but you might as well get earlier feedback with `gotsyntax`. 129 | 130 | `flattablesc` also invokes the Google `FlatTables` `flatc` code generator. It doesn't seem to police the 131 | [FlatBuffers style guide](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) 132 | but `flattablesc` does. `flattablesc` also guards against some gotchas specific to generating `Go` code. 133 | 134 | `FlatTables` is also a little more strict than `gotables` syntax: 135 | * Table names must be `UpperCamelCase` as per the [FlatBuffers style guide](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) 136 | * Column names must be `lowerCamelCase` as per the [FlatBuffers style guide](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) 137 | * Table names or column names that so much as **look** like `Go` key words are not permitted. Table and column names end up as 138 | function and variable names in generated `Go` code, and the `Go` compiler doesn't like key words as names. So we don't risk it. 139 | * Transfers between `Go` slices and `FlatBuffers` require the field names to be exported (hence `UpperCamelCase`) which is 140 | done by code generation. So there's a (managed) difference between the 141 | [FlatBuffers style guide](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) 142 | and the need to export `Go` fields to user code. Languages such as Java convert field names to `lowerCamelCase`, which is what `FlatTables` 143 | requires here, consistent with `Go` unexported fields. But they are exported as `UpperCamelCase` where needed in raw `Go` structs 144 | used by `FlatTables`, namely: 145 | ``` 146 | type RootTableSlice struct {...} 147 | ``` 148 | 149 | See a sample `RootTableSlice` definition in 150 | [flattables_sample_NewSliceFromFlatBuffers.go](https://github.com/urban-wombat/flattables_sample/blob/master/flattables_sample_NewSliceFromFlatBuffers.go) 151 | (near line 70) 152 | 153 | `type RootTableSlice` is generated for you based on your `tables.got` schema file and the `*fbs` schema file. 154 | 155 | 6. From within dir `my_package` run the `FlatTables` utility `flattablesc` 156 | 157 | ``` 158 | $ flattablesc -f ../my_package/tables.got -n my_package -p github.com/your-github-name/my_package 159 | ``` 160 | 161 | `flattablesc` creates a flatbuffers schema `*.fbs` file and a number of `Go` source files in `../my_package`. 162 | 163 | 7. Run the tests 164 | 165 | ``` 166 | $ go test -bench=. 167 | ``` 168 | 169 | 170 | ## `FlatTables` is a simplified tabular subset of `FlatBuffers` 171 | 172 | Have a look at the Google `FlatBuffers` official documentation to see 173 | why you should seriously consider `FlatBuffers` (and `FlatTables`) 174 | for **very fast** binary data transfer: 175 | * [Google FlatBuffers official documentation](https://google.github.io/flatbuffers) 176 | 177 | If your data is tabular (or can be easily normalised to tabular) then `FlatTables` 178 | may be right for your project. 179 | 180 | The `FlatTables` utility `flattablesc` will generate all the code needed to convert 181 | from [gotables](https://github.com/urban-wombat/gotables#gotables) tabular format to 182 | FlatBuffers and back again. 183 | 184 | `flattablesc` also generates a `Go` main program with constructor and getter methods 185 | specific to your `FlatBuffers` schema. 186 | 187 | The generated code includes: 188 | * conversion functions (which include all the code generated by the `FlatBuffers` utility `flatc`) 189 | * test code 190 | * test Example code (an Example for each of the key functions) 191 | * benchmark tests which will run with the data you put into your `tables.got` schema file. 192 | * a main program with sample code 193 | 194 | * There is a [sample implementation](https://godoc.org/github.com/urban-wombat/flattables_sample) 195 | using a `gotables` file 196 | [tables.got](https://github.com/urban-wombat/flattables_sample/blob/master/tables.got) as input to the `flattablesc` utility. 197 | The same way you would create your own implementation. 198 | It is called `flattables_sample`. It is an actual implementation, and not just a toy sample. 199 | 200 | When you run `flattablesc` on your own `gotables` schema file, it will generate 201 | your raw `Go` struct tables, functions, examples and benchtests. 202 | 203 | Have a look at [urban-wombat/flattables_sample](https://github.com/urban-wombat/flattables_sample) 204 | which is a sample of `FlatBuffers` code generated entirely by `flatc` (FlatBuffers utility) 205 | and `flattablesc` (gotables `FlatTables` utility). 206 | 207 | Here is the GoDoc of all 90 or so `Go` functions generated by the Google `flatc` utility and `gotables` `flattablesc` utility: 208 | * [https://godoc.org/github.com/urban-wombat/flattables_sample](https://godoc.org/github.com/urban-wombat/flattables_sample) 209 | * [https://github.com/urban-wombat/flattables_sample/blob/master/cmd/flattables_sample/flattables_sample_main.go](https://github.com/urban-wombat/flattables_sample/blob/master/cmd/flattables_sample/flattables_sample_main.go) 210 | 211 | And test and benchmark code: 212 | * [https://github.com/urban-wombat/flattables_sample/blob/master/flattables_sample_test.go](https://github.com/urban-wombat/flattables_sample/blob/master/flattables_sample_test.go) 213 | 214 | The main function in [urban-wombat/flattables_sample_main](https://github.com/urban-wombat/flattables_sample_main) 215 | is the simplest possible conversion code, if you don't want to get into 216 | the weeds of moving data into and out of `FlatBuffers`. 217 | 218 | ALL of the code, including the `FlatBuffers` schema and all `Go` code, 219 | was generated automatically from `flatc` and `flattablesc`. 220 | 221 | When you download and run `flattablesc` (referencing a simple 222 | [gotables](https://github.com/urban-wombat/gotables) file you write yourself) 223 | you can run the tests and benchtest and see the speed of `FlatBuffers`. 224 | 225 | ## Advantages 226 | 227 | ### Auto-Generation of schema and `Go` code: 228 | 229 | * `FlatTables` auto-generates the `FlatBuffers` schema from your data. 230 | All you need to do is write a very simple self-describing `gotables` data file (sample below). 231 | This means normalising your objects to one or more tables (tabular tables, like database tables). 232 | 233 | `FlatBuffers` and `FlatTables` use 'table' in a slightly different sense, but if you see them as tabular 234 | tables, it makes sense. 235 | 236 | `gotables` is the supporting library and file format used by `FlatTables`. 237 | 238 | * `FlatBuffers` utility `flatc` generates `Go` code to write (and read) data conforming to the `FlatBuffers` schema. 239 | 240 | * `FlatTables` generates `Go` code to write `gotables`-formatted data to a `FlatBuffers` []byte array. 241 | 242 | * `FlatTables` generates `Go` code to test that data has been written to `FlatBuffers` correctly. 243 | 244 | * The read step is **very fast**. There is no additional code between you and the auto-generated `FlatBuffers` code. 245 | (Note: the read step at this stage is read-only. This may get better with the implementation of mutable tables) 246 | 247 | * You write only your own code to call these auto-generated methods, and denormalise the data from tables to 248 | your prefered data structures. 249 | 250 | * `FlatTables` uses a subset of Google `FlatBuffers` as a binary format for `gotables` Table objects. 251 | 252 | * `FlatTables` is general purpose because it consists of tables, and your own data is probably capable of being 253 | normalised (in Ted Codd, C J Date fashion) to one or more relational tables ready for transmission and re-assembly 254 | at the receiving end. 255 | 256 | * The Example functions in the `*.test.go` file will get you started coding data transfers. 257 | 258 | * You don't have to write the `*.fbs` flatbuffers schema `flattables_sample.fbs`. It is done for you. 259 | 260 | * You don't have to write the glue code to get data from `tables.got` to a flatbuffers []byte array. 261 | -------------------------------------------------------------------------------- /cmd/flattablesc/README.md: -------------------------------------------------------------------------------- 1 | # `flattablesc` - FlatBuffers code generator 2 | 3 | `flattablesc [-h] [-v] [-d] -f -n -p [-o ] [-s ]` 4 | 5 | Run `flattablesc -h` to see usage. 6 | 7 | Try especially `-d` dry-run (also turns on verbose) to find out ahead of time what and where generated code will be written. 8 | 9 | Generates `FlatBuffers` and `FlatTables` code (to call from your programs) to write and read `FlatBuffers []byte` arrays. 10 | 11 | See [urban-wombat/flattables](https://github.com/urban-wombat/flattables#getting-started-with-google-flatbuffers-via-flattables) 12 | for some of the how and why of `flattablesc` 13 | -------------------------------------------------------------------------------- /cmd/flattablesc/flattablesc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | // "path" 10 | "os" 11 | "os/exec" 12 | "os/user" 13 | "path/filepath" 14 | "strings" 15 | 16 | "github.com/urban-wombat/flattables" 17 | "github.com/urban-wombat/gotables" 18 | "github.com/urban-wombat/util" 19 | ) 20 | 21 | // import "github.com/davecgh/go-spew/spew" 22 | 23 | type Flags struct { 24 | f string // BOTH schema AND data file name 25 | n string // (also sets TableSet name) 26 | p string // 27 | o string // 28 | O string // 29 | s string // defaults to /cmd/.go 30 | m bool // mutable // Note: mutable (non-const) FlatBuffers apparently unavailable in Go 31 | v bool // verbose 32 | d bool // Dry Run 33 | h bool // help 34 | } 35 | 36 | var flags Flags 37 | 38 | var globalGotablesFileNameAbsolute string // from flags.f via filepath.Abs() 39 | // var globalRelationsFileName string // from flags.r 40 | var globalNameSpace string // from flags.n 41 | var globalPackageName string // from flags.p 42 | var globalOutDirAbsolute string // from (optional) flags.o or flags.O via filepath.Abs() 43 | var globalFlagOWarnOnly bool // if flags.O (capital O) is set 44 | var globalOutDirMainAbsolute string // from (optional) flags.s via filepath.Abs() 45 | var globalMutableFlag string // Pass to flatc. Note: mutable (non-const) FlatBuffers apparently unavailable in Go. 46 | var globalUtilName string = "flattablesc" 47 | var globalUtilDir string = "../flattables/cmd/flattablesc" 48 | 49 | func init() { 50 | log.SetFlags(log.Lshortfile) // For var where 51 | } 52 | 53 | var where = log.Print 54 | 55 | // Custom flag. 56 | // Note: For custom flags that satisfy the Value interface, the default value is just the initial value of the variable. 57 | // See: https://golang.org/pkg/flag 58 | type stringFlag struct { 59 | set bool 60 | val string 61 | } 62 | 63 | // Custom flag. 64 | func (sf *stringFlag) Set(x string) error { 65 | sf.val = x 66 | sf.set = true 67 | return nil 68 | } 69 | 70 | // Custom flag. 71 | func (sf *stringFlag) String() string { 72 | return sf.val 73 | } 74 | 75 | var tablesTemplateInfo flattables.TablesTemplateInfoType 76 | 77 | func initFlags() { 78 | /* 79 | 1. variable pointer 80 | 2. -flagname 81 | 3. default value (except for custom flags that satisfy the Value interface, which default to the initial value of the variable) 82 | 4. help message for flagname 83 | */ 84 | var err error 85 | 86 | flag.StringVar(&flags.f, "f", "", fmt.Sprintf(" of schema/data tables")) 87 | flag.StringVar(&flags.n, "n", "", fmt.Sprintf("Sets schema file .fbs, schema root_type, FlatBuffers namespace, TableSet name")) 88 | flag.StringVar(&flags.p, "p", "", fmt.Sprintf(" Sets package name")) 89 | flag.StringVar(&flags.o, "o", "", fmt.Sprintf(" Default is ../")) 90 | flag.StringVar(&flags.O, "O", "", fmt.Sprintf(" Default is ../")) 91 | flag.StringVar(&flags.s, "s", "", fmt.Sprintf(" Default is ..//cmd/")) 92 | flag.BoolVar(&flags.m, "m", false, fmt.Sprintf("generate additional non-const accessors for mutating FlatBuffers in-place")) 93 | flag.BoolVar(&flags.v, "v", false, fmt.Sprintf("verbose")) 94 | flag.BoolVar(&flags.d, "d", false, fmt.Sprintf("dry run")) 95 | flag.BoolVar(&flags.h, "h", false, fmt.Sprintf("print flattables usage")) 96 | 97 | flag.Parse() 98 | 99 | if flags.h { 100 | // help 101 | printUsage() 102 | os.Exit(1) 103 | } 104 | 105 | const ( 106 | compulsoryFlag = true 107 | optionalFlag = false 108 | ) 109 | 110 | var flagExists bool 111 | 112 | // Input file of gotables tables to be used as a schema, and possibly data. 113 | checkStringFlagReplaceWithUtilVersion("f", flags.f, compulsoryFlag) 114 | var globalGotablesFileName string = flags.f 115 | globalGotablesFileNameAbsolute, err = util.FilepathAbs(globalGotablesFileName) 116 | if err != nil { 117 | fmt.Fprintf(os.Stderr, "%s\n", err) 118 | printUsage() 119 | os.Exit(14) 120 | } 121 | // Change backslashes to forward slashes. Otherwise strings interpret them as escape chars. 122 | globalGotablesFileNameAbsolute = filepath.ToSlash(globalGotablesFileNameAbsolute) 123 | 124 | // Namespace 125 | // Compulsory flag. 126 | checkStringFlagReplaceWithUtilVersion("n", flags.n, compulsoryFlag) 127 | globalNameSpace = flags.n 128 | // globalNameSpace has the same validity criteria as gotables col names and table names. 129 | isValid, _ := gotables.IsValidColName(globalNameSpace) 130 | if !isValid { 131 | fmt.Fprintf(os.Stderr, "Error: non-alpha-numeric-underscore chars in -n : %q\n", flags.n) 132 | fmt.Fprintf(os.Stderr, "Note: is not a file or dir name. Though it is used in file and dir names.\n") 133 | printUsage() 134 | os.Exit(9) 135 | } 136 | 137 | if flags.m { 138 | globalMutableFlag = "--gen-mutable" // Generate additional non-const accessors to mutate FlatBuffers in-place. 139 | } 140 | 141 | // Package 142 | checkStringFlagReplaceWithUtilVersion("p", flags.p, compulsoryFlag) 143 | globalPackageName = flags.p 144 | // Package name must include namespace. 145 | if !strings.HasSuffix(globalPackageName, globalNameSpace) { 146 | fmt.Fprintf(os.Stderr, "tail end of -p %q must match -n %q\n", globalPackageName, globalNameSpace) 147 | printUsage() 148 | os.Exit(12) 149 | } 150 | // Detect an easy package name error (looks like relative path name). 151 | if strings.HasPrefix(globalPackageName, ".") { 152 | fmt.Fprintf(os.Stderr, "invalid -p %s (leading '.')\n", globalPackageName) 153 | printUsage() 154 | os.Exit(12) 155 | } 156 | 157 | // Set default outDir. May be provided (optionally) with -o 158 | var outDir string = "../" + globalNameSpace // Package level, where globalNameSpace is package name. 159 | flagExists = checkStringFlagReplaceWithUtilVersion("o", flags.o, optionalFlag) 160 | if flagExists { // Has been set explicitly with -o 161 | outDir = flags.o 162 | } 163 | // Capital -O means warnOnly. 164 | flagExists = checkStringFlagReplaceWithUtilVersion("O", flags.O, optionalFlag) 165 | if flagExists { // Has been set explicitly with -O 166 | outDir = flags.O 167 | globalFlagOWarnOnly = true 168 | } 169 | globalOutDirAbsolute, err = filepath.Abs(outDir) 170 | if err != nil { 171 | fmt.Fprintf(os.Stderr, "%s\n", err) 172 | printUsage() 173 | os.Exit(14) 174 | } 175 | // Change backslashes to forward slashes. Otherwise strings interpret them as escape chars. 176 | globalOutDirAbsolute = filepath.ToSlash(globalOutDirAbsolute) 177 | if inconsistent, err := inconsistentPackageAndOutDir(globalPackageName, globalOutDirAbsolute); inconsistent { 178 | if globalFlagOWarnOnly { 179 | fmt.Fprintf(os.Stderr, "WARNING: %v\n", err) 180 | fmt.Fprintf(os.Stderr, " go test will work, but main will not be able to find its package\n") 181 | } else { 182 | fmt.Fprintf(os.Stderr, "%v\n", err) 183 | printUsage() 184 | os.Exit(12) 185 | } 186 | } 187 | 188 | // Set default globalOutDirMainAbsolute. May be provided (optionally) with -s 189 | // defaults to /cmd/ 190 | globalOutDirMainAbsolute = fmt.Sprintf("%s/cmd/%s", globalOutDirAbsolute, globalNameSpace) 191 | flagExists = checkStringFlagReplaceWithUtilVersion("s", flags.s, optionalFlag) 192 | if flagExists { // Has been set explicitly with -s 193 | globalOutDirMainAbsolute = flags.s 194 | } 195 | globalOutDirMainAbsolute, err = filepath.Abs(globalOutDirMainAbsolute) 196 | if err != nil { 197 | fmt.Fprintf(os.Stderr, "%s\n", err) 198 | printUsage() 199 | os.Exit(14) 200 | } 201 | // Change backslashes to forward slashes. Otherwise strings interpret them as escape chars. 202 | globalOutDirMainAbsolute = filepath.ToSlash(globalOutDirMainAbsolute) 203 | } 204 | 205 | func progName() string { 206 | return filepath.Base(os.Args[0]) 207 | } 208 | 209 | /* 210 | This function does naughty things:- 211 | - Does not return any values. 212 | - Has global side-effects: Calls os.Exit(). 213 | It avoids heaps of boilerplate code testing flags. 214 | It can be called and:- 215 | - Compulsory flags can trust the existence of an argument. 216 | - Optional flags can test exists. 217 | The flag library itself does some global stuff: bails out if a flag does not have an argument. 218 | */ 219 | func checkStringFlagReplaceWithUtilVersion(name string, arg string, compulsory bool) (exists bool) { 220 | var hasArg bool 221 | 222 | if arg != "" { 223 | exists = true 224 | } 225 | 226 | // Try to detect missing flag argument. 227 | // If an argument is another flag, argument has not been provided. 228 | if exists && !strings.HasPrefix(arg, "-") { 229 | // Option expecting an argument but has been followed by another flag. 230 | hasArg = true 231 | } 232 | /* 233 | where(fmt.Sprintf("-%s compulsory = %t", name, compulsory)) 234 | where(fmt.Sprintf("-%s exists = %t", name, exists)) 235 | where(fmt.Sprintf("-%s hasArg = %t", name, hasArg)) 236 | where(fmt.Sprintf("-%s value = %s", name, arg)) 237 | */ 238 | 239 | if compulsory && !exists { 240 | fmt.Fprintf(os.Stderr, "compulsory flag: -%s\n", name) 241 | printUsage() 242 | os.Exit(2) 243 | } 244 | 245 | if exists && !hasArg { 246 | fmt.Fprintf(os.Stderr, "flag -%s needs a valid argument (not: %s)\n", name, arg) 247 | printUsage() 248 | os.Exit(3) 249 | } 250 | 251 | return 252 | } 253 | 254 | func printUsage() { 255 | var usageSlice []string = []string{ 256 | "usage: ${globalUtilName} [-v] [-d] -f -n -p [-o ] [-s ]", 257 | "purpose: (1) Generate a FlatBuffers schema file .fbs from a set of tables.", 258 | " (2) Generate official Flatbuffers Go code (from .fbs) using flatc --go", 259 | " (3) Generate additional Go code to read/write these specific table types from gotables objects.", 260 | "flags: -f Input text file containing one or more gotables tables (generates schema).", 261 | " See flattables_sample: https://github.com/urban-wombat/flattables_sample/blob/master/tables.got", 262 | " Note: The file need not contain data. Only metadata (names and types) will be used for Go code generation.", 263 | " If there is data in the input file, it will be used for running benchmarks.", 264 | " -n Namespace Sets schema file .fbs, schema root_type, FlatBuffers namespace, TableSet name.", 265 | " Note: Generated Go code will be placed adjacently at Go package level.", 266 | " This assumes you are running ${globalUtilName} at package level.", 267 | " You may override this with -o ", 268 | " -p Package Sets Go package name. Needs to include Namespace.", 269 | " [-o] Where to put generated Go code files. Default is ../", 270 | " Note: The tail end of must match -p ", 271 | " [-O] Allow generated code to go where does NOT match -p (will print WARNING)", 272 | " Note: go test will work, but main will not be able to find its package", 273 | " [-s] Where to put generated sample main Go code file. Default is /cmd/", 274 | // " -m Mutable Tells flatc to add mutable methods to its Go code generation: Mutate...()", 275 | "types: Architecture-dependent Go types int and uint are not used. Instead use e.g. int64, uint32, etc.", 276 | " Go types not implemented: complex32 complex64.", 277 | // "names: Table names are UpperCamelCase, column names are lowerCamelCase, as per the FlatBuffers style guide.", 278 | // "deprecation: To deprecate a column, append its name with _DEPRECATED_ (warning: deprecation may break tests and old code).", 279 | " [-v] Verbose", 280 | " [-d] Dry run (also turns on Verbose)", 281 | " [-h] Help", 282 | "sample: This sample assumes package name \"github.com/urban-wombat/flattables_sample\".", 283 | " Make a Go package dir: $ mkdir flattables_sample", 284 | " $ cd flattables_sample", 285 | " Create a gotables file: tables.got", 286 | " $ ${globalUtilName} -v -f ../flattables_sample/tables.got -n flattables_sample -p github.com/urban-wombat/flattables_sample", 287 | // " $ ${globalUtilName} -v -f ../flattables_sample/tables.got -n flattables_sample -p github.com/urban-wombat/flattables_sample -m", 288 | // " $ go run ${globalUtilName}.go -v -f ../flattables_sample/tables.got -n flattables_sample -p github.com/urban-wombat/flattables_sample", 289 | " $ Test: cd ../flattables_sample; go test -bench=.", 290 | " $ Test: cd ../flattables_sample/cmd/flattables_sample; go run flattables_sample_main.go", 291 | } 292 | 293 | var usageString string 294 | for i := 0; i < len(usageSlice); i++ { 295 | usageString += usageSlice[i] + "\n" 296 | } 297 | 298 | // For debugging or new code, conditionally add provisional command line examples under development. 299 | if user, _ := user.Current(); user.Username == "Malcolm-PC\\Malcolm" { 300 | // We are testing. Provide a useful sample. Does not appear in final product. 301 | usageString += "additional commands in development mode:\n" 302 | usageString += " $ go run ${globalUtilDir}/${globalUtilName}.go -v -f ../flattables_sample/tables.got -n flattables_sample -p github.com/urban-wombat/flattables_sample\n" 303 | usageString += " $ go install ${globalUtilDir}/${globalUtilName}.go\n" 304 | usageString += util.BuildDateTime() 305 | } 306 | 307 | usageString = strings.Replace(usageString, "${globalUtilName}", globalUtilName, -1) 308 | usageString = strings.Replace(usageString, "${globalUtilDir}", globalUtilDir, -1) 309 | 310 | fmt.Fprintf(os.Stderr, "%s\n", usageString) 311 | } 312 | 313 | func main() { 314 | 315 | if strings.Contains(os.Args[0], globalUtilName) { 316 | } else { 317 | fmt.Fprintf(os.Stderr, `expecting to be called "flattablesc" not %q`, os.Args[0]) 318 | os.Exit(2) 319 | } 320 | 321 | if len(os.Args) == 1 { 322 | // No args. 323 | fmt.Fprintf(os.Stderr, "%s expects at least 1 argument\n", globalUtilName) 324 | printUsage() 325 | os.Exit(2) 326 | } 327 | 328 | if len(os.Args) < 3 { 329 | fmt.Fprintf(os.Stderr, "Expecting -f containing one or more gotables tables\n") 330 | printUsage() 331 | os.Exit(13) 332 | } 333 | 334 | flag.Usage = printUsage // Override the default flag.Usage variable. 335 | initFlags() 336 | 337 | if flags.d { // dry run, turn on verbose 338 | flags.v = true 339 | fmt.Printf(" *** -d DRY-RUN ***\n") 340 | } 341 | 342 | if flags.v { 343 | fmt.Printf(" (1) Reading gotables file: %s\n", globalGotablesFileNameAbsolute) 344 | } 345 | tableSet, err := gotables.NewTableSetFromFile(globalGotablesFileNameAbsolute) 346 | if err != nil { 347 | fmt.Fprintf(os.Stderr, "%s\n", err) 348 | printUsage() 349 | os.Exit(14) 350 | } 351 | 352 | if flags.v { 353 | fmt.Printf(" (2) Setting gotables.TableSet name to %q (from -n %s)\n", globalNameSpace, globalNameSpace) 354 | } 355 | tableSet.SetName(globalNameSpace) 356 | tableSet.SetFileName(globalGotablesFileNameAbsolute) 357 | 358 | if flags.v { 359 | fmt.Printf(" (3) Setting package name to %q (from -p %s)\n", globalPackageName, globalPackageName) 360 | } 361 | 362 | if !pathExists(globalOutDirAbsolute) { 363 | if flags.v { 364 | fmt.Printf(" (4) Creating dir %s\n", globalOutDirAbsolute) 365 | } 366 | permissions := 0777 367 | if flags.d { 368 | fmt.Printf(" *** -d dry-run: Would have created %s\n", globalOutDirAbsolute) 369 | } else { 370 | err = os.MkdirAll(globalOutDirAbsolute, os.FileMode(permissions)) 371 | if err != nil { 372 | fmt.Fprintf(os.Stderr, "%s\n", err) 373 | os.Exit(15) 374 | } 375 | } 376 | } else { 377 | if flags.v { 378 | fmt.Printf(" (4) Dir already exists (okay) %s\n", globalOutDirAbsolute) 379 | } 380 | } 381 | 382 | if !pathExists(globalOutDirMainAbsolute) { 383 | if flags.v { 384 | fmt.Printf(" (5) Creating dir %s\n", globalOutDirMainAbsolute) 385 | } 386 | permissions := 0777 387 | if flags.d { 388 | fmt.Printf(" *** -d dry-run: Would have created %s\n", globalOutDirMainAbsolute) 389 | } else { 390 | err = os.MkdirAll(globalOutDirMainAbsolute, os.FileMode(permissions)) 391 | if err != nil { 392 | fmt.Fprintf(os.Stderr, "%s\n", err) 393 | os.Exit(16) 394 | } 395 | } 396 | } else { 397 | if flags.v { 398 | fmt.Printf(" (5) Dir already exists (okay) %s\n", globalOutDirMainAbsolute) 399 | } 400 | } 401 | 402 | // Must be called before flattables.InitTablesTemplateInfo() 403 | err = flattables.DeleteEmptyTables(tableSet) 404 | if err != nil { 405 | fmt.Fprintf(os.Stderr, "%s\n", err) 406 | os.Exit(17) 407 | } 408 | 409 | // spew.Dump(tablesTemplateInfo) 410 | 411 | // Template info for ALL the templates. 412 | if flags.v { 413 | fmt.Printf(" (6) Preparing tables for schema generation ...\n") 414 | } 415 | tablesTemplateInfo, err = flattables.InitTablesTemplateInfo(tableSet, globalPackageName) 416 | if err != nil { 417 | fmt.Fprintf(os.Stderr, "%s\n", err) 418 | os.Exit(18) 419 | } 420 | 421 | // Make these assignments AFTER calling flattables.InitTablesTemplateInfo() 422 | tablesTemplateInfo.NameSpace = globalNameSpace 423 | tablesTemplateInfo.GotablesFileNameAbsolute = globalGotablesFileNameAbsolute 424 | 425 | tablesTemplateInfo.OutDirAbsolute = globalOutDirAbsolute 426 | tablesTemplateInfo.OutDirMainAbsolute = globalOutDirMainAbsolute 427 | 428 | // spew.Dump(tablesTemplateInfo) 429 | 430 | var tableCount int = tableSet.TableCount() 431 | if flags.v { 432 | fmt.Printf(" Adding gotables tables * to FlatBuffers schema: (%d table%s added)\n", tableCount, plural(tableCount)) 433 | } 434 | 435 | flatBuffersSchemaFileName := globalOutDirAbsolute + "/" + globalNameSpace + ".fbs" 436 | tablesTemplateInfo.GeneratedFile = filepath.Base(flatBuffersSchemaFileName) 437 | if flags.v { 438 | fmt.Printf(" (7) Generating FlatBuffers schema from gotables file %s \n", globalGotablesFileNameAbsolute) 439 | fmt.Printf(" Generating: %s\n", flatBuffersSchemaFileName) 440 | } 441 | flatBuffersSchema, err := flattables.FlatBuffersSchemaFromTableSet(tablesTemplateInfo) 442 | if err != nil { 443 | fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) 444 | os.Exit(19) 445 | } 446 | 447 | flatBuffersSchema = flattables.RemoveExcessTabsAndNewLines(flatBuffersSchema) 448 | 449 | if flags.d { 450 | fmt.Printf(" *** -d dry-run: Would have written file: %s\n", flatBuffersSchemaFileName) 451 | } else { 452 | err = ioutil.WriteFile(flatBuffersSchemaFileName, []byte(flatBuffersSchema), 0644) 453 | if err != nil { 454 | fmt.Fprintf(os.Stderr, "%s\n", err) 455 | os.Exit(20) 456 | } 457 | } 458 | 459 | // Note: each arg part needs to be passed to exec.Command separately. 460 | executable := "flatc" 461 | goFlag := "--go" 462 | outFlag := "-o" // for flatc 463 | // outDirFlatC := ".." // flatc creates a subdir under this. 464 | // flatc appends nameSpace to its outDirFlatC. So we need to snip nameSpace from end of outDirFlatC 465 | outDirFlatC := globalOutDirAbsolute[:len(globalOutDirAbsolute)-len(globalNameSpace)] // Snip off globalNameSpace 466 | 467 | // flatc creates subdir under outDirFlatC 468 | if flags.v { 469 | fmt.Printf(" (8) From FlatBuffers schema %s\n", flatBuffersSchemaFileName) 470 | } 471 | if flags.v { 472 | fmt.Printf(" generating standard generic Google FlatBuffers Go code:\n") 473 | } 474 | if flags.v { 475 | fmt.Printf(" %s\n", flatBuffersSchemaFileName) 476 | } 477 | fmtString := " $ %s %s %s %s %s\n %s\n" 478 | if flags.m { // Mutable 479 | // if flags.v { fmt.Printf(" $ %s %s %s %s %s %s\n", executable, goFlag, globalMutableFlag, outFlag, outDirFlatC, flatBuffersSchemaFileName) } 480 | if flags.v { 481 | fmt.Printf(fmtString, executable, goFlag, globalMutableFlag, outFlag, outDirFlatC, flatBuffersSchemaFileName) 482 | } 483 | } else { 484 | // if flags.v { fmt.Printf(" $ %s %s %s %s %s\n", executable, goFlag, outFlag, outDirFlatC, flatBuffersSchemaFileName) } 485 | if flags.v { 486 | fmt.Printf(fmtString, executable, goFlag, "", outFlag, outDirFlatC, flatBuffersSchemaFileName) 487 | } 488 | } 489 | var cmd *exec.Cmd 490 | if flags.m { // Mutable 491 | cmd = exec.Command(executable, goFlag, globalMutableFlag, outFlag, outDirFlatC, flatBuffersSchemaFileName) 492 | } else { 493 | cmd = exec.Command(executable, goFlag, outFlag, outDirFlatC, flatBuffersSchemaFileName) 494 | } 495 | var out bytes.Buffer 496 | cmd.Stdout = &out 497 | if flags.d { 498 | fmt.Printf(" *** -d dry-run: Would have run command: %s\n", cmd.Args) 499 | } else { 500 | err = cmd.Run() 501 | if err != nil { 502 | // Note: err contains the exit code. (?) 503 | // out contains the actual error message. (?) 504 | fmt.Fprintf(os.Stderr, "(a) %s\n", err) 505 | fmt.Fprintf(os.Stderr, "(b) %s\n", out.String()) 506 | fmt.Fprintf(os.Stderr, "(c) Have you installed flatc ?\n") 507 | printUsage() 508 | os.Exit(21) 509 | } 510 | } 511 | 512 | if flags.v { 513 | fmt.Printf(" (*) Generating user Go code ...\n") 514 | } 515 | err = flattables.GenerateAll(tablesTemplateInfo, flags.v, flags.d) 516 | if err != nil { 517 | fmt.Fprintf(os.Stderr, "%s\n", err) 518 | os.Exit(22) 519 | } 520 | 521 | if flags.d { 522 | fmt.Println(" *** -d DRY-RUN *** (Didn't do anything!)") 523 | } else { 524 | fmt.Println(" DONE") 525 | fmt.Printf(" Test: cd %s; go test -bench=.\n", globalOutDirAbsolute) 526 | fmt.Printf(" Test: cd %s; go run %s_main.go\n", globalOutDirMainAbsolute, globalNameSpace) 527 | } 528 | 529 | if user, _ := user.Current(); user.Username == "Malcolm-PC\\Malcolm" { 530 | fmt.Printf(" %s\n", util.BuildDateTime()) 531 | } 532 | } 533 | 534 | // From: http://www.musingscafe.com/check-if-a-file-or-folder-exists-in-golang 535 | // Also checks directories. 536 | // fileExists 537 | // dirExists 538 | func pathExists(path string) (exists bool) { 539 | exists = true 540 | if _, err := os.Stat(path); os.IsNotExist(err) { 541 | exists = false 542 | } 543 | return 544 | } 545 | 546 | func plural(items int) string { 547 | if items == 1 || items == -1 { 548 | // Singular 549 | return "" 550 | } else { 551 | // Plural 552 | return "s" 553 | } 554 | } 555 | 556 | func inconsistentPackageAndOutDir(packageName string, outDir string) (consistent bool, err error) { 557 | // Convert outDir to absolute and forward slashes for valid comparison with packageName. 558 | absolute, err := filepath.Abs(outDir) 559 | if err != nil { 560 | return true, err 561 | } 562 | 563 | // Change backslashes to forward slashes. Otherwise strings interpret them as escape chars. 564 | absolute = filepath.ToSlash(absolute) 565 | 566 | // See if outDir even contains packageName. Deal-breaker if it doesn't. 567 | index := strings.Index(absolute, packageName) 568 | if index < 0 { 569 | err := fmt.Errorf("-p %q is missing from (absolute) -o %s", packageName, absolute) 570 | return true, err 571 | } 572 | 573 | absolute = absolute[index:] 574 | if absolute != packageName { 575 | err := fmt.Errorf("-p does not match END OF -o : -p %s -o %s (end of)", packageName, absolute) 576 | lenOutDir := len(absolute) 577 | lenPackageName := len(packageName) 578 | var length int 579 | if lenOutDir > lenPackageName { 580 | length = lenOutDir 581 | } else { 582 | length = lenPackageName 583 | } 584 | where(fmt.Sprintf("%-*s\n", length, packageName)) 585 | where(fmt.Sprintf("%-*s\n", length, absolute)) 586 | return true, err 587 | } 588 | 589 | return false, nil 590 | } 591 | -------------------------------------------------------------------------------- /flattables.go: -------------------------------------------------------------------------------- 1 | package flattables 2 | 3 | // This is a library of helper functions for utility: flattablesc 4 | 5 | // See: https://github.com/urban-wombat/flattables#flattables-is-a-simplified-tabular-subset-of-flatbuffers 6 | 7 | import ( 8 | "bufio" 9 | "bytes" 10 | "fmt" 11 | "go/token" 12 | "io/ioutil" 13 | "log" 14 | "os" 15 | "path/filepath" 16 | "strings" 17 | "text/template" 18 | "time" 19 | "unicode" 20 | 21 | "github.com/urban-wombat/gotables" 22 | "github.com/urban-wombat/util" 23 | ) 24 | 25 | /* 26 | Copyright (c) 2017 Malcolm Gorman 27 | 28 | Permission is hereby granted, free of charge, to any person obtaining a copy 29 | of this software and associated documentation files (the "Software"), to deal 30 | in the Software without restriction, including without limitation the rights 31 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 32 | copies of the Software, and to permit persons to whom the Software is 33 | furnished to do so, subject to the following conditions: 34 | 35 | The above copyright notice and this permission notice shall be included in all 36 | copies or substantial portions of the Software. 37 | 38 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 39 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 40 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 41 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 42 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 43 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 44 | SOFTWARE. 45 | */ 46 | 47 | func init() { 48 | log.SetFlags(log.Lshortfile) // For var where 49 | } 50 | 51 | var where = log.Print 52 | 53 | /* 54 | Note: the following are synonyms:- 55 | synonyms: flattables FlatTables flatbuffers FlatBuffers 56 | */ 57 | 58 | // FlatBuffers schema types: bool | byte | ubyte | short | ushort | int | uint | float | long | ulong | double | string 59 | // From: https://google.github.io/flatbuffers/flatbuffers_grammar.html 60 | 61 | // Built-in scalar types are: 62 | // 8 bit: byte, ubyte, bool 63 | // 16 bit: short, ushort 64 | // 32 bit: int, uint, float 65 | // 64 bit: long, ulong, double 66 | // From: https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html 67 | 68 | var goToFlatBuffersTypes = map[string]string{ 69 | "bool": "bool", 70 | "int8": "byte", // Signed. 71 | "int16": "short", 72 | "int32": "int", // (Go rune is an alias for Go int32. For future reference.) 73 | "int64": "long", 74 | "byte": "ubyte", // Unsigned. Go byte is an alias for Go uint8. 75 | "[]byte": "[ubyte]", // Unsigned. Go byte is an alias for Go uint8. NOTE: This [ubyte] IS NOT IMPLEMENTED IN FLATTABLES! 76 | "uint8": "ubyte", 77 | "uint16": "ushort", 78 | "uint32": "uint", 79 | "uint64": "ulong", 80 | "float32": "float", 81 | "float64": "double", 82 | "string": "string", 83 | // "int": "long", // Assume largest int size: 64 bit. NO, DON'T DO THIS AUTOMATICALLY. REQUIRE USER DECISION. 84 | // "uint": "ulong", // Assume largest uint size: 64 bit. NO, DON'T DO THIS AUTOMATICALLY. REQUIRE USER DECISION. 85 | } 86 | 87 | var goFlatBuffersScalarTypes = map[string]string{ 88 | "bool": "bool", // Scalar from FlatBuffers point of view. 89 | "int8": "byte", // Signed. 90 | "int16": "short", 91 | "int32": "int", // (Go rune is an alias for Go int32. For future reference.) 92 | "int64": "long", 93 | "byte": "ubyte", // Unsigned. Go byte is an alias for Go uint8. 94 | "uint8": "ubyte", 95 | "uint16": "ushort", 96 | "uint32": "uint", 97 | "uint64": "ulong", 98 | "float32": "float", 99 | "float64": "double", 100 | } 101 | 102 | const deprecated = "_deprecated_" 103 | 104 | func schemaType(colType string) (string, error) { 105 | schemaType, exists := goToFlatBuffersTypes[colType] 106 | if exists { 107 | return schemaType, nil 108 | } else { 109 | // Build a useful error message. 110 | var suggestChangeTypeTo string 111 | switch colType { 112 | case "int": 113 | suggestChangeTypeTo = "int32 or int64" 114 | case "uint": 115 | suggestChangeTypeTo = "uint32 or uint64" 116 | default: 117 | return "", fmt.Errorf("no FlatBuffers-compatible Go type suggestion for Go type: %s", colType) 118 | } 119 | return "", fmt.Errorf("no FlatBuffers type available for Go type: %s (suggest change it to Go type: %s)", 120 | colType, suggestChangeTypeTo) 121 | } 122 | } 123 | 124 | func isDeprecated(colName string) bool { 125 | return strings.Contains(colName, deprecated) 126 | } 127 | 128 | func IsFlatBuffersScalar(colType string) bool { 129 | _, exists := goFlatBuffersScalarTypes[colType] 130 | return exists 131 | } 132 | 133 | // This is possibly unused. 134 | func isScalar(table *gotables.Table, colName string) bool { 135 | colType, err := table.ColType(colName) 136 | if err != nil { 137 | _, _ = fmt.Fprintf(os.Stderr, "%s [%s].%s ERROR: %v\n", util.FuncName(), table.Name(), colName, err) 138 | return false 139 | } 140 | 141 | isNumeric, err := gotables.IsNumericColType(colType) 142 | if err != nil { 143 | _, _ = fmt.Fprintf(os.Stderr, "%s [%s].%s ERROR: %v\n", util.FuncName(), table.Name(), colName, err) 144 | return false 145 | } 146 | 147 | return isNumeric || colType == "bool" 148 | } 149 | 150 | func indentText(indent string, text string) string { 151 | var indentedText string = "" 152 | scanner := bufio.NewScanner(strings.NewReader(text)) 153 | for scanner.Scan() { 154 | indentedText += fmt.Sprintf("%s%s\n", indent, scanner.Text()) 155 | } 156 | return indentedText 157 | } 158 | 159 | func FlatBuffersSchemaFromTableSet(tablesTemplateInfo TablesTemplateInfoType) (string, error) { 160 | 161 | var err error 162 | 163 | var buf *bytes.Buffer = bytes.NewBufferString("") 164 | 165 | const FlatBuffersSchemaFromTableSetTemplateFile = "../flattables/FlatBuffersSchema.template" 166 | // Use the file name as the template name so that file name appears in error output. 167 | // We still use the file name for diagnostics, even though the template is now embedded in flattables_templates.go 168 | // Although no longer used to OPEN the file, it is still used in err to locate the original (non-embedded) file source. 169 | var tplate *template.Template = template.New(FlatBuffersSchemaFromTableSetTemplateFile) 170 | 171 | // Add a user-defined function to schema tplate. 172 | tplate.Funcs(template.FuncMap{"firstCharToUpper": firstCharToUpper}) 173 | tplate.Funcs(template.FuncMap{"yearRangeFromFirstYear": yearRangeFromFirstYear}) 174 | 175 | // From embedded template in flattables_templates.go 176 | var data []byte = FlatBuffersSchema_template 177 | /* 178 | NOTE: This []byte slice may be what egonelbre is referring to when he says: 179 | This https://github.com/urban-wombat/flattables/blob/master/flattables.go#L202 breaks with unicode. 180 | */ 181 | 182 | tplate, err = tplate.Parse(string(data)) 183 | if err != nil { 184 | return "", err 185 | } 186 | 187 | err = tplate.Execute(buf, tablesTemplateInfo) 188 | if err != nil { 189 | return "", err 190 | } 191 | 192 | return buf.String(), nil 193 | } 194 | 195 | func startsWithLowerCase(s string) bool { 196 | if len(s) > 0 { 197 | rune0 := rune(s[0]) 198 | return unicode.IsLower(rune0) 199 | } else { 200 | return false 201 | } 202 | } 203 | 204 | func startsWithUpperCase(s string) bool { 205 | if len(s) > 0 { 206 | rune0 := rune(s[0]) 207 | return unicode.IsUpper(rune0) 208 | } else { 209 | return false 210 | } 211 | } 212 | 213 | func firstCharToUpper(s string) string { 214 | var upper string 215 | if len(s) > 0 { 216 | rune0 := rune(s[0]) 217 | upper = string(unicode.ToUpper(rune0)) + s[1:] 218 | } else { 219 | upper = "" 220 | } 221 | return upper 222 | } 223 | 224 | func firstCharToLower(s string) string { 225 | var lower string 226 | if len(s) > 0 { 227 | rune0 := rune(s[0]) 228 | lower = string(unicode.ToLower(rune0)) + s[1:] 229 | } else { 230 | lower = "" 231 | } 232 | return lower 233 | } 234 | 235 | func tableName(table *gotables.Table) string { 236 | return "// " + table.Name() 237 | } 238 | 239 | func rowCount(table *gotables.Table) int { 240 | return table.RowCount() 241 | } 242 | 243 | // Information specific to each generated function. 244 | type GenerationInfo struct { 245 | TemplateType string 246 | FuncName string // Used as basename of *.template and *.go files. Not always a function name. 247 | Imports []string // imports for this template. 248 | TemplateText []byte 249 | } 250 | 251 | var generations = []GenerationInfo{ 252 | {TemplateType: "flattables", 253 | FuncName: "README", 254 | TemplateText: README_template, 255 | Imports: []string{}, 256 | }, 257 | {TemplateType: "flattables", 258 | FuncName: "test", // Not really a function name. 259 | TemplateText: test_template, 260 | Imports: []string{ 261 | `"bytes"`, 262 | `"fmt"`, 263 | `"github.com/urban-wombat/gotables"`, 264 | `"reflect"`, 265 | `"testing"`, 266 | }, 267 | }, 268 | {TemplateType: "flattables", 269 | FuncName: "helpers", 270 | TemplateText: helpers_template, 271 | Imports: []string{ 272 | `"path/filepath"`, 273 | `"runtime"`, 274 | `"strings"`, 275 | }, 276 | }, 277 | {TemplateType: "flattables", 278 | FuncName: "NewFlatBuffersFromSlice", 279 | TemplateText: NewFlatBuffersFromSlice_template, 280 | Imports: []string{ 281 | `flatbuffers "github.com/google/flatbuffers/go"`, 282 | `"fmt"`, 283 | }, 284 | }, 285 | {TemplateType: "flattables", 286 | FuncName: "NewFlatBuffersFromTableSet", 287 | TemplateText: NewFlatBuffersFromTableSet_template, 288 | Imports: []string{ 289 | `flatbuffers "github.com/google/flatbuffers/go"`, 290 | `"github.com/urban-wombat/gotables"`, 291 | `"fmt"`, 292 | }, 293 | }, 294 | {TemplateType: "flattables", 295 | FuncName: "NewSliceFromFlatBuffers", 296 | TemplateText: NewSliceFromFlatBuffers_template, 297 | Imports: []string{ 298 | `"fmt"`, 299 | }, 300 | }, 301 | {TemplateType: "flattables", 302 | FuncName: "NewTableSetFromFlatBuffers", 303 | TemplateText: NewTableSetFromFlatBuffers_template, 304 | Imports: []string{ 305 | `"github.com/urban-wombat/gotables"`, 306 | `"fmt"`, 307 | }, 308 | }, 309 | {TemplateType: "flattables", 310 | FuncName: "OldSliceFromFlatBuffers", 311 | TemplateText: OldSliceFromFlatBuffers_template, 312 | Imports: []string{ 313 | `"fmt"`, 314 | }, 315 | }, 316 | {TemplateType: "flattables", 317 | FuncName: "main", // Not really a function name. 318 | TemplateText: main_template, 319 | Imports: []string{ 320 | `"github.com/urban-wombat/gotables"`, 321 | `"fmt"`, 322 | }, 323 | }, 324 | } 325 | 326 | func GenerateAll(tablesTemplateInfo TablesTemplateInfoType, verbose bool, dryRun bool) error { 327 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 328 | for _, generation := range generations { 329 | // tablesTemplateInfo is global. 330 | err := generateGoCodeFromTemplate(generation, tablesTemplateInfo, verbose, dryRun) 331 | if err != nil { 332 | return err 333 | } 334 | } 335 | 336 | return nil 337 | } 338 | 339 | func generateGoCodeFromTemplate(generationInfo GenerationInfo, tablesTemplateInfo TablesTemplateInfoType, verbose bool, dryRun bool) (err error) { 340 | //gotables.PrintCaller() 341 | 342 | var templateFile string 343 | var outDir string 344 | var generatedFile string 345 | 346 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 347 | // Calculate input template file name. 348 | // Although no longer used to OPEN the file, it is still used in err to locate the original (non-embedded) file source. 349 | templateFile = fmt.Sprintf("../%s/%s.template", generationInfo.TemplateType, generationInfo.FuncName) 350 | 351 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 352 | /* 353 | where("USING") 354 | where(tablesTemplateInfo.OutDirMainAbsolute) 355 | where(tablesTemplateInfo.OutDir) 356 | */ 357 | // Calculate output dir name. 358 | if strings.Contains(generationInfo.FuncName, "main") { 359 | // outDir = "../" + nameSpace + "_main" // main is in its own directory 360 | outDir = tablesTemplateInfo.OutDirMainAbsolute // main is in its own directory 361 | //where(fmt.Sprintf("outDir = %s", outDir)) 362 | } else { 363 | // outDir = "../" + nameSpace // put it in with all the rest 364 | outDir = tablesTemplateInfo.OutDirAbsolute // put it in with all the rest 365 | //where(fmt.Sprintf("outDir = %s", outDir)) 366 | } 367 | 368 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 369 | // Calculate output file name. 370 | switch generationInfo.FuncName { 371 | case "README": // README is a markdown .md file 372 | generatedFile = outDir + "/" + generationInfo.FuncName + ".md" 373 | default: // For both function files and main files. Retain FuncName for main functions to differentiate multiple mains. 374 | generatedFile = outDir + "/" + tablesTemplateInfo.NameSpace + "_" + generationInfo.FuncName + ".go" 375 | } 376 | if verbose { 377 | fmt.Printf(" Generating: %-12s %s\n", fmt.Sprintf("(%s)", generationInfo.TemplateType), generatedFile) 378 | } 379 | 380 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 381 | tablesTemplateInfo.SchemaFileName = outDir + "/" + tablesTemplateInfo.NameSpace + ".fbs" 382 | tablesTemplateInfo.SchemaFileNameBase = filepath.Base(tablesTemplateInfo.SchemaFileName) 383 | tablesTemplateInfo.GeneratedFile = generatedFile 384 | tablesTemplateInfo.GeneratedFileBaseName = filepath.Base(tablesTemplateInfo.GeneratedFile) 385 | tablesTemplateInfo.FuncName = generationInfo.FuncName 386 | tablesTemplateInfo.Imports = generationInfo.Imports 387 | 388 | /* 389 | fmt.Printf("\n") 390 | fmt.Printf("%#v\n", generationInfo) 391 | fmt.Printf("templateFile = %s\n", templateFile) 392 | fmt.Printf("outDir = %s\n", outDir) 393 | fmt.Printf("generatedFile = %s\n", generatedFile) 394 | fmt.Printf("\n") 395 | */ 396 | 397 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 398 | var stringBuffer *bytes.Buffer = bytes.NewBufferString("") 399 | 400 | // Use the file name as the template name so that file name appears in error output. 401 | // We still use the file name for diagnostics, even though the template is now embedded in flattables_templates.go 402 | // Although no longer used to OPEN the file, it is still used in err to locate the original (non-embedded) file source. 403 | var tplate *template.Template = template.New(templateFile) 404 | 405 | // Add functions. 406 | tplate.Funcs(template.FuncMap{"firstCharToUpper": firstCharToUpper}) 407 | tplate.Funcs(template.FuncMap{"firstCharToLower": firstCharToLower}) 408 | tplate.Funcs(template.FuncMap{"rowCount": rowCount}) 409 | tplate.Funcs(template.FuncMap{"yearRangeFromFirstYear": yearRangeFromFirstYear}) 410 | 411 | /* 412 | // Open and read file explicitly to avoid calling tplate.ParseFile() which has problems. 413 | templateText, err := ioutil.ReadFile(templateFile) 414 | if err != nil { return } 415 | */ 416 | 417 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 418 | // Template from embedded templates in flattables_templates.go 419 | var templateText []byte = generationInfo.TemplateText 420 | 421 | tplate, err = tplate.Parse(string(templateText)) 422 | if err != nil { 423 | return 424 | } 425 | 426 | err = tplate.Execute(stringBuffer, tablesTemplateInfo) 427 | if err != nil { 428 | return 429 | } 430 | 431 | var goCode string = stringBuffer.String() 432 | 433 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 434 | // The code generator has a lot of quirks (such as extra lines and tabs) which are hard to 435 | // eliminate within the templates themselves. Use gofmt to tidy up Go code. 436 | 437 | // We don't want gofmt to mess with non-Go files (such as README.md which it crunches). 438 | if strings.HasSuffix(generatedFile, ".go") { 439 | goCode = RemoveExcessTabsAndNewLines(goCode) // handwritten formatter 440 | 441 | goCode, err = util.FormatSource(goCode) 442 | if err != nil { 443 | return 444 | } 445 | } 446 | 447 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 448 | if dryRun { 449 | fmt.Printf(" *** -d dry-run: Would have written file: %s\n", generatedFile) 450 | } else { 451 | err = ioutil.WriteFile(generatedFile, []byte(goCode), 0644) 452 | if err != nil { 453 | _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) 454 | os.Exit(30) 455 | } 456 | } 457 | //where(fmt.Sprintf("WHAT? %s", tablesTemplateInfo.OutDirMainAbsolute)) 458 | 459 | return 460 | } 461 | 462 | // Compilation will fail if a user inadvertently uses a Go key word as a name. 463 | var goKeyWordsDEPRECATED = map[string]string{ 464 | "break": "break", 465 | "default": "default", 466 | "func": "func", 467 | "interface": "interface", 468 | "select": "select", 469 | "case": "case", 470 | "defer": "defer", 471 | "go": "go", 472 | "map": "map", 473 | "struct": "struct", 474 | "chan": "chan", 475 | "else": "else", 476 | "goto": "goto", 477 | "package": "package", 478 | "switch": "switch", 479 | "const": "const", 480 | "fallthrough": "fallthrough", 481 | "if": "if", 482 | "range": "range", 483 | "type": "type", 484 | "continue": "continue", 485 | "for": "for", 486 | "import": "import", 487 | "return": "return", 488 | "var": "var", 489 | } 490 | 491 | func isGoKeywordDEPRECATED(name string) bool { 492 | nameLower := strings.ToLower(name) 493 | _, exists := goKeyWordsDEPRECATED[nameLower] 494 | return exists 495 | } 496 | 497 | // See https://www.reddit.com/r/golang/comments/9umtp2/beta_release_of_flattables_go_flatbuffers/e95iffn/?context=3 498 | // This avoids manually providing a lookup map. 499 | func isGoKeyword(name string) bool { 500 | nameLower := strings.ToLower(name) 501 | var isKeyword bool = token.Lookup(nameLower).IsKeyword() 502 | return isKeyword 503 | } 504 | 505 | // Could be tricky if a user inadvertently uses a word used in FlatBuffers schemas. 506 | var flatBuffersOrFlatTablesKeyWords = map[string]string{ 507 | "flattables": "flattables", // FlatTables is used as the root table name and root_type. 508 | "table": "table", 509 | "namespace": "namespace", 510 | "root_type": "root_type", 511 | "ubyte": "ubyte", 512 | "float": "float", 513 | "long": "long", 514 | "ulong": "ulong", 515 | "short": "short", 516 | "ushort": "ushort", 517 | "double": "double", 518 | "enum": "enum", 519 | "union": "union", 520 | "include": "include", 521 | } 522 | 523 | func isFlatBuffersOrFlatTablesKeyWord(name string) bool { 524 | name = strings.ToLower(name) 525 | _, exists := flatBuffersOrFlatTablesKeyWords[name] 526 | return exists 527 | } 528 | 529 | type ColInfo struct { 530 | ColName string 531 | ColType string 532 | FbsType string 533 | ColIndex int 534 | IsScalar bool // FlatBuffers Scalar includes bool 535 | IsString bool 536 | IsBool bool 537 | IsDeprecated bool 538 | } 539 | 540 | type Row []string 541 | 542 | type TableInfo struct { 543 | Table *gotables.Table 544 | TableIndex int 545 | TableName string 546 | RowCount int 547 | ColCount int 548 | Cols []ColInfo 549 | Rows []Row 550 | ColNames []string 551 | ColTypes []string 552 | } 553 | 554 | type TablesTemplateInfoType struct { 555 | GeneratedDateFromFile string 556 | GeneratedDateFromFileBaseName string 557 | GeneratedFromFile string 558 | UsingCommand string 559 | UsingCommandMinusG string 560 | NameSpace string // Included in PackageName. 561 | PackageName string // Includes NameSpace 562 | Year string 563 | OutDirAbsolute string 564 | OutDirMainAbsolute string 565 | SchemaFileName string 566 | SchemaFileNameBase string 567 | GeneratedFile string 568 | GeneratedFileBaseName string 569 | FuncName string 570 | Imports []string 571 | // GotablesFileName string // We want to replace this with the following TWO file names. 572 | GotablesFileNameAbsolute string 573 | GotablesFileNameBase string 574 | TableSetMetadata string 575 | TableSetData string 576 | Tables []TableInfo 577 | } 578 | 579 | var TablesTemplateInfo TablesTemplateInfoType 580 | 581 | func (tablesTemplateInfo TablesTemplateInfoType) Name(tableIndex int) string { 582 | return tablesTemplateInfo.Tables[0].Table.Name() 583 | } 584 | 585 | func DeleteEmptyTables(tableSet *gotables.TableSet) error { 586 | 587 | for tableIndex := tableSet.TableCount() - 1; tableIndex >= 0; tableIndex-- { 588 | table, err := tableSet.TableByTableIndex(tableIndex) 589 | if err != nil { 590 | return err 591 | } 592 | 593 | if table.ColCount() == 0 { 594 | err = tableSet.DeleteTableByTableIndex(tableIndex) 595 | if err != nil { 596 | return err 597 | } 598 | return fmt.Errorf("table has zero cols: [%s] (remove or populate)", table.Name()) 599 | } 600 | } 601 | 602 | return nil 603 | } 604 | 605 | // Assumes flattables.RemoveEmptyTables() has been called first. 606 | func InitTablesTemplateInfo(tableSet *gotables.TableSet, packageName string) (TablesTemplateInfoType, error) { 607 | 608 | var emptyTemplateInfo TablesTemplateInfoType 609 | var tablesTemplateInfo TablesTemplateInfoType 610 | 611 | var tables []TableInfo = make([]TableInfo, tableSet.TableCount()) 612 | for tableIndex := 0; tableIndex < tableSet.TableCount(); tableIndex++ { 613 | table, err := tableSet.TableByTableIndex(tableIndex) 614 | if err != nil { 615 | return emptyTemplateInfo, err 616 | } 617 | 618 | if table.ColCount() > 0 { 619 | _, _ = fmt.Fprintf(os.Stderr, " Adding gotables table %2d to FlatBuffers schema: [%s] \n", tableIndex, table.Name()) 620 | } else { 621 | // Skip tables with zero cols. 622 | return emptyTemplateInfo, fmt.Errorf("--- FlatTables: table [%s] has no cols", table.Name()) 623 | } 624 | 625 | if startsWithLowerCase(table.Name()) { 626 | // See: https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html 627 | return emptyTemplateInfo, fmt.Errorf("the FlatBuffers style guide requires UpperCamelCase table names. Rename [%s] to [%s]", 628 | table.Name(), firstCharToUpper(table.Name())) 629 | } 630 | 631 | if isGoKeyword(table.Name()) { 632 | return emptyTemplateInfo, 633 | fmt.Errorf("cannot use a Go key word as a table name, even if it's upper case. Rename [%s]", table.Name()) 634 | } 635 | 636 | if isFlatBuffersOrFlatTablesKeyWord(table.Name()) { 637 | return emptyTemplateInfo, 638 | fmt.Errorf("cannot use a FlatBuffers or FlatTables key word as a table name, even if it's merely similar. Rename [%s]", 639 | table.Name()) 640 | } 641 | 642 | // I don't see documentation on this, but undescores in field names affect code generation. 643 | if strings.ContainsRune(table.Name(), '_') { 644 | return emptyTemplateInfo, 645 | fmt.Errorf("cannot use underscores '_' in table names. Rename [%s]", table.Name()) 646 | } 647 | 648 | tables[tableIndex].Table = table // An array of Table accessible as .Tables 649 | 650 | var cols []ColInfo = make([]ColInfo, table.ColCount()) 651 | for colIndex := 0; colIndex < table.ColCount(); colIndex++ { 652 | colName, err := table.ColNameByColIndex(colIndex) 653 | if err != nil { 654 | return emptyTemplateInfo, err 655 | } 656 | 657 | // Enforce FlatBuffers style guide. 658 | if startsWithUpperCase(colName) { 659 | // See: https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html 660 | return emptyTemplateInfo, fmt.Errorf("#1 the FlatBuffers style guide requires lowerCamelCase field names. In table [%s] rename %s to %s", 661 | table.Name(), colName, firstCharToLower(colName)) 662 | } 663 | 664 | if isGoKeyword(colName) { 665 | return emptyTemplateInfo, fmt.Errorf("cannot use a Go key word as a col name, even if it's upper case. Rename: %s", colName) 666 | } 667 | 668 | if isFlatBuffersOrFlatTablesKeyWord(colName) { 669 | return emptyTemplateInfo, 670 | fmt.Errorf("cannot use a FlatBuffers or FlatTables key word as a col name, even if it's merely similar. Rename: %s", colName) 671 | } 672 | 673 | // I don't see documentation on this, but undescores in field names affect code generation. 674 | if strings.ContainsRune(colName, '_') { 675 | return emptyTemplateInfo, 676 | fmt.Errorf("cannot use underscores '_' in col names. Rename %s", colName) 677 | } 678 | 679 | colType, err := table.ColTypeByColIndex(colIndex) 680 | if err != nil { 681 | return emptyTemplateInfo, err 682 | } 683 | 684 | cols[colIndex].IsDeprecated = isDeprecated(colName) 685 | if cols[colIndex].IsDeprecated { 686 | // Restore the col name by removing _DEPRECATED_ indicator. 687 | colName = strings.Replace(colName, deprecated, "", 1) 688 | _, _ = fmt.Fprintf(os.Stderr, "*** FlatTables: Tagged table [%s] column %q is deprecated\n", table.Name(), colName) 689 | } 690 | 691 | cols[colIndex].ColName = colName 692 | cols[colIndex].ColType = colType 693 | cols[colIndex].FbsType, err = schemaType(colType) 694 | if err != nil { 695 | return emptyTemplateInfo, err 696 | } 697 | cols[colIndex].ColIndex = colIndex 698 | cols[colIndex].IsScalar = IsFlatBuffersScalar(colType) // FlatBuffers Scalar includes bool 699 | cols[colIndex].IsString = colType == "string" 700 | cols[colIndex].IsBool = colType == "bool" 701 | } 702 | 703 | // Populate Rows with a string representation of each table cell. 704 | var rows []Row = make([]Row, table.RowCount()) 705 | // where(fmt.Sprintf("RowCount = %d", table.RowCount())) 706 | for rowIndex := 0; rowIndex < table.RowCount(); rowIndex++ { 707 | var row []string = make([]string, table.ColCount()) 708 | for colIndex := 0; colIndex < table.ColCount(); colIndex++ { 709 | var cell string 710 | cell, err = table.GetValAsStringByColIndex(colIndex, rowIndex) 711 | if err != nil { 712 | return emptyTemplateInfo, err 713 | } 714 | var isStringType bool 715 | isStringType, err = table.IsColTypeByColIndex(colIndex, "string") 716 | if err != nil { 717 | return emptyTemplateInfo, err 718 | } 719 | if isStringType { 720 | cell = fmt.Sprintf("%q", cell) // Add delimiters. 721 | } 722 | row[colIndex] = cell 723 | } 724 | rows[rowIndex] = row 725 | // where(fmt.Sprintf("row[%d] = %v", rowIndex, rows[rowIndex])) 726 | } 727 | 728 | var colNames []string = make([]string, table.ColCount()) 729 | var colTypes []string = make([]string, table.ColCount()) 730 | for colIndex := 0; colIndex < table.ColCount(); colIndex++ { 731 | colName, err := table.ColName(colIndex) 732 | if err != nil { 733 | return emptyTemplateInfo, err 734 | } 735 | colNames[colIndex] = colName 736 | 737 | colType, err := table.ColTypeByColIndex(colIndex) 738 | if err != nil { 739 | return emptyTemplateInfo, err 740 | } 741 | colTypes[colIndex] = colType 742 | } 743 | 744 | tables[tableIndex].Cols = cols 745 | tables[tableIndex].TableIndex = tableIndex 746 | tables[tableIndex].TableName = table.Name() 747 | tables[tableIndex].RowCount = table.RowCount() 748 | tables[tableIndex].ColCount = table.ColCount() 749 | tables[tableIndex].Rows = rows 750 | tables[tableIndex].ColNames = colNames 751 | tables[tableIndex].ColTypes = colTypes 752 | } 753 | 754 | // Get tableset metadata. 755 | // Make a copy of the tables and use them as metadata-only. 756 | // We end up with 2 instances of TableSet:- 757 | // (1) tableSet which contains data. Is accessible in templates as: .Tables (an array of Table) 758 | // (2) metadataTableSet which contains NO data. Is accessible in templates as: .TableSetMetadata (a TableSet) 759 | 760 | const copyRows = false // i.e., don't copy rows. 761 | metadataTableSet, err := tableSet.Copy(copyRows) // Accessible as 762 | if err != nil { 763 | return emptyTemplateInfo, err 764 | } 765 | 766 | for tableIndex := 0; tableIndex < metadataTableSet.TableCount(); tableIndex++ { 767 | table, err := metadataTableSet.TableByTableIndex(tableIndex) 768 | if err != nil { 769 | return emptyTemplateInfo, err 770 | } 771 | 772 | err = table.SetStructShape(true) 773 | if err != nil { 774 | return emptyTemplateInfo, err 775 | } 776 | } 777 | 778 | tableSetMetadata := metadataTableSet.String() 779 | tableSetMetadata = indentText("\t\t", tableSetMetadata) 780 | 781 | tableSetData := tableSet.String() 782 | // tableSetData = indentText("\t", tableSetData) 783 | 784 | tablesTemplateInfo = TablesTemplateInfoType{ 785 | GeneratedDateFromFile: generatedDateFromFile(tableSet), 786 | GeneratedDateFromFileBaseName: filepath.Base(generatedDateFromFile(tableSet)), 787 | GeneratedFromFile: generatedFromFile(tableSet), 788 | UsingCommand: usingCommand(tableSet, packageName), 789 | UsingCommandMinusG: usingCommandMinusG(tableSet, packageName), 790 | GotablesFileNameAbsolute: tableSet.FileName(), 791 | GotablesFileNameBase: filepath.Base(tableSet.FileName()), 792 | Year: copyrightYear(), 793 | NameSpace: tableSet.Name(), 794 | PackageName: packageName, 795 | TableSetMetadata: tableSetMetadata, 796 | TableSetData: tableSetData, 797 | Tables: tables, 798 | } 799 | 800 | return tablesTemplateInfo, nil 801 | } 802 | 803 | // Assumes flattables.RemoveEmptyTables() has been called first. 804 | // THIS NEEDS TO ADD TO, NOT REPLACE, EXISTING TEMPLATE INFORMATION. 805 | func InitRelationsTemplateInfo(tableSet *gotables.TableSet, packageName string) (TablesTemplateInfoType, error) { 806 | 807 | var emptyTemplateInfo TablesTemplateInfoType 808 | var tablesTemplateInfo TablesTemplateInfoType 809 | 810 | var tables []TableInfo = make([]TableInfo, tableSet.TableCount()) 811 | for tableIndex := 0; tableIndex < tableSet.TableCount(); tableIndex++ { 812 | table, err := tableSet.TableByTableIndex(tableIndex) 813 | if err != nil { 814 | return emptyTemplateInfo, err 815 | } 816 | 817 | if table.ColCount() > 0 { 818 | _, _ = fmt.Fprintf(os.Stderr, " Adding gotables table %2d to FlatBuffers schema: [%s] \n", tableIndex, table.Name()) 819 | } else { 820 | // Skip tables with zero cols. 821 | return emptyTemplateInfo, fmt.Errorf("--- FlatTables: table [%s] has no cols", table.Name()) 822 | } 823 | 824 | if startsWithLowerCase(table.Name()) { 825 | // See: https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html 826 | return emptyTemplateInfo, fmt.Errorf("the FlatBuffers style guide requires UpperCamelCase table names. Rename [%s] to [%s]", 827 | table.Name(), firstCharToUpper(table.Name())) 828 | } 829 | 830 | if isGoKeyword(table.Name()) { 831 | return emptyTemplateInfo, 832 | fmt.Errorf("cannot use a Go key word as a table name, even if it's upper case. Rename [%s]", table.Name()) 833 | } 834 | 835 | if isFlatBuffersOrFlatTablesKeyWord(table.Name()) { 836 | return emptyTemplateInfo, 837 | fmt.Errorf("cannot use a FlatBuffers or FlatTables key word as a table name, even if it's merely similar. Rename [%s]", 838 | table.Name()) 839 | } 840 | 841 | // I don't see documentation on this, but undescores in field names affect code generation. 842 | if strings.ContainsRune(table.Name(), '_') { 843 | return emptyTemplateInfo, 844 | fmt.Errorf("cannot use underscores '_' in table names. Rename [%s]", table.Name()) 845 | } 846 | 847 | tables[tableIndex].Table = table // An array of Table accessible as .Tables 848 | 849 | var cols []ColInfo = make([]ColInfo, table.ColCount()) 850 | for colIndex := 0; colIndex < table.ColCount(); colIndex++ { 851 | colName, err := table.ColNameByColIndex(colIndex) 852 | if err != nil { 853 | return emptyTemplateInfo, err 854 | } 855 | 856 | // Enforce FlatBuffers style guide. 857 | if startsWithUpperCase(colName) { 858 | // See: https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html 859 | return emptyTemplateInfo, fmt.Errorf("#2 the FlatBuffers style guide requires lowerCamelCase field names. In table [%s] rename %s to %s", 860 | table.Name(), colName, firstCharToLower(colName)) 861 | } 862 | 863 | if isGoKeyword(colName) { 864 | return emptyTemplateInfo, fmt.Errorf("cannot use a Go key word as a col name, even if it's upper case. Rename: %s", colName) 865 | } 866 | 867 | if isFlatBuffersOrFlatTablesKeyWord(colName) { 868 | return emptyTemplateInfo, 869 | fmt.Errorf("cannot use a FlatBuffers or FlatTables key word as a col name, even if it's merely similar. Rename: %s", colName) 870 | } 871 | 872 | // I don't see documentation on this, but undescores in field names affect code generation. 873 | if strings.ContainsRune(colName, '_') { 874 | return emptyTemplateInfo, 875 | fmt.Errorf("cannot use underscores '_' in col names. Rename %s", colName) 876 | } 877 | 878 | colType, err := table.ColTypeByColIndex(colIndex) 879 | if err != nil { 880 | return emptyTemplateInfo, err 881 | } 882 | 883 | cols[colIndex].IsDeprecated = isDeprecated(colName) 884 | if cols[colIndex].IsDeprecated { 885 | // Restore the col name by removing _DEPRECATED_ indicator. 886 | colName = strings.Replace(colName, deprecated, "", 1) 887 | _, _ = fmt.Fprintf(os.Stderr, "*** FlatTables: Tagged table [%s] column %q is deprecated\n", table.Name(), colName) 888 | } 889 | 890 | cols[colIndex].ColName = colName 891 | cols[colIndex].ColType = colType 892 | cols[colIndex].FbsType, err = schemaType(colType) 893 | if err != nil { 894 | return emptyTemplateInfo, err 895 | } 896 | cols[colIndex].ColIndex = colIndex 897 | cols[colIndex].IsScalar = IsFlatBuffersScalar(colType) // FlatBuffers Scalar includes bool 898 | cols[colIndex].IsString = colType == "string" 899 | cols[colIndex].IsBool = colType == "bool" 900 | } 901 | 902 | // Populate Rows with a string representation of each table cell. 903 | var rows []Row = make([]Row, table.RowCount()) 904 | // where(fmt.Sprintf("RowCount = %d", table.RowCount())) 905 | for rowIndex := 0; rowIndex < table.RowCount(); rowIndex++ { 906 | var row []string = make([]string, table.ColCount()) 907 | for colIndex := 0; colIndex < table.ColCount(); colIndex++ { 908 | var cell string 909 | cell, err = table.GetValAsStringByColIndex(colIndex, rowIndex) 910 | if err != nil { 911 | return emptyTemplateInfo, err 912 | } 913 | var isStringType bool 914 | isStringType, err = table.IsColTypeByColIndex(colIndex, "string") 915 | if err != nil { 916 | return emptyTemplateInfo, err 917 | } 918 | if isStringType { 919 | cell = fmt.Sprintf("%q", cell) // Add delimiters. 920 | } 921 | row[colIndex] = cell 922 | } 923 | rows[rowIndex] = row 924 | // where(fmt.Sprintf("row[%d] = %v", rowIndex, rows[rowIndex])) 925 | } 926 | 927 | var colNames []string = make([]string, table.ColCount()) 928 | var colTypes []string = make([]string, table.ColCount()) 929 | for colIndex := 0; colIndex < table.ColCount(); colIndex++ { 930 | colName, err := table.ColName(colIndex) 931 | if err != nil { 932 | return emptyTemplateInfo, err 933 | } 934 | colNames[colIndex] = colName 935 | 936 | colType, err := table.ColTypeByColIndex(colIndex) 937 | if err != nil { 938 | return emptyTemplateInfo, err 939 | } 940 | colTypes[colIndex] = colType 941 | } 942 | 943 | tables[tableIndex].Cols = cols 944 | tables[tableIndex].TableIndex = tableIndex 945 | tables[tableIndex].TableName = table.Name() 946 | tables[tableIndex].RowCount = table.RowCount() 947 | tables[tableIndex].ColCount = table.ColCount() 948 | tables[tableIndex].Rows = rows 949 | tables[tableIndex].ColNames = colNames 950 | tables[tableIndex].ColTypes = colTypes 951 | } 952 | 953 | // Get tableset metadata. 954 | // Make a copy of the tables and use them as metadata-only. 955 | // We end up with 2 instances of TableSet:- 956 | // (1) tableSet which contains data. Is accessible in templates as: .Tables (an array of Table) 957 | // (2) metadataTableSet which contains NO data. Is accessible in templates as: .TableSetMetadata (a TableSet) 958 | 959 | const copyRows = false // i.e., don't copy rows. 960 | metadataTableSet, err := tableSet.Copy(copyRows) // Accessible as 961 | if err != nil { 962 | return emptyTemplateInfo, err 963 | } 964 | 965 | for tableIndex := 0; tableIndex < metadataTableSet.TableCount(); tableIndex++ { 966 | table, err := metadataTableSet.TableByTableIndex(tableIndex) 967 | if err != nil { 968 | return emptyTemplateInfo, err 969 | } 970 | 971 | err = table.SetStructShape(true) 972 | if err != nil { 973 | return emptyTemplateInfo, err 974 | } 975 | } 976 | 977 | tableSetMetadata := metadataTableSet.String() 978 | tableSetMetadata = indentText("\t\t", tableSetMetadata) 979 | 980 | tableSetData := tableSet.String() 981 | // tableSetData = indentText("\t", tableSetData) 982 | 983 | tablesTemplateInfo = TablesTemplateInfoType{ 984 | GeneratedDateFromFile: generatedDateFromFile(tableSet), 985 | GeneratedFromFile: generatedFromFile(tableSet), 986 | UsingCommand: usingCommand(tableSet, packageName), 987 | UsingCommandMinusG: usingCommandMinusG(tableSet, packageName), 988 | GotablesFileNameAbsolute: tableSet.FileName(), 989 | Year: copyrightYear(), 990 | NameSpace: tableSet.Name(), 991 | PackageName: packageName, 992 | TableSetMetadata: tableSetMetadata, 993 | TableSetData: tableSetData, 994 | Tables: tables, 995 | } 996 | 997 | return tablesTemplateInfo, nil 998 | } 999 | 1000 | func copyrightYear() (copyrightYear string) { 1001 | firstYear := "2017" // See github dates. 1002 | copyrightYear = fmt.Sprintf("%s-%s", firstYear, thisYear()) 1003 | return 1004 | } 1005 | 1006 | func yearRangeFromFirstYear(firstYear string) (yearRange string) { 1007 | thisYear := thisYear() 1008 | if firstYear == thisYear { 1009 | yearRange = firstYear 1010 | } else { 1011 | yearRange = fmt.Sprintf("%s-%s", firstYear, thisYear) 1012 | } 1013 | return 1014 | } 1015 | 1016 | func thisYear() (thisYear string) { 1017 | thisYear = time.Now().Format("2006") 1018 | return 1019 | } 1020 | 1021 | func generatedDateFromFile(tableSet *gotables.TableSet) string { 1022 | return fmt.Sprintf("Generated %s from your gotables file %s", time.Now().Format("Monday 2 Jan 2006"), tableSet.FileName()) 1023 | } 1024 | 1025 | func generatedFromFile(tableSet *gotables.TableSet) string { 1026 | return tableSet.FileName() 1027 | } 1028 | 1029 | func usingCommand(tableSet *gotables.TableSet, packageName string) string { 1030 | var usingCommand string 1031 | 1032 | // Sample: 1033 | // flattablesc -v -f ../flattables_sample/tables.got -n flattables_sample -p package_name 1034 | 1035 | nameSpace := tableSet.Name() 1036 | fileName := filepath.Base(tableSet.FileName()) 1037 | 1038 | indent := "\t" 1039 | usingCommand = "using the following command:\n" 1040 | usingCommand += indentText(indent, fmt.Sprintf("$ cd %s\t# Where you defined your tables in file %s\n", nameSpace, fileName)) 1041 | usingCommand += indentText(indent, fmt.Sprintf("$ flattablesc -v -f ../%s/%s -n %s -p %s\n", 1042 | nameSpace, fileName, nameSpace, packageName)) 1043 | usingCommand += indentText(indent, "See instructions at: https://github.com/urban-wombat/flattables") 1044 | 1045 | return usingCommand 1046 | } 1047 | 1048 | func usingCommandMinusG(tableSet *gotables.TableSet, packageName string) string { 1049 | var usingCommand string 1050 | 1051 | // Sample: 1052 | // flattablesc -v -g -f ../flattables_sample/tables.got -n flattables_sample -p package_name 1053 | 1054 | nameSpace := tableSet.Name() 1055 | fileName := filepath.Base(tableSet.FileName()) 1056 | 1057 | indent := "\t" 1058 | usingCommand = "using the following command:\n" 1059 | usingCommand += indentText(indent, fmt.Sprintf("$ cd %s\t# Where you defined your tables in file %s\n", nameSpace, fileName)) 1060 | usingCommand += indentText(indent, fmt.Sprintf("$ flattablesc -v -g -f ../%s/%s -n %s -p %s\n", 1061 | nameSpace, fileName, nameSpace, packageName)) 1062 | usingCommand += indentText(indent, "See instructions at: https://github.com/urban-wombat/flattables") 1063 | 1064 | return usingCommand 1065 | } 1066 | 1067 | type removeStruct struct { 1068 | replace string 1069 | with string 1070 | id string 1071 | count int 1072 | } 1073 | 1074 | var rmstr = []removeStruct{ 1075 | {"\r\n", "\n", "rn", 0}, // Remove ^M 1076 | // { "\r", "", "r", 0 }, // Maybe replace by rn 1077 | {"\n\n\n", "\n\n", "02", 0}, 1078 | {"\n\n\n", "\n\n", "03", 0}, 1079 | {"\n\t\n", "\n\n", "04", 0}, 1080 | {"\n\n}", "\n}", "05", 0}, 1081 | {"\n\n)", "\n)", "06", 0}, 1082 | {"\n\n\n\n", "\n\n", "07", 0}, 1083 | {"\n\n\n", "\n\n", "08", 0}, 1084 | {"\n\t\t\n", "\n\n", "09", 0}, 1085 | {"\n\t\t\t\n", "\n", "10", 0}, 1086 | {"\n\t\t\t\n", "\n", "11", 0}, 1087 | {"\t\n", "", "12", 0}, 1088 | {"\t\n", "", "13", 0}, 1089 | {"\n\n\n\n", "\n\n", "14", 0}, 1090 | {"\n\n\n\n", "\n\n", "15", 0}, 1091 | {"\n\n\n", "\n\n", "16", 0}, 1092 | {"\n\n}", "\n}", "17", 0}, 1093 | {"\n\n\t\t}", "\n\t}", "18", 0}, 1094 | {"\n\n\t}", "\n\t}", "19", 0}, 1095 | {"\n \n)", "\n)", "20", 0}, 1096 | {"\t\t\t\t\t\t}", "\t\t\t}", "21", 0}, 1097 | {"{\n\n", "{\n", "22", 0}, 1098 | {"{\n\n", "{\n", "22", 0}, 1099 | {"\t\n}", "}", "23", 0}, // Why doesn't this do anything? 1100 | } 1101 | 1102 | func RemoveExcessTabsAndNewLines(code string) string { 1103 | // Use cat -A flattables_sample_flattables.go to detect non-printing characters. 1104 | 1105 | for i := 0; i < len(rmstr); i++ { 1106 | var codeIn = code 1107 | code = strings.Replace(code, rmstr[i].replace, rmstr[i].with, -1) 1108 | if code != codeIn { 1109 | rmstr[i].count++ 1110 | } 1111 | } 1112 | 1113 | var verbose bool = false 1114 | if verbose { 1115 | fmt.Println() 1116 | 1117 | // Used 1118 | for i := 0; i < len(rmstr); i++ { 1119 | if rmstr[i].count > 0 { 1120 | fmt.Printf(" Used %d times id %q\n", rmstr[i].count, rmstr[i].id) 1121 | } 1122 | } 1123 | 1124 | // Unused 1125 | for i := 0; i < len(rmstr); i++ { 1126 | if rmstr[i].count == 0 { 1127 | fmt.Printf("UNUSED %d times id %q\n", rmstr[i].count, rmstr[i].id) 1128 | } 1129 | } 1130 | } 1131 | 1132 | return code 1133 | } 1134 | -------------------------------------------------------------------------------- /flattables_test.go: -------------------------------------------------------------------------------- 1 | package flattables 2 | 3 | import ( 4 | "go/token" 5 | "testing" 6 | ) 7 | 8 | func TestIsGoKeyword(t *testing.T) { 9 | var tests = []struct { 10 | maybeKeyword string 11 | isKeyword bool 12 | }{ 13 | {"break", true}, 14 | {"case", true}, 15 | {"chan", true}, 16 | {"const", true}, 17 | {"continue", true}, 18 | {"default", true}, 19 | {"defer", true}, 20 | {"else", true}, 21 | {"fallthrough", true}, 22 | {"for", true}, 23 | {"func", true}, 24 | {"go", true}, 25 | {"goto", true}, 26 | {"if", true}, 27 | {"import", true}, 28 | {"interface", true}, 29 | {"map", true}, 30 | {"package", true}, 31 | {"range", true}, 32 | {"return", true}, 33 | {"select", true}, 34 | {"struct", true}, 35 | {"switch", true}, 36 | {"type", true}, 37 | {"var", true}, 38 | {"int", false}, 39 | } 40 | 41 | for i, test := range tests { 42 | var result bool = token.Lookup(test.maybeKeyword).IsKeyword() 43 | 44 | if result != test.isKeyword && test.isKeyword == true { 45 | t.Errorf("test[%d] expected %s to be a Go keyword, but it's not", i, test.maybeKeyword) 46 | } 47 | 48 | if result != test.isKeyword && test.isKeyword == false { 49 | t.Errorf("test[%d] expected %s to NOT be a Go keyword, but it is", i, test.maybeKeyword) 50 | } 51 | } 52 | } 53 | --------------------------------------------------------------------------------