├── testdata ├── empty.json ├── test_simple_json.json ├── test_invalid_field_chars.json ├── test_simple_array.json ├── test_nested_json.json ├── test_mixed_nulls.json ├── test_nullable_json.json ├── test_mixed_nulls.go ├── test_simple_json.go ├── test_invalid_field_chars.go ├── test_repeated_json.json ├── test_simple_array.go ├── test_nullable_json.go ├── test_nested_json.go ├── test_repeated_json.go ├── template-header.txt ├── roundtrip_edge_cases.txtar ├── simple_roundtrip.txtar ├── more_complex_example.json ├── ndjson-support.txtar ├── more_complex_example.go ├── extract-structs.txtar ├── ndjson.txtar ├── nested_result.txtar ├── all.txtar ├── template-roundtrip-test.txt ├── large-payload.txtar ├── multi-eslog.txtar └── exotic.txtar ├── docs ├── json-to-struct.wasm ├── index.html └── wasm_exec.js ├── .gitignore ├── .github ├── slsa-configs │ ├── darwin-amd64.yml │ ├── darwin-arm64.yml │ ├── linux-amd64.yml │ ├── linux-arm64.yml │ ├── windows-amd64.yml │ └── windows-arm64.yml └── workflows │ └── release.yml ├── txtar_legacy_test.go ├── txtar_mode_default_test.go ├── go.mod ├── json-to-struct_test.go ├── main_js.go ├── go.sum ├── templates.txt ├── Makefile ├── CHANGELOG.md ├── templates_legacy.go ├── doc.go ├── README.md ├── generate_legacy.go ├── main.go ├── extract.go ├── stream.go ├── txtar_test.go ├── type.go ├── roundtrip.go └── generate.go /testdata/empty.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/test_simple_json.json: -------------------------------------------------------------------------------- 1 | {"f.o-o" : 42} 2 | -------------------------------------------------------------------------------- /testdata/test_invalid_field_chars.json: -------------------------------------------------------------------------------- 1 | {"foo" : "bar"} 2 | -------------------------------------------------------------------------------- /testdata/test_simple_array.json: -------------------------------------------------------------------------------- 1 | {"foo" : "bar", "baz" : null} 2 | -------------------------------------------------------------------------------- /testdata/test_nested_json.json: -------------------------------------------------------------------------------- 1 | {"foo" : {"bar": 24}, "baz" : [42,43]} 2 | -------------------------------------------------------------------------------- /testdata/test_mixed_nulls.json: -------------------------------------------------------------------------------- 1 | [{"bar" : 85},{"bar" : null},{"bar" : 81}] 2 | -------------------------------------------------------------------------------- /testdata/test_nullable_json.json: -------------------------------------------------------------------------------- 1 | {"foo" : [{"bar": 24}, {"bar" : 42}]} 2 | -------------------------------------------------------------------------------- /docs/json-to-struct.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tmc/json-to-struct/HEAD/docs/json-to-struct.wasm -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.swo 3 | *.swn 4 | *.db 5 | *.log 6 | build/ 7 | conf.go 8 | gojson 9 | json-to-struct 10 | -------------------------------------------------------------------------------- /.github/slsa-configs/darwin-amd64.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | goos: darwin 3 | goarch: amd64 4 | binary: json-to-struct-darwin-amd64 5 | -------------------------------------------------------------------------------- /.github/slsa-configs/darwin-arm64.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | goos: darwin 3 | goarch: arm64 4 | binary: json-to-struct-darwin-arm64 5 | -------------------------------------------------------------------------------- /.github/slsa-configs/linux-amd64.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | goos: linux 3 | goarch: amd64 4 | binary: json-to-struct-linux-amd64 5 | -------------------------------------------------------------------------------- /.github/slsa-configs/linux-arm64.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | goos: linux 3 | goarch: arm64 4 | binary: json-to-struct-linux-arm64 5 | -------------------------------------------------------------------------------- /.github/slsa-configs/windows-amd64.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | goos: windows 3 | goarch: amd64 4 | binary: json-to-struct-windows-amd64.exe 5 | -------------------------------------------------------------------------------- /.github/slsa-configs/windows-arm64.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | goos: windows 3 | goarch: arm64 4 | binary: json-to-struct-windows-arm64.exe 5 | -------------------------------------------------------------------------------- /testdata/test_mixed_nulls.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type test_mixed_nulls struct { 4 | Bar *float64 `json:"bar,omitempty"` 5 | } 6 | -------------------------------------------------------------------------------- /testdata/test_simple_json.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type test_simple_json struct { 4 | F_O_O float64 `json:"f.o-o,omitempty"` 5 | } 6 | -------------------------------------------------------------------------------- /testdata/test_invalid_field_chars.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type test_invalid_field_chars struct { 4 | Foo string `json:"foo,omitempty"` 5 | } 6 | -------------------------------------------------------------------------------- /txtar_legacy_test.go: -------------------------------------------------------------------------------- 1 | //go:build legacy 2 | 3 | package main 4 | 5 | // legacyMode is true when legacy build tag is present 6 | const legacyMode = true 7 | -------------------------------------------------------------------------------- /testdata/test_repeated_json.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"foo" : 42}, 3 | {"foo" : 42, "bar": 22}, 4 | {"foo" : 42, "baz": {}}, 5 | {"foo" : 42, "baz": {"zap": true}} 6 | ] 7 | -------------------------------------------------------------------------------- /txtar_mode_default_test.go: -------------------------------------------------------------------------------- 1 | //go:build !legacy 2 | 3 | package main 4 | 5 | // legacyMode is false when legacy build tag is not present 6 | const legacyMode = false 7 | -------------------------------------------------------------------------------- /testdata/test_simple_array.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type test_simple_array struct { 4 | Baz any `json:"baz,omitempty"` 5 | Foo string `json:"foo,omitempty"` 6 | } 7 | -------------------------------------------------------------------------------- /testdata/test_nullable_json.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type test_nullable_json struct { 4 | Foo []struct { 5 | Bar float64 `json:"bar,omitempty"` 6 | } `json:"foo,omitempty"` 7 | } 8 | -------------------------------------------------------------------------------- /testdata/test_nested_json.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type test_nested_json struct { 4 | Baz []float64 `json:"baz,omitempty"` 5 | Foo struct { 6 | Bar float64 `json:"bar,omitempty"` 7 | } `json:"foo,omitempty"` 8 | } 9 | -------------------------------------------------------------------------------- /testdata/test_repeated_json.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type test_repeated_json struct { 4 | Foo float64 `json:"foo,omitempty"` 5 | Bar float64 `json:"bar,omitempty"` 6 | Baz struct { 7 | Zap bool `json:"zap,omitempty"` 8 | } `json:"baz,omitempty"` 9 | } 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tmc/json-to-struct 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.6 6 | 7 | require ( 8 | github.com/google/go-cmp v0.6.0 9 | golang.org/x/term v0.35.0 10 | golang.org/x/tools v0.37.0 11 | ) 12 | 13 | require golang.org/x/sys v0.36.0 // indirect 14 | -------------------------------------------------------------------------------- /json-to-struct_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Legacy table-driven test harness has been replaced with txtar-based tests. 4 | // See txtar_test.go for the new test approach. 5 | // 6 | // To run tests: 7 | // go test # Run all txtar tests 8 | // go test -write-txtar-golden # Update golden files 9 | // go test -force-legacy-pattern=X # Force legacy mode for files matching pattern X 10 | -------------------------------------------------------------------------------- /main_js.go: -------------------------------------------------------------------------------- 1 | //go:build js 2 | // +build js 3 | 4 | package main 5 | 6 | import ( 7 | "strings" 8 | "syscall/js" 9 | ) 10 | 11 | func jsonToStructFunction(this js.Value, p []js.Value) any { 12 | in := strings.NewReader(p[0].String()) 13 | if output, err := generate(in, "Type", "main", &generator{}); err != nil { 14 | return js.ValueOf(err.Error()) 15 | } else { 16 | return js.ValueOf(string(output)) 17 | } 18 | return js.Null() 19 | } 20 | 21 | func main() { 22 | c := make(chan struct{}, 0) 23 | 24 | js.Global().Set("jsonToStruct", js.FuncOf(jsonToStructFunction)) 25 | 26 | <-c 27 | } 28 | -------------------------------------------------------------------------------- /testdata/template-header.txt: -------------------------------------------------------------------------------- 1 | -- file.tmpl -- 2 | // Code generated by json-to-struct; DO NOT EDIT. 3 | package {{.Package}} 4 | {{if .Imports}} 5 | import ( 6 | {{range .Imports}} "{{.}}" 7 | {{end}}) 8 | {{end}} 9 | {{.Content}} 10 | -- type.tmpl -- 11 | {{if eq .Type "struct"}}// {{.Name}} represents the JSON structure 12 | type {{.Name}} {{.GetType}} { 13 | {{range .Children}} {{.Name}} {{if eq .Type "struct"}}{{if .Repeated}}[]{{end}}struct { 14 | {{range .Children}} {{.Name}} {{.GetType}} {{.GetTags}} 15 | {{end}} }{{else}}{{.GetType}}{{end}} {{.GetTags}} 16 | {{end}}} {{.GetTags}}{{else}}type {{.Name}} {{.GetType}} {{.GetTags}}{{end}} -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 2 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 3 | golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= 4 | golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 5 | golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= 6 | golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= 7 | golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= 8 | golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= 9 | -------------------------------------------------------------------------------- /templates.txt: -------------------------------------------------------------------------------- 1 | -- file.tmpl -- 2 | package {{.Package}} 3 | {{if .Imports}} 4 | import ( 5 | {{range .Imports}} "{{.}}" 6 | {{end}}) 7 | {{end}} 8 | {{.Content}} 9 | -- type.tmpl -- 10 | {{if eq .Type "struct"}}type {{.Name}} struct { 11 | {{range .Children}} {{.Name}} {{if or (eq .Type "struct") (eq .Type "*struct")}}{{RenderInlineStruct . 1}}{{else}}{{.GetType}}{{end}} {{.GetTags}} 12 | {{end}}}{{else}}type {{.Name}} {{.GetType}} {{.GetTags}}{{end}} 13 | -- type-with-stats.tmpl -- 14 | {{if eq .Type "struct"}}type {{.Name}} struct { 15 | {{range .Children}} {{.Name}} {{if or (eq .Type "struct") (eq .Type "*struct")}}{{RenderInlineStruct . 1}}{{else}}{{.GetType}}{{end}} {{.GetTags}}{{.GetStatComment}} 16 | {{end}}}{{else}}type {{.Name}} {{.GetType}} {{.GetTags}}{{end}} 17 | -------------------------------------------------------------------------------- /testdata/roundtrip_edge_cases.txtar: -------------------------------------------------------------------------------- 1 | # Edge cases for roundtrip validation 2 | # Tests complex JSON structures and field variations 3 | 4 | -- nested.json -- 5 | {"user":{"name":"John","age":30},"tags":["admin","user"],"metadata":{"created":"2024-01-01","updated":null}} 6 | 7 | -- nested.go -- 8 | package test_package 9 | 10 | type nested struct { 11 | Metadata struct { 12 | Created string `json:"created,omitempty"` 13 | Updated any `json:"updated,omitempty"` 14 | } `json:"metadata,omitempty"` 15 | Tags []string `json:"tags,omitempty"` 16 | User struct { 17 | Age float64 `json:"age,omitempty"` 18 | Name string `json:"name,omitempty"` 19 | } `json:"user,omitempty"` 20 | } 21 | -- nested.roundtrip -- 22 | ✓ Round-trip: 1/1 records validated successfully 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - "v*" 6 | 7 | permissions: read-all 8 | 9 | jobs: 10 | build: 11 | strategy: 12 | matrix: 13 | include: 14 | - {os: linux, arch: amd64} 15 | - {os: linux, arch: arm64} 16 | - {os: darwin, arch: amd64} 17 | - {os: darwin, arch: arm64} 18 | - {os: windows, arch: amd64} 19 | - {os: windows, arch: arm64} 20 | permissions: {id-token: write, contents: write, actions: read} 21 | uses: slsa-framework/slsa-github-generator/.github/workflows/builder_go_slsa3.yml@v2.0.0 22 | with: 23 | go-version: "1.24" 24 | upload-assets: true 25 | config-file: .github/slsa-configs/${{ matrix.os }}-${{ matrix.arch }}.yml 26 | private-repository: true 27 | draft-release: true 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | all: install docs 3 | 4 | .PHONY: install 5 | install: 6 | go install 7 | 8 | .PHONY: docs 9 | docs: 10 | @LATEST_TAG=$$(gh release list --exclude-pre-releases -L1 --json tagName --jq '.[0].tagName'); \ 11 | SHAS=$$(gh api repos/tmc/json-to-struct/releases/tags/$$LATEST_TAG --jq '.assets[] | select(.name | startswith("json-to-struct-") and contains("amd64") and (endswith(".jsonl")|not)) | .name + " " + .digest' | sed 's/sha256://'); \ 12 | HELP="$$(json-to-struct -h 2>&1)" \ 13 | LATEST_TAG="$$LATEST_TAG" \ 14 | LINUX_SHASUM=$$(echo "$$SHAS" | awk '/linux/{print $$2}') \ 15 | MACOS_SHASUM=$$(echo "$$SHAS" | awk '/darwin/{print $$2}') \ 16 | tmpl < README.in > README.md 17 | 18 | .PHONY: wasm 19 | wasm: docs/json-to-struct.wasm 20 | 21 | docs/json-to-struct.wasm: *.go 22 | @cp "$(shell go env GOROOT)/misc/wasm/wasm_exec.js" docs/ 23 | @GOOS=js GOARCH=wasm go build -o $@ 24 | -------------------------------------------------------------------------------- /testdata/simple_roundtrip.txtar: -------------------------------------------------------------------------------- 1 | # Test for roundtrip validation 2 | # This test ensures that the generated struct can properly round-trip the input JSON 3 | 4 | -- simple.json -- 5 | {"name":"John","age":30,"active":true} 6 | 7 | -- simple.go -- 8 | package test_package 9 | 10 | type simple struct { 11 | Active bool `json:"active,omitempty"` 12 | Age float64 `json:"age,omitempty"` 13 | Name string `json:"name,omitempty"` 14 | } 15 | -- simple.roundtrip -- 16 | ✓ Round-trip: 1/1 records validated successfully 17 | Successfully round-tripped 1/1 single objects 18 | 19 | -- array.json -- 20 | [{"name":"John","age":30},{"name":"Jane","age":null,"active":false}] 21 | 22 | -- array.go -- 23 | package test_package 24 | 25 | type array struct { 26 | Active bool `json:"active,omitempty"` 27 | Age *float64 `json:"age,omitempty"` 28 | Name string `json:"name,omitempty"` 29 | } 30 | -- array.roundtrip -- 31 | ✓ Round-trip: 2/1 records validated successfully 32 | -------------------------------------------------------------------------------- /testdata/more_complex_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "login": "tmc", 3 | "id": 3977, 4 | "avatar_url": "https://1.gravatar.com/avatar/68f0049842700597b89972e1fbf6f542?d=https%3A%2F%2Fidenticons.github.com%2F7d571e5c15bad5ef8c4352ce7a1d9e78.png", 5 | "gravatar_id": "68f0049842700597b89972e1fbf6f542", 6 | "url": "https://api.github.com/users/tmc", 7 | "html_url": "https://github.com/tmc", 8 | "followers_url": "https://api.github.com/users/tmc/followers", 9 | "following_url": "https://api.github.com/users/tmc/following{/other_user}", 10 | "gists_url": "https://api.github.com/users/tmc/gists{/gist_id}", 11 | "starred_url": "https://api.github.com/users/tmc/starred{/owner}{/repo}", 12 | "subscriptions_url": "https://api.github.com/users/tmc/subscriptions", 13 | "organizations_url": "https://api.github.com/users/tmc/orgs", 14 | "repos_url": "https://api.github.com/users/tmc/repos", 15 | "events_url": "https://api.github.com/users/tmc/events{/privacy}", 16 | "received_events_url": "https://api.github.com/users/tmc/received_events", 17 | "type": "User", 18 | "name": "Travis Cline", 19 | "company": "Facebook", 20 | "blog": "", 21 | "location": "lawrence, ks", 22 | "email": "travis.cline@gmail.com", 23 | "hireable": true, 24 | "bio": null, 25 | "public_repos": 87, 26 | "followers": 103, 27 | "following": 88, 28 | "created_at": "2008-03-27T15:49:13Z", 29 | "updated_at": "2013-09-05T00:03:43Z", 30 | "public_gists": 44 31 | } 32 | -------------------------------------------------------------------------------- /testdata/ndjson-support.txtar: -------------------------------------------------------------------------------- 1 | # Test NDJSON (newline-delimited JSON) support 2 | # NDJSON is common in log files and streaming data 3 | 4 | -- simple_ndjson.json -- 5 | {"name": "Alice", "age": 30, "city": "New York"} 6 | {"name": "Bob", "age": 25, "city": "San Francisco"} 7 | {"name": "Charlie", "age": 35, "city": "Chicago"} 8 | 9 | -- simple_ndjson.go -- 10 | package test_package 11 | 12 | type simple_ndjson struct { 13 | Age float64 `json:"age,omitempty"` 14 | City string `json:"city,omitempty"` 15 | Name string `json:"name,omitempty"` 16 | } 17 | -- mixed_types_ndjson.json -- 18 | {"id": 1, "type": "user", "data": {"username": "alice", "email": "alice@example.com"}} 19 | {"id": 2, "type": "product", "data": {"name": "Widget", "price": 19.99}} 20 | {"id": 3, "type": "user", "data": {"username": "bob", "verified": true}} 21 | 22 | -- mixed_types_ndjson.go -- 23 | package test_package 24 | 25 | type mixed_types_ndjson struct { 26 | Data struct { 27 | Email string `json:"email,omitempty"` 28 | Name string `json:"name,omitempty"` 29 | Price float64 `json:"price,omitempty"` 30 | Username string `json:"username,omitempty"` 31 | Verified bool `json:"verified,omitempty"` 32 | } `json:"data,omitempty"` 33 | ID float64 `json:"id,omitempty"` 34 | Type string `json:"type,omitempty"` 35 | } 36 | -- empty_lines_ndjson.json -- 37 | {"value": 1} 38 | 39 | {"value": 2} 40 | 41 | {"value": 3} 42 | -- empty_lines_ndjson.go -- 43 | package test_package 44 | 45 | type empty_lines_ndjson struct { 46 | Value float64 `json:"value,omitempty"` 47 | } 48 | -------------------------------------------------------------------------------- /testdata/more_complex_example.go: -------------------------------------------------------------------------------- 1 | package test_package 2 | 3 | type more_complex_example struct { 4 | AvatarURL string `json:"avatar_url,omitempty"` 5 | Bio any `json:"bio,omitempty"` 6 | Blog string `json:"blog,omitempty"` 7 | Company string `json:"company,omitempty"` 8 | CreatedAt string `json:"created_at,omitempty"` 9 | Email string `json:"email,omitempty"` 10 | EventsURL string `json:"events_url,omitempty"` 11 | Followers float64 `json:"followers,omitempty"` 12 | FollowersURL string `json:"followers_url,omitempty"` 13 | Following float64 `json:"following,omitempty"` 14 | FollowingURL string `json:"following_url,omitempty"` 15 | GistsURL string `json:"gists_url,omitempty"` 16 | GravatarID string `json:"gravatar_id,omitempty"` 17 | Hireable bool `json:"hireable,omitempty"` 18 | HtmlURL string `json:"html_url,omitempty"` 19 | ID float64 `json:"id,omitempty"` 20 | Location string `json:"location,omitempty"` 21 | Login string `json:"login,omitempty"` 22 | Name string `json:"name,omitempty"` 23 | OrganizationsURL string `json:"organizations_url,omitempty"` 24 | PublicGists float64 `json:"public_gists,omitempty"` 25 | PublicRepos float64 `json:"public_repos,omitempty"` 26 | ReceivedEventsURL string `json:"received_events_url,omitempty"` 27 | ReposURL string `json:"repos_url,omitempty"` 28 | StarredURL string `json:"starred_url,omitempty"` 29 | SubscriptionsURL string `json:"subscriptions_url,omitempty"` 30 | Type string `json:"type,omitempty"` 31 | UpdatedAt string `json:"updated_at,omitempty"` 32 | URL string `json:"url,omitempty"` 33 | } 34 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [Unreleased] 4 | 5 | ### Added 6 | 7 | - **Struct Extraction**: New `-extract-structs` flag automatically identifies and extracts repeated nested struct patterns to reduce code duplication 8 | - **Streaming Output Mode**: New `-stream` flag for progressive output display with terminal clearing, ideal for large datasets 9 | - **Roundtrip Validation**: New `-roundtrip` flag generates and executes test programs to verify JSON marshaling/unmarshaling correctness 10 | - **Field Statistics**: New `-stat-comments` flag adds occurrence rates and type distribution information as struct field comments 11 | - **Field Ordering Options**: New `-field-order` flag supports multiple strategies: 12 | - `alphabetical`: Sort fields alphabetically (default) 13 | - `encounter`: Maintain order of first encounter 14 | - `common-first`: Sort by occurrence frequency (most common first) 15 | - `rare-first`: Sort by occurrence frequency (least common first) 16 | - **Template-Based Generation**: Custom output formatting via `-template` flag with txtar template files 17 | - **NDJSON Support**: Full support for newline-delimited JSON input 18 | - **Performance Profiling**: Added `-cpuprofile` and `-pprof` flags for performance analysis 19 | - **Comprehensive Test Suite**: Added extensive txtar-based test coverage including: 20 | - Edge case handling (exotic types, unicode, deeply nested structures) 21 | - Roundtrip validation tests 22 | - Large payload handling 23 | - Multi-document NDJSON processing 24 | 25 | ### Changed 26 | 27 | - Replaced `interface{}` with `any` type alias throughout codebase 28 | - Improved field ordering determinism by sorting keys during JSON processing 29 | - Enhanced template rendering with proper function mapping for both default and legacy modes 30 | - Reorganized test files for better clarity (renamed legacy detector files) 31 | - Updated default behavior to use `omitempty` struct tags 32 | 33 | ### Fixed 34 | 35 | - Improved handling of nullable fields with proper pointer type detection 36 | - Better extraction and naming of nested structs with type-based prefixes 37 | - Enhanced legacy mode compatibility with proper template function support 38 | 39 | ## Previous Releases 40 | 41 | See git history for earlier changes. 42 | -------------------------------------------------------------------------------- /testdata/extract-structs.txtar: -------------------------------------------------------------------------------- 1 | # Test cases for struct extraction feature 2 | # Tests the -extract-structs flag to reduce duplication 3 | # flags: -extract-structs 4 | 5 | -- simple_duplicate.json -- 6 | { 7 | "user": { 8 | "id": 1, 9 | "name": "Alice", 10 | "active": true 11 | }, 12 | "admin": { 13 | "id": 2, 14 | "name": "Bob", 15 | "active": false 16 | } 17 | } 18 | 19 | -- simple_duplicate.go -- 20 | package test_package 21 | 22 | type simple_duplicateStructC801770D struct { 23 | Active bool `json:"active,omitempty"` 24 | ID float64 `json:"id,omitempty"` 25 | Name string `json:"name,omitempty"` 26 | } 27 | 28 | type simple_duplicate struct { 29 | Admin simple_duplicateStructC801770D `json:"admin,omitempty"` 30 | User simple_duplicateStructC801770D `json:"user,omitempty"` 31 | } 32 | -- repeated_nested.json -- 33 | { 34 | "source": { 35 | "file": { 36 | "path": "/src/main.go", 37 | "size": 1024, 38 | "mode": 644 39 | } 40 | }, 41 | "target": { 42 | "file": { 43 | "path": "/dst/main.go", 44 | "size": 1024, 45 | "mode": 644 46 | } 47 | }, 48 | "backup": { 49 | "file": { 50 | "path": "/bak/main.go", 51 | "size": 1024, 52 | "mode": 644 53 | } 54 | } 55 | } 56 | 57 | -- repeated_nested.go -- 58 | package test_package 59 | 60 | type repeated_nestedStruct9A925295 struct { 61 | Mode float64 `json:"mode,omitempty"` 62 | Path string `json:"path,omitempty"` 63 | Size float64 `json:"size,omitempty"` 64 | } 65 | 66 | type repeated_nested struct { 67 | Backup struct { 68 | File repeated_nestedStruct9A925295 `json:"file,omitempty"` 69 | } `json:"backup,omitempty"` 70 | Source struct { 71 | File repeated_nestedStruct9A925295 `json:"file,omitempty"` 72 | } `json:"source,omitempty"` 73 | Target struct { 74 | File repeated_nestedStruct9A925295 `json:"file,omitempty"` 75 | } `json:"target,omitempty"` 76 | } 77 | -- deeply_nested.json -- 78 | { 79 | "action": { 80 | "result": { 81 | "data": { 82 | "auth": 1 83 | }, 84 | "type": 0 85 | } 86 | } 87 | } 88 | 89 | -- deeply_nested.go -- 90 | package test_package 91 | 92 | type deeply_nested struct { 93 | Action struct { 94 | Result struct { 95 | Data struct { 96 | Auth float64 `json:"auth,omitempty"` 97 | } `json:"data,omitempty"` 98 | Type float64 `json:"type,omitempty"` 99 | } `json:"result,omitempty"` 100 | } `json:"action,omitempty"` 101 | } 102 | -------------------------------------------------------------------------------- /templates_legacy.go: -------------------------------------------------------------------------------- 1 | //go:build legacy 2 | 3 | package main 4 | 5 | import ( 6 | "bytes" 7 | _ "embed" 8 | "os" 9 | "text/template" 10 | 11 | "golang.org/x/tools/txtar" 12 | ) 13 | 14 | var ( 15 | fileTemplate *template.Template 16 | typeTemplate *template.Template 17 | ) 18 | 19 | func init() { 20 | loadTemplates("") 21 | } 22 | 23 | func loadTemplates(templatePath string) { 24 | var templateData string 25 | 26 | // Try to load from specified template file first 27 | if templatePath != "" { 28 | if data, err := os.ReadFile(templatePath); err == nil { 29 | templateData = string(data) 30 | } 31 | } 32 | 33 | // Fallback to embedded templates if external file not found or not specified 34 | if templateData == "" { 35 | templateData = defaultTemplates 36 | } 37 | 38 | // Parse the template data (either external or embedded) 39 | archive := txtar.Parse([]byte(templateData)) 40 | templates := make(map[string]string) 41 | for _, file := range archive.Files { 42 | templates[file.Name] = string(file.Data) 43 | } 44 | 45 | // Template functions - note: these are stubs for legacy mode 46 | // Full implementation would require generator context 47 | funcMap := template.FuncMap{ 48 | "RenderInlineStruct": func(t *Type, depth int) string { 49 | // Simple inline struct rendering for legacy compatibility 50 | if t.Type == "struct" || t.Type == "*struct" { 51 | return t.Type + " {...}" 52 | } 53 | return t.Type 54 | }, 55 | } 56 | 57 | if fileTmpl, ok := templates["file.tmpl"]; ok { 58 | fileTemplate = template.Must(template.New("file").Funcs(funcMap).Parse(fileTmpl)) 59 | } 60 | if typeTmpl, ok := templates["type.tmpl"]; ok { 61 | typeTemplate = template.Must(template.New("type").Funcs(funcMap).Parse(typeTmpl)) 62 | } 63 | } 64 | 65 | func (t *Type) templateString() string { 66 | var buf bytes.Buffer 67 | template := typeTemplate 68 | if t.Config != nil && t.Config.typeTemplate != nil { 69 | template = t.Config.typeTemplate 70 | } 71 | if err := template.Execute(&buf, t); err != nil { 72 | panic(err) 73 | } 74 | return buf.String() 75 | } 76 | 77 | func renderFile(packageName, content string, cfg *generator) string { 78 | data := struct { 79 | Package string 80 | Imports []string 81 | Content string 82 | }{ 83 | Package: packageName, 84 | Imports: nil, // No imports needed for now 85 | Content: content, 86 | } 87 | 88 | var buf bytes.Buffer 89 | template := fileTemplate 90 | if cfg != nil && cfg.fileTemplate != nil { 91 | template = cfg.fileTemplate 92 | } 93 | if err := template.Execute(&buf, data); err != nil { 94 | panic(err) 95 | } 96 | return buf.String() 97 | } 98 | -------------------------------------------------------------------------------- /testdata/ndjson.txtar: -------------------------------------------------------------------------------- 1 | ndjson test cases - runs in non-legacy mode only 2 | NDJSON (Newline Delimited JSON) test cases for line-by-line JSON processing. 3 | 4 | -- user_events.ndjson -- 5 | {"user": "alice", "event": "login", "timestamp": "2024-01-01T10:00:00Z"} 6 | {"user": "bob", "event": "logout", "timestamp": "2024-01-01T10:05:00Z", "session_duration": 3600} 7 | {"user": "alice", "event": "purchase", "timestamp": "2024-01-01T10:10:00Z", "amount": 29.99, "product": "book"} 8 | {"user": "charlie", "event": "login", "timestamp": "2024-01-01T10:15:00Z", "ip_address": "192.168.1.100"} 9 | 10 | -- mixed_schemas.ndjson -- 11 | {"name": "Product A", "price": 19.99, "category": "electronics"} 12 | {"name": "Product B", "price": 39.99, "category": "electronics", "discount": true} 13 | {"name": "Product C", "price": 9.99, "category": "books", "isbn": "978-1234567890", "pages": 300} 14 | {"name": "Product D", "price": null, "category": "clothing", "sizes": ["S", "M", "L"]} 15 | 16 | -- sensor_data.ndjson -- 17 | {"sensor_id": "temp_01", "value": 23.5, "unit": "celsius", "location": {"room": "living_room", "floor": 1}} 18 | {"sensor_id": "humid_01", "value": 45.2, "unit": "percent", "location": {"room": "living_room", "floor": 1}, "battery": 87} 19 | {"sensor_id": "temp_02", "value": 19.8, "unit": "celsius", "location": {"room": "bedroom", "floor": 2}} 20 | {"sensor_id": "motion_01", "triggered": true, "location": {"room": "entrance", "floor": 1}, "sensitivity": "high"} 21 | 22 | -- user_events_expected.go -- 23 | package main 24 | 25 | type user_events struct { 26 | User string `json:"user,omitempty"` 27 | Event string `json:"event,omitempty"` 28 | Timestamp string `json:"timestamp,omitempty"` 29 | SessionDuration *float64 `json:"session_duration,omitempty"` 30 | Amount *float64 `json:"amount,omitempty"` 31 | Product *string `json:"product,omitempty"` 32 | IPAddress *string `json:"ip_address,omitempty"` 33 | } 34 | 35 | -- mixed_schemas_expected.go -- 36 | package main 37 | 38 | type mixed_schemas struct { 39 | Name string `json:"name,omitempty"` 40 | Price *float64 `json:"price,omitempty"` 41 | Category string `json:"category,omitempty"` 42 | Discount *bool `json:"discount,omitempty"` 43 | ISBN *string `json:"isbn,omitempty"` 44 | Pages *float64 `json:"pages,omitempty"` 45 | Sizes *[]string `json:"sizes,omitempty"` 46 | } 47 | 48 | -- sensor_data_expected.go -- 49 | package main 50 | 51 | type sensor_data struct { 52 | SensorID string `json:"sensor_id,omitempty"` 53 | Value *float64 `json:"value,omitempty"` 54 | Unit *string `json:"unit,omitempty"` 55 | Location struct { 56 | Room string `json:"room,omitempty"` 57 | Floor float64 `json:"floor,omitempty"` 58 | } `json:"location,omitempty"` 59 | Battery *float64 `json:"battery,omitempty"` 60 | Triggered *bool `json:"triggered,omitempty"` 61 | Sensitivity *string `json:"sensitivity,omitempty"` 62 | } -------------------------------------------------------------------------------- /testdata/nested_result.txtar: -------------------------------------------------------------------------------- 1 | # Test cases for nested structures with field named "result" 2 | # This tests a regression where templates incorrectly used {{.GetType}} for root struct 3 | 4 | -- simple_result.json -- 5 | {"result": 123} 6 | 7 | -- simple_result.expected -- 8 | package test_package 9 | 10 | type simple_result struct { 11 | Result float64 `json:"result,omitempty"` 12 | } 13 | 14 | -- nested_result.json -- 15 | {"result": {"value": 456}} 16 | 17 | -- nested_result.expected -- 18 | package test_package 19 | 20 | type nested_result struct { 21 | Result struct { 22 | Value float64 `json:"value,omitempty"` 23 | } `json:"result,omitempty"` 24 | } 25 | 26 | -- double_nested_result.json -- 27 | {"action": {"result": {"result": {"auth": 0}, "result_type": 0}}} 28 | 29 | -- double_nested_result.expected -- 30 | package test_package 31 | 32 | type double_nested_result struct { 33 | Action struct { 34 | Result struct { 35 | Result struct { 36 | Auth float64 `json:"auth,omitempty"` 37 | } `json:"result,omitempty"` 38 | ResultType float64 `json:"result_type,omitempty"` 39 | } `json:"result,omitempty"` 40 | } `json:"action,omitempty"` 41 | } 42 | 43 | -- empty_object.json -- 44 | {"empty_object": {}} 45 | 46 | -- empty_object.expected -- 47 | package test_package 48 | 49 | type empty_object struct { 50 | EmptyObject struct { 51 | } `json:"empty_object,omitempty"` 52 | } 53 | 54 | -- mixed_empty_and_result.json -- 55 | {"data": {}, "result": {"status": "ok"}} 56 | 57 | -- mixed_empty_and_result.expected -- 58 | package test_package 59 | 60 | type mixed_empty_and_result struct { 61 | Data struct { 62 | } `json:"data,omitempty"` 63 | Result struct { 64 | Status string `json:"status,omitempty"` 65 | } `json:"result,omitempty"` 66 | } 67 | -- mixed_empty_and_result.go -- 68 | package test_package 69 | 70 | type mixed_empty_and_result struct { 71 | Data struct{} `json:"data,omitempty"` 72 | Result struct { 73 | Status string `json:"status,omitempty"` 74 | } `json:"result,omitempty"` 75 | } 76 | -- simple_result.go -- 77 | package test_package 78 | 79 | type simple_result struct { 80 | Result float64 `json:"result,omitempty"` 81 | } 82 | -- nested_result.go -- 83 | package test_package 84 | 85 | type nested_result struct { 86 | Result struct { 87 | Value float64 `json:"value,omitempty"` 88 | } `json:"result,omitempty"` 89 | } 90 | -- double_nested_result.go -- 91 | package test_package 92 | 93 | type double_nested_result struct { 94 | Action struct { 95 | Result struct { 96 | Result struct { 97 | Auth float64 `json:"auth,omitempty"` 98 | } `json:"result,omitempty"` 99 | ResultType float64 `json:"result_type,omitempty"` 100 | } `json:"result,omitempty"` 101 | } `json:"action,omitempty"` 102 | } 103 | -- empty_object.go -- 104 | package test_package 105 | 106 | type empty_object struct { 107 | EmptyObject struct{} `json:"empty_object,omitempty"` 108 | } 109 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Command json-to-struct generates Go struct definitions from JSON documents. 2 | // 3 | // # json-to-struct 4 | // 5 | // Command json-to-struct generates Go struct definitions from JSON documents. 6 | // 7 | // json-to-struct reads from stdin and prints to stdout, making it easy to integrate into 8 | // shell pipelines and build workflows. 9 | // 10 | // ```sh 11 | // $ json-to-struct -h 12 | // Usage of json-to-struct: 13 | // 14 | // -cpuprofile string 15 | // write CPU profile to file 16 | // -extract-structs 17 | // if true, extracts repeated nested structs to reduce duplication 18 | // -field-order string 19 | // field ordering: alphabetical, encounter, common-first, or rare-first (default "alphabetical") 20 | // -name string 21 | // the name of the struct (default "Foo") 22 | // -omitempty 23 | // if true, emits struct field tags with 'omitempty' (default true) 24 | // -pkg string 25 | // the name of the package for the generated code (default "main") 26 | // -pprof string 27 | // pprof server address (e.g., :6060) 28 | // -roundtrip 29 | // if true, generates and runs a round-trip validation test 30 | // -stat-comments 31 | // if true, adds field statistics as comments 32 | // -stream 33 | // if true, shows progressive output with terminal clearing 34 | // -template string 35 | // path to txtar template file 36 | // -update-interval int 37 | // milliseconds between stream mode updates (default 500) 38 | // 39 | // ``` 40 | // 41 | // It effectively exposes JSON-to-Go struct conversion for use in shells. 42 | // 43 | // # Example 44 | // 45 | // Given a JSON API response: 46 | // 47 | // $ curl -s https://api.github.com/users/tmc | json-to-struct -name=User 48 | // 49 | // # Produces 50 | // 51 | // package main 52 | // 53 | // type User struct { 54 | // AvatarURL string `json:"avatar_url,omitempty"` 55 | // Bio string `json:"bio,omitempty"` 56 | // Blog string `json:"blog,omitempty"` 57 | // Company string `json:"company,omitempty"` 58 | // CreatedAt string `json:"created_at,omitempty"` 59 | // Email any `json:"email,omitempty"` 60 | // EventsURL string `json:"events_url,omitempty"` 61 | // Followers float64 `json:"followers,omitempty"` 62 | // FollowersURL string `json:"followers_url,omitempty"` 63 | // Following float64 `json:"following,omitempty"` 64 | // FollowingURL string `json:"following_url,omitempty"` 65 | // GistsURL string `json:"gists_url,omitempty"` 66 | // GravatarID string `json:"gravatar_id,omitempty"` 67 | // Hireable bool `json:"hireable,omitempty"` 68 | // HtmlURL string `json:"html_url,omitempty"` 69 | // ID float64 `json:"id,omitempty"` 70 | // Location string `json:"location,omitempty"` 71 | // Login string `json:"login,omitempty"` 72 | // Name string `json:"name,omitempty"` 73 | // NodeID string `json:"node_id,omitempty"` 74 | // OrganizationsURL string `json:"organizations_url,omitempty"` 75 | // PublicGists float64 `json:"public_gists,omitempty"` 76 | // PublicRepos float64 `json:"public_repos,omitempty"` 77 | // ReceivedEventsURL string `json:"received_events_url,omitempty"` 78 | // ReposURL string `json:"repos_url,omitempty"` 79 | // SiteAdmin bool `json:"site_admin,omitempty"` 80 | // StarredURL string `json:"starred_url,omitempty"` 81 | // SubscriptionsURL string `json:"subscriptions_url,omitempty"` 82 | // Type string `json:"type,omitempty"` 83 | // UpdatedAt string `json:"updated_at,omitempty"` 84 | // URL string `json:"url,omitempty"` 85 | // } 86 | package main 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | json-to-struct 2 | ====== 3 | 4 | [![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev/spec/v1.0/levels#build-l3) 5 | 6 | json-to-struct attempts to generate go struct definitions from json documents 7 | 8 | [Online Version Here](https://tmc.github.io/json-to-struct/) 9 | 10 | Example 11 | ---------- 12 | 13 | ```sh 14 | $ curl -s https://api.github.com/users/tmc | json-to-struct -name=User 15 | package main 16 | 17 | type User struct { 18 | AvatarURL string `json:"avatar_url"` 19 | Bio interface{} `json:"bio"` 20 | Blog string `json:"blog"` 21 | Company string `json:"company"` 22 | CreatedAt string `json:"created_at"` 23 | Email string `json:"email"` 24 | EventsURL string `json:"events_url"` 25 | Followers float64 `json:"followers"` 26 | FollowersURL string `json:"followers_url"` 27 | Following float64 `json:"following"` 28 | FollowingURL string `json:"following_url"` 29 | GistsURL string `json:"gists_url"` 30 | GravatarID string `json:"gravatar_id"` 31 | Hireable bool `json:"hireable"` 32 | HtmlURL string `json:"html_url"` 33 | ID float64 `json:"id"` 34 | Location string `json:"location"` 35 | Login string `json:"login"` 36 | Name string `json:"name"` 37 | OrganizationsURL string `json:"organizations_url"` 38 | PublicGists float64 `json:"public_gists"` 39 | PublicRepos float64 `json:"public_repos"` 40 | ReceivedEventsURL string `json:"received_events_url"` 41 | ReposURL string `json:"repos_url"` 42 | StarredURL string `json:"starred_url"` 43 | SubscriptionsURL string `json:"subscriptions_url"` 44 | Type string `json:"type"` 45 | UpdatedAt string `json:"updated_at"` 46 | URL string `json:"url"` 47 | } 48 | ``` 49 | 50 | Installation 51 | ------------ 52 | 53 | ```sh 54 | $ go install github.com/tmc/json-to-struct@latest 55 | ``` 56 | 57 | Features 58 | -------- 59 | 60 | - **Struct Extraction**: Automatically extract repeated nested structures with `-extract-structs` 61 | - **Streaming Mode**: Progressive output for large datasets with `-stream` 62 | - **Roundtrip Validation**: Verify generated structs with `-roundtrip` 63 | - **Field Statistics**: Add occurrence metadata with `-stat-comments` 64 | - **Custom Field Ordering**: Control field order with `-field-order` (alphabetical, encounter, common-first, rare-first) 65 | - **Template Support**: Custom output formatting via `-template` 66 | - **NDJSON Support**: Process newline-delimited JSON 67 | 68 | Usage 69 | ----- 70 | 71 | ```sh 72 | # Basic usage 73 | $ curl -s https://api.github.com/users/tmc | json-to-struct -name=User 74 | 75 | # Extract repeated structs to reduce duplication 76 | $ cat data.json | json-to-struct -name=Data -extract-structs 77 | 78 | # Generate with field statistics comments 79 | $ cat data.json | json-to-struct -name=Data -stat-comments 80 | 81 | # Validate generated structs with roundtrip testing 82 | $ cat data.json | json-to-struct -name=Data -roundtrip 83 | 84 | # Process large datasets with streaming output 85 | $ cat large.json | json-to-struct -name=Data -stream 86 | 87 | # Custom field ordering by occurrence frequency 88 | $ cat data.json | json-to-struct -name=Data -field-order=common-first 89 | ``` 90 | 91 | Related Work 92 | ------------ 93 | 94 | github.com/ChimeraCoder/gojson 95 | github.com/str1ngs/jflect 96 | 97 | License 98 | ---------- 99 | 100 | json-to-struct is free software distributed under Version 3 of the GNU Public License. 101 | 102 | As of the time of writing, this is the same license used for gcc (and therefore gccgo), so it is unlikely to restrict use in any way. Note that the GPL does not extend to any output generated by json-to-struct; the GPL only applies to software which includes copies of json-to-struct itself. 103 | -------------------------------------------------------------------------------- /generate_legacy.go: -------------------------------------------------------------------------------- 1 | //go:build legacy 2 | 3 | package main 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | "go/format" 9 | "io" 10 | "reflect" 11 | "sort" 12 | "strings" 13 | "unicode" 14 | ) 15 | 16 | func init() { 17 | legacyGenerateFunc = generate 18 | } 19 | 20 | func generate(input io.Reader, structName, pkgName string, cfg *generator) ([]byte, error) { 21 | var iresult any 22 | if cfg == nil { 23 | cfg = &generator{OmitEmpty: true} 24 | } 25 | if err := json.NewDecoder(input).Decode(&iresult); err != nil { 26 | return nil, err 27 | } 28 | 29 | var typ *Type 30 | switch iresult := iresult.(type) { 31 | case map[string]any: 32 | typ = generateType(structName, iresult, cfg) 33 | case []map[string]any: 34 | if len(iresult) == 0 { 35 | return nil, fmt.Errorf("empty array") 36 | } 37 | typ = generateType(structName, iresult[0], cfg) 38 | for _, r := range iresult[0:] { 39 | t2 := generateType(structName, r, cfg) 40 | if err := typ.Merge(t2); err != nil { 41 | return nil, fmt.Errorf("issue merging: %w", err) 42 | } 43 | } 44 | case []any: 45 | // TODO: reduce repetition 46 | if len(iresult) == 0 { 47 | return nil, fmt.Errorf("empty array") 48 | } 49 | typ = generateType(structName, iresult[0], cfg) 50 | for _, r := range iresult[0:] { 51 | t2 := generateType(structName, r, cfg) 52 | if err := typ.Merge(t2); err != nil { 53 | return nil, fmt.Errorf("issue merging: %w", err) 54 | } 55 | } 56 | default: 57 | return nil, fmt.Errorf("unexpected type: %T", iresult) 58 | } 59 | 60 | src := renderFile(pkgName, typ.String(), cfg) 61 | formatted, err := format.Source([]byte(src)) 62 | if err != nil { 63 | err = fmt.Errorf("error generating struct: %w", err) 64 | } 65 | return formatted, err 66 | } 67 | 68 | func generateType(name string, value any, cfg *generator) *Type { 69 | result := &Type{Name: name, Config: cfg} 70 | switch v := value.(type) { 71 | case []any: 72 | types := make(map[reflect.Type]bool, 0) 73 | for _, o := range v { 74 | types[reflect.TypeOf(o)] = true 75 | } 76 | result.Repeated = true 77 | if len(types) == 1 { 78 | t := generateType("", v[0], cfg) 79 | result.Type = t.Type 80 | result.Children = t.Children 81 | } else { 82 | result.Type = "any" 83 | } 84 | case map[string]any: 85 | result.Type = "struct" 86 | result.Children = generateFieldTypes(v, cfg) 87 | default: 88 | if reflect.TypeOf(value) == nil { 89 | result.Type = "nil" 90 | } else { 91 | result.Type = reflect.TypeOf(value).Name() 92 | } 93 | } 94 | return result 95 | } 96 | 97 | func generateFieldTypes(obj map[string]any, cfg *generator) []*Type { 98 | result := []*Type{} 99 | 100 | keys := make([]string, 0, len(obj)) 101 | for key := range obj { 102 | keys = append(keys, key) 103 | } 104 | sort.Strings(keys) 105 | 106 | for _, key := range keys { 107 | var typ *Type 108 | switch v := obj[key].(type) { 109 | case map[string]any: 110 | typ = generateType(key, v, cfg) 111 | default: 112 | typ = generateType(key, obj[key], cfg) 113 | } 114 | typ.Name = fmtFieldName(key) 115 | // if we need to rewrite the field name we need to record the json field in a tag. 116 | if typ.Name != key { 117 | typ.Tags = map[string]string{"json": key} 118 | } 119 | result = append(result, typ) 120 | } 121 | return result 122 | } 123 | 124 | func fmtFieldName(s string) string { 125 | uppercaseFixups := map[string]bool{"id": true, "url": true} 126 | parts := strings.Split(s, "_") 127 | for i := range parts { 128 | parts[i] = strings.Title(parts[i]) 129 | } 130 | if len(parts) > 0 { 131 | last := parts[len(parts)-1] 132 | if uppercaseFixups[strings.ToLower(last)] { 133 | parts[len(parts)-1] = strings.ToUpper(last) 134 | } 135 | } 136 | assembled := strings.Join(parts, "") 137 | runes := []rune(assembled) 138 | for i, c := range runes { 139 | ok := unicode.IsLetter(c) || unicode.IsDigit(c) 140 | if i == 0 { 141 | ok = unicode.IsLetter(c) 142 | } 143 | if !ok { 144 | runes[i] = '_' 145 | } 146 | } 147 | return string(runes) 148 | } 149 | -------------------------------------------------------------------------------- /testdata/all.txtar: -------------------------------------------------------------------------------- 1 | # Original test suite coverage 2 | 3 | legacy-compat: Contains original test cases that are compatible with legacy test approach. 4 | 5 | -- more_complex_example.go -- 6 | package test_package 7 | 8 | type more_complex_example struct { 9 | AvatarURL string `json:"avatar_url,omitempty"` 10 | Bio any `json:"bio,omitempty"` 11 | Blog string `json:"blog,omitempty"` 12 | Company string `json:"company,omitempty"` 13 | CreatedAt string `json:"created_at,omitempty"` 14 | Email string `json:"email,omitempty"` 15 | EventsURL string `json:"events_url,omitempty"` 16 | Followers float64 `json:"followers,omitempty"` 17 | FollowersURL string `json:"followers_url,omitempty"` 18 | Following float64 `json:"following,omitempty"` 19 | FollowingURL string `json:"following_url,omitempty"` 20 | GistsURL string `json:"gists_url,omitempty"` 21 | GravatarID string `json:"gravatar_id,omitempty"` 22 | Hireable bool `json:"hireable,omitempty"` 23 | HtmlURL string `json:"html_url,omitempty"` 24 | ID float64 `json:"id,omitempty"` 25 | Location string `json:"location,omitempty"` 26 | Login string `json:"login,omitempty"` 27 | Name string `json:"name,omitempty"` 28 | OrganizationsURL string `json:"organizations_url,omitempty"` 29 | PublicGists float64 `json:"public_gists,omitempty"` 30 | PublicRepos float64 `json:"public_repos,omitempty"` 31 | ReceivedEventsURL string `json:"received_events_url,omitempty"` 32 | ReposURL string `json:"repos_url,omitempty"` 33 | StarredURL string `json:"starred_url,omitempty"` 34 | SubscriptionsURL string `json:"subscriptions_url,omitempty"` 35 | Type string `json:"type,omitempty"` 36 | UpdatedAt string `json:"updated_at,omitempty"` 37 | URL string `json:"url,omitempty"` 38 | } 39 | -- test_invalid_field_chars.go -- 40 | package test_package 41 | 42 | type test_invalid_field_chars struct { 43 | Foo string `json:"foo,omitempty"` 44 | } 45 | -- test_mixed_nulls.go -- 46 | package test_package 47 | 48 | type test_mixed_nulls struct { 49 | Bar *float64 `json:"bar,omitempty"` 50 | } 51 | -- test_nested_json.go -- 52 | package test_package 53 | 54 | type test_nested_json struct { 55 | Baz []float64 `json:"baz,omitempty"` 56 | Foo struct { 57 | Bar float64 `json:"bar,omitempty"` 58 | } `json:"foo,omitempty"` 59 | } 60 | -- test_nullable_json.go -- 61 | package test_package 62 | 63 | type test_nullable_json struct { 64 | Foo []struct { 65 | Bar float64 `json:"bar,omitempty"` 66 | } `json:"foo,omitempty"` 67 | } 68 | -- test_repeated_json.go -- 69 | package test_package 70 | 71 | type test_repeated_json struct { 72 | Bar float64 `json:"bar,omitempty"` 73 | Baz struct { 74 | Zap bool `json:"zap,omitempty"` 75 | } `json:"baz,omitempty"` 76 | Foo float64 `json:"foo,omitempty"` 77 | } 78 | -- test_simple_array.go -- 79 | package test_package 80 | 81 | type test_simple_array struct { 82 | Baz any `json:"baz,omitempty"` 83 | Foo string `json:"foo,omitempty"` 84 | } 85 | -- test_simple_json.go -- 86 | package test_package 87 | 88 | type test_simple_json struct { 89 | F_o_o float64 `json:"f.o-o,omitempty"` 90 | } 91 | -- empty.json -- 92 | -- more_complex_example.json -- 93 | { 94 | "login": "exampleuser", 95 | "id": 3977, 96 | "avatar_url": "https://1.gravatar.com/avatar/68f0049842700597b89972e1fbf6f542?d=https%3A%2F%2Fidenticons.github.com%2F7d571e5c15bad5ef8c4352ce7a1d9e78.png", 97 | "gravatar_id": "68f0049842700597b89972e1fbf6f542", 98 | "url": "https://api.github.com/users/exampleuser", 99 | "html_url": "https://github.com/exampleuser", 100 | "followers_url": "https://api.github.com/users/exampleuser/followers", 101 | "following_url": "https://api.github.com/users/exampleuser/following{/other_user}", 102 | "gists_url": "https://api.github.com/users/exampleuser/gists{/gist_id}", 103 | "starred_url": "https://api.github.com/users/exampleuser/starred{/owner}{/repo}", 104 | "subscriptions_url": "https://api.github.com/users/exampleuser/subscriptions", 105 | "organizations_url": "https://api.github.com/users/exampleuser/orgs", 106 | "repos_url": "https://api.github.com/users/exampleuser/repos", 107 | "events_url": "https://api.github.com/users/exampleuser/events{/privacy}", 108 | "received_events_url": "https://api.github.com/users/exampleuser/received_events", 109 | "type": "User", 110 | "name": "Example User", 111 | "company": "Example Corp", 112 | "blog": "", 113 | "location": "Example City, ST", 114 | "email": "user@example.com", 115 | "hireable": true, 116 | "bio": null, 117 | "public_repos": 87, 118 | "followers": 103, 119 | "following": 88, 120 | "created_at": "2008-03-27T15:49:13Z", 121 | "updated_at": "2013-09-05T00:03:43Z", 122 | "public_gists": 44 123 | } 124 | -- test_invalid_field_chars.json -- 125 | {"foo" : "bar"} 126 | -- test_mixed_nulls.json -- 127 | [{"bar" : 85},{"bar" : null},{"bar" : 81}] 128 | -- test_nested_json.json -- 129 | {"foo" : {"bar": 24}, "baz" : [42,43]} 130 | -- test_nullable_json.json -- 131 | {"foo" : [{"bar": 24}, {"bar" : 42}]} 132 | -- test_repeated_json.json -- 133 | [ 134 | {"foo" : 42}, 135 | {"foo" : 42, "bar": 22}, 136 | {"foo" : 42, "baz": {}}, 137 | {"foo" : 42, "baz": {"zap": true}} 138 | ] 139 | -- test_simple_array.json -- 140 | {"foo" : "bar", "baz" : null} 141 | -- test_simple_json.json -- 142 | {"f.o-o" : 42} 143 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | //go:build !js 2 | // +build !js 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "flag" 9 | "fmt" 10 | "io" 11 | "log" 12 | "net/http" 13 | _ "net/http/pprof" 14 | "os" 15 | "runtime/pprof" 16 | "strings" 17 | ) 18 | 19 | var ( 20 | flagName = flag.String("name", "Foo", "the name of the struct") 21 | flagPkg = flag.String("pkg", "main", "the name of the package for the generated code") 22 | flagOmitEmpty = flag.Bool("omitempty", true, "if true, emits struct field tags with 'omitempty'") 23 | flagTemplate = flag.String("template", "", "path to txtar template file") 24 | flagRoundtrip = flag.Bool("roundtrip", false, "if true, generates and runs a round-trip validation test") 25 | flagStatComments = flag.Bool("stat-comments", false, "if true, adds field statistics as comments") 26 | flagStream = flag.Bool("stream", false, "if true, shows progressive output with terminal clearing") 27 | flagExtractStructs = flag.Bool("extract-structs", false, "if true, extracts repeated nested structs to reduce duplication") 28 | flagUpdateInterval = flag.Int("update-interval", 500, "milliseconds between stream mode updates") 29 | flagPprofAddr = flag.String("pprof", "", "pprof server address (e.g., :6060)") 30 | flagCpuProfile = flag.String("cpuprofile", "", "write CPU profile to file") 31 | flagFieldOrder = flag.String("field-order", "alphabetical", "field ordering: alphabetical, encounter, common-first, or rare-first") 32 | ) 33 | 34 | func main() { 35 | flag.Parse() 36 | 37 | // Start pprof server if requested 38 | if *flagPprofAddr != "" { 39 | go func() { 40 | log.Printf("Starting pprof server on %s", *flagPprofAddr) 41 | log.Printf("CPU profile: http://%s/debug/pprof/profile?seconds=30", *flagPprofAddr) 42 | log.Printf("Heap profile: http://%s/debug/pprof/heap", *flagPprofAddr) 43 | if err := http.ListenAndServe(*flagPprofAddr, nil); err != nil { 44 | log.Printf("pprof server failed: %v", err) 45 | } 46 | }() 47 | } 48 | 49 | // Start CPU profiling if requested 50 | if *flagCpuProfile != "" { 51 | f, err := os.Create(*flagCpuProfile) 52 | if err != nil { 53 | log.Fatal("could not create CPU profile: ", err) 54 | } 55 | defer f.Close() 56 | if err := pprof.StartCPUProfile(f); err != nil { 57 | log.Fatal("could not start CPU profile: ", err) 58 | } 59 | defer pprof.StopCPUProfile() 60 | } 61 | 62 | if err := run(); err != nil { 63 | // Check if it's a FormatError 64 | if fmtErr, ok := err.(*FormatError); ok { 65 | displayFormatError(fmtErr) 66 | } else { 67 | fmt.Fprintln(os.Stderr, "json-to-struct error:", err) 68 | } 69 | os.Exit(1) 70 | } 71 | } 72 | 73 | func displayFormatError(e *FormatError) { 74 | lines := strings.Split(e.Source, "\n") 75 | 76 | fmt.Fprintf(os.Stderr, "\n🔴 Syntax error in generated Go code:\n") 77 | fmt.Fprintf(os.Stderr, " %s\n\n", e.OriginalError) 78 | 79 | if e.LineNum > 0 && e.LineNum <= len(lines) { 80 | // Show context around the error 81 | start := e.LineNum - 5 82 | if start < 1 { 83 | start = 1 84 | } 85 | end := e.LineNum + 5 86 | if end > len(lines) { 87 | end = len(lines) 88 | } 89 | 90 | fmt.Fprintf(os.Stderr, "Code around line %d:\n", e.LineNum) 91 | fmt.Fprintf(os.Stderr, "─────────────────────────────────────────────\n") 92 | 93 | for i := start; i <= end; i++ { 94 | marker := " " 95 | if i == e.LineNum { 96 | marker = "→ " 97 | fmt.Fprintf(os.Stderr, "\033[31m%s%3d: %s\033[0m\n", marker, i, lines[i-1]) 98 | } else { 99 | fmt.Fprintf(os.Stderr, "%s%3d: %s\n", marker, i, lines[i-1]) 100 | } 101 | } 102 | fmt.Fprintf(os.Stderr, "─────────────────────────────────────────────\n\n") 103 | } else { 104 | // Show first 50 lines if we can't pinpoint the error 105 | fmt.Fprintf(os.Stderr, "First 50 lines of problematic code:\n") 106 | fmt.Fprintf(os.Stderr, "─────────────────────────────────────────────\n") 107 | for i, line := range lines { 108 | if i >= 50 { 109 | fmt.Fprintf(os.Stderr, "... (%d more lines)\n", len(lines)-50) 110 | break 111 | } 112 | fmt.Fprintf(os.Stderr, "%3d: %s\n", i+1, line) 113 | } 114 | fmt.Fprintf(os.Stderr, "─────────────────────────────────────────────\n") 115 | } 116 | } 117 | 118 | func run() error { 119 | if isInteractive() { 120 | flag.Usage() 121 | return fmt.Errorf("no input on stdin") 122 | } 123 | 124 | g := &generator{ 125 | OmitEmpty: *flagOmitEmpty, 126 | Template: *flagTemplate, 127 | TypeName: *flagName, 128 | PackageName: *flagPkg, 129 | StatComments: *flagStatComments, 130 | Stream: *flagStream, 131 | ExtractStructs: *flagExtractStructs, 132 | UpdateInterval: *flagUpdateInterval, 133 | FieldOrder: *flagFieldOrder, 134 | } 135 | if err := g.loadTemplates(); err != nil { 136 | fmt.Fprintln(os.Stderr, "warning: failed to load templates, using defaults:", err) 137 | } 138 | 139 | // If we need roundtrip, capture input with TeeReader 140 | var input io.Reader = os.Stdin 141 | var capturedInput bytes.Buffer 142 | 143 | if *flagRoundtrip { 144 | // Use TeeReader to capture input for both generation and validation 145 | input = io.TeeReader(os.Stdin, &capturedInput) 146 | } 147 | 148 | // Generate the struct (output to stdout) 149 | if err := g.generate(os.Stdout, input); err != nil { 150 | return err 151 | } 152 | 153 | // Run roundtrip validation if requested 154 | if *flagRoundtrip { 155 | return runRoundtripTestWithData(g, capturedInput.Bytes()) 156 | } 157 | 158 | return nil 159 | } 160 | 161 | func isInteractive() bool { 162 | fileInfo, err := os.Stdin.Stat() 163 | if err != nil { 164 | return false 165 | } 166 | return fileInfo.Mode()&(os.ModeCharDevice|os.ModeCharDevice) != 0 167 | } 168 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 21 | 22 | 23 |
24 | 173 | 174 | 175 | -------------------------------------------------------------------------------- /extract.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "sort" 7 | "strings" 8 | ) 9 | 10 | // extractRepeatedStructs identifies and extracts repeated struct patterns 11 | func (g *generator) extractRepeatedStructs(root *Type) { 12 | if !g.ExtractStructs { 13 | return 14 | } 15 | 16 | g.extractedTypes = make(map[string]*Type) 17 | 18 | // Build a map of struct signatures to track duplicates 19 | structMap := make(map[string][]*Type) 20 | g.collectStructSignatures(root, structMap) 21 | 22 | // Extract structs that appear multiple times, or nullable structs (to avoid *struct without braces) 23 | for signature, types := range structMap { 24 | shouldExtract := len(types) > 1 || strings.HasSuffix(signature, ":nullable") 25 | 26 | if shouldExtract { 27 | // This struct appears multiple times or is nullable, extract it 28 | extracted := g.createExtractedType(types[0], signature) 29 | if extracted != nil { 30 | g.extractedTypes[extracted.Name] = extracted 31 | 32 | // Replace all occurrences with references 33 | for _, t := range types { 34 | // For nullable structs, the extracted type is a pointer 35 | if t.Type == "*struct" { 36 | t.ExtractedTypeName = "*" + extracted.Name 37 | t.Type = "*" + extracted.Name 38 | } else { 39 | t.ExtractedTypeName = extracted.Name 40 | t.Type = extracted.Name // Change type from "struct" to the extracted type name 41 | } 42 | t.Children = nil // Clear children since we're using a reference 43 | } 44 | } 45 | } 46 | } 47 | } 48 | 49 | // collectStructSignatures recursively collects all struct signatures 50 | func (g *generator) collectStructSignatures(t *Type, structMap map[string][]*Type) { 51 | if t == nil { 52 | return 53 | } 54 | 55 | // Process both inline structs and nullable structs 56 | if (t.Type == "struct" || t.Type == "*struct") && len(t.Children) > 0 { 57 | sig := g.getStructSignature(t) 58 | if sig != "" { 59 | structMap[sig] = append(structMap[sig], t) 60 | } 61 | } 62 | 63 | // For nullable structs, we want to force extraction even if they only appear once 64 | // This prevents the `*struct` without braces issue 65 | if t.Type == "*struct" && len(t.Children) > 0 { 66 | sig := g.getStructSignature(t) + ":nullable" 67 | if sig != "" { 68 | // Force this to be extracted by adding it to a separate signature 69 | structMap[sig] = append(structMap[sig], t) 70 | } 71 | } 72 | 73 | // Recurse into children 74 | for _, child := range t.Children { 75 | g.collectStructSignatures(child, structMap) 76 | } 77 | } 78 | 79 | // getStructSignature generates a signature for a struct based on its fields 80 | func (g *generator) getStructSignature(t *Type) string { 81 | if (t.Type != "struct" && t.Type != "*struct") || len(t.Children) == 0 { 82 | return "" 83 | } 84 | 85 | // Build a signature from sorted field names and types 86 | var fields []string 87 | for _, child := range t.Children { 88 | fieldSig := fmt.Sprintf("%s:%s", child.Name, child.Type) 89 | if child.Repeated { 90 | fieldSig = "[]" + fieldSig 91 | } 92 | fields = append(fields, fieldSig) 93 | } 94 | 95 | sort.Strings(fields) 96 | signature := strings.Join(fields, ",") 97 | 98 | // For nullable structs, we want to extract even with fewer fields to avoid *struct rendering issues 99 | // For regular structs, only extract structs with at least 3 fields to avoid over-extraction 100 | if t.Type != "*struct" && len(fields) < 3 { 101 | return "" 102 | } 103 | 104 | return signature 105 | } 106 | 107 | // createExtractedType creates a new named type from a struct 108 | func (g *generator) createExtractedType(t *Type, signature string) *Type { 109 | if t.Type != "struct" && t.Type != "*struct" { 110 | return nil 111 | } 112 | 113 | // Generate a name based on the struct's content 114 | name := g.generateStructName(t, signature) 115 | 116 | // Create a copy of the type with the new name 117 | // Always make the extracted type a regular struct, even if the original was *struct 118 | extracted := &Type{ 119 | Name: name, 120 | Type: "struct", 121 | Children: make([]*Type, len(t.Children)), 122 | Config: t.Config, 123 | } 124 | 125 | // Deep copy children 126 | for i, child := range t.Children { 127 | extracted.Children[i] = child.Copy() 128 | } 129 | 130 | return extracted 131 | } 132 | 133 | // generateStructName generates a meaningful name for an extracted struct 134 | func (g *generator) generateStructName(t *Type, signature string) string { 135 | // Start with the root type name as prefix 136 | prefix := g.TypeName 137 | if prefix == "" { 138 | prefix = "Foo" // Default fallback 139 | } 140 | 141 | // Try to find a meaningful name from the fields 142 | // Look for common patterns like "stat", "token", etc. 143 | 144 | // Check if all fields start with a common prefix 145 | if len(t.Children) > 0 { 146 | // Look for fields like st_* which suggest "Stat" 147 | if hasCommonPrefix(t.Children, "St") { 148 | return prefix + "Stat" 149 | } 150 | 151 | } 152 | 153 | // Fallback: generate a name from a hash of the signature 154 | hash := md5.Sum([]byte(signature)) 155 | return fmt.Sprintf("%sStruct%X", prefix, hash[:4]) 156 | } 157 | 158 | // hasCommonPrefix checks if all fields share a common prefix 159 | func hasCommonPrefix(fields []*Type, prefix string) bool { 160 | if len(fields) == 0 { 161 | return false 162 | } 163 | 164 | count := 0 165 | for _, field := range fields { 166 | if strings.HasPrefix(field.Name, prefix) { 167 | count++ 168 | } 169 | } 170 | 171 | // Consider it a common prefix if at least 80% of fields have it 172 | return float64(count) >= float64(len(fields))*0.8 173 | } 174 | 175 | // Copy creates a deep copy of a Type 176 | func (t *Type) Copy() *Type { 177 | if t == nil { 178 | return nil 179 | } 180 | 181 | copied := &Type{ 182 | Name: t.Name, 183 | Type: t.Type, 184 | Repeated: t.Repeated, 185 | Tags: make(map[string]string), 186 | Config: t.Config, 187 | Stat: t.Stat, 188 | ExtractedTypeName: t.ExtractedTypeName, 189 | } 190 | 191 | // Copy tags 192 | for k, v := range t.Tags { 193 | copied.Tags[k] = v 194 | } 195 | 196 | // Deep copy children 197 | if len(t.Children) > 0 { 198 | copied.Children = make([]*Type, len(t.Children)) 199 | for i, child := range t.Children { 200 | copied.Children[i] = child.Copy() 201 | } 202 | } 203 | 204 | return copied 205 | } 206 | -------------------------------------------------------------------------------- /testdata/template-roundtrip-test.txt: -------------------------------------------------------------------------------- 1 | -- file.tmpl -- 2 | package {{.Package}} 3 | 4 | import ( 5 | "bufio" 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "os" 10 | "reflect" 11 | "strings" 12 | ) 13 | 14 | {{.Content}} 15 | 16 | type ValidationStats struct { 17 | TotalRecords int `json:"total_records"` 18 | SuccessfulParse int `json:"successful_parse"` 19 | ParseErrors int `json:"parse_errors"` 20 | FieldStats map[string]FieldValidation `json:"field_stats"` 21 | TypeMismatches []TypeMismatch `json:"type_mismatches,omitempty"` 22 | } 23 | 24 | type FieldValidation struct { 25 | ExpectedCount int `json:"expected_count"` 26 | ActualCount int `json:"actual_count"` 27 | NilCount int `json:"nil_count"` 28 | TypeErrors []string `json:"type_errors,omitempty"` 29 | } 30 | 31 | type TypeMismatch struct { 32 | Record int `json:"record"` 33 | Field string `json:"field"` 34 | Expected string `json:"expected"` 35 | Actual string `json:"actual"` 36 | OriginalVal any `json:"original_value"` 37 | } 38 | 39 | func main() { 40 | stats := &ValidationStats{ 41 | FieldStats: make(map[string]FieldValidation), 42 | } 43 | 44 | scanner := bufio.NewScanner(os.Stdin) 45 | recordNum := 0 46 | 47 | for scanner.Scan() { 48 | line := scanner.Text() 49 | if line == "" { 50 | continue 51 | } 52 | 53 | recordNum++ 54 | stats.TotalRecords++ 55 | 56 | // Parse as raw JSON first to get original structure 57 | var rawData map[string]any 58 | if err := json.Unmarshal([]byte(line), &rawData); err != nil { 59 | // Try as array 60 | var rawArray []map[string]any 61 | if err := json.Unmarshal([]byte(line), &rawArray); err != nil { 62 | log.Printf("Record %d: Failed to parse as JSON: %v", recordNum, err) 63 | stats.ParseErrors++ 64 | continue 65 | } 66 | // Process each object in array 67 | for i, obj := range rawArray { 68 | validateRecord(obj, recordNum*1000+i, stats) 69 | } 70 | continue 71 | } 72 | 73 | validateRecord(rawData, recordNum, stats) 74 | } 75 | 76 | if err := scanner.Err(); err != nil { 77 | log.Fatalf("Error reading input: %v", err) 78 | } 79 | 80 | // Output validation statistics 81 | output, err := json.MarshalIndent(stats, "", " ") 82 | if err != nil { 83 | log.Fatalf("Error marshaling stats: %v", err) 84 | } 85 | 86 | fmt.Printf("=== ROUND-TRIP VALIDATION RESULTS ===\n") 87 | fmt.Printf("%s\n", output) 88 | 89 | // Summary 90 | fmt.Printf("\n=== SUMMARY ===\n") 91 | fmt.Printf("Total Records: %d\n", stats.TotalRecords) 92 | fmt.Printf("Successful Parse: %d (%.1f%%)\n", 93 | stats.SuccessfulParse, 94 | float64(stats.SuccessfulParse)/float64(stats.TotalRecords)*100) 95 | fmt.Printf("Parse Errors: %d\n", stats.ParseErrors) 96 | fmt.Printf("Type Mismatches: %d\n", len(stats.TypeMismatches)) 97 | 98 | if len(stats.TypeMismatches) > 0 { 99 | fmt.Printf("\n=== TYPE MISMATCHES ===\n") 100 | for _, mismatch := range stats.TypeMismatches { 101 | fmt.Printf("Record %d, Field '%s': Expected %s, got %s (value: %v)\n", 102 | mismatch.Record, mismatch.Field, mismatch.Expected, mismatch.Actual, mismatch.OriginalVal) 103 | } 104 | } 105 | 106 | // Field coverage analysis 107 | fmt.Printf("\n=== FIELD COVERAGE ===\n") 108 | for fieldName, validation := range stats.FieldStats { 109 | coverage := float64(validation.ActualCount) / float64(stats.SuccessfulParse) * 100 110 | fmt.Printf("%s: %d/%d records (%.1f%%), %d nil values\n", 111 | fieldName, validation.ActualCount, stats.SuccessfulParse, coverage, validation.NilCount) 112 | 113 | if len(validation.TypeErrors) > 0 { 114 | fmt.Printf(" Type errors: %v\n", validation.TypeErrors) 115 | } 116 | } 117 | } 118 | 119 | func validateRecord(rawData map[string]any, recordNum int, stats *ValidationStats) { 120 | // Parse into generated struct 121 | var generated {{.TypeName}} 122 | rawBytes, _ := json.Marshal(rawData) 123 | 124 | if err := json.Unmarshal(rawBytes, &generated); err != nil { 125 | log.Printf("Record %d: Failed to unmarshal into generated struct: %v", recordNum, err) 126 | stats.ParseErrors++ 127 | return 128 | } 129 | 130 | stats.SuccessfulParse++ 131 | 132 | // Analyze each field 133 | generatedValue := reflect.ValueOf(generated) 134 | generatedType := reflect.TypeOf(generated) 135 | 136 | for i := 0; i < generatedValue.NumField(); i++ { 137 | field := generatedType.Field(i) 138 | fieldValue := generatedValue.Field(i) 139 | 140 | // Get JSON tag name 141 | jsonTag := field.Tag.Get("json") 142 | if jsonTag == "" { 143 | jsonTag = field.Name 144 | } 145 | // Remove ,omitempty suffix 146 | if comma := strings.Index(jsonTag, ","); comma != -1 { 147 | jsonTag = jsonTag[:comma] 148 | } 149 | 150 | // Initialize field stats if not exists 151 | if _, exists := stats.FieldStats[field.Name]; !exists { 152 | stats.FieldStats[field.Name] = FieldValidation{} 153 | } 154 | fieldStat := stats.FieldStats[field.Name] 155 | 156 | // Check if field exists in original data 157 | originalValue, exists := rawData[jsonTag] 158 | if exists { 159 | fieldStat.ActualCount++ 160 | 161 | // Check for type compatibility 162 | if err := validateFieldType(field.Name, fieldValue, originalValue, recordNum, stats); err != nil { 163 | fieldStat.TypeErrors = append(fieldStat.TypeErrors, err.Error()) 164 | } 165 | } 166 | 167 | // Check for nil values in pointer fields 168 | if fieldValue.Kind() == reflect.Ptr && fieldValue.IsNil() { 169 | fieldStat.NilCount++ 170 | } 171 | 172 | stats.FieldStats[field.Name] = fieldStat 173 | } 174 | } 175 | 176 | func validateFieldType(fieldName string, structField reflect.Value, originalValue any, recordNum int, stats *ValidationStats) error { 177 | if originalValue == nil { 178 | // Nil values should work with pointer types 179 | if structField.Kind() != reflect.Ptr { 180 | mismatch := TypeMismatch{ 181 | Record: recordNum, 182 | Field: fieldName, 183 | Expected: "pointer type (for nil)", 184 | Actual: structField.Type().String(), 185 | OriginalVal: originalValue, 186 | } 187 | stats.TypeMismatches = append(stats.TypeMismatches, mismatch) 188 | return fmt.Errorf("nil value but field is not pointer") 189 | } 190 | return nil 191 | } 192 | 193 | // Check type compatibility 194 | originalType := reflect.TypeOf(originalValue) 195 | expectedType := structField.Type() 196 | 197 | // Handle pointer types 198 | if expectedType.Kind() == reflect.Ptr { 199 | expectedType = expectedType.Elem() 200 | } 201 | 202 | compatible := false 203 | switch { 204 | case originalType == expectedType: 205 | compatible = true 206 | case originalType.Kind() == reflect.Float64 && expectedType.Kind() == reflect.Float64: 207 | compatible = true 208 | case originalType.Kind() == reflect.String && expectedType.Kind() == reflect.String: 209 | compatible = true 210 | case originalType.Kind() == reflect.Bool && expectedType.Kind() == reflect.Bool: 211 | compatible = true 212 | case expectedType == reflect.TypeOf((*any)(nil)).Elem(): // interface{} 213 | compatible = true 214 | case expectedType.Kind() == reflect.Slice || expectedType.Kind() == reflect.Array: 215 | // Arrays/slices need more complex validation 216 | compatible = true // Simplified for now 217 | case expectedType.Kind() == reflect.Struct: 218 | // Nested structs need recursive validation 219 | compatible = true // Simplified for now 220 | } 221 | 222 | if !compatible { 223 | mismatch := TypeMismatch{ 224 | Record: recordNum, 225 | Field: fieldName, 226 | Expected: expectedType.String(), 227 | Actual: originalType.String(), 228 | OriginalVal: originalValue, 229 | } 230 | stats.TypeMismatches = append(stats.TypeMismatches, mismatch) 231 | return fmt.Errorf("type mismatch: expected %s, got %s", expectedType, originalType) 232 | } 233 | 234 | return nil 235 | } 236 | 237 | -- type.tmpl -- 238 | {{.Name}} {{.GetType}}{{if .Children}} { 239 | {{range .Children}} {{.}} 240 | {{end}}}{{end}}{{.GetTags}} -------------------------------------------------------------------------------- /stream.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/json" 7 | "fmt" 8 | "go/format" 9 | "io" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "golang.org/x/term" 15 | ) 16 | 17 | // ANSI escape codes 18 | const ( 19 | clearScreen = "\033[2J" 20 | moveCursor = "\033[H" 21 | ) 22 | 23 | // generateStream processes JSON input line by line with progressive display 24 | func (g *generator) generateStream(output io.Writer, input io.Reader) error { 25 | stats := NewStructStats() 26 | g.stats = stats 27 | 28 | scanner := bufio.NewScanner(input) 29 | scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024) // 10MB max line 30 | 31 | lineNum := 0 32 | var lastOutput string 33 | lastUpdateTime := time.Now() 34 | updateInterval := time.Duration(g.UpdateInterval) * time.Millisecond 35 | if updateInterval <= 0 { 36 | updateInterval = 500 * time.Millisecond // Default 37 | } 38 | const updateBatchSize = 10 // Or every 10 objects 39 | 40 | // Check if input looks like a JSON array 41 | var buffer bytes.Buffer 42 | teeReader := io.TeeReader(input, &buffer) 43 | firstByte := make([]byte, 1) 44 | _, err := teeReader.Read(firstByte) 45 | if err != nil && err != io.EOF { 46 | return err 47 | } 48 | 49 | // If it starts with '[', we need to handle it as an array 50 | if len(firstByte) > 0 && firstByte[0] == '[' { 51 | // Read entire array and process 52 | allBytes, err := io.ReadAll(teeReader) 53 | if err != nil { 54 | return err 55 | } 56 | fullInput := append(firstByte, allBytes...) 57 | return g.generateStreamFromArray(output, fullInput) 58 | } 59 | 60 | // Otherwise process line by line (JSONL format) 61 | combined := io.MultiReader(bytes.NewReader(firstByte), &buffer, input) 62 | scanner = bufio.NewScanner(combined) 63 | scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024) 64 | 65 | for scanner.Scan() { 66 | line := strings.TrimSpace(scanner.Text()) 67 | if line == "" { 68 | continue 69 | } 70 | 71 | lineNum++ 72 | 73 | // Try to parse as JSON object 74 | var obj map[string]any 75 | if err := json.Unmarshal([]byte(line), &obj); err != nil { 76 | // Skip non-JSON lines 77 | continue 78 | } 79 | 80 | // Process this object 81 | stats.ProcessJSON(obj, g) 82 | 83 | // Only update display periodically - use logarithmic scale for large datasets 84 | timeSinceUpdate := time.Since(lastUpdateTime) 85 | 86 | // Adaptive batch size: grows logarithmically with data size 87 | adaptiveBatchSize := updateBatchSize 88 | if lineNum > 1000 { 89 | adaptiveBatchSize = 100 90 | } 91 | if lineNum > 10000 { 92 | adaptiveBatchSize = 1000 93 | } 94 | if lineNum > 100000 { 95 | adaptiveBatchSize = 10000 96 | } 97 | 98 | shouldUpdate := timeSinceUpdate >= updateInterval || 99 | lineNum%adaptiveBatchSize == 0 || 100 | lineNum <= 5 || // Always show first few updates for responsiveness 101 | lineNum == 10 || lineNum == 100 || lineNum == 1000 || lineNum == 10000 || lineNum == 100000 || lineNum == 1000000 // Milestones 102 | 103 | if shouldUpdate { 104 | // Generate current struct 105 | typ := g.buildTypeFromStats(stats) 106 | src := g.renderFile(typ.String()) 107 | 108 | // Format the code 109 | formatted, err := format.Source([]byte(src)) 110 | if err != nil { 111 | // If formatting fails, use unformatted 112 | formatted = []byte(src) 113 | } 114 | 115 | // Clear screen and display 116 | currentOutput := string(formatted) 117 | if currentOutput != lastOutput { 118 | g.displayStreamOutput(output, currentOutput, lineNum, stats.TotalLines) 119 | lastOutput = currentOutput 120 | lastUpdateTime = time.Now() 121 | } 122 | } 123 | } 124 | 125 | if err := scanner.Err(); err != nil { 126 | return fmt.Errorf("error reading input: %w", err) 127 | } 128 | 129 | if stats.TotalLines == 0 { 130 | return fmt.Errorf("no valid JSON objects found") 131 | } 132 | 133 | // Final output without clearing 134 | typ := g.buildTypeFromStats(stats) 135 | src := g.renderFile(typ.String()) 136 | formatted, err := format.Source([]byte(src)) 137 | if err != nil { 138 | return fmt.Errorf("error formatting generated code: %w", err) 139 | } 140 | 141 | // Clear one more time and show final result 142 | g.displayStreamOutput(output, string(formatted), stats.TotalLines, stats.TotalLines) 143 | 144 | return nil 145 | } 146 | 147 | // generateStreamFromArray processes a JSON array with progressive display 148 | func (g *generator) generateStreamFromArray(output io.Writer, input []byte) error { 149 | stats := NewStructStats() 150 | g.stats = stats 151 | 152 | // Parse as array 153 | var array []any 154 | if err := json.Unmarshal(input, &array); err != nil { 155 | return fmt.Errorf("error parsing JSON array: %w", err) 156 | } 157 | 158 | var lastOutput string 159 | lastUpdateTime := time.Now() 160 | updateInterval := time.Duration(g.UpdateInterval) * time.Millisecond 161 | if updateInterval <= 0 { 162 | updateInterval = 500 * time.Millisecond // Default 163 | } 164 | const updateBatchSize = 10 165 | 166 | for i, item := range array { 167 | if obj, ok := item.(map[string]any); ok { 168 | stats.ProcessJSON(obj, g) 169 | 170 | // Only update display periodically - use logarithmic scale for large datasets 171 | timeSinceUpdate := time.Since(lastUpdateTime) 172 | 173 | // Adaptive batch size for large arrays 174 | adaptiveBatchSize := updateBatchSize 175 | if i > 1000 { 176 | adaptiveBatchSize = 100 177 | } 178 | if i > 10000 { 179 | adaptiveBatchSize = 1000 180 | } 181 | if i > 100000 { 182 | adaptiveBatchSize = 10000 183 | } 184 | 185 | shouldUpdate := timeSinceUpdate >= updateInterval || 186 | (i+1)%adaptiveBatchSize == 0 || 187 | i < 5 || // Show first few 188 | i == 9 || i == 99 || i == 999 || i == 9999 || i == 99999 || i == 999999 || // Milestones 189 | i == len(array)-1 // Always show final 190 | 191 | if shouldUpdate { 192 | // Generate current struct 193 | typ := g.buildTypeFromStats(stats) 194 | src := g.renderFile(typ.String()) 195 | 196 | // Format the code 197 | formatted, err := format.Source([]byte(src)) 198 | if err != nil { 199 | formatted = []byte(src) 200 | } 201 | 202 | // Display progressive output 203 | currentOutput := string(formatted) 204 | if currentOutput != lastOutput { 205 | g.displayStreamOutput(output, currentOutput, i+1, len(array)) 206 | lastOutput = currentOutput 207 | lastUpdateTime = time.Now() 208 | } 209 | } 210 | } 211 | } 212 | 213 | if stats.TotalLines == 0 { 214 | return fmt.Errorf("no valid JSON objects found in array") 215 | } 216 | 217 | // Final output 218 | typ := g.buildTypeFromStats(stats) 219 | src := g.renderFile(typ.String()) 220 | formatted, err := format.Source([]byte(src)) 221 | if err != nil { 222 | return fmt.Errorf("error formatting generated code: %w", err) 223 | } 224 | 225 | g.displayStreamOutput(output, string(formatted), len(array), len(array)) 226 | 227 | return nil 228 | } 229 | 230 | // displayStreamOutput clears the terminal and displays the current output 231 | func (g *generator) displayStreamOutput(w io.Writer, content string, current, total int) { 232 | // Check if output is a terminal 233 | if file, ok := w.(*os.File); ok && isTerminal(file) { 234 | // For final output, show everything 235 | if current == total { 236 | // Clear screen and show full final result 237 | fmt.Fprint(w, clearScreen+moveCursor) 238 | fmt.Fprint(w, content) 239 | fmt.Fprintf(w, "\n\n✅ Complete! Processed %d objects\n", total) 240 | return 241 | } 242 | 243 | // Get terminal height for progressive display 244 | rows := getTerminalRows() 245 | 246 | // Clear screen and move cursor to top 247 | fmt.Fprint(w, clearScreen+moveCursor) 248 | 249 | // Show progress header (2 lines) 250 | fmt.Fprintf(w, "=== Processing JSON objects: %d/%d ===\n\n", current, total) 251 | 252 | // Calculate available lines (rows - header(2) - footer(3) - safety margin(2)) 253 | availableLines := rows - 7 254 | if availableLines < 10 { 255 | availableLines = 10 // Minimum to show something useful 256 | } 257 | 258 | // Split content into lines and truncate if needed 259 | lines := strings.Split(content, "\n") 260 | if len(lines) > availableLines { 261 | // Write truncated content 262 | for i := 0; i < availableLines-1; i++ { 263 | fmt.Fprintln(w, lines[i]) 264 | } 265 | fmt.Fprintf(w, "... (%d more lines)", len(lines)-availableLines+1) 266 | } else { 267 | // Write full content 268 | fmt.Fprint(w, content) 269 | } 270 | 271 | // Add footer 272 | fmt.Fprintf(w, "\n\n⏳ Processing... (%d/%d)", current, total) 273 | } else { 274 | // Non-terminal output: just write content 275 | fmt.Fprint(w, content) 276 | } 277 | } 278 | 279 | // getTerminalRows returns the terminal height using term.GetSize 280 | func getTerminalRows() int { 281 | _, rows, err := term.GetSize(int(os.Stdout.Fd())) 282 | if err != nil || rows <= 0 { 283 | // Default to 24 rows if not a terminal or error 284 | return 24 285 | } 286 | return rows 287 | } 288 | 289 | // isTerminal checks if a file descriptor is a terminal 290 | func isTerminal(f *os.File) bool { 291 | fileInfo, err := f.Stat() 292 | if err != nil { 293 | return false 294 | } 295 | return fileInfo.Mode()&os.ModeCharDevice != 0 296 | } 297 | -------------------------------------------------------------------------------- /testdata/large-payload.txtar: -------------------------------------------------------------------------------- 1 | # Large payload test case 2 | # This tests handling of complex nested JSON structures from real-world data 3 | 4 | -- eslog.json -- 5 | {"process":{"executable":{"path":"\/Users\/tmc\/.local\/homebrew\/Cellar\/node\/24.4.1\/bin\/node","stat":{"st_flags":0,"st_dev":16777239,"st_atimespec":"2025-09-29T08:17:05.863502863Z","st_ino":65930357,"st_blksize":4096,"st_uid":501,"st_nlink":1,"st_birthtimespec":"2025-07-19T15:52:48.169930626Z","st_mtimespec":"2025-07-19T15:52:48.169930626Z","st_gid":20,"st_blocks":173912,"st_ctimespec":"2025-07-19T15:52:57.775450573Z","st_mode":33133,"st_gen":0,"st_size":89042552,"st_rdev":0},"path_truncated":false},"parent_audit_token":{"egid":20,"pid":29351,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":19845966},"ppid":29351,"start_time":"2025-09-28T02:05:45.339970Z","audit_token":{"egid":20,"pid":42221,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":19867158},"cs_validation_category":10,"codesigning_flags":570556963,"signing_id":"node","original_ppid":29351,"session_id":29342,"team_id":null,"responsible_audit_token":{"egid":20,"pid":20792,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":12130782},"group_id":42221,"tty":{"path":"\/dev\/ttys027","stat":{"st_flags":0,"st_dev":1552022631,"st_atimespec":"2025-09-29T03:55:25.904783000Z","st_ino":1969,"st_blksize":65536,"st_uid":501,"st_nlink":1,"st_birthtimespec":"1970-01-01T00:00:00.000000000Z","st_mtimespec":"2025-09-29T08:17:17.877803000Z","st_gid":4,"st_blocks":0,"st_ctimespec":"2025-09-29T08:17:17.877803000Z","st_mode":8592,"st_gen":0,"st_size":0,"st_rdev":268435483},"path_truncated":false},"is_platform_binary":false,"cdhash":"3F85C87E5B5E72EAB33D2C7A1362BA86A34D0283","is_es_client":false},"global_seq_num":0,"time":"2025-09-29T08:17:17.924250682Z","thread":{"thread_id":21612566},"schema_version":1,"action":{"result":{"result":{"auth":0},"result_type":0}},"mach_time":4210846481572,"event_type":43,"action_type":1,"seq_num":0,"version":10,"event":{"lookup":{"source_dir":{"stat":{"st_dev":16777239,"st_atimespec":"2025-09-09T07:15:49.000000000Z","st_rdev":0,"st_blksize":4096,"st_size":704,"st_mtimespec":"2025-09-09T07:15:49.000000000Z","st_birthtimespec":"2025-09-09T07:15:49.000000000Z","st_blocks":0,"st_flags":1048576,"st_gen":0,"st_uid":0,"st_mode":16877,"st_gid":0,"st_ino":2,"st_nlink":22,"st_ctimespec":"2025-09-09T07:15:49.000000000Z"},"path_truncated":false,"path":"\/"},"relative_target":"Users\/tmc\/.claude\/.config.json"}}} 6 | 7 | -- eslog.go -- 8 | package test_package 9 | 10 | type eslog struct { 11 | Action struct { 12 | Result struct { 13 | Result struct { 14 | Auth float64 `json:"auth,omitempty"` 15 | } `json:"result,omitempty"` 16 | ResultType float64 `json:"result_type,omitempty"` 17 | } `json:"result,omitempty"` 18 | } `json:"action,omitempty"` 19 | ActionType float64 `json:"action_type,omitempty"` 20 | Event struct { 21 | Lookup struct { 22 | RelativeTarget string `json:"relative_target,omitempty"` 23 | SourceDir struct { 24 | Path string `json:"path,omitempty"` 25 | PathTruncated bool `json:"path_truncated,omitempty"` 26 | Stat struct { 27 | StAtimespec string `json:"st_atimespec,omitempty"` 28 | StBirthtimespec string `json:"st_birthtimespec,omitempty"` 29 | StBlksize float64 `json:"st_blksize,omitempty"` 30 | StBlocks float64 `json:"st_blocks,omitempty"` 31 | StCtimespec string `json:"st_ctimespec,omitempty"` 32 | StDev float64 `json:"st_dev,omitempty"` 33 | StFlags float64 `json:"st_flags,omitempty"` 34 | StGen float64 `json:"st_gen,omitempty"` 35 | StGid float64 `json:"st_gid,omitempty"` 36 | StIno float64 `json:"st_ino,omitempty"` 37 | StMode float64 `json:"st_mode,omitempty"` 38 | StMtimespec string `json:"st_mtimespec,omitempty"` 39 | StNlink float64 `json:"st_nlink,omitempty"` 40 | StRdev float64 `json:"st_rdev,omitempty"` 41 | StSize float64 `json:"st_size,omitempty"` 42 | StUid float64 `json:"st_uid,omitempty"` 43 | } `json:"stat,omitempty"` 44 | } `json:"source_dir,omitempty"` 45 | } `json:"lookup,omitempty"` 46 | } `json:"event,omitempty"` 47 | EventType float64 `json:"event_type,omitempty"` 48 | GlobalSeqNum float64 `json:"global_seq_num,omitempty"` 49 | MachTime float64 `json:"mach_time,omitempty"` 50 | Process struct { 51 | AuditToken struct { 52 | Asid float64 `json:"asid,omitempty"` 53 | Auid float64 `json:"auid,omitempty"` 54 | Egid float64 `json:"egid,omitempty"` 55 | Euid float64 `json:"euid,omitempty"` 56 | Pid float64 `json:"pid,omitempty"` 57 | Pidversion float64 `json:"pidversion,omitempty"` 58 | Rgid float64 `json:"rgid,omitempty"` 59 | Ruid float64 `json:"ruid,omitempty"` 60 | } `json:"audit_token,omitempty"` 61 | Cdhash string `json:"cdhash,omitempty"` 62 | CodesigningFlags float64 `json:"codesigning_flags,omitempty"` 63 | CsValidationCategory float64 `json:"cs_validation_category,omitempty"` 64 | Executable struct { 65 | Path string `json:"path,omitempty"` 66 | PathTruncated bool `json:"path_truncated,omitempty"` 67 | Stat struct { 68 | StAtimespec string `json:"st_atimespec,omitempty"` 69 | StBirthtimespec string `json:"st_birthtimespec,omitempty"` 70 | StBlksize float64 `json:"st_blksize,omitempty"` 71 | StBlocks float64 `json:"st_blocks,omitempty"` 72 | StCtimespec string `json:"st_ctimespec,omitempty"` 73 | StDev float64 `json:"st_dev,omitempty"` 74 | StFlags float64 `json:"st_flags,omitempty"` 75 | StGen float64 `json:"st_gen,omitempty"` 76 | StGid float64 `json:"st_gid,omitempty"` 77 | StIno float64 `json:"st_ino,omitempty"` 78 | StMode float64 `json:"st_mode,omitempty"` 79 | StMtimespec string `json:"st_mtimespec,omitempty"` 80 | StNlink float64 `json:"st_nlink,omitempty"` 81 | StRdev float64 `json:"st_rdev,omitempty"` 82 | StSize float64 `json:"st_size,omitempty"` 83 | StUid float64 `json:"st_uid,omitempty"` 84 | } `json:"stat,omitempty"` 85 | } `json:"executable,omitempty"` 86 | GroupID float64 `json:"group_id,omitempty"` 87 | IsEsClient bool `json:"is_es_client,omitempty"` 88 | IsPlatformBinary bool `json:"is_platform_binary,omitempty"` 89 | OriginalPpid float64 `json:"original_ppid,omitempty"` 90 | ParentAuditToken struct { 91 | Asid float64 `json:"asid,omitempty"` 92 | Auid float64 `json:"auid,omitempty"` 93 | Egid float64 `json:"egid,omitempty"` 94 | Euid float64 `json:"euid,omitempty"` 95 | Pid float64 `json:"pid,omitempty"` 96 | Pidversion float64 `json:"pidversion,omitempty"` 97 | Rgid float64 `json:"rgid,omitempty"` 98 | Ruid float64 `json:"ruid,omitempty"` 99 | } `json:"parent_audit_token,omitempty"` 100 | Ppid float64 `json:"ppid,omitempty"` 101 | ResponsibleAuditToken struct { 102 | Asid float64 `json:"asid,omitempty"` 103 | Auid float64 `json:"auid,omitempty"` 104 | Egid float64 `json:"egid,omitempty"` 105 | Euid float64 `json:"euid,omitempty"` 106 | Pid float64 `json:"pid,omitempty"` 107 | Pidversion float64 `json:"pidversion,omitempty"` 108 | Rgid float64 `json:"rgid,omitempty"` 109 | Ruid float64 `json:"ruid,omitempty"` 110 | } `json:"responsible_audit_token,omitempty"` 111 | SessionID float64 `json:"session_id,omitempty"` 112 | SigningID string `json:"signing_id,omitempty"` 113 | StartTime string `json:"start_time,omitempty"` 114 | TeamID any `json:"team_id,omitempty"` 115 | Tty struct { 116 | Path string `json:"path,omitempty"` 117 | PathTruncated bool `json:"path_truncated,omitempty"` 118 | Stat struct { 119 | StAtimespec string `json:"st_atimespec,omitempty"` 120 | StBirthtimespec string `json:"st_birthtimespec,omitempty"` 121 | StBlksize float64 `json:"st_blksize,omitempty"` 122 | StBlocks float64 `json:"st_blocks,omitempty"` 123 | StCtimespec string `json:"st_ctimespec,omitempty"` 124 | StDev float64 `json:"st_dev,omitempty"` 125 | StFlags float64 `json:"st_flags,omitempty"` 126 | StGen float64 `json:"st_gen,omitempty"` 127 | StGid float64 `json:"st_gid,omitempty"` 128 | StIno float64 `json:"st_ino,omitempty"` 129 | StMode float64 `json:"st_mode,omitempty"` 130 | StMtimespec string `json:"st_mtimespec,omitempty"` 131 | StNlink float64 `json:"st_nlink,omitempty"` 132 | StRdev float64 `json:"st_rdev,omitempty"` 133 | StSize float64 `json:"st_size,omitempty"` 134 | StUid float64 `json:"st_uid,omitempty"` 135 | } `json:"stat,omitempty"` 136 | } `json:"tty,omitempty"` 137 | } `json:"process,omitempty"` 138 | SchemaVersion float64 `json:"schema_version,omitempty"` 139 | SeqNum float64 `json:"seq_num,omitempty"` 140 | Thread struct { 141 | ThreadID float64 `json:"thread_id,omitempty"` 142 | } `json:"thread,omitempty"` 143 | Time string `json:"time,omitempty"` 144 | Version float64 `json:"version,omitempty"` 145 | } 146 | -------------------------------------------------------------------------------- /testdata/multi-eslog.txtar: -------------------------------------------------------------------------------- 1 | # Multiple ES log lines test case 2 | # Tests handling of multiple JSON lines from real-world data as an array 3 | 4 | -- eslog2.json -- 5 | [ 6 | {"process":{"executable":{"path":"\/Users\/tmc\/.local\/homebrew\/Cellar\/node\/24.4.1\/bin\/node","stat":{"st_flags":0,"st_dev":16777239,"st_atimespec":"2025-09-29T08:17:05.863502863Z","st_ino":65930357,"st_blksize":4096,"st_uid":501,"st_nlink":1,"st_birthtimespec":"2025-07-19T15:52:48.169930626Z","st_mtimespec":"2025-07-19T15:52:48.169930626Z","st_gid":20,"st_blocks":173912,"st_ctimespec":"2025-07-19T15:52:57.775450573Z","st_mode":33133,"st_gen":0,"st_size":89042552,"st_rdev":0},"path_truncated":false},"parent_audit_token":{"egid":20,"pid":29351,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":19845966},"ppid":29351,"start_time":"2025-09-28T02:05:45.339970Z","audit_token":{"egid":20,"pid":42221,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":19867158},"cs_validation_category":10,"codesigning_flags":570556963,"signing_id":"node","original_ppid":29351,"session_id":29342,"team_id":null,"responsible_audit_token":{"egid":20,"pid":20792,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":12130782},"group_id":42221,"tty":{"path":"\/dev\/ttys027","stat":{"st_flags":0,"st_dev":1552022631,"st_atimespec":"2025-09-29T03:55:25.904783000Z","st_ino":1969,"st_blksize":65536,"st_uid":501,"st_nlink":1,"st_birthtimespec":"1970-01-01T00:00:00.000000000Z","st_mtimespec":"2025-09-29T08:17:17.877803000Z","st_gid":4,"st_blocks":0,"st_ctimespec":"2025-09-29T08:17:17.877803000Z","st_mode":8592,"st_gen":0,"st_size":0,"st_rdev":268435483},"path_truncated":false},"is_platform_binary":false,"cdhash":"3F85C87E5B5E72EAB33D2C7A1362BA86A34D0283","is_es_client":false},"global_seq_num":0,"time":"2025-09-29T08:17:17.924250682Z","thread":{"thread_id":21612566},"schema_version":1,"action":{"result":{"result":{"auth":0},"result_type":0}},"mach_time":4210846481572,"event_type":43,"action_type":1,"seq_num":0,"version":10,"event":{"lookup":{"source_dir":{"stat":{"st_dev":16777239,"st_atimespec":"2025-09-09T07:15:49.000000000Z","st_rdev":0,"st_blksize":4096,"st_size":704,"st_mtimespec":"2025-09-09T07:15:49.000000000Z","st_birthtimespec":"2025-09-09T07:15:49.000000000Z","st_blocks":0,"st_flags":1048576,"st_gen":0,"st_uid":0,"st_mode":16877,"st_gid":0,"st_ino":2,"st_nlink":22,"st_ctimespec":"2025-09-09T07:15:49.000000000Z"},"path_truncated":false,"path":"\/"},"relative_target":"Users\/tmc\/.claude\/.config.json"}}} 7 | , 8 | {"process":{"codesigning_flags":570556963,"tty":{"path":"\/dev\/ttys027","path_truncated":false,"stat":{"st_rdev":268435483,"st_birthtimespec":"1970-01-01T00:00:00.000000000Z","st_nlink":1,"st_ctimespec":"2025-09-29T08:17:17.877803000Z","st_ino":1969,"st_size":0,"st_gid":4,"st_blocks":0,"st_mtimespec":"2025-09-29T08:17:17.877803000Z","st_blksize":65536,"st_atimespec":"2025-09-29T03:55:25.904783000Z","st_mode":8592,"st_uid":501,"st_dev":1552022631,"st_flags":0,"st_gen":0}},"is_es_client":false,"cs_validation_category":10,"original_ppid":29351,"ppid":29351,"start_time":"2025-09-28T02:05:45.339970Z","team_id":null,"cdhash":"3F85C87E5B5E72EAB33D2C7A1362BA86A34D0283","signing_id":"node","is_platform_binary":false,"session_id":29342,"executable":{"path":"\/Users\/tmc\/.local\/homebrew\/Cellar\/node\/24.4.1\/bin\/node","path_truncated":false,"stat":{"st_blocks":173912,"st_atimespec":"2025-09-29T08:17:05.863502863Z","st_size":89042552,"st_rdev":0,"st_uid":501,"st_birthtimespec":"2025-07-19T15:52:48.169930626Z","st_blksize":4096,"st_nlink":1,"st_flags":0,"st_mode":33133,"st_ctimespec":"2025-07-19T15:52:57.775450573Z","st_gid":20,"st_dev":16777239,"st_ino":65930357,"st_mtimespec":"2025-07-19T15:52:48.169930626Z","st_gen":0}},"group_id":42221,"responsible_audit_token":{"egid":20,"pid":20792,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":12130782},"parent_audit_token":{"egid":20,"pid":29351,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":19845966},"audit_token":{"egid":20,"pid":42221,"asid":100024,"euid":501,"ruid":501,"auid":501,"rgid":20,"pidversion":19867158}},"global_seq_num":1,"time":"2025-09-29T08:17:17.924273515Z","thread":{"thread_id":21612566},"schema_version":1,"action":{"result":{"result":{"auth":0},"result_type":0}},"mach_time":4210846482120,"event_type":43,"action_type":1,"seq_num":1,"version":10,"event":{"lookup":{"source_dir":{"stat":{"st_rdev":0,"st_birthtimespec":"2025-09-09T07:15:49.000000000Z","st_nlink":22,"st_ctimespec":"2025-09-09T07:15:49.000000000Z","st_ino":2,"st_size":704,"st_gid":0,"st_blocks":0,"st_mtimespec":"2025-09-09T07:15:49.000000000Z","st_blksize":4096,"st_atimespec":"2025-09-09T07:15:49.000000000Z","st_mode":16877,"st_uid":0,"st_dev":16777239,"st_flags":1048576,"st_gen":0},"path_truncated":false,"path":"\/"},"relative_target":"Users\/tmc\/.claude.json"}}} 9 | 10 | ] 11 | 12 | -- eslog2.go -- 13 | package test_package 14 | 15 | type eslog2 struct { 16 | Action struct { 17 | Result struct { 18 | Result struct { 19 | Auth float64 `json:"auth,omitempty"` 20 | } `json:"result,omitempty"` 21 | ResultType float64 `json:"result_type,omitempty"` 22 | } `json:"result,omitempty"` 23 | } `json:"action,omitempty"` 24 | ActionType float64 `json:"action_type,omitempty"` 25 | Event struct { 26 | Lookup struct { 27 | RelativeTarget string `json:"relative_target,omitempty"` 28 | SourceDir struct { 29 | Path string `json:"path,omitempty"` 30 | PathTruncated bool `json:"path_truncated,omitempty"` 31 | Stat struct { 32 | StAtimespec string `json:"st_atimespec,omitempty"` 33 | StBirthtimespec string `json:"st_birthtimespec,omitempty"` 34 | StBlksize float64 `json:"st_blksize,omitempty"` 35 | StBlocks float64 `json:"st_blocks,omitempty"` 36 | StCtimespec string `json:"st_ctimespec,omitempty"` 37 | StDev float64 `json:"st_dev,omitempty"` 38 | StFlags float64 `json:"st_flags,omitempty"` 39 | StGen float64 `json:"st_gen,omitempty"` 40 | StGid float64 `json:"st_gid,omitempty"` 41 | StIno float64 `json:"st_ino,omitempty"` 42 | StMode float64 `json:"st_mode,omitempty"` 43 | StMtimespec string `json:"st_mtimespec,omitempty"` 44 | StNlink float64 `json:"st_nlink,omitempty"` 45 | StRdev float64 `json:"st_rdev,omitempty"` 46 | StSize float64 `json:"st_size,omitempty"` 47 | StUid float64 `json:"st_uid,omitempty"` 48 | } `json:"stat,omitempty"` 49 | } `json:"source_dir,omitempty"` 50 | } `json:"lookup,omitempty"` 51 | } `json:"event,omitempty"` 52 | EventType float64 `json:"event_type,omitempty"` 53 | GlobalSeqNum float64 `json:"global_seq_num,omitempty"` 54 | MachTime float64 `json:"mach_time,omitempty"` 55 | Process struct { 56 | AuditToken struct { 57 | Asid float64 `json:"asid,omitempty"` 58 | Auid float64 `json:"auid,omitempty"` 59 | Egid float64 `json:"egid,omitempty"` 60 | Euid float64 `json:"euid,omitempty"` 61 | Pid float64 `json:"pid,omitempty"` 62 | Pidversion float64 `json:"pidversion,omitempty"` 63 | Rgid float64 `json:"rgid,omitempty"` 64 | Ruid float64 `json:"ruid,omitempty"` 65 | } `json:"audit_token,omitempty"` 66 | Cdhash string `json:"cdhash,omitempty"` 67 | CodesigningFlags float64 `json:"codesigning_flags,omitempty"` 68 | CsValidationCategory float64 `json:"cs_validation_category,omitempty"` 69 | Executable struct { 70 | Path string `json:"path,omitempty"` 71 | PathTruncated bool `json:"path_truncated,omitempty"` 72 | Stat struct { 73 | StAtimespec string `json:"st_atimespec,omitempty"` 74 | StBirthtimespec string `json:"st_birthtimespec,omitempty"` 75 | StBlksize float64 `json:"st_blksize,omitempty"` 76 | StBlocks float64 `json:"st_blocks,omitempty"` 77 | StCtimespec string `json:"st_ctimespec,omitempty"` 78 | StDev float64 `json:"st_dev,omitempty"` 79 | StFlags float64 `json:"st_flags,omitempty"` 80 | StGen float64 `json:"st_gen,omitempty"` 81 | StGid float64 `json:"st_gid,omitempty"` 82 | StIno float64 `json:"st_ino,omitempty"` 83 | StMode float64 `json:"st_mode,omitempty"` 84 | StMtimespec string `json:"st_mtimespec,omitempty"` 85 | StNlink float64 `json:"st_nlink,omitempty"` 86 | StRdev float64 `json:"st_rdev,omitempty"` 87 | StSize float64 `json:"st_size,omitempty"` 88 | StUid float64 `json:"st_uid,omitempty"` 89 | } `json:"stat,omitempty"` 90 | } `json:"executable,omitempty"` 91 | GroupID float64 `json:"group_id,omitempty"` 92 | IsEsClient bool `json:"is_es_client,omitempty"` 93 | IsPlatformBinary bool `json:"is_platform_binary,omitempty"` 94 | OriginalPpid float64 `json:"original_ppid,omitempty"` 95 | ParentAuditToken struct { 96 | Asid float64 `json:"asid,omitempty"` 97 | Auid float64 `json:"auid,omitempty"` 98 | Egid float64 `json:"egid,omitempty"` 99 | Euid float64 `json:"euid,omitempty"` 100 | Pid float64 `json:"pid,omitempty"` 101 | Pidversion float64 `json:"pidversion,omitempty"` 102 | Rgid float64 `json:"rgid,omitempty"` 103 | Ruid float64 `json:"ruid,omitempty"` 104 | } `json:"parent_audit_token,omitempty"` 105 | Ppid float64 `json:"ppid,omitempty"` 106 | ResponsibleAuditToken struct { 107 | Asid float64 `json:"asid,omitempty"` 108 | Auid float64 `json:"auid,omitempty"` 109 | Egid float64 `json:"egid,omitempty"` 110 | Euid float64 `json:"euid,omitempty"` 111 | Pid float64 `json:"pid,omitempty"` 112 | Pidversion float64 `json:"pidversion,omitempty"` 113 | Rgid float64 `json:"rgid,omitempty"` 114 | Ruid float64 `json:"ruid,omitempty"` 115 | } `json:"responsible_audit_token,omitempty"` 116 | SessionID float64 `json:"session_id,omitempty"` 117 | SigningID string `json:"signing_id,omitempty"` 118 | StartTime string `json:"start_time,omitempty"` 119 | TeamID any `json:"team_id,omitempty"` 120 | Tty struct { 121 | Path string `json:"path,omitempty"` 122 | PathTruncated bool `json:"path_truncated,omitempty"` 123 | Stat struct { 124 | StAtimespec string `json:"st_atimespec,omitempty"` 125 | StBirthtimespec string `json:"st_birthtimespec,omitempty"` 126 | StBlksize float64 `json:"st_blksize,omitempty"` 127 | StBlocks float64 `json:"st_blocks,omitempty"` 128 | StCtimespec string `json:"st_ctimespec,omitempty"` 129 | StDev float64 `json:"st_dev,omitempty"` 130 | StFlags float64 `json:"st_flags,omitempty"` 131 | StGen float64 `json:"st_gen,omitempty"` 132 | StGid float64 `json:"st_gid,omitempty"` 133 | StIno float64 `json:"st_ino,omitempty"` 134 | StMode float64 `json:"st_mode,omitempty"` 135 | StMtimespec string `json:"st_mtimespec,omitempty"` 136 | StNlink float64 `json:"st_nlink,omitempty"` 137 | StRdev float64 `json:"st_rdev,omitempty"` 138 | StSize float64 `json:"st_size,omitempty"` 139 | StUid float64 `json:"st_uid,omitempty"` 140 | } `json:"stat,omitempty"` 141 | } `json:"tty,omitempty"` 142 | } `json:"process,omitempty"` 143 | SchemaVersion float64 `json:"schema_version,omitempty"` 144 | SeqNum float64 `json:"seq_num,omitempty"` 145 | Thread struct { 146 | ThreadID float64 `json:"thread_id,omitempty"` 147 | } `json:"thread,omitempty"` 148 | Time string `json:"time,omitempty"` 149 | Version float64 `json:"version,omitempty"` 150 | } 151 | -------------------------------------------------------------------------------- /txtar_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "regexp" 10 | "strings" 11 | "sync" 12 | "testing" 13 | 14 | "github.com/google/go-cmp/cmp" 15 | "golang.org/x/tools/txtar" 16 | ) 17 | 18 | var writeTxtarGolden = flag.String("write-txtar-golden", "", "If set, writes out golden files in txtar archives matching this regexp pattern (use '.' to match all)") 19 | var forceLegacyPattern = flag.String("force-legacy-pattern", "", "If set, forces legacy mode for txtar files matching this regexp pattern") 20 | 21 | // shouldWriteGolden determines if golden files should be written for a txtar file 22 | func shouldWriteGolden(filename string) bool { 23 | if *writeTxtarGolden == "" { 24 | // Not set, don't write golden files 25 | return false 26 | } 27 | // Check regexp pattern 28 | matched, err := regexp.MatchString(*writeTxtarGolden, filename) 29 | return err == nil && matched 30 | } 31 | 32 | // shouldRunTxtarFile determines if a txtar file should run based on mode and comment 33 | func shouldRunTxtarFile(comment string, filename string) bool { 34 | hasLegacyCompat := strings.Contains(strings.ToLower(comment), "legacy-compat") 35 | 36 | // Check if this file matches the force legacy pattern 37 | isForceMatch := false 38 | if *forceLegacyPattern != "" { 39 | matched, err := regexp.MatchString(*forceLegacyPattern, filename) 40 | if err == nil && matched { 41 | isForceMatch = true 42 | } 43 | } 44 | 45 | isLegacy := legacyMode || isForceMatch 46 | 47 | if isLegacy { 48 | // In legacy mode, only run files with legacy-compat 49 | return hasLegacyCompat 50 | } 51 | 52 | // Default mode: run all files (both legacy-compat and non-legacy-compat) 53 | return true 54 | } 55 | 56 | func TestTxtarGenerate(t *testing.T) { 57 | // Look for txtar files in testdata and current directory 58 | txtarFiles, err := filepath.Glob("testdata/*.txtar") 59 | if err != nil { 60 | t.Fatalf("failed to find txtar files in testdata: %v", err) 61 | } 62 | 63 | moreTxtarFiles, err := filepath.Glob("*.txtar") 64 | if err != nil { 65 | t.Fatalf("failed to find txtar files in current dir: %v", err) 66 | } 67 | 68 | txtarFiles = append(txtarFiles, moreTxtarFiles...) 69 | 70 | if len(txtarFiles) == 0 { 71 | t.Skip("no txtar files found") 72 | } 73 | 74 | for _, txtarFile := range txtarFiles { 75 | // Check if this file should run in the current mode 76 | archive, err := txtar.ParseFile(txtarFile) 77 | if err != nil { 78 | t.Errorf("failed to parse txtar file %s for mode check: %v", txtarFile, err) 79 | continue 80 | } 81 | 82 | if !shouldRunTxtarFile(string(archive.Comment), filepath.Base(txtarFile)) { 83 | t.Logf("Skipping %s (mode filter)", filepath.Base(txtarFile)) 84 | continue 85 | } 86 | 87 | t.Run(filepath.Base(txtarFile), func(t *testing.T) { 88 | runTxtarTest(t, txtarFile) 89 | }) 90 | } 91 | } 92 | 93 | func runTxtarTest(t *testing.T, txtarFile string) { 94 | archive, err := txtar.ParseFile(txtarFile) 95 | if err != nil { 96 | t.Fatalf("failed to parse txtar file %s: %v", txtarFile, err) 97 | } 98 | 99 | // Parse flags from comment header 100 | // Example: # flags: -extract-structs -stat-comments 101 | comment := string(archive.Comment) 102 | var extractStructs bool 103 | var statComments bool 104 | for _, line := range strings.Split(comment, "\n") { 105 | if strings.HasPrefix(strings.TrimSpace(line), "# flags:") { 106 | flagLine := strings.TrimPrefix(strings.TrimSpace(line), "# flags:") 107 | if strings.Contains(flagLine, "-extract-structs") { 108 | extractStructs = true 109 | } 110 | if strings.Contains(flagLine, "-stat-comments") { 111 | statComments = true 112 | } 113 | } 114 | } 115 | 116 | // Group files by test case (based on prefix before first dot) 117 | testCases := make(map[string]struct { 118 | json []byte 119 | golden []byte 120 | expectedErr []byte 121 | roundtrip []byte 122 | name string 123 | }) 124 | 125 | for _, file := range archive.Files { 126 | name := file.Name 127 | if strings.HasSuffix(name, ".json") { 128 | testName := strings.TrimSuffix(name, ".json") 129 | tc := testCases[testName] 130 | tc.json = file.Data 131 | tc.name = testName 132 | testCases[testName] = tc 133 | } else if strings.HasSuffix(name, ".go") { 134 | testName := strings.TrimSuffix(name, ".go") 135 | tc := testCases[testName] 136 | tc.golden = file.Data 137 | tc.name = testName 138 | testCases[testName] = tc 139 | } else if strings.HasSuffix(name, ".err") { 140 | testName := strings.TrimSuffix(name, ".err") 141 | tc := testCases[testName] 142 | tc.expectedErr = file.Data 143 | tc.name = testName 144 | testCases[testName] = tc 145 | } else if strings.HasSuffix(name, ".roundtrip") { 146 | testName := strings.TrimSuffix(name, ".roundtrip") 147 | tc := testCases[testName] 148 | tc.roundtrip = file.Data 149 | tc.name = testName 150 | testCases[testName] = tc 151 | } 152 | } 153 | 154 | var modifiedArchive *txtar.Archive 155 | var needsUpdate bool 156 | 157 | for testName, tc := range testCases { 158 | t.Run(testName, func(t *testing.T) { 159 | if len(tc.json) == 0 { 160 | t.Skip("no JSON input found") 161 | return 162 | } 163 | 164 | g := &generator{ 165 | TypeName: testName, 166 | PackageName: "test_package", 167 | OmitEmpty: true, 168 | ExtractStructs: extractStructs, 169 | StatComments: statComments, 170 | } 171 | 172 | var buf bytes.Buffer 173 | err := g.generate(&buf, bytes.NewReader(tc.json)) 174 | 175 | // Check if we expect an error 176 | if len(tc.expectedErr) > 0 { 177 | expectedErrStr := strings.TrimSpace(string(tc.expectedErr)) 178 | if err == nil { 179 | t.Errorf("expected error containing %q, but got none", expectedErrStr) 180 | return 181 | } 182 | if !strings.Contains(err.Error(), expectedErrStr) { 183 | t.Errorf("expected error containing %q, got %q", expectedErrStr, err.Error()) 184 | } 185 | t.Logf("generator.generate() got expected error = %v", err) 186 | return 187 | } 188 | 189 | // If no error expected, but we got one 190 | if err != nil { 191 | if shouldWriteGolden(filepath.Base(txtarFile)) { 192 | // Write error expectation file 193 | if modifiedArchive == nil { 194 | modifiedArchive = &txtar.Archive{ 195 | Comment: archive.Comment, 196 | Files: make([]txtar.File, len(archive.Files)), 197 | } 198 | copy(modifiedArchive.Files, archive.Files) 199 | } 200 | 201 | // Find and update the corresponding .err file 202 | errFileName := testName + ".err" 203 | found := false 204 | for i, file := range modifiedArchive.Files { 205 | if file.Name == errFileName { 206 | modifiedArchive.Files[i].Data = []byte(err.Error()) 207 | found = true 208 | needsUpdate = true 209 | break 210 | } 211 | } 212 | 213 | // If not found, append new error file 214 | if !found { 215 | modifiedArchive.Files = append(modifiedArchive.Files, txtar.File{ 216 | Name: errFileName, 217 | Data: []byte(err.Error()), 218 | }) 219 | needsUpdate = true 220 | } 221 | 222 | t.Logf("wrote error expectation for %s: %v", testName, err) 223 | return 224 | } 225 | // Log the raw generated output if it's a FormatError 226 | if formatErr, ok := err.(*FormatError); ok { 227 | t.Logf("Error at line %d, column %d", formatErr.LineNum, formatErr.Column) 228 | } 229 | t.Errorf("generator.generate() error = %v", err) 230 | return 231 | } 232 | 233 | got := buf.String() 234 | 235 | // Always log generated code for debugging 236 | t.Logf("Generated code for %s:\n%s", testName, got) 237 | 238 | if shouldWriteGolden(filepath.Base(txtarFile)) { 239 | // Update the golden file in the archive 240 | if modifiedArchive == nil { 241 | modifiedArchive = &txtar.Archive{ 242 | Comment: archive.Comment, 243 | Files: make([]txtar.File, len(archive.Files)), 244 | } 245 | copy(modifiedArchive.Files, archive.Files) 246 | } 247 | 248 | // Find and update the corresponding .go file 249 | goldenFileName := testName + ".go" 250 | found := false 251 | for i, file := range modifiedArchive.Files { 252 | if file.Name == goldenFileName { 253 | modifiedArchive.Files[i].Data = []byte(got) 254 | found = true 255 | needsUpdate = true 256 | break 257 | } 258 | } 259 | 260 | // If not found, append new golden file 261 | if !found { 262 | modifiedArchive.Files = append(modifiedArchive.Files, txtar.File{ 263 | Name: goldenFileName, 264 | Data: []byte(got), 265 | }) 266 | needsUpdate = true 267 | } 268 | 269 | t.Logf("updated golden file for %s in txtar archive", testName) 270 | return 271 | } 272 | 273 | if len(tc.golden) == 0 { 274 | t.Logf("no golden file found for %s, generated:\n%s", testName, got) 275 | return 276 | } 277 | 278 | want := string(tc.golden) 279 | 280 | if diff := cmp.Diff(want, got); diff != "" { 281 | t.Errorf("generate() mismatch for %s (-want +got):\n%s", testName, diff) 282 | } 283 | 284 | // Run roundtrip test if .roundtrip file exists 285 | if len(tc.roundtrip) > 0 { 286 | t.Run("roundtrip", func(t *testing.T) { 287 | // Run roundtrip validation with stderr capture 288 | var stderrBuf bytes.Buffer 289 | 290 | // Temporarily redirect stderr 291 | origStderr := os.Stderr 292 | r, w, _ := os.Pipe() 293 | os.Stderr = w 294 | 295 | // Run test in goroutine to avoid deadlock 296 | var wg sync.WaitGroup 297 | wg.Add(1) 298 | var testErr error 299 | go func() { 300 | defer wg.Done() 301 | testErr = runRoundtripTestWithData(g, tc.json) 302 | w.Close() 303 | }() 304 | 305 | // Read from pipe 306 | io.Copy(&stderrBuf, r) 307 | wg.Wait() 308 | 309 | // Restore stderr 310 | os.Stderr = origStderr 311 | 312 | if testErr != nil { 313 | t.Errorf("roundtrip test failed: %v", testErr) 314 | return 315 | } 316 | 317 | gotRoundtrip := strings.TrimSpace(stderrBuf.String()) 318 | wantRoundtrip := strings.TrimSpace(string(tc.roundtrip)) 319 | 320 | if shouldWriteGolden(filepath.Base(txtarFile)) { 321 | // Update roundtrip golden file 322 | if modifiedArchive == nil { 323 | modifiedArchive = &txtar.Archive{ 324 | Comment: archive.Comment, 325 | Files: make([]txtar.File, len(archive.Files)), 326 | } 327 | copy(modifiedArchive.Files, archive.Files) 328 | } 329 | 330 | roundtripFileName := testName + ".roundtrip" 331 | found := false 332 | for i, file := range modifiedArchive.Files { 333 | if file.Name == roundtripFileName { 334 | modifiedArchive.Files[i].Data = []byte(gotRoundtrip) 335 | found = true 336 | needsUpdate = true 337 | break 338 | } 339 | } 340 | 341 | if !found { 342 | modifiedArchive.Files = append(modifiedArchive.Files, txtar.File{ 343 | Name: roundtripFileName, 344 | Data: []byte(gotRoundtrip), 345 | }) 346 | needsUpdate = true 347 | } 348 | 349 | t.Logf("updated roundtrip golden for %s", testName) 350 | return 351 | } 352 | 353 | if diff := cmp.Diff(wantRoundtrip, gotRoundtrip); diff != "" { 354 | t.Errorf("roundtrip output mismatch (-want +got):\n%s", diff) 355 | } 356 | }) 357 | } 358 | }) 359 | } 360 | 361 | // Write updated txtar file if golden files were updated 362 | if shouldWriteGolden(filepath.Base(txtarFile)) && needsUpdate && modifiedArchive != nil { 363 | data := txtar.Format(modifiedArchive) 364 | err := os.WriteFile(txtarFile, data, 0644) 365 | if err != nil { 366 | t.Errorf("failed to write updated txtar file %s: %v", txtarFile, err) 367 | } else { 368 | t.Logf("wrote updated txtar file: %s", txtarFile) 369 | } 370 | } 371 | } 372 | -------------------------------------------------------------------------------- /type.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | ) 8 | 9 | type Fields []*Type 10 | 11 | func (f Fields) String() string { 12 | result := []string{} 13 | for _, field := range f { 14 | result = append(result, field.String()) 15 | } 16 | return strings.Join(result, "\n") 17 | } 18 | 19 | type Type struct { 20 | Name string 21 | Repeated bool 22 | Type string 23 | Tags map[string]string 24 | Children Fields 25 | Config *generator 26 | Stat *FieldStat // Optional field statistics for comments 27 | ExtractedTypeName string // If set, use this type name instead of inline struct 28 | } 29 | 30 | func (t *Type) GetType() string { 31 | // Use extracted type name if available 32 | if t.ExtractedTypeName != "" { 33 | if t.Repeated { 34 | return "[]" + t.ExtractedTypeName 35 | } 36 | return t.ExtractedTypeName 37 | } 38 | 39 | if t.Type == "nil" { 40 | t.Type = "any" 41 | } 42 | 43 | if t.Repeated { 44 | return "[]" + t.Type 45 | } 46 | return t.Type 47 | } 48 | 49 | func (t *Type) GetTags() string { 50 | if len(t.Tags) == 0 { 51 | return "" 52 | } 53 | 54 | keys := make([]string, 0, len(t.Tags)) 55 | for key := range t.Tags { 56 | keys = append(keys, key) 57 | } 58 | sort.Strings(keys) 59 | parts := []string{} 60 | for _, k := range keys { 61 | v := t.Tags[k] 62 | if k == "json" && t.Config.OmitEmpty { 63 | v += ",omitempty" 64 | } 65 | parts = append(parts, fmt.Sprintf(`%v:"%v"`, k, v)) 66 | } 67 | return fmt.Sprintf("`%v`", strings.Join(parts, ",")) 68 | } 69 | 70 | func (t *Type) GetStatComment() string { 71 | if t.Stat == nil || t.Config == nil || !t.Config.StatComments { 72 | return "" 73 | } 74 | 75 | // Build comment with field statistics 76 | comments := []string{} 77 | 78 | // Add occurrence count 79 | if t.Config.stats != nil && t.Config.stats.TotalLines > 0 { 80 | percentage := float64(t.Stat.TotalCount) * 100.0 / float64(t.Config.stats.TotalLines) 81 | comments = append(comments, fmt.Sprintf("seen in %.1f%% (%d/%d)", 82 | percentage, t.Stat.TotalCount, t.Config.stats.TotalLines)) 83 | } 84 | 85 | // Add type distribution if multiple types seen 86 | if len(t.Stat.Types) > 1 { 87 | typeInfo := []string{} 88 | for typeName, count := range t.Stat.Types { 89 | typeInfo = append(typeInfo, fmt.Sprintf("%s:%d", typeName, count)) 90 | } 91 | sort.Strings(typeInfo) 92 | comments = append(comments, "types: "+strings.Join(typeInfo, ", ")) 93 | } 94 | 95 | // For numeric fields, show percentiles if they appear to be continuous 96 | if t.Type == "float64" && len(t.Stat.NumericVals) > 0 { 97 | // Check if values look like continuous data (not just small integers/enums) 98 | continuousData := false 99 | for _, v := range t.Stat.NumericVals { 100 | if v != float64(int(v)) || v < -100 || v > 100 { 101 | continuousData = true 102 | break 103 | } 104 | } 105 | 106 | if continuousData && len(t.Stat.NumericVals) > 2 { 107 | // Calculate percentiles 108 | sorted := make([]float64, len(t.Stat.NumericVals)) 109 | copy(sorted, t.Stat.NumericVals) 110 | sort.Float64s(sorted) 111 | 112 | getPercentile := func(p float64) float64 { 113 | index := p * float64(len(sorted)-1) 114 | lower := int(index) 115 | upper := lower + 1 116 | if upper >= len(sorted) { 117 | return sorted[lower] 118 | } 119 | weight := index - float64(lower) 120 | return sorted[lower]*(1-weight) + sorted[upper]*weight 121 | } 122 | 123 | min := sorted[0] 124 | max := sorted[len(sorted)-1] 125 | 126 | // Format based on the range and values 127 | formatVal := func(v float64) string { 128 | if v == float64(int(v)) && v > -1000000 && v < 1000000 { 129 | return fmt.Sprintf("%.0f", v) 130 | } 131 | return fmt.Sprintf("%.2g", v) 132 | } 133 | 134 | if len(sorted) >= 10 { 135 | comments = append(comments, fmt.Sprintf("range: [%s, p25:%s, p50:%s, p75:%s, p90:%s, p99:%s, %s]", 136 | formatVal(min), formatVal(getPercentile(0.25)), formatVal(getPercentile(0.5)), 137 | formatVal(getPercentile(0.75)), formatVal(getPercentile(0.90)), formatVal(getPercentile(0.99)), formatVal(max))) 138 | } else { 139 | comments = append(comments, fmt.Sprintf("range: [%s, %s]", formatVal(min), formatVal(max))) 140 | } 141 | } else if len(t.Stat.Values) > 0 && len(t.Stat.Values) < 10 { 142 | // For enum-like numbers, show the values in order of appearance with percentages 143 | valueStrings := make([]string, 0, len(t.Stat.ValueOrder)) 144 | for _, val := range t.Stat.ValueOrder { 145 | if count, exists := t.Stat.Values[val]; exists { 146 | percentage := float64(count) * 100.0 / float64(t.Stat.TotalCount) 147 | valueStrings = append(valueStrings, fmt.Sprintf("%s:%.1f%%", val, percentage)) 148 | } 149 | } 150 | comments = append(comments, fmt.Sprintf("values: %s", strings.Join(valueStrings, ", "))) 151 | } 152 | } else if len(t.Stat.Values) > 0 && len(t.Stat.Values) < 10 && t.Type != "float64" { 153 | // For non-numeric fields with low cardinality, show in order of appearance with percentages 154 | valueStrings := make([]string, 0, len(t.Stat.ValueOrder)) 155 | for _, val := range t.Stat.ValueOrder { 156 | if count, exists := t.Stat.Values[val]; exists { 157 | percentage := float64(count) * 100.0 / float64(t.Stat.TotalCount) 158 | displayVal := val 159 | if len(val) > 20 { 160 | // Truncate long values 161 | displayVal = fmt.Sprintf("%s...", val[:17]) 162 | } 163 | valueStrings = append(valueStrings, fmt.Sprintf("%q:%.1f%%", displayVal, percentage)) 164 | } 165 | } 166 | comments = append(comments, fmt.Sprintf("values: %s", strings.Join(valueStrings, ", "))) 167 | } else if len(t.Stat.Values) >= 10 { 168 | // Just show cardinality if too many unique values 169 | comments = append(comments, fmt.Sprintf("%d unique values", len(t.Stat.Values))) 170 | } 171 | 172 | if len(comments) > 0 { 173 | return " // " + strings.Join(comments, ", ") 174 | } 175 | return "" 176 | } 177 | 178 | func (t *Type) String() string { 179 | if t.Config != nil && t.Config.typeTemplate != nil { 180 | return t.Config.renderTypeWithTemplate(t) 181 | } 182 | return t.Config.renderType(t) 183 | } 184 | 185 | func (t *Type) Merge(t2 *Type) error { 186 | if strings.Trim(t.Type, "*") != strings.Trim(t2.Type, "*") { 187 | if t.Type == "nil" { 188 | // When merging nil with a struct, copy the whole type (including children) 189 | if t2.Type == "struct" { 190 | t.Type = "*struct" 191 | t.Children = t2.Children 192 | } else { 193 | t.Type = fmt.Sprintf("*%s", strings.Trim(t2.Type, "*")) 194 | } 195 | return nil 196 | } else if t2.Type == "nil" { 197 | // When merging struct with nil, make it a pointer 198 | if t.Type == "struct" { 199 | t.Type = "*struct" 200 | // Children remain the same 201 | } else { 202 | t.Type = fmt.Sprintf("*%s", strings.Trim(t.Type, "*")) 203 | } 204 | return nil 205 | } else { 206 | t.Type = "any" 207 | return nil 208 | } 209 | } 210 | 211 | fields := map[string]*Type{} 212 | for _, typ := range t.Children { 213 | fields[typ.Name] = typ 214 | } 215 | for _, typ := range t2.Children { 216 | field, ok := fields[typ.Name] 217 | if !ok { 218 | t.Children = append(t.Children, typ) 219 | continue 220 | } 221 | if err := field.Merge(typ); err != nil { 222 | return fmt.Errorf("issue with '%v': %w", field.Name, err) 223 | } 224 | } 225 | 226 | return nil 227 | } 228 | 229 | // renderType renders the type as a Go struct definition. 230 | func (g *generator) renderType(t *Type) string { 231 | return g.renderTypeWithKeyword(t, true) 232 | } 233 | 234 | // renderInlineStruct renders a struct type inline (for nested anonymous structs) 235 | func (g *generator) renderInlineStruct(t *Type, depth int) string { 236 | indent := strings.Repeat("\t", depth) 237 | 238 | // Handle pointer to struct 239 | if t.Type == "*struct" { 240 | if len(t.Children) == 0 { 241 | return "*struct{}" 242 | } 243 | // Build the pointer struct with proper indentation 244 | var result strings.Builder 245 | result.WriteString("*struct {\n") 246 | for _, child := range t.Children { 247 | result.WriteString(indent + "\t") 248 | result.WriteString(child.Name) 249 | result.WriteString(" ") 250 | 251 | if child.Type == "struct" && len(child.Children) > 0 { 252 | // Recursively render nested struct 253 | result.WriteString(g.renderInlineStruct(child, depth+1)) 254 | } else if child.Type == "*struct" && len(child.Children) > 0 { 255 | // Recursively render nested pointer struct 256 | result.WriteString(g.renderInlineStruct(child, depth+1)) 257 | } else if child.Type == "struct" && len(child.Children) == 0 { 258 | // Empty struct 259 | result.WriteString("struct{}") 260 | } else if child.Type == "*struct" && len(child.Children) == 0 { 261 | // Empty pointer struct 262 | result.WriteString("*struct{}") 263 | } else { 264 | // Simple type 265 | if child.Type == "nil" { 266 | child.Type = "any" 267 | } 268 | typeStr := child.Type 269 | if child.Repeated { 270 | typeStr = "[]" + typeStr 271 | } 272 | if child.ExtractedTypeName != "" { 273 | typeStr = child.ExtractedTypeName 274 | if child.Repeated { 275 | typeStr = "[]" + child.ExtractedTypeName 276 | } 277 | } 278 | result.WriteString(typeStr) 279 | } 280 | 281 | if tags := child.GetTags(); tags != "" { 282 | result.WriteString(" ") 283 | result.WriteString(tags) 284 | } 285 | // Add stat comments if enabled 286 | if g.StatComments { 287 | if comment := child.GetStatComment(); comment != "" { 288 | result.WriteString(comment) 289 | } 290 | } 291 | result.WriteString("\n") 292 | } 293 | result.WriteString(indent + "}") 294 | return result.String() 295 | } 296 | 297 | if t.Type != "struct" { 298 | // Not a struct, just return the type 299 | if t.Repeated { 300 | return "[]" + t.Type 301 | } 302 | return t.Type 303 | } 304 | 305 | if len(t.Children) == 0 { 306 | // Empty struct 307 | if t.Repeated { 308 | return "[]struct{}" 309 | } 310 | return "struct{}" 311 | } 312 | 313 | // Build the struct with proper indentation 314 | var result strings.Builder 315 | if t.Repeated { 316 | result.WriteString("[]") 317 | } 318 | result.WriteString("struct {\n") 319 | 320 | for _, child := range t.Children { 321 | result.WriteString(indent + "\t") 322 | result.WriteString(child.Name) 323 | result.WriteString(" ") 324 | 325 | if child.Type == "struct" && len(child.Children) > 0 { 326 | // Recursively render nested struct 327 | result.WriteString(g.renderInlineStruct(child, depth+1)) 328 | } else if child.Type == "struct" && len(child.Children) == 0 { 329 | // Empty struct 330 | result.WriteString("struct{}") 331 | } else { 332 | // Simple type - don't call GetType to avoid infinite recursion 333 | if child.Type == "nil" { 334 | child.Type = "any" 335 | } 336 | typeStr := child.Type 337 | if child.Repeated { 338 | typeStr = "[]" + typeStr 339 | } 340 | if child.ExtractedTypeName != "" { 341 | typeStr = child.ExtractedTypeName 342 | if child.Repeated { 343 | typeStr = "[]" + child.ExtractedTypeName 344 | } 345 | } 346 | result.WriteString(typeStr) 347 | } 348 | 349 | if tags := child.GetTags(); tags != "" { 350 | result.WriteString(" ") 351 | result.WriteString(tags) 352 | } 353 | // Add stat comments if enabled 354 | if g.StatComments { 355 | if comment := child.GetStatComment(); comment != "" { 356 | result.WriteString(comment) 357 | } 358 | } 359 | result.WriteString("\n") 360 | } 361 | 362 | result.WriteString(indent + "}") 363 | return result.String() 364 | } 365 | 366 | // renderTypeWithKeyword renders the type, optionally including the 'type' keyword 367 | func (g *generator) renderTypeWithKeyword(t *Type, includeTypeKeyword bool) string { 368 | // If this is using an extracted type, don't render children 369 | if t.ExtractedTypeName != "" { 370 | if includeTypeKeyword { 371 | return fmt.Sprintf("type %s %s%s", t.Name, t.GetType(), t.GetTags()) 372 | } 373 | return fmt.Sprintf("%s %s%s", t.Name, t.GetType(), t.GetTags()) 374 | } 375 | 376 | // Check if this is a struct with no children 377 | if t.Type == "struct" && len(t.Children) == 0 { 378 | // Empty struct needs braces 379 | if includeTypeKeyword { 380 | return fmt.Sprintf("type %s struct {}%s", t.Name, t.GetTags()) 381 | } 382 | return fmt.Sprintf("%s struct {}%s", t.Name, t.GetTags()) 383 | } 384 | 385 | if len(t.Children) == 0 { 386 | // Non-struct types (like string, int, etc.) 387 | if includeTypeKeyword { 388 | return fmt.Sprintf("type %s %s%s", t.Name, t.GetType(), t.GetTags()) 389 | } 390 | return fmt.Sprintf("%s %s%s", t.Name, t.GetType(), t.GetTags()) 391 | } 392 | 393 | result := []string{} 394 | if includeTypeKeyword { 395 | result = append(result, fmt.Sprintf("type %s %s {", t.Name, t.GetType())) 396 | } else { 397 | result = append(result, fmt.Sprintf("%s %s {", t.Name, t.GetType())) 398 | } 399 | 400 | for _, child := range t.Children { 401 | result = append(result, fmt.Sprintf(" %s", g.renderTypeWithKeyword(child, false))) 402 | } 403 | result = append(result, fmt.Sprintf("}%s", t.GetTags())) 404 | return strings.Join(result, "\n") 405 | } 406 | -------------------------------------------------------------------------------- /testdata/exotic.txtar: -------------------------------------------------------------------------------- 1 | This txtar archive contains exotic and edge case JSON inputs to test the generator's robustness. 2 | 3 | -- unicode_fields.json -- 4 | { 5 | "🚀": "rocket", 6 | "café": "coffee", 7 | "東京": "tokyo", 8 | "Москва": "moscow", 9 | "ñoño": "spanish", 10 | "naïve": "french" 11 | } 12 | 13 | -- deeply_nested.json -- 14 | { 15 | "level1": { 16 | "level2": { 17 | "level3": { 18 | "level4": { 19 | "level5": { 20 | "level6": { 21 | "level7": { 22 | "level8": { 23 | "level9": { 24 | "level10": { 25 | "deep_value": "found it!" 26 | } 27 | } 28 | } 29 | } 30 | } 31 | } 32 | } 33 | } 34 | } 35 | } 36 | } 37 | 38 | -- mixed_array_types.json -- 39 | { 40 | "mixed": [ 41 | "string", 42 | 42, 43 | 3.14, 44 | true, 45 | null, 46 | {"nested": "object"}, 47 | ["nested", "array"] 48 | ] 49 | } 50 | 51 | -- extreme_numbers.json -- 52 | { 53 | "big_int": 9223372036854775807, 54 | "negative": -9223372036854775808, 55 | "zero": 0, 56 | "float_precision": 3.141592653589793238462643383279, 57 | "scientific": 1.23e10, 58 | "negative_scientific": -4.56e-7 59 | } 60 | 61 | -- special_strings.json -- 62 | { 63 | "empty": "", 64 | "whitespace": " \t\n\r ", 65 | "quotes": "\"quoted\" and 'single'", 66 | "backslashes": "\\path\\to\\file", 67 | "unicode_escape": "\u0048\u0065\u006C\u006C\u006F", 68 | "newlines": "line1\nline2\nline3", 69 | "emoji": "👋🌍🚀💻🎉", 70 | "special_chars": "!@#$%^&*()_+-=[]{}|;:,.<>?" 71 | } 72 | 73 | -- boolean_variations.json -- 74 | { 75 | "true_val": true, 76 | "false_val": false, 77 | "string_true": "true", 78 | "string_false": "false", 79 | "number_one": 1, 80 | "number_zero": 0 81 | } 82 | 83 | -- null_variations.json -- 84 | { 85 | "explicit_null": null, 86 | "string_null": "null", 87 | "empty_object": {}, 88 | "empty_array": [], 89 | "nested_nulls": { 90 | "inner": null, 91 | "array_with_nulls": [null, "value", null] 92 | } 93 | } 94 | 95 | -- complex_arrays.json -- 96 | { 97 | "array_of_objects": [ 98 | {"id": 1, "name": "first"}, 99 | {"id": 2, "name": "second", "extra": "field"}, 100 | {"id": 3, "different": "structure"} 101 | ], 102 | "nested_arrays": [ 103 | [1, 2, 3], 104 | ["a", "b", "c"], 105 | [true, false, null] 106 | ], 107 | "mixed_nested": [ 108 | {"array": [1, 2, 3]}, 109 | {"object": {"nested": true}}, 110 | {"mixed": [{"deep": "value"}]} 111 | ] 112 | } 113 | 114 | -- numeric_strings.json -- 115 | { 116 | "looks_like_int": "123", 117 | "looks_like_float": "45.67", 118 | "leading_zero": "007", 119 | "phone_number": "+1-555-123-4567", 120 | "zip_code": "12345-6789", 121 | "version": "1.2.3" 122 | } 123 | 124 | -- reserved_keywords.json -- 125 | { 126 | "type": "keyword", 127 | "func": "function", 128 | "var": "variable", 129 | "const": "constant", 130 | "package": "pkg", 131 | "import": "imp", 132 | "interface": "iface", 133 | "struct": "structure", 134 | "chan": "channel", 135 | "go": "goroutine" 136 | } 137 | 138 | -- case_variations.json -- 139 | { 140 | "camelCase": "camel", 141 | "PascalCase": "pascal", 142 | "snake_case": "snake", 143 | "kebab-case": "kebab", 144 | "SCREAMING_SNAKE": "screaming", 145 | "Mixed_Case-Types": "mixed", 146 | "dotted.field": "dotted", 147 | "spaced field": "spaced" 148 | } 149 | 150 | -- large_object.json -- 151 | { 152 | "field001": "value001", "field002": "value002", "field003": "value003", "field004": "value004", "field005": "value005", 153 | "field006": "value006", "field007": "value007", "field008": "value008", "field009": "value009", "field010": "value010", 154 | "field011": "value011", "field012": "value012", "field013": "value013", "field014": "value014", "field015": "value015", 155 | "field016": "value016", "field017": "value017", "field018": "value018", "field019": "value019", "field020": "value020", 156 | "nested": { 157 | "subfield001": 1, "subfield002": 2, "subfield003": 3, "subfield004": 4, "subfield005": 5, 158 | "subfield006": 6, "subfield007": 7, "subfield008": 8, "subfield009": 9, "subfield010": 10 159 | } 160 | } 161 | 162 | -- recursive_structure.json -- 163 | { 164 | "name": "root", 165 | "children": [ 166 | { 167 | "name": "child1", 168 | "children": [ 169 | { 170 | "name": "grandchild1", 171 | "children": [] 172 | }, 173 | { 174 | "name": "grandchild2", 175 | "children": [ 176 | { 177 | "name": "great-grandchild", 178 | "children": [] 179 | } 180 | ] 181 | } 182 | ] 183 | }, 184 | { 185 | "name": "child2", 186 | "children": [] 187 | } 188 | ] 189 | } 190 | 191 | -- single_value_types.json -- 192 | "just a string" 193 | 194 | -- number_only.json -- 195 | 42 196 | 197 | -- boolean_only.json -- 198 | true 199 | 200 | -- null_only.json -- 201 | null 202 | 203 | -- empty_array_only.json -- 204 | [] 205 | 206 | -- array_of_primitives.json -- 207 | [1, "two", 3.0, true, null] 208 | 209 | -- inconsistent_objects.json -- 210 | [ 211 | {"a": 1, "b": 2}, 212 | {"a": 1, "c": 3}, 213 | {"b": 2, "d": 4}, 214 | {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5} 215 | ] 216 | -- unicode_fields.go -- 217 | package test_package 218 | 219 | type unicode_fields struct { 220 | Café string `json:"café,omitempty"` 221 | Naïve string `json:"naïve,omitempty"` 222 | __oño string `json:"ñoño,omitempty"` 223 | __осква string `json:"Москва,omitempty"` 224 | ___京 string `json:"東京,omitempty"` 225 | ____ string `json:"🚀,omitempty"` 226 | } 227 | -- large_object.go -- 228 | package test_package 229 | 230 | type large_object struct { 231 | Field001 string `json:"field001,omitempty"` 232 | Field002 string `json:"field002,omitempty"` 233 | Field003 string `json:"field003,omitempty"` 234 | Field004 string `json:"field004,omitempty"` 235 | Field005 string `json:"field005,omitempty"` 236 | Field006 string `json:"field006,omitempty"` 237 | Field007 string `json:"field007,omitempty"` 238 | Field008 string `json:"field008,omitempty"` 239 | Field009 string `json:"field009,omitempty"` 240 | Field010 string `json:"field010,omitempty"` 241 | Field011 string `json:"field011,omitempty"` 242 | Field012 string `json:"field012,omitempty"` 243 | Field013 string `json:"field013,omitempty"` 244 | Field014 string `json:"field014,omitempty"` 245 | Field015 string `json:"field015,omitempty"` 246 | Field016 string `json:"field016,omitempty"` 247 | Field017 string `json:"field017,omitempty"` 248 | Field018 string `json:"field018,omitempty"` 249 | Field019 string `json:"field019,omitempty"` 250 | Field020 string `json:"field020,omitempty"` 251 | Nested struct { 252 | Subfield001 float64 `json:"subfield001,omitempty"` 253 | Subfield002 float64 `json:"subfield002,omitempty"` 254 | Subfield003 float64 `json:"subfield003,omitempty"` 255 | Subfield004 float64 `json:"subfield004,omitempty"` 256 | Subfield005 float64 `json:"subfield005,omitempty"` 257 | Subfield006 float64 `json:"subfield006,omitempty"` 258 | Subfield007 float64 `json:"subfield007,omitempty"` 259 | Subfield008 float64 `json:"subfield008,omitempty"` 260 | Subfield009 float64 `json:"subfield009,omitempty"` 261 | Subfield010 float64 `json:"subfield010,omitempty"` 262 | } `json:"nested,omitempty"` 263 | } 264 | -- inconsistent_objects.go -- 265 | package test_package 266 | 267 | type inconsistent_objects struct { 268 | A float64 `json:"a,omitempty"` 269 | B float64 `json:"b,omitempty"` 270 | C float64 `json:"c,omitempty"` 271 | D float64 `json:"d,omitempty"` 272 | E float64 `json:"e,omitempty"` 273 | } 274 | -- deeply_nested.go -- 275 | package test_package 276 | 277 | type deeply_nested struct { 278 | Level1 struct { 279 | Level2 struct { 280 | Level3 struct { 281 | Level4 struct { 282 | Level5 struct { 283 | Level6 struct { 284 | Level7 struct { 285 | Level8 struct { 286 | Level9 struct { 287 | Level10 struct { 288 | DeepValue string `json:"deep_value,omitempty"` 289 | } `json:"level10,omitempty"` 290 | } `json:"level9,omitempty"` 291 | } `json:"level8,omitempty"` 292 | } `json:"level7,omitempty"` 293 | } `json:"level6,omitempty"` 294 | } `json:"level5,omitempty"` 295 | } `json:"level4,omitempty"` 296 | } `json:"level3,omitempty"` 297 | } `json:"level2,omitempty"` 298 | } `json:"level1,omitempty"` 299 | } 300 | -- extreme_numbers.go -- 301 | package test_package 302 | 303 | type extreme_numbers struct { 304 | BigInt float64 `json:"big_int,omitempty"` 305 | FloatPrecision float64 `json:"float_precision,omitempty"` 306 | Negative float64 `json:"negative,omitempty"` 307 | NegativeScientific float64 `json:"negative_scientific,omitempty"` 308 | Scientific float64 `json:"scientific,omitempty"` 309 | Zero float64 `json:"zero,omitempty"` 310 | } 311 | -- complex_arrays.go -- 312 | package test_package 313 | 314 | type complex_arrays struct { 315 | ArrayOfObjects []struct { 316 | ID float64 `json:"id,omitempty"` 317 | Name string `json:"name,omitempty"` 318 | } `json:"array_of_objects,omitempty"` 319 | MixedNested []struct { 320 | Array []float64 `json:"array,omitempty"` 321 | } `json:"mixed_nested,omitempty"` 322 | NestedArrays [][]any `json:"nested_arrays,omitempty"` 323 | } 324 | -- reserved_keywords.go -- 325 | package test_package 326 | 327 | type reserved_keywords struct { 328 | Chan string `json:"chan,omitempty"` 329 | Const string `json:"const,omitempty"` 330 | Func string `json:"func,omitempty"` 331 | Go string `json:"go,omitempty"` 332 | Import string `json:"import,omitempty"` 333 | Interface string `json:"interface,omitempty"` 334 | Package string `json:"package,omitempty"` 335 | Struct string `json:"struct,omitempty"` 336 | Type string `json:"type,omitempty"` 337 | Var string `json:"var,omitempty"` 338 | } 339 | -- case_variations.go -- 340 | package test_package 341 | 342 | type case_variations struct { 343 | Camelcase string `json:"camelCase,omitempty"` 344 | Dotted_field string `json:"dotted.field,omitempty"` 345 | Kebab_case string `json:"kebab-case,omitempty"` 346 | MixedCase_types string `json:"Mixed_Case-Types,omitempty"` 347 | Pascalcase string `json:"PascalCase,omitempty"` 348 | ScreamingSnake string `json:"SCREAMING_SNAKE,omitempty"` 349 | SnakeCase string `json:"snake_case,omitempty"` 350 | Spaced_field string `json:"spaced field,omitempty"` 351 | } 352 | -- special_strings.go -- 353 | package test_package 354 | 355 | type special_strings struct { 356 | Backslashes string `json:"backslashes,omitempty"` 357 | Emoji string `json:"emoji,omitempty"` 358 | Empty string `json:"empty,omitempty"` 359 | Newlines string `json:"newlines,omitempty"` 360 | Quotes string `json:"quotes,omitempty"` 361 | SpecialChars string `json:"special_chars,omitempty"` 362 | UnicodeEscape string `json:"unicode_escape,omitempty"` 363 | Whitespace string `json:"whitespace,omitempty"` 364 | } 365 | -- boolean_variations.go -- 366 | package test_package 367 | 368 | type boolean_variations struct { 369 | FalseVal bool `json:"false_val,omitempty"` 370 | NumberOne float64 `json:"number_one,omitempty"` 371 | NumberZero float64 `json:"number_zero,omitempty"` 372 | StringFalse string `json:"string_false,omitempty"` 373 | StringTrue string `json:"string_true,omitempty"` 374 | TrueVal bool `json:"true_val,omitempty"` 375 | } 376 | -- numeric_strings.go -- 377 | package test_package 378 | 379 | type numeric_strings struct { 380 | LeadingZero string `json:"leading_zero,omitempty"` 381 | LooksLikeFloat string `json:"looks_like_float,omitempty"` 382 | LooksLikeInt string `json:"looks_like_int,omitempty"` 383 | PhoneNumber string `json:"phone_number,omitempty"` 384 | Version string `json:"version,omitempty"` 385 | ZipCode string `json:"zip_code,omitempty"` 386 | } 387 | -- recursive_structure.go -- 388 | package test_package 389 | 390 | type recursive_structure struct { 391 | Children []struct { 392 | Children []struct { 393 | Children []any `json:"children,omitempty"` 394 | Name string `json:"name,omitempty"` 395 | } `json:"children,omitempty"` 396 | Name string `json:"name,omitempty"` 397 | } `json:"children,omitempty"` 398 | Name string `json:"name,omitempty"` 399 | } 400 | -- mixed_array_types.go -- 401 | package test_package 402 | 403 | type mixed_array_types struct { 404 | Mixed []string `json:"mixed,omitempty"` 405 | } 406 | -- null_variations.go -- 407 | package test_package 408 | 409 | type null_variations struct { 410 | EmptyArray []any `json:"empty_array,omitempty"` 411 | EmptyObject struct{} `json:"empty_object,omitempty"` 412 | ExplicitNull any `json:"explicit_null,omitempty"` 413 | NestedNulls struct { 414 | ArrayWithNulls []any `json:"array_with_nulls,omitempty"` 415 | Inner any `json:"inner,omitempty"` 416 | } `json:"nested_nulls,omitempty"` 417 | StringNull string `json:"string_null,omitempty"` 418 | } 419 | -- null_only.err -- 420 | unsupported JSON structure: 421 | -- empty_array_only.err -- 422 | no valid JSON objects found 423 | -- boolean_only.err -- 424 | unsupported JSON structure: bool 425 | -- single_value_types.err -- 426 | unsupported JSON structure: string 427 | -- number_only.err -- 428 | unsupported JSON structure: float64 429 | -- array_of_primitives.err -- 430 | no valid JSON objects found 431 | -------------------------------------------------------------------------------- /roundtrip.go: -------------------------------------------------------------------------------- 1 | //go:build !js 2 | // +build !js 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "io" 10 | "os" 11 | "os/exec" 12 | "path/filepath" 13 | "strings" 14 | ) 15 | 16 | // runRoundtripTest generates a round-trip validation program, compiles it, and runs it with the input data 17 | func runRoundtripTest(g *generator) error { 18 | // Read input data first - we'll need it twice 19 | inputData, err := io.ReadAll(os.Stdin) 20 | if err != nil { 21 | return fmt.Errorf("failed to read input: %w", err) 22 | } 23 | return runRoundtripTestWithData(g, inputData) 24 | } 25 | 26 | // runRoundtripTestWithData runs a round-trip validation test with the provided input data 27 | func runRoundtripTestWithData(g *generator, inputData []byte) error { 28 | 29 | // Force main package for roundtrip test 30 | origPkg := g.PackageName 31 | g.PackageName = "main" 32 | 33 | // Generate the struct definition 34 | var structBuf bytes.Buffer 35 | if err := g.generate(&structBuf, bytes.NewReader(inputData)); err != nil { 36 | return fmt.Errorf("failed to generate struct: %w", err) 37 | } 38 | 39 | g.PackageName = origPkg // Restore original 40 | 41 | // Create temp directory for the test program 42 | tmpDir, err := os.MkdirTemp("", "json-to-struct-roundtrip-*") 43 | if err != nil { 44 | return fmt.Errorf("failed to create temp directory: %w", err) 45 | } 46 | defer os.RemoveAll(tmpDir) 47 | 48 | // Generate the round-trip test program 49 | testProgram := generateRoundtripProgram(g.TypeName, structBuf.String()) 50 | 51 | // Write the program to a file 52 | programPath := filepath.Join(tmpDir, "main.go") 53 | if err := os.WriteFile(programPath, []byte(testProgram), 0644); err != nil { 54 | return fmt.Errorf("failed to write test program: %w", err) 55 | } 56 | 57 | // Run the program directly with go run and capture output 58 | runCmd := exec.Command("go", "run", "main.go") 59 | runCmd.Dir = tmpDir // Set working directory to the temp directory 60 | runCmd.Stdin = bytes.NewReader(inputData) 61 | 62 | output, err := runCmd.CombinedOutput() 63 | if err != nil { 64 | return fmt.Errorf("round-trip test failed: %w\nOutput: %s", err, output) 65 | } 66 | 67 | // Parse the output to extract summary statistics 68 | lines := strings.Split(string(output), "\n") 69 | var totalRecords, successfulParse, parseErrors int 70 | var fieldCoverage []string 71 | 72 | for _, line := range lines { 73 | line = strings.TrimSpace(line) 74 | if strings.HasPrefix(line, "Total Records:") { 75 | fmt.Sscanf(line, "Total Records: %d", &totalRecords) 76 | } else if strings.HasPrefix(line, "Successful Parse:") { 77 | fmt.Sscanf(line, "Successful Parse: %d", &successfulParse) 78 | } else if strings.HasPrefix(line, "Parse Errors:") { 79 | fmt.Sscanf(line, "Parse Errors: %d", &parseErrors) 80 | } else if strings.Contains(line, "records (") && strings.Contains(line, "%)") { 81 | // Field coverage line 82 | fieldCoverage = append(fieldCoverage, line) 83 | } else if strings.HasPrefix(line, "Successfully round-tripped") { 84 | fieldCoverage = append(fieldCoverage, line) 85 | } 86 | } 87 | 88 | // Report summary to stderr 89 | if parseErrors > 0 { 90 | fmt.Fprintf(os.Stderr, "⚠️ Round-trip: %d/%d records parsed (%d errors)\n", 91 | successfulParse, totalRecords, parseErrors) 92 | } else { 93 | fmt.Fprintf(os.Stderr, "✓ Round-trip: %d/%d records validated successfully\n", 94 | successfulParse, totalRecords) 95 | } 96 | 97 | // If there are issues, show more details 98 | if parseErrors > 0 || len(fieldCoverage) > 0 { 99 | for _, fc := range fieldCoverage { 100 | if strings.Contains(fc, "round-tripped") { 101 | fmt.Fprintf(os.Stderr, " %s\n", fc) 102 | } 103 | } 104 | } 105 | 106 | return nil 107 | } 108 | 109 | // generateRoundtripProgram generates a complete Go program for round-trip testing 110 | func generateRoundtripProgram(typeName, structDef string) string { 111 | // Extract just the struct definition, removing package declaration 112 | lines := strings.Split(structDef, "\n") 113 | var structOnly []string 114 | for _, line := range lines { 115 | if !strings.HasPrefix(line, "package ") && line != "" { 116 | structOnly = append(structOnly, line) 117 | } 118 | } 119 | structDefClean := strings.Join(structOnly, "\n") 120 | 121 | return fmt.Sprintf(`package main 122 | 123 | import ( 124 | "bufio" 125 | "encoding/json" 126 | "fmt" 127 | "log" 128 | "os" 129 | "reflect" 130 | "strings" 131 | ) 132 | 133 | %s 134 | 135 | type ValidationStats struct { 136 | TotalRecords int `+"`json:\"total_records\"`"+` 137 | SuccessfulParse int `+"`json:\"successful_parse\"`"+` 138 | ParseErrors int `+"`json:\"parse_errors\"`"+` 139 | FieldStats map[string]FieldValidation `+"`json:\"field_stats\"`"+` 140 | TypeMismatches []TypeMismatch `+"`json:\"type_mismatches,omitempty\"`"+` 141 | } 142 | 143 | type FieldValidation struct { 144 | ActualCount int `+"`json:\"actual_count\"`"+` 145 | NilCount int `+"`json:\"nil_count\"`"+` 146 | TypeErrors []string `+"`json:\"type_errors,omitempty\"`"+` 147 | } 148 | 149 | type TypeMismatch struct { 150 | Record int `+"`json:\"record\"`"+` 151 | Field string `+"`json:\"field\"`"+` 152 | Expected string `+"`json:\"expected\"`"+` 153 | Actual string `+"`json:\"actual\"`"+` 154 | OriginalVal any `+"`json:\"original_value\"`"+` 155 | } 156 | 157 | func main() { 158 | stats := &ValidationStats{ 159 | FieldStats: make(map[string]FieldValidation), 160 | } 161 | 162 | scanner := bufio.NewScanner(os.Stdin) 163 | recordNum := 0 164 | var allInputs []string 165 | 166 | for scanner.Scan() { 167 | line := scanner.Text() 168 | if line == "" { 169 | continue 170 | } 171 | allInputs = append(allInputs, line) 172 | recordNum++ 173 | stats.TotalRecords++ 174 | 175 | // Try to parse as single object or array 176 | if strings.HasPrefix(strings.TrimSpace(line), "[") { 177 | // Handle array 178 | var rawArray []map[string]any 179 | if err := json.Unmarshal([]byte(line), &rawArray); err != nil { 180 | var objArray []any 181 | if err := json.Unmarshal([]byte(line), &objArray); err != nil { 182 | log.Printf("Record %%d: Failed to parse array: %%v", recordNum, err) 183 | stats.ParseErrors++ 184 | continue 185 | } 186 | // Process each object in array 187 | for i, obj := range objArray { 188 | if objMap, ok := obj.(map[string]any); ok { 189 | validateRecord(objMap, recordNum*1000+i, stats) 190 | } 191 | } 192 | continue 193 | } 194 | // Process each object in array 195 | for i, obj := range rawArray { 196 | validateRecord(obj, recordNum*1000+i, stats) 197 | } 198 | } else { 199 | // Parse as single object 200 | var rawData map[string]any 201 | if err := json.Unmarshal([]byte(line), &rawData); err != nil { 202 | log.Printf("Record %%d: Failed to parse as JSON object: %%v", recordNum, err) 203 | stats.ParseErrors++ 204 | continue 205 | } 206 | validateRecord(rawData, recordNum, stats) 207 | } 208 | } 209 | 210 | if err := scanner.Err(); err != nil { 211 | log.Fatalf("Error reading input: %%v", err) 212 | } 213 | 214 | // Output validation statistics 215 | fmt.Printf("\n=== ROUND-TRIP VALIDATION RESULTS ===\n") 216 | fmt.Printf("Total Records: %%d\n", stats.TotalRecords) 217 | fmt.Printf("Successful Parse: %%d (%%0.1f%%%%)\n", 218 | stats.SuccessfulParse, 219 | float64(stats.SuccessfulParse)/float64(stats.TotalRecords)*100) 220 | fmt.Printf("Parse Errors: %%d\n", stats.ParseErrors) 221 | 222 | if len(stats.TypeMismatches) > 0 { 223 | fmt.Printf("\n=== TYPE MISMATCHES ===\n") 224 | for _, mismatch := range stats.TypeMismatches { 225 | fmt.Printf("Record %%d, Field '%%s': Expected %%s, got %%s (value: %%v)\n", 226 | mismatch.Record, mismatch.Field, mismatch.Expected, mismatch.Actual, mismatch.OriginalVal) 227 | } 228 | } 229 | 230 | // Field coverage analysis 231 | fmt.Printf("\n=== FIELD COVERAGE ===\n") 232 | for fieldName, validation := range stats.FieldStats { 233 | coverage := float64(validation.ActualCount) / float64(stats.SuccessfulParse) * 100 234 | fmt.Printf("%%s: %%d/%%d records (%%0.1f%%%%), %%d nil values\n", 235 | fieldName, validation.ActualCount, stats.SuccessfulParse, coverage, validation.NilCount) 236 | 237 | if len(validation.TypeErrors) > 0 { 238 | fmt.Printf(" Type errors: %%v\n", validation.TypeErrors) 239 | } 240 | } 241 | 242 | // Re-marshal test 243 | fmt.Printf("\n=== RE-MARSHAL TEST ===\n") 244 | successCount := 0 245 | for i, input := range allInputs { 246 | var original map[string]any 247 | if strings.HasPrefix(strings.TrimSpace(input), "[") { 248 | // Skip arrays for re-marshal test 249 | continue 250 | } 251 | 252 | if err := json.Unmarshal([]byte(input), &original); err != nil { 253 | continue 254 | } 255 | 256 | var generated %s 257 | if err := json.Unmarshal([]byte(input), &generated); err != nil { 258 | fmt.Printf("Record %%d: Failed to unmarshal: %%v\n", i+1, err) 259 | continue 260 | } 261 | 262 | remarshaled, err := json.Marshal(generated) 263 | if err != nil { 264 | fmt.Printf("Record %%d: Failed to re-marshal: %%v\n", i+1, err) 265 | continue 266 | } 267 | 268 | var remarshaledMap map[string]any 269 | if err := json.Unmarshal(remarshaled, &remarshaledMap); err != nil { 270 | fmt.Printf("Record %%d: Failed to parse re-marshaled data: %%v\n", i+1, err) 271 | continue 272 | } 273 | 274 | // Deep comparison of values 275 | mismatch := false 276 | for key, origVal := range original { 277 | remarshaledVal, exists := remarshaledMap[key] 278 | if !exists { 279 | fmt.Printf("Record %%d: Missing field '%%s' in remarshaled data\n", i+1, key) 280 | mismatch = true 281 | continue 282 | } 283 | // Compare values (note: JSON numbers are always float64) 284 | if !compareJSONValues(origVal, remarshaledVal) { 285 | fmt.Printf("Record %%d: Value mismatch for field '%%s': original=%%v, remarshaled=%%v\n", 286 | i+1, key, origVal, remarshaledVal) 287 | mismatch = true 288 | } 289 | } 290 | 291 | // Check for extra fields in remarshaled 292 | for key := range remarshaledMap { 293 | if _, exists := original[key]; !exists { 294 | fmt.Printf("Record %%d: Extra field '%%s' in remarshaled data\n", i+1, key) 295 | mismatch = true 296 | } 297 | } 298 | 299 | if !mismatch { 300 | successCount++ 301 | } 302 | } 303 | 304 | if successCount > 0 { 305 | fmt.Printf("Successfully round-tripped %%d/%%d single objects\n", successCount, len(allInputs)) 306 | } 307 | } 308 | 309 | func validateRecord(rawData map[string]any, recordNum int, stats *ValidationStats) { 310 | // Parse into generated struct 311 | var generated %s 312 | rawBytes, _ := json.Marshal(rawData) 313 | 314 | if err := json.Unmarshal(rawBytes, &generated); err != nil { 315 | log.Printf("Record %%d: Failed to unmarshal into generated struct: %%v", recordNum, err) 316 | stats.ParseErrors++ 317 | return 318 | } 319 | 320 | stats.SuccessfulParse++ 321 | 322 | // Analyze each field 323 | generatedValue := reflect.ValueOf(generated) 324 | generatedType := reflect.TypeOf(generated) 325 | 326 | for i := 0; i < generatedValue.NumField(); i++ { 327 | field := generatedType.Field(i) 328 | fieldValue := generatedValue.Field(i) 329 | 330 | // Get JSON tag name 331 | jsonTag := field.Tag.Get("json") 332 | if jsonTag == "" { 333 | jsonTag = field.Name 334 | } 335 | // Remove ,omitempty suffix 336 | if comma := strings.Index(jsonTag, ","); comma != -1 { 337 | jsonTag = jsonTag[:comma] 338 | } 339 | 340 | // Initialize field stats if not exists 341 | if _, exists := stats.FieldStats[field.Name]; !exists { 342 | stats.FieldStats[field.Name] = FieldValidation{} 343 | } 344 | fieldStat := stats.FieldStats[field.Name] 345 | 346 | // Check if field exists in original data 347 | originalValue, exists := rawData[jsonTag] 348 | if exists { 349 | fieldStat.ActualCount++ 350 | 351 | // Check for type compatibility 352 | if err := validateFieldType(field.Name, fieldValue, originalValue, recordNum, stats); err != nil { 353 | fieldStat.TypeErrors = append(fieldStat.TypeErrors, err.Error()) 354 | } 355 | } 356 | 357 | // Check for nil values in pointer fields 358 | if fieldValue.Kind() == reflect.Ptr && fieldValue.IsNil() { 359 | fieldStat.NilCount++ 360 | } 361 | 362 | stats.FieldStats[field.Name] = fieldStat 363 | } 364 | } 365 | 366 | // compareJSONValues compares two JSON values for equality 367 | func compareJSONValues(a, b any) bool { 368 | // Handle nil 369 | if a == nil && b == nil { 370 | return true 371 | } 372 | if a == nil || b == nil { 373 | return false 374 | } 375 | 376 | // Compare types 377 | switch aVal := a.(type) { 378 | case float64: 379 | if bVal, ok := b.(float64); ok { 380 | return aVal == bVal 381 | } 382 | case string: 383 | if bVal, ok := b.(string); ok { 384 | return aVal == bVal 385 | } 386 | case bool: 387 | if bVal, ok := b.(bool); ok { 388 | return aVal == bVal 389 | } 390 | case map[string]any: 391 | if bVal, ok := b.(map[string]any); ok { 392 | if len(aVal) != len(bVal) { 393 | return false 394 | } 395 | for key, aSubVal := range aVal { 396 | if bSubVal, exists := bVal[key]; !exists || !compareJSONValues(aSubVal, bSubVal) { 397 | return false 398 | } 399 | } 400 | return true 401 | } 402 | case []any: 403 | if bVal, ok := b.([]any); ok { 404 | if len(aVal) != len(bVal) { 405 | return false 406 | } 407 | for i := range aVal { 408 | if !compareJSONValues(aVal[i], bVal[i]) { 409 | return false 410 | } 411 | } 412 | return true 413 | } 414 | } 415 | return false 416 | } 417 | 418 | func validateFieldType(fieldName string, structField reflect.Value, originalValue any, recordNum int, stats *ValidationStats) error { 419 | if originalValue == nil { 420 | // Nil values should work with pointer types 421 | if structField.Kind() != reflect.Ptr { 422 | mismatch := TypeMismatch{ 423 | Record: recordNum, 424 | Field: fieldName, 425 | Expected: "pointer type (for nil)", 426 | Actual: structField.Type().String(), 427 | OriginalVal: originalValue, 428 | } 429 | stats.TypeMismatches = append(stats.TypeMismatches, mismatch) 430 | return fmt.Errorf("nil value but field is not pointer") 431 | } 432 | return nil 433 | } 434 | 435 | // Check basic type compatibility 436 | originalType := reflect.TypeOf(originalValue) 437 | expectedType := structField.Type() 438 | 439 | // Handle pointer types 440 | if expectedType.Kind() == reflect.Ptr { 441 | expectedType = expectedType.Elem() 442 | } 443 | 444 | // Basic compatibility check (simplified) 445 | compatible := false 446 | switch originalType.Kind() { 447 | case reflect.Float64: 448 | compatible = expectedType.Kind() == reflect.Float64 449 | case reflect.String: 450 | compatible = expectedType.Kind() == reflect.String 451 | case reflect.Bool: 452 | compatible = expectedType.Kind() == reflect.Bool 453 | case reflect.Map, reflect.Slice: 454 | compatible = true // Complex types need deeper validation 455 | } 456 | 457 | if !compatible && expectedType != reflect.TypeOf((*any)(nil)).Elem() { 458 | mismatch := TypeMismatch{ 459 | Record: recordNum, 460 | Field: fieldName, 461 | Expected: expectedType.String(), 462 | Actual: originalType.String(), 463 | OriginalVal: originalValue, 464 | } 465 | stats.TypeMismatches = append(stats.TypeMismatches, mismatch) 466 | return fmt.Errorf("type mismatch: expected %%s, got %%s", expectedType, originalType) 467 | } 468 | 469 | return nil 470 | } 471 | `, structDefClean, typeName, typeName) 472 | } 473 | -------------------------------------------------------------------------------- /docs/wasm_exec.js: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | (() => { 6 | // Map multiple JavaScript environments to a single common API, 7 | // preferring web standards over Node.js API. 8 | // 9 | // Environments considered: 10 | // - Browsers 11 | // - Node.js 12 | // - Electron 13 | // - Parcel 14 | 15 | if (typeof global !== "undefined") { 16 | // global already exists 17 | } else if (typeof window !== "undefined") { 18 | window.global = window; 19 | } else if (typeof self !== "undefined") { 20 | self.global = self; 21 | } else { 22 | throw new Error("cannot export Go (neither global, window nor self is defined)"); 23 | } 24 | 25 | if (!global.require && typeof require !== "undefined") { 26 | global.require = require; 27 | } 28 | 29 | if (!global.fs && global.require) { 30 | global.fs = require("fs"); 31 | } 32 | 33 | const enosys = () => { 34 | const err = new Error("not implemented"); 35 | err.code = "ENOSYS"; 36 | return err; 37 | }; 38 | 39 | if (!global.fs) { 40 | let outputBuf = ""; 41 | global.fs = { 42 | constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused 43 | writeSync(fd, buf) { 44 | outputBuf += decoder.decode(buf); 45 | const nl = outputBuf.lastIndexOf("\n"); 46 | if (nl != -1) { 47 | console.log(outputBuf.substr(0, nl)); 48 | outputBuf = outputBuf.substr(nl + 1); 49 | } 50 | return buf.length; 51 | }, 52 | write(fd, buf, offset, length, position, callback) { 53 | if (offset !== 0 || length !== buf.length || position !== null) { 54 | callback(enosys()); 55 | return; 56 | } 57 | const n = this.writeSync(fd, buf); 58 | callback(null, n); 59 | }, 60 | chmod(path, mode, callback) { callback(enosys()); }, 61 | chown(path, uid, gid, callback) { callback(enosys()); }, 62 | close(fd, callback) { callback(enosys()); }, 63 | fchmod(fd, mode, callback) { callback(enosys()); }, 64 | fchown(fd, uid, gid, callback) { callback(enosys()); }, 65 | fstat(fd, callback) { callback(enosys()); }, 66 | fsync(fd, callback) { callback(null); }, 67 | ftruncate(fd, length, callback) { callback(enosys()); }, 68 | lchown(path, uid, gid, callback) { callback(enosys()); }, 69 | link(path, link, callback) { callback(enosys()); }, 70 | lstat(path, callback) { callback(enosys()); }, 71 | mkdir(path, perm, callback) { callback(enosys()); }, 72 | open(path, flags, mode, callback) { callback(enosys()); }, 73 | read(fd, buffer, offset, length, position, callback) { callback(enosys()); }, 74 | readdir(path, callback) { callback(enosys()); }, 75 | readlink(path, callback) { callback(enosys()); }, 76 | rename(from, to, callback) { callback(enosys()); }, 77 | rmdir(path, callback) { callback(enosys()); }, 78 | stat(path, callback) { callback(enosys()); }, 79 | symlink(path, link, callback) { callback(enosys()); }, 80 | truncate(path, length, callback) { callback(enosys()); }, 81 | unlink(path, callback) { callback(enosys()); }, 82 | utimes(path, atime, mtime, callback) { callback(enosys()); }, 83 | }; 84 | } 85 | 86 | if (!global.process) { 87 | global.process = { 88 | getuid() { return -1; }, 89 | getgid() { return -1; }, 90 | geteuid() { return -1; }, 91 | getegid() { return -1; }, 92 | getgroups() { throw enosys(); }, 93 | pid: -1, 94 | ppid: -1, 95 | umask() { throw enosys(); }, 96 | cwd() { throw enosys(); }, 97 | chdir() { throw enosys(); }, 98 | } 99 | } 100 | 101 | if (!global.crypto) { 102 | const nodeCrypto = require("crypto"); 103 | global.crypto = { 104 | getRandomValues(b) { 105 | nodeCrypto.randomFillSync(b); 106 | }, 107 | }; 108 | } 109 | 110 | if (!global.performance) { 111 | global.performance = { 112 | now() { 113 | const [sec, nsec] = process.hrtime(); 114 | return sec * 1000 + nsec / 1000000; 115 | }, 116 | }; 117 | } 118 | 119 | if (!global.TextEncoder) { 120 | global.TextEncoder = require("util").TextEncoder; 121 | } 122 | 123 | if (!global.TextDecoder) { 124 | global.TextDecoder = require("util").TextDecoder; 125 | } 126 | 127 | // End of polyfills for common API. 128 | 129 | const encoder = new TextEncoder("utf-8"); 130 | const decoder = new TextDecoder("utf-8"); 131 | 132 | global.Go = class { 133 | constructor() { 134 | this.argv = ["js"]; 135 | this.env = {}; 136 | this.exit = (code) => { 137 | if (code !== 0) { 138 | console.warn("exit code:", code); 139 | } 140 | }; 141 | this._exitPromise = new Promise((resolve) => { 142 | this._resolveExitPromise = resolve; 143 | }); 144 | this._pendingEvent = null; 145 | this._scheduledTimeouts = new Map(); 146 | this._nextCallbackTimeoutID = 1; 147 | 148 | const setInt64 = (addr, v) => { 149 | this.mem.setUint32(addr + 0, v, true); 150 | this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true); 151 | } 152 | 153 | const getInt64 = (addr) => { 154 | const low = this.mem.getUint32(addr + 0, true); 155 | const high = this.mem.getInt32(addr + 4, true); 156 | return low + high * 4294967296; 157 | } 158 | 159 | const loadValue = (addr) => { 160 | const f = this.mem.getFloat64(addr, true); 161 | if (f === 0) { 162 | return undefined; 163 | } 164 | if (!isNaN(f)) { 165 | return f; 166 | } 167 | 168 | const id = this.mem.getUint32(addr, true); 169 | return this._values[id]; 170 | } 171 | 172 | const storeValue = (addr, v) => { 173 | const nanHead = 0x7FF80000; 174 | 175 | if (typeof v === "number") { 176 | if (isNaN(v)) { 177 | this.mem.setUint32(addr + 4, nanHead, true); 178 | this.mem.setUint32(addr, 0, true); 179 | return; 180 | } 181 | if (v === 0) { 182 | this.mem.setUint32(addr + 4, nanHead, true); 183 | this.mem.setUint32(addr, 1, true); 184 | return; 185 | } 186 | this.mem.setFloat64(addr, v, true); 187 | return; 188 | } 189 | 190 | switch (v) { 191 | case undefined: 192 | this.mem.setFloat64(addr, 0, true); 193 | return; 194 | case null: 195 | this.mem.setUint32(addr + 4, nanHead, true); 196 | this.mem.setUint32(addr, 2, true); 197 | return; 198 | case true: 199 | this.mem.setUint32(addr + 4, nanHead, true); 200 | this.mem.setUint32(addr, 3, true); 201 | return; 202 | case false: 203 | this.mem.setUint32(addr + 4, nanHead, true); 204 | this.mem.setUint32(addr, 4, true); 205 | return; 206 | } 207 | 208 | let id = this._ids.get(v); 209 | if (id === undefined) { 210 | id = this._idPool.pop(); 211 | if (id === undefined) { 212 | id = this._values.length; 213 | } 214 | this._values[id] = v; 215 | this._goRefCounts[id] = 0; 216 | this._ids.set(v, id); 217 | } 218 | this._goRefCounts[id]++; 219 | let typeFlag = 1; 220 | switch (typeof v) { 221 | case "string": 222 | typeFlag = 2; 223 | break; 224 | case "symbol": 225 | typeFlag = 3; 226 | break; 227 | case "function": 228 | typeFlag = 4; 229 | break; 230 | } 231 | this.mem.setUint32(addr + 4, nanHead | typeFlag, true); 232 | this.mem.setUint32(addr, id, true); 233 | } 234 | 235 | const loadSlice = (addr) => { 236 | const array = getInt64(addr + 0); 237 | const len = getInt64(addr + 8); 238 | return new Uint8Array(this._inst.exports.mem.buffer, array, len); 239 | } 240 | 241 | const loadSliceOfValues = (addr) => { 242 | const array = getInt64(addr + 0); 243 | const len = getInt64(addr + 8); 244 | const a = new Array(len); 245 | for (let i = 0; i < len; i++) { 246 | a[i] = loadValue(array + i * 8); 247 | } 248 | return a; 249 | } 250 | 251 | const loadString = (addr) => { 252 | const saddr = getInt64(addr + 0); 253 | const len = getInt64(addr + 8); 254 | return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len)); 255 | } 256 | 257 | const timeOrigin = Date.now() - performance.now(); 258 | this.importObject = { 259 | go: { 260 | // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) 261 | // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported 262 | // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function). 263 | // This changes the SP, thus we have to update the SP used by the imported function. 264 | 265 | // func wasmExit(code int32) 266 | "runtime.wasmExit": (sp) => { 267 | const code = this.mem.getInt32(sp + 8, true); 268 | this.exited = true; 269 | delete this._inst; 270 | delete this._values; 271 | delete this._goRefCounts; 272 | delete this._ids; 273 | delete this._idPool; 274 | this.exit(code); 275 | }, 276 | 277 | // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) 278 | "runtime.wasmWrite": (sp) => { 279 | const fd = getInt64(sp + 8); 280 | const p = getInt64(sp + 16); 281 | const n = this.mem.getInt32(sp + 24, true); 282 | fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n)); 283 | }, 284 | 285 | // func resetMemoryDataView() 286 | "runtime.resetMemoryDataView": (sp) => { 287 | this.mem = new DataView(this._inst.exports.mem.buffer); 288 | }, 289 | 290 | // func nanotime1() int64 291 | "runtime.nanotime1": (sp) => { 292 | setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000); 293 | }, 294 | 295 | // func walltime1() (sec int64, nsec int32) 296 | "runtime.walltime1": (sp) => { 297 | const msec = (new Date).getTime(); 298 | setInt64(sp + 8, msec / 1000); 299 | this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true); 300 | }, 301 | 302 | // func scheduleTimeoutEvent(delay int64) int32 303 | "runtime.scheduleTimeoutEvent": (sp) => { 304 | const id = this._nextCallbackTimeoutID; 305 | this._nextCallbackTimeoutID++; 306 | this._scheduledTimeouts.set(id, setTimeout( 307 | () => { 308 | this._resume(); 309 | while (this._scheduledTimeouts.has(id)) { 310 | // for some reason Go failed to register the timeout event, log and try again 311 | // (temporary workaround for https://github.com/golang/go/issues/28975) 312 | console.warn("scheduleTimeoutEvent: missed timeout event"); 313 | this._resume(); 314 | } 315 | }, 316 | getInt64(sp + 8) + 1, // setTimeout has been seen to fire up to 1 millisecond early 317 | )); 318 | this.mem.setInt32(sp + 16, id, true); 319 | }, 320 | 321 | // func clearTimeoutEvent(id int32) 322 | "runtime.clearTimeoutEvent": (sp) => { 323 | const id = this.mem.getInt32(sp + 8, true); 324 | clearTimeout(this._scheduledTimeouts.get(id)); 325 | this._scheduledTimeouts.delete(id); 326 | }, 327 | 328 | // func getRandomData(r []byte) 329 | "runtime.getRandomData": (sp) => { 330 | crypto.getRandomValues(loadSlice(sp + 8)); 331 | }, 332 | 333 | // func finalizeRef(v ref) 334 | "syscall/js.finalizeRef": (sp) => { 335 | const id = this.mem.getUint32(sp + 8, true); 336 | this._goRefCounts[id]--; 337 | if (this._goRefCounts[id] === 0) { 338 | const v = this._values[id]; 339 | this._values[id] = null; 340 | this._ids.delete(v); 341 | this._idPool.push(id); 342 | } 343 | }, 344 | 345 | // func stringVal(value string) ref 346 | "syscall/js.stringVal": (sp) => { 347 | storeValue(sp + 24, loadString(sp + 8)); 348 | }, 349 | 350 | // func valueGet(v ref, p string) ref 351 | "syscall/js.valueGet": (sp) => { 352 | const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16)); 353 | sp = this._inst.exports.getsp(); // see comment above 354 | storeValue(sp + 32, result); 355 | }, 356 | 357 | // func valueSet(v ref, p string, x ref) 358 | "syscall/js.valueSet": (sp) => { 359 | Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32)); 360 | }, 361 | 362 | // func valueDelete(v ref, p string) 363 | "syscall/js.valueDelete": (sp) => { 364 | Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16)); 365 | }, 366 | 367 | // func valueIndex(v ref, i int) ref 368 | "syscall/js.valueIndex": (sp) => { 369 | storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16))); 370 | }, 371 | 372 | // valueSetIndex(v ref, i int, x ref) 373 | "syscall/js.valueSetIndex": (sp) => { 374 | Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24)); 375 | }, 376 | 377 | // func valueCall(v ref, m string, args []ref) (ref, bool) 378 | "syscall/js.valueCall": (sp) => { 379 | try { 380 | const v = loadValue(sp + 8); 381 | const m = Reflect.get(v, loadString(sp + 16)); 382 | const args = loadSliceOfValues(sp + 32); 383 | const result = Reflect.apply(m, v, args); 384 | sp = this._inst.exports.getsp(); // see comment above 385 | storeValue(sp + 56, result); 386 | this.mem.setUint8(sp + 64, 1); 387 | } catch (err) { 388 | storeValue(sp + 56, err); 389 | this.mem.setUint8(sp + 64, 0); 390 | } 391 | }, 392 | 393 | // func valueInvoke(v ref, args []ref) (ref, bool) 394 | "syscall/js.valueInvoke": (sp) => { 395 | try { 396 | const v = loadValue(sp + 8); 397 | const args = loadSliceOfValues(sp + 16); 398 | const result = Reflect.apply(v, undefined, args); 399 | sp = this._inst.exports.getsp(); // see comment above 400 | storeValue(sp + 40, result); 401 | this.mem.setUint8(sp + 48, 1); 402 | } catch (err) { 403 | storeValue(sp + 40, err); 404 | this.mem.setUint8(sp + 48, 0); 405 | } 406 | }, 407 | 408 | // func valueNew(v ref, args []ref) (ref, bool) 409 | "syscall/js.valueNew": (sp) => { 410 | try { 411 | const v = loadValue(sp + 8); 412 | const args = loadSliceOfValues(sp + 16); 413 | const result = Reflect.construct(v, args); 414 | sp = this._inst.exports.getsp(); // see comment above 415 | storeValue(sp + 40, result); 416 | this.mem.setUint8(sp + 48, 1); 417 | } catch (err) { 418 | storeValue(sp + 40, err); 419 | this.mem.setUint8(sp + 48, 0); 420 | } 421 | }, 422 | 423 | // func valueLength(v ref) int 424 | "syscall/js.valueLength": (sp) => { 425 | setInt64(sp + 16, parseInt(loadValue(sp + 8).length)); 426 | }, 427 | 428 | // valuePrepareString(v ref) (ref, int) 429 | "syscall/js.valuePrepareString": (sp) => { 430 | const str = encoder.encode(String(loadValue(sp + 8))); 431 | storeValue(sp + 16, str); 432 | setInt64(sp + 24, str.length); 433 | }, 434 | 435 | // valueLoadString(v ref, b []byte) 436 | "syscall/js.valueLoadString": (sp) => { 437 | const str = loadValue(sp + 8); 438 | loadSlice(sp + 16).set(str); 439 | }, 440 | 441 | // func valueInstanceOf(v ref, t ref) bool 442 | "syscall/js.valueInstanceOf": (sp) => { 443 | this.mem.setUint8(sp + 24, loadValue(sp + 8) instanceof loadValue(sp + 16)); 444 | }, 445 | 446 | // func copyBytesToGo(dst []byte, src ref) (int, bool) 447 | "syscall/js.copyBytesToGo": (sp) => { 448 | const dst = loadSlice(sp + 8); 449 | const src = loadValue(sp + 32); 450 | if (!(src instanceof Uint8Array)) { 451 | this.mem.setUint8(sp + 48, 0); 452 | return; 453 | } 454 | const toCopy = src.subarray(0, dst.length); 455 | dst.set(toCopy); 456 | setInt64(sp + 40, toCopy.length); 457 | this.mem.setUint8(sp + 48, 1); 458 | }, 459 | 460 | // func copyBytesToJS(dst ref, src []byte) (int, bool) 461 | "syscall/js.copyBytesToJS": (sp) => { 462 | const dst = loadValue(sp + 8); 463 | const src = loadSlice(sp + 16); 464 | if (!(dst instanceof Uint8Array)) { 465 | this.mem.setUint8(sp + 48, 0); 466 | return; 467 | } 468 | const toCopy = src.subarray(0, dst.length); 469 | dst.set(toCopy); 470 | setInt64(sp + 40, toCopy.length); 471 | this.mem.setUint8(sp + 48, 1); 472 | }, 473 | 474 | "debug": (value) => { 475 | console.log(value); 476 | }, 477 | } 478 | }; 479 | } 480 | 481 | async run(instance) { 482 | this._inst = instance; 483 | this.mem = new DataView(this._inst.exports.mem.buffer); 484 | this._values = [ // JS values that Go currently has references to, indexed by reference id 485 | NaN, 486 | 0, 487 | null, 488 | true, 489 | false, 490 | global, 491 | this, 492 | ]; 493 | this._goRefCounts = []; // number of references that Go has to a JS value, indexed by reference id 494 | this._ids = new Map(); // mapping from JS values to reference ids 495 | this._idPool = []; // unused ids that have been garbage collected 496 | this.exited = false; // whether the Go program has exited 497 | 498 | // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory. 499 | let offset = 4096; 500 | 501 | const strPtr = (str) => { 502 | const ptr = offset; 503 | const bytes = encoder.encode(str + "\0"); 504 | new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes); 505 | offset += bytes.length; 506 | if (offset % 8 !== 0) { 507 | offset += 8 - (offset % 8); 508 | } 509 | return ptr; 510 | }; 511 | 512 | const argc = this.argv.length; 513 | 514 | const argvPtrs = []; 515 | this.argv.forEach((arg) => { 516 | argvPtrs.push(strPtr(arg)); 517 | }); 518 | argvPtrs.push(0); 519 | 520 | const keys = Object.keys(this.env).sort(); 521 | keys.forEach((key) => { 522 | argvPtrs.push(strPtr(`${key}=${this.env[key]}`)); 523 | }); 524 | argvPtrs.push(0); 525 | 526 | const argv = offset; 527 | argvPtrs.forEach((ptr) => { 528 | this.mem.setUint32(offset, ptr, true); 529 | this.mem.setUint32(offset + 4, 0, true); 530 | offset += 8; 531 | }); 532 | 533 | this._inst.exports.run(argc, argv); 534 | if (this.exited) { 535 | this._resolveExitPromise(); 536 | } 537 | await this._exitPromise; 538 | } 539 | 540 | _resume() { 541 | if (this.exited) { 542 | throw new Error("Go program has already exited"); 543 | } 544 | this._inst.exports.resume(); 545 | if (this.exited) { 546 | this._resolveExitPromise(); 547 | } 548 | } 549 | 550 | _makeFuncWrapper(id) { 551 | const go = this; 552 | return function () { 553 | const event = { id: id, this: this, args: arguments }; 554 | go._pendingEvent = event; 555 | go._resume(); 556 | return event.result; 557 | }; 558 | } 559 | } 560 | 561 | if ( 562 | global.require && 563 | global.require.main === module && 564 | global.process && 565 | global.process.versions && 566 | !global.process.versions.electron 567 | ) { 568 | if (process.argv.length < 3) { 569 | console.error("usage: go_js_wasm_exec [wasm binary] [arguments]"); 570 | process.exit(1); 571 | } 572 | 573 | const go = new Go(); 574 | go.argv = process.argv.slice(2); 575 | go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env); 576 | go.exit = process.exit; 577 | WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => { 578 | process.on("exit", (code) => { // Node.js exits if no event handler is pending 579 | if (code === 0 && !go.exited) { 580 | // deadlock, make Go print error and stack traces 581 | go._pendingEvent = { id: 0 }; 582 | go._resume(); 583 | } 584 | }); 585 | return go.run(result.instance); 586 | }).catch((err) => { 587 | console.error(err); 588 | process.exit(1); 589 | }); 590 | } 591 | })(); 592 | -------------------------------------------------------------------------------- /generate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "encoding/json" 7 | "fmt" 8 | "go/format" 9 | "io" 10 | "os" 11 | "sort" 12 | "strings" 13 | "text/template" 14 | "unicode" 15 | 16 | "golang.org/x/tools/txtar" 17 | ) 18 | 19 | // FormatError is returned when generated code fails to format 20 | type FormatError struct { 21 | OriginalError error 22 | Source string // The unformatted source code 23 | LineNum int 24 | Column int 25 | } 26 | 27 | func (e *FormatError) Error() string { 28 | return fmt.Sprintf("formatting error at line %d:%d: %v", e.LineNum, e.Column, e.OriginalError) 29 | } 30 | 31 | func (e *FormatError) Unwrap() error { 32 | return e.OriginalError 33 | } 34 | 35 | //go:embed templates.txt 36 | var defaultTemplates string 37 | 38 | // legacyGenerateFunc can be set by build tags to use legacy implementation 39 | var legacyGenerateFunc func(input io.Reader, structName, pkgName string, cfg *generator) ([]byte, error) 40 | 41 | type generator struct { 42 | PackageName string // package name to use in generated code 43 | TypeName string // struct name to use in generated code 44 | 45 | OmitEmpty bool // use omitempty in json tags 46 | StatComments bool // add field statistics as comments 47 | Stream bool // show progressive output with terminal clearing 48 | ExtractStructs bool // extract repeated structs to reduce duplication 49 | UpdateInterval int // milliseconds between stream updates 50 | FieldOrder string // field ordering strategy: common-first, rare-first, alphabetical 51 | 52 | Template string // custom template to use instead of default 53 | 54 | fileTemplate *template.Template 55 | typeTemplate *template.Template 56 | 57 | // Statistics gathered during parsing 58 | stats *StructStats 59 | 60 | // Extracted struct definitions 61 | extractedTypes map[string]*Type 62 | 63 | // Cache for fmtFieldName to avoid repeated expensive string operations 64 | fieldNameCache map[string]string 65 | } 66 | 67 | // FieldStat tracks statistics about a field across multiple JSON objects 68 | type FieldStat struct { 69 | Name string 70 | Types map[string]int // type name -> count 71 | TotalCount int // how many times this field appeared 72 | IsArray map[string]bool // type -> whether it was seen as array 73 | JsonName string // original JSON field name 74 | NestedObjs []any // store nested objects for proper struct generation 75 | Values map[string]int // for string/number fields, track unique values and their counts 76 | NumericVals []float64 // for numeric fields, track all values for percentile calculation 77 | ValueOrder []string // track order of first appearance for values 78 | } 79 | 80 | // StructStats tracks field statistics for building consolidated struct 81 | type StructStats struct { 82 | Fields map[string]*FieldStat 83 | TotalLines int 84 | FieldOrder []string // Track order of first field encounter 85 | } 86 | 87 | func (g *generator) loadTemplates() error { 88 | var templateData string 89 | 90 | // Try to load from specified template file first 91 | if g.Template != "" { 92 | if data, err := os.ReadFile(g.Template); err == nil { 93 | templateData = string(data) 94 | } 95 | } 96 | 97 | // Fallback to embedded templates if external file not found or not specified 98 | if templateData == "" { 99 | templateData = defaultTemplates 100 | } 101 | 102 | // Parse the template data (either external or embedded) 103 | archive := txtar.Parse([]byte(templateData)) 104 | templates := make(map[string]string) 105 | for _, file := range archive.Files { 106 | templates[file.Name] = string(file.Data) 107 | } 108 | 109 | // Choose template based on stat-comments flag 110 | var typeTemplateKey string 111 | if g.StatComments { 112 | // Try to use stat-comments version first 113 | if _, ok := templates["type-with-stats.tmpl"]; ok { 114 | typeTemplateKey = "type-with-stats.tmpl" 115 | } else { 116 | typeTemplateKey = "type.tmpl" 117 | } 118 | } else { 119 | typeTemplateKey = "type.tmpl" 120 | } 121 | 122 | if fileTmpl, ok := templates["file.tmpl"]; ok { 123 | g.fileTemplate = template.Must(template.New("file").Parse(fileTmpl)) 124 | } 125 | if typeTmpl, ok := templates[typeTemplateKey]; ok { 126 | g.typeTemplate = template.Must(template.New("type").Funcs(template.FuncMap{ 127 | "GetStatComment": func(t *Type) string { 128 | return t.GetStatComment() 129 | }, 130 | "RenderInlineStruct": func(t *Type, depth int) string { 131 | return g.renderInlineStruct(t, depth) 132 | }, 133 | }).Parse(typeTmpl)) 134 | } 135 | 136 | return nil 137 | } 138 | 139 | // NewStructStats creates a new StructStats instance 140 | func NewStructStats() *StructStats { 141 | return &StructStats{ 142 | Fields: make(map[string]*FieldStat), 143 | FieldOrder: make([]string, 0), 144 | } 145 | } 146 | 147 | // ProcessValue processes a single value and updates field statistics 148 | func (s *StructStats) ProcessValue(key string, value any, g *generator) { 149 | fieldName := g.fmtFieldName(key) 150 | 151 | if s.Fields[fieldName] == nil { 152 | s.Fields[fieldName] = &FieldStat{ 153 | Name: fieldName, 154 | JsonName: key, 155 | Types: make(map[string]int), 156 | IsArray: make(map[string]bool), 157 | NestedObjs: make([]any, 0), 158 | Values: make(map[string]int), 159 | } 160 | // Track the order of first encounter 161 | s.FieldOrder = append(s.FieldOrder, fieldName) 162 | } 163 | 164 | field := s.Fields[fieldName] 165 | field.TotalCount++ 166 | 167 | switch v := value.(type) { 168 | case []any: 169 | if len(v) > 0 { 170 | elementType := g.getGoType(v[0]) 171 | field.Types[elementType]++ 172 | field.IsArray[elementType] = true 173 | // Store nested objects from arrays 174 | if elementType == "struct" { 175 | field.NestedObjs = append(field.NestedObjs, v[0]) 176 | } 177 | } else { 178 | field.Types["any"]++ 179 | field.IsArray["any"] = true 180 | } 181 | case map[string]any: 182 | field.Types["struct"]++ 183 | // Store the nested object for proper struct generation 184 | field.NestedObjs = append(field.NestedObjs, v) 185 | case string: 186 | field.Types["string"]++ 187 | // Track string values for cardinality 188 | if len(field.Values) < 100 { // Limit tracking to avoid memory issues 189 | if _, exists := field.Values[v]; !exists { 190 | field.ValueOrder = append(field.ValueOrder, v) 191 | } 192 | field.Values[v]++ 193 | } 194 | case float64: 195 | field.Types["float64"]++ 196 | // Track all numeric values for statistics 197 | if field.NumericVals == nil { 198 | field.NumericVals = make([]float64, 0) 199 | } 200 | field.NumericVals = append(field.NumericVals, v) 201 | 202 | // Track numeric values if they look like enums (small integers) 203 | if v == float64(int(v)) && v >= -100 && v <= 100 { 204 | valStr := fmt.Sprintf("%d", int(v)) 205 | if _, exists := field.Values[valStr]; !exists { 206 | field.ValueOrder = append(field.ValueOrder, valStr) 207 | } 208 | field.Values[valStr]++ 209 | } 210 | case bool: 211 | field.Types["bool"]++ 212 | valStr := fmt.Sprintf("%v", v) 213 | if _, exists := field.Values[valStr]; !exists { 214 | field.ValueOrder = append(field.ValueOrder, valStr) 215 | } 216 | field.Values[valStr]++ 217 | case nil: 218 | field.Types["nil"]++ 219 | default: 220 | goType := g.getGoType(value) 221 | field.Types[goType]++ 222 | } 223 | } 224 | 225 | // ProcessJSON processes a single JSON object 226 | func (s *StructStats) ProcessJSON(data map[string]any, g *generator) { 227 | s.TotalLines++ 228 | // Process all fields - ordering will be handled when building the final type 229 | for key, value := range data { 230 | s.ProcessValue(key, value, g) 231 | } 232 | } 233 | 234 | // getGoType returns the Go type name for a JSON value 235 | func (g *generator) getGoType(value any) string { 236 | if value == nil { 237 | return "nil" 238 | } 239 | 240 | switch value.(type) { 241 | case bool: 242 | return "bool" 243 | case float64: 244 | return "float64" 245 | case string: 246 | return "string" 247 | case map[string]any: 248 | return "struct" 249 | case []any: 250 | return "[]any" // This will be refined by the caller 251 | default: 252 | return "any" 253 | } 254 | } 255 | 256 | // GetMostCommonType returns the most frequently seen type for a field 257 | func (f *FieldStat) GetMostCommonType() string { 258 | var maxType string 259 | maxCount := 0 260 | hasNil := false 261 | 262 | for typeName, count := range f.Types { 263 | if typeName == "nil" { 264 | hasNil = true 265 | } else if count > maxCount { 266 | maxCount = count 267 | maxType = typeName 268 | } 269 | } 270 | 271 | // If we have both nil and non-nil values, make it a pointer type 272 | if hasNil && maxType != "" { 273 | return "*" + maxType 274 | } 275 | 276 | if maxType == "" { 277 | maxType = "any" 278 | } 279 | 280 | return maxType 281 | } 282 | 283 | func (g *generator) generate(output io.Writer, input io.Reader) error { 284 | // Check if legacy implementation is available and use it 285 | if legacyGenerateFunc != nil { 286 | b, err := legacyGenerateFunc(input, g.TypeName, g.PackageName, g) 287 | if err != nil { 288 | return err 289 | } 290 | _, err = output.Write(b) 291 | return err 292 | } 293 | 294 | // Use streaming mode if requested 295 | if g.Stream { 296 | return g.generateStream(output, input) 297 | } 298 | 299 | // New multi-line implementation 300 | stats := NewStructStats() 301 | g.stats = stats 302 | 303 | // Read all input 304 | inputBytes, err := io.ReadAll(input) 305 | if err != nil { 306 | return fmt.Errorf("error reading input: %w", err) 307 | } 308 | 309 | inputStr := strings.TrimSpace(string(inputBytes)) 310 | if inputStr == "" { 311 | return fmt.Errorf("no input provided") 312 | } 313 | 314 | // Try to parse as different JSON structures 315 | var iresult any 316 | if err := json.Unmarshal(inputBytes, &iresult); err != nil { 317 | // Not valid JSON, try NDJSON (newline-delimited JSON) 318 | lines := strings.Split(inputStr, "\n") 319 | hasValidJSON := false 320 | for _, line := range lines { 321 | line = strings.TrimSpace(line) 322 | if line == "" { 323 | continue 324 | } 325 | var obj map[string]any 326 | if err := json.Unmarshal([]byte(line), &obj); err == nil { 327 | stats.ProcessJSON(obj, g) 328 | hasValidJSON = true 329 | } 330 | } 331 | if !hasValidJSON { 332 | return fmt.Errorf("error parsing JSON: %w", err) 333 | } 334 | } else { 335 | // Successfully parsed as regular JSON 336 | switch result := iresult.(type) { 337 | case map[string]any: 338 | // Single JSON object 339 | stats.ProcessJSON(result, g) 340 | case []any: 341 | // Array of objects - process each one 342 | for _, item := range result { 343 | if obj, ok := item.(map[string]any); ok { 344 | stats.ProcessJSON(obj, g) 345 | } 346 | } 347 | case []map[string]any: 348 | // Array of maps - process each one 349 | for _, obj := range result { 350 | stats.ProcessJSON(obj, g) 351 | } 352 | default: 353 | return fmt.Errorf("unsupported JSON structure: %T", iresult) 354 | } 355 | } 356 | 357 | if stats.TotalLines == 0 { 358 | return fmt.Errorf("no valid JSON objects found") 359 | } 360 | 361 | // Generate the struct definition 362 | typ := g.buildTypeFromStats(stats) 363 | 364 | // Extract repeated structs if requested 365 | if g.ExtractStructs { 366 | g.extractRepeatedStructs(typ) 367 | } 368 | 369 | // Build the complete output with extracted types 370 | var src string 371 | if g.ExtractStructs && len(g.extractedTypes) > 0 { 372 | // Render extracted types first, then main type 373 | var parts []string 374 | 375 | // Sort extracted type names for deterministic output 376 | var names []string 377 | for name := range g.extractedTypes { 378 | names = append(names, name) 379 | } 380 | sort.Strings(names) 381 | 382 | // Add extracted types 383 | for _, name := range names { 384 | parts = append(parts, g.extractedTypes[name].String()) 385 | } 386 | 387 | // Add main type 388 | parts = append(parts, typ.String()) 389 | 390 | src = g.renderFile(strings.Join(parts, "\n\n")) 391 | } else { 392 | src = g.renderFile(typ.String()) 393 | } 394 | 395 | formatted, err := format.Source([]byte(src)) 396 | if err != nil { 397 | // Write the unformatted source to output anyway so user can see what was generated 398 | output.Write([]byte(src)) 399 | 400 | // Parse go/format error which is like "61:17: expected '{', found `json:"result,omitempty"`" 401 | var lineNum, colNum int 402 | fmt.Sscanf(err.Error(), "%d:%d:", &lineNum, &colNum) 403 | 404 | // Return a FormatError with all the info 405 | // The error will be printed to stderr but we still wrote the output 406 | return &FormatError{ 407 | OriginalError: err, 408 | Source: src, 409 | LineNum: lineNum, 410 | Column: colNum, 411 | } 412 | } 413 | 414 | _, err = output.Write(formatted) 415 | return err 416 | } 417 | 418 | // buildTypeFromStats creates a Type from accumulated statistics 419 | func (g *generator) buildTypeFromStats(stats *StructStats) *Type { 420 | result := &Type{ 421 | Name: g.TypeName, 422 | Type: "struct", 423 | Config: g, 424 | } 425 | 426 | // Convert field stats to Type children 427 | var children []*Type 428 | 429 | // Sort fields by occurrence count (most common first), then alphabetically for determinism 430 | type fieldInfo struct { 431 | name string 432 | jsonName string 433 | count int 434 | order int // first encounter order 435 | } 436 | 437 | fields := make([]fieldInfo, 0, len(stats.Fields)) 438 | orderMap := make(map[string]int) 439 | for i, name := range stats.FieldOrder { 440 | orderMap[name] = i 441 | } 442 | 443 | for fieldName, stat := range stats.Fields { 444 | fields = append(fields, fieldInfo{ 445 | name: fieldName, 446 | jsonName: stat.JsonName, 447 | count: stat.TotalCount, 448 | order: orderMap[fieldName], 449 | }) 450 | } 451 | 452 | // Sort based on configured field ordering strategy 453 | switch g.FieldOrder { 454 | case "encounter": 455 | // Use encounter order (no sorting by count) 456 | sort.Slice(fields, func(i, j int) bool { 457 | return fields[i].order < fields[j].order 458 | }) 459 | case "rare-first": 460 | // Rare fields first (ascending count), then by encounter order 461 | sort.Slice(fields, func(i, j int) bool { 462 | if fields[i].count != fields[j].count { 463 | return fields[i].count < fields[j].count // Lower count first 464 | } 465 | return fields[i].order < fields[j].order 466 | }) 467 | case "common-first": 468 | // Common fields first (descending count), then by encounter order 469 | sort.Slice(fields, func(i, j int) bool { 470 | if fields[i].count != fields[j].count { 471 | return fields[i].count > fields[j].count // Higher count first 472 | } 473 | return fields[i].order < fields[j].order 474 | }) 475 | default: // "alphabetical" or unspecified 476 | // Alphabetical by JSON key name (legacy default) 477 | sort.Slice(fields, func(i, j int) bool { 478 | return strings.ToLower(fields[i].jsonName) < strings.ToLower(fields[j].jsonName) 479 | }) 480 | } 481 | 482 | fieldNames := make([]string, len(fields)) 483 | for i, f := range fields { 484 | fieldNames[i] = f.name 485 | } 486 | 487 | for _, fieldName := range fieldNames { 488 | stat := stats.Fields[fieldName] 489 | child := &Type{ 490 | Name: stat.Name, 491 | Config: g, 492 | Stat: stat, // Add statistics for comment generation 493 | } 494 | 495 | // Determine the most common type 496 | mostCommonType := stat.GetMostCommonType() 497 | 498 | // Check if it's an array type 499 | isArray := false 500 | for typeName, isArr := range stat.IsArray { 501 | if stat.Types[typeName] > 0 && isArr { 502 | isArray = true 503 | child.Type = typeName 504 | break 505 | } 506 | } 507 | 508 | if !isArray { 509 | child.Type = mostCommonType 510 | } 511 | 512 | child.Repeated = isArray 513 | 514 | // For struct types, create proper nested structures by merging all nested objects 515 | if child.Type == "struct" && len(stat.NestedObjs) > 0 { 516 | child.Type = "struct" 517 | // Merge all nested objects like the legacy implementation does 518 | child.Children = g.mergeNestedObjects(stat.NestedObjs, child.Name) 519 | } 520 | 521 | // Set JSON tags if field name differs from JSON name 522 | if stat.Name != stat.JsonName { 523 | child.Tags = map[string]string{"json": stat.JsonName} 524 | } 525 | 526 | // Legacy implementation doesn't use pointer types for optional fields 527 | // It just relies on json:",omitempty" tags 528 | 529 | children = append(children, child) 530 | } 531 | 532 | result.Children = children 533 | return result 534 | } 535 | 536 | // renderFile renders the complete Go file with package and type definition 537 | func (g *generator) renderFile(content string) string { 538 | if g.fileTemplate != nil { 539 | data := struct { 540 | Package string 541 | Imports []string 542 | Content string 543 | }{ 544 | Package: g.PackageName, 545 | Imports: nil, // No imports needed for basic struct types 546 | Content: content, 547 | } 548 | 549 | var buf bytes.Buffer 550 | if err := g.fileTemplate.Execute(&buf, data); err != nil { 551 | // Fallback to simple format 552 | return fmt.Sprintf("package %s\n\n%s", g.PackageName, content) 553 | } 554 | return buf.String() 555 | } 556 | 557 | // Default format 558 | return fmt.Sprintf("package %s\n\n%s", g.PackageName, content) 559 | } 560 | 561 | var uppercaseFixups = map[string]bool{"id": true, "url": true} 562 | 563 | // fmtFieldName formats a JSON field name as a Go struct field name 564 | func (g *generator) fmtFieldName(s string) string { 565 | // Initialize cache if needed 566 | if g.fieldNameCache == nil { 567 | g.fieldNameCache = make(map[string]string) 568 | } 569 | 570 | // Check cache first 571 | if cached, ok := g.fieldNameCache[s]; ok { 572 | return cached 573 | } 574 | 575 | // Compute the formatted name efficiently 576 | parts := strings.Split(s, "_") 577 | for i := range parts { 578 | // Replace deprecated strings.Title with efficient implementation 579 | if len(parts[i]) > 0 { 580 | parts[i] = strings.ToUpper(parts[i][:1]) + strings.ToLower(parts[i][1:]) 581 | } 582 | } 583 | if len(parts) > 0 { 584 | last := parts[len(parts)-1] 585 | lastLower := strings.ToLower(last) 586 | if uppercaseFixups[lastLower] { 587 | parts[len(parts)-1] = strings.ToUpper(last) 588 | } 589 | } 590 | assembled := strings.Join(parts, "") 591 | runes := []rune(assembled) 592 | for i, c := range runes { 593 | ok := unicode.IsLetter(c) || unicode.IsDigit(c) 594 | if i == 0 { 595 | ok = unicode.IsLetter(c) 596 | } 597 | if !ok { 598 | runes[i] = '_' 599 | } 600 | } 601 | result := string(runes) 602 | 603 | // Cache the result 604 | g.fieldNameCache[s] = result 605 | return result 606 | } 607 | 608 | // generateFieldTypesFromMap creates Type structures from a map, similar to legacy generateFieldTypes 609 | func (g *generator) generateFieldTypesFromMap(obj map[string]any) []*Type { 610 | result := []*Type{} 611 | 612 | keys := make([]string, 0, len(obj)) 613 | for key := range obj { 614 | keys = append(keys, key) 615 | } 616 | sort.Strings(keys) 617 | 618 | for _, key := range keys { 619 | typ := g.generateTypeFromValue(key, obj[key]) 620 | typ.Name = g.fmtFieldName(key) 621 | // if we need to rewrite the field name we need to record the json field in a tag. 622 | if typ.Name != key { 623 | typ.Tags = map[string]string{"json": key} 624 | } 625 | result = append(result, typ) 626 | } 627 | return result 628 | } 629 | 630 | // generateTypeFromValue creates a Type from a value, similar to legacy generateType 631 | func (g *generator) generateTypeFromValue(name string, value any) *Type { 632 | result := &Type{Name: name, Config: g} 633 | switch v := value.(type) { 634 | case []any: 635 | result.Repeated = true 636 | if len(v) > 0 { 637 | // For now, handle arrays of basic types 638 | t := g.generateTypeFromValue("", v[0]) 639 | result.Type = t.Type 640 | result.Children = t.Children 641 | } else { 642 | result.Type = "any" 643 | } 644 | case map[string]any: 645 | result.Type = "struct" 646 | result.Children = g.generateFieldTypesFromMap(v) 647 | default: 648 | if value == nil { 649 | result.Type = "nil" 650 | } else { 651 | result.Type = g.getGoType(value) 652 | } 653 | } 654 | return result 655 | } 656 | 657 | // mergeNestedObjects merges multiple nested objects into a single Type structure with statistics 658 | func (g *generator) mergeNestedObjects(nestedObjs []any, name string) []*Type { 659 | if len(nestedObjs) == 0 { 660 | return nil 661 | } 662 | 663 | // Create a nested stats collector 664 | nestedStats := NewStructStats() 665 | 666 | // Process all nested objects to gather statistics 667 | for _, obj := range nestedObjs { 668 | if objMap, ok := obj.(map[string]any); ok { 669 | nestedStats.ProcessJSON(objMap, g) 670 | } 671 | } 672 | 673 | // Build the type from the statistics (recursive) 674 | nestedType := g.buildTypeFromStats(nestedStats) 675 | return nestedType.Children 676 | } 677 | 678 | // renderTypeWithTemplate renders the type using the configured template 679 | func (g *generator) renderTypeWithTemplate(t *Type) string { 680 | var buf bytes.Buffer 681 | if err := g.typeTemplate.Execute(&buf, t); err != nil { 682 | panic(err) 683 | } 684 | return buf.String() 685 | } 686 | --------------------------------------------------------------------------------