├── testdata ├── tmpl │ ├── two.html │ ├── dir │ │ ├── three.tmpl │ │ └── subdir │ │ │ └── five.html │ ├── seven.parens │ ├── four.tmpl │ ├── one.txt │ └── five.tmpl ├── project │ ├── foo │ │ └── foo.go │ ├── unimported │ │ └── unimported.go │ ├── uninteresting_imports.go │ ├── go.mod │ ├── templates.go │ ├── main.go │ ├── sub │ │ └── sub.go │ ├── errors.go │ ├── globalassign.go │ ├── varassign.go │ ├── struct.go │ ├── maps.go │ ├── funcreturn.go │ ├── go.sum │ ├── slices.go │ └── funccall.go └── unit │ ├── go.mod │ ├── structs.go │ ├── funcs.go │ ├── vars.go │ └── go.sum ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── ISSUE_TEMPLATE.md └── workflows │ ├── test.yml │ └── release.yml ├── extract ├── processors │ ├── comment_cleaner_test.go │ ├── processor.go │ ├── skip_empty_msgid_test.go │ ├── skip_empty_msgid.go │ ├── skip_errors.go │ ├── skip_ignore.go │ ├── prepare_key.go │ ├── prepare_key_test.go │ ├── unprintable_checker.go │ └── comment_cleaner.go ├── comments.go ├── extractor.go ├── extractors │ ├── globalassign_test.go │ ├── error_test.go │ ├── varassign_test.go │ ├── structdef_test.go │ ├── funcreturn_test.go │ ├── mapsdef_test.go │ ├── slicedef_test.go │ ├── inline_test.go │ ├── funccall_test.go │ ├── error.go │ ├── globalassign.go │ ├── testlib_test.go │ ├── varassign.go │ ├── mapsdef.go │ ├── inline.go │ ├── slicedef.go │ ├── funcreturn.go │ ├── utils.go │ ├── funccall.go │ └── structdef.go ├── etype │ └── token.go ├── loader │ ├── comments_test.go │ ├── comments.go │ ├── package_cleaner.go │ ├── definition_test.go │ ├── package_loader.go │ └── definition.go ├── issue.go ├── runner │ └── runner.go ├── definition.go ├── context_test.go └── context.go ├── util ├── time.go ├── flags_test.go ├── flags.go └── ast.go ├── main.go ├── .gitignore ├── encoder ├── encoder.go ├── encoder_test.go ├── pot.go ├── json_test.go └── json.go ├── Makefile ├── tmpl ├── keyword_test.go ├── parse_test.go ├── walk.go ├── parse.go ├── keyword.go └── inspector.go ├── go.mod ├── config ├── paths.go └── config.go ├── LICENSE ├── .golangci.yml ├── merger ├── json_test.go └── json.go ├── tmplextractors ├── testlib_test.go ├── command.go └── command_test.go ├── commands ├── merge.go ├── extractor.go └── root.go ├── go.sum └── README.md /testdata/tmpl/two.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | #### Summary -------------------------------------------------------------------------------- /extract/processors/comment_cleaner_test.go: -------------------------------------------------------------------------------- 1 | package processors 2 | -------------------------------------------------------------------------------- /testdata/tmpl/dir/three.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | {{.T.NGet "NGet-Singular" }} -------------------------------------------------------------------------------- /testdata/tmpl/seven.parens: -------------------------------------------------------------------------------- 1 | 2 | 3 | {{ printf "%s" ( .X "foo" ) }} 4 | {{ .X "bar" }} -------------------------------------------------------------------------------- /testdata/project/foo/foo.go: -------------------------------------------------------------------------------- 1 | package foo 2 | 3 | import "github.com/vorlif/spreak" 4 | 5 | var T *spreak.Localizer 6 | -------------------------------------------------------------------------------- /testdata/tmpl/four.tmpl: -------------------------------------------------------------------------------- 1 | 2 | {{.i18n.Tr "custom keyword"}} 3 | 4 | {{.i18n.Trp .Foo "trp-singular" .Bar "trp-plural" .Dog .Cat}} -------------------------------------------------------------------------------- /testdata/unit/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vorlif/xspreakunit 2 | 3 | go 1.18 4 | 5 | require github.com/vorlif/spreak v0.1.2-0.20220511142256-22c219833d0b 6 | -------------------------------------------------------------------------------- /extract/comments.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import "go/ast" 4 | 5 | type Comments map[string]map[string]ast.CommentMap // pkg -> file -> node -> comments 6 | -------------------------------------------------------------------------------- /testdata/project/unimported/unimported.go: -------------------------------------------------------------------------------- 1 | package unimported 2 | 3 | import "github.com/vorlif/spreak/localize" 4 | 5 | type UnimportedStruct struct { 6 | UnimportedField localize.Singular 7 | } 8 | -------------------------------------------------------------------------------- /extract/processors/processor.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "github.com/vorlif/xspreak/extract" 5 | ) 6 | 7 | type Processor interface { 8 | Process(issues []extract.Issue) ([]extract.Issue, error) 9 | Name() string 10 | } 11 | -------------------------------------------------------------------------------- /util/time.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "time" 5 | 6 | log "github.com/sirupsen/logrus" 7 | ) 8 | 9 | func TrackTime(start time.Time, name string) { 10 | elapsed := time.Since(start) 11 | log.Debugf("%s took %s", name, elapsed) 12 | } 13 | -------------------------------------------------------------------------------- /testdata/unit/structs.go: -------------------------------------------------------------------------------- 1 | package unit 2 | 3 | import "github.com/vorlif/spreak/localize" 4 | 5 | type TranslationStruct struct { 6 | Singular localize.Singular 7 | plural localize.Plural 8 | Domain localize.Domain 9 | context localize.Context 10 | } 11 | -------------------------------------------------------------------------------- /testdata/project/uninteresting_imports.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/sirupsen/logrus" 4 | 5 | // This file contains imports of packages that are not interesting for us. 6 | 7 | func log() { 8 | logrus.Info("Logging from uninteresting_imports.go") 9 | } 10 | -------------------------------------------------------------------------------- /testdata/project/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vorlif/testdata 2 | 3 | go 1.24.0 4 | 5 | require github.com/vorlif/spreak v1.0.0 6 | 7 | require ( 8 | github.com/sirupsen/logrus v1.9.3 9 | golang.org/x/text v0.29.0 10 | ) 11 | 12 | require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect 13 | -------------------------------------------------------------------------------- /testdata/project/templates.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // xspreak: template 4 | var t = ` 5 | {{.T.Get "Hello"}} 6 | ` 7 | 8 | // xspreak: template 9 | const constTemplate = ` 10 | 11 | 12 | {{.T.NGet "Dog" "Dogs" 2 "2" "b"}} 13 | 14 | 15 | ` 16 | 17 | // xspreak: template 18 | const multiline = "{{ .T.Get `Multiline String\nwith\n newlines` }}" 19 | -------------------------------------------------------------------------------- /testdata/tmpl/dir/subdir/five.html: -------------------------------------------------------------------------------- 1 | {{- define "todos" -}} 2 |

{{.T "todos"}}

3 | 12 | {{- end -}} 13 | 14 | {{- define "help" -}} 15 | {{.T "Help" }} 16 | {{- end -}} -------------------------------------------------------------------------------- /testdata/project/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/vorlif/spreak/localize" 7 | ) 8 | 9 | const ( 10 | constCtx = "constCtxVal" 11 | ) 12 | 13 | type M struct { 14 | Test localize.Singular 15 | Hello string 16 | } 17 | 18 | func main() { 19 | 20 | // test comment 21 | fmt.Println(localize.Singular("init")) 22 | } 23 | -------------------------------------------------------------------------------- /extract/extractor.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // Extractor uses the context to search for strings to be translated and returns all strings found as an issue. 8 | type Extractor interface { 9 | Run(ctx context.Context, extractCtx *Context) ([]Issue, error) 10 | 11 | // Name returns the name of the extractor 12 | Name() string 13 | } 14 | -------------------------------------------------------------------------------- /testdata/unit/funcs.go: -------------------------------------------------------------------------------- 1 | package unit 2 | 3 | import "github.com/vorlif/spreak/localize" 4 | 5 | func TranslateFunc(one, tow, three localize.Singular, plural localize.Plural, Domain localize.Domain, 6 | context localize.Context) { 7 | 8 | } 9 | 10 | func GetTranslateFunc() (localize.Singular, localize.Plural, localize.Domain, localize.Context) { 11 | return "", "", "", "" 12 | } 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | #### Summary 2 | 3 | Bug report in one concise sentence 4 | 5 | #### Steps to reproduce 6 | 7 | How can we reproduce the issue (what version are you using?) 8 | 9 | #### Expected behavior 10 | 11 | Describe your issue in detail 12 | 13 | #### Observed behavior (that appears unintentional) 14 | 15 | What did you see happen? Please include relevant error messages. 16 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/sirupsen/logrus" 7 | 8 | "github.com/vorlif/xspreak/commands" 9 | ) 10 | 11 | func init() { 12 | logrus.SetOutput(os.Stdout) 13 | logrus.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true}) 14 | logrus.SetLevel(logrus.InfoLevel) 15 | } 16 | 17 | func main() { 18 | if err := commands.Execute(); err != nil { 19 | logrus.Warn(err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /testdata/unit/vars.go: -------------------------------------------------------------------------------- 1 | package unit 2 | 3 | import "github.com/vorlif/spreak/localize" 4 | 5 | var ( 6 | one localize.Singular = "one-singular" 7 | two localize.Singular = "two-singular" 8 | nonLocalize = "non-localize" 9 | ) 10 | 11 | var three localize.Singular = "three-singular" 12 | 13 | var ignored = "ignored" 14 | 15 | var pluralIgnored localize.Plural = "plural without singular is ignored" 16 | -------------------------------------------------------------------------------- /extract/extractors/globalassign_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGlobalAssignExtractor(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewGlobalAssignExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | got := collectIssueStrings(issues) 14 | want := []string{"app", "monday", "tuesday", "wednesday", "thursday", "friday"} 15 | assert.ElementsMatch(t, want, got) 16 | } 17 | -------------------------------------------------------------------------------- /testdata/project/sub/sub.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "github.com/vorlif/testdata/foo" 5 | 6 | "github.com/vorlif/spreak/localize" 7 | ) 8 | 9 | type Sub struct { 10 | Text localize.Singular 11 | Plural localize.Plural 12 | } 13 | 14 | func Func(msgID localize.Singular, plural localize.Plural) string { 15 | 16 | var t localize.Singular 17 | 18 | t = `This is an 19 | multiline string` 20 | 21 | t = "Newline remains\n" 22 | foo.T.Getf("foo test") 23 | return t 24 | } 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cover.out 2 | coverage.html 3 | 4 | # IDE paths 5 | .project 6 | .settings 7 | .buildpath 8 | .idea 9 | 10 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 11 | *.o 12 | *.a 13 | *.so 14 | 15 | # Folders 16 | _obj 17 | _test 18 | 19 | # Architecture specific extensions/prefixes 20 | *.[568vq] 21 | [568vq].out 22 | 23 | *.cgo1.go 24 | *.cgo2.c 25 | _cgo_defun.c 26 | _cgo_gotypes.go 27 | _cgo_export.* 28 | 29 | _testmain.go 30 | 31 | *.exe 32 | *.test 33 | *.prof 34 | /xspreak 35 | -------------------------------------------------------------------------------- /encoder/encoder.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/vorlif/xspreak/extract" 7 | ) 8 | 9 | // Regexp that matches if the string contains a string formatting verb 10 | // like %s, %d, %f, etc. 11 | // Recognizes not all cases but most. - See Unit tests. 12 | var reGoStringFormat = regexp.MustCompile(`%([#+\-*0.])?(\[\d])?(([1-9])\.([1-9])|([1-9])|([1-9])\.|\.([1-9]))?[xsvTtbcdoOqXUeEfFgGp]`) 13 | 14 | type Encoder interface { 15 | Encode(issues []extract.Issue) error 16 | } 17 | -------------------------------------------------------------------------------- /extract/extractors/error_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestErrorExtractor(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewErrorExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | got := collectIssueStrings(issues) 14 | want := []string{"global error", "errors", "global alias error", 15 | "errors", "local error", "errors", "local alias error", "errors", "return error", "errors"} 16 | assert.ElementsMatch(t, want, got) 17 | } 18 | -------------------------------------------------------------------------------- /extract/processors/skip_empty_msgid_test.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/vorlif/xspreak/extract" 10 | ) 11 | 12 | func TestSkipEmptyMsgID(t *testing.T) { 13 | issuses := []extract.Issue{ 14 | {PluralID: "p", Context: "ctx"}, 15 | {MsgID: "id"}, 16 | } 17 | 18 | p := NewSkipEmptyMsgID() 19 | res, err := p.Process(issuses) 20 | assert.NoError(t, err) 21 | require.Len(t, res, 1) 22 | assert.EqualValues(t, extract.Issue{MsgID: "id"}, res[0]) 23 | } 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | 3 | gofmt: 4 | gofmt -s -w . 5 | goimports -w -local github.com/vorlif/xspreak ./ 6 | 7 | 8 | lint: 9 | @echo Running golangci-lint 10 | golangci-lint run --fix ./... 11 | 12 | 13 | test-race: 14 | go test -race -run=. ./... || exit 1; 15 | 16 | 17 | test: 18 | go test -short ./... 19 | 20 | 21 | coverage: 22 | go test -short -v -coverprofile cover.out ./... 23 | go tool cover -func cover.out 24 | go tool cover -html=cover.out -o coverage.html 25 | 26 | 27 | clean: 28 | @echo Cleaning 29 | 30 | go clean -i ./... 31 | rm -f cover.out 32 | rm -f coverage.html 33 | rm -rf dist 34 | -------------------------------------------------------------------------------- /extract/extractors/varassign_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestVariablesExtractorRun(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewVariablesExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | want := []string{ 14 | "Bob", "Bobby", "application", "john", "doe", "assign function param", "struct attr assign", 15 | "Newline remains\n", "This is an\nmultiline string", 16 | "backtrace init", "backtrace assign", 17 | } 18 | got := collectIssueStrings(issues) 19 | assert.ElementsMatch(t, want, got) 20 | } 21 | -------------------------------------------------------------------------------- /testdata/project/errors.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | alias "errors" 6 | ) 7 | 8 | /* TRANSLATORS: comment a 9 | comment b 10 | comment c */ 11 | // comment d 12 | var ErrGlobal = errors.New( 13 | // test comment 14 | "global error") 15 | 16 | // TRANSLATORS: comment e 17 | // comment f 18 | // 19 | // comment g 20 | var ErrAliasGloba = alias.New("global alias error") 21 | 22 | func localError() error { 23 | // comment local error 24 | _ = errors.New("local error") 25 | // comment local alias error 26 | _ = errors.New("local alias error") 27 | 28 | return errors.New("return error") 29 | } 30 | -------------------------------------------------------------------------------- /testdata/project/globalassign.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | alias "github.com/vorlif/spreak/localize" 5 | ) 6 | 7 | // TRANSLATORS: Name of the app 8 | // 9 | //goland:noinspection GoVarAndConstTypeMayBeOmitted 10 | var applicationName alias.Singular = "app" 11 | 12 | var ignored = "no localize assign global" 13 | 14 | var backtrace = "backtrace" 15 | 16 | const ( 17 | // TRANSLATORS: Weekday 18 | Monday alias.Singular = "monday" 19 | ) 20 | 21 | const ( 22 | Tuesday alias.Singular = "tuesday" 23 | Wednesday alias.Singular = "wednesday" 24 | Thursday, Friday alias.Singular = "thursday", "friday" 25 | ) 26 | -------------------------------------------------------------------------------- /extract/etype/token.go: -------------------------------------------------------------------------------- 1 | package etype 2 | 3 | type Token string 4 | 5 | const ( 6 | None Token = "" 7 | Singular Token = "Singular" 8 | Key Token = "Key" 9 | PluralKey Token = "PluralKey" 10 | Plural Token = "Plural" 11 | Domain Token = "Domain" 12 | Context Token = "Context" 13 | ) 14 | 15 | func IsMessageID(tok Token) bool { 16 | return tok == Singular || tok == Key || tok == PluralKey 17 | } 18 | 19 | var StringExtractNames = map[string]Token{ 20 | "MsgID": Singular, 21 | "Singular": Singular, 22 | "Plural": Plural, 23 | "Domain": Domain, 24 | "Context": Context, 25 | "Key": Key, 26 | "PluralKey": PluralKey, 27 | } 28 | -------------------------------------------------------------------------------- /extract/loader/comments_test.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/vorlif/xspreak/config" 11 | ) 12 | 13 | const testdataDir = "../../testdata/project" 14 | 15 | func TestCommentsExtractor(t *testing.T) { 16 | cfg := config.NewDefault() 17 | cfg.SourceDir = testdataDir 18 | ctx := context.Background() 19 | 20 | contextLoader := NewPackageLoader(cfg) 21 | 22 | extractCtx, err := contextLoader.Load(ctx) 23 | require.NoError(t, err) 24 | 25 | assert.NotNil(t, extractCtx.CommentMaps) 26 | assert.NotEmpty(t, extractCtx.CommentMaps) 27 | } 28 | -------------------------------------------------------------------------------- /util/flags_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestRangRe(t *testing.T) { 10 | tests := []struct { 11 | text string 12 | assertionFunc assert.BoolAssertionFunc 13 | }{ 14 | {"range: 1..6", assert.True}, 15 | {"range: 100..6", assert.True}, 16 | {"range: 1..600", assert.True}, 17 | {"range: 1..6 ", assert.True}, 18 | {"range: 1..6 ", assert.True}, 19 | {"range: 1...6", assert.False}, 20 | {"range: a..6", assert.False}, 21 | {"range: a..6 bb", assert.False}, 22 | } 23 | 24 | for _, tt := range tests { 25 | tt.assertionFunc(t, reRange.MatchString(tt.text)) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tmpl/keyword_test.go: -------------------------------------------------------------------------------- 1 | package tmpl 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestDefaultKeywords(t *testing.T) { 11 | keywords := DefaultKeywords("..T.", false) 12 | assert.Len(t, keywords, 32) 13 | pointCount := 0 14 | dollarCount := 0 15 | formatCount := 0 16 | for _, kw := range keywords { 17 | if strings.HasPrefix(kw.Name, ".") { 18 | pointCount++ 19 | } 20 | if strings.HasPrefix(kw.Name, "$") { 21 | dollarCount++ 22 | } 23 | if strings.HasSuffix(kw.Name, "f") { 24 | formatCount++ 25 | } 26 | } 27 | 28 | assert.Equal(t, 16, pointCount) 29 | assert.Equal(t, 16, dollarCount) 30 | assert.Equal(t, 16, formatCount) 31 | } 32 | -------------------------------------------------------------------------------- /extract/extractors/structdef_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestStructDefExtractor(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewStructDefExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | got := collectIssueStrings(issues) 14 | want := []string{ 15 | "global struct msgid", "global struct plural", 16 | "local struct msgid", "local struct plural", 17 | "struct msgid arr1", "struct plural arr1", 18 | "struct msgid arr2", "struct plural arr2", 19 | "A3", "B3", "C3", 20 | "A4", "B4", "C4", 21 | "GA3", "GB3", "GC3", 22 | "GA4", "GB4", "GC4", 23 | } 24 | assert.ElementsMatch(t, want, got) 25 | } 26 | -------------------------------------------------------------------------------- /extract/processors/skip_empty_msgid.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "slices" 5 | "time" 6 | 7 | "github.com/vorlif/xspreak/extract" 8 | "github.com/vorlif/xspreak/util" 9 | ) 10 | 11 | type skipEmptyMsgID struct{} 12 | 13 | // NewSkipEmptyMsgID create a new processor that removes issues with an empty msgId. 14 | func NewSkipEmptyMsgID() Processor { return &skipEmptyMsgID{} } 15 | 16 | func (s skipEmptyMsgID) Name() string { return "skip_empty_msgid" } 17 | 18 | func (s skipEmptyMsgID) Process(issues []extract.Issue) ([]extract.Issue, error) { 19 | util.TrackTime(time.Now(), "Clean empty msgid") 20 | issues = slices.DeleteFunc(issues, func(iss extract.Issue) bool { return iss.MsgID == "" }) 21 | return issues, nil 22 | } 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vorlif/xspreak 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/mattn/go-zglob v0.0.6 7 | github.com/sirupsen/logrus v1.9.3 8 | github.com/spf13/cobra v1.10.1 9 | github.com/stretchr/testify v1.11.1 10 | github.com/vorlif/spreak v1.0.0 11 | golang.org/x/text v0.29.0 12 | golang.org/x/tools v0.37.0 13 | ) 14 | 15 | require ( 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | github.com/spf13/pflag v1.0.10 // indirect 20 | golang.org/x/mod v0.28.0 // indirect 21 | golang.org/x/sync v0.17.0 // indirect 22 | golang.org/x/sys v0.36.0 // indirect 23 | gopkg.in/yaml.v3 v3.0.1 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /extract/processors/skip_errors.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "slices" 5 | "time" 6 | 7 | "github.com/vorlif/xspreak/extract" 8 | "github.com/vorlif/xspreak/util" 9 | ) 10 | 11 | type skipErrors struct{} 12 | 13 | // NewSkipErrors creates a new processor that removes all issues which result from the error extractor. 14 | func NewSkipErrors() Processor { return &skipErrors{} } 15 | 16 | func (s skipErrors) Name() string { return "skip-errors" } 17 | 18 | func (s skipErrors) Process(issues []extract.Issue) ([]extract.Issue, error) { 19 | util.TrackTime(time.Now(), "Skip errors") 20 | 21 | issues = slices.DeleteFunc(issues, func(iss extract.Issue) bool { return iss.FromExtractor == "error_extractor" }) 22 | return issues, nil 23 | } 24 | -------------------------------------------------------------------------------- /extract/processors/skip_ignore.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "slices" 5 | "time" 6 | 7 | "github.com/vorlif/xspreak/extract" 8 | "github.com/vorlif/xspreak/util" 9 | ) 10 | 11 | type skipIgnoreFlag struct{} 12 | 13 | // NewSkipIgnore creates a new processor that skips issues with the "ignore" flag. 14 | func NewSkipIgnore() Processor { return &skipIgnoreFlag{} } 15 | 16 | func (s skipIgnoreFlag) Name() string { return "skip-ignore" } 17 | 18 | func (s skipIgnoreFlag) Process(issues []extract.Issue) ([]extract.Issue, error) { 19 | util.TrackTime(time.Now(), "Skip ignore") 20 | 21 | issues = slices.DeleteFunc(issues, func(iss extract.Issue) bool { 22 | return slices.Contains(iss.Flags, "ignore") 23 | }) 24 | 25 | return issues, nil 26 | } 27 | -------------------------------------------------------------------------------- /extract/processors/prepare_key.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/vorlif/xspreak/extract" 7 | "github.com/vorlif/xspreak/extract/etype" 8 | "github.com/vorlif/xspreak/util" 9 | ) 10 | 11 | type prepareKey struct{} 12 | 13 | func NewPrepareKey() Processor { return &prepareKey{} } 14 | 15 | func (p prepareKey) Name() string { return "prepare-key" } 16 | 17 | func (p prepareKey) Process(inIssues []extract.Issue) ([]extract.Issue, error) { 18 | util.TrackTime(time.Now(), "Prepare key") 19 | outIssues := make([]extract.Issue, 0, len(inIssues)) 20 | 21 | for _, iss := range inIssues { 22 | if iss.IDToken == etype.PluralKey { 23 | iss.PluralID = iss.MsgID 24 | } 25 | outIssues = append(outIssues, iss) 26 | } 27 | return outIssues, nil 28 | } 29 | -------------------------------------------------------------------------------- /extract/extractors/funcreturn_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestFuncReturnExtractor(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewFuncReturnExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | want := []string{ 14 | "single", 15 | "plural_s", "plural_p", 16 | "context_s", "context_c", "context_p", 17 | "full_s", "full_c", "full_p", "full_d", 18 | 19 | // backtracking 20 | "bt_ctx_a", "bt_ctx_b", 21 | "bt_domain_A", "bt_domain_B", 22 | "bt_msgA", 23 | "bt_plural_a", "bt_plural_b", 24 | "bt_ctx_c", "bt_domain_c", 25 | "bt_msg_c", "bt_msgB", 26 | "bt_plural_a", 27 | } 28 | got := collectIssueStrings(issues) 29 | assert.ElementsMatch(t, want, got) 30 | assert.Len(t, issues, 7) 31 | } 32 | -------------------------------------------------------------------------------- /extract/extractors/mapsdef_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestMapsDefExtractor(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewMapsDefExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | got := collectIssueStrings(issues) 14 | want := []string{ 15 | "globalKeyMap-a", "globalKeyMap-b", 16 | "globalValueMap-a", "globalValueMap-b", 17 | "globalMap-ka", "globalMap-va", "globalMap-kb", "globalMap-vb", 18 | 19 | "localKeyMap-a", "localKeyMap-b", 20 | "localValueMap-a", "localValueMap-b", 21 | "localMap-ka", "localMap-va", "localMap-kb", "localMap-vb", 22 | 23 | "map struct msgid", "map struct plural", 24 | "map pointer struct msgid", "map pointer struct plural", 25 | } 26 | assert.ElementsMatch(t, want, got) 27 | } 28 | -------------------------------------------------------------------------------- /testdata/project/varassign.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import alias "github.com/vorlif/spreak/localize" 4 | 5 | func varAssignFunc() (string, string) { 6 | var name alias.Singular 7 | 8 | name = "Bob" 9 | name = "Bobby" 10 | 11 | var alternativeName string 12 | alternativeName = "no localize assign local" 13 | 14 | return name, alternativeName 15 | } 16 | 17 | func changeApplicationName() { 18 | applicationName = "application" 19 | } 20 | 21 | type assignStruct struct{} 22 | 23 | func (assignStruct) testAssign() string { 24 | var name alias.Singular 25 | name = "john" 26 | name = "doe" 27 | return name 28 | } 29 | 30 | func assignFunc(singular alias.Singular) { 31 | singular = "assign function param" 32 | 33 | backtraceAssign := "backtrace init" 34 | backtraceAssign = "backtrace assign" 35 | singular = backtraceAssign 36 | } 37 | -------------------------------------------------------------------------------- /extract/issue.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "fmt" 5 | "go/token" 6 | 7 | "golang.org/x/tools/go/packages" 8 | 9 | "github.com/vorlif/xspreak/extract/etype" 10 | ) 11 | 12 | // Issue represents a single issue found by an extractor. 13 | type Issue struct { 14 | // FromExtractor is the name of the extractor that found this issue. 15 | FromExtractor string 16 | 17 | IDToken etype.Token 18 | 19 | Domain string 20 | Context string 21 | MsgID string 22 | PluralID string 23 | 24 | Comments []string 25 | Flags []string 26 | 27 | Pkg *packages.Package 28 | 29 | Pos token.Position 30 | } 31 | 32 | func (i *Issue) FilePath() string { return i.Pos.Filename } 33 | func (i *Issue) Line() int { return i.Pos.Line } 34 | func (i *Issue) Column() int { return i.Pos.Column } 35 | func (i *Issue) Description() string { return fmt.Sprintf("%s: %s", i.FromExtractor, i.MsgID) } 36 | -------------------------------------------------------------------------------- /extract/processors/prepare_key_test.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/vorlif/xspreak/extract" 10 | "github.com/vorlif/xspreak/extract/etype" 11 | ) 12 | 13 | func TestPrepareKey(t *testing.T) { 14 | noKey := extract.Issue{IDToken: etype.Singular, MsgID: "msgid", PluralID: "pluralid"} 15 | key := extract.Issue{IDToken: etype.Key, MsgID: "keyid", PluralID: "keypluralid"} 16 | pluralKey := extract.Issue{IDToken: etype.PluralKey, MsgID: "id"} 17 | 18 | p := NewPrepareKey() 19 | res, err := p.Process([]extract.Issue{noKey, key, pluralKey}) 20 | assert.NoError(t, err) 21 | require.Len(t, res, 3) 22 | 23 | assert.EqualValues(t, noKey, res[0]) 24 | assert.EqualValues(t, key, res[1]) 25 | 26 | pluralKey.PluralID = pluralKey.MsgID 27 | assert.EqualValues(t, pluralKey, res[2]) 28 | } 29 | -------------------------------------------------------------------------------- /extract/extractors/slicedef_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSliceDefExtractor(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewSliceDefExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | got := collectIssueStrings(issues) 14 | want := []string{ 15 | "one", "two", "three", "four", 16 | "six", "seven", "eight", "nine", 17 | "global struct slice singular", "global struct slice plural", "global ctx", 18 | "local struct slice singular", "local struct slice plural", "local ctx", 19 | "global struct slice singular 2", "global struct slice plural 2", 20 | "local struct slice singular 2", "local struct slice plural 2", "local struct slice ctx 2", 21 | "A1", "B1", "C1", 22 | "A2", "B2", "C2", 23 | "struct slice msgid", "struct slice plural", 24 | "backtrace init", "backtrace assign", 25 | } 26 | assert.ElementsMatch(t, want, got) 27 | } 28 | -------------------------------------------------------------------------------- /config/paths.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "strings" 4 | 5 | const ( 6 | SpreakPackagePath = "github.com/vorlif/spreak" 7 | SpreakLocalizePackagePath = SpreakPackagePath + "/localize" 8 | XSpreakPackagePath = SpreakPackagePath + "/xspreak" 9 | ) 10 | 11 | func IsValidSpreakPackage(pkg string) bool { 12 | return pkg == SpreakPackagePath || 13 | pkg == SpreakLocalizePackagePath || 14 | strings.HasPrefix(pkg, XSpreakPackagePath) 15 | } 16 | 17 | func ShouldScanStruct(pkg string) bool { 18 | if !strings.HasPrefix(pkg, SpreakPackagePath) { 19 | return true 20 | } 21 | 22 | // We need the definitions of the message. All other packages can be ignored. 23 | return pkg == SpreakLocalizePackagePath || strings.HasPrefix(pkg, XSpreakPackagePath) 24 | } 25 | 26 | func ShouldExtractPackage(pkg string) bool { 27 | if !strings.HasPrefix(pkg, SpreakPackagePath) { 28 | return true 29 | } 30 | 31 | return strings.HasPrefix(pkg, XSpreakPackagePath) 32 | } 33 | -------------------------------------------------------------------------------- /testdata/unit/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 3 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 4 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 5 | github.com/vorlif/spreak v0.1.2-0.20220511142256-22c219833d0b h1:oxvJLYsbxUjlVPm7e+RTVJC9pw6UrJQdpQX8dXGc7zk= 6 | github.com/vorlif/spreak v0.1.2-0.20220511142256-22c219833d0b/go.mod h1:/mYnzorAXrvSyN5a81dWy/e1Xhm+0vGOUHIcq8Wif2w= 7 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 8 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 10 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 11 | -------------------------------------------------------------------------------- /testdata/project/struct.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/vorlif/testdata/sub" 5 | 6 | "github.com/vorlif/spreak/localize" 7 | ) 8 | 9 | var _ = sub.Sub{ 10 | Text: "global struct msgid", 11 | Plural: "global struct plural", 12 | } 13 | 14 | type OneLineStruct struct { 15 | A, B, C localize.Singular 16 | } 17 | 18 | type OneLineGenericStruct[T any] struct { 19 | A, B, C localize.Singular 20 | } 21 | 22 | func structLocalTest() []*sub.Sub { 23 | 24 | // TRANSLATORS: Struct init 25 | _ = OneLineStruct{ 26 | A: "A3", 27 | B: "B3", 28 | C: "C3", 29 | } 30 | 31 | _ = OneLineStruct{"A4", "B4", "C4"} 32 | 33 | // TRANSLATORS: Generic struct init 34 | _ = OneLineGenericStruct[string]{ 35 | A: "GA3", 36 | B: "GB3", 37 | C: "GC3", 38 | } 39 | 40 | _ = OneLineGenericStruct[string]{"GA4", "GB4", "GC4"} 41 | 42 | item := &sub.Sub{ 43 | Text: "local struct msgid", 44 | Plural: "local struct plural", 45 | } 46 | 47 | item.Text = "struct attr assign" 48 | 49 | return []*sub.Sub{item} 50 | } 51 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | 9 | golangci: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v5 13 | - uses: actions/setup-go@v6 14 | with: 15 | go-version: '1.24' 16 | cache: false 17 | - name: lint 18 | uses: golangci/golangci-lint-action@v8 19 | with: 20 | version: latest 21 | 22 | test: 23 | strategy: 24 | matrix: 25 | platform: 26 | - ubuntu 27 | - macOS 28 | - windows 29 | go: 30 | - 24 31 | - 25 32 | name: '${{ matrix.platform }} | 1.${{ matrix.go }}.x' 33 | runs-on: ${{ matrix.platform }}-latest 34 | steps: 35 | - uses: actions/checkout@v5 36 | - uses: actions/setup-go@v6 37 | with: 38 | go-version: 1.${{ matrix.go }}.x 39 | 40 | - name: Build 41 | run: go build -v ./... 42 | 43 | - name: Test 44 | run: go test -v ./... 45 | -------------------------------------------------------------------------------- /testdata/tmpl/one.txt: -------------------------------------------------------------------------------- 1 | {{/* TRANSLATORS: ignored */}} 2 | 3 | {{/* TRANSLATORS: added */}} 4 | {{.T.Get "Get-Singular"}} 5 | {{.T.Getf "Getf-Singular"}} 6 | 7 | {{.T.NGet "NGet-Singular" "NGet-Plural"}} 8 | {{.T.NGetf "NGetf-Singular" "NGetf-Plural"}} 9 | 10 | {{.T.DGet "DGet-Domain" "DGet-Singular" }} 11 | {{.T.DGetf "DGetf-Domain" "DGetf-Singular" }} 12 | 13 | {{ .T.DNGet "DNGet-Domain" "DNGet-Singular" "DNGet-Plural" }} 14 | {{ .T.DNGetf "DNGetf-Domain" "DNGetf-Singular" "DNGetf-Plural" }} 15 | 16 | {{ .T.PGet "PGet-Context" "PGet-Singular" }} 17 | {{ .T.PGetf "PGetf-Context" "PGetf-Singular" }} 18 | 19 | {{ .T.DPGet "DPGet-Domain" "DPGet-Context" "DPGet-Singular" }} 20 | {{ .T.DPGetf "DPGetf-Domain" "DPGetf-Context" "DPGetf-Singular" }} 21 | 22 | {{ .T.NPGet "NPGet-Context" "NPGet-Singular" "NPGet-Plural" }} 23 | {{ .T.NPGetf "NPGetf-Context" "NPGetf-Singular" "NPGetf-Plural" }} 24 | 25 | {{ .T.DNPGet "DNPGet-Context" "DNPGet-Context" "DNPGet-Singular" "DNPGet-Plural" }} 26 | {{ .T.DNPGetf "DNPGetf-Context" "DNPGetf-Context" "DNPGetf-Singular" "DNPGetf-Plural" }} -------------------------------------------------------------------------------- /extract/loader/comments.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "go/ast" 5 | "time" 6 | 7 | "golang.org/x/tools/go/packages" 8 | 9 | "github.com/vorlif/xspreak/extract" 10 | "github.com/vorlif/xspreak/util" 11 | ) 12 | 13 | // extractComments extracts all comments from the transferred packages and processes them in such a way 14 | // that they can be easily accessed. 15 | func extractComments(pkgs []*packages.Package) extract.Comments { 16 | util.TrackTime(time.Now(), "Extract comments") 17 | 18 | comments := make(extract.Comments) 19 | 20 | for _, pkg := range pkgs { 21 | for _, file := range pkg.Syntax { 22 | position := pkg.Fset.Position(file.Pos()) 23 | if !position.IsValid() { 24 | continue 25 | } 26 | 27 | commentMap := ast.NewCommentMap(pkg.Fset, file, file.Comments) 28 | if len(commentMap) == 0 { 29 | continue 30 | } 31 | 32 | if _, hasPkg := comments[pkg.ID]; !hasPkg { 33 | comments[pkg.ID] = make(map[string]ast.CommentMap) 34 | } 35 | comments[pkg.ID][position.Filename] = commentMap 36 | } 37 | } 38 | 39 | return comments 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2022 Florian Vogt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /extract/extractors/inline_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/vorlif/xspreak/config" 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/extract/loader" 13 | "github.com/vorlif/xspreak/extract/runner" 14 | "github.com/vorlif/xspreak/tmpl" 15 | ) 16 | 17 | func TestInlineExtraction(t *testing.T) { 18 | cfg := config.NewDefault() 19 | cfg.SourceDir = testdataDir 20 | cfg.ExtractErrors = true 21 | cfg.Keywords = tmpl.DefaultKeywords("T", false) 22 | require.NoError(t, cfg.Prepare()) 23 | ctx := context.Background() 24 | contextLoader := loader.NewPackageLoader(cfg) 25 | 26 | extractCtx, err := contextLoader.Load(ctx) 27 | require.NoError(t, err) 28 | 29 | runner, err := runner.New(cfg, extractCtx.Packages) 30 | require.NoError(t, err) 31 | 32 | e := []extract.Extractor{NewInlineTemplateExtractor()} 33 | issues, err := runner.Run(ctx, extractCtx, e) 34 | require.NoError(t, err) 35 | assert.Empty(t, issues) 36 | 37 | assert.Equal(t, 3, len(extractCtx.Templates)) 38 | } 39 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | enable: 4 | - forbidigo 5 | - godot 6 | - misspell 7 | - revive 8 | - unconvert 9 | settings: 10 | cyclop: 11 | max-complexity: 15 12 | package-average: 0 13 | forbidigo: 14 | forbid: 15 | - pattern: fmt\.Print.* 16 | godot: 17 | scope: declarations 18 | exclude: 19 | - '^fixme:' 20 | - '^todo:' 21 | capital: false 22 | period: true 23 | exclusions: 24 | generated: lax 25 | presets: 26 | - comments 27 | - common-false-positives 28 | - legacy 29 | - std-error-handling 30 | paths: 31 | - third_party$ 32 | - builtin$ 33 | - examples$ 34 | rules: 35 | - path: util 36 | text: 'var-naming: avoid meaningless package names' 37 | linters: 38 | - revive 39 | issues: 40 | fix: true 41 | formatters: 42 | enable: 43 | - gofmt 44 | - goimports 45 | settings: 46 | goimports: 47 | local-prefixes: 48 | - github.com/vorlif/spreak 49 | exclusions: 50 | generated: lax 51 | paths: 52 | - third_party$ 53 | - builtin$ 54 | - examples$ 55 | -------------------------------------------------------------------------------- /testdata/project/maps.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | alias "github.com/vorlif/spreak/localize" 5 | 6 | "github.com/vorlif/testdata/sub" 7 | ) 8 | 9 | var globalKeyMap = map[alias.Singular]string{ 10 | "globalKeyMap-a": "a", 11 | "globalKeyMap-b": "b", 12 | } 13 | 14 | var globalValueMap = map[string]alias.Singular{ 15 | "a": "globalValueMap-a", 16 | "b": "globalValueMap-b", 17 | } 18 | 19 | var globalMap = map[alias.MsgID]alias.Singular{ 20 | "globalMap-ka": "globalMap-va", 21 | "globalMap-kb": "globalMap-vb", 22 | } 23 | 24 | func mapFunc() { 25 | _ = map[alias.Singular]string{ 26 | "localKeyMap-a": "a", 27 | "localKeyMap-b": "b", 28 | } 29 | 30 | _ = map[string]alias.Singular{ 31 | "a": "localValueMap-a", 32 | "b": "localValueMap-b", 33 | } 34 | 35 | _ = map[alias.MsgID]alias.Singular{ 36 | "localMap-ka": "localMap-va", 37 | "localMap-kb": "localMap-vb", 38 | } 39 | 40 | _ = map[string]sub.Sub{ 41 | "key": { 42 | Text: "map struct msgid", 43 | Plural: "map struct plural", 44 | }, 45 | } 46 | 47 | _ = map[string]*sub.Sub{ 48 | "key": { 49 | Text: "map pointer struct msgid", 50 | Plural: "map pointer struct plural", 51 | }, 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /testdata/project/funcreturn.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/vorlif/spreak/localize" 7 | alias "github.com/vorlif/spreak/localize" 8 | ) 9 | 10 | const msgB = "bt_msgB" 11 | 12 | func returnSingle() localize.MsgID { 13 | return "single" 14 | } 15 | 16 | func returnPlural() (alias.MsgID, localize.Plural) { 17 | return "plural_s", "plural_p" 18 | } 19 | 20 | func returnContext() (alias.MsgID, localize.Context, localize.Plural) { 21 | return "context_s", "context_c", "context_p" 22 | } 23 | 24 | func returnFull() (alias.MsgID, alias.Context, alias.Plural, alias.Domain) { 25 | return "full_s", "full_c", "full_p", "full_d" 26 | } 27 | 28 | func returnBacktracking() (alias.MsgID, alias.Context, alias.Plural, alias.Domain) { 29 | ctxA := "bt_ctx_a" 30 | ctxB := "bt_ctx_b" 31 | 32 | domainA := "bt_domain_A" 33 | domainB := "bt_domain_B" 34 | 35 | msgA := "bt_msgA" 36 | 37 | pluralA := "bt_plural_a" 38 | pluralB := "bt_plural_b" 39 | 40 | switch rand.Intn(101) { 41 | case 1: 42 | return msgA, "bt_ctx_c", pluralA, "bt_domain_c" 43 | case 2: 44 | return msgB, ctxA, pluralB, domainA 45 | default: 46 | return "bt_msg_c", ctxB, pluralA, domainB 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /tmpl/parse_test.go: -------------------------------------------------------------------------------- 1 | package tmpl 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestParse(t *testing.T) { 12 | text := `This is an test {{.T.Get "hello" "world"}} 13 | {{.T.Get "foo"}} 14 | 15 | ` 16 | res, err := ParseBytes("test", []byte(text)) 17 | require.NoError(t, err) 18 | require.NotNil(t, res) 19 | assert.Len(t, res.OffsetLookup, 4) 20 | assert.Equal(t, 1, res.OffsetLookup[0].Line) 21 | } 22 | 23 | func TestExtractComments(t *testing.T) { 24 | text := `{{/* start comment */}} This is an test {{.T.Get "hello" "world"}} 25 | 26 | {{/*a comment 27 | with 28 | multiline */}} 29 | {{.T.Get "foo"}} 30 | 31 | {{- /* also a comment */ -}} 32 | ` 33 | res, err := ParseBytes("test", []byte(text)) 34 | assert.NoError(t, err) 35 | require.NotNil(t, res) 36 | 37 | assert.Len(t, res.Comments, 0) 38 | res.ExtractComments() 39 | assert.Len(t, res.Comments, 3) 40 | } 41 | 42 | func TestParseHtml(t *testing.T) { 43 | text, err := os.ReadFile("../testdata/tmpl/five.tmpl") 44 | if err != nil { 45 | panic(err) 46 | } 47 | 48 | res, err := ParseBytes("test", text) 49 | assert.NoError(t, err) 50 | require.NotNil(t, res) 51 | } 52 | -------------------------------------------------------------------------------- /extract/processors/unprintable_checker.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "go/token" 5 | "os" 6 | "path/filepath" 7 | "strconv" 8 | "unicode" 9 | 10 | log "github.com/sirupsen/logrus" 11 | 12 | "github.com/vorlif/xspreak/extract" 13 | ) 14 | 15 | var workingDir, _ = os.Getwd() 16 | 17 | type unprintableChecker struct{} 18 | 19 | func NewUnprintableCheck() Processor { 20 | return &unprintableChecker{} 21 | } 22 | 23 | func (u unprintableChecker) Process(issues []extract.Issue) ([]extract.Issue, error) { 24 | 25 | for _, iss := range issues { 26 | checkForUnprintableChars(iss.MsgID, iss.Pos) 27 | checkForUnprintableChars(iss.PluralID, iss.Pos) 28 | } 29 | 30 | return issues, nil 31 | } 32 | 33 | func (u unprintableChecker) Name() string { return "unprintable check" } 34 | 35 | func checkForUnprintableChars(s string, pos token.Position) { 36 | for _, r := range s { 37 | if !unicode.IsPrint(r) && !unicode.IsSpace(r) { 38 | filename := pos.Filename 39 | if relPath, err := filepath.Rel(workingDir, filename); err == nil { 40 | filename = relPath 41 | } 42 | 43 | log.Warnf("%s:%d internationalized messages should not contain the %s character", filename, pos.Line, strconv.QuoteRune(r)) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /merger/json_test.go: -------------------------------------------------------------------------------- 1 | package merger 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/vorlif/spreak/catalog/cldrplural" 10 | ) 11 | 12 | func TestMergeJson(t *testing.T) { 13 | 14 | t.Run("missing IDs will be created", func(t *testing.T) { 15 | src := []byte(`{ 16 | "a": "", "c_ctx": {"context": "ctx", "other": "c"}, "d": "", "b": "" 17 | }`) 18 | dst := []byte(`{ 19 | "a": "A", 20 | "d": "D" 21 | }`) 22 | res := MergeJSON(src, dst, []cldrplural.Category{cldrplural.One, cldrplural.Many, cldrplural.Other}) 23 | require.NotNil(t, res) 24 | 25 | want := `{ 26 | "a": "A", 27 | "b": "", 28 | "c_ctx": { 29 | "context": "ctx", 30 | "other": "c" 31 | }, 32 | "d": "D" 33 | }` 34 | assert.JSONEq(t, want, string(res)) 35 | }) 36 | 37 | t.Run("new plurals are created", func(t *testing.T) { 38 | src := []byte(`{ 39 | "a": {"zero": "", "other": "O"}, 40 | "b_ctx": {"context": "ctx", "zero": "", "other": ""} 41 | }`) 42 | 43 | res := MergeJSON(src, nil, []cldrplural.Category{cldrplural.One, cldrplural.Many, cldrplural.Other}) 44 | require.NotNil(t, res) 45 | 46 | want := `{ 47 | "a": {"one": "", "many": "", "other": "O"}, 48 | "b_ctx": {"context": "ctx", "one": "", "many": "", "other": ""} 49 | }` 50 | assert.JSONEq(t, want, string(res)) 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /encoder/encoder_test.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestReGoStringFormat(t *testing.T) { 10 | tests := []struct { 11 | text string 12 | assert assert.BoolAssertionFunc 13 | }{ 14 | {"", assert.False}, 15 | {"text", assert.False}, 16 | {"text %%", assert.False}, 17 | {"1234 %%", assert.False}, 18 | {"%v", assert.True}, 19 | {"%#v", assert.True}, 20 | {"%T", assert.True}, 21 | {"%+v", assert.True}, 22 | {"%t", assert.True}, 23 | {"%b", assert.True}, 24 | {"%c", assert.True}, 25 | {"%d", assert.True}, 26 | {"%o", assert.True}, 27 | {"%O", assert.True}, 28 | {"%x", assert.True}, 29 | {"%X", assert.True}, 30 | {"%U", assert.True}, 31 | {"%e", assert.True}, 32 | {"%E", assert.True}, 33 | {"%g", assert.True}, 34 | {"%G", assert.True}, 35 | {"%s", assert.True}, 36 | {"%q", assert.True}, 37 | {"%p", assert.True}, 38 | {"%f", assert.True}, 39 | {"%9f", assert.True}, 40 | {"%.2f", assert.True}, 41 | {"%9.2f", assert.True}, 42 | {"%9.f", assert.True}, 43 | {"%[2]d", assert.True}, 44 | // {"1234 %%s", assert.False}, 45 | {"%#[1]x", assert.True}, 46 | {"%*[2]d", assert.True}, 47 | {"%.[2]d", assert.True}, 48 | } 49 | 50 | for _, tt := range tests { 51 | t.Run(tt.text, func(t *testing.T) { 52 | tt.assert(t, reGoStringFormat.MatchString(tt.text)) 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /extract/extractors/funccall_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestFuncCallExtractor(t *testing.T) { 10 | issues := runExtraction(t, testdataDir, NewFuncCallExtractor()) 11 | assert.NotEmpty(t, issues) 12 | 13 | want := []string{ 14 | "f-msgid", "f-plural", "f-context", "f-domain", 15 | "init", "localizer func call", 16 | "noop-msgid", "noop-plural", "noop-context", "noop-domain", 17 | "msgid", 18 | "msgid-n", "pluralid-n", 19 | "domain-d", "msgid-d", 20 | "domain-dn", "msgid-dn", "pluralid-dn", 21 | "context-pg", "msgid-pg", 22 | "context-np", "msgid-np", "pluralid-np", 23 | "domain-dp", "context-dp", "singular-dp", 24 | "domain-dnp", "context-dnp", "msgid-dnp", "pluralid-dnp", 25 | "submsgid", "subplural", "foo test", 26 | "generic-call", 27 | "pre-variadic", "variadic-a", "variadic-b", 28 | "no-param-msgid", "no-param-plural", 29 | "multi-names-a", "multi-names-b", 30 | "init backtrace", "assign backtrace", 31 | "inline function", 32 | 33 | "constCtxMsg", "constCtxVal", 34 | 35 | "struct-method-call", "generic-struct-method-call", 36 | } 37 | got := collectIssueStrings(issues) 38 | assert.ElementsMatch(t, want, got) 39 | 40 | for _, iss := range issues { 41 | switch iss.MsgID { 42 | case "constCtxMsg": 43 | assert.Equal(t, "constCtxVal", iss.Context) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /util/flags.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | const ( 12 | flagPrefix = "xspreak:" 13 | templateMarkerLong = "template" 14 | templateMarkerShort = "tmpl" 15 | ) 16 | 17 | var reRange = regexp.MustCompile(`^range:\s+\d+\.\.\d+\s*$`) 18 | 19 | func IsInlineTemplate(comment string) bool { 20 | for _, line := range strings.Split(comment, "\n") { 21 | line = strings.ToLower(strings.TrimSpace(line)) 22 | if strings.HasPrefix(line, flagPrefix) && (strings.Contains(line, templateMarkerLong) || strings.Contains(line, templateMarkerShort)) { 23 | return true 24 | } 25 | } 26 | 27 | return false 28 | } 29 | 30 | func ParseFlags(line string) []string { 31 | possibleFlags := strings.Split(strings.TrimPrefix(line, flagPrefix), ",") 32 | flags := make([]string, 0, len(possibleFlags)) 33 | for _, flag := range possibleFlags { 34 | flag = strings.ToLower(strings.TrimSpace(flag)) 35 | 36 | if strings.HasPrefix(flag, "range:") { 37 | if !reRange.MatchString(flag) { 38 | log.WithField("input", flag).Warn("Invalid range flag") 39 | continue 40 | } 41 | 42 | rangeFlag := fmt.Sprintf("range: %s", strings.TrimSpace(strings.TrimPrefix(flag, "range:"))) 43 | flags = append(flags, rangeFlag) 44 | } 45 | 46 | if flag == "ignore" { 47 | flags = append(flags, flag) 48 | } 49 | } 50 | 51 | return flags 52 | } 53 | -------------------------------------------------------------------------------- /util/ast.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "go/ast" 6 | "go/types" 7 | "strings" 8 | ) 9 | 10 | func ObjToKey(obj types.Object) string { 11 | switch v := obj.Type().(type) { 12 | case *types.Signature: 13 | if recv := v.Recv(); recv != nil { 14 | // Strip out the generic type declaration from the type name. 15 | // The ast.CallExpr reports its receiver as the actual type 16 | // (e.g.`Generic[string]`), whereas the ast.FuncDecl on the 17 | // same type as `Generic[T]`. The returned key values need 18 | // to be consistent between different invocation patterns. 19 | recv, _, _ := strings.Cut(recv.Type().String(), "[") 20 | 21 | return fmt.Sprintf("%s.%s", recv, obj.Name()) 22 | } 23 | 24 | return fmt.Sprintf("%s.%s", obj.Pkg().Path(), obj.Name()) 25 | case *types.Pointer: 26 | return v.Elem().String() 27 | default: 28 | return fmt.Sprintf("%s.%s", obj.Pkg().Path(), obj.Name()) 29 | } 30 | } 31 | 32 | func SearchSelector(expr any) *ast.SelectorExpr { 33 | current := expr 34 | for current != nil { 35 | switch v := current.(type) { 36 | case *ast.SelectorExpr: 37 | return v 38 | case *ast.Ident: 39 | if v.Obj == nil { 40 | return nil 41 | } 42 | current = v.Obj.Decl 43 | case *ast.ValueSpec: 44 | current = v.Type 45 | case *ast.Field: 46 | current = v.Type 47 | case *ast.Ellipsis: 48 | current = v.Elt 49 | default: 50 | return nil 51 | } 52 | } 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /testdata/tmpl/five.tmpl: -------------------------------------------------------------------------------- 1 |
2 |

Privacy

3 |
4 |

Emails you share with this site will be used to 5 | provide you with the data about them you've requested. That data, including the entire content of the email, is made available without any authentication needed at a semi-private URL.

6 |

We don't currently plan on using any of that data for anything other than ensuring smooth running of the site.

7 |

Contact us if you have any questions.

8 |
9 |
-------------------------------------------------------------------------------- /extract/extractors/error.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "time" 7 | 8 | "github.com/vorlif/xspreak/extract" 9 | "github.com/vorlif/xspreak/util" 10 | 11 | "github.com/vorlif/xspreak/config" 12 | ) 13 | 14 | type errorExtractor struct{} 15 | 16 | func NewErrorExtractor() extract.Extractor { 17 | return &errorExtractor{} 18 | } 19 | 20 | func (v errorExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 21 | util.TrackTime(time.Now(), "extract errors") 22 | var issues []extract.Issue 23 | 24 | extractCtx.Inspector.Nodes([]ast.Node{&ast.CallExpr{}}, func(rawNode ast.Node, push bool) (proceed bool) { 25 | proceed = true 26 | if !push { 27 | return 28 | } 29 | 30 | node := rawNode.(*ast.CallExpr) 31 | if len(node.Args) != 1 { 32 | return 33 | } 34 | 35 | selector := util.SearchSelector(node.Fun) 36 | if selector == nil { 37 | return 38 | } 39 | 40 | pkg, obj := extractCtx.GetType(selector.Sel) 41 | if pkg == nil { 42 | return 43 | } 44 | 45 | if obj.Pkg().Path() != "errors" || !config.ShouldExtractPackage(pkg.PkgPath) { 46 | return 47 | } 48 | 49 | msgID, msgNode := extract.StringLiteral(node.Args[0]) 50 | if msgID == "" { 51 | return 52 | } 53 | 54 | issue := extract.Issue{ 55 | FromExtractor: v.Name(), 56 | MsgID: msgID, 57 | Pkg: pkg, 58 | Context: extractCtx.Config.ErrorContext, 59 | Comments: extractCtx.GetComments(pkg, msgNode), 60 | Pos: extractCtx.GetPosition(msgNode.Pos()), 61 | } 62 | 63 | issues = append(issues, issue) 64 | 65 | return 66 | }) 67 | 68 | return issues, nil 69 | } 70 | 71 | func (v errorExtractor) Name() string { 72 | return "error_extractor" 73 | } 74 | -------------------------------------------------------------------------------- /tmplextractors/testlib_test.go: -------------------------------------------------------------------------------- 1 | package tmplextractors 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/vorlif/xspreak/config" 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/extract/loader" 13 | "github.com/vorlif/xspreak/extract/runner" 14 | ) 15 | 16 | var ( 17 | testdataDir = filepath.FromSlash("../testdata/project") 18 | testdataTemplates = filepath.FromSlash("../testdata/tmpl") 19 | ) 20 | 21 | func runExtraction(t *testing.T, dir string, testExtractors ...extract.Extractor) []extract.Issue { 22 | cfg := config.NewDefault() 23 | cfg.SourceDir = dir 24 | cfg.ExtractErrors = false 25 | cfg.TemplatePatterns = []string{ 26 | testdataTemplates + "/**/*.txt", 27 | testdataTemplates + "/**/*.html", 28 | testdataTemplates + "/**/*.tmpl", 29 | } 30 | 31 | require.NoError(t, cfg.Prepare()) 32 | 33 | ctx := context.Background() 34 | contextLoader := loader.NewPackageLoader(cfg) 35 | 36 | extractCtx, err := contextLoader.Load(ctx) 37 | require.NoError(t, err) 38 | 39 | runner, err := runner.New(cfg, extractCtx.Packages) 40 | require.NoError(t, err) 41 | 42 | issues, err := runner.Run(ctx, extractCtx, testExtractors) 43 | require.NoError(t, err) 44 | return issues 45 | } 46 | 47 | func collectIssueStrings(issues []extract.Issue) []string { 48 | collection := make([]string, 0, len(issues)) 49 | for _, issue := range issues { 50 | collection = append(collection, issue.MsgID) 51 | if issue.PluralID != "" { 52 | collection = append(collection, issue.PluralID) 53 | } 54 | 55 | if issue.Context != "" { 56 | collection = append(collection, issue.Context) 57 | } 58 | 59 | if issue.Domain != "" { 60 | collection = append(collection, issue.Domain) 61 | } 62 | } 63 | return collection 64 | } 65 | -------------------------------------------------------------------------------- /testdata/project/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 7 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 8 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 9 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 10 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 11 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 12 | github.com/vorlif/spreak v1.0.0 h1:SUaD/p+cWcGgLAdHi73cTmBBejpPpztlgM5I4kPSewY= 13 | github.com/vorlif/spreak v1.0.0/go.mod h1:oJ0AuinQV2XPy8WkdkbGejGDHQ3dCoB9brQMj5dsEyc= 14 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= 15 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 16 | golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= 17 | golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 19 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 20 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 21 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 22 | -------------------------------------------------------------------------------- /extract/extractors/globalassign.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "time" 7 | 8 | "github.com/vorlif/xspreak/extract" 9 | "github.com/vorlif/xspreak/extract/etype" 10 | "github.com/vorlif/xspreak/util" 11 | ) 12 | 13 | type globalAssignExtractor struct{} 14 | 15 | func NewGlobalAssignExtractor() extract.Extractor { 16 | return &globalAssignExtractor{} 17 | } 18 | 19 | func (v globalAssignExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 20 | util.TrackTime(time.Now(), "extract global assign") 21 | var issues []extract.Issue 22 | 23 | extractCtx.Inspector.Nodes([]ast.Node{&ast.ValueSpec{}}, func(rawNode ast.Node, push bool) (proceed bool) { 24 | proceed = true 25 | if !push { 26 | return 27 | } 28 | node := rawNode.(*ast.ValueSpec) 29 | 30 | selector := util.SearchSelector(node.Type) 31 | if selector == nil { 32 | return 33 | } 34 | 35 | tok := extractCtx.GetLocalizeTypeToken(selector) 36 | if tok != etype.Singular { 37 | if tok != etype.None { 38 | writeMissingMessageID(extractCtx.GetPosition(selector.Pos()), tok, "") 39 | } 40 | return 41 | } 42 | 43 | in, ok := selector.X.(*ast.Ident) 44 | if !ok { 45 | return 46 | } 47 | 48 | pkg, _ := extractCtx.GetType(in) 49 | if pkg == nil { 50 | return 51 | } 52 | 53 | for _, value := range node.Values { 54 | for _, res := range extractCtx.SearchStrings(value) { 55 | issue := extract.Issue{ 56 | FromExtractor: v.Name(), 57 | MsgID: res.Raw, 58 | Pkg: pkg, 59 | Comments: extractCtx.GetComments(pkg, res.Node), 60 | Pos: extractCtx.GetPosition(res.Node.Pos()), 61 | } 62 | 63 | issues = append(issues, issue) 64 | } 65 | } 66 | 67 | return 68 | }) 69 | 70 | return issues, nil 71 | } 72 | 73 | func (v globalAssignExtractor) Name() string { 74 | return "global_assign" 75 | } 76 | -------------------------------------------------------------------------------- /extract/loader/package_cleaner.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "golang.org/x/tools/go/packages" 8 | 9 | "github.com/vorlif/xspreak/config" 10 | "github.com/vorlif/xspreak/util" 11 | ) 12 | 13 | type packageCleaner struct { 14 | // All loaded packages 15 | allPackages []*packages.Package 16 | 17 | // Packages that were visited 18 | visited map[string]bool 19 | 20 | cleanedPackages []*packages.Package 21 | } 22 | 23 | func cleanPackages(originalPackages []*packages.Package) []*packages.Package { 24 | pc := &packageCleaner{ 25 | allPackages: originalPackages, 26 | visited: make(map[string]bool), 27 | cleanedPackages: make([]*packages.Package, 0, len(originalPackages)), 28 | } 29 | 30 | pc.performCleanup() 31 | 32 | return pc.cleanedPackages 33 | } 34 | 35 | func (pc *packageCleaner) performCleanup() { 36 | defer util.TrackTime(time.Now(), "Collect packages") 37 | 38 | for _, startPck := range pc.allPackages { 39 | pc.collectPackages(startPck) 40 | } 41 | } 42 | 43 | func (pc *packageCleaner) collectPackages(startPck *packages.Package) { 44 | queue := []*packages.Package{startPck} 45 | 46 | for len(queue) > 0 { 47 | current := queue[0] 48 | queue = queue[1:] 49 | 50 | if pc.visited[current.ID] { 51 | continue 52 | } 53 | 54 | pc.visited[current.ID] = true 55 | pc.cleanedPackages = append(pc.cleanedPackages, current) 56 | 57 | for _, importedPackage := range current.Imports { 58 | if pc.visited[importedPackage.ID] { 59 | continue 60 | } 61 | 62 | if !pc.isPartOfDirectory(importedPackage) { 63 | continue 64 | } 65 | 66 | queue = append(queue, importedPackage) 67 | } 68 | } 69 | } 70 | 71 | func (pc *packageCleaner) isPartOfDirectory(pkg *packages.Package) bool { 72 | if config.IsValidSpreakPackage(pkg.PkgPath) { 73 | return true 74 | } 75 | 76 | for _, src := range pc.allPackages { 77 | if strings.HasPrefix(pkg.PkgPath, src.PkgPath) { 78 | return true 79 | } 80 | } 81 | 82 | return false 83 | } 84 | -------------------------------------------------------------------------------- /testdata/project/slices.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/vorlif/testdata/sub" 5 | 6 | "github.com/vorlif/spreak/localize" 7 | ) 8 | 9 | // TRANSLATORS: This is not extracted 10 | var globalSlice = []localize.Singular{ 11 | // TRANSLATORS: numbers 1 to 4 12 | // only for "one" extracted 13 | "one", "two", "three", "four", 14 | } 15 | 16 | var globalStructSlice = []localize.Message{ 17 | { 18 | // TRANSLATORS: For singular extracted 19 | Singular: "global struct slice singular", 20 | Plural: "global struct slice plural", 21 | Context: "global ctx", 22 | Vars: nil, 23 | Count: 0, 24 | }, 25 | { 26 | "global struct slice singular 2", 27 | // TRANSLATORS: For plural extracted 28 | "global struct slice plural 2", 29 | "", 30 | nil, 31 | 0, 32 | }, 33 | } 34 | 35 | func localSliceFunc() []string { 36 | globalSlice = append(globalSlice, "five") 37 | 38 | backtrace := "backtrace init" 39 | backtrace = "backtrace assign" 40 | 41 | localSlice := []localize.Singular{"six", "seven", "eight", "nine", backtrace} 42 | localSlice = append(localSlice, "ten") 43 | 44 | _ = []localize.Message{ 45 | { 46 | Singular: "local struct slice singular", 47 | Plural: "local struct slice plural", 48 | Context: "local ctx", 49 | Vars: nil, 50 | Count: 0, 51 | }, 52 | { 53 | "local struct slice singular 2", 54 | "local struct slice plural 2", 55 | "local struct slice ctx 2", 56 | nil, 57 | 0, 58 | }, 59 | } 60 | 61 | subs := []*sub.Sub{ 62 | { 63 | Text: "struct slice msgid", 64 | Plural: "struct slice plural", 65 | }, 66 | } 67 | 68 | _ = []OneLineStruct{ 69 | { 70 | A: "A1", 71 | B: "B1", 72 | C: "C1", 73 | }, 74 | {"A2", "B2", "C2"}, 75 | } 76 | 77 | _ = append(subs, 78 | &sub.Sub{ 79 | Text: "struct msgid arr1", 80 | Plural: "struct plural arr1", 81 | }, 82 | &sub.Sub{ 83 | Text: "struct msgid arr2", 84 | Plural: "struct plural arr2", 85 | }, 86 | ) 87 | 88 | return localSlice 89 | } 90 | -------------------------------------------------------------------------------- /tmpl/walk.go: -------------------------------------------------------------------------------- 1 | package tmpl 2 | 3 | import ( 4 | "fmt" 5 | "text/template/parse" 6 | ) 7 | 8 | type Visitor interface { 9 | Visit(node parse.Node) (w Visitor) 10 | } 11 | 12 | func Walk(v Visitor, node parse.Node) { 13 | if v = v.Visit(node); v == nil { 14 | return 15 | } 16 | 17 | switch n := node.(type) { 18 | case *parse.ActionNode: 19 | if n.Pipe != nil { 20 | Walk(v, n.Pipe) 21 | } 22 | case *parse.BranchNode: 23 | if n.Pipe != nil { 24 | Walk(v, n.Pipe) 25 | } 26 | if n.List != nil { 27 | Walk(v, n.List) 28 | } 29 | if n.ElseList != nil { 30 | Walk(v, n.ElseList) 31 | } 32 | case *parse.ChainNode: 33 | Walk(v, n.Node) 34 | case *parse.CommandNode: 35 | for _, arg := range n.Args { 36 | Walk(v, arg) 37 | } 38 | case *parse.IfNode: 39 | Walk(v, &n.BranchNode) 40 | case *parse.ListNode: 41 | if len(n.Nodes) > 0 { 42 | for _, arg := range n.Nodes { 43 | Walk(v, arg) 44 | } 45 | } 46 | case *parse.PipeNode: 47 | if len(n.Decl) > 0 { 48 | for _, arg := range n.Decl { 49 | Walk(v, arg) 50 | } 51 | } 52 | if len(n.Cmds) > 0 { 53 | for _, arg := range n.Cmds { 54 | Walk(v, arg) 55 | } 56 | } 57 | case *parse.RangeNode: 58 | Walk(v, &n.BranchNode) 59 | case *parse.TemplateNode: 60 | if n.Pipe != nil { 61 | Walk(v, n.Pipe) 62 | } 63 | case *parse.WithNode: 64 | Walk(v, &n.BranchNode) 65 | case *parse.BoolNode, *parse.BreakNode, *parse.CommentNode, *parse.ContinueNode, *parse.DotNode, 66 | *parse.FieldNode, *parse.IdentifierNode, *parse.NilNode, *parse.NumberNode, *parse.StringNode, 67 | *parse.TextNode, *parse.VariableNode: 68 | // Walk(v, n) 69 | default: 70 | panic(fmt.Sprintf("tmpl.Walk: unexpected node type %T", n)) 71 | } 72 | 73 | v.Visit(nil) 74 | } 75 | 76 | type inspector func(parse.Node) bool 77 | 78 | func (f inspector) Visit(node parse.Node) Visitor { 79 | if f(node) { 80 | return f 81 | } 82 | return nil 83 | } 84 | 85 | func Inspect(node parse.Node, f func(parse.Node) bool) { 86 | Walk(inspector(f), node) 87 | } 88 | -------------------------------------------------------------------------------- /extract/extractors/testlib_test.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "go/parser" 7 | "go/token" 8 | "path/filepath" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | 14 | "github.com/vorlif/xspreak/config" 15 | "github.com/vorlif/xspreak/extract" 16 | "github.com/vorlif/xspreak/extract/loader" 17 | "github.com/vorlif/xspreak/extract/runner" 18 | ) 19 | 20 | const testdataDir = "../../testdata/project" 21 | 22 | func TestPrintAst(t *testing.T) { 23 | fset := token.NewFileSet() // positions are relative to fset 24 | f, err := parser.ParseFile(fset, filepath.Join(testdataDir, "funccall.go"), nil, 0) 25 | require.NoError(t, err) 26 | 27 | err = ast.Print(fset, f) 28 | assert.NoError(t, err) 29 | } 30 | 31 | func runExtraction(t *testing.T, dir string, testExtractors ...extract.Extractor) []extract.Issue { 32 | cfg := config.NewDefault() 33 | cfg.SourceDir = dir 34 | cfg.ExtractErrors = true 35 | require.NoError(t, cfg.Prepare()) 36 | ctx := context.Background() 37 | contextLoader := loader.NewPackageLoader(cfg) 38 | 39 | extractCtx, err := contextLoader.Load(ctx) 40 | require.NoError(t, err) 41 | 42 | runner, err := runner.New(cfg, extractCtx.Packages) 43 | require.NoError(t, err) 44 | 45 | var e []extract.Extractor 46 | if len(testExtractors) > 0 { 47 | e = append(e, testExtractors...) 48 | } 49 | issues, err := runner.Run(ctx, extractCtx, e) 50 | require.NoError(t, err) 51 | return issues 52 | } 53 | 54 | func collectIssueStrings(issues []extract.Issue) []string { 55 | collection := make([]string, 0, len(issues)) 56 | for _, issue := range issues { 57 | collection = append(collection, issue.MsgID) 58 | if issue.PluralID != "" { 59 | collection = append(collection, issue.PluralID) 60 | } 61 | 62 | if issue.Context != "" { 63 | collection = append(collection, issue.Context) 64 | } 65 | 66 | if issue.Domain != "" { 67 | collection = append(collection, issue.Domain) 68 | } 69 | } 70 | return collection 71 | } 72 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Upload Release Assets 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | release: 9 | name: Upload Release Asset 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout repository 13 | uses: actions/checkout@v5 14 | - name: Install Go 15 | uses: actions/setup-go@v6 16 | with: 17 | go-version: '>=1.24.0' 18 | - name: Build binaries 19 | run: | 20 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ 21 | -ldflags="-X 'github.com/vorlif/xspreak/commands.Version=$(git describe --tags)'" \ 22 | -o "xspreak-$(git describe --tags)-linux-amd64" 23 | CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build \ 24 | -ldflags="-X 'github.com/vorlif/xspreak/commands.Version=$(git describe --tags)'" \ 25 | -o "xspreak-$(git describe --tags)-darwin-amd64" 26 | CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build \ 27 | -ldflags="-X 'github.com/vorlif/xspreak/commands.Version=$(git describe --tags)'" \ 28 | -o "xspreak-$(git describe --tags)-windows-amd64.exe" 29 | - name: Upload release artifacts 30 | uses: actions/github-script@v7 31 | with: 32 | github-token: ${{ secrets.GITHUB_TOKEN }} 33 | script: | 34 | const fs = require("fs").promises; 35 | const { repo: { owner, repo }, sha } = context; 36 | 37 | const release = await github.repos.getReleaseByTag({ 38 | owner, repo, 39 | tag: process.env.GITHUB_REF.replace("refs/tags/", ""), 40 | }); 41 | console.log("Release:", { release }); 42 | 43 | for (let file of await fs.readdir(".")) { 44 | if (!file.startsWith("xspreak-")) continue; 45 | console.log("Uploading", file); 46 | await github.repos.uploadReleaseAsset({ 47 | owner, repo, 48 | release_id: release.data.id, 49 | name: file, 50 | data: await fs.readFile(file), 51 | }); 52 | } 53 | -------------------------------------------------------------------------------- /extract/loader/definition_test.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/vorlif/xspreak/config" 11 | ) 12 | 13 | func TestDefinitionExtractor(t *testing.T) { 14 | cfg := config.NewDefault() 15 | cfg.SourceDir = testdataDir 16 | require.NoError(t, cfg.Prepare()) 17 | ctx := context.Background() 18 | contextLoader := NewPackageLoader(cfg) 19 | 20 | extractCtx, err := contextLoader.Load(ctx) 21 | require.NoError(t, err) 22 | 23 | defs := extractCtx.Definitions 24 | 25 | key := "github.com/vorlif/testdata.M" 26 | if assert.Contains(t, defs, key) { 27 | assert.Contains(t, defs[key], "Test") 28 | } 29 | 30 | key = "github.com/vorlif/testdata.methodStruct.Method" 31 | if assert.Contains(t, defs, key) { 32 | assert.Contains(t, defs[key], "0") 33 | } 34 | 35 | key = "github.com/vorlif/testdata.genericMethodStruct.Method" 36 | if assert.Contains(t, defs, key) { 37 | assert.Contains(t, defs[key], "0") 38 | } 39 | 40 | key = "github.com/vorlif/testdata.noop" 41 | if assert.Contains(t, defs, key) { 42 | assert.Contains(t, defs[key], "sing") 43 | assert.Contains(t, defs[key], "plural") 44 | assert.Contains(t, defs[key], "context") 45 | assert.Contains(t, defs[key], "domain") 46 | } 47 | 48 | key = "github.com/vorlif/testdata.multiNamesFunc" 49 | if assert.Contains(t, defs, key) { 50 | assert.Contains(t, defs[key], "a") 51 | assert.Contains(t, defs[key], "b") 52 | } 53 | 54 | key = "github.com/vorlif/testdata.noParamNames" 55 | if assert.Contains(t, defs, key) { 56 | assert.Contains(t, defs[key], "0") 57 | assert.Contains(t, defs[key], "1") 58 | } 59 | 60 | key = "github.com/vorlif/testdata.variadicFunc" 61 | if assert.Contains(t, defs, key) { 62 | if assert.Contains(t, defs[key], "a") { 63 | assert.Equal(t, 0, defs[key]["a"].FieldPos) 64 | assert.False(t, defs[key]["a"].IsVariadic) 65 | } 66 | 67 | if assert.Contains(t, defs[key], "vars") { 68 | assert.Equal(t, 1, defs[key]["vars"].FieldPos) 69 | assert.True(t, defs[key]["vars"].IsVariadic) 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /extract/extractors/varassign.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "go/types" 7 | "time" 8 | 9 | "github.com/vorlif/xspreak/extract" 10 | "github.com/vorlif/xspreak/extract/etype" 11 | "github.com/vorlif/xspreak/util" 12 | ) 13 | 14 | type varAssignExtractor struct{} 15 | 16 | func NewVariablesExtractor() extract.Extractor { 17 | return &varAssignExtractor{} 18 | } 19 | 20 | func (v varAssignExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 21 | util.TrackTime(time.Now(), "extract var assign") 22 | var issues []extract.Issue 23 | 24 | extractCtx.Inspector.Nodes([]ast.Node{&ast.AssignStmt{}}, func(rawNode ast.Node, push bool) (proceed bool) { 25 | proceed = true 26 | if !push { 27 | return 28 | } 29 | 30 | node := rawNode.(*ast.AssignStmt) 31 | if len(node.Lhs) == 0 || len(node.Rhs) == 0 { 32 | return 33 | } 34 | 35 | token, ident := extractCtx.SearchIdentAndToken(node.Lhs[0]) 36 | if token == etype.None { 37 | return 38 | } 39 | 40 | pkg, obj := extractCtx.GetType(ident) 41 | if pkg == nil { 42 | return 43 | } 44 | 45 | if etype.IsMessageID(token) { 46 | for _, res := range extractCtx.SearchStrings(node.Rhs[0]) { 47 | issue := extract.Issue{ 48 | FromExtractor: v.Name(), 49 | IDToken: token, 50 | MsgID: res.Raw, 51 | Pkg: pkg, 52 | Comments: extractCtx.GetComments(pkg, res.Node), 53 | Pos: extractCtx.GetPosition(res.Node.Pos()), 54 | } 55 | 56 | issues = append(issues, issue) 57 | } 58 | } else if token != etype.None { 59 | shouldPrint := true 60 | objType := obj.Type() 61 | if objType != nil { 62 | if pointerT, ok := objType.(*types.Pointer); ok { 63 | objType = pointerT.Elem() 64 | } 65 | 66 | if _, isNamed := objType.(*types.Named); isNamed { 67 | shouldPrint = false 68 | } 69 | } 70 | 71 | if shouldPrint { 72 | writeMissingMessageID(extractCtx.GetPosition(node.Pos()), token, "") 73 | } 74 | } 75 | 76 | return 77 | }) 78 | 79 | return issues, nil 80 | } 81 | 82 | func (v varAssignExtractor) Name() string { 83 | return "varassign_extractor" 84 | } 85 | -------------------------------------------------------------------------------- /extract/runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/sirupsen/logrus" 8 | "golang.org/x/tools/go/packages" 9 | 10 | "github.com/vorlif/xspreak/config" 11 | "github.com/vorlif/xspreak/extract" 12 | processors2 "github.com/vorlif/xspreak/extract/processors" 13 | "github.com/vorlif/xspreak/util" 14 | ) 15 | 16 | type Runner struct { 17 | Processors []processors2.Processor 18 | Log *logrus.Entry 19 | } 20 | 21 | func New(cfg *config.Config, _ []*packages.Package) (*Runner, error) { 22 | p := []processors2.Processor{ 23 | processors2.NewSkipEmptyMsgID(), 24 | } 25 | 26 | if !cfg.ExtractErrors { 27 | p = append(p, processors2.NewSkipErrors()) 28 | } 29 | 30 | p = append(p, 31 | processors2.NewCommentCleaner(cfg), 32 | processors2.NewSkipIgnore(), 33 | processors2.NewUnprintableCheck(), 34 | processors2.NewPrepareKey(), 35 | ) 36 | 37 | ret := &Runner{ 38 | Processors: p, 39 | Log: logrus.WithField("service", "Runner"), 40 | } 41 | 42 | return ret, nil 43 | } 44 | 45 | func (r Runner) Run(ctx context.Context, extractCtx *extract.Context, extractors []extract.Extractor) ([]extract.Issue, error) { 46 | r.Log.Debug("Start issue extracting") 47 | defer util.TrackTime(time.Now(), "Extracting the issues") 48 | 49 | issues := make([]extract.Issue, 0, 100) 50 | for _, extractor := range extractors { 51 | extractedIssues, err := extractor.Run(ctx, extractCtx) 52 | if err != nil { 53 | r.Log.Warnf("Can't run extractor %s: %v", extractor.Name(), err) 54 | } else { 55 | issues = append(issues, extractedIssues...) 56 | } 57 | } 58 | 59 | return r.processIssues(issues), nil 60 | } 61 | 62 | func (r *Runner) processIssues(issues []extract.Issue) []extract.Issue { 63 | defer util.TrackTime(time.Now(), "Process the issues") 64 | 65 | for _, p := range r.Processors { 66 | var newIssues []extract.Issue 67 | var err error 68 | 69 | newIssues, err = p.Process(issues) 70 | if err != nil { 71 | r.Log.Warnf("Can't process result by %s processor: %s", p.Name(), err) 72 | } else { 73 | issues = newIssues 74 | } 75 | 76 | if issues == nil { 77 | issues = []extract.Issue{} 78 | } 79 | } 80 | 81 | return issues 82 | } 83 | -------------------------------------------------------------------------------- /extract/definition.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "go/ast" 5 | "go/types" 6 | 7 | "golang.org/x/tools/go/packages" 8 | 9 | "github.com/vorlif/xspreak/extract/etype" 10 | ) 11 | 12 | type DefinitionType int 13 | 14 | const ( 15 | VarSingular DefinitionType = iota 16 | Array 17 | FunctionReturn 18 | FunctionParam 19 | StructField 20 | ) 21 | 22 | // Definition represents a data type in the program code that uses values from the spreak/localize package. 23 | // 24 | // Is used to determine that a text must be extracted when using the data type. 25 | // 26 | // Example: 27 | // 28 | // func noop(sing alias.MsgID, plural alias.Plural) {} 29 | // type M struct { Test localize.Singular} 30 | // var applicationName alias.Singular 31 | type Definition struct { 32 | Type DefinitionType 33 | Token etype.Token 34 | Pck *packages.Package 35 | Ident *ast.Ident 36 | Path string // github.com/name/repo/package/pack 37 | ID string // github.com/name/repo/package/pack.StructName 38 | Obj types.Object 39 | 40 | // -- BEGIN: Only for functions and structs -- 41 | FieldIdent *ast.Ident 42 | 43 | // FieldName is the name of the function parameter or the name of the struct field. 44 | // Example: 45 | // For the fuction noop(sing alias.MsgID, plu alias.Plural) the FieldName is "sing" or "plu". 46 | // For the struct M { Test localize.Singular} the FieldName is "Test". 47 | FieldName string 48 | IsVariadic bool 49 | // -- END: Only for functions and structs -- 50 | 51 | // FieldPos is the position of the parameter within the function definition. 52 | FieldPos int 53 | } 54 | 55 | func (d *Definition) Key() string { 56 | return d.ID 57 | } 58 | 59 | // Definitions Is a map of all definitions used in the source code. 60 | // 61 | // path.name -> field || "" -> Definition 62 | // Example: 63 | // 64 | // github.com/vorlif/testdata.noop -> sing -> Definition 65 | // github.com/vorlif/testdata.noop -> plural -> Definition 66 | type Definitions map[string]map[string]*Definition 67 | 68 | func (defs Definitions) Get(key, fieldName string) *Definition { 69 | if _, ok := defs[key]; !ok { 70 | return nil 71 | } 72 | 73 | if _, ok := defs[key][fieldName]; !ok { 74 | return nil 75 | } 76 | 77 | return defs[key][fieldName] 78 | } 79 | 80 | func (defs Definitions) GetFields(key string) map[string]*Definition { 81 | if _, ok := defs[key]; !ok { 82 | return nil 83 | } 84 | 85 | if len(defs[key]) == 0 { 86 | return nil 87 | } 88 | 89 | return defs[key] 90 | } 91 | -------------------------------------------------------------------------------- /extract/processors/comment_cleaner.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/vorlif/xspreak/config" 8 | "github.com/vorlif/xspreak/extract" 9 | "github.com/vorlif/xspreak/util" 10 | ) 11 | 12 | const flagPrefix = "xspreak:" 13 | 14 | type commentCleaner struct { 15 | allowPrefixes []string 16 | } 17 | 18 | var _ Processor = (*commentCleaner)(nil) 19 | 20 | func NewCommentCleaner(cfg *config.Config) Processor { 21 | c := &commentCleaner{ 22 | allowPrefixes: make([]string, 0, len(cfg.CommentPrefixes)), 23 | } 24 | 25 | for _, prefix := range cfg.CommentPrefixes { 26 | prefix = strings.TrimSpace(prefix) 27 | if prefix != "" { 28 | c.allowPrefixes = append(c.allowPrefixes, prefix) 29 | } 30 | } 31 | 32 | return c 33 | } 34 | 35 | func (s commentCleaner) Process(inIssues []extract.Issue) ([]extract.Issue, error) { 36 | util.TrackTime(time.Now(), "Clean comments") 37 | outIssues := make([]extract.Issue, 0, len(inIssues)) 38 | 39 | for _, iss := range inIssues { 40 | cleanedComments := make([]string, 0) 41 | 42 | // remove duplicates and extract text 43 | commentLines := make(map[string][]string) 44 | for _, com := range iss.Comments { 45 | commentLines[com] = strings.Split(com, "\n") 46 | } 47 | 48 | // filter text 49 | for _, lines := range commentLines { 50 | isTranslatorComment := false 51 | 52 | var cleanedLines []string 53 | for _, line := range lines { 54 | line = strings.TrimSpace(line) 55 | if s.hasTranslatorPrefix(line) { 56 | isTranslatorComment = true 57 | } else if strings.HasPrefix(line, flagPrefix) { 58 | iss.Flags = append(iss.Flags, util.ParseFlags(line)...) 59 | isTranslatorComment = false 60 | continue 61 | } else if len(line) == 0 { 62 | isTranslatorComment = false 63 | continue 64 | } 65 | 66 | if isTranslatorComment { 67 | cleanedLines = append(cleanedLines, line) 68 | } 69 | } 70 | 71 | if len(cleanedLines) > 0 { 72 | cleanedComments = append(cleanedComments, strings.Join(cleanedLines, " ")) 73 | } 74 | } 75 | 76 | iss.Comments = cleanedComments 77 | outIssues = append(outIssues, iss) 78 | } 79 | 80 | return outIssues, nil 81 | } 82 | 83 | func (s commentCleaner) hasTranslatorPrefix(line string) bool { 84 | for _, prefix := range s.allowPrefixes { 85 | if strings.HasPrefix(line, prefix) { 86 | return true 87 | } 88 | } 89 | 90 | return false 91 | } 92 | 93 | func (s commentCleaner) Name() string { 94 | return "comment_cleaner" 95 | } 96 | -------------------------------------------------------------------------------- /extract/context_test.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "go/parser" 5 | "testing" 6 | ) 7 | 8 | func TestExtractStringLiteral(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | code string 12 | wantStr string 13 | wantFound bool 14 | }{ 15 | { 16 | name: "String extracted", 17 | code: `"Extracted string"`, 18 | wantStr: `Extracted string`, 19 | wantFound: true, 20 | }, 21 | { 22 | name: "Even addition is merged", 23 | code: `"Extracted " + "string"`, 24 | wantStr: `Extracted string`, 25 | wantFound: true, 26 | }, 27 | { 28 | name: "Odd addition is merged", 29 | code: `"Extracted " + "string" + " is combined"`, 30 | wantStr: `Extracted string is combined`, 31 | wantFound: true, 32 | }, 33 | { 34 | name: "Backquotes are removed", 35 | code: "`Extracted string`", 36 | wantStr: "Extracted string", 37 | wantFound: true, 38 | }, 39 | { 40 | name: "Multiline text with backquotes are formatted correctly", 41 | code: "`This is an multiline\nstring`", 42 | wantStr: "This is an multiline\nstring", 43 | wantFound: true, 44 | }, 45 | { 46 | name: "Backqoutes with qoutes", 47 | code: "`This is an \"Test\" abc`", 48 | wantStr: "This is an \"Test\" abc", 49 | wantFound: true, 50 | }, 51 | { 52 | name: "Backqoutes with qoutes", 53 | code: `"This is an \"Test\" abc"`, 54 | wantStr: "This is an \"Test\" abc", 55 | wantFound: true, 56 | }, 57 | { 58 | name: "Escaping", 59 | code: `"\\caf\u00e9"`, 60 | wantStr: `\café`, 61 | wantFound: true, 62 | }, 63 | } 64 | for _, tt := range tests { 65 | t.Run(tt.name, func(t *testing.T) { 66 | expr, err := parser.ParseExpr(tt.code) 67 | if err != nil { 68 | t.Errorf("Expression %s could not be parsed: %v", tt.code, expr) 69 | } 70 | extractedStr, found := StringLiteral(expr) 71 | if extractedStr != tt.wantStr { 72 | t.Errorf("StringLiteral() string = %v, want %v", extractedStr, tt.wantStr) 73 | } 74 | if (found == nil) == tt.wantFound { 75 | t.Errorf("StringLiteral() got1 = %v, want %v", found, found == nil) 76 | } 77 | }) 78 | } 79 | 80 | t.Run("Nil is ignored", func(t *testing.T) { 81 | extractedStr, found := StringLiteral(nil) 82 | if extractedStr != "" { 83 | t.Errorf("StringLiteral() string = %v, want %v", extractedStr, "") 84 | } 85 | if found != nil { 86 | t.Errorf("StringLiteral() got1 = %v, want %v", found, false) 87 | } 88 | }) 89 | } 90 | -------------------------------------------------------------------------------- /merger/json.go: -------------------------------------------------------------------------------- 1 | package merger 2 | 3 | import ( 4 | "encoding/json" 5 | "sort" 6 | "strings" 7 | 8 | log "github.com/sirupsen/logrus" 9 | 10 | "github.com/vorlif/spreak/catalog/cldrplural" 11 | 12 | "github.com/vorlif/xspreak/encoder" 13 | ) 14 | 15 | func MergeJSON(src []byte, dst []byte, cats []cldrplural.Category) []byte { 16 | if len(src) == 0 { 17 | log.Fatal("Source file is empty") 18 | } 19 | 20 | var sourceFile encoder.JSONFile 21 | var targetFile encoder.JSONFile 22 | if err := json.Unmarshal(src, &sourceFile); err != nil { 23 | log.WithError(err).Fatal("Source file could not be decoded") 24 | } 25 | 26 | if len(dst) == 0 { 27 | targetFile = make(encoder.JSONFile, len(sourceFile)) 28 | } else { 29 | if err := json.Unmarshal(dst, &targetFile); err != nil { 30 | log.WithError(err).Fatal("Target file could not be decoded") 31 | } 32 | } 33 | 34 | newItems := make(map[string]encoder.JSONItem) 35 | 36 | isTargetCat := func(key string) bool { 37 | for _, cat := range cats { 38 | if catKey(cat) == key { 39 | return true 40 | } 41 | } 42 | return false 43 | } 44 | 45 | for _, srcItem := range sourceFile { 46 | msg := make(encoder.JSONMessage) 47 | 48 | if ctx, hasCtx := srcItem.Message["context"]; hasCtx { 49 | msg["context"] = ctx 50 | } 51 | 52 | keyCount := 0 53 | for k := range srcItem.Message { 54 | if isCategory(k) { 55 | keyCount++ 56 | } 57 | if isTargetCat(k) { 58 | msg[k] = "" 59 | } 60 | } 61 | 62 | if keyCount > 1 { 63 | for _, cat := range cats { 64 | msg[catKey(cat)] = "" 65 | } 66 | } 67 | 68 | for k, v := range srcItem.Message { 69 | if _, ok := msg[k]; ok { 70 | msg[k] = v 71 | } 72 | } 73 | 74 | newItems[srcItem.Key] = encoder.JSONItem{ 75 | Key: srcItem.Key, 76 | Message: msg, 77 | } 78 | } 79 | 80 | for _, oldItem := range targetFile { 81 | newItem, ok := newItems[oldItem.Key] 82 | if !ok { 83 | continue 84 | } 85 | 86 | for k, v := range oldItem.Message { 87 | if _, ok = newItem.Message[k]; ok && v != "" { 88 | newItem.Message[k] = v 89 | } 90 | } 91 | } 92 | 93 | file := make(encoder.JSONFile, 0, len(newItems)) 94 | for _, v := range newItems { 95 | file = append(file, v) 96 | } 97 | sort.Slice(file, func(i, j int) bool { 98 | return file[i].Key < file[j].Key 99 | }) 100 | 101 | data, err := json.MarshalIndent(file, "", " ") 102 | if err != nil { 103 | log.WithError(err).Fatal("Marshal failed") 104 | } 105 | return data 106 | } 107 | 108 | func catKey(cat cldrplural.Category) string { 109 | return strings.ToLower(cat.String()) 110 | } 111 | 112 | func isCategory(key string) bool { 113 | for cat := range cldrplural.CategoryNames { 114 | if catKey(cat) == key { 115 | return true 116 | } 117 | } 118 | return false 119 | } 120 | -------------------------------------------------------------------------------- /testdata/project/funccall.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "golang.org/x/text/language" 5 | 6 | "github.com/vorlif/testdata/sub" 7 | 8 | "github.com/vorlif/spreak" 9 | sp "github.com/vorlif/spreak" 10 | alias "github.com/vorlif/spreak/localize" 11 | ) 12 | 13 | func noop(sing alias.MsgID, plural alias.Plural, context alias.Context, domain alias.Domain) {} 14 | func noParamNames(alias.MsgID, alias.Plural) {} 15 | func variadicFunc(a alias.Singular, vars ...alias.Singular) {} 16 | func multiNamesFunc(a, b alias.MsgID) {} 17 | func ctxAndMsg(c alias.Context, msgid alias.Singular) {} 18 | 19 | func GenericFunc[V int64 | float64](log alias.Singular, i V) V { 20 | return i 21 | } 22 | 23 | type methodStruct struct{} 24 | 25 | func (methodStruct) Method(alias.Singular) {} 26 | 27 | type genericMethodStruct[T any] struct{} 28 | 29 | func (genericMethodStruct[T]) Method(alias.Singular) {} 30 | 31 | func outerFuncDef() { 32 | f := func(msgid alias.Singular, plural alias.Plural, context alias.Context, domain alias.Domain) {} 33 | 34 | variadicFunc("pre-variadic", "variadic-a", "variadic-b") 35 | noParamNames("no-param-msgid", "no-param-plural") 36 | multiNamesFunc("multi-names-a", "multi-names-b") 37 | 38 | // not extracted 39 | f("f-msgid", "f-plural", "f-context", "f-domain") 40 | 41 | // extracted 42 | noop("noop-msgid", "noop-plural", "noop-context", "noop-domain") 43 | sub.Func("submsgid", "subplural") 44 | _ = GenericFunc[int64]("generic-call", 5) 45 | } 46 | 47 | // TRANSLATORS: this is not extracted 48 | func localizerCall(loc *sp.Localizer) { 49 | // TRANSLATORS: this is extracted 50 | loc.Getf("localizer func call") 51 | 52 | initBacktrace := "init backtrace" 53 | loc.Get(initBacktrace) 54 | 55 | var assignBacktrace string 56 | assignBacktrace = "assign backtrace" 57 | loc.Get(assignBacktrace) 58 | 59 | ctxAndMsg(constCtx, "constCtxMsg") 60 | } 61 | 62 | func builtInFunctions() { 63 | bundle, err := spreak.NewBundle( 64 | spreak.WithDefaultDomain(spreak.NoDomain), 65 | spreak.WithDomainPath(spreak.NoDomain, "./"), 66 | spreak.WithLanguage(language.English), 67 | ) 68 | if err != nil { 69 | panic(err) 70 | } 71 | 72 | inlineFunc := func(inlineParam alias.Singular) {} 73 | 74 | inlineFunc("inline function") 75 | 76 | t := spreak.NewLocalizer(bundle, "en") 77 | // TRANSLATORS: Test 78 | // multiline 79 | t.Getf("msgid") 80 | t.NGetf("msgid-n", "pluralid-n", 10, 10) 81 | t.DGetf("domain-d", "msgid-d") 82 | t.DNGetf("domain-dn", "msgid-dn", "pluralid-dn", 10) 83 | t.PGetf("context-pg", "msgid-pg") 84 | t.NPGetf("context-np", "msgid-np", "pluralid-np", 10) 85 | t.DPGetf("domain-dp", "context-dp", "singular-dp") 86 | t.DNPGetf("domain-dnp", "context-dnp", "msgid-dnp", "pluralid-dnp", 10) 87 | } 88 | 89 | func methodCall() { 90 | (methodStruct{}).Method("struct-method-call") 91 | (genericMethodStruct[string]{}).Method("generic-struct-method-call") 92 | } 93 | -------------------------------------------------------------------------------- /extract/extractors/mapsdef.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "time" 7 | 8 | "github.com/vorlif/xspreak/extract" 9 | "github.com/vorlif/xspreak/extract/etype" 10 | "github.com/vorlif/xspreak/util" 11 | ) 12 | 13 | type mapsDefExtractor struct{} 14 | 15 | func NewMapsDefExtractor() extract.Extractor { 16 | return &mapsDefExtractor{} 17 | } 18 | 19 | func (v mapsDefExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 20 | util.TrackTime(time.Now(), "extract maps") 21 | var issues []extract.Issue 22 | 23 | extractCtx.Inspector.Nodes([]ast.Node{&ast.CompositeLit{}}, func(rawNode ast.Node, push bool) (proceed bool) { 24 | proceed = true 25 | if !push { 26 | return 27 | } 28 | 29 | node := rawNode.(*ast.CompositeLit) 30 | if len(node.Elts) == 0 { 31 | return 32 | } 33 | 34 | mapT, isMap := node.Type.(*ast.MapType) 35 | if !isMap { 36 | return 37 | } 38 | 39 | for _, expr := range []ast.Expr{mapT.Key, mapT.Value} { 40 | 41 | ident := extractCtx.SearchIdent(expr) 42 | if ident == nil { 43 | continue 44 | } 45 | 46 | pkg, obj := extractCtx.GetType(ident) 47 | if pkg == nil { 48 | continue 49 | } 50 | 51 | token := extractCtx.GetLocalizeTypeToken(ident) 52 | 53 | // Map of strings 54 | if etype.IsMessageID(token) { 55 | for _, elt := range node.Elts { 56 | kvExpr, isKv := elt.(*ast.KeyValueExpr) 57 | if !isKv { 58 | continue 59 | } 60 | var target ast.Expr 61 | if expr == mapT.Key { 62 | target = kvExpr.Key 63 | } else { 64 | target = kvExpr.Value 65 | } 66 | 67 | for _, res := range extractCtx.SearchStrings(target) { 68 | issue := extract.Issue{ 69 | FromExtractor: v.Name(), 70 | IDToken: token, 71 | MsgID: res.Raw, 72 | Pkg: pkg, 73 | Comments: extractCtx.GetComments(pkg, res.Node), 74 | Pos: extractCtx.GetPosition(res.Node.Pos()), 75 | } 76 | 77 | issues = append(issues, issue) 78 | } 79 | } 80 | 81 | continue 82 | } else if token != etype.None { 83 | writeMissingMessageID(extractCtx.GetPosition(ident.Pos()), token, "") 84 | } 85 | 86 | // Array of structs 87 | structAttr := extractCtx.Definitions.GetFields(util.ObjToKey(obj)) 88 | if structAttr == nil { 89 | return 90 | } 91 | 92 | for _, elt := range node.Elts { 93 | kvExpr, isKv := elt.(*ast.KeyValueExpr) 94 | if !isKv { 95 | continue 96 | } 97 | var target ast.Expr 98 | if expr == mapT.Key { 99 | target = kvExpr.Key 100 | } else { 101 | target = kvExpr.Value 102 | } 103 | 104 | compLit, isCompLit := target.(*ast.CompositeLit) 105 | if !isCompLit { 106 | continue 107 | } 108 | 109 | structIssues := extractStruct(extractCtx, compLit, obj, pkg) 110 | issues = append(issues, structIssues...) 111 | } 112 | } 113 | 114 | return 115 | }) 116 | 117 | return issues, nil 118 | } 119 | 120 | func (v mapsDefExtractor) Name() string { 121 | return "mapsdef_extractor" 122 | } 123 | -------------------------------------------------------------------------------- /commands/merge.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | "golang.org/x/text/language" 9 | 10 | "github.com/vorlif/spreak/catalog/cldrplural" 11 | 12 | "github.com/vorlif/xspreak/merger" 13 | ) 14 | 15 | var mergeCmd = &cobra.Command{ 16 | Use: "merge", 17 | Short: "Create or update JSON translation files", 18 | Long: `Merge creates new translation files or merges existing files. 19 | Already existing translations in the target file are preserved. 20 | Available translations from the source file will be taken over. 21 | The language of the target file must be specified to create the correct templates for the plural forms.`, 22 | Run: mergeCmdF, 23 | Example: ` xspreak merge -i locale/httptempl.json -o locale/de.json -l de`, 24 | } 25 | 26 | func init() { 27 | fs := mergeCmd.Flags() 28 | fs.SortFlags = false 29 | fs.StringP("input", "i", "", "source file") 30 | fs.StringP("output", "o", "", "output file") 31 | fs.StringP("lang", "l", "", "destination language") 32 | 33 | rootCmd.AddCommand(mergeCmd) 34 | } 35 | 36 | func mergeCmdF(cmd *cobra.Command, _ []string) { 37 | targetLang, errL := cmd.Flags().GetString("lang") 38 | if errL != nil { 39 | log.WithError(errL).Fatal("Invalid source file") 40 | } else if targetLang == "" { 41 | log.Fatal("Target language must be specified") 42 | } 43 | 44 | lang, errP := language.Parse(targetLang) 45 | if errP != nil { 46 | log.WithError(errP).Fatal("Language could not be parsed") 47 | } 48 | ruleSet, found := cldrplural.ForLanguage(lang) 49 | if !found { 50 | log.Fatal("No rules for language found") 51 | } 52 | 53 | srcPath, errS := cmd.Flags().GetString("input") 54 | if errS != nil { 55 | log.WithError(errS).Fatal("Invalid source file") 56 | } else if srcPath == "" { 57 | log.Fatal("Source required") 58 | } 59 | 60 | dstPath, errD := cmd.Flags().GetString("output") 61 | if errD != nil { 62 | log.WithError(errD).Fatal("Invalid destination file") 63 | } else if dstPath == "" { 64 | log.Fatal("Destination required") 65 | } 66 | 67 | var sourceContent []byte 68 | if fi, err := os.Stat(srcPath); err != nil { 69 | log.WithError(err).Fatal("Source file could not be verified") 70 | } else if fi.IsDir() { 71 | log.Fatal("Source file must be a file, but is a folder") 72 | } else { 73 | sourceContent, err = os.ReadFile(srcPath) 74 | if err != nil { 75 | log.WithError(err).Fatal("Source file could not be read") 76 | } 77 | } 78 | 79 | var destinationContent []byte 80 | if fi, err := os.Stat(dstPath); err != nil { 81 | if !os.IsNotExist(err) { 82 | log.WithError(err).Fatal("Destination file could not be verified") 83 | } 84 | } else if fi.IsDir() { 85 | log.Fatal("Destination file must be a file, but is a folder") 86 | } else { 87 | destinationContent, err = os.ReadFile(dstPath) 88 | if err != nil { 89 | log.WithError(err).Fatal("Destination file could not be read") 90 | } 91 | } 92 | 93 | newContent := merger.MergeJSON(sourceContent, destinationContent, ruleSet.Categories) 94 | if err := os.WriteFile(dstPath, newContent, 0666); err != nil { 95 | log.WithError(err).Fatal("Target file could not be written") 96 | } 97 | log.Printf("Target file written %s\n", dstPath) 98 | } 99 | -------------------------------------------------------------------------------- /extract/extractors/inline.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "go/token" 7 | "strconv" 8 | "time" 9 | 10 | log "github.com/sirupsen/logrus" 11 | "golang.org/x/tools/go/packages" 12 | 13 | "github.com/vorlif/xspreak/extract" 14 | "github.com/vorlif/xspreak/tmpl" 15 | "github.com/vorlif/xspreak/util" 16 | ) 17 | 18 | type inlineTemplateExtractor struct{} 19 | 20 | func NewInlineTemplateExtractor() extract.Extractor { 21 | return &inlineTemplateExtractor{} 22 | } 23 | 24 | func (i *inlineTemplateExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 25 | util.TrackTime(time.Now(), "extract inline templates") 26 | 27 | if len(extractCtx.Config.Keywords) == 0 { 28 | log.Debug("Skip inline template extraction, no keywords present") 29 | return []extract.Issue{}, nil 30 | } 31 | 32 | extractCtx.Inspector.WithStack([]ast.Node{&ast.BasicLit{}}, func(rawNode ast.Node, push bool, stack []ast.Node) (proceed bool) { 33 | proceed = true 34 | if !push { 35 | return 36 | } 37 | 38 | node := rawNode.(*ast.BasicLit) 39 | if node.Kind != token.STRING { 40 | return 41 | } 42 | // Search for ident to get the package 43 | var pkg *packages.Package 44 | for i := len(stack) - 1; i >= 0; i-- { 45 | if stack[i] == nil { 46 | break 47 | } 48 | if ident := extractIdent(stack[i]); ident != nil { 49 | pkg, _ = extractCtx.GetType(ident) 50 | if pkg != nil { 51 | break 52 | } 53 | } 54 | } 55 | 56 | if pkg == nil { 57 | return 58 | } 59 | 60 | comments := extractCtx.GetComments(pkg, node) 61 | if comments == nil { 62 | return 63 | } 64 | pos := extractCtx.GetPosition(node.Pos()) 65 | templateString, err := strconv.Unquote(node.Value) 66 | if err != nil { 67 | return 68 | } 69 | 70 | for _, comment := range comments { 71 | if !util.IsInlineTemplate(comment) { 72 | continue 73 | } 74 | 75 | template, errP := tmpl.ParseString(pos.Filename, templateString) 76 | if errP != nil { 77 | log.WithError(errP).WithField("pos", pos).Warn("Template could not be parsed") 78 | break 79 | } 80 | template.GoFilePos = pos 81 | extractCtx.Templates = append(extractCtx.Templates, template) 82 | } 83 | 84 | return 85 | }) 86 | 87 | return []extract.Issue{}, nil 88 | } 89 | 90 | func (i *inlineTemplateExtractor) Name() string { 91 | return "inline_template_extractor" 92 | } 93 | 94 | func extractIdent(node ast.Node) *ast.Ident { 95 | switch v := node.(type) { 96 | case *ast.Ident: 97 | return v 98 | case *ast.ValueSpec: 99 | if len(v.Names) > 0 { 100 | return v.Names[0] 101 | } 102 | case *ast.SelectorExpr: 103 | return v.Sel 104 | case *ast.CallExpr: 105 | if ident, ok := v.Fun.(*ast.Ident); ok { 106 | return ident 107 | } 108 | case *ast.StarExpr: 109 | switch pointerExpr := v.X.(type) { 110 | case *ast.SelectorExpr: 111 | return pointerExpr.Sel 112 | } 113 | case *ast.KeyValueExpr: 114 | if ident, ok := v.Key.(*ast.Ident); ok { 115 | return ident 116 | } 117 | if ident, ok := v.Value.(*ast.Ident); ok { 118 | return ident 119 | } 120 | } 121 | 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /extract/extractors/slicedef.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "go/types" 7 | "time" 8 | 9 | "golang.org/x/tools/go/packages" 10 | 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/extract/etype" 13 | "github.com/vorlif/xspreak/util" 14 | ) 15 | 16 | type sliceDefExtractor struct{} 17 | 18 | func NewSliceDefExtractor() extract.Extractor { 19 | return &sliceDefExtractor{} 20 | } 21 | 22 | func (v sliceDefExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 23 | util.TrackTime(time.Now(), "extract slices") 24 | var issues []extract.Issue 25 | 26 | extractCtx.Inspector.Nodes([]ast.Node{&ast.CompositeLit{}}, func(rawNode ast.Node, push bool) (proceed bool) { 27 | proceed = true 28 | if !push { 29 | return 30 | } 31 | 32 | node := rawNode.(*ast.CompositeLit) 33 | if len(node.Elts) == 0 { 34 | return 35 | } 36 | 37 | arrayTye, ok := node.Type.(*ast.ArrayType) 38 | if !ok { 39 | return 40 | } 41 | 42 | var obj types.Object 43 | var pkg *packages.Package 44 | var token etype.Token 45 | switch val := arrayTye.Elt.(type) { 46 | case *ast.SelectorExpr: 47 | pkg, obj = extractCtx.GetType(val.Sel) 48 | if pkg == nil { 49 | return 50 | } 51 | token = extractCtx.GetLocalizeTypeToken(val.Sel) 52 | case *ast.Ident: 53 | pkg, obj = extractCtx.GetType(val) 54 | if pkg == nil { 55 | return 56 | } 57 | token = extractCtx.GetLocalizeTypeToken(val) 58 | case *ast.StarExpr: 59 | switch pointerExpr := val.X.(type) { 60 | case *ast.SelectorExpr: 61 | pkg, obj = extractCtx.GetType(pointerExpr.Sel) 62 | if pkg == nil { 63 | return 64 | } 65 | token = extractCtx.GetLocalizeTypeToken(pointerExpr.Sel) 66 | case *ast.Ident: 67 | pkg, obj = extractCtx.GetType(pointerExpr) 68 | if pkg == nil { 69 | return 70 | } 71 | token = extractCtx.GetLocalizeTypeToken(pointerExpr) 72 | 73 | default: 74 | return 75 | } 76 | default: 77 | return 78 | } 79 | 80 | // Array of strings 81 | if etype.IsMessageID(token) { 82 | for _, elt := range node.Elts { 83 | for _, res := range extractCtx.SearchStrings(elt) { 84 | issue := extract.Issue{ 85 | FromExtractor: v.Name(), 86 | IDToken: token, 87 | MsgID: res.Raw, 88 | Pkg: pkg, 89 | Comments: extractCtx.GetComments(pkg, res.Node), 90 | Pos: extractCtx.GetPosition(res.Node.Pos()), 91 | } 92 | 93 | issues = append(issues, issue) 94 | } 95 | } 96 | 97 | return 98 | } else if token != etype.None { 99 | writeMissingMessageID(extractCtx.GetPosition(node.Pos()), token, "") 100 | } 101 | 102 | structAttr := extractCtx.Definitions.GetFields(util.ObjToKey(obj)) 103 | if structAttr == nil { 104 | return 105 | } 106 | 107 | for _, elt := range node.Elts { 108 | compLit, isCompLit := elt.(*ast.CompositeLit) 109 | if !isCompLit { 110 | continue 111 | } 112 | 113 | structIssues := extractStruct(extractCtx, compLit, obj, pkg) 114 | issues = append(issues, structIssues...) 115 | } 116 | 117 | return 118 | }) 119 | 120 | return issues, nil 121 | } 122 | 123 | func (v sliceDefExtractor) Name() string { 124 | return "slicedef_extractor" 125 | } 126 | -------------------------------------------------------------------------------- /encoder/pot.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/sirupsen/logrus" 11 | 12 | "github.com/vorlif/spreak/catalog/po" 13 | 14 | "github.com/vorlif/xspreak/config" 15 | "github.com/vorlif/xspreak/extract" 16 | "github.com/vorlif/xspreak/util" 17 | ) 18 | 19 | type potEncoder struct { 20 | cfg *config.Config 21 | w *po.Encoder 22 | } 23 | 24 | func NewPotEncoder(cfg *config.Config, w io.Writer) Encoder { 25 | enc := po.NewEncoder(w) 26 | enc.SetWrapWidth(cfg.WrapWidth) 27 | enc.SetWriteHeader(!cfg.OmitHeader) 28 | enc.SetWriteReferences(!cfg.WriteNoLocation) 29 | 30 | return &potEncoder{cfg: cfg, w: enc} 31 | } 32 | 33 | func (e *potEncoder) Encode(issues []extract.Issue) error { 34 | file := &po.File{ 35 | Header: e.buildHeader(), 36 | Messages: make(map[string]map[string]*po.Message), 37 | } 38 | 39 | for _, msg := range e.buildMessages(issues) { 40 | file.AddMessage(msg) 41 | } 42 | 43 | return e.w.Encode(file) 44 | } 45 | 46 | func (e *potEncoder) buildMessages(issues []extract.Issue) []*po.Message { 47 | util.TrackTime(time.Now(), "Build messages") 48 | messages := make([]*po.Message, 0, len(issues)) 49 | 50 | absOut, errA := filepath.Abs(e.cfg.OutputDir) 51 | if errA != nil { 52 | absOut = e.cfg.OutputDir 53 | } 54 | 55 | for _, iss := range issues { 56 | path, errP := filepath.Rel(absOut, iss.Pos.Filename) 57 | if errP != nil { 58 | logrus.WithError(errP).Warn("Relative path could not be created, use absolute") 59 | path = iss.Pos.Filename 60 | } 61 | 62 | ref := &po.Reference{ 63 | Path: filepath.ToSlash(path), 64 | Line: iss.Pos.Line, 65 | Column: iss.Pos.Column, 66 | } 67 | 68 | if reGoStringFormat.MatchString(iss.MsgID) || reGoStringFormat.MatchString(iss.PluralID) { 69 | iss.Flags = append(iss.Flags, "go-format") 70 | } 71 | 72 | msg := &po.Message{ 73 | Comment: &po.Comment{ 74 | Extracted: strings.Join(iss.Comments, "\n"), 75 | References: []*po.Reference{ref}, 76 | Flags: iss.Flags, 77 | }, 78 | Context: iss.Context, 79 | ID: iss.MsgID, 80 | IDPlural: iss.PluralID, 81 | } 82 | 83 | messages = append(messages, msg) 84 | } 85 | 86 | return messages 87 | } 88 | 89 | func (e *potEncoder) buildHeader() *po.Header { 90 | headerComment := fmt.Sprintf(`SOME DESCRIPTIVE TITLE. 91 | Copyright (C) YEAR %s 92 | This file is distributed under the same license as the %s package. 93 | FIRST AUTHOR , YEAR. 94 | `, e.cfg.CopyrightHolder, e.cfg.PackageName) 95 | return &po.Header{ 96 | Comment: &po.Comment{ 97 | Translator: headerComment, 98 | Extracted: "", 99 | References: nil, 100 | Flags: []string{"fuzzy"}, 101 | PrevMsgContext: "", 102 | PrevMsgID: "", 103 | }, 104 | ProjectIDVersion: e.cfg.PackageName, 105 | ReportMsgidBugsTo: e.cfg.BugsAddress, 106 | POTCreationDate: time.Now().Format("2006-01-02 15:04-0700"), 107 | PORevisionDate: "YEAR-MO-DA HO:MI+ZONE", 108 | LastTranslator: "FULL NAME ", 109 | LanguageTeam: "LANGUAGE ", 110 | Language: "", 111 | MimeVersion: "1.0", 112 | ContentType: "text/plain; charset=UTF-8", 113 | ContentTransferEncoding: "8bit", 114 | PluralForms: "", // alternative "nplurals=INTEGER; plural=EXPRESSION;" 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "github.com/vorlif/xspreak/tmpl" 12 | ) 13 | 14 | const ( 15 | ExtractFormatPot = "pot" 16 | ExtractFormatJSON = "json" 17 | ) 18 | 19 | type Config struct { 20 | IsVerbose bool 21 | CurrentDir string 22 | SourceDir string 23 | // OutputDir is the directory where the output file will be written. 24 | OutputDir string 25 | OutputFile string 26 | // CommentPrefixes are the prefixes that will be used to identify translator comments. 27 | // For example, "TRANSLATORS:". 28 | CommentPrefixes []string 29 | ExtractErrors bool 30 | ErrorContext string 31 | 32 | TemplatePatterns []string 33 | Keywords []*tmpl.Keyword 34 | 35 | DefaultDomain string 36 | WriteNoLocation bool 37 | WrapWidth int 38 | DontWrap bool 39 | 40 | OmitHeader bool 41 | CopyrightHolder string 42 | PackageName string 43 | BugsAddress string 44 | 45 | Args []string 46 | 47 | LoadedPackages []string 48 | 49 | Timeout time.Duration 50 | 51 | // ExtractFormat is the format of the output file. 52 | // Possible values: "po", "pot", "json" 53 | ExtractFormat string 54 | TmplIsMonolingual bool 55 | } 56 | 57 | func NewDefault() *Config { 58 | return &Config{ 59 | IsVerbose: false, 60 | SourceDir: "", 61 | OutputDir: filepath.Clean("./"), 62 | OutputFile: "", 63 | CommentPrefixes: []string{"TRANSLATORS"}, 64 | ExtractErrors: false, 65 | ErrorContext: "errors", 66 | 67 | DefaultDomain: "messages", 68 | WriteNoLocation: false, 69 | WrapWidth: 80, 70 | DontWrap: false, 71 | 72 | OmitHeader: false, 73 | CopyrightHolder: "THE PACKAGE'S COPYRIGHT HOLDER", 74 | PackageName: "PACKAGE VERSION", 75 | BugsAddress: "", 76 | 77 | Timeout: 15 * time.Minute, 78 | 79 | ExtractFormat: ExtractFormatPot, 80 | } 81 | } 82 | 83 | func (c *Config) Prepare() error { 84 | c.ErrorContext = strings.TrimSpace(c.ErrorContext) 85 | c.DefaultDomain = strings.TrimSpace(c.DefaultDomain) 86 | if c.DefaultDomain == "" { 87 | return errors.New("a default domain is required") 88 | } 89 | 90 | if c.Timeout < 1*time.Minute { 91 | return errors.New("the value for Timeout must be at least one minute") 92 | } 93 | 94 | currentDir, errC := os.Getwd() 95 | if errC != nil { 96 | return errC 97 | } 98 | c.CurrentDir = currentDir 99 | 100 | if c.SourceDir == "" { 101 | c.SourceDir = c.CurrentDir 102 | } 103 | 104 | var err error 105 | c.SourceDir, err = filepath.Abs(c.SourceDir) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | if c.OutputFile != "" { 111 | c.OutputDir = filepath.Dir(c.OutputFile) 112 | c.OutputFile = filepath.Base(c.OutputFile) 113 | } else { 114 | c.OutputFile = fmt.Sprintf("%s.%s", c.DefaultDomain, c.ExtractFormat) 115 | } 116 | 117 | c.OutputDir, err = filepath.Abs(c.OutputDir) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | if c.DontWrap { 123 | c.WrapWidth = -1 124 | } 125 | 126 | switch c.ExtractFormat { 127 | case "po": 128 | c.ExtractFormat = ExtractFormatPot 129 | case ExtractFormatJSON, ExtractFormatPot: 130 | break 131 | default: 132 | return fmt.Errorf("only the JSON and pot format is supported, you want %v", c.ExtractFormat) 133 | } 134 | 135 | if len(c.TemplatePatterns) > 0 && len(c.Keywords) == 0 { 136 | c.Keywords = tmpl.DefaultKeywords("T", c.TmplIsMonolingual) 137 | } 138 | 139 | return nil 140 | } 141 | -------------------------------------------------------------------------------- /extract/extractors/funcreturn.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "time" 7 | 8 | "github.com/vorlif/xspreak/extract" 9 | "github.com/vorlif/xspreak/extract/etype" 10 | "github.com/vorlif/xspreak/util" 11 | ) 12 | 13 | type funcReturnExtractor struct{} 14 | 15 | func NewFuncReturnExtractor() extract.Extractor { 16 | return &funcReturnExtractor{} 17 | } 18 | 19 | func (e *funcReturnExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 20 | util.TrackTime(time.Now(), "extract func return values") 21 | var issues []extract.Issue 22 | 23 | extractCtx.Inspector.WithStack([]ast.Node{&ast.FuncDecl{}}, func(rawNode ast.Node, push bool, _ []ast.Node) (proceed bool) { 24 | proceed = true 25 | if !push { 26 | return 27 | } 28 | 29 | node := rawNode.(*ast.FuncDecl) 30 | if node.Body == nil || node.Type == nil || node.Type.Results == nil || len(node.Type.Results.List) == 0 { 31 | return 32 | } 33 | 34 | // Extract the return types if from the localise package 35 | extractedResults := make([]etype.Token, len(node.Type.Results.List)) 36 | var foundType bool 37 | for i, res := range node.Type.Results.List { 38 | tok, _ := extractCtx.SearchIdentAndToken(res) 39 | if tok == etype.None { 40 | extractedResults[i] = etype.None 41 | continue 42 | } 43 | 44 | extractedResults[i] = tok 45 | foundType = true 46 | } 47 | 48 | if !foundType { 49 | return 50 | } 51 | 52 | pkg, _ := extractCtx.GetType(node.Name) 53 | if pkg == nil { 54 | return 55 | } 56 | 57 | // Extract the values from the return statements 58 | ast.Inspect(node.Body, func(node ast.Node) bool { 59 | if node == nil { 60 | return true 61 | } 62 | 63 | retNode, isReturn := node.(*ast.ReturnStmt) 64 | if !isReturn || len(retNode.Results) != len(extractedResults) { 65 | return true 66 | } 67 | 68 | collector := newSearchCollector() 69 | collector.ExtraNodes = append(collector.ExtraNodes, node) 70 | 71 | for i, extractedResult := range extractedResults { 72 | foundResults := extractCtx.SearchStrings(retNode.Results[i]) 73 | if len(foundResults) == 0 { 74 | continue 75 | } 76 | 77 | switch extractedResult { 78 | case etype.Singular, etype.Key, etype.PluralKey: 79 | collector.AddSingulars(extractedResult, foundResults) 80 | case etype.Plural: 81 | collector.Plurals = append(collector.Plurals, foundResults...) 82 | case etype.Context: 83 | collector.Contexts = append(collector.Contexts, foundResults...) 84 | case etype.Domain: 85 | collector.Domains = append(collector.Domains, foundResults...) 86 | } 87 | } 88 | 89 | collector.CheckMissingMessageID(extractCtx) 90 | for i, singularResult := range collector.Singulars { 91 | issue := extract.Issue{ 92 | FromExtractor: e.Name(), 93 | IDToken: collector.SingularType[i], 94 | MsgID: singularResult.Raw, 95 | Domain: collector.GetDomain(), 96 | Context: collector.GetContext(), 97 | PluralID: collector.GetPlural(), 98 | Comments: extractCtx.GetComments(pkg, singularResult.Node), 99 | Pkg: pkg, 100 | Pos: extractCtx.GetPosition(singularResult.Node.Pos()), 101 | } 102 | issues = append(issues, issue) 103 | } 104 | 105 | return true 106 | }) 107 | 108 | return 109 | }) 110 | 111 | return issues, nil 112 | } 113 | 114 | func (e *funcReturnExtractor) Name() string { 115 | return "func_return" 116 | } 117 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 6 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 7 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 8 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 9 | github.com/mattn/go-zglob v0.0.6 h1:mP8RnmCgho4oaUYDIDn6GNxYk+qJGUs8fJLn+twYj2A= 10 | github.com/mattn/go-zglob v0.0.6/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= 11 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 12 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 13 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 14 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 15 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 16 | github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= 17 | github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= 18 | github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 19 | github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= 20 | github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 21 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 22 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 24 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 25 | github.com/vorlif/spreak v1.0.0 h1:SUaD/p+cWcGgLAdHi73cTmBBejpPpztlgM5I4kPSewY= 26 | github.com/vorlif/spreak v1.0.0/go.mod h1:oJ0AuinQV2XPy8WkdkbGejGDHQ3dCoB9brQMj5dsEyc= 27 | golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= 28 | golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= 29 | golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= 30 | golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= 31 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 32 | golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= 33 | golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 34 | golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= 35 | golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= 36 | golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= 37 | golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= 38 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 39 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 40 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 41 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 42 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 43 | -------------------------------------------------------------------------------- /encoder/json_test.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/extract/etype" 13 | ) 14 | 15 | func TestJSONEncoder(t *testing.T) { 16 | var buf bytes.Buffer 17 | enc := NewJSONEncoder(&buf, "") 18 | 19 | t.Run("key contains context", func(t *testing.T) { 20 | buf.Reset() 21 | 22 | iss := extract.Issue{Context: "ctx", MsgID: "id"} 23 | err := enc.Encode([]extract.Issue{iss}) 24 | require.NoError(t, err) 25 | 26 | want := `{"id_ctx":{"context":"ctx","other":"id"}} 27 | ` 28 | assert.Equal(t, want, buf.String()) 29 | }) 30 | 31 | t.Run("by key type text is removed", func(t *testing.T) { 32 | buf.Reset() 33 | 34 | err := enc.Encode([]extract.Issue{ 35 | {MsgID: "s.id", IDToken: etype.Key}, 36 | {MsgID: "p.id", PluralID: "p.id", IDToken: etype.PluralKey}, 37 | {Context: "ctx", MsgID: "sc.id", IDToken: etype.Key}, 38 | {Context: "ctx", MsgID: "pc.id", PluralID: "pc.id", IDToken: etype.PluralKey}, 39 | }) 40 | require.NoError(t, err) 41 | want := `{"p.id":{"one":"","other":""},"pc.id_ctx":{"context":"ctx","one":"","other":""},"s.id":"","sc.id_ctx":{"context":"ctx","other":""}} 42 | ` 43 | assert.JSONEq(t, want, buf.String()) 44 | }) 45 | } 46 | 47 | func TestJSONMessage_MarshalJSON(t *testing.T) { 48 | t.Run("empty returns empty string", func(t *testing.T) { 49 | msg := make(JSONMessage) 50 | data, err := json.Marshal(msg) 51 | require.NoError(t, err) 52 | assert.Equal(t, `""`, string(data)) 53 | }) 54 | 55 | t.Run("single entry returns string", func(t *testing.T) { 56 | msg := make(JSONMessage) 57 | msg["other"] = "a" 58 | data, err := json.Marshal(msg) 59 | require.NoError(t, err) 60 | assert.Equal(t, `"a"`, string(data)) 61 | }) 62 | 63 | t.Run("keeps order", func(t *testing.T) { 64 | msg := make(JSONMessage) 65 | msg["other"] = "a" 66 | msg["many"] = "b" 67 | msg["one"] = "c" 68 | msg["context"] = "d" 69 | 70 | data, err := json.Marshal(msg) 71 | require.NoError(t, err) 72 | want := `{"context":"d","one":"c","many":"b","other":"a"}` 73 | assert.Equal(t, want, string(data)) 74 | }) 75 | } 76 | 77 | func TestJSONMessage_UnmarshalJSON(t *testing.T) { 78 | t.Run("string as other", func(t *testing.T) { 79 | data := []byte(`"a"`) 80 | var msg JSONMessage 81 | 82 | require.NoError(t, json.Unmarshal(data, &msg)) 83 | assert.Len(t, msg, 1) 84 | if assert.Contains(t, msg, "other") { 85 | assert.Equal(t, "a", msg["other"]) 86 | } 87 | }) 88 | 89 | t.Run("object as map", func(t *testing.T) { 90 | data := []byte(`{"context":"ctx", "one":"b", "other":"a"}`) 91 | var msg JSONMessage 92 | 93 | require.NoError(t, json.Unmarshal(data, &msg)) 94 | assert.Len(t, msg, 3) 95 | if assert.Contains(t, msg, "context") { 96 | assert.Equal(t, "ctx", msg["context"]) 97 | } 98 | if assert.Contains(t, msg, "one") { 99 | assert.Equal(t, "b", msg["one"]) 100 | } 101 | if assert.Contains(t, msg, "other") { 102 | assert.Equal(t, "a", msg["other"]) 103 | } 104 | }) 105 | 106 | t.Run("empty object", func(t *testing.T) { 107 | data := []byte(`{}`) 108 | var msg JSONMessage 109 | 110 | require.NoError(t, json.Unmarshal(data, &msg)) 111 | if assert.Contains(t, msg, "other") { 112 | assert.Equal(t, "", msg["other"]) 113 | } 114 | }) 115 | 116 | t.Run("empty string", func(t *testing.T) { 117 | data := []byte(`""`) 118 | var msg JSONMessage 119 | 120 | require.NoError(t, json.Unmarshal(data, &msg)) 121 | if assert.Contains(t, msg, "other") { 122 | assert.Equal(t, "", msg["other"]) 123 | } 124 | }) 125 | } 126 | -------------------------------------------------------------------------------- /extract/extractors/utils.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "go/ast" 5 | "go/token" 6 | "os" 7 | "path/filepath" 8 | 9 | log "github.com/sirupsen/logrus" 10 | 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/extract/etype" 13 | ) 14 | 15 | var workingDir, _ = os.Getwd() 16 | 17 | type searchCollector struct { 18 | Singulars []*extract.SearchResult 19 | SingularType []etype.Token 20 | Plurals []*extract.SearchResult 21 | Contexts []*extract.SearchResult 22 | Domains []*extract.SearchResult 23 | ExtraNodes []ast.Node 24 | } 25 | 26 | func writeMissingMessageID(position token.Position, token etype.Token, text string) { 27 | var typeName string 28 | switch token { 29 | case etype.Plural: 30 | typeName = "plural" 31 | case etype.Context: 32 | typeName = "context" 33 | case etype.Domain: 34 | typeName = "domain" 35 | default: 36 | typeName = "unknown" 37 | } 38 | 39 | filename := position.Filename 40 | if relPath, err := filepath.Rel(workingDir, filename); err == nil { 41 | filename = relPath 42 | } 43 | 44 | if text != "" { 45 | log.Warnf("%s:%d usage of %s without MessageID is not supported", filename, position.Line, typeName) 46 | } else { 47 | log.Warnf("%s:%d usage of %s without MessageID is not supported: %q", filename, position.Line, typeName, text) 48 | } 49 | } 50 | 51 | func newSearchCollector() *searchCollector { 52 | return &searchCollector{ 53 | Singulars: make([]*extract.SearchResult, 0), 54 | Plurals: make([]*extract.SearchResult, 0), 55 | Contexts: make([]*extract.SearchResult, 0), 56 | Domains: make([]*extract.SearchResult, 0), 57 | ExtraNodes: make([]ast.Node, 0), 58 | } 59 | } 60 | 61 | func (sc *searchCollector) AddSingulars(token etype.Token, singulars []*extract.SearchResult) { 62 | sc.Singulars = append(sc.Singulars, singulars...) 63 | for i := 0; i < len(singulars); i++ { 64 | sc.SingularType = append(sc.SingularType, token) 65 | } 66 | } 67 | 68 | func (sc *searchCollector) GetPlural() string { 69 | if len(sc.Plurals) > 0 { 70 | return sc.Plurals[0].Raw 71 | } 72 | return "" 73 | } 74 | 75 | func (sc *searchCollector) GetContext() string { 76 | if len(sc.Contexts) > 0 { 77 | for _, c := range sc.Contexts { 78 | if c != nil && c.Raw != "" { 79 | return c.Raw 80 | } 81 | } 82 | } 83 | return "" 84 | } 85 | 86 | func (sc *searchCollector) GetDomain() string { 87 | if len(sc.Domains) > 0 { 88 | return sc.Domains[0].Raw 89 | } 90 | return "" 91 | } 92 | 93 | func (sc *searchCollector) GetNodes() []ast.Node { 94 | nodes := make([]ast.Node, 0, len(sc.Singulars)+3+len(sc.ExtraNodes)) 95 | nodes = append(nodes, sc.ExtraNodes...) 96 | 97 | for _, sing := range sc.Singulars { 98 | nodes = append(nodes, sing.Node) 99 | } 100 | if len(sc.Plurals) > 0 { 101 | nodes = append(nodes, sc.Plurals[0].Node) 102 | } 103 | if len(sc.Contexts) > 0 { 104 | nodes = append(nodes, sc.Contexts[0].Node) 105 | } 106 | if len(sc.Domains) > 0 { 107 | nodes = append(nodes, sc.Domains[0].Node) 108 | } 109 | 110 | return nodes 111 | } 112 | 113 | func (sc *searchCollector) CheckMissingMessageID(extractCtx *extract.Context) { 114 | for _, sing := range sc.Singulars { 115 | if sing.Raw != "" { 116 | return 117 | } 118 | } 119 | 120 | for _, plural := range sc.Plurals { 121 | writeMissingMessageID(extractCtx.GetPosition(plural.Node.Pos()), etype.Plural, plural.Raw) 122 | } 123 | 124 | for _, ctx := range sc.Contexts { 125 | writeMissingMessageID(extractCtx.GetPosition(ctx.Node.Pos()), etype.Plural, ctx.Raw) 126 | } 127 | 128 | for _, domain := range sc.Domains { 129 | writeMissingMessageID(extractCtx.GetPosition(domain.Node.Pos()), etype.Plural, domain.Raw) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /tmpl/parse.go: -------------------------------------------------------------------------------- 1 | package tmpl 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "go/token" 7 | "io" 8 | "os" 9 | "strings" 10 | "text/template/parse" 11 | ) 12 | 13 | type Template struct { 14 | Filename string 15 | Trees map[string]*parse.Tree 16 | Inspector *Inspector 17 | 18 | // GoFilePos holds the position from which .go file the template originates 19 | GoFilePos token.Position 20 | 21 | // OffsetLookup holds the first position of all line starts. 22 | OffsetLookup []token.Position 23 | 24 | // Comments holds the comments of each line number 25 | Comments map[int][]string 26 | } 27 | 28 | func ParseFile(filepath string) (*Template, error) { 29 | src, errF := os.ReadFile(filepath) 30 | if errF != nil { 31 | return nil, errF 32 | } 33 | 34 | return ParseBytes(filepath, src) 35 | } 36 | 37 | func ParseString(name, content string) (*Template, error) { 38 | return ParseBytes(name, []byte(content)) 39 | } 40 | 41 | func ParseBytes(name string, src []byte) (*Template, error) { 42 | t := &Template{ 43 | Filename: name, 44 | Trees: make(map[string]*parse.Tree), 45 | Comments: make(map[int][]string), 46 | Inspector: nil, 47 | } 48 | 49 | tree := &parse.Tree{ 50 | Name: name, 51 | Mode: parse.ParseComments | parse.SkipFuncCheck, 52 | } 53 | 54 | _, err := tree.Parse(string(src), "{{", "}}", t.Trees, map[string]any{}) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | roodNotes := make([]parse.Node, 0, len(t.Trees)) 60 | for _, tree := range t.Trees { 61 | roodNotes = append(roodNotes, tree.Root) 62 | } 63 | 64 | t.OffsetLookup = extractLineInfos(name, bytes.NewReader(src)) 65 | t.Inspector = newInspector(roodNotes) 66 | 67 | return t, nil 68 | } 69 | 70 | func extractLineInfos(filename string, src io.Reader) []token.Position { 71 | infos := make([]token.Position, 0, 50) 72 | infos = append(infos, token.Position{Filename: filename, Offset: 0, Line: 1, Column: 1}) 73 | 74 | scanner := bufio.NewScanner(src) 75 | offset := 0 76 | line := 1 77 | for scanner.Scan() { 78 | offset += len(scanner.Text()) 79 | infos = append(infos, token.Position{Filename: filename, Offset: offset, Line: line, Column: len(scanner.Text())}) 80 | offset++ 81 | line++ 82 | } 83 | 84 | return infos 85 | } 86 | 87 | func (t *Template) ExtractComments() { 88 | t.Inspector.Nodes([]parse.Node{&parse.CommentNode{}}, func(rawNode parse.Node, _ bool) (proceed bool) { 89 | proceed = false 90 | node := rawNode.(*parse.CommentNode) 91 | 92 | comment := strings.TrimSpace(node.Text) 93 | comment = strings.TrimPrefix(comment, "/*") 94 | comment = strings.TrimSuffix(comment, "*/") 95 | comment = strings.TrimSpace(comment) 96 | 97 | pos := t.Position(node.Pos) 98 | t.Comments[pos.Line] = append(t.Comments[pos.Line], comment) 99 | 100 | return 101 | }) 102 | } 103 | 104 | func (t *Template) Position(offset parse.Pos) token.Position { 105 | var pos token.Position 106 | pos.Filename = t.Filename 107 | 108 | for i, p := range t.OffsetLookup { 109 | if p.Offset > int(offset) { 110 | pos = t.OffsetLookup[i-1] 111 | break 112 | } 113 | } 114 | 115 | if pos.IsValid() { 116 | pos.Column = int(offset) - pos.Offset 117 | pos.Offset += int(offset) 118 | } 119 | 120 | if t.GoFilePos.IsValid() { 121 | pos.Filename = t.GoFilePos.Filename 122 | pos.Line += t.GoFilePos.Line - 1 123 | pos.Column += t.GoFilePos.Column - 1 124 | pos.Offset += t.GoFilePos.Offset 125 | } 126 | 127 | return pos 128 | } 129 | 130 | func (t *Template) GetComments(offset parse.Pos) []string { 131 | pos := t.Position(offset) 132 | var comments []string 133 | if _, ok := t.Comments[pos.Line]; ok { 134 | comments = append(comments, t.Comments[pos.Line]...) 135 | } 136 | 137 | if _, ok := t.Comments[pos.Line-1]; ok { 138 | comments = append(comments, t.Comments[pos.Line-1]...) 139 | } 140 | 141 | return comments 142 | } 143 | -------------------------------------------------------------------------------- /extract/extractors/funccall.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "time" 7 | 8 | "github.com/vorlif/xspreak/extract" 9 | "github.com/vorlif/xspreak/extract/etype" 10 | "github.com/vorlif/xspreak/util" 11 | ) 12 | 13 | type funcCallExtractor struct{} 14 | 15 | func NewFuncCallExtractor() extract.Extractor { return &funcCallExtractor{} } 16 | 17 | func (v funcCallExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 18 | util.TrackTime(time.Now(), "extract func calls") 19 | var issues []extract.Issue 20 | 21 | extractCtx.Inspector.Nodes([]ast.Node{&ast.CallExpr{}}, func(rawNode ast.Node, push bool) (proceed bool) { 22 | proceed = true 23 | if !push { 24 | return 25 | } 26 | 27 | node := rawNode.(*ast.CallExpr) 28 | if len(node.Args) == 0 { 29 | return 30 | } 31 | 32 | var ident *ast.Ident 33 | switch fun := node.Fun.(type) { 34 | case *ast.Ident: 35 | ident = fun 36 | case *ast.IndexExpr: 37 | switch x := fun.X.(type) { 38 | case *ast.Ident: 39 | ident = x 40 | } 41 | } 42 | 43 | if ident == nil { 44 | if selector := util.SearchSelector(node.Fun); selector != nil { 45 | ident = selector.Sel 46 | } else { 47 | return 48 | } 49 | } 50 | 51 | pkg, obj := extractCtx.GetType(ident) 52 | if pkg == nil { 53 | return 54 | } 55 | 56 | // type conversions, e.g. localize.Singular("init") 57 | if tok := extractCtx.GetLocalizeTypeToken(ident); etype.IsMessageID(tok) { 58 | for _, res := range extractCtx.SearchStrings(node.Args[0]) { 59 | issue := extract.Issue{ 60 | FromExtractor: v.Name(), 61 | IDToken: tok, 62 | MsgID: res.Raw, 63 | Pkg: pkg, 64 | Comments: extractCtx.GetComments(pkg, res.Node), 65 | Pos: extractCtx.GetPosition(res.Node.Pos()), 66 | } 67 | 68 | issues = append(issues, issue) 69 | } 70 | } else if tok != etype.None { 71 | writeMissingMessageID(extractCtx.GetPosition(ident.Pos()), tok, "") 72 | } 73 | 74 | funcParameterDefs := extractCtx.Definitions.GetFields(util.ObjToKey(obj)) 75 | if funcParameterDefs == nil { 76 | return 77 | } 78 | 79 | collector := newSearchCollector() 80 | 81 | // Function calls 82 | for _, def := range funcParameterDefs { 83 | for i, arg := range node.Args { 84 | if (def.FieldPos != i) && (i < def.FieldPos || !def.IsVariadic) { 85 | continue 86 | } 87 | 88 | foundResults := extractCtx.SearchStrings(arg) 89 | if len(foundResults) == 0 { 90 | continue 91 | } 92 | 93 | switch def.Token { 94 | case etype.Singular, etype.Key, etype.PluralKey: 95 | collector.AddSingulars(def.Token, foundResults) 96 | case etype.Plural: 97 | collector.Plurals = append(collector.Plurals, foundResults...) 98 | case etype.Context: 99 | collector.Contexts = append(collector.Contexts, foundResults...) 100 | case etype.Domain: 101 | collector.Domains = append(collector.Domains, foundResults...) 102 | } 103 | } 104 | } 105 | 106 | collector.CheckMissingMessageID(extractCtx) 107 | for i, singularResult := range collector.Singulars { 108 | issue := extract.Issue{ 109 | FromExtractor: v.Name(), 110 | IDToken: collector.SingularType[i], 111 | MsgID: singularResult.Raw, 112 | Domain: collector.GetDomain(), 113 | Context: collector.GetContext(), 114 | PluralID: collector.GetPlural(), 115 | Comments: extractCtx.GetComments(pkg, singularResult.Node), 116 | Pkg: pkg, 117 | Pos: extractCtx.GetPosition(singularResult.Node.Pos()), 118 | } 119 | issues = append(issues, issue) 120 | } 121 | 122 | return 123 | }) 124 | 125 | return issues, nil 126 | } 127 | 128 | func (v funcCallExtractor) Name() string { 129 | return "funccall_extractor" 130 | } 131 | -------------------------------------------------------------------------------- /commands/extractor.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | 10 | log "github.com/sirupsen/logrus" 11 | 12 | "github.com/vorlif/xspreak/config" 13 | "github.com/vorlif/xspreak/encoder" 14 | "github.com/vorlif/xspreak/extract" 15 | "github.com/vorlif/xspreak/extract/extractors" 16 | "github.com/vorlif/xspreak/extract/loader" 17 | "github.com/vorlif/xspreak/extract/runner" 18 | "github.com/vorlif/xspreak/tmplextractors" 19 | "github.com/vorlif/xspreak/util" 20 | ) 21 | 22 | type Extractor struct { 23 | cfg *config.Config 24 | log *log.Entry 25 | 26 | contextLoader *loader.PackageLoader 27 | } 28 | 29 | func NewExtractor() *Extractor { 30 | return &Extractor{ 31 | cfg: extractCfg, 32 | log: log.WithField("service", "extractor"), 33 | contextLoader: loader.NewPackageLoader(extractCfg), 34 | } 35 | } 36 | 37 | func (e *Extractor) extract() { 38 | ctx, cancel := context.WithTimeout(context.Background(), e.cfg.Timeout) 39 | defer cancel() 40 | 41 | extractedIssues, errE := e.runExtraction(ctx) 42 | if errE != nil { 43 | e.log.Fatalf("Running error: %s", errE) 44 | } 45 | 46 | domainIssues := make(map[string][]extract.Issue) 47 | start := time.Now() 48 | for _, iss := range extractedIssues { 49 | if _, ok := domainIssues[iss.Domain]; !ok { 50 | domainIssues[iss.Domain] = []extract.Issue{iss} 51 | } else { 52 | domainIssues[iss.Domain] = append(domainIssues[iss.Domain], iss) 53 | } 54 | } 55 | log.Debugf("sort extractions took %s", time.Since(start)) 56 | 57 | if len(extractedIssues) == 0 { 58 | domainIssues[""] = make([]extract.Issue, 0) 59 | log.Println("No Strings found") 60 | } 61 | 62 | e.saveDomains(domainIssues) 63 | } 64 | 65 | func (e *Extractor) runExtraction(ctx context.Context) ([]extract.Issue, error) { 66 | util.TrackTime(time.Now(), "run all extractors") 67 | extractorsToRun := []extract.Extractor{ 68 | extractors.NewFuncCallExtractor(), 69 | extractors.NewFuncReturnExtractor(), 70 | extractors.NewGlobalAssignExtractor(), 71 | extractors.NewSliceDefExtractor(), 72 | extractors.NewMapsDefExtractor(), 73 | extractors.NewStructDefExtractor(), 74 | extractors.NewVariablesExtractor(), 75 | extractors.NewErrorExtractor(), 76 | extractors.NewInlineTemplateExtractor(), 77 | tmplextractors.NewCommandExtractor(), 78 | } 79 | 80 | extractCtx, err := e.contextLoader.Load(ctx) 81 | if err != nil { 82 | return nil, fmt.Errorf("context loading failed: %w", err) 83 | } 84 | 85 | runner, err := runner.New(e.cfg, extractCtx.Packages) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | issues, err := runner.Run(ctx, extractCtx, extractorsToRun) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | return issues, nil 96 | } 97 | 98 | func (e *Extractor) saveDomains(domains map[string][]extract.Issue) { 99 | util.TrackTime(time.Now(), "save files") 100 | for domainName, issues := range domains { 101 | var outputFile string 102 | if domainName == "" { 103 | outputFile = filepath.Join(e.cfg.OutputDir, e.cfg.OutputFile) 104 | } else { 105 | outputFile = filepath.Join(e.cfg.OutputDir, domainName+"."+e.cfg.ExtractFormat) 106 | } 107 | 108 | outputDir := filepath.Dir(outputFile) 109 | if _, err := os.Stat(outputDir); os.IsNotExist(err) { 110 | log.Printf("Output folder does not exist, trying to create it: %s\n", outputDir) 111 | if errC := os.MkdirAll(outputDir, os.ModePerm); errC != nil { 112 | log.Fatalf("Output folder does not exist and could not be created: %s", errC) 113 | } 114 | } 115 | 116 | dst, err := os.Create(outputFile) 117 | if err != nil { 118 | e.log.WithError(err).Fatal("Output file could not be created") 119 | } 120 | defer dst.Close() 121 | 122 | var enc encoder.Encoder 123 | if e.cfg.ExtractFormat == config.ExtractFormatPot { 124 | enc = encoder.NewPotEncoder(e.cfg, dst) 125 | } else { 126 | enc = encoder.NewJSONEncoder(dst, " ") 127 | } 128 | 129 | if errEnc := enc.Encode(issues); errEnc != nil { 130 | e.log.WithError(errEnc).Fatal("Output file could not be written") 131 | } 132 | 133 | _ = dst.Close() 134 | log.Printf("File written: %s\n", outputFile) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /extract/extractors/structdef.go: -------------------------------------------------------------------------------- 1 | package extractors 2 | 3 | import ( 4 | "context" 5 | "go/ast" 6 | "go/types" 7 | "time" 8 | 9 | "golang.org/x/tools/go/packages" 10 | 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/extract/etype" 13 | "github.com/vorlif/xspreak/util" 14 | ) 15 | 16 | type structDefExtractor struct{} 17 | 18 | func NewStructDefExtractor() extract.Extractor { 19 | return &structDefExtractor{} 20 | } 21 | 22 | func (v structDefExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 23 | util.TrackTime(time.Now(), "extract structs") 24 | var issues []extract.Issue 25 | 26 | extractCtx.Inspector.Nodes([]ast.Node{&ast.CompositeLit{}}, func(rawNode ast.Node, push bool) (proceed bool) { 27 | proceed = true 28 | if !push { 29 | return 30 | } 31 | 32 | node := rawNode.(*ast.CompositeLit) 33 | if len(node.Elts) == 0 { 34 | return 35 | } 36 | 37 | var ident *ast.Ident 38 | switch val := node.Type.(type) { 39 | case *ast.SelectorExpr: 40 | ident = val.Sel 41 | case *ast.Ident: 42 | ident = val 43 | case *ast.IndexExpr: 44 | switch x := val.X.(type) { 45 | case *ast.Ident: 46 | ident = x 47 | } 48 | default: 49 | return 50 | } 51 | 52 | pkg, obj := extractCtx.GetType(ident) 53 | if pkg == nil { 54 | return 55 | } 56 | 57 | if structAttr := extractCtx.Definitions.GetFields(util.ObjToKey(obj)); structAttr == nil { 58 | return 59 | } 60 | 61 | structIssues := extractStruct(extractCtx, node, obj, pkg) 62 | issues = append(issues, structIssues...) 63 | 64 | return 65 | }) 66 | 67 | return issues, nil 68 | } 69 | 70 | func (v structDefExtractor) Name() string { 71 | return "struct_extractor" 72 | } 73 | 74 | func extractStruct(extractCtx *extract.Context, node *ast.CompositeLit, obj types.Object, pkg *packages.Package) []extract.Issue { 75 | var issues []extract.Issue 76 | 77 | definitionKey := util.ObjToKey(obj) 78 | collector := newSearchCollector() 79 | 80 | if _, isKv := node.Elts[0].(*ast.KeyValueExpr); isKv { 81 | for _, elt := range node.Elts { 82 | kve, ok := elt.(*ast.KeyValueExpr) 83 | if !ok { 84 | continue 85 | } 86 | 87 | idt, ok := kve.Key.(*ast.Ident) 88 | if !ok { 89 | continue 90 | } 91 | 92 | def := extractCtx.Definitions.Get(definitionKey, idt.Name) 93 | if def == nil { 94 | continue 95 | } 96 | 97 | foundResults := extractCtx.SearchStrings(kve.Value) 98 | if len(foundResults) == 0 { 99 | continue 100 | } 101 | 102 | switch def.Token { 103 | case etype.Singular, etype.Key, etype.PluralKey: 104 | collector.AddSingulars(def.Token, foundResults) 105 | case etype.Plural: 106 | collector.Plurals = append(collector.Plurals, foundResults...) 107 | case etype.Context: 108 | collector.Contexts = append(collector.Contexts, foundResults...) 109 | case etype.Domain: 110 | collector.Domains = append(collector.Domains, foundResults...) 111 | } 112 | } 113 | } else { 114 | for _, attrDef := range extractCtx.Definitions.GetFields(definitionKey) { 115 | for i, elt := range node.Elts { 116 | if attrDef.FieldPos != i { 117 | continue 118 | } 119 | 120 | foundResults := extractCtx.SearchStrings(elt) 121 | if len(foundResults) == 0 { 122 | continue 123 | } 124 | 125 | switch attrDef.Token { 126 | case etype.Singular, etype.Key, etype.PluralKey: 127 | collector.AddSingulars(attrDef.Token, foundResults) 128 | case etype.Plural: 129 | collector.Plurals = append(collector.Plurals, foundResults...) 130 | case etype.Context: 131 | collector.Contexts = append(collector.Contexts, foundResults...) 132 | case etype.Domain: 133 | collector.Domains = append(collector.Domains, foundResults...) 134 | } 135 | } 136 | } 137 | } 138 | 139 | collector.CheckMissingMessageID(extractCtx) 140 | for i, singularResult := range collector.Singulars { 141 | issue := extract.Issue{ 142 | FromExtractor: "extract_struct", 143 | IDToken: collector.SingularType[i], 144 | MsgID: singularResult.Raw, 145 | Domain: collector.GetDomain(), 146 | Context: collector.GetContext(), 147 | PluralID: collector.GetPlural(), 148 | Comments: extractCtx.GetComments(pkg, singularResult.Node), 149 | Pkg: pkg, 150 | Pos: extractCtx.GetPosition(singularResult.Node.Pos()), 151 | } 152 | issues = append(issues, issue) 153 | } 154 | 155 | return issues 156 | } 157 | -------------------------------------------------------------------------------- /tmplextractors/command.go: -------------------------------------------------------------------------------- 1 | package tmplextractors 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "text/template/parse" 7 | "time" 8 | 9 | log "github.com/sirupsen/logrus" 10 | 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/tmpl" 13 | "github.com/vorlif/xspreak/util" 14 | ) 15 | 16 | type commandExtractor struct{} 17 | 18 | func NewCommandExtractor() extract.Extractor { 19 | return &commandExtractor{} 20 | } 21 | 22 | func (c *commandExtractor) Run(_ context.Context, extractCtx *extract.Context) ([]extract.Issue, error) { 23 | util.TrackTime(time.Now(), "extract templates") 24 | 25 | var issues []extract.Issue 26 | if len(extractCtx.Config.Keywords) == 0 { 27 | log.Debug("Skip template extraction, no keywords present") 28 | return issues, nil 29 | } 30 | 31 | for _, template := range extractCtx.Templates { 32 | template.ExtractComments() 33 | 34 | template.Inspector.WithStack([]parse.Node{&parse.PipeNode{}}, func(n parse.Node, push bool, _ []parse.Node) (proceed bool) { 35 | proceed = false 36 | if !push { 37 | return 38 | } 39 | pipe := n.(*parse.PipeNode) 40 | if pipe.IsAssign { 41 | return 42 | } 43 | 44 | for _, cmd := range pipe.Cmds { 45 | isses := extractIssues(cmd, extractCtx, template) 46 | for _, iss := range isses { 47 | iss.Flags = append(iss.Flags, "go-template") 48 | issues = append(issues, iss) 49 | } 50 | } 51 | 52 | return 53 | }) 54 | } 55 | return issues, nil 56 | } 57 | 58 | func (c *commandExtractor) Name() string { 59 | return "tmpl_command" 60 | } 61 | 62 | func walkNode(n parse.Node, extractCtx *extract.Context, template *tmpl.Template, results *[]extract.Issue) { 63 | switch v := n.(type) { 64 | case *parse.CommandNode: 65 | iss := extractIssue(v, extractCtx, template) 66 | if iss != nil { 67 | *results = append(*results, *iss) 68 | } 69 | for _, node := range v.Args { 70 | walkNode(node, extractCtx, template, results) 71 | } 72 | case *parse.PipeNode: 73 | for _, node := range v.Cmds { 74 | walkNode(node, extractCtx, template, results) 75 | } 76 | } 77 | } 78 | 79 | func extractIssues(cmd *parse.CommandNode, extractCtx *extract.Context, template *tmpl.Template) []extract.Issue { 80 | var ret []extract.Issue 81 | walkNode(cmd, extractCtx, template, &ret) 82 | return ret 83 | } 84 | 85 | func extractIssue(cmd *parse.CommandNode, extractCtx *extract.Context, template *tmpl.Template) *extract.Issue { 86 | if cmd == nil { 87 | return nil 88 | } 89 | raw := cmd.String() 90 | for _, keyword := range extractCtx.Config.Keywords { 91 | if !strings.HasPrefix(raw, keyword.Name+" ") { 92 | continue 93 | } 94 | 95 | if keyword.MaxPosition() >= len(cmd.Args)-1 { // The first index contains the keyword itself 96 | log.Warnf("Template keyword found but not enough arguments available: %s %s", template.Position(cmd.Pos), raw) 97 | continue 98 | } 99 | 100 | return extractArgs(cmd.Args[1:], keyword, template) 101 | } 102 | 103 | return nil 104 | } 105 | 106 | func extractArgs(args []parse.Node, keyword *tmpl.Keyword, template *tmpl.Template) *extract.Issue { 107 | iss := &extract.Issue{} 108 | 109 | if stringNode, ok := args[keyword.SingularPos].(*parse.StringNode); ok { 110 | iss.MsgID = stringNode.Text 111 | iss.IDToken = keyword.IDToken 112 | iss.Comments = template.GetComments(args[keyword.SingularPos].Position()) 113 | iss.Pos = template.Position(args[keyword.SingularPos].Position()) 114 | } else { 115 | log.Warnf("Template keyword is not passed a string: %s", args[keyword.SingularPos]) 116 | return nil 117 | } 118 | 119 | if keyword.PluralPos >= 0 { 120 | if stringNode, ok := args[keyword.PluralPos].(*parse.StringNode); ok { 121 | iss.PluralID = stringNode.Text 122 | } else { 123 | log.Warnf("Template keyword is not passed a string: %s", args[keyword.PluralPos]) 124 | return nil 125 | } 126 | } 127 | 128 | if keyword.ContextPos >= 0 { 129 | if stringNode, ok := args[keyword.ContextPos].(*parse.StringNode); ok { 130 | iss.Context = stringNode.Text 131 | } else { 132 | log.Warnf("Template keyword is not passed a string: %s", args[keyword.ContextPos]) 133 | return nil 134 | } 135 | } 136 | 137 | if keyword.DomainPos >= 0 { 138 | if stringNode, ok := args[keyword.DomainPos].(*parse.StringNode); ok { 139 | iss.Domain = stringNode.Text 140 | } else { 141 | log.Warnf("Template keyword is not passed a string: %s", args[keyword.DomainPos]) 142 | return nil 143 | } 144 | } 145 | 146 | return iss 147 | } 148 | -------------------------------------------------------------------------------- /encoder/json.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "sort" 9 | "strings" 10 | "time" 11 | 12 | "github.com/vorlif/spreak/catalog/cldrplural" 13 | 14 | "github.com/vorlif/xspreak/extract" 15 | "github.com/vorlif/xspreak/extract/etype" 16 | "github.com/vorlif/xspreak/util" 17 | ) 18 | 19 | type jsonEncoder struct { 20 | w *json.Encoder 21 | } 22 | 23 | func NewJSONEncoder(w io.Writer, ident string) Encoder { 24 | enc := json.NewEncoder(w) 25 | enc.SetIndent("", ident) 26 | 27 | return &jsonEncoder{w: enc} 28 | } 29 | 30 | func (e *jsonEncoder) Encode(issues []extract.Issue) error { 31 | util.TrackTime(time.Now(), "Build messages") 32 | 33 | items := make(map[string]JSONItem, len(issues)) 34 | 35 | for _, iss := range issues { 36 | msg := make(JSONMessage) 37 | msg[catKey(cldrplural.Other)] = "" 38 | 39 | key := iss.MsgID 40 | 41 | if iss.Context != "" { 42 | msg["context"] = iss.Context 43 | key = fmt.Sprintf("%s_%s", iss.MsgID, iss.Context) 44 | } 45 | 46 | if iss.PluralID != "" { 47 | msg[catKey(cldrplural.One)] = "" 48 | 49 | if iss.IDToken != etype.PluralKey && iss.IDToken != etype.Key { 50 | msg[catKey(cldrplural.One)] = iss.MsgID 51 | msg[catKey(cldrplural.Other)] = iss.PluralID 52 | } 53 | } else { 54 | if iss.IDToken != etype.PluralKey && iss.IDToken != etype.Key { 55 | msg[catKey(cldrplural.Other)] = iss.MsgID 56 | } 57 | } 58 | 59 | items[key] = JSONItem{Key: key, Message: msg} 60 | } 61 | 62 | file := make(JSONFile, 0, len(issues)) 63 | for _, item := range items { 64 | file = append(file, item) 65 | } 66 | 67 | sort.Slice(file, func(i, j int) bool { 68 | return file[i].Key < file[j].Key 69 | }) 70 | 71 | return e.w.Encode(file) 72 | } 73 | 74 | type JSONItem struct { 75 | Key string 76 | Message JSONMessage 77 | } 78 | 79 | type JSONFile []JSONItem 80 | 81 | func (f JSONFile) MarshalJSON() ([]byte, error) { 82 | var buf bytes.Buffer 83 | buf.WriteString("{") 84 | 85 | for i, kv := range f { 86 | if i != 0 { 87 | buf.WriteString(",") 88 | } 89 | // marshal key 90 | key, err := json.Marshal(kv.Key) 91 | if err != nil { 92 | return nil, err 93 | } 94 | buf.Write(key) 95 | buf.WriteString(":") 96 | // marshal value 97 | val, err := json.Marshal(&kv.Message) 98 | if err != nil { 99 | return nil, err 100 | } 101 | buf.Write(val) 102 | } 103 | 104 | buf.WriteString("}") 105 | return buf.Bytes(), nil 106 | } 107 | 108 | func (f *JSONFile) UnmarshalJSON(data []byte) error { 109 | messages := make(map[string]JSONMessage) 110 | if err := json.Unmarshal(data, &messages); err != nil { 111 | return err 112 | } 113 | 114 | for k, msg := range messages { 115 | *f = append(*f, JSONItem{Key: k, Message: msg}) 116 | } 117 | 118 | return nil 119 | } 120 | 121 | type JSONMessage map[string]string 122 | 123 | var messageKey = []string{"context", "zero", "one", "two", "few", "many", "other"} 124 | 125 | func (m JSONMessage) MarshalJSON() ([]byte, error) { 126 | switch len(m) { 127 | case 0: 128 | return json.Marshal("") 129 | case 1: 130 | for _, v := range m { 131 | return json.Marshal(v) 132 | } 133 | } 134 | 135 | var buf bytes.Buffer 136 | buf.WriteString("{") 137 | 138 | keywordNum := 0 139 | for _, keyword := range messageKey { 140 | text, ok := m[keyword] 141 | if !ok { 142 | continue 143 | } 144 | 145 | if keywordNum != 0 { 146 | buf.WriteString(",") 147 | } 148 | keywordNum++ 149 | 150 | // marshal key 151 | key, err := json.Marshal(keyword) 152 | if err != nil { 153 | return nil, err 154 | } 155 | buf.Write(key) 156 | buf.WriteString(":") 157 | // marshal value 158 | val, err := json.Marshal(text) 159 | if err != nil { 160 | return nil, err 161 | } 162 | buf.Write(val) 163 | } 164 | 165 | buf.WriteString("}") 166 | return buf.Bytes(), nil 167 | } 168 | 169 | func (m *JSONMessage) UnmarshalJSON(data []byte) error { 170 | if (*m) == nil { 171 | *m = make(JSONMessage) 172 | } 173 | 174 | var other string 175 | if err := json.Unmarshal(data, &other); err == nil { 176 | (*m)[catKey(cldrplural.Other)] = other 177 | return nil 178 | } 179 | 180 | var mm map[string]string 181 | if err := json.Unmarshal(data, &mm); err != nil { 182 | return err 183 | } 184 | 185 | if len(mm) == 0 { 186 | (*m)[catKey(cldrplural.Other)] = "" 187 | return nil 188 | } 189 | 190 | for k, v := range mm { 191 | (*m)[k] = v 192 | } 193 | return nil 194 | } 195 | 196 | func catKey(cat cldrplural.Category) string { 197 | return strings.ToLower(cat.String()) 198 | } 199 | -------------------------------------------------------------------------------- /tmpl/keyword.go: -------------------------------------------------------------------------------- 1 | package tmpl 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/vorlif/xspreak/extract/etype" 9 | ) 10 | 11 | type Keyword struct { 12 | Name string 13 | IDToken etype.Token 14 | SingularPos int 15 | PluralPos int 16 | ContextPos int 17 | DomainPos int 18 | } 19 | 20 | func ParseKeywords(spec string, isMonolingual bool) (*Keyword, error) { 21 | var functionName string 22 | var args []string 23 | idx := strings.IndexByte(spec, ':') 24 | if idx >= 0 { 25 | functionName = spec[:idx] 26 | args = strings.Split(spec[idx+1:], ",") 27 | } else { 28 | functionName = spec 29 | } 30 | 31 | k := &Keyword{ 32 | Name: functionName, 33 | IDToken: etype.Singular, 34 | SingularPos: 0, 35 | PluralPos: -1, 36 | ContextPos: -1, 37 | DomainPos: -1, 38 | } 39 | 40 | if isMonolingual { 41 | k.IDToken = etype.Key 42 | } 43 | 44 | inputType := 0 45 | for _, arg := range args { 46 | if len(arg) == 0 { 47 | continue 48 | } 49 | 50 | lastSign := arg[len(arg)-1] 51 | if lastSign == 'c' || lastSign == 'd' { 52 | val, err := strconv.Atoi(arg[:len(arg)-1]) 53 | if err != nil { 54 | return nil, fmt.Errorf("bad keyword number: %s %w", arg, err) 55 | } 56 | if lastSign == 'c' { 57 | k.ContextPos = val - 1 58 | } else { 59 | k.DomainPos = val - 1 60 | } 61 | continue 62 | } 63 | 64 | val, err := strconv.Atoi(arg) 65 | if err != nil { 66 | return nil, fmt.Errorf("bad keyword number: %s", arg) 67 | } 68 | switch inputType { 69 | case 0: 70 | k.SingularPos = val - 1 71 | case 1: 72 | k.PluralPos = val - 1 73 | default: 74 | return nil, fmt.Errorf("bad keyword number: %s", arg) 75 | } 76 | inputType++ 77 | } 78 | 79 | return k, nil 80 | } 81 | 82 | var bilingualKeywordTemplates = []Keyword{ 83 | {Name: "Get", IDToken: etype.Singular, SingularPos: 0, PluralPos: -1, ContextPos: -1, DomainPos: -1}, 84 | {Name: "DGet", IDToken: etype.Singular, DomainPos: 0, SingularPos: 1, PluralPos: -1, ContextPos: -1}, 85 | {Name: "NGet", IDToken: etype.Singular, SingularPos: 0, PluralPos: 1, ContextPos: -1, DomainPos: -1}, 86 | {Name: "DNGet", IDToken: etype.Singular, DomainPos: 0, SingularPos: 1, PluralPos: 2, ContextPos: -1}, 87 | {Name: "PGet", IDToken: etype.Singular, ContextPos: 0, SingularPos: 1, PluralPos: -1, DomainPos: -1}, 88 | {Name: "DPGet", IDToken: etype.Singular, DomainPos: 0, ContextPos: 1, SingularPos: 2, PluralPos: -1}, 89 | {Name: "NPGet", IDToken: etype.Singular, ContextPos: 0, SingularPos: 1, PluralPos: 2, DomainPos: -1}, 90 | {Name: "DNPGet", IDToken: etype.Singular, DomainPos: 0, ContextPos: 1, SingularPos: 2, PluralPos: 3}, 91 | } 92 | 93 | var monolingualKeywordTemplates = []Keyword{ 94 | {Name: "Get", IDToken: etype.Key, SingularPos: 0, PluralPos: -1, ContextPos: -1, DomainPos: -1}, 95 | {Name: "DGet", IDToken: etype.Key, DomainPos: 0, SingularPos: 1, PluralPos: -1, ContextPos: -1}, 96 | {Name: "NGet", IDToken: etype.PluralKey, SingularPos: 0, PluralPos: 0, ContextPos: -1, DomainPos: -1}, 97 | {Name: "DNGet", IDToken: etype.PluralKey, DomainPos: 0, SingularPos: 1, PluralPos: 1, ContextPos: -1}, 98 | {Name: "PGet", IDToken: etype.Singular, ContextPos: 0, SingularPos: 1, PluralPos: -1, DomainPos: -1}, 99 | {Name: "DPGet", IDToken: etype.Singular, DomainPos: 0, ContextPos: 1, SingularPos: 2, PluralPos: -1}, 100 | {Name: "NPGet", IDToken: etype.PluralKey, ContextPos: 0, SingularPos: 1, PluralPos: 1, DomainPos: -1}, 101 | {Name: "DNPGet", IDToken: etype.PluralKey, DomainPos: 0, ContextPos: 1, SingularPos: 2, PluralPos: 2}, 102 | } 103 | 104 | func DefaultKeywords(name string, isMonolingual bool) []*Keyword { 105 | name = strings.Trim(strings.TrimSpace(name), ".") 106 | if name == "" { 107 | name = "T" 108 | } 109 | 110 | var keywordTemplates []Keyword 111 | if isMonolingual { 112 | keywordTemplates = monolingualKeywordTemplates 113 | } else { 114 | keywordTemplates = bilingualKeywordTemplates 115 | } 116 | 117 | keywords := make([]*Keyword, 0, len(keywordTemplates)*4) 118 | 119 | for _, prefix := range []string{"$.", "."} { 120 | for _, suffix := range []string{"", "f"} { 121 | for _, tmpl := range keywordTemplates { 122 | keyword := tmpl 123 | keyword.Name = prefix + name + "." + tmpl.Name + suffix 124 | keywords = append(keywords, &keyword) 125 | } 126 | } 127 | } 128 | 129 | return keywords 130 | } 131 | 132 | func (k *Keyword) MaxPosition() int { 133 | start := maxOf(k.SingularPos, -1) 134 | start = maxOf(start, k.PluralPos) 135 | start = maxOf(start, k.ContextPos) 136 | return maxOf(start, k.DomainPos) 137 | } 138 | 139 | func maxOf(x, y int) int { 140 | if x > y { 141 | return x 142 | } 143 | return y 144 | } 145 | -------------------------------------------------------------------------------- /tmpl/inspector.go: -------------------------------------------------------------------------------- 1 | package tmpl 2 | 3 | import ( 4 | "text/template/parse" 5 | ) 6 | 7 | // Based on https://cs.opensource.google/go/x/tools/+/refs/tags/v0.1.10:go/ast/inspector/ 8 | 9 | const ( 10 | nAction = iota 11 | nBool 12 | nBranch 13 | nBreak 14 | nChain 15 | nCommand 16 | nComment 17 | nContinue 18 | nDot 19 | nField 20 | nIdentifier 21 | nIf 22 | nList 23 | nNil 24 | nNumber 25 | nPipe 26 | nRange 27 | nString 28 | nTemplate 29 | nText 30 | NVariable 31 | nWith 32 | ) 33 | 34 | type Inspector struct { 35 | events []event 36 | } 37 | 38 | func newInspector(nodes []parse.Node) *Inspector { 39 | return &Inspector{traverse(nodes)} 40 | } 41 | 42 | type event struct { 43 | node parse.Node 44 | typ uint64 // typeOf(node) 45 | index int // 1 + index of corresponding pop event, or 0 if this is a pop 46 | } 47 | 48 | func (in *Inspector) Preorder(types []parse.Node, f func(parse.Node)) { 49 | mask := maskOf(types) 50 | for i := 0; i < len(in.events); { 51 | ev := in.events[i] 52 | if ev.typ&mask != 0 { 53 | if ev.index > 0 { 54 | f(ev.node) 55 | } 56 | } 57 | i++ 58 | } 59 | } 60 | 61 | func (in *Inspector) Nodes(types []parse.Node, f func(n parse.Node, push bool) (proceed bool)) { 62 | mask := maskOf(types) 63 | for i := 0; i < len(in.events); { 64 | ev := in.events[i] 65 | if ev.typ&mask != 0 { 66 | if ev.index > 0 { 67 | // push 68 | if !f(ev.node, true) { 69 | i = ev.index // jump to corresponding pop + 1 70 | continue 71 | } 72 | } else { 73 | // pop 74 | f(ev.node, false) 75 | } 76 | } 77 | i++ 78 | } 79 | } 80 | 81 | func (in *Inspector) WithStack(types []parse.Node, f func(n parse.Node, push bool, stack []parse.Node) (proceed bool)) { 82 | mask := maskOf(types) 83 | var stack []parse.Node 84 | for i := 0; i < len(in.events); { 85 | ev := in.events[i] 86 | if ev.index > 0 { 87 | // push 88 | stack = append(stack, ev.node) 89 | if ev.typ&mask != 0 { 90 | if !f(ev.node, true, stack) { 91 | i = ev.index 92 | stack = stack[:len(stack)-1] 93 | continue 94 | } 95 | } 96 | } else { 97 | // pop 98 | if ev.typ&mask != 0 { 99 | f(ev.node, false, stack) 100 | } 101 | stack = stack[:len(stack)-1] 102 | } 103 | i++ 104 | } 105 | } 106 | 107 | func traverse(nodes []parse.Node) []event { 108 | // This estimate is based on the net/http package. 109 | capacity := len(nodes) * 33 / 100 110 | if capacity > 1e6 { 111 | capacity = 1e6 // impose some reasonable maximum 112 | } 113 | events := make([]event, 0, capacity) 114 | 115 | var stack []event 116 | for _, root := range nodes { 117 | Inspect(root, func(n parse.Node) bool { 118 | if n != nil { 119 | // push 120 | ev := event{ 121 | node: n, 122 | typ: typeOf(n), 123 | index: len(events), // push event temporarily holds own index 124 | } 125 | stack = append(stack, ev) 126 | events = append(events, ev) 127 | } else { 128 | // pop 129 | ev := stack[len(stack)-1] 130 | stack = stack[:len(stack)-1] 131 | 132 | events[ev.index].index = len(events) + 1 // make push refer to pop 133 | 134 | ev.index = 0 // turn ev into a pop event 135 | events = append(events, ev) 136 | } 137 | return true 138 | }) 139 | } 140 | 141 | return events 142 | } 143 | 144 | func typeOf(n parse.Node) uint64 { 145 | switch n.(type) { 146 | case *parse.ActionNode: 147 | return 1 << nAction 148 | case *parse.BranchNode: 149 | return 1 << nBranch 150 | case *parse.ChainNode: 151 | return 1 << nChain 152 | case *parse.CommandNode: 153 | return 1 << nCommand 154 | case *parse.IfNode: 155 | return 1 << nIf 156 | case *parse.ListNode: 157 | return 1 << nList 158 | case *parse.PipeNode: 159 | return 1 << nPipe 160 | case *parse.RangeNode: 161 | return 1 << nRange 162 | case *parse.TemplateNode: 163 | return 1 << nTemplate 164 | case *parse.WithNode: 165 | return 1 << nWith 166 | case *parse.BoolNode: 167 | return 1 << nBool 168 | case *parse.BreakNode: 169 | return 1 << nBreak 170 | case *parse.CommentNode: 171 | return 1 << nComment 172 | case *parse.ContinueNode: 173 | return 1 << nContinue 174 | case *parse.DotNode: 175 | return 1 << nDot 176 | case *parse.FieldNode: 177 | return 1 << nField 178 | case *parse.IdentifierNode: 179 | return 1 << nIdentifier 180 | case *parse.NilNode: 181 | return 1 << nNil 182 | case *parse.NumberNode: 183 | return 1 << nNumber 184 | case *parse.StringNode: 185 | return 1 << nString 186 | case *parse.TextNode: 187 | return 1 << nText 188 | case *parse.VariableNode: 189 | return 1 << NVariable 190 | } 191 | 192 | return 0 193 | } 194 | 195 | func maskOf(nodes []parse.Node) uint64 { 196 | if nodes == nil { 197 | return 1<<64 - 1 // match all node types 198 | } 199 | var mask uint64 200 | for _, n := range nodes { 201 | mask |= typeOf(n) 202 | } 203 | return mask 204 | } 205 | -------------------------------------------------------------------------------- /tmplextractors/command_test.go: -------------------------------------------------------------------------------- 1 | package tmplextractors 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/vorlif/xspreak/config" 11 | "github.com/vorlif/xspreak/extract" 12 | "github.com/vorlif/xspreak/extract/loader" 13 | "github.com/vorlif/xspreak/extract/runner" 14 | "github.com/vorlif/xspreak/tmpl" 15 | ) 16 | 17 | func TestVariablesExtractorRun(t *testing.T) { 18 | issues := runExtraction(t, testdataDir, NewCommandExtractor()) 19 | assert.NotEmpty(t, issues) 20 | 21 | want := []string{ 22 | "Get-Singular", 23 | "Getf-Singular", 24 | 25 | "NGet-Singular", "NGet-Plural", 26 | "NGetf-Singular", "NGetf-Plural", 27 | 28 | "DGet-Domain", "DGet-Singular", 29 | "DGetf-Domain", "DGetf-Singular", 30 | 31 | "DNGet-Domain", "DNGet-Singular", "DNGet-Plural", 32 | "DNGetf-Domain", "DNGetf-Singular", "DNGetf-Plural", 33 | 34 | "PGet-Context", "PGet-Singular", 35 | "PGetf-Context", "PGetf-Singular", 36 | 37 | "DPGet-Domain", "DPGet-Context", "DPGet-Singular", 38 | "DPGetf-Domain", "DPGetf-Context", "DPGetf-Singular", 39 | 40 | "NPGet-Context", "NPGet-Singular", "NPGet-Plural", 41 | "NPGetf-Context", "NPGetf-Singular", "NPGetf-Plural", 42 | 43 | "DNPGet-Context", "DNPGet-Context", "DNPGet-Singular", "DNPGet-Plural", 44 | "DNPGetf-Context", "DNPGetf-Context", "DNPGetf-Singular", "DNPGetf-Plural", 45 | } 46 | got := collectIssueStrings(issues) 47 | assert.ElementsMatch(t, want, got) 48 | } 49 | 50 | func TestKeyword(t *testing.T) { 51 | cfg := config.NewDefault() 52 | cfg.SourceDir = testdataDir 53 | cfg.ExtractErrors = false 54 | cfg.Keywords = []*tmpl.Keyword{ 55 | { 56 | Name: ".i18n.Tr", 57 | SingularPos: 0, 58 | PluralPos: -1, 59 | ContextPos: -1, 60 | DomainPos: -1, 61 | }, 62 | { 63 | Name: ".i18n.Trp", 64 | SingularPos: 1, 65 | PluralPos: 3, 66 | ContextPos: -1, 67 | DomainPos: -1, 68 | }, 69 | } 70 | cfg.TemplatePatterns = []string{ 71 | testdataTemplates + "/**/*.txt", 72 | testdataTemplates + "/**/*.html", 73 | testdataTemplates + "/**/*.tmpl", 74 | } 75 | 76 | require.NoError(t, cfg.Prepare()) 77 | 78 | ctx := context.Background() 79 | contextLoader := loader.NewPackageLoader(cfg) 80 | 81 | extractCtx, err := contextLoader.Load(ctx) 82 | require.NoError(t, err) 83 | 84 | runner, err := runner.New(cfg, extractCtx.Packages) 85 | require.NoError(t, err) 86 | 87 | issues, err := runner.Run(ctx, extractCtx, []extract.Extractor{NewCommandExtractor()}) 88 | require.NoError(t, err) 89 | 90 | want := []string{ 91 | "custom keyword", 92 | 93 | "trp-singular", "trp-plural", 94 | } 95 | got := collectIssueStrings(issues) 96 | assert.ElementsMatch(t, want, got) 97 | } 98 | 99 | func TestCommentExtraction(t *testing.T) { 100 | issues := runExtraction(t, testdataDir, NewCommandExtractor()) 101 | require.NotEmpty(t, issues) 102 | 103 | comments := make([]string, 0, len(issues)) 104 | for _, issue := range issues { 105 | comments = append(comments, issue.Comments...) 106 | } 107 | assert.Len(t, comments, 1) 108 | } 109 | 110 | func TestComplex(t *testing.T) { 111 | cfg := config.NewDefault() 112 | cfg.SourceDir = testdataDir 113 | cfg.ExtractErrors = false 114 | cfg.Keywords = []*tmpl.Keyword{ 115 | { 116 | Name: ".T", 117 | SingularPos: 0, 118 | PluralPos: -1, 119 | ContextPos: -1, 120 | DomainPos: -1, 121 | }, 122 | } 123 | cfg.TemplatePatterns = []string{ 124 | testdataTemplates + "/**/*.txt", 125 | testdataTemplates + "/**/*.html", 126 | testdataTemplates + "/**/*.tmpl", 127 | } 128 | 129 | require.NoError(t, cfg.Prepare()) 130 | 131 | ctx := context.Background() 132 | contextLoader := loader.NewPackageLoader(cfg) 133 | 134 | extractCtx, err := contextLoader.Load(ctx) 135 | require.NoError(t, err) 136 | 137 | runner, err := runner.New(cfg, extractCtx.Packages) 138 | require.NoError(t, err) 139 | 140 | issues, err := runner.Run(ctx, extractCtx, []extract.Extractor{NewCommandExtractor()}) 141 | require.NoError(t, err) 142 | 143 | want := []string{ 144 | "todos", 145 | "Help", 146 | } 147 | got := collectIssueStrings(issues) 148 | assert.ElementsMatch(t, want, got) 149 | } 150 | 151 | func TestParenthesised(t *testing.T) { 152 | cfg := config.NewDefault() 153 | cfg.SourceDir = testdataDir 154 | cfg.ExtractErrors = false 155 | cfg.Keywords = []*tmpl.Keyword{ 156 | { 157 | Name: ".X", 158 | SingularPos: 0, 159 | PluralPos: -1, 160 | ContextPos: -1, 161 | DomainPos: -1, 162 | }, 163 | } 164 | cfg.TemplatePatterns = []string{ 165 | testdataTemplates + "/**/seven.parens", 166 | } 167 | 168 | require.NoError(t, cfg.Prepare()) 169 | 170 | ctx := context.Background() 171 | contextLoader := loader.NewPackageLoader(cfg) 172 | 173 | extractCtx, err := contextLoader.Load(ctx) 174 | require.NoError(t, err) 175 | 176 | runner, err := runner.New(cfg, extractCtx.Packages) 177 | require.NoError(t, err) 178 | 179 | issues, err := runner.Run(ctx, extractCtx, []extract.Extractor{NewCommandExtractor()}) 180 | require.NoError(t, err) 181 | 182 | want := []string{ 183 | "foo", 184 | "bar", 185 | } 186 | got := collectIssueStrings(issues) 187 | assert.ElementsMatch(t, want, got) 188 | } 189 | -------------------------------------------------------------------------------- /commands/root.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "runtime/debug" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/vorlif/xspreak/config" 10 | "github.com/vorlif/xspreak/tmpl" 11 | ) 12 | 13 | // Version is initialized via ldflags or debug.BuildInfo. 14 | var Version = "" 15 | 16 | var ( 17 | extractCfg = config.NewDefault() 18 | 19 | rootCmd = &cobra.Command{ 20 | Use: "xspreak", 21 | Version: Version, 22 | Short: "String extraction for spreak.", 23 | Long: `Simple tool to extract strings and create POT/JSON files for application translations.`, 24 | Run: extractCmdF, 25 | } 26 | ) 27 | 28 | func Execute() error { 29 | return rootCmd.Execute() 30 | } 31 | 32 | func init() { 33 | initVersionNumber() 34 | 35 | def := config.NewDefault() 36 | 37 | rootCmd.PersistentFlags().BoolVarP(&extractCfg.IsVerbose, "verbose", "V", def.IsVerbose, "increase verbosity level") 38 | rootCmd.PersistentFlags().DurationVar(&extractCfg.Timeout, "timeout", def.Timeout, "Timeout for total work") 39 | 40 | fs := rootCmd.Flags() 41 | fs.SortFlags = false 42 | fs.StringVarP(&extractCfg.ExtractFormat, "format", "f", def.ExtractFormat, "Output format of the extraction. Valid values are 'pot' and 'json'.") 43 | fs.StringVarP(&extractCfg.SourceDir, "directory", "D", def.SourceDir, "Directory with the Go source files") 44 | fs.StringVarP(&extractCfg.OutputDir, "output-dir", "p", def.OutputDir, "Directory in which the pot files are stored.") 45 | fs.StringVarP(&extractCfg.OutputFile, "output", "o", def.OutputFile, "Write output to specified file") 46 | fs.StringSliceVarP(&extractCfg.CommentPrefixes, "add-comments", "c", def.CommentPrefixes, "Place comment blocks starting with TAG and preceding keyword lines in output file") 47 | fs.BoolVarP(&extractCfg.ExtractErrors, "extract-errors", "e", def.ExtractErrors, "Strings from errors.New(STRING) are extracted") 48 | fs.StringVar(&extractCfg.ErrorContext, "errors-context", def.ErrorContext, "Context which is automatically assigned to extracted errors") 49 | 50 | fs.StringArrayVarP(&extractCfg.TemplatePatterns, "template-directory", "t", []string{}, "Set a list of paths to which the template files contain. Regular expressions can be used.") 51 | fs.String("template-prefix", "", "Sets a prefix for the translation functions, which is used within the templates") 52 | fs.BoolVar(&extractCfg.TmplIsMonolingual, "template-use-kv", false, "Determines whether the strings from templates should be handled as key-value") 53 | fs.StringArrayP("template-keyword", "k", []string{}, "Sets a keyword that is used within templates to identify translation functions") 54 | 55 | fs.StringVarP(&extractCfg.DefaultDomain, "default-domain", "d", def.DefaultDomain, "Use name.pot for output (instead of messages.pot)") 56 | fs.BoolVar(&extractCfg.WriteNoLocation, "no-location", def.WriteNoLocation, "Do not write '#: filename:line' lines") 57 | 58 | fs.IntVarP(&extractCfg.WrapWidth, "width", "w", def.WrapWidth, "Set output page width") 59 | fs.BoolVar(&extractCfg.DontWrap, "no-wrap", def.DontWrap, "Do not break long message lines, longer than the output page width, into several lines") 60 | 61 | fs.BoolVar(&extractCfg.OmitHeader, "omit-header", def.OmitHeader, "Don't write header with 'msgid \"\"' entry") 62 | fs.StringVar(&extractCfg.CopyrightHolder, "copyright-holder", def.CopyrightHolder, "Set copyright holder in output") 63 | fs.StringVar(&extractCfg.PackageName, "package-name", def.PackageName, "Set package name in output") 64 | fs.StringVar(&extractCfg.BugsAddress, "msgid-bugs-address", def.BugsAddress, "Set report address for msgid bugs") 65 | fs.StringSliceVarP(&extractCfg.LoadedPackages, "loaded-packages", "l", []string{}, "List of packages divided by comma to search for translations") 66 | } 67 | 68 | func initVersionNumber() { 69 | // If already set via ldflags, the value is retained. 70 | if Version != "" { 71 | return 72 | } 73 | 74 | info, available := debug.ReadBuildInfo() 75 | if available { 76 | Version = info.Main.Version 77 | } else { 78 | Version = "dev" 79 | } 80 | 81 | rootCmd.Version = Version 82 | } 83 | 84 | func extractCmdF(cmd *cobra.Command, args []string) { 85 | validateExtractConfig(cmd) 86 | extractCfg.Args = args 87 | 88 | extractor := NewExtractor() 89 | extractor.extract() 90 | } 91 | 92 | func validateExtractConfig(cmd *cobra.Command) { 93 | fs := cmd.Flags() 94 | if keywordPrefix, errP := fs.GetString("template-prefix"); errP != nil { 95 | log.WithError(errP).Fatal("Args could not be parsed") 96 | } else if keywordPrefix != "" { 97 | extractCfg.Keywords = tmpl.DefaultKeywords(keywordPrefix, extractCfg.TmplIsMonolingual) 98 | } 99 | 100 | if rawKeywords, err := fs.GetStringArray("template-keyword"); err != nil { 101 | log.WithError(err).Fatal("Args could not be parsed") 102 | } else { 103 | for _, raw := range rawKeywords { 104 | if kw, errKw := tmpl.ParseKeywords(raw, extractCfg.TmplIsMonolingual); errKw != nil { 105 | log.WithError(errKw).Fatalf("Arg could not be parsed %s", raw) 106 | } else { 107 | extractCfg.Keywords = append(extractCfg.Keywords, kw) 108 | } 109 | } 110 | } 111 | 112 | if err := extractCfg.Prepare(); err != nil { 113 | log.Fatalf("Configuration could not be processed: %v", err) 114 | } 115 | 116 | if extractCfg.IsVerbose { 117 | log.SetLevel(log.DebugLevel) 118 | } 119 | 120 | log.Debug("Starting execution...") 121 | } 122 | -------------------------------------------------------------------------------- /extract/loader/package_loader.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "go/ast" 8 | "path/filepath" 9 | "strings" 10 | "time" 11 | 12 | "github.com/mattn/go-zglob" 13 | "github.com/sirupsen/logrus" 14 | "golang.org/x/tools/go/ast/inspector" 15 | "golang.org/x/tools/go/packages" 16 | 17 | "github.com/vorlif/xspreak/config" 18 | "github.com/vorlif/xspreak/extract" 19 | "github.com/vorlif/xspreak/tmpl" 20 | "github.com/vorlif/xspreak/util" 21 | ) 22 | 23 | var loadMode = packages.NeedName | 24 | packages.NeedFiles | 25 | packages.NeedSyntax | 26 | packages.NeedTypes | 27 | packages.NeedTypesInfo | 28 | packages.NeedImports | 29 | packages.NeedDeps 30 | 31 | type PackageLoader struct { 32 | config *config.Config 33 | log *logrus.Entry 34 | } 35 | 36 | func NewPackageLoader(cfg *config.Config) *PackageLoader { 37 | return &PackageLoader{ 38 | config: cfg, 39 | log: logrus.WithField("service", "PackageLoader"), 40 | } 41 | } 42 | 43 | func (pl *PackageLoader) buildArgs() []string { 44 | args := pl.config.Args 45 | if len(args) == 0 { 46 | return []string{"./..."} 47 | } 48 | 49 | var retArgs []string 50 | for _, arg := range args { 51 | if strings.HasPrefix(arg, ".") || filepath.IsAbs(arg) { 52 | retArgs = append(retArgs, arg) 53 | } else if strings.ContainsRune(arg, filepath.Separator) { 54 | retArgs = append(retArgs, fmt.Sprintf(".%c%s", filepath.Separator, arg), arg) 55 | } else { 56 | retArgs = append(retArgs, arg) 57 | } 58 | } 59 | 60 | return retArgs 61 | } 62 | 63 | func (pl *PackageLoader) Load(ctx context.Context) (*extract.Context, error) { 64 | pkgConf := &packages.Config{ 65 | Context: ctx, 66 | Mode: loadMode, 67 | Dir: pl.config.SourceDir, 68 | Logf: logrus.WithField("service", "package-loader").Debugf, 69 | Tests: false, 70 | } 71 | 72 | originalPkgs, err := pl.loadPackages(ctx, pkgConf) 73 | if err != nil { 74 | return nil, fmt.Errorf("failed to load packages: %w", err) 75 | } 76 | 77 | // Add loaded packages from config to originalPkgs 78 | if len(pl.config.LoadedPackages) > 0 { 79 | loadedPkgs, err := loadPackagesFromDir(pkgConf, pl.config.LoadedPackages) 80 | if err != nil { 81 | return nil, fmt.Errorf("failed to load specified packages: %w", err) 82 | } 83 | originalPkgs = append(originalPkgs, loadedPkgs...) 84 | } 85 | 86 | if len(originalPkgs) == 0 { 87 | return nil, errors.New("no go files to analyze") 88 | } 89 | 90 | ret := &extract.Context{ 91 | OriginalPackages: originalPkgs, 92 | Packages: cleanPackages(originalPkgs), 93 | Config: pl.config, 94 | Log: pl.log, 95 | Definitions: make(extract.Definitions, 200), 96 | } 97 | 98 | ret.Inspector = createInspector(ret.Packages) 99 | extractDefinitions(ret) 100 | ret.CommentMaps = extractComments(ret.Packages) 101 | 102 | templateFiles, errTmpl := pl.searchTemplate() 103 | if errTmpl != nil { 104 | return nil, errTmpl 105 | } 106 | ret.Templates = templateFiles 107 | 108 | return ret, nil 109 | } 110 | 111 | func (pl *PackageLoader) loadPackages(ctx context.Context, pkgCfg *packages.Config) ([]*packages.Package, error) { 112 | args := pl.buildArgs() 113 | pl.log.Debugf("Built loader args are %s", args) 114 | 115 | pkgs, err := loadPackagesFromDir(pkgCfg, args) 116 | if err != nil { 117 | return nil, fmt.Errorf("%w failed to load with go/packages", err) 118 | } 119 | 120 | if ctx.Err() != nil { 121 | return nil, fmt.Errorf("%w timed out to load packages", ctx.Err()) 122 | } 123 | 124 | pl.debugPrintLoadedPackages(pkgs) 125 | return pkgs, nil 126 | } 127 | 128 | func (pl *PackageLoader) debugPrintLoadedPackages(pkgs []*packages.Package) { 129 | pl.log.Debugf("loaded %d pkgs", len(pkgs)) 130 | for i, pkg := range pkgs { 131 | var syntaxFiles []string 132 | for _, sf := range pkg.Syntax { 133 | syntaxFiles = append(syntaxFiles, pkg.Fset.Position(sf.Pos()).Filename) 134 | } 135 | pl.log.Debugf("Loaded pkg #%d: ID=%s GoFiles=%d Syntax=%d", 136 | i, pkg.ID, len(pkg.GoFiles), len(syntaxFiles)) 137 | } 138 | } 139 | 140 | func (pl *PackageLoader) searchTemplate() ([]*tmpl.Template, error) { 141 | defer util.TrackTime(time.Now(), "Template file search") 142 | patterns := pl.config.TemplatePatterns 143 | if len(patterns) == 0 { 144 | return []*tmpl.Template{}, nil 145 | } 146 | 147 | files := make([]*tmpl.Template, 0, len(patterns)*5) 148 | 149 | for _, pattern := range patterns { 150 | foundFiles, err := zglob.Glob(pattern) 151 | if err != nil { 152 | return nil, err 153 | } 154 | for _, file := range foundFiles { 155 | pl.log.Debugf("found template file %s", file) 156 | pathAbs, errAbs := filepath.Abs(file) 157 | if errAbs != nil { 158 | logrus.WithError(errAbs).Warn("Template could not be parsed") 159 | continue 160 | } 161 | 162 | parsed, errP := tmpl.ParseFile(pathAbs) 163 | if errP != nil { 164 | logrus.WithError(errP).Warn("Template could not be parsed") 165 | continue 166 | } 167 | 168 | files = append(files, parsed) 169 | } 170 | } 171 | 172 | pl.log.Debugf("found %d template files", len(files)) 173 | return files, nil 174 | } 175 | 176 | func createInspector(pkgs []*packages.Package) *inspector.Inspector { 177 | files := make([]*ast.File, 0, 200) 178 | for _, pkg := range pkgs { 179 | files = append(files, pkg.Syntax...) 180 | } 181 | 182 | return inspector.New(files) 183 | } 184 | 185 | func loadPackagesFromDir(pkgCfg *packages.Config, args []string) ([]*packages.Package, error) { 186 | defer util.TrackTime(time.Now(), "Loading source packages") 187 | pkgs, err := packages.Load(pkgCfg, args...) 188 | if err != nil { 189 | return nil, err 190 | } 191 | 192 | if packages.PrintErrors(pkgs) > 0 { 193 | logrus.Warn("There are files with errors, the extraction may fail") 194 | } 195 | 196 | return pkgs, nil 197 | } 198 | -------------------------------------------------------------------------------- /extract/loader/definition.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "go/ast" 5 | "go/token" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/vorlif/xspreak/config" 10 | "github.com/vorlif/xspreak/extract" 11 | "github.com/vorlif/xspreak/extract/etype" 12 | "github.com/vorlif/xspreak/util" 13 | ) 14 | 15 | func extractDefinitions(extractCtx *extract.Context) { 16 | defer util.TrackTime(time.Now(), "Extract definitions") 17 | runner := &definitionExtractorRunner{extractCtx: extractCtx} 18 | extractCtx.Inspector.Nodes(nil, runner.searchDefinitions) 19 | } 20 | 21 | type definitionExtractorRunner struct { 22 | extractCtx *extract.Context 23 | } 24 | 25 | func (de *definitionExtractorRunner) searchDefinitions(n ast.Node, push bool) bool { 26 | if !push { 27 | return true 28 | } 29 | 30 | switch v := n.(type) { 31 | case *ast.FuncDecl: 32 | de.extractFunc(v) 33 | case *ast.AssignStmt: 34 | de.extractInlineFunc(v) 35 | case *ast.GenDecl: 36 | switch v.Tok { 37 | case token.VAR: 38 | de.extractVar(v) 39 | case token.TYPE: 40 | de.extractStruct(v) 41 | } 42 | } 43 | 44 | return true 45 | } 46 | 47 | // Extracts global variables. 48 | // 49 | // Example: 50 | // 51 | // var t localize.Singular. 52 | func (de *definitionExtractorRunner) extractVar(decl *ast.GenDecl) { 53 | for _, spec := range decl.Specs { 54 | valueSpec, ok := spec.(*ast.ValueSpec) 55 | if !ok { 56 | continue 57 | } 58 | 59 | selector := util.SearchSelector(valueSpec.Type) 60 | if selector == nil { 61 | continue 62 | } 63 | 64 | tok := de.extractCtx.GetLocalizeTypeToken(selector) 65 | if tok != etype.Singular { 66 | // TODO(fv): log hint 67 | continue 68 | } 69 | 70 | for _, name := range valueSpec.Names { 71 | pkg, obj := de.extractCtx.GetType(name) 72 | if pkg == nil { 73 | continue 74 | } 75 | 76 | def := &extract.Definition{ 77 | Type: extract.VarSingular, 78 | Token: tok, 79 | Pck: pkg, 80 | Ident: name, 81 | Path: obj.Pkg().Path(), 82 | ID: util.ObjToKey(obj), 83 | Obj: obj, 84 | } 85 | 86 | de.addDefinition(def) 87 | } 88 | 89 | } 90 | 91 | } 92 | 93 | // Extracts struct fields. 94 | // 95 | // Example: 96 | // 97 | // type T struct { 98 | // sing localize.Singular 99 | // p localize.Plural 100 | // } 101 | func (de *definitionExtractorRunner) extractStruct(decl *ast.GenDecl) { 102 | for _, spec := range decl.Specs { 103 | typeSpec, ok := spec.(*ast.TypeSpec) 104 | if !ok { 105 | continue 106 | } 107 | 108 | structType, ok := typeSpec.Type.(*ast.StructType) 109 | if !ok { 110 | continue 111 | } 112 | 113 | pkg, obj := de.extractCtx.GetType(typeSpec.Name) 114 | if obj == nil { 115 | continue 116 | } 117 | 118 | if !config.ShouldScanStruct(pkg.PkgPath) { 119 | continue 120 | } 121 | 122 | for i, field := range structType.Fields.List { 123 | 124 | var tok etype.Token 125 | switch field.Type.(type) { 126 | case *ast.Ident: 127 | if pkg.PkgPath == config.SpreakLocalizePackagePath { 128 | tok = de.extractCtx.GetLocalizeTypeToken(field.Type) 129 | break 130 | } 131 | 132 | selector := util.SearchSelector(field.Type) 133 | if selector == nil { 134 | continue 135 | } 136 | 137 | tok = de.extractCtx.GetLocalizeTypeToken(selector) 138 | 139 | default: 140 | selector := util.SearchSelector(field.Type) 141 | if selector == nil { 142 | continue 143 | } 144 | 145 | tok = de.extractCtx.GetLocalizeTypeToken(selector) 146 | } 147 | 148 | if tok == etype.None { 149 | continue 150 | } 151 | 152 | for ii, fieldName := range field.Names { 153 | def := &extract.Definition{ 154 | Type: extract.StructField, 155 | Token: tok, 156 | Pck: pkg, 157 | Ident: typeSpec.Name, 158 | Path: obj.Pkg().Path(), 159 | ID: util.ObjToKey(obj), 160 | Obj: obj, 161 | FieldIdent: fieldName, 162 | FieldName: fieldName.Name, 163 | FieldPos: calculatePosIdx(ii, i), 164 | } 165 | 166 | de.addDefinition(def) 167 | } 168 | } 169 | } 170 | } 171 | 172 | // Extracts function definitions. 173 | // 174 | // Example: 175 | // 176 | // func translate(msgid localize.Singular, plural localize.Plural) {} 177 | // func getTranslation() (localize.Singular, localize.Plural) {} 178 | func (de *definitionExtractorRunner) extractFunc(decl *ast.FuncDecl) { 179 | if decl.Type == nil || decl.Type.Params == nil { 180 | return 181 | } 182 | 183 | de.extractFunctionsParams(decl.Name, decl.Type) 184 | } 185 | 186 | func (de *definitionExtractorRunner) extractInlineFunc(assign *ast.AssignStmt) { 187 | if len(assign.Lhs) == 0 || len(assign.Lhs) != len(assign.Rhs) { 188 | return 189 | } 190 | 191 | ident, ok := assign.Lhs[0].(*ast.Ident) 192 | if !ok { 193 | return 194 | } 195 | 196 | funcLit, ok := assign.Rhs[0].(*ast.FuncLit) 197 | if !ok || funcLit.Type == nil || funcLit.Type.Params == nil { 198 | return 199 | } 200 | 201 | de.extractFunctionsParams(ident, funcLit.Type) 202 | } 203 | 204 | func (de *definitionExtractorRunner) extractFunctionsParams(ident *ast.Ident, t *ast.FuncType) { 205 | pck, obj := de.extractCtx.GetType(ident) 206 | if pck == nil { 207 | return 208 | } 209 | 210 | // function call 211 | for i, param := range t.Params.List { 212 | tok, _ := de.extractCtx.SearchIdentAndToken(param) 213 | if tok == etype.None { 214 | continue 215 | } 216 | 217 | if len(param.Names) == 0 { 218 | def := &extract.Definition{ 219 | Type: extract.FunctionParam, 220 | Token: tok, 221 | Pck: pck, 222 | Ident: ident, 223 | Path: obj.Pkg().Path(), 224 | ID: util.ObjToKey(obj), 225 | Obj: obj, 226 | FieldIdent: nil, 227 | FieldName: strconv.Itoa(i), 228 | 229 | FieldPos: i, 230 | IsVariadic: isEllipsis(param.Type), 231 | } 232 | de.addDefinition(def) 233 | } 234 | 235 | for ii, name := range param.Names { 236 | def := &extract.Definition{ 237 | Type: extract.FunctionParam, 238 | Token: tok, 239 | Pck: pck, 240 | Ident: ident, 241 | Path: obj.Pkg().Path(), 242 | ID: util.ObjToKey(obj), 243 | Obj: obj, 244 | FieldIdent: name, 245 | FieldName: name.Name, 246 | IsVariadic: isEllipsis(param.Type), 247 | 248 | FieldPos: calculatePosIdx(i, ii), 249 | } 250 | de.addDefinition(def) 251 | } 252 | } 253 | } 254 | 255 | func (de *definitionExtractorRunner) addDefinition(d *extract.Definition) { 256 | key := d.Key() 257 | if _, ok := de.extractCtx.Definitions[key]; !ok { 258 | de.extractCtx.Definitions[key] = make(map[string]*extract.Definition) 259 | } 260 | 261 | de.extractCtx.Definitions[key][d.FieldName] = d 262 | } 263 | 264 | func isEllipsis(node ast.Node) bool { 265 | switch node.(type) { 266 | case *ast.Ellipsis: 267 | return true 268 | default: 269 | return false 270 | } 271 | } 272 | 273 | func calculatePosIdx(first, second int) int { 274 | if first > 0 { 275 | if second > 0 { 276 | return first * second 277 | } 278 | 279 | return first 280 | } 281 | 282 | return second 283 | } 284 | -------------------------------------------------------------------------------- /extract/context.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "go/ast" 5 | "go/constant" 6 | "go/token" 7 | "go/types" 8 | "strconv" 9 | "strings" 10 | 11 | log "github.com/sirupsen/logrus" 12 | "golang.org/x/tools/go/ast/inspector" 13 | "golang.org/x/tools/go/packages" 14 | 15 | "github.com/vorlif/xspreak/config" 16 | "github.com/vorlif/xspreak/extract/etype" 17 | "github.com/vorlif/xspreak/tmpl" 18 | "github.com/vorlif/xspreak/util" 19 | ) 20 | 21 | // Context holds all the important information needed to scan the program code and template files. 22 | type Context struct { 23 | Config *config.Config 24 | Log *log.Entry 25 | 26 | // OriginalPackages contains the packages that were loaded by golang.org/x/tools/go/packages. 27 | // It contains the packages of the scanned directory and may contain duplicates. 28 | OriginalPackages []*packages.Package 29 | 30 | // Packages contains the packages that are of interest to us 31 | // 32 | // In addition include: 33 | // * Packages of the scanned directory 34 | // * Packages of the Spreak library 35 | Packages []*packages.Package 36 | 37 | // Inspector to run through the AST of all Packages. 38 | Inspector *inspector.Inspector 39 | 40 | // CommentMaps for quick access to comments. 41 | CommentMaps Comments 42 | 43 | Definitions Definitions 44 | 45 | Templates []*tmpl.Template 46 | } 47 | 48 | func (c *Context) GetPosition(pos token.Pos) token.Position { 49 | for _, pkg := range c.Packages { 50 | if position := pkg.Fset.Position(pos); position.IsValid() { 51 | return position 52 | } 53 | } 54 | 55 | return token.Position{} 56 | } 57 | 58 | func (c *Context) GetType(ident *ast.Ident) (*packages.Package, types.Object) { 59 | for _, pkg := range c.Packages { 60 | if pkg.Types == nil { 61 | continue 62 | } 63 | if obj, ok := pkg.TypesInfo.Defs[ident]; ok { 64 | if obj == nil || obj.Type() == nil || obj.Pkg() == nil { 65 | return nil, nil 66 | } 67 | return pkg, obj 68 | } 69 | if obj, ok := pkg.TypesInfo.Uses[ident]; ok { 70 | if obj == nil || obj.Type() == nil || obj.Pkg() == nil { 71 | return nil, nil 72 | } 73 | return pkg, obj 74 | } 75 | if obj, ok := pkg.TypesInfo.Implicits[ident]; ok { 76 | if obj == nil || obj.Type() == nil || obj.Pkg() == nil { 77 | return nil, nil 78 | } 79 | return pkg, obj 80 | } 81 | } 82 | return nil, nil 83 | } 84 | 85 | func (c *Context) GetLocalizeTypeToken(expr ast.Expr) etype.Token { 86 | if expr == nil { 87 | return etype.None 88 | } 89 | 90 | switch v := expr.(type) { 91 | case *ast.SelectorExpr: 92 | return c.GetLocalizeTypeToken(v.Sel) 93 | case *ast.Ident: 94 | _, vType := c.GetType(v) 95 | if vType == nil { 96 | return etype.None 97 | } 98 | 99 | if vType.Pkg() == nil || vType.Pkg().Path() != config.SpreakLocalizePackagePath { 100 | return etype.None 101 | } 102 | 103 | tok, ok := etype.StringExtractNames[vType.Name()] 104 | if !ok { 105 | return etype.None 106 | } 107 | 108 | return tok 109 | default: 110 | return etype.None 111 | } 112 | } 113 | 114 | func (c *Context) SearchIdent(start ast.Node) *ast.Ident { 115 | switch v := start.(type) { 116 | case *ast.Ident: 117 | pkg, _ := c.GetType(v) 118 | if pkg != nil { 119 | return v 120 | } 121 | case *ast.SelectorExpr: 122 | pkg, _ := c.GetType(v.Sel) 123 | if pkg != nil { 124 | return v.Sel 125 | } 126 | 127 | return c.SearchIdent(v.X) 128 | case *ast.StarExpr: 129 | return c.SearchIdent(v.X) 130 | } 131 | 132 | return nil 133 | } 134 | 135 | func (c *Context) SearchIdentAndToken(start ast.Node) (etype.Token, *ast.Ident) { 136 | switch val := start.(type) { 137 | case *ast.Ident: 138 | if tok := c.GetLocalizeTypeToken(val); tok != etype.None { 139 | return tok, val 140 | } 141 | 142 | pkg, obj := c.GetType(val) 143 | if pkg == nil { 144 | break 145 | } 146 | 147 | if def := c.Definitions.Get(util.ObjToKey(obj), ""); def != nil { 148 | return def.Token, val 149 | } 150 | case *ast.StarExpr: 151 | tok, ident := c.SearchIdentAndToken(val.X) 152 | if ident != nil { 153 | pkg, _ := c.GetType(ident) 154 | if pkg != nil { 155 | return tok, ident 156 | } 157 | } 158 | } 159 | 160 | selector := util.SearchSelector(start) 161 | if selector == nil { 162 | return etype.None, nil 163 | } 164 | 165 | switch ident := selector.X.(type) { 166 | case *ast.Ident: 167 | if tok := c.GetLocalizeTypeToken(ident); tok != etype.None { 168 | return tok, ident 169 | } 170 | 171 | pkg, obj := c.GetType(ident) 172 | if pkg == nil { 173 | break 174 | } 175 | 176 | if def := c.Definitions.Get(util.ObjToKey(obj), ""); def != nil { 177 | return def.Token, ident 178 | } 179 | if def := c.Definitions.Get(util.ObjToKey(obj), selector.Sel.Name); def != nil { 180 | return def.Token, ident 181 | } 182 | 183 | if obj.Type() == nil { 184 | break 185 | } 186 | } 187 | 188 | if tok := c.GetLocalizeTypeToken(selector.Sel); tok != etype.None { 189 | return tok, selector.Sel 190 | } 191 | 192 | pkg, obj := c.GetType(selector.Sel) 193 | if pkg == nil { 194 | return etype.None, nil 195 | } 196 | 197 | if def := c.Definitions.Get(util.ObjToKey(obj), ""); def != nil { 198 | return def.Token, selector.Sel 199 | } 200 | if def := c.Definitions.Get(util.ObjToKey(obj), selector.Sel.Name); def != nil { 201 | return def.Token, selector.Sel 202 | } 203 | 204 | return etype.None, nil 205 | } 206 | 207 | type SearchResult struct { 208 | Raw string 209 | Node ast.Node 210 | } 211 | 212 | func (c *Context) SearchStrings(startExpr ast.Expr) []*SearchResult { 213 | results := make([]*SearchResult, 0) 214 | visited := make(map[ast.Node]bool) 215 | 216 | // String was created at the current position 217 | extracted, originNode := StringLiteral(startExpr) 218 | if extracted != "" { 219 | results = append(results, &SearchResult{Raw: extracted, Node: originNode}) 220 | visited[originNode] = true 221 | } 222 | 223 | // Backtracking the string 224 | startIdent, ok := startExpr.(*ast.Ident) 225 | if !ok { 226 | return results 227 | } 228 | 229 | if startIdent.Obj == nil { 230 | _, obj := c.GetType(startIdent) 231 | if constObj, ok := obj.(*types.Const); ok && constObj.Val().Kind() == constant.String { 232 | if stringVal := constant.StringVal(constObj.Val()); stringVal != "" { 233 | results = append(results, &SearchResult{Raw: stringVal, Node: originNode}) 234 | } 235 | } 236 | return results 237 | } 238 | 239 | c.Inspector.WithStack([]ast.Node{&ast.AssignStmt{}}, func(raw ast.Node, _ bool, _ []ast.Node) (proceed bool) { 240 | proceed = false 241 | 242 | node := raw.(*ast.AssignStmt) 243 | if len(node.Lhs) != len(node.Rhs) || len(node.Lhs) == 0 { 244 | return 245 | } 246 | 247 | for i, left := range node.Lhs { 248 | leftIdent, isIdent := left.(*ast.Ident) 249 | if !isIdent { 250 | continue 251 | } 252 | if leftIdent.Obj != startIdent.Obj { 253 | continue 254 | } 255 | 256 | if visited[node.Rhs[i]] { 257 | continue 258 | } 259 | 260 | extracted, originNode = StringLiteral(node.Rhs[i]) 261 | if extracted != "" { 262 | visited[node.Rhs[i]] = true 263 | results = append(results, &SearchResult{Raw: extracted, Node: originNode}) 264 | } 265 | 266 | } 267 | return 268 | }) 269 | 270 | return results 271 | } 272 | 273 | // GetComments extracts the Go comments for a list of nodes. 274 | func (c *Context) GetComments(pkg *packages.Package, node ast.Node) []string { 275 | var comments []string 276 | 277 | pkgComments, pkgHashComments := c.CommentMaps[pkg.PkgPath] 278 | if !pkgHashComments { 279 | return comments 280 | } 281 | 282 | pos := c.GetPosition(node.Pos()) 283 | 284 | fileComments, fileHasComments := pkgComments[pos.Filename] 285 | if !fileHasComments { 286 | return comments 287 | } 288 | 289 | visited := make(map[*ast.CommentGroup]bool) 290 | 291 | c.Inspector.WithStack([]ast.Node{node}, func(n ast.Node, _ bool, stack []ast.Node) (proceed bool) { 292 | proceed = false 293 | // Search stack for our node 294 | if n != node { 295 | return 296 | } 297 | 298 | // Find the first node of the line 299 | var topNode = node 300 | for i := len(stack) - 1; i >= 0; i-- { 301 | entry := stack[i] 302 | entryPos := c.GetPosition(entry.Pos()) 303 | if !entryPos.IsValid() || entryPos.Line < pos.Line { 304 | break 305 | } 306 | 307 | topNode = entry 308 | } 309 | 310 | // Search for all comments for this line 311 | ast.Inspect(topNode, func(node ast.Node) bool { 312 | nodeComments := fileComments[node] 313 | for _, comment := range nodeComments { 314 | if visited[comment] { 315 | continue 316 | } 317 | 318 | visited[comment] = true 319 | comments = append(comments, comment.Text()) 320 | } 321 | return true 322 | }) 323 | return 324 | }) 325 | 326 | return comments 327 | } 328 | 329 | // StringLiteral extracts and concatenates string literals from an AST expression. 330 | // It returns the extracted string and the originating AST node. 331 | func StringLiteral(expr ast.Expr) (string, ast.Node) { 332 | stack := []ast.Expr{expr} 333 | var b strings.Builder 334 | var elem ast.Expr 335 | 336 | for len(stack) != 0 { 337 | n := len(stack) - 1 338 | elem = stack[n] 339 | stack = stack[:n] 340 | 341 | switch v := elem.(type) { 342 | // Simple string with quotes or backquotes 343 | case *ast.BasicLit: 344 | if v.Kind != token.STRING { 345 | continue 346 | } 347 | 348 | if unqouted, err := strconv.Unquote(v.Value); err != nil { 349 | b.WriteString(v.Value) 350 | } else { 351 | b.WriteString(unqouted) 352 | } 353 | // Concatenation of several string literals 354 | case *ast.BinaryExpr: 355 | if v.Op != token.ADD { 356 | continue 357 | } 358 | stack = append(stack, v.Y, v.X) 359 | case *ast.Ident: 360 | if v.Obj == nil { 361 | continue 362 | } 363 | switch z := v.Obj.Decl.(type) { 364 | case *ast.ValueSpec: 365 | if len(z.Values) == 0 { 366 | continue 367 | } 368 | stack = append(stack, z.Values[0]) 369 | case *ast.AssignStmt: 370 | if len(z.Rhs) == 0 { 371 | continue 372 | } 373 | stack = append(stack, z.Rhs...) 374 | } 375 | default: 376 | continue 377 | } 378 | } 379 | 380 | return b.String(), elem 381 | } 382 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | xspreak is the command line program for extracting strings for the [spreak library](https://github.com/vorlif/spreak). 2 | 3 | # xspreak ![Test status](https://github.com/vorlif/xspreak/workflows/Test/badge.svg) [![MIT license](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) 4 | 5 | xspreak automatically extracts strings that use a string alias from 6 | the [`localize` package](https://pkg.go.dev/github.com/vorlif/spreak/localize). 7 | The extracted strings are stored in a `.pot` or `.json` file and can then be easily translated. 8 | 9 | The extracted strings can then be passed to a [Localizer](https://pkg.go.dev/github.com/vorlif/spreak#Localizer) 10 | or a [KeyLocalizer](https://pkg.go.dev/github.com/vorlif/spreak#KeyLocalizer) which returns the matching translation. 11 | 12 | Example: 13 | 14 | ```go 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | 20 | "golang.org/x/text/language" 21 | 22 | "github.com/vorlif/spreak" 23 | "github.com/vorlif/spreak/localize" 24 | ) 25 | 26 | // This string is extracted because the type is localize.Singular 27 | var ApplicationName localize.Singular = "Beautiful app" 28 | 29 | func main() { 30 | bundle, err := spreak.NewBundle( 31 | spreak.WithSourceLanguage(language.English), 32 | spreak.WithDomainPath(spreak.NoDomain, "../locale"), 33 | spreak.WithLanguage(language.German, language.Spanish, language.Chinese), 34 | ) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | t := spreak.NewLocalizer(bundle, language.Spanish) 40 | 41 | // Message lookup of the extracted string 42 | fmt.Println(t.Get(ApplicationName)) 43 | // Output: 44 | // Hermosa app 45 | } 46 | ``` 47 | 48 | ## Requirements 49 | 50 | * Your project must be a go module (must have a `go.mod` and `go.sum`) 51 | * The dependencies of your project must be installed `go mod tidy` 52 | * xspreak searches all files for strings to extract. This can take a lot of memory or CPU time for larger projects. 53 | 54 | ## How to install 55 | 56 | Download a [pre-built binary from the releases](https://github.com/vorlif/xspreak/releases/latest) or create it from 57 | source: 58 | 59 | ```bash 60 | go install github.com/vorlif/xspreak@latest 61 | ``` 62 | 63 | Tests installation with: 64 | 65 | ```bash 66 | xspreak --help 67 | ``` 68 | 69 | ## What can be extracted? 70 | 71 | ### spreak functions calls 72 | 73 | All function calls of spreak translation functions, where a string is passed, are extracted. 74 | 75 | ```go 76 | t.Get("Hello world") 77 | t.Nget("Hello world", "Hello worlds") 78 | // ... 79 | ``` 80 | 81 | ### Global variables and constants 82 | 83 | Global variables and constants are extracted if the type is `localize.Singular` or `localize.MsgID`. 84 | Thereby `localize.Singular` and `localize.MsgID` are always equivalent and can be used synonymously. 85 | 86 | ```go 87 | package main 88 | 89 | import "github.com/vorlif/spreak/localize" 90 | 91 | const Weekday localize.Singular = "weekday" 92 | 93 | var ApplicationName localize.Singular = "app" 94 | ``` 95 | 96 | ### Local variables 97 | 98 | Local variables are extracted if the type is `localize.Singular` or `localize.MsgID`. 99 | 100 | ```go 101 | package main 102 | 103 | import "github.com/vorlif/spreak/localize" 104 | 105 | func init() { 106 | holiday := localize.Singular("Christmas") 107 | } 108 | ``` 109 | 110 | ### Variable assignments 111 | 112 | Assignments to variables are extracted if the type is `localize.Singular` or `localize.MsgID`. 113 | 114 | ```go 115 | package main 116 | 117 | import "github.com/vorlif/spreak/localize" 118 | 119 | var ApplicationName = "app" 120 | 121 | func init() { 122 | var holiday localize.Singular 123 | 124 | holiday = "Mother's Day" 125 | 126 | ApplicationName = "App for you" 127 | } 128 | ``` 129 | 130 | ### Argument of function calls 131 | 132 | Function calls to **global functions** are extracted if the parameter type is from the `localize` package. 133 | The parameters of a function are grouped together to form a message. 134 | Thus, a message can be created with singular, plural, a context and a domain. 135 | 136 | ```go 137 | package main 138 | 139 | import "github.com/vorlif/spreak/localize" 140 | 141 | func noop(name localize.Singular, plural localize.Plural, ctx localize.Context) {} 142 | 143 | func init() { 144 | // Extracted as a message with singular, plural and a context 145 | noop("I have %d car", "I have %d cars", "cars") 146 | } 147 | ``` 148 | 149 | ### Return values of functions 150 | 151 | Return values of functions are extracted if the parameter type is from the `localize` package. 152 | The parameters of a function are grouped together to form a message. 153 | Thus, a message can be created with singular, plural, a context and a domain. 154 | Named return values are currently not supported. 155 | 156 | ```go 157 | package main 158 | 159 | import "github.com/vorlif/spreak/localize" 160 | 161 | func noop() (localize.Singular, localize.Plural, localize.Context) { 162 | // Extracted as a message with singular, plural and a context 163 | return "I have %d car", "I have %d cars", "cars" 164 | } 165 | ``` 166 | 167 | ### Attributes at struct initialization 168 | 169 | Struct initializations are extracted if the struct was **defined globally** and 170 | the attribute type *comes from the `localize` package*. 171 | The attributes of a struct are grouped together to create a message. 172 | Thus, a message can be created with singular, plural, a context and a domain. 173 | 174 | ```go 175 | package main 176 | 177 | import "github.com/vorlif/spreak/localize" 178 | 179 | type MyMessage struct { 180 | // Defined as singular and plural 181 | Text localize.Singular 182 | Plural localize.Plural 183 | Tmp string 184 | } 185 | 186 | func main() { 187 | msg := &MyMessage{ 188 | // Extracted as a message with singular and plural 189 | Text: "Hello planet", 190 | Plural: "Hello planets", 191 | 192 | // not extracted - type string 193 | Tmp: "tmp", 194 | } 195 | } 196 | ``` 197 | 198 | ### Values from an array initialization 199 | 200 | Arrays are extracted if the type is `localize.Singular` or a struct that contains parameter 201 | types from the `localize` package. 202 | 203 | ```go 204 | package main 205 | 206 | import "github.com/vorlif/spreak/localize" 207 | 208 | var weekdays = []localize.MsgID{"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"} 209 | 210 | type MyMessage struct { 211 | Text localize.Singular 212 | Plural localize.Plural 213 | Tmp string 214 | } 215 | 216 | func main() { 217 | animals := []MyMessage{ 218 | {Text: "%d dog", Plural: "%d dogs"}, 219 | {Text: "%d cat", Plural: "%d cat"}, 220 | {Text: "%d horse", Plural: "%d horses"}, 221 | } 222 | } 223 | ``` 224 | 225 | ### Values from a map initialization 226 | 227 | During map initialization keys and values are extracted, 228 | if the type is `localize.Singular` or a struct that contains parameter types from the `localize` package. 229 | 230 | ```go 231 | package main 232 | 233 | import "github.com/vorlif/spreak/localize" 234 | 235 | var weekdays = map[localize.MsgID]int{ 236 | "Monday": 1, 237 | "Tuesday": 2, 238 | } 239 | 240 | var reverseWeekdays = map[int]localize.Singular{ 241 | 1: "Monday", 242 | 2: "Tuesday", 243 | } 244 | 245 | ``` 246 | 247 | ### Error texts 248 | 249 | Strings can be extracted from `errors.New` if xspreak is called with the `-e` option. 250 | 251 | ```go 252 | package main 253 | 254 | import "errors" 255 | 256 | var ErrInvalidAnimal = errors.New("this is not a valid animal") 257 | 258 | ``` 259 | 260 | ### Comments 261 | 262 | Comments can be left for translators. 263 | These are extracted, stored in the `.pot` file and displayed to the translator. 264 | For JSON files the comments are ignored. 265 | 266 | ```go 267 | package main 268 | 269 | import "github.com/vorlif/spreak/localize" 270 | 271 | // TRANSLATORS: This comment is automatically extracted by xspreak 272 | // and can be used to leave useful hints for the translators. 273 | // 274 | // This comment is not extracted because a blank line was inserted above it. 275 | const InvalidName localize.Singular = "The name has an invalid format" 276 | ``` 277 | 278 | ### Exclude from extraction 279 | 280 | Strings can be ignored. 281 | 282 | ```go 283 | package main 284 | 285 | import "github.com/vorlif/spreak/localize" 286 | 287 | // xspreak: ignore 288 | const MagicName localize.Singular = ".%$($§($(%" 289 | ``` 290 | 291 | ### Templates (Experimental) 292 | 293 | With `-t` a template directory can be specified. 294 | 295 | * `-t "path/to/templates/*.html"`: Scans all HTML files in the `templates` directory 296 | * `-t "path/to/templates/**/*.html"` Scans all HTML files in the `templates` directory 297 | and in subdirectories of the templates directory. 298 | 299 | With `--template-prefix` you can specify a prefix for the template function. 300 | 301 | For example, with `--template-prefix "T"` the following function calls are extracted 302 | 303 | ```text 304 | {{.T.Get "Hello world"}} 305 | {{.T.NGet "I see a planet" "I see planets" 2}} 306 | ``` 307 | 308 | Two messages are extracted here: 309 | 310 | * Message 1 311 | * Singular `Hello world` 312 | * Message 2 313 | * Singular `I see a planet` 314 | * Plural `I see planets` 315 | 316 | Instead of `--template-prefix` you can also use `-k` to define your own keywords. 317 | The definition follows 318 | the [xgettext notation](https://www.gnu.org/software/gettext/manual/html_node/xgettext-Invocation.html), 319 | but only applies to templates. 320 | 321 | The default is: `.T.Get .T.Getf .T.DGet:1d,2 .T.DGetf:1d,2 .T.NGet:1,2 .T.NGetf:1,2 .T.DNGet:1d,2,3 .T.DNGetf:1d,2,3 322 | .T.PGet:1c,2 .T.PGetf:1c,2 .T.DPGet:1d,2c,3 .T.DPGetf:1d,2c,3 .T.NPGet:1c,2,3 .T.NPGetf:1c,2,3 .T.DNPGet:1d,2c,3,4 323 | .T.DNPGetf:1d,2c,3,4` 324 | 325 | Inline templates must be marked with `xspreak: template`: 326 | 327 | ```go 328 | package main 329 | 330 | // xspreak: template 331 | const tmpl = `{{.T.Get "Hello"}}` 332 | ``` 333 | 334 | There is also a [detailed example](https://github.com/vorlif/spreak/tree/main/examples/features/httptempl) how to use 335 | spreak with templates and your own keywords. 336 | 337 | ### Variables tracing 338 | 339 | For `localize.Singular` and `localize.MsgID`, variable tracing is supported for simple cases. 340 | The following example extracts two strings. One string with "Yes" and one string with "No, better a beer". 341 | 342 | ```go 343 | package main 344 | 345 | import "github.com/vorlif/spreak/localize" 346 | 347 | func WantCoffee() localize.Singular { 348 | answer := "Yes" 349 | 350 | switch time.Now().Weekday() { 351 | case time.Friday, time.Saturday, time.Sunday: 352 | answer = "No, better a beer" 353 | } 354 | 355 | return answer 356 | } 357 | ``` 358 | 359 | Whether tracing works for a particular use case should be verified separately for each use case. 360 | 361 | ### Using monolingual format (e.g. Key-Value) 362 | 363 | To use monolingual format, the following changes must be made. 364 | 365 | 1. Instead of `localize.Singular` and `localize.MsgID` use `localize.Key`. 366 | 2. If a key also has a plural form, `localize.PluralKey` must be used. 367 | 3. When extracting templates `--template-use-kv` must be added 368 | 4. If a separate key is defined for templates, the position of singular and plural must be identical for plural: 369 | ```shell 370 | # Example 371 | xspreak -f json -D ./ -p locale/ -k "i18n.TrN:1,1" --template-use-kv -t "templates/*.html" 372 | ``` 373 | 374 | All of the above functions also apply to `localize.Key` and `localize.PluralKey`. 375 | 376 | 5. Use [KeyLocalizer](https://pkg.go.dev/github.com/vorlif/spreak#KeyLocalizer) instead of Localizer in the code 377 | 378 | ### Supported export formats 379 | 380 | 1. `po`/`pot` (Default) `xspreak ...` 381 | 2. `json`: `xspreak -f json ...` 382 | --------------------------------------------------------------------------------