├── httputil ├── testdata │ ├── empty.html │ ├── euc-jp.html │ └── euc-jp.nohint.html ├── errhandler.go ├── limitedtransport.go ├── limitedtransport_test.go ├── httputil.go ├── charsettransport_test.go └── charsettransport.go ├── chardet ├── testdata │ ├── utf-8.txt │ ├── euc-jp.txt │ └── shift_jis.txt ├── chardet_test.go ├── LICENSE_charset.txt ├── chardet.go └── charset.go ├── README.md ├── roundtime ├── duration.go └── duration_test.go ├── .github └── workflows │ └── test.yml ├── ctxlog ├── ctxlog.go ├── ctxlog_test.go ├── ctxlog_pre17.go ├── ctxlog_17.go └── ctxlog_appengine.go ├── go.mod ├── logwriter ├── logwriter_test.go └── logwriter.go ├── gomockutil └── matchers │ ├── matchers_test.go │ └── matchers.go ├── LICENSE ├── broadcastwriter ├── broadcastwriter_test.go └── broadcastwriter.go ├── minipipeline ├── pipeline_test.go └── pipeline.go ├── bytesutil ├── transformer_test.go └── transformer.go ├── giturl ├── giturl.go └── giturl_test.go ├── netutil ├── blocklist_test.go └── blocklist.go ├── urlutil ├── normalize_test.go └── normalize.go ├── oauth2util └── oauth2util.go ├── stringstringmap ├── stringstringmap_test.go └── stringstringmap.go └── go.sum /httputil/testdata/empty.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chardet/testdata/utf-8.txt: -------------------------------------------------------------------------------- 1 | ファッション誌 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-nuts 2 | Go code I want sometimes but too small to be a separate library 3 | -------------------------------------------------------------------------------- /chardet/testdata/euc-jp.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/motemen/go-nuts/HEAD/chardet/testdata/euc-jp.txt -------------------------------------------------------------------------------- /chardet/testdata/shift_jis.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/motemen/go-nuts/HEAD/chardet/testdata/shift_jis.txt -------------------------------------------------------------------------------- /httputil/testdata/euc-jp.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/motemen/go-nuts/HEAD/httputil/testdata/euc-jp.html -------------------------------------------------------------------------------- /httputil/testdata/euc-jp.nohint.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/motemen/go-nuts/HEAD/httputil/testdata/euc-jp.nohint.html -------------------------------------------------------------------------------- /httputil/errhandler.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type ErrorHandlerFunc func(http.ResponseWriter, *http.Request) error 8 | 9 | func (f ErrorHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { 10 | err := f(w, r) 11 | if err != nil { 12 | http.Error(w, err.Error(), http.StatusInternalServerError) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /roundtime/duration.go: -------------------------------------------------------------------------------- 1 | package roundtime 2 | 3 | import ( 4 | "math" 5 | "time" 6 | ) 7 | 8 | func Duration(d time.Duration, dicimal int) time.Duration { 9 | shift := int(math.Pow10(dicimal)) 10 | 11 | units := []time.Duration{time.Second, time.Millisecond, time.Microsecond, time.Nanosecond} 12 | for _, u := range units { 13 | if d > u { 14 | div := u / time.Duration(shift) 15 | if div == 0 { 16 | break 17 | } 18 | d = d / div * div 19 | break 20 | } 21 | } 22 | 23 | return d 24 | } 25 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches: [ master ] 7 | 8 | jobs: 9 | 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | 15 | - name: staticcheck 16 | uses: reviewdog/action-staticcheck@v1 17 | with: 18 | reporter: github-check 19 | 20 | - name: Set up Go 21 | uses: actions/setup-go@v2 22 | with: 23 | go-version: 1.16 24 | 25 | - name: Build 26 | run: go build -v ./... 27 | 28 | - name: Test 29 | run: go test -v ./... 30 | -------------------------------------------------------------------------------- /ctxlog/ctxlog.go: -------------------------------------------------------------------------------- 1 | package ctxlog 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "os" 7 | "sync" 8 | ) 9 | 10 | var Logger = log.New(os.Stderr, "", log.LstdFlags) 11 | 12 | var ( 13 | outputMu sync.Mutex 14 | //lint:ignore U1000 required for ctxlog_17.go 15 | output io.Writer = os.Stderr 16 | ) 17 | 18 | type contextKey struct { 19 | name string 20 | } 21 | 22 | func SetOutput(w io.Writer) { 23 | outputMu.Lock() 24 | defer outputMu.Unlock() 25 | Logger.SetOutput(w) 26 | output = w 27 | } 28 | 29 | var LoggerContextKey = &contextKey{"logger"} 30 | var PrefixContextKey = &contextKey{"prefix"} 31 | -------------------------------------------------------------------------------- /httputil/limitedtransport.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | ) 7 | 8 | type LimitedTransport struct { 9 | Base http.RoundTripper 10 | N int64 11 | } 12 | 13 | func (t *LimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) { 14 | base := t.Base 15 | if base == nil { 16 | base = http.DefaultTransport 17 | } 18 | 19 | resp, err := base.RoundTrip(req) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | resp.Body = readCloser{ 25 | Reader: io.LimitReader(resp.Body, t.N), 26 | Closer: resp.Body, 27 | } 28 | return resp, nil 29 | } 30 | -------------------------------------------------------------------------------- /chardet/chardet_test.go: -------------------------------------------------------------------------------- 1 | package chardet 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestDetectEncoding(t *testing.T) { 14 | ff, _ := filepath.Glob("testdata/*.txt") 15 | detector := NewDetector(WithLanguage("ja", "")) 16 | for _, f := range ff { 17 | filename := filepath.Base(f) 18 | t.Run(filename, func(t *testing.T) { 19 | b, _ := os.ReadFile(f) 20 | enc, name := detector.DetectEncoding(b) 21 | if assert.NotEqual(t, "", name) { 22 | return 23 | } 24 | assert.Equal( 25 | t, 26 | strings.ToLower(strings.TrimSuffix(filename, ".txt")), 27 | strings.Replace(strings.ToLower(fmt.Sprint(enc)), " ", "_", -1), 28 | ) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ctxlog/ctxlog_test.go: -------------------------------------------------------------------------------- 1 | // +build go1.7 2 | 3 | package ctxlog 4 | 5 | import ( 6 | "testing" 7 | 8 | "bytes" 9 | "context" 10 | ) 11 | 12 | func testPrefix(t *testing.T, ctx context.Context, expect string) { 13 | var buf bytes.Buffer 14 | 15 | logger := LoggerFromContext(ctx) 16 | logger.SetOutput(&buf) 17 | logger.SetFlags(0) 18 | Infof(ctx, "") 19 | 20 | if o := buf.String(); o[:len(o)-len("info: \n")] != expect { 21 | t.Errorf("prefix should be %q: got %q", expect, o[:len(o)-1]) 22 | } 23 | } 24 | 25 | func TestNewContext(t *testing.T) { 26 | ctx := NewContext(context.Background(), "prefix: ") 27 | testPrefix(t, ctx, "prefix: ") 28 | 29 | { 30 | ctx := NewContext(ctx, "foo: ") 31 | testPrefix(t, ctx, "prefix: foo: ") 32 | } 33 | 34 | testPrefix(t, ctx, "prefix: ") 35 | } 36 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/motemen/go-nuts 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/golang/mock v1.6.0 7 | github.com/google/go-cmp v0.7.0 8 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d 9 | github.com/stretchr/testify v1.11.1 10 | golang.org/x/net v0.46.0 11 | golang.org/x/oauth2 v0.32.0 12 | golang.org/x/sync v0.17.0 13 | golang.org/x/text v0.30.0 14 | google.golang.org/appengine v1.6.8 15 | ) 16 | 17 | require ( 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/golang/protobuf v1.5.4 // indirect 20 | github.com/kr/pretty v0.1.0 // indirect 21 | github.com/pmezard/go-difflib v1.0.0 // indirect 22 | google.golang.org/protobuf v1.36.10 // indirect 23 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 24 | gopkg.in/yaml.v3 v3.0.1 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /roundtime/duration_test.go: -------------------------------------------------------------------------------- 1 | package roundtime 2 | 3 | import ( 4 | "testing" 5 | 6 | "time" 7 | ) 8 | 9 | func TestDuration(t *testing.T) { 10 | cases := []struct { 11 | from string 12 | prec int 13 | to string 14 | }{ 15 | {from: "12.863722988s", prec: 2, to: "12.86s"}, 16 | {from: "225.128274ms", prec: 3, to: "225.128ms"}, 17 | {from: "13m35.436221022s", prec: 1, to: "13m35.4s"}, 18 | {from: "13m35.436221022s", prec: 1, to: "13m35.4s"}, 19 | {from: "2.039480015s", prec: 0, to: "2s"}, 20 | {from: "90.112µs", prec: 5, to: "90.112µs"}, 21 | {from: "2h56m24s", prec: 2, to: "2h56m24s"}, 22 | } 23 | 24 | for _, c := range cases { 25 | original, _ := time.ParseDuration(c.from) 26 | rounded := Duration(original, c.prec) 27 | if got, expected := rounded.String(), c.to; got != expected { 28 | t.Errorf("%q precision %d -> %q, expected %q", original, c.prec, rounded, expected) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /logwriter/logwriter_test.go: -------------------------------------------------------------------------------- 1 | package logwriter 2 | 3 | import ( 4 | "testing" 5 | 6 | "bytes" 7 | "fmt" 8 | "log" 9 | ) 10 | 11 | func mustEqual(t *testing.T, got, expected string) { 12 | if got != expected { 13 | t.Errorf("got %q but expected %q", got, expected) 14 | } 15 | } 16 | 17 | func TestLogWriter(t *testing.T) { 18 | var buf bytes.Buffer 19 | w := &LogWriter{ 20 | Logger: log.New(&buf, "", log.Lshortfile), 21 | Format: "[test] %s", 22 | } 23 | 24 | fmt.Fprintln(w, "foo") 25 | fmt.Fprint(w, "bar-") 26 | 27 | mustEqual(t, buf.String(), "logwriter_test.go:24: [test] foo\n") 28 | 29 | fmt.Fprintln(w, "baz") 30 | 31 | mustEqual(t, buf.String(), "logwriter_test.go:24: [test] foo\nlogwriter_test.go:29: [test] bar-baz\n") 32 | 33 | fmt.Fprint(w, "qux") 34 | 35 | mustEqual(t, buf.String(), "logwriter_test.go:24: [test] foo\nlogwriter_test.go:29: [test] bar-baz\n") 36 | 37 | w.Close() 38 | 39 | mustEqual(t, buf.String(), "logwriter_test.go:24: [test] foo\nlogwriter_test.go:29: [test] bar-baz\nlogwriter_test.go:37: [test] qux\n") 40 | } 41 | -------------------------------------------------------------------------------- /gomockutil/matchers/matchers_test.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/golang/mock/gomock" 9 | ) 10 | 11 | func TestMatchers(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | matcher gomock.Matcher 15 | yes, no []interface{} 16 | }{ 17 | {"String()", String("123"), []interface{}{123, "123", fmt.Errorf("123")}, []interface{}{123.4, "x"}}, 18 | {"Function()", Function(func(x interface{}) bool { 19 | rv := reflect.ValueOf(x) 20 | return (rv.Kind() == reflect.Slice || rv.Kind() == reflect.String) && rv.Len() == 3 21 | }), []interface{}{"xxx", []int{6, 6, 6}}, []interface{}{1, true}}, 22 | } 23 | for _, tt := range tests { 24 | t.Run(tt.name, func(t *testing.T) { 25 | for _, v := range tt.yes { 26 | if !tt.matcher.Matches(v) { 27 | t.Errorf("%#v should match %T", v, tt.matcher) 28 | } 29 | } 30 | for _, v := range tt.no { 31 | if tt.matcher.Matches(v) { 32 | t.Errorf("%#v should not match %T", v, tt.matcher) 33 | } 34 | } 35 | }) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /httputil/limitedtransport_test.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "strconv" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | func TestLimitedTransport(t *testing.T) { 14 | h := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 15 | n, _ := strconv.Atoi(req.URL.Query().Get("n")) 16 | fmt.Fprint(w, strings.Repeat("x", n)) 17 | }) 18 | s := httptest.NewServer(h) 19 | defer s.Close() 20 | 21 | client := &http.Client{ 22 | Transport: &LimitedTransport{N: 5000}, 23 | } 24 | 25 | tests := []int{10000, 5000, 300} 26 | for _, test := range tests { 27 | resp, err := client.Get(s.URL + "?n=" + fmt.Sprint(test)) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | 32 | defer resp.Body.Close() 33 | b, err := ioutil.ReadAll(resp.Body) 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | 38 | expected := test 39 | if expected > 5000 { 40 | expected = 5000 41 | } 42 | if got := len(b); got != expected { 43 | t.Errorf("got=%v, expected=%v", got, expected) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Hironao OTSUBO 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /gomockutil/matchers/matchers.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/golang/mock/gomock" 8 | ) 9 | 10 | type String string 11 | 12 | var _ gomock.Matcher = (*String)(nil) 13 | 14 | func (s String) Matches(x interface{}) bool { 15 | return fmt.Sprint(x) == string(s) 16 | } 17 | 18 | func (s String) String() string { 19 | return string(s) 20 | } 21 | 22 | type functionSpec struct { 23 | rv reflect.Value 24 | } 25 | 26 | var _ gomock.Matcher = (*functionSpec)(nil) 27 | 28 | func Function(f interface{}) gomock.Matcher { 29 | rv := reflect.ValueOf(f) 30 | rt := rv.Type() 31 | if rt.Kind() != reflect.Func || rt.NumIn() != 1 { 32 | panic(fmt.Errorf("must be a function with 1 arg: %v", f)) 33 | } 34 | return functionSpec{ 35 | rv: reflect.ValueOf(f), 36 | } 37 | } 38 | 39 | func (f functionSpec) Matches(x interface{}) bool { 40 | rv := reflect.ValueOf(x) 41 | if !rv.Type().AssignableTo(f.rv.Type().In(0)) { 42 | return false 43 | } 44 | return f.rv.Call([]reflect.Value{rv})[0].Bool() 45 | } 46 | 47 | func (f functionSpec) String() string { 48 | return fmt.Sprintf("%v matching some predicates", f.rv.Type().In(0)) 49 | } 50 | -------------------------------------------------------------------------------- /broadcastwriter/broadcastwriter_test.go: -------------------------------------------------------------------------------- 1 | package broadcastwriter 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | "testing" 7 | 8 | "fmt" 9 | ) 10 | 11 | func TestBroadcastWriter(t *testing.T) { 12 | var wg sync.WaitGroup 13 | consumeListenerC := func(c <-chan []byte) <-chan string { 14 | outc := make(chan string) 15 | wg.Add(1) 16 | go func() { 17 | var buf bytes.Buffer 18 | for b := range c { 19 | buf.Write(b) 20 | } 21 | outc <- buf.String() 22 | wg.Done() 23 | }() 24 | return outc 25 | } 26 | 27 | bw := NewBroadcastWriter() 28 | 29 | l1 := bw.NewListener() 30 | c1 := consumeListenerC(l1) 31 | 32 | fmt.Fprintln(bw, "foo") 33 | 34 | l2 := bw.NewListener() 35 | c2 := consumeListenerC(l2) 36 | 37 | fmt.Fprintln(bw, "bar") 38 | 39 | l3 := bw.NewListener() 40 | c3 := consumeListenerC(l3) 41 | 42 | bw.Close() 43 | 44 | l4 := bw.NewListener() 45 | c4 := consumeListenerC(l4) 46 | 47 | check := func(name string, c <-chan string) { 48 | s := <-c 49 | expected := "foo\nbar\n" 50 | if s != expected { 51 | t.Errorf("%s: got %q but expected %q", name, s, expected) 52 | } 53 | } 54 | check("c1", c1) 55 | check("c2", c2) 56 | check("c3", c3) 57 | check("c4", c4) 58 | 59 | wg.Wait() 60 | } 61 | -------------------------------------------------------------------------------- /httputil/httputil.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type HTTPError struct { 8 | StatusCode int 9 | Status string 10 | } 11 | 12 | func (e *HTTPError) Error() string { 13 | return e.Status 14 | } 15 | 16 | func Successful(resp *http.Response, err error) (*http.Response, error) { 17 | if err != nil { 18 | return resp, err 19 | } 20 | if resp.StatusCode < 200 || resp.StatusCode >= 300 { 21 | return resp, &HTTPError{StatusCode: resp.StatusCode, Status: resp.Status} 22 | } 23 | return resp, nil 24 | } 25 | 26 | type TransportWrapFunc func(req *http.Request, base http.RoundTripper) (*http.Response, error) 27 | 28 | func WrapTransport(base http.RoundTripper, wrapper ...TransportWrapFunc) http.RoundTripper { 29 | transport := base 30 | for _, w := range wrapper { 31 | transport = &wrappedTransport{ 32 | roundTrip: w, 33 | base: transport, 34 | } 35 | } 36 | return transport 37 | } 38 | 39 | type wrappedTransport struct { 40 | roundTrip TransportWrapFunc 41 | base http.RoundTripper 42 | } 43 | 44 | func (t *wrappedTransport) RoundTrip(req *http.Request) (*http.Response, error) { 45 | base := t.base 46 | if base == nil { 47 | base = http.DefaultTransport 48 | } 49 | 50 | return t.roundTrip(req, base) 51 | } 52 | -------------------------------------------------------------------------------- /minipipeline/pipeline_test.go: -------------------------------------------------------------------------------- 1 | package minipipeline 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "testing" 7 | ) 8 | 9 | func TestSet_Pipeline(t *testing.T) { 10 | var set Set 11 | p := set.Pipeline("test1") 12 | 13 | done := make(chan struct{}) 14 | 15 | go func() { 16 | for { 17 | select { 18 | case <-done: 19 | return 20 | default: 21 | } 22 | 23 | for _, p := range set.Snapshot() { 24 | for _, s := range p.Steps { 25 | s.String() 26 | } 27 | } 28 | } 29 | }() 30 | 31 | logSteps := func() { 32 | steps := p.Steps 33 | ss := make([]string, len(steps)) 34 | for i, s := range steps { 35 | ss[i] = s.String() 36 | } 37 | log.Printf("%v", ss) 38 | } 39 | 40 | p.Step("step1", func() error { 41 | logSteps() 42 | return nil 43 | }) 44 | logSteps() 45 | 46 | p.Step("step2", func() error { 47 | logSteps() 48 | pg := p.Current().ProgressGroup() 49 | for i := 0; i < 3; i++ { 50 | pg.Go(func() error { 51 | logSteps() 52 | return nil 53 | }) 54 | } 55 | return pg.Wait() 56 | }) 57 | logSteps() 58 | 59 | p.Step("step3", func() error { 60 | logSteps() 61 | return fmt.Errorf("stop") 62 | }) 63 | logSteps() 64 | 65 | p.Step("step4", func() error { 66 | t.Error("this step should not be executed") 67 | return nil 68 | }) 69 | logSteps() 70 | } 71 | -------------------------------------------------------------------------------- /bytesutil/transformer_test.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import "testing" 4 | 5 | import ( 6 | "bytes" 7 | "golang.org/x/text/transform" 8 | "io/ioutil" 9 | ) 10 | 11 | func TestRemove(t *testing.T) { 12 | type test struct { 13 | remove string 14 | in string 15 | out string 16 | } 17 | tests := []test{ 18 | { 19 | "\x00", 20 | "abcde", 21 | "abcde", 22 | }, 23 | { 24 | "\x00", 25 | "abc\x00de", 26 | "abcde", 27 | }, 28 | { 29 | "\x00", 30 | "\x00abc\x00de", 31 | "abcde", 32 | }, 33 | { 34 | "\x00", 35 | "\x00abc\x00de\x00", 36 | "abcde", 37 | }, 38 | { 39 | "\x00", 40 | "abc\x00\x00\x00de", 41 | "abcde", 42 | }, 43 | { 44 | "\x00", 45 | "", 46 | "", 47 | }, 48 | { 49 | "\x00", 50 | "\x00", 51 | "", 52 | }, 53 | { 54 | "\x00\x01", 55 | "\x00\x01", 56 | "", 57 | }, 58 | { 59 | "\x00\x01", 60 | "ab\x00cd\x01ef", 61 | "abcdef", 62 | }, 63 | { 64 | "\xe3", 65 | "あ", 66 | "\x81\x82", 67 | }, 68 | } 69 | 70 | for _, test := range tests { 71 | buf := bytes.NewBufferString(test.in) 72 | remover := Remove(In(test.remove)) 73 | r := transform.NewReader(buf, remover) 74 | b, err := ioutil.ReadAll(r) 75 | if err != nil { 76 | t.Error(err) 77 | continue 78 | } 79 | 80 | if string(b) != test.out { 81 | t.Errorf("got %q != %q", string(b), test.out) 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /ctxlog/ctxlog_pre17.go: -------------------------------------------------------------------------------- 1 | // +build !appengine,!go1.7 2 | 3 | package ctxlog 4 | 5 | import ( 6 | "log" 7 | 8 | "golang.org/x/net/context" 9 | ) 10 | 11 | func FromContext(ctx context.Context) *log.Logger { 12 | logger, ok := ctx.Value(LoggerContextKey).(*log.Logger) 13 | if !ok { 14 | logger = Logger 15 | } 16 | return logger 17 | } 18 | 19 | func NewContext(ctx context.Context, prefix string) context.Context { 20 | logger := FromContext(ctx) 21 | newLogger := log.New(output, logger.Prefix()+prefix, logger.Flags()) 22 | return context.WithValue(ctx, LoggerContextKey, newLogger) 23 | } 24 | 25 | func logf(ctx context.Context, level string, format string, args ...interface{}) { 26 | logger := FromContext(ctx) 27 | args = append([]interface{}{level}, args...) 28 | logger.Printf("%s: "+format, args...) 29 | } 30 | 31 | func Debugf(ctx context.Context, format string, args ...interface{}) { 32 | logf(ctx, "debug", format, args...) 33 | } 34 | 35 | func Infof(ctx context.Context, format string, args ...interface{}) { 36 | logf(ctx, "info", format, args...) 37 | } 38 | 39 | func Warningf(ctx context.Context, format string, args ...interface{}) { 40 | logf(ctx, "warning", format, args...) 41 | } 42 | 43 | func Errorf(ctx context.Context, format string, args ...interface{}) { 44 | logf(ctx, "error", format, args...) 45 | } 46 | 47 | func Criticalf(ctx context.Context, format string, args ...interface{}) { 48 | logf(ctx, "critical", format, args...) 49 | } 50 | -------------------------------------------------------------------------------- /chardet/LICENSE_charset.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 The Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /bytesutil/transformer.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "strings" 5 | 6 | "golang.org/x/text/transform" 7 | ) 8 | 9 | type Set interface { 10 | Contains(b byte) bool 11 | } 12 | 13 | type setFunc func(byte) bool 14 | 15 | func (s setFunc) Contains(b byte) bool { 16 | return s(b) 17 | } 18 | 19 | func In(s string) Set { 20 | return setFunc(func(b byte) bool { 21 | return strings.IndexByte(s, b) != -1 22 | }) 23 | } 24 | 25 | func Predicate(f func(byte) bool) Set { 26 | return setFunc(f) 27 | } 28 | 29 | func Remove(s Set) removeTransformer { 30 | return removeTransformer{s} 31 | } 32 | 33 | type removeTransformer struct { 34 | set Set 35 | } 36 | 37 | // Reset implements transform.Transformer. 38 | func (t removeTransformer) Reset() {} 39 | 40 | // Transform implements transform.Transformer. 41 | func (t removeTransformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 42 | for nSrc < len(src) { 43 | p := IndexFunc(src[nSrc:], t.set.Contains) 44 | if p == 0 { 45 | nSrc++ 46 | continue 47 | } 48 | 49 | var end, skip int 50 | if p == -1 { 51 | end = len(src) 52 | skip = 0 53 | } else { 54 | end = nSrc + p 55 | skip = 1 56 | } 57 | 58 | if cap(dst[nDst:]) < end-nSrc+1 { 59 | err = transform.ErrShortDst 60 | return 61 | } 62 | 63 | n := copy(dst[nDst:], src[nSrc:end]) 64 | nDst += n 65 | nSrc += n + skip 66 | } 67 | 68 | return 69 | } 70 | 71 | func IndexFunc(s []byte, f func(b byte) bool) int { 72 | for i := 0; i < len(s); i++ { 73 | if f(s[i]) { 74 | return i 75 | } 76 | } 77 | return -1 78 | } 79 | -------------------------------------------------------------------------------- /ctxlog/ctxlog_17.go: -------------------------------------------------------------------------------- 1 | // +build !appengine,go1.7 2 | 3 | package ctxlog 4 | 5 | import ( 6 | "context" 7 | "log" 8 | ) 9 | 10 | func LoggerFromContext(ctx context.Context) *log.Logger { 11 | logger, ok := ctx.Value(LoggerContextKey).(*log.Logger) 12 | if !ok { 13 | logger = Logger 14 | } 15 | return logger 16 | } 17 | 18 | func PrefixFromContext(ctx context.Context) string { 19 | prefix, _ := ctx.Value(PrefixContextKey).(string) 20 | return prefix 21 | } 22 | 23 | func NewContext(ctx context.Context, prefix string) context.Context { 24 | prefix = PrefixFromContext(ctx) + prefix 25 | ctx = context.WithValue(ctx, PrefixContextKey, prefix) 26 | return ctx 27 | } 28 | 29 | func logf(ctx context.Context, level string, format string, args ...interface{}) { 30 | logger := LoggerFromContext(ctx) 31 | prefix := PrefixFromContext(ctx) 32 | args = append([]interface{}{prefix}, args...) 33 | logger.Printf("%s"+level+": "+format, args...) 34 | } 35 | 36 | func Debugf(ctx context.Context, format string, args ...interface{}) { 37 | logf(ctx, "debug", format, args...) 38 | } 39 | 40 | func Infof(ctx context.Context, format string, args ...interface{}) { 41 | logf(ctx, "info", format, args...) 42 | } 43 | 44 | func Warningf(ctx context.Context, format string, args ...interface{}) { 45 | logf(ctx, "warning", format, args...) 46 | } 47 | 48 | func Errorf(ctx context.Context, format string, args ...interface{}) { 49 | logf(ctx, "error", format, args...) 50 | } 51 | 52 | func Criticalf(ctx context.Context, format string, args ...interface{}) { 53 | logf(ctx, "critical", format, args...) 54 | } 55 | -------------------------------------------------------------------------------- /logwriter/logwriter.go: -------------------------------------------------------------------------------- 1 | package logwriter 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "log" 9 | ) 10 | 11 | type LogWriter struct { 12 | Logger *log.Logger 13 | Format string 14 | FormatArgs []interface{} 15 | Calldepth int 16 | 17 | buf []byte 18 | } 19 | 20 | func (lw *LogWriter) Write(p []byte) (n int, err error) { 21 | var buf []byte 22 | if lw.buf == nil { 23 | buf = make([]byte, len(lw.buf)+len(p)) 24 | copy(buf, p) 25 | } else { 26 | buf = append(lw.buf, p...) 27 | } 28 | 29 | for len(buf) > 0 { 30 | n := bytes.IndexByte(buf, '\n') 31 | if n == -1 { 32 | lw.buf = buf 33 | break 34 | } 35 | 36 | lw.writeln(buf[0:n+1], 0) 37 | buf = buf[n+1:] 38 | } 39 | 40 | lw.buf = buf 41 | 42 | return len(p), nil 43 | } 44 | 45 | func (lw *LogWriter) Close() error { 46 | if len(lw.buf) > 0 { 47 | lw.writeln(lw.buf, -1) 48 | lw.buf = nil 49 | } 50 | 51 | return nil 52 | } 53 | 54 | func (lw *LogWriter) ReadFrom(r io.Reader) (n int64, err error) { 55 | s := bufio.NewScanner(r) 56 | for s.Scan() { 57 | lw.writeln(s.Bytes(), 0) 58 | } 59 | 60 | return 0, s.Err() 61 | } 62 | 63 | func (lw LogWriter) writeln(line []byte, delta int) { 64 | var s string 65 | if lw.Format == "" { 66 | s = fmt.Sprintln(string(line)) 67 | } else { 68 | args := lw.FormatArgs 69 | if args == nil { 70 | args = []interface{}{string(line)} 71 | } else { 72 | args = append(args, string(line)) 73 | } 74 | s = fmt.Sprintf(lw.Format, args...) 75 | } 76 | 77 | calldepth := lw.Calldepth 78 | if calldepth == 0 { 79 | calldepth = 4 80 | } 81 | 82 | lw.Logger.Output(calldepth+delta, s) 83 | } 84 | -------------------------------------------------------------------------------- /broadcastwriter/broadcastwriter.go: -------------------------------------------------------------------------------- 1 | package broadcastwriter 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | const chanBufSize = 256 8 | 9 | type BroadcastWriter struct { 10 | backlog *bytes.Buffer 11 | 12 | listeners []chan<- []byte 13 | addListenerC chan chan<- []byte 14 | 15 | writeC chan []byte 16 | 17 | closeC chan struct{} 18 | closed bool 19 | } 20 | 21 | func NewBroadcastWriter() *BroadcastWriter { 22 | bw := &BroadcastWriter{ 23 | backlog: new(bytes.Buffer), 24 | listeners: []chan<- []byte{}, 25 | addListenerC: make(chan chan<- []byte), 26 | writeC: make(chan []byte), 27 | closeC: make(chan struct{}), 28 | } 29 | go bw.loop() 30 | return bw 31 | } 32 | 33 | func (bw *BroadcastWriter) loop() { 34 | for { 35 | select { 36 | case l := <-bw.addListenerC: 37 | if bw.backlog.Len() > 0 { 38 | l <- bw.backlog.Bytes() 39 | } 40 | if bw.closed { 41 | close(l) 42 | } 43 | bw.listeners = append(bw.listeners, l) 44 | 45 | case buf := <-bw.writeC: 46 | bw.backlog.Write(buf) 47 | for _, l := range bw.listeners { 48 | l <- buf 49 | } 50 | 51 | case <-bw.closeC: 52 | for _, l := range bw.listeners { 53 | close(l) 54 | } 55 | bw.closed = true 56 | } 57 | } 58 | } 59 | 60 | func (bw *BroadcastWriter) Write(p []byte) (n int, err error) { 61 | buf := make([]byte, len(p)) 62 | copy(buf, p) 63 | bw.writeC <- buf 64 | return len(p), nil 65 | } 66 | 67 | func (bw *BroadcastWriter) Close() error { 68 | bw.closeC <- struct{}{} 69 | return nil 70 | } 71 | 72 | func (bw *BroadcastWriter) NewListener() <-chan []byte { 73 | l := make(chan []byte, chanBufSize) 74 | bw.addListenerC <- l 75 | return l 76 | } 77 | -------------------------------------------------------------------------------- /ctxlog/ctxlog_appengine.go: -------------------------------------------------------------------------------- 1 | // +build appengine 2 | 3 | package ctxlog 4 | 5 | import ( 6 | "log" 7 | 8 | "golang.org/x/net/context" 9 | aelog "google.golang.org/appengine/log" 10 | ) 11 | 12 | func FromContext(ctx context.Context) *log.Logger { 13 | logger, ok := ctx.Value(LoggerContextKey).(*log.Logger) 14 | if !ok { 15 | logger = Logger 16 | } 17 | return logger 18 | } 19 | 20 | func NewContext(ctx context.Context, prefix string) context.Context { 21 | logger := FromContext(ctx) 22 | newLogger := log.New(output, logger.Prefix()+prefix, logger.Flags()) 23 | return context.WithValue(ctx, LoggerContextKey, newLogger) 24 | } 25 | 26 | func Debugf(ctx context.Context, format string, args ...interface{}) { 27 | prefix := FromContext(ctx).Prefix() 28 | args = append([]interface{}{prefix}, args...) 29 | aelog.Debugf(ctx, "%s"+format, args...) 30 | } 31 | 32 | func Infof(ctx context.Context, format string, args ...interface{}) { 33 | prefix := FromContext(ctx).Prefix() 34 | args = append([]interface{}{prefix}, args...) 35 | aelog.Infof(ctx, "%s"+format, args...) 36 | } 37 | 38 | func Warningf(ctx context.Context, format string, args ...interface{}) { 39 | prefix := FromContext(ctx).Prefix() 40 | args = append([]interface{}{prefix}, args...) 41 | aelog.Warningf(ctx, "%s"+format, args...) 42 | } 43 | 44 | func Errorf(ctx context.Context, format string, args ...interface{}) { 45 | prefix := FromContext(ctx).Prefix() 46 | args = append([]interface{}{prefix}, args...) 47 | aelog.Errorf(ctx, "%s"+format, args...) 48 | } 49 | 50 | func Criticalf(ctx context.Context, format string, args ...interface{}) { 51 | prefix := FromContext(ctx).Prefix() 52 | args = append([]interface{}{prefix}, args...) 53 | aelog.Criticalf(ctx, "%s"+format, args...) 54 | } 55 | -------------------------------------------------------------------------------- /httputil/charsettransport_test.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "io/ioutil" 5 | "mime" 6 | "net/http" 7 | "net/http/httptest" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/motemen/go-nuts/chardet" 12 | ) 13 | 14 | func TestCharsetTransport(t *testing.T) { 15 | mime.AddExtensionType(".html", "text/html; charset=unknown") 16 | 17 | s := httptest.NewServer(http.FileServer(http.Dir("testdata"))) 18 | defer s.Close() 19 | 20 | client := &http.Client{ 21 | Transport: &CharsetTransport{}, 22 | } 23 | 24 | resp, err := client.Get(s.URL + "/euc-jp.html") 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | 29 | defer resp.Body.Close() 30 | b, err := ioutil.ReadAll(resp.Body) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | if !strings.Contains(string(b), "こんにちは、世界") { 35 | t.Fatal(string(b)) 36 | } 37 | 38 | _, err = client.Get(s.URL + "/empty.html") 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | } 43 | 44 | func TestChardetTransport(t *testing.T) { 45 | mime.AddExtensionType(".html", "text/html; charset=unknown") 46 | 47 | s := httptest.NewServer(http.FileServer(http.Dir("testdata"))) 48 | defer s.Close() 49 | 50 | client := &http.Client{ 51 | Transport: &ChardetTransport{ 52 | Options: []chardet.DetectorOption{ 53 | chardet.WithLanguage("ja", ""), 54 | }, 55 | }, 56 | } 57 | 58 | resp, err := client.Get(s.URL + "/euc-jp.nohint.html") 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | defer resp.Body.Close() 64 | b, err := ioutil.ReadAll(resp.Body) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | if !strings.Contains(string(b), "こんにちは、世界") { 69 | t.Fatal(string(b)) 70 | } 71 | 72 | _, err = client.Get(s.URL + "/empty.html") 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /giturl/giturl.go: -------------------------------------------------------------------------------- 1 | // Package giturl provides ParseGitURL which parses remote URLs under the way 2 | // that git does. 3 | package giturl 4 | 5 | import ( 6 | "net/url" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | var ( 13 | rxURLLike = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9+.-]*://`) 14 | rxHostAndPort = regexp.MustCompile(`^([^:]+|\[.+?\]):([0-9]+)$`) 15 | rxSCPLikeV6 = regexp.MustCompile(`^(.+?@)?\[(.+?)\]:(.*)`) 16 | ) 17 | 18 | func ParseGitURL(giturl string) (proto string, host string, port uint, path string, exotic bool, err error) { 19 | // ref: parse_connect_url() in connect.c 20 | 21 | if rxURLLike.MatchString(giturl) { 22 | var u *url.URL 23 | u, err = url.Parse(giturl) 24 | if err != nil { 25 | return 26 | } 27 | 28 | proto = u.Scheme 29 | if proto == "git+ssh" || proto == "ssh+git" { 30 | proto = "ssh" 31 | } 32 | 33 | host = u.Host 34 | path = u.Path 35 | 36 | if proto == "ssh" { 37 | if m := rxHostAndPort.FindStringSubmatch(host); m != nil { 38 | if port64, err := strconv.ParseUint(m[2], 10, 16); err == nil { 39 | host = m[1] 40 | port = uint(port64) 41 | } 42 | } 43 | if host[0] == '[' && host[len(host)-1] == ']' { 44 | host = host[1 : len(host)-1] 45 | } 46 | } 47 | 48 | if u.User != nil { 49 | host = u.User.String() + "@" + host 50 | } 51 | 52 | if proto == "git" || proto == "ssh" { 53 | if path[1] == '~' { 54 | path = path[1:] 55 | } 56 | } else if proto == "file" { 57 | host = "" 58 | path = u.Host + u.Path 59 | } else { 60 | exotic = true 61 | } 62 | } else { 63 | colon := strings.IndexByte(giturl, ':') 64 | slash := strings.IndexByte(giturl, '/') 65 | 66 | if colon > -1 && (slash == -1 || colon < slash) /*&& !hasDosDrivePrefix(giturl)*/ { 67 | // For SCP-like URLs, colon must appear and be before any slashes 68 | // - user@host.xyz:path/to/repo.git/ 69 | // - host.xyz:path/to/repo.git/ 70 | // - user@[::1]:path/to/repo.git/ 71 | // - [::1]:path/to/repo.git/ 72 | proto = "ssh" 73 | m := rxSCPLikeV6.FindStringSubmatch(giturl) 74 | if m != nil { 75 | host = m[1] + m[2] 76 | path = m[3] 77 | } else { 78 | host = giturl[:colon] 79 | path = giturl[colon+1:] 80 | } 81 | if path[1] == '~' { 82 | path = path[1:] 83 | } 84 | } else { 85 | proto = "file" 86 | path = giturl 87 | } 88 | } 89 | 90 | return 91 | } 92 | -------------------------------------------------------------------------------- /httputil/charsettransport.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "net/http" 7 | "sync" 8 | 9 | "golang.org/x/net/html/charset" 10 | "golang.org/x/text/encoding" 11 | "golang.org/x/text/transform" 12 | 13 | "github.com/motemen/go-nuts/chardet" 14 | ) 15 | 16 | // CharsetTransport is an http.Transport which automatically decodes resp.Body by its charset. 17 | type CharsetTransport struct { 18 | Base http.RoundTripper 19 | } 20 | 21 | type readCloser struct { 22 | io.Reader 23 | io.Closer 24 | } 25 | 26 | // RoundTrip implements http.RoundTripper. 27 | func (t *CharsetTransport) RoundTrip(req *http.Request) (*http.Response, error) { 28 | base := t.Base 29 | if base == nil { 30 | base = http.DefaultTransport 31 | } 32 | 33 | resp, err := base.RoundTrip(req) 34 | if err != nil { 35 | return resp, err 36 | } 37 | 38 | r, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) 39 | if err != nil && err != io.EOF { 40 | return resp, err 41 | } 42 | 43 | if r == nil { 44 | r = bytes.NewReader(nil) 45 | } 46 | 47 | resp.Body = &readCloser{ 48 | Reader: r, 49 | Closer: resp.Body, 50 | } 51 | return resp, nil 52 | } 53 | 54 | type ChardetTransport struct { 55 | Base http.RoundTripper 56 | Options []chardet.DetectorOption 57 | once sync.Once 58 | detector *chardet.Detector 59 | } 60 | 61 | func (t *ChardetTransport) RoundTrip(req *http.Request) (*http.Response, error) { 62 | base := t.Base 63 | if base == nil { 64 | base = http.DefaultTransport 65 | } 66 | 67 | t.once.Do(func() { 68 | t.detector = chardet.NewDetector(t.Options...) 69 | }) 70 | 71 | resp, err := base.RoundTrip(req) 72 | if err != nil { 73 | return resp, err 74 | } 75 | 76 | // from golang.org/x/net/html/charset.NewReader 77 | var r io.Reader = resp.Body 78 | 79 | preview := make([]byte, 1024) 80 | n, err := io.ReadFull(resp.Body, preview) 81 | switch { 82 | case err == io.ErrUnexpectedEOF || err == io.EOF: 83 | preview = preview[:n] 84 | r = bytes.NewReader(preview) 85 | case err != nil: 86 | return nil, err 87 | default: 88 | r = io.MultiReader(bytes.NewReader(preview), r) 89 | } 90 | 91 | if n > 0 { 92 | enc, _, _ := chardet.DetermineEncoding(preview, resp.Header.Get("Content-Type"), t.detector.DetectEncoding) 93 | if enc != encoding.Nop { 94 | r = transform.NewReader(r, enc.NewDecoder()) 95 | } 96 | } 97 | 98 | resp.Body = &readCloser{ 99 | Reader: r, 100 | Closer: resp.Body, 101 | } 102 | return resp, nil 103 | } 104 | -------------------------------------------------------------------------------- /chardet/chardet.go: -------------------------------------------------------------------------------- 1 | package chardet 2 | 3 | import ( 4 | "sort" 5 | 6 | "golang.org/x/text/encoding" 7 | "golang.org/x/text/encoding/ianaindex" 8 | 9 | "github.com/saintfish/chardet" 10 | ) 11 | 12 | var detector = chardet.NewTextDetector() 13 | 14 | type Detector struct { 15 | resultFilter 16 | } 17 | 18 | type DetectorOption func(resultFilter) resultFilter 19 | 20 | var ErrNotDetected = chardet.NotDetectedError 21 | 22 | func NewDetector(opts ...DetectorOption) *Detector { 23 | var d Detector 24 | for _, o := range opts { 25 | d.resultFilter = o(d.resultFilter) 26 | } 27 | return &d 28 | } 29 | 30 | func (d Detector) DetectEncoding(b []byte) (encoding.Encoding, string) { 31 | results, err := detector.DetectAll(b) 32 | if err != nil { 33 | return nil, "" 34 | } 35 | 36 | results = d.resultFilter.filter(results) 37 | if len(results) == 0 { 38 | return nil, "" 39 | } 40 | 41 | charset := results[0].Charset 42 | 43 | enc, err := ianaindex.IANA.Encoding(charset) 44 | if err != nil { 45 | return nil, "" 46 | } 47 | 48 | return enc, charset 49 | } 50 | 51 | func WithCharset(charsets ...string) DetectorOption { 52 | return func(dc resultFilter) resultFilter { 53 | dc.charsets = charsets 54 | return dc 55 | } 56 | } 57 | 58 | func WithLanguage(langs ...string) DetectorOption { 59 | return func(dc resultFilter) resultFilter { 60 | dc.languages = langs 61 | return dc 62 | } 63 | } 64 | 65 | func WithPrefer(f func(a, b chardet.Result) bool) DetectorOption { 66 | return func(dc resultFilter) resultFilter { 67 | dc.prefer = f 68 | return dc 69 | } 70 | } 71 | 72 | type resultFilter struct { 73 | charsets []string 74 | languages []string 75 | prefer func(a, b chardet.Result) bool // return true if a is preferable than b 76 | } 77 | 78 | func (dc resultFilter) filter(results []chardet.Result) []chardet.Result { 79 | if dc.charsets != nil { 80 | m := map[string]bool{} 81 | for _, k := range dc.charsets { 82 | m[k] = true 83 | } 84 | 85 | filtered := []chardet.Result{} 86 | for _, r := range results { 87 | if m[r.Charset] { 88 | filtered = append(filtered, r) 89 | } 90 | } 91 | 92 | results = filtered 93 | } 94 | 95 | if dc.languages != nil { 96 | m := map[string]bool{} 97 | for _, k := range dc.languages { 98 | m[k] = true 99 | } 100 | 101 | filtered := []chardet.Result{} 102 | for _, r := range results { 103 | if m[r.Language] { 104 | filtered = append(filtered, r) 105 | } 106 | } 107 | 108 | results = filtered 109 | } 110 | 111 | if dc.prefer != nil { 112 | sort.Slice( 113 | results, 114 | func(i, j int) bool { 115 | return dc.prefer(results[i], results[j]) 116 | }, 117 | ) 118 | } 119 | 120 | return results 121 | } 122 | -------------------------------------------------------------------------------- /netutil/blocklist_test.go: -------------------------------------------------------------------------------- 1 | package netutil 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestPrivateNetworkBlocklist(t *testing.T) { 16 | dialer := &net.Dialer{ 17 | Control: PrivateNetworkBlocklist.Control, 18 | } 19 | transport := http.DefaultTransport.(*http.Transport).Clone() 20 | transport.DialContext = dialer.DialContext 21 | client := &http.Client{ 22 | Transport: transport, 23 | } 24 | 25 | tests := []struct { 26 | addr string 27 | shouldBlock bool 28 | }{ 29 | {"localhost", true}, 30 | {"10.255.0.1", true}, 31 | {"www.example.com", false}, 32 | {"203.0.113.1", true}, 33 | {"192.0.0.170", true}, 34 | {"255.255.255.255", true}, 35 | {"169.254.169.25", true}, 36 | {"192.88.99.1", true}, 37 | {"[::]", true}, 38 | {"[::0]", true}, 39 | {"[::1]", true}, 40 | {"[::2]", false}, 41 | {"[2001:2::1]", true}, 42 | {"[2001:4860:4802:32::a]", false}, 43 | // TODO: Uncomment when Go 1.26 is released or backport is available 44 | // IPv4-mapped IPv6 addresses are rejected by net/url.Parse in Go 1.25.3 due to CVE-2025-47912 fix 45 | // This is fixed in Go 1.26: https://github.com/golang/go/issues/75815 46 | // {"[::ffff:192.168.0.1]", true}, 47 | } 48 | 49 | for _, test := range tests { 50 | ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) 51 | req, err := http.NewRequestWithContext(ctx, "GET", "http://"+test.addr, nil) 52 | if err != nil { 53 | t.Fatalf("failed to create request for %s: %v", test.addr, err) 54 | } 55 | _, err = client.Do(req) 56 | var berr ErrBlocked 57 | if test.shouldBlock { 58 | assert.ErrorAs(t, err, &ErrBlocked{}) 59 | } else { 60 | if errors.As(err, &berr) { 61 | t.Errorf("should not block %s but got error: %s", test.addr, err) 62 | } 63 | } 64 | cancel() 65 | } 66 | } 67 | 68 | // may be flaky 69 | func TestNetworkBlocklist_Control(t *testing.T) { 70 | blocklist := NetworkBlocklist{ 71 | V4: []NamedNetwork{ 72 | {IPNet: MustParseCIDR("8.8.4.4/32")}, 73 | {IPNet: MustParseCIDR("8.8.8.8/32")}, 74 | }, 75 | V6: []NamedNetwork{ 76 | {IPNet: MustParseCIDR("2001:4860:4860::8844/128")}, 77 | {IPNet: MustParseCIDR("2001:4860:4860::8888/128")}, 78 | }, 79 | } 80 | dialer := net.Dialer{ 81 | Control: blocklist.Control, 82 | } 83 | 84 | ctx := context.Background() 85 | conn, err := dialer.DialContext(ctx, "udp", "8.8.8.8:53") 86 | assert.ErrorAs(t, err, &ErrBlocked{}) 87 | if conn != nil { 88 | conn.Close() 89 | } 90 | 91 | conn, err = dialer.DialContext(ctx, "udp", "dns.google:53") 92 | assert.ErrorAs(t, err, &ErrBlocked{}) 93 | if conn != nil { 94 | conn.Close() 95 | } 96 | } 97 | 98 | func ExampleNetworkBlocklist_Control() { 99 | transport := http.DefaultTransport.(*http.Transport).Clone() 100 | transport.DialContext = (&net.Dialer{ 101 | Timeout: 30 * time.Second, 102 | KeepAlive: 30 * time.Second, 103 | Control: PrivateNetworkBlocklist.Control, 104 | }).DialContext 105 | 106 | client := &http.Client{ 107 | Transport: transport, 108 | } 109 | 110 | _, err := client.Get("http://[::1]/") 111 | fmt.Println(err) 112 | // Output: Get "http://[::1]/": dial tcp [::1]:80: host is blocked (Loopback Address) 113 | } 114 | -------------------------------------------------------------------------------- /urlutil/normalize_test.go: -------------------------------------------------------------------------------- 1 | package urlutil 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | ) 7 | 8 | func TestNormalizeURL(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | in string 12 | want string 13 | wantErr bool 14 | }{ 15 | { 16 | name: "Remove default port (http)", 17 | in: "http://www.example.com:80/", 18 | want: "http://www.example.com/", 19 | }, 20 | { 21 | name: "Remove default port (https)", 22 | in: "https://www.example.com:443/", 23 | want: "https://www.example.com/", 24 | }, 25 | { 26 | name: "Remove empty port", 27 | in: "http://www.example.com:/", 28 | want: "http://www.example.com/", 29 | }, 30 | { 31 | name: "Keep non-default port", 32 | in: "http://www.example.com:9999/", 33 | want: "http://www.example.com:9999/", 34 | }, 35 | { 36 | name: "Encode to Punycode", 37 | in: "https://はじめよう.みんな/はじめよう.みんな", 38 | want: "https://xn--p8j9a0d9c9a.xn--q9jyb4c/%E3%81%AF%E3%81%98%E3%82%81%E3%82%88%E3%81%86.%E3%81%BF%E3%82%93%E3%81%AA", 39 | }, 40 | { 41 | name: "Encode/decode characters", 42 | in: "https://localhost/%7e%41%5E/🤗?q=%7E%41%5E🤗/", 43 | want: "https://localhost/~A%5E/%F0%9F%A4%97?q=~A%5E%F0%9F%A4%97%2F", 44 | }, 45 | { 46 | name: "Encode/decode characters", 47 | in: `https://localhost/!"$%ef%41`, 48 | want: "https://localhost/%21%22$%EFA", 49 | }, 50 | { 51 | name: "Uppercase percent encodings", 52 | in: "https://localhost/%5e", 53 | want: "https://localhost/%5E", 54 | }, 55 | { 56 | name: "Space in path/query", 57 | in: "https://localhost/foo bar?q=foo bar+baz", 58 | want: "https://localhost/foo%20bar?q=foo+bar+baz", 59 | }, 60 | { 61 | name: "Empty path", 62 | in: "https://localhost", 63 | want: "https://localhost/", 64 | }, 65 | { 66 | name: "Lowercase scheme/host", 67 | in: "HTTPS://WWW.EXAMPLE.COM/", 68 | want: "https://www.example.com/", 69 | }, 70 | { 71 | name: "Clean path", 72 | in: "https://localhost/a/./b/../c//d/", 73 | want: "https://localhost/a/c//d/", 74 | }, 75 | } 76 | for _, tt := range tests { 77 | name := tt.name 78 | if name == "" { 79 | name = tt.in 80 | } 81 | t.Run(name, func(t *testing.T) { 82 | u, _ := url.Parse(tt.in) 83 | got, err := NormalizeURL(u) 84 | if (err != nil) != tt.wantErr { 85 | t.Errorf("NormalizeURL() error = %v, wantErr %v", err, tt.wantErr) 86 | return 87 | } 88 | if got.String() != tt.want { 89 | t.Errorf("NormalizeURL() = %v, want %v", got, tt.want) 90 | } 91 | }) 92 | } 93 | } 94 | 95 | func Test_removeDotSegments(t *testing.T) { 96 | tests := []struct { 97 | name string 98 | in string 99 | want string 100 | }{ 101 | { 102 | in: "/a/b/c/./../../g", 103 | want: "/a/g", 104 | }, 105 | { 106 | in: "mid/content=5/../6", 107 | want: "mid/6", 108 | }, 109 | { 110 | in: "foo/bar/.", 111 | want: "foo/bar/", 112 | }, 113 | { 114 | in: "foo/bar/..", 115 | want: "foo/", 116 | }, 117 | { 118 | in: "/../.", 119 | want: "/", 120 | }, 121 | } 122 | for _, tt := range tests { 123 | t.Run(tt.name, func(t *testing.T) { 124 | if got := removeDotSegments(tt.in); got != tt.want { 125 | t.Errorf("removeDotSegments(%q) = %v, want %v", tt.in, got, tt.want) 126 | } 127 | }) 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /minipipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | package minipipeline 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | "time" 8 | 9 | "golang.org/x/sync/errgroup" 10 | ) 11 | 12 | type Set struct { 13 | sync.RWMutex 14 | pipelines []*Pipeline 15 | } 16 | 17 | type Pipeline struct { 18 | sync.RWMutex 19 | Name string 20 | Steps []*Step 21 | Err error 22 | } 23 | 24 | type Step struct { 25 | sync.RWMutex 26 | Name string 27 | StartedAt time.Time 28 | FinishedAt time.Time 29 | Err error 30 | 31 | progressDone uint32 32 | progressAll uint32 33 | } 34 | 35 | func (s *Step) ProgressDone(delta uint32) { 36 | s.Lock() 37 | defer s.Unlock() 38 | 39 | s.progressDone += delta 40 | } 41 | 42 | func (s *Step) ProgressAll(delta uint32) { 43 | s.Lock() 44 | defer s.Unlock() 45 | 46 | s.progressAll += delta 47 | } 48 | 49 | func (s *Set) Snapshot() []*Pipeline { 50 | s.RLock() 51 | defer s.RUnlock() 52 | 53 | pipelines := make([]*Pipeline, len(s.pipelines)) 54 | for i, p := range s.pipelines { 55 | pipelines[i] = p.Copy() 56 | } 57 | return pipelines 58 | } 59 | 60 | func (s *Set) Pipeline(name string) *Pipeline { 61 | s.Lock() 62 | defer s.Unlock() 63 | 64 | p := &Pipeline{Name: name} 65 | s.pipelines = append(s.pipelines, p) 66 | 67 | return p 68 | } 69 | 70 | func (p *Pipeline) Step(name string, fn func() error) error { 71 | p.Lock() 72 | 73 | if p.Err != nil { 74 | p.Unlock() 75 | return p.Err 76 | } 77 | 78 | step := &Step{Name: name} 79 | p.Steps = append(p.Steps, step) 80 | 81 | log.Printf("%s {", name) 82 | 83 | step.StartedAt = time.Now() 84 | p.Unlock() 85 | err := fn() 86 | p.Lock() 87 | step.FinishedAt = time.Now() 88 | 89 | log.Printf("} // %s", name) 90 | 91 | step.Err = err 92 | p.Err = err 93 | 94 | p.Unlock() 95 | return err 96 | } 97 | 98 | func (p *Pipeline) Current() *Step { 99 | p.RLock() 100 | defer p.RUnlock() 101 | 102 | if len(p.Steps) == 0 { 103 | return nil 104 | } 105 | 106 | step := p.Steps[len(p.Steps)-1] 107 | if step.FinishedAt.IsZero() { 108 | // step is running 109 | return step 110 | } 111 | 112 | return nil 113 | } 114 | 115 | func (p *Pipeline) Copy() *Pipeline { 116 | p.RLock() 117 | defer p.RUnlock() 118 | 119 | var copy = *p 120 | 121 | steps := make([]*Step, len(p.Steps)) 122 | for i, s := range p.Steps { 123 | s.RLock() 124 | var copy = *s 125 | s.RUnlock() 126 | steps[i] = © 127 | } 128 | 129 | copy.Steps = steps 130 | 131 | return © 132 | } 133 | 134 | type ProgressGroup struct { 135 | step *Step 136 | *errgroup.Group 137 | } 138 | 139 | func (s *Step) ProgressGroup() ProgressGroup { 140 | return ProgressGroup{ 141 | step: s, 142 | Group: new(errgroup.Group), 143 | } 144 | } 145 | 146 | func (s *Step) String() string { 147 | if s.FinishedAt.IsZero() { 148 | // running 149 | s.Lock() 150 | defer s.Unlock() 151 | 152 | if s.progressDone != 0 || s.progressAll != 0 { 153 | return fmt.Sprintf("● %d/%d", s.progressDone, s.progressAll) 154 | } else { 155 | return "●" 156 | } 157 | } else if s.Err != nil { 158 | return fmt.Sprintf("✗ %s", s.Err) 159 | } else { 160 | return fmt.Sprintf("✓ %s", s.FinishedAt.Sub(s.StartedAt)) 161 | } 162 | } 163 | 164 | func (pg *ProgressGroup) Go(fn func() error) { 165 | pg.step.ProgressAll(1) 166 | pg.Group.Go(func() error { 167 | defer pg.step.ProgressDone(1) 168 | return fn() 169 | }) 170 | } 171 | -------------------------------------------------------------------------------- /urlutil/normalize.go: -------------------------------------------------------------------------------- 1 | package urlutil 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | 7 | "golang.org/x/net/idna" 8 | ) 9 | 10 | func CloneURL(u *url.URL) *url.URL { 11 | u2 := *u 12 | return &u2 13 | } 14 | 15 | var defaultPorts = map[string]string{ 16 | "https": "443", 17 | "http": "80", 18 | } 19 | 20 | // NormalizeURL normalizes URL u in such manner: 21 | // - all components should be represented in ASCII 22 | // - precent encoding in upper case 23 | // https://datatracker.ietf.org/doc/html/rfc3986#section-6 24 | func NormalizeURL(u *url.URL) (*url.URL, error) { 25 | u = CloneURL(u) 26 | 27 | u.Scheme = strings.ToLower(u.Scheme) 28 | 29 | port := u.Port() 30 | if port == defaultPorts[u.Scheme] { 31 | port = "" 32 | } 33 | 34 | hostname, err := idna.ToASCII(u.Hostname()) 35 | if err != nil { 36 | return nil, err 37 | } 38 | hostname = strings.ToLower(hostname) 39 | 40 | u.Host = hostname 41 | if port != "" { 42 | u.Host += ":" + port 43 | } 44 | 45 | path := u.RawPath 46 | if path == "" { 47 | path = u.Path 48 | } 49 | if path == "" { 50 | path = "/" 51 | } 52 | 53 | path, err = normalizeComponent(path, "/", escapeModePath) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | path = removeDotSegments(path) 59 | 60 | u.Path, err = url.PathUnescape(path) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | u.RawPath = "" 66 | 67 | u.RawQuery, err = normalizeComponent(u.RawQuery, "&;=", escapeModeQuery) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | return u, nil 73 | } 74 | 75 | // https://datatracker.ietf.org/doc/html/rfc3986#section-5.2.4 76 | func removeDotSegments(in string) string { 77 | segs := strings.Split(in, "/") 78 | result := make([]string, 0, len(segs)) 79 | 80 | isAbsolute := false 81 | if segs[0] == "" { 82 | isAbsolute = true 83 | segs = segs[1:] 84 | } 85 | 86 | for i, seg := range segs { 87 | switch seg { 88 | case ".": 89 | // nop 90 | if i == len(segs)-1 { 91 | result = append(result, "") 92 | } 93 | case "..": 94 | if len(result) > 0 { 95 | result = result[:len(result)-1] 96 | } 97 | if i == len(segs)-1 { 98 | result = append(result, "") 99 | } 100 | default: 101 | result = append(result, seg) 102 | } 103 | } 104 | 105 | resultPath := strings.Join(result, "/") 106 | if isAbsolute { 107 | resultPath = "/" + resultPath 108 | } 109 | return resultPath 110 | } 111 | 112 | type escapeMode int 113 | 114 | const ( 115 | escapeModeQuery escapeMode = iota 116 | escapeModePath 117 | ) 118 | 119 | func normalizeComponent(component string, special string, mode escapeMode) (string, error) { 120 | escaped := "" 121 | 122 | type escapeFuncs struct { 123 | escape func(string) string 124 | unescape func(string) (string, error) 125 | } 126 | 127 | var e escapeFuncs 128 | if mode == escapeModeQuery { 129 | e = escapeFuncs{ 130 | escape: url.QueryEscape, 131 | unescape: url.QueryUnescape, 132 | } 133 | } else if mode == escapeModePath { 134 | e = escapeFuncs{ 135 | escape: url.PathEscape, 136 | unescape: url.PathUnescape, 137 | } 138 | } 139 | 140 | for component != "" { 141 | var part, sep string 142 | if i := strings.IndexAny(component, special); i >= 0 { 143 | part, sep, component = component[:i], component[i:i+1], component[i+1:] 144 | } else { 145 | part, sep, component = component, "", "" 146 | } 147 | 148 | unescaped, err := e.unescape(part) 149 | if err != nil { 150 | return "", err 151 | } 152 | 153 | escaped += e.escape(unescaped) + sep 154 | } 155 | 156 | return escaped, nil 157 | } 158 | -------------------------------------------------------------------------------- /giturl/giturl_test.go: -------------------------------------------------------------------------------- 1 | package giturl 2 | 3 | import "testing" 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "strings" 10 | ) 11 | 12 | func checkParseGitURL(t *testing.T, url string) { 13 | t.Logf("URL: %s", url) 14 | 15 | got := map[string]string{} 16 | proto, host, port, path, exotic, err := ParseGitURL(url) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | if exotic == true { 21 | t.Errorf("protocol should not be exotic: %q", url) 22 | } 23 | 24 | got["url"] = url 25 | got["protocol"] = proto 26 | got["path"] = path 27 | 28 | if proto == "ssh" { 29 | got["userandhost"] = host 30 | } else { 31 | got["hostandport"] = host 32 | } 33 | 34 | if port != 0 { 35 | got["port"] = fmt.Sprint(port) 36 | } else { 37 | got["port"] = "NONE" 38 | } 39 | 40 | b, err := exec.Command("git", "fetch-pack", "--diag-url", url).Output() 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | expected := map[string]string{} 46 | lines := strings.Split(string(b), "\n") 47 | for _, line := range lines { 48 | if !strings.HasPrefix(line, "Diag: ") { 49 | continue 50 | } 51 | kv := strings.SplitN(line[len("Diag: "):], "=", 2) 52 | expected[kv[0]] = kv[1] 53 | } 54 | 55 | for k, v := range expected { 56 | if got[k] != v { 57 | t.Errorf("%s expected %q but got %q", k, v, got[k]) 58 | } 59 | } 60 | } 61 | 62 | func TestMain(m *testing.M) { 63 | cmd := exec.Command("git", "version") 64 | b, err := cmd.Output() 65 | if err != nil { 66 | fmt.Fprintln(os.Stderr, err) 67 | os.Exit(2) 68 | } 69 | 70 | version := string(b[:len(b)-1]) 71 | if !strings.HasPrefix(version, "git version 2.") { 72 | fmt.Fprintf(os.Stderr, "git version ~2 required: got [%s]\n", version) 73 | os.Exit(0) 74 | } 75 | 76 | fmt.Fprintf(os.Stderr, "# %s\n", b[:len(b)-1]) 77 | os.Exit(m.Run()) 78 | } 79 | 80 | // source: t/t5500-fetch-pack.sh 81 | func TestParseGitURL(t *testing.T) { 82 | for _, repo := range []string{"repo", "re:po", "re/po"} { 83 | for _, proto := range []string{"ssh+git", "git+ssh", "git", "ssh"} { 84 | for _, host := range []string{"host", "user@host", "user@[::1]", "user@::1"} { 85 | checkParseGitURL(t, fmt.Sprintf("%s://%s/%s", proto, host, repo)) 86 | checkParseGitURL(t, fmt.Sprintf("%s://%s/~%s", proto, host, repo)) 87 | } 88 | for _, host := range []string{"host", "User@host", "User@[::1]"} { 89 | checkParseGitURL(t, fmt.Sprintf("%s://%s:22/%s", proto, host, repo)) 90 | } 91 | } 92 | for _, proto := range []string{"file"} { 93 | checkParseGitURL(t, fmt.Sprintf("%s:///%s", proto, repo)) 94 | checkParseGitURL(t, fmt.Sprintf("%s:///~%s", proto, repo)) 95 | } 96 | for _, host := range []string{"nohost", "nohost:12", "[::1]", "[::1]:23", "[", "[:aa"} { 97 | checkParseGitURL(t, fmt.Sprintf("./%s:%s", host, repo)) 98 | checkParseGitURL(t, fmt.Sprintf("./:%s/~%s", host, repo)) 99 | } 100 | for _, host := range []string{"host", "[::1]"} { 101 | checkParseGitURL(t, fmt.Sprintf("%s:%s", host, repo)) 102 | checkParseGitURL(t, fmt.Sprintf("%s:/~%s", host, repo)) 103 | } 104 | } 105 | } 106 | 107 | func TestParseGitURL_ExtraSCPLike(t *testing.T) { 108 | for _, repo := range []string{"repo", "re:po", "re/po"} { 109 | for _, host := range []string{"user@host", "user@[::1]"} { 110 | checkParseGitURL(t, fmt.Sprintf("%s:%s", host, repo)) 111 | checkParseGitURL(t, fmt.Sprintf("%s:/~%s", host, repo)) 112 | } 113 | } 114 | } 115 | 116 | func TestParseGitURL_HTTP(t *testing.T) { 117 | for _, repo := range []string{"repo", "re:po", "re/po"} { 118 | for _, host := range []string{"host", "host:80"} { 119 | for _, proto := range []string{"http", "https"} { 120 | p, _, _, _, exotic, err := ParseGitURL(fmt.Sprintf("%s://%s/%s", proto, host, repo)) 121 | if err != nil { 122 | t.Error(err) 123 | } else if p != proto { 124 | t.Errorf("expected protocol %q but got %q", proto, p) 125 | } else if exotic == false { 126 | t.Errorf("protocol %q should be exotic", p) 127 | } 128 | } 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /netutil/blocklist.go: -------------------------------------------------------------------------------- 1 | // Package netutil provides net-related utility functions/types. 2 | package netutil 3 | 4 | import ( 5 | "fmt" 6 | "net" 7 | "syscall" 8 | ) 9 | 10 | // PrivateNetworkBlocklist is a blocklist that blocks dialing to private networks. 11 | var PrivateNetworkBlocklist NetworkBlocklist 12 | 13 | // NetworkBlocklist is a blocklist that blocks dialing to specified networks. 14 | type NetworkBlocklist struct { 15 | V4 []NamedNetwork 16 | V6 []NamedNetwork 17 | } 18 | 19 | // Control is intended to be passed to net.Dialer.Control in order to block dialing to networks specified in l. 20 | func (l NetworkBlocklist) Control(network, address string, c syscall.RawConn) error { 21 | host, _, err := net.SplitHostPort(address) 22 | if err != nil { 23 | return fmt.Errorf("cannot parse address %q: %w", address, err) 24 | } 25 | 26 | addr := net.ParseIP(host) 27 | if addr == nil { 28 | return fmt.Errorf("cannot parse host %q", host) 29 | } 30 | 31 | if addr.To4() != nil { 32 | for _, n := range l.V4 { 33 | if n.IPNet.Contains(addr) { 34 | return ErrBlocked{ 35 | Host: host, 36 | Network: n, 37 | } 38 | } 39 | } 40 | } else if addr.To16() != nil { 41 | for _, n := range l.V6 { 42 | if n.IPNet.Contains(addr) { 43 | return ErrBlocked{ 44 | Host: host, 45 | Network: n, 46 | } 47 | } 48 | } 49 | } else { 50 | return fmt.Errorf("BUG: unreachable") 51 | } 52 | 53 | return nil 54 | } 55 | 56 | // ErrBlocked is an error returned by NetworkBlocklist.Control (thus net.Dialer.DialContext) when 57 | // outgoing host is blocked by NetworkBlocklist. 58 | type ErrBlocked struct { 59 | Host string 60 | Network NamedNetwork 61 | } 62 | 63 | func (e ErrBlocked) Error() string { 64 | message := "host is blocked" 65 | if e.Network.Name != "" { 66 | message += fmt.Sprintf(" (%s)", e.Network.Name) 67 | } 68 | return message 69 | } 70 | 71 | type NamedNetwork struct { 72 | IPNet *net.IPNet 73 | Name string 74 | } 75 | 76 | type unparsedNamedNetwork struct { 77 | ip string 78 | name string 79 | } 80 | 81 | func MustParseCIDR(cidr string) *net.IPNet { 82 | _, ipNet, err := net.ParseCIDR(cidr) 83 | if err != nil { 84 | panic(fmt.Sprintf("cannot parse CIDR %q: %s", cidr, err)) 85 | } 86 | 87 | return ipNet 88 | } 89 | 90 | func init() { 91 | PrivateNetworkBlocklist.V4 = make([]NamedNetwork, len(privateNetworksV4)) 92 | for i, p := range privateNetworksV4 { 93 | PrivateNetworkBlocklist.V4[i] = NamedNetwork{ 94 | IPNet: MustParseCIDR(p.ip), 95 | Name: p.name, 96 | } 97 | } 98 | 99 | PrivateNetworkBlocklist.V6 = make([]NamedNetwork, len(privateNetworksV6)) 100 | for i, p := range privateNetworksV6 { 101 | PrivateNetworkBlocklist.V6[i] = NamedNetwork{ 102 | IPNet: MustParseCIDR(p.ip), 103 | Name: p.name, 104 | } 105 | } 106 | } 107 | 108 | // https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml 109 | var privateNetworksV4 = []unparsedNamedNetwork{ 110 | {"0.0.0.0/8", `"This network"`}, 111 | {"0.0.0.0/32", `"This host on this network"`}, 112 | {"127.0.0.0/8", "Loopback"}, 113 | {"255.255.255.255/32", "Limited Broadcast"}, 114 | {"240.0.0.0/4", "Reserved"}, 115 | {"10.0.0.0/8", "Private-Use"}, 116 | {"172.16.0.0/12", "Private-Use"}, 117 | {"192.168.0.0/16", "Private-Use"}, 118 | {"198.18.0.0/15", "Benchmarking"}, 119 | {"192.88.99.0/24", "Deprecated (6to4 Relay Anycast)"}, 120 | {"169.254.0.0/16", "Link Local"}, 121 | {"192.0.0.0/24", "IETF Protocol Assignments"}, 122 | {"192.0.2.0/24", "Documentation (TEST-NET-1)"}, 123 | {"198.51.100.0/24", "Documentation (TEST-NET-2)"}, 124 | {"203.0.113.0/24", "Documentation (TEST-NET-3)"}, 125 | {"192.0.0.0/29", "IPv4 Service Continuity Prefix"}, 126 | {"100.64.0.0/10", "Shared Address Space"}, 127 | {"192.0.0.170/32", "NAT64/DNS64 Discovery"}, 128 | {"192.0.0.171/32", "NAT64/DNS64 Discovery"}, 129 | {"192.0.0.8/32", "IPv4 dummy address"}, 130 | } 131 | 132 | // https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml 133 | var privateNetworksV6 = []unparsedNamedNetwork{ 134 | {"2001::/23", "IETF Protocol Assignments"}, 135 | {"2002::/16", "6to4"}, 136 | {"2001:db8::/32", "Documentation"}, 137 | {"fc00::/7", "Unique-Local"}, 138 | {"2001::/32", "TEREDO"}, 139 | {"::1/128", "Loopback Address"}, 140 | {"::/128", "Unspecified Address"}, 141 | {"::ffff:0:0/96", "IPv4-mapped Address"}, 142 | {"fe80::/10", "Link-Local Unicast"}, 143 | {"2001:10::/28", "Deprecated (previously ORCHID)"}, 144 | {"2001:2::/48", "Benchmarking"}, 145 | {"100::/64", "Discard-Only Address Block"}, 146 | {"64:ff9b:1::/48", "IPv4-IPv6 Translat."}, 147 | } 148 | -------------------------------------------------------------------------------- /chardet/charset.go: -------------------------------------------------------------------------------- 1 | // Original code is golang.org/x/net/html/charset. See LICENSE_charset.txt. 2 | 3 | package chardet 4 | 5 | import ( 6 | "bytes" 7 | "mime" 8 | "strings" 9 | "unicode/utf8" 10 | 11 | "golang.org/x/net/html" 12 | "golang.org/x/net/html/charset" 13 | "golang.org/x/text/encoding" 14 | "golang.org/x/text/encoding/charmap" 15 | ) 16 | 17 | func DetermineEncoding(content []byte, contentType string, detect func([]byte) (encoding.Encoding, string)) (e encoding.Encoding, name string, certain bool) { 18 | if len(content) > 1024 { 19 | content = content[:1024] 20 | } 21 | 22 | for _, b := range boms { 23 | if bytes.HasPrefix(content, b.bom) { 24 | e, name = charset.Lookup(b.enc) 25 | return e, name, true 26 | } 27 | } 28 | 29 | if _, params, err := mime.ParseMediaType(contentType); err == nil { 30 | if cs, ok := params["charset"]; ok { 31 | if e, name = charset.Lookup(cs); e != nil { 32 | return e, name, true 33 | } 34 | } 35 | } 36 | 37 | if len(content) > 0 { 38 | e, name = prescan(content) 39 | if e != nil { 40 | return e, name, false 41 | } 42 | } 43 | 44 | if detect != nil { 45 | e, name = detect(content) 46 | if e != nil { 47 | return e, name, false 48 | } 49 | } 50 | 51 | // Try to detect UTF-8. 52 | // First eliminate any partial rune at the end. 53 | for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- { 54 | b := content[i] 55 | if b < 0x80 { 56 | break 57 | } 58 | if utf8.RuneStart(b) { 59 | content = content[:i] 60 | break 61 | } 62 | } 63 | hasHighBit := false 64 | for _, c := range content { 65 | if c >= 0x80 { 66 | hasHighBit = true 67 | break 68 | } 69 | } 70 | if hasHighBit && utf8.Valid(content) { 71 | return encoding.Nop, "utf-8", false 72 | } 73 | 74 | // TODO: change default depending on user's locale? 75 | return charmap.Windows1252, "windows-1252", false 76 | } 77 | 78 | func prescan(content []byte) (e encoding.Encoding, name string) { 79 | z := html.NewTokenizer(bytes.NewReader(content)) 80 | for { 81 | switch z.Next() { 82 | case html.ErrorToken: 83 | return nil, "" 84 | 85 | case html.StartTagToken, html.SelfClosingTagToken: 86 | tagName, hasAttr := z.TagName() 87 | if !bytes.Equal(tagName, []byte("meta")) { 88 | continue 89 | } 90 | attrList := make(map[string]bool) 91 | gotPragma := false 92 | 93 | const ( 94 | dontKnow = iota 95 | doNeedPragma 96 | doNotNeedPragma 97 | ) 98 | needPragma := dontKnow 99 | 100 | name = "" 101 | e = nil 102 | for hasAttr { 103 | var key, val []byte 104 | key, val, hasAttr = z.TagAttr() 105 | ks := string(key) 106 | if attrList[ks] { 107 | continue 108 | } 109 | attrList[ks] = true 110 | for i, c := range val { 111 | if 'A' <= c && c <= 'Z' { 112 | val[i] = c + 0x20 113 | } 114 | } 115 | 116 | switch ks { 117 | case "http-equiv": 118 | if bytes.Equal(val, []byte("content-type")) { 119 | gotPragma = true 120 | } 121 | 122 | case "content": 123 | if e == nil { 124 | name = fromMetaElement(string(val)) 125 | if name != "" { 126 | e, name = charset.Lookup(name) 127 | if e != nil { 128 | needPragma = doNeedPragma 129 | } 130 | } 131 | } 132 | 133 | case "charset": 134 | e, name = charset.Lookup(string(val)) 135 | needPragma = doNotNeedPragma 136 | } 137 | } 138 | 139 | if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma { 140 | continue 141 | } 142 | 143 | if strings.HasPrefix(name, "utf-16") { 144 | name = "utf-8" 145 | e = encoding.Nop 146 | } 147 | 148 | if e != nil { 149 | return e, name 150 | } 151 | } 152 | } 153 | } 154 | 155 | func fromMetaElement(s string) string { 156 | for s != "" { 157 | csLoc := strings.Index(s, "charset") 158 | if csLoc == -1 { 159 | return "" 160 | } 161 | s = s[csLoc+len("charset"):] 162 | s = strings.TrimLeft(s, " \t\n\f\r") 163 | if !strings.HasPrefix(s, "=") { 164 | continue 165 | } 166 | s = s[1:] 167 | s = strings.TrimLeft(s, " \t\n\f\r") 168 | if s == "" { 169 | return "" 170 | } 171 | if q := s[0]; q == '"' || q == '\'' { 172 | s = s[1:] 173 | closeQuote := strings.IndexRune(s, rune(q)) 174 | if closeQuote == -1 { 175 | return "" 176 | } 177 | return s[:closeQuote] 178 | } 179 | 180 | end := strings.IndexAny(s, "; \t\n\f\r") 181 | if end == -1 { 182 | end = len(s) 183 | } 184 | return s[:end] 185 | } 186 | return "" 187 | } 188 | 189 | var boms = []struct { 190 | bom []byte 191 | enc string 192 | }{ 193 | {[]byte{0xfe, 0xff}, "utf-16be"}, 194 | {[]byte{0xff, 0xfe}, "utf-16le"}, 195 | {[]byte{0xef, 0xbb, 0xbf}, "utf-8"}, 196 | } 197 | -------------------------------------------------------------------------------- /oauth2util/oauth2util.go: -------------------------------------------------------------------------------- 1 | package oauth2util 2 | 3 | import ( 4 | "context" 5 | "crypto/rand" 6 | "crypto/sha256" 7 | "encoding/json" 8 | "fmt" 9 | "net/http" 10 | "net/http/httptest" 11 | "os" 12 | "path/filepath" 13 | 14 | "golang.org/x/oauth2" 15 | ) 16 | 17 | // Config encapsulates typical OAuth2 authorization flow: 18 | // 1. Try to restore previously-saved token, 19 | // 2. If not available, start a local server for receiving code and prompt its URL, 20 | // 3. Obtain an access token when code is received, 21 | // 4. Store the token for later use. 22 | type Config struct { 23 | // Required 24 | OAuth2Config *oauth2.Config 25 | 26 | // Required if TokenFile is empty 27 | Name string 28 | 29 | AuthCodeOptions []oauth2.AuthCodeOption 30 | 31 | // Defaults to //token.json 32 | TokenFile string 33 | } 34 | 35 | func (c *Config) DeleteTokenFile() (string, error) { 36 | tokenFile, err := c.getTokenFile() 37 | if err != nil { 38 | return "", err 39 | } 40 | 41 | return tokenFile, os.Remove(tokenFile) 42 | } 43 | 44 | func (c *Config) getTokenFile() (string, error) { 45 | if c.TokenFile != "" { 46 | return c.TokenFile, nil 47 | } 48 | 49 | cacheDirBase, err := os.UserCacheDir() 50 | if err != nil { 51 | return "", (fmt.Errorf("os.UserCacheDir: %w", err)) 52 | } 53 | 54 | c.TokenFile = filepath.Join(cacheDirBase, c.Name, "token.json") 55 | 56 | return c.TokenFile, nil 57 | } 58 | 59 | // CreateOAuth2Client handles a typical authorization flow. See Config. 60 | func (c *Config) CreateOAuth2Client(ctx context.Context) (*http.Client, error) { 61 | token, err := c.restoreToken() 62 | if err != nil { 63 | token, err = c.AuthorizeByTemporaryServer( 64 | ctx, 65 | func(authURL string) error { 66 | fmt.Printf("Visit below to authorize:\n%s\n", authURL) 67 | return nil 68 | }, 69 | ) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | err = c.storeToken(token) 75 | if err != nil { 76 | return nil, err 77 | } 78 | } 79 | 80 | return c.OAuth2Config.Client(ctx, token), nil 81 | } 82 | 83 | type CodeReceiver struct { 84 | ch chan string 85 | State string 86 | *httptest.Server 87 | } 88 | 89 | func (c CodeReceiver) Code() <-chan string { 90 | return c.ch 91 | } 92 | 93 | func NewCodeReceiver() (*CodeReceiver, error) { 94 | ch := make(chan string) 95 | 96 | buf := make([]byte, 16) 97 | _, err := rand.Read(buf) 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | state := fmt.Sprintf("%x", sha256.Sum256(buf)) 103 | 104 | s := httptest.NewServer( 105 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 106 | if r.URL.Path == "/favicon.ico" { 107 | http.Error(w, "Not Found", 404) 108 | return 109 | } 110 | 111 | if code := r.FormValue("code"); code != "" { 112 | if r.FormValue("state") != state { 113 | http.Error(w, "State mismatch", 400) 114 | return 115 | } 116 | 117 | w.Header().Set("Content-Type", "text/plain") 118 | fmt.Fprintln(w, "Authorized.") 119 | ch <- code 120 | return 121 | } 122 | })) 123 | 124 | return &CodeReceiver{ 125 | ch: ch, 126 | State: state, 127 | Server: s, 128 | }, nil 129 | } 130 | 131 | func (c *Config) AuthorizeByTemporaryServer(ctx context.Context, prompt func(url string) error) (*oauth2.Token, error) { 132 | recv, err := NewCodeReceiver() 133 | if err != nil { 134 | return nil, err 135 | } 136 | 137 | defer recv.Close() 138 | 139 | var oauth2ConfigCopy oauth2.Config = *c.OAuth2Config 140 | oauth2ConfigCopy.RedirectURL = recv.URL 141 | 142 | err = prompt(oauth2ConfigCopy.AuthCodeURL(recv.State, c.AuthCodeOptions...)) 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | select { 148 | case <-ctx.Done(): 149 | return nil, ctx.Err() 150 | 151 | case code := <-recv.Code(): 152 | return oauth2ConfigCopy.Exchange(ctx, code, c.AuthCodeOptions...) 153 | } 154 | } 155 | 156 | func (c *Config) restoreToken() (*oauth2.Token, error) { 157 | tokenFile, err := c.getTokenFile() 158 | if err != nil { 159 | return nil, err 160 | } 161 | 162 | f, err := os.Open(tokenFile) 163 | if err != nil { 164 | return nil, err 165 | } 166 | defer f.Close() 167 | 168 | var token oauth2.Token 169 | err = json.NewDecoder(f).Decode(&token) 170 | return &token, err 171 | } 172 | 173 | func (c *Config) storeToken(token *oauth2.Token) error { 174 | tokenFile, err := c.getTokenFile() 175 | if err != nil { 176 | return err 177 | } 178 | 179 | err = os.MkdirAll(filepath.Dir(tokenFile), 0o777) 180 | if err != nil { 181 | return err 182 | } 183 | 184 | f, err := os.OpenFile(tokenFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) 185 | if err != nil { 186 | return err 187 | } 188 | defer f.Close() 189 | 190 | return json.NewEncoder(f).Encode(token) 191 | } 192 | -------------------------------------------------------------------------------- /stringstringmap/stringstringmap_test.go: -------------------------------------------------------------------------------- 1 | package stringstringmap 2 | 3 | import ( 4 | "encoding" 5 | "encoding/base64" 6 | "reflect" 7 | "strconv" 8 | "testing" 9 | "time" 10 | 11 | "github.com/google/go-cmp/cmp" 12 | "github.com/google/go-cmp/cmp/cmpopts" 13 | ) 14 | 15 | type base64EncodedString string 16 | 17 | var ( 18 | _ encoding.TextMarshaler = (*base64EncodedString)(nil) 19 | _ encoding.TextUnmarshaler = (*base64EncodedString)(nil) 20 | ) 21 | 22 | func (s base64EncodedString) MarshalText() ([]byte, error) { 23 | return []byte(base64.StdEncoding.EncodeToString([]byte(s))), nil 24 | } 25 | 26 | func (s *base64EncodedString) UnmarshalText(b []byte) error { 27 | b, err := base64.StdEncoding.DecodeString(string(b)) 28 | *s = base64EncodedString(string(b)) 29 | return err 30 | } 31 | 32 | type s struct { 33 | Int int 34 | Uint uint 35 | Float float64 36 | String string 37 | Time time.Time 38 | Bool bool 39 | Omitempty int `stringstringmap:",omitempty"` 40 | Base64 base64EncodedString 41 | Embedded 42 | Enum enum 43 | Skip string `stringstringmap:"-"` 44 | Named string `stringstringmap:"customname"` 45 | //lint:ignore U1000 testing purpose 46 | unexported int 47 | } 48 | 49 | type enum int 50 | 51 | const ( 52 | e0 enum = iota 53 | e1 54 | e2 55 | e3 56 | ) 57 | 58 | type Embedded struct { 59 | String2 string 60 | } 61 | 62 | func TestEncodeDecode(t *testing.T) { 63 | tests := []struct { 64 | name string 65 | value interface{} 66 | marshaled map[string]string 67 | unmarshaled interface{} 68 | error string 69 | }{ 70 | { 71 | name: "complicated struct", 72 | value: s{ 73 | Int: -99, 74 | Uint: 100, 75 | Float: 3.14, 76 | String: "foo", 77 | Time: time.Unix(12345, 0).UTC(), 78 | Bool: true, 79 | Base64: "hello", 80 | Embedded: Embedded{ 81 | String2: "bar", 82 | }, 83 | Enum: e1, 84 | Skip: "skipthis", 85 | Named: "named", 86 | }, 87 | unmarshaled: &s{}, 88 | marshaled: map[string]string{ 89 | "Int": "-99", 90 | "Uint": "100", 91 | "Float": "3.14", 92 | "String": "foo", 93 | "Time": "12345", 94 | "Bool": "true", 95 | "Base64": "aGVsbG8=", 96 | "String2": "bar", 97 | "Enum": "1", 98 | "customname": "named", 99 | }, 100 | }, 101 | { 102 | name: "unsupported type", 103 | value: struct { 104 | C chan struct{} 105 | }{ 106 | make(chan struct{}), 107 | }, 108 | error: "encoding field C: unsupported type chan struct {}", 109 | }, 110 | { 111 | name: "marshalling pointer", 112 | value: &struct { 113 | S string 114 | }{ 115 | S: "a", 116 | }, 117 | unmarshaled: &struct{ S string }{}, 118 | marshaled: map[string]string{ 119 | "S": "a", 120 | }, 121 | }, 122 | } 123 | 124 | e := Encoder{ 125 | OverrideEncode: func(v interface{}, field reflect.StructField) (string, error) { 126 | if t, ok := v.(time.Time); ok { 127 | n := t.Unix() 128 | return strconv.Itoa(int(n)), nil 129 | } 130 | 131 | return "", ErrSkipOverride 132 | }, 133 | } 134 | 135 | d := Decoder{ 136 | OverrideDecode: func(s string, v interface{}, field reflect.StructField) error { 137 | if t, ok := v.(*time.Time); ok { 138 | n, err := strconv.Atoi(s) 139 | if err != nil { 140 | return err 141 | } 142 | *t = time.Unix(int64(n), 0) 143 | return nil 144 | } 145 | 146 | return ErrSkipOverride 147 | }, 148 | } 149 | 150 | for _, test := range tests { 151 | t.Run(test.name, func(t *testing.T) { 152 | if test.error != "" { 153 | _, err := e.Encode(test.value) 154 | if err == nil { 155 | t.Error("should err") 156 | } else if test.error != err.Error() { 157 | t.Errorf("expected error %s but got %s", test.error, err.Error()) 158 | } 159 | return 160 | } 161 | 162 | m, err := e.Encode(test.value) 163 | if err != nil { 164 | t.Fatal(err) 165 | } 166 | 167 | if diff := cmp.Diff(test.marshaled, m); diff != "" { 168 | t.Fatalf("comparing marshaled got diff:\n%s", diff) 169 | } 170 | 171 | unmarshalled := test.unmarshaled 172 | err = d.Decode(m, unmarshalled) 173 | if err != nil { 174 | t.Fatal(err) 175 | } 176 | 177 | value := test.value 178 | if reflect.ValueOf(value).Kind() == reflect.Ptr { 179 | value = indirect(value) 180 | } 181 | if diff := cmp.Diff( 182 | value, 183 | indirect(unmarshalled), 184 | cmp.FilterPath(func(p cmp.Path) bool { return p.String() == "Skip" }, cmp.Ignore()), 185 | cmpopts.IgnoreUnexported(value), 186 | ); diff != "" { 187 | t.Fatalf("comparing unmarshaled got diff:\n%s", diff) 188 | } 189 | }) 190 | } 191 | } 192 | 193 | func indirect(v interface{}) interface{} { 194 | return reflect.Indirect(reflect.ValueOf(v)).Interface() 195 | } 196 | 197 | func TestEncodeDecode_Omitempty(t *testing.T) { 198 | e := Encoder{ 199 | Omitempty: true, 200 | } 201 | d := Decoder{ 202 | Omitempty: true, 203 | } 204 | 205 | v := struct { 206 | Int int 207 | String string 208 | }{ 209 | 0, 210 | "", 211 | } 212 | 213 | m, err := e.Encode(v) 214 | if err != nil { 215 | t.Fatalf("got error: %v", err) 216 | } 217 | 218 | err = d.Decode(m, &v) 219 | if err != nil { 220 | t.Fatalf("got error: %v", err) 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 4 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 5 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 6 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 7 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 8 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 9 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 10 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 11 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 12 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 13 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 14 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 15 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 16 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 17 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 18 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 19 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= 20 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= 21 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 22 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 23 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 24 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 25 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 26 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 27 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 28 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 29 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 30 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 31 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 32 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 33 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 34 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 35 | golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= 36 | golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= 37 | golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= 38 | golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= 39 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 40 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 41 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 42 | golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= 43 | golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= 44 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 45 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 46 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 47 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 48 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 50 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 51 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 52 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 53 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 54 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 55 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 56 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 57 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 58 | golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= 59 | golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= 60 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 61 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 62 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 63 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 64 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 65 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 66 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 67 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 68 | google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= 69 | google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= 70 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 71 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 72 | google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= 73 | google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 74 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 75 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 76 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 77 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 78 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 79 | -------------------------------------------------------------------------------- /stringstringmap/stringstringmap.go: -------------------------------------------------------------------------------- 1 | package stringstringmap 2 | 3 | import ( 4 | "encoding" 5 | "fmt" 6 | "reflect" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | var ErrSkipOverride = fmt.Errorf("skip this override") 12 | 13 | type Encoder struct { 14 | OverrideEncode func(v interface{}, field reflect.StructField) (string, error) 15 | Omitempty bool 16 | } 17 | 18 | type Decoder struct { 19 | OverrideDecode func(s string, v interface{}, field reflect.StructField) error 20 | Omitempty bool 21 | } 22 | 23 | func (e Encoder) encodeToText(v interface{}, field reflect.StructField) (string, error) { 24 | if e.OverrideEncode != nil { 25 | s, err := e.OverrideEncode(v, field) 26 | if err == nil { 27 | return s, nil 28 | } else if err != ErrSkipOverride { 29 | return "", err 30 | } 31 | } 32 | 33 | if m, ok := v.(encoding.TextMarshaler); ok { 34 | b, err := m.MarshalText() 35 | if err != nil { 36 | return "", err 37 | } 38 | return string(b), nil 39 | } 40 | 41 | rv := reflect.ValueOf(v) 42 | switch rv.Kind() { 43 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 44 | return strconv.FormatInt(rv.Int(), 10), nil 45 | 46 | case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: 47 | return strconv.FormatUint(rv.Uint(), 10), nil 48 | 49 | case reflect.String: 50 | return rv.String(), nil 51 | 52 | case reflect.Bool: 53 | return strconv.FormatBool(rv.Bool()), nil 54 | 55 | case reflect.Float32, reflect.Float64: 56 | return strconv.FormatFloat(rv.Float(), 'g', -1, 64), nil 57 | 58 | case reflect.Array: 59 | case reflect.Chan: 60 | case reflect.Complex128: 61 | case reflect.Complex64: 62 | case reflect.Func: 63 | case reflect.Interface: 64 | case reflect.Invalid: 65 | case reflect.Map: 66 | case reflect.Ptr: 67 | case reflect.Slice: 68 | case reflect.Struct: 69 | case reflect.Uintptr: 70 | case reflect.UnsafePointer: 71 | } 72 | 73 | return "", fmt.Errorf("unsupported type %T", v) 74 | } 75 | 76 | func (d Decoder) decodeFromText(s string, v interface{}, field reflect.StructField) error { 77 | if d.OverrideDecode != nil { 78 | err := d.OverrideDecode(s, v, field) 79 | if err == nil { 80 | return nil 81 | } else if err != ErrSkipOverride { 82 | return err 83 | } 84 | } 85 | 86 | if u, ok := v.(encoding.TextUnmarshaler); ok { 87 | return u.UnmarshalText([]byte(s)) 88 | } 89 | 90 | pv := reflect.ValueOf(v) 91 | if pv.Kind() != reflect.Ptr { 92 | return fmt.Errorf("want pointer; got %v (%T)", v, v) 93 | } 94 | 95 | rv := pv.Elem() 96 | switch rv.Kind() { 97 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 98 | i, err := strconv.ParseInt(s, 10, 0) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | rv.SetInt(i) 104 | return nil 105 | 106 | case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: 107 | i, err := strconv.ParseUint(s, 10, 0) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | rv.SetUint(i) 113 | return nil 114 | 115 | case reflect.Bool: 116 | b, err := strconv.ParseBool(s) 117 | if err != nil { 118 | return err 119 | } 120 | rv.SetBool(b) 121 | return nil 122 | 123 | case reflect.String: 124 | rv.SetString(s) 125 | return nil 126 | 127 | case reflect.Float32, reflect.Float64: 128 | f, err := strconv.ParseFloat(s, 64) 129 | if err != nil { 130 | return err 131 | } 132 | rv.SetFloat(f) 133 | return nil 134 | 135 | case reflect.Array: 136 | case reflect.Chan: 137 | case reflect.Complex128: 138 | case reflect.Complex64: 139 | case reflect.Func: 140 | case reflect.Interface: 141 | case reflect.Invalid: 142 | case reflect.Map: 143 | case reflect.Ptr: 144 | case reflect.Slice: 145 | case reflect.Struct: 146 | case reflect.Uintptr: 147 | case reflect.UnsafePointer: 148 | } 149 | 150 | return fmt.Errorf("unsupported type: %s", rv.Type()) 151 | } 152 | 153 | func (e Encoder) Encode(v interface{}) (map[string]string, error) { 154 | m := map[string]string{} 155 | rv := reflect.ValueOf(v) 156 | return e.encodeToStringStringMap(rv, m) 157 | } 158 | 159 | func (e Encoder) encodeToStringStringMap(rv reflect.Value, m map[string]string) (map[string]string, error) { 160 | rt := rv.Type() 161 | 162 | if rt.Kind() == reflect.Ptr { 163 | rt = rt.Elem() 164 | rv = rv.Elem() 165 | } 166 | 167 | embeddedIdx := []int{} 168 | for i, n := 0, rt.NumField(); i < n; i++ { 169 | if rt.Field(i).PkgPath != "" { 170 | // unexported field 171 | continue 172 | } 173 | if rt.Field(i).Anonymous { 174 | embeddedIdx = append(embeddedIdx, i) 175 | } 176 | } 177 | 178 | // process embedded fields first because they have lower priority 179 | // TODO(motemen): how about decoding? 180 | for _, i := range embeddedIdx { 181 | fv := rv.Field(i) 182 | _, err := e.encodeToStringStringMap(fv, m) 183 | if err != nil { 184 | return nil, fmt.Errorf("encodeToStringStringMap: %w", err) 185 | } 186 | continue 187 | } 188 | 189 | for i, n := 0, rt.NumField(); i < n; i++ { 190 | fv := rv.Field(i) 191 | field := rt.Field(i) 192 | if field.PkgPath != "" { 193 | // unexported field 194 | continue 195 | } 196 | if field.Anonymous { 197 | continue 198 | } 199 | 200 | tag := field.Tag.Get("stringstringmap") 201 | name, opts := parseFieldTag(tag) 202 | 203 | if name == "-" { 204 | continue 205 | } 206 | if name == "" { 207 | name = field.Name 208 | } 209 | if (e.Omitempty || opts == "omitempty") && fv.IsZero() { 210 | continue 211 | } 212 | 213 | var err error 214 | m[name], err = e.encodeToText(fv.Interface(), field) 215 | if err != nil { 216 | return nil, fmt.Errorf("encoding field %s: %w", field.Name, err) 217 | } 218 | } 219 | 220 | return m, nil 221 | } 222 | 223 | func (d Decoder) decodeFromStringStringMap(rv reflect.Value, m map[string]string) error { 224 | rt := rv.Type() 225 | 226 | for i, n := 0, rt.NumField(); i < n; i++ { 227 | fv := rv.Field(i) 228 | field := rt.Field(i) 229 | if !fv.CanSet() { 230 | continue 231 | } 232 | if field.Anonymous { 233 | err := d.decodeFromStringStringMap(fv, m) 234 | if err != nil { 235 | return fmt.Errorf("decoding embedded field %v: %w", rt.Field(i).Name, err) 236 | } 237 | continue 238 | } 239 | 240 | tag := field.Tag.Get("stringstringmap") 241 | name, opts := parseFieldTag(tag) 242 | if name == "-" { 243 | continue 244 | } 245 | if name == "" { 246 | name = field.Name 247 | } 248 | if (d.Omitempty || opts == "omitempty") && m[name] == "" { 249 | continue 250 | } 251 | 252 | err := d.decodeFromText(m[name], fv.Addr().Interface(), field) 253 | if err != nil { 254 | return fmt.Errorf("decoding field %v: %w", field.Name, err) 255 | } 256 | } 257 | 258 | return nil 259 | } 260 | 261 | func (d Decoder) Decode(m map[string]string, v interface{}) error { 262 | pv := reflect.ValueOf(v) 263 | if pv.Elem().Kind() == reflect.Ptr { 264 | pv.Elem().Set(reflect.New(pv.Elem().Type())) 265 | pv = pv.Elem() 266 | } 267 | 268 | // make a copy as decodeFromStringStringMap destroys it 269 | m2 := make(map[string]string, len(m)) 270 | for k, v := range m { 271 | m2[k] = v 272 | } 273 | 274 | return d.decodeFromStringStringMap(pv.Elem(), m2) 275 | } 276 | 277 | func Marshal(v interface{}) (map[string]string, error) { 278 | return Encoder{}.Encode(v) 279 | } 280 | 281 | func Unmarshal(m map[string]string, v interface{}) error { 282 | return Decoder{}.Decode(m, v) 283 | } 284 | 285 | func parseFieldTag(tag string) (name string, opts string) { 286 | if i := strings.Index(tag, ","); i != -1 { 287 | name = tag[:i] 288 | opts = tag[i+1:] 289 | } else { 290 | name = tag 291 | } 292 | return 293 | } 294 | --------------------------------------------------------------------------------