├── behavioral-tests ├── .gitignore ├── pipe-mode │ ├── custom-run.sh │ └── expected.log ├── args-and-signals │ ├── custom-run.sh │ ├── expected.log │ └── custom-fake-server.sh ├── default │ └── expected.log ├── fake-server.sh ├── custom │ ├── expected.log │ └── pplog.env └── run.sh ├── cmd └── pplog │ ├── const.go │ ├── run_nosubproc.go │ ├── run.go │ ├── run_common.go │ └── main.go ├── codecov.yml ├── slogtotext ├── xjson_test.go ├── example_test.go ├── xxjson_test.go ├── xjson.go ├── flat.go ├── flat_test.go ├── example_slog_test.go ├── reader_test.go ├── reader.go ├── xxjson.go ├── formatter.go └── formatter_test.go ├── go.mod ├── LICENSE ├── .golangci.yaml ├── .github └── workflows │ └── ci.yaml ├── go.sum └── README.md /behavioral-tests/.gitignore: -------------------------------------------------------------------------------- 1 | output.log 2 | -------------------------------------------------------------------------------- /cmd/pplog/const.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const buffSize = 32768 4 | -------------------------------------------------------------------------------- /behavioral-tests/pipe-mode/custom-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ../fake-server.sh | 4 | go run ../../cmd/... | 5 | tee output.log 6 | -------------------------------------------------------------------------------- /behavioral-tests/args-and-signals/custom-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | go run ../../cmd/... -c ./custom-fake-server.sh ARG1 ARG2 | tee output.log 4 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | # do not forget to validate it 2 | # curl --data-binary @codecov.yml https://codecov.io/validate 3 | 4 | comment: off 5 | coverage: 6 | precision: 2 7 | range: [75, 90] 8 | round: up 9 | -------------------------------------------------------------------------------- /behavioral-tests/args-and-signals/expected.log: -------------------------------------------------------------------------------- 1 | INVALID JSON: "Arguments: ARG1 ARG2" 2 | INVALID JSON: "Sleeping..." 3 | INVALID JSON: "Going to kill parent process..." 4 | INVALID JSON: "Getting SIGNAL SIGINT. Exiting" 5 | -------------------------------------------------------------------------------- /cmd/pplog/run_nosubproc.go: -------------------------------------------------------------------------------- 1 | //go:build windows || (linux && (arm64 || loong64 || riscv64)) 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | ) 9 | 10 | func runSubprocessModeChild() { 11 | fmt.Fprintln(os.Stderr, "Child mode is not supported in MS Windows") 12 | os.Exit(1) 13 | } 14 | -------------------------------------------------------------------------------- /behavioral-tests/default/expected.log: -------------------------------------------------------------------------------- 1 | 2024-12-02T12:00:00-05:00 [INFO] OK source.file=/Users/slog/main.go source.function=main.main source.line=14 user=1 2 | INVALID JSON: "Broken raw error message" 3 | 2024-12-02T12:05:00-05:00 [INFO] OK source.file=/Users/slog/main.go source.function=main.main source.line=14 user=2 4 | -------------------------------------------------------------------------------- /behavioral-tests/pipe-mode/expected.log: -------------------------------------------------------------------------------- 1 | 2024-12-02T12:00:00-05:00 [INFO] OK source.file=/Users/slog/main.go source.function=main.main source.line=14 user=1 2 | INVALID JSON: "Broken raw error message" 3 | 2024-12-02T12:05:00-05:00 [INFO] OK source.file=/Users/slog/main.go source.function=main.main source.line=14 user=2 4 | -------------------------------------------------------------------------------- /behavioral-tests/fake-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo '{"time":"2024-12-02T12:00:00-05:00","level":"INFO","source":{"function":"main.main","file":"/Users/slog/main.go","line":14},"msg":"OK","user":1}' 4 | echo 'Broken raw error message' 5 | echo '{"time":"2024-12-02T12:05:00-05:00","level":"INFO","source":{"function":"main.main","file":"/Users/slog/main.go","line":14},"msg":"OK","user":2}' 6 | -------------------------------------------------------------------------------- /behavioral-tests/custom/expected.log: -------------------------------------------------------------------------------- 1 | 12:00:00 INFO OK source.file=/Users/slog/main.go source.function=main.main source.line=14 user=1 2 | NOJSON: Broken raw error message 3 | 12:05:00 INFO OK source.file=/Users/slog/main.go source.function=main.main source.line=14 user=2 4 | -------------------------------------------------------------------------------- /behavioral-tests/args-and-signals/custom-fake-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | xtrap() { 4 | echo "Getting SIGNAL $1. Exiting" 5 | exit 0 # it must to be forced 0; some shells can return 130 by default 6 | } 7 | 8 | trap "xtrap SIGINT" 2 # graceful shutdown handler 9 | 10 | echo "Arguments: $@" 11 | 12 | ( 13 | # pretending someone kills process 14 | echo "Sleeping..." 15 | sleep 1 16 | echo "Going to kill parent process..." 17 | kill -2 $$ 18 | ) & 19 | 20 | wait 21 | -------------------------------------------------------------------------------- /slogtotext/xjson_test.go: -------------------------------------------------------------------------------- 1 | package slogtotext 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestTXJson(t *testing.T) { 10 | t.Run("json", func(t *testing.T) { 11 | data := `{"a": true, "b": null, "c": 1, "d": "xxx", "e": "[1,\"{\\\"p\\\":55}\",3]"}` 12 | res := tXJson(data) 13 | assert.Equal(t, `{"a":true,"b":null,"c":1,"d":"xxx","e":[1,{"p":55},3]}`, res) //nolint:testifylint 14 | }) 15 | t.Run("raw", func(t *testing.T) { 16 | data := `xxx` 17 | res := tXJson(data) 18 | assert.Equal(t, `xxx`, res) 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /behavioral-tests/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -x # debug: display commands 4 | set -e # exit immediately on any error 5 | 6 | cleanup() { 7 | test "$?" = 0 && echo "OK" || echo "FAIL" 8 | } 9 | 10 | trap cleanup EXIT 11 | 12 | cd "$(dirname $0)" 13 | 14 | for cs in $(find . -mindepth 1 -maxdepth 1 -type d | sort) 15 | do 16 | ( 17 | cd $cs 18 | if test -x ./custom-run.sh 19 | then 20 | ./custom-run.sh 21 | else 22 | go run ../../cmd/... ../fake-server.sh | tee output.log 23 | fi 24 | diff expected.log output.log # will cause interruption due to -e 25 | rm output.log 26 | echo "OK: $cs" 27 | ) 28 | done -------------------------------------------------------------------------------- /slogtotext/example_test.go: -------------------------------------------------------------------------------- 1 | package slogtotext_test 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | 7 | "github.com/michurin/human-readable-json-logging/slogtotext" 8 | ) 9 | 10 | func Example() { 11 | f := slogtotext.MustFormatter(os.Stdout, `x={{.x}}{{if .ALL | rm "x"}} UNKNOWN:{{range .ALL | rm "x" "p" "q"}} {{.K}}={{.V}}{{end}}{{end}}`+"\n") 12 | g := slogtotext.MustFormatter(os.Stdout, `INVALID LINE: {{ .TEXT | printf "%q" }}`+"\n") 13 | buf := strings.NewReader(`{"x": 100} 14 | {"x": 1, "y": { "a": 2, "b": 3 }, "p": 9, "q": 9} 15 | here`) 16 | err := slogtotext.Read(buf, f, g, 1024) 17 | if err != nil { 18 | panic(err) 19 | } 20 | // Output: 21 | // x=100 22 | // x=1 UNKNOWN: y.a=2 y.b=3 23 | // INVALID LINE: "here" 24 | } 25 | -------------------------------------------------------------------------------- /slogtotext/xxjson_test.go: -------------------------------------------------------------------------------- 1 | //nolint:lll // it's ok for tests 2 | package slogtotext 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTXXJson(t *testing.T) { 11 | t.Run("json", func(t *testing.T) { 12 | data := `{"b":null, "a":true, "c": 1, "d": "xxx", "e": "[1,\"{\\\"p\\\":55}\",3]", "f": false}` // keys must be unordered to cover Swap() 13 | res := tXXJson(data) 14 | assert.Equal(t, "{\x1b[93ma\x1b[0m:\x1b[92mT\x1b[0m,\x1b[93mb\x1b[0m:\x1b[95mN\x1b[0m,\x1b[93mc\x1b[0m:\x1b[95m1\x1b[0m,\x1b[93md\x1b[0m:\x1b[35mxxx\x1b[0m,\x1b[93me\x1b[0m:[\x1b[95m1\x1b[0m,{\x1b[93mp\x1b[0m:\x1b[95m55\x1b[0m},\x1b[95m3\x1b[0m],\x1b[93mf\x1b[0m:\x1b[91mF\x1b[0m}", res) 15 | }) 16 | t.Run("raw", func(t *testing.T) { 17 | data := `xxx` 18 | res := tXXJson(data) 19 | assert.Equal(t, `xxx`, res) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/michurin/human-readable-json-logging 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/Masterminds/sprig/v3 v3.3.0 7 | github.com/michurin/systemd-env-file v0.0.0-20250606021327-1e826f3c7c79 8 | github.com/stretchr/testify v1.11.1 9 | ) 10 | 11 | require ( 12 | dario.cat/mergo v1.0.1 // indirect 13 | github.com/Masterminds/goutils v1.1.1 // indirect 14 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/google/uuid v1.6.0 // indirect 17 | github.com/huandu/xstrings v1.5.0 // indirect 18 | github.com/mitchellh/copystructure v1.2.0 // indirect 19 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 20 | github.com/pmezard/go-difflib v1.0.0 // indirect 21 | github.com/shopspring/decimal v1.4.0 // indirect 22 | github.com/spf13/cast v1.7.0 // indirect 23 | golang.org/x/crypto v0.26.0 // indirect 24 | gopkg.in/yaml.v3 v3.0.1 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /slogtotext/xjson.go: -------------------------------------------------------------------------------- 1 | package slogtotext 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | ) 7 | 8 | func fixAny(x any) any { 9 | switch q := x.(type) { 10 | case map[string]any: 11 | r := make(map[string]any, len(q)) 12 | for k, v := range q { 13 | r[k] = fixAny(v) 14 | } 15 | return r 16 | case []any: 17 | r := make([]any, len(q)) 18 | for i, e := range q { 19 | r[i] = fixAny(e) 20 | } 21 | return r 22 | case string: 23 | if a, ok := strToAny(q); ok { 24 | return a 25 | } 26 | } 27 | return x 28 | } 29 | 30 | func strToAny(x string) (any, bool) { 31 | d := []byte(x) 32 | if !json.Valid(d) { 33 | return nil, false 34 | } 35 | q := any(nil) 36 | buf := bytes.NewBuffer(d) 37 | dec := json.NewDecoder(buf) 38 | dec.UseNumber() 39 | err := dec.Decode(&q) 40 | if err != nil { 41 | return nil, false 42 | } 43 | return fixAny(q), true 44 | } 45 | 46 | func tXJson(x string) string { 47 | p, ok := strToAny(x) 48 | if !ok { 49 | return x 50 | } 51 | b, err := json.Marshal(p) 52 | if err != nil { 53 | return err.Error() 54 | } 55 | return string(b) 56 | } 57 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023-2025 Alexey Michurin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /slogtotext/flat.go: -------------------------------------------------------------------------------- 1 | package slogtotext 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | type Pair struct { 12 | K string 13 | V string 14 | } 15 | 16 | type pair struct { 17 | k string 18 | v any 19 | } 20 | 21 | func fill(m *[]Pair, pfx []string, d any) { 22 | switch x := d.(type) { 23 | case nil: 24 | case bool: 25 | t := "false" 26 | if x { 27 | t = "true" 28 | } 29 | *m = append(*m, Pair{K: kjoin(pfx), V: t}) 30 | case string: 31 | *m = append(*m, Pair{K: kjoin(pfx), V: x}) 32 | case json.Number: 33 | *m = append(*m, Pair{K: kjoin(pfx), V: x.String()}) 34 | case []any: 35 | for i, v := range x { 36 | fill(m, append(pfx, strconv.Itoa(i)), v) 37 | } 38 | case map[string]any: 39 | kv := make([]pair, len(x)) 40 | n := 0 41 | for k, v := range x { 42 | kv[n].k = k 43 | kv[n].v = v 44 | n++ 45 | } 46 | sort.Slice(kv, func(i, j int) bool { return kv[i].k < kv[j].k }) 47 | for _, e := range kv { 48 | fill(m, append(pfx, e.k), e.v) 49 | } 50 | default: 51 | panic(fmt.Errorf("unknown type (pfx=%[2]s) %[1]T: %[1]v", x, pfx)) 52 | } 53 | } 54 | 55 | func kjoin(pfx []string) string { 56 | if len(pfx) == 0 { 57 | return "NOKEY" // in case the root element is not object or array 58 | } 59 | return strings.Join(pfx, ".") 60 | } 61 | 62 | func flat(d any) []Pair { 63 | res := []Pair(nil) 64 | fill(&res, nil, d) 65 | return res 66 | } 67 | -------------------------------------------------------------------------------- /slogtotext/flat_test.go: -------------------------------------------------------------------------------- 1 | package slogtotext 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestFlat(t *testing.T) { 14 | t.Parallel() 15 | for i, cs := range []struct { 16 | in string 17 | exp []Pair 18 | }{{ 19 | in: `null`, // it skips nulls 20 | exp: nil, 21 | }, { 22 | in: `true`, 23 | exp: []Pair{{"NOKEY", "true"}}, 24 | }, { 25 | in: `false`, 26 | exp: []Pair{{"NOKEY", "false"}}, 27 | }, { 28 | in: `"x"`, 29 | exp: []Pair{{"NOKEY", "x"}}, 30 | }, { 31 | in: `1`, 32 | exp: []Pair{{"NOKEY", "1"}}, 33 | }, { 34 | in: `[]`, 35 | exp: []Pair(nil), 36 | }, { 37 | in: `{}`, 38 | exp: nil, 39 | }, { 40 | in: `{"a": null, "b": [1, null, "str", {"p": "q"}], "c": [], "d": true}`, 41 | exp: []Pair{ 42 | {"b.0", "1"}, 43 | {"b.2", "str"}, 44 | {"b.3.p", "q"}, 45 | {"d", "true"}, 46 | }, 47 | }} { 48 | cs := cs 49 | t.Run(fmt.Sprintf("test_%d", i), func(t *testing.T) { 50 | t.Parallel() 51 | d := json.NewDecoder(strings.NewReader(cs.in)) 52 | d.UseNumber() 53 | x := any(nil) 54 | err := d.Decode(&x) 55 | require.NoError(t, err) 56 | r := flat(x) 57 | assert.Equal(t, cs.exp, r, "in="+cs.in) 58 | }) 59 | } 60 | t.Run("invalid_type", func(t *testing.T) { 61 | require.PanicsWithError(t, "unknown type (pfx=[]) int: 1", func() { 62 | _ = flat(1) 63 | }) 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: all 4 | disable: 5 | - copyloopvar 6 | - err113 7 | - godox 8 | - intrange 9 | - nlreturn 10 | - paralleltest 11 | - revive 12 | - testpackage 13 | - tparallel 14 | - wastedassign 15 | - wrapcheck 16 | - wsl 17 | - wsl_v5 18 | settings: 19 | cyclop: 20 | max-complexity: 15 21 | funlen: 22 | lines: 70 23 | gocognit: 24 | min-complexity: 40 25 | lll: 26 | line-length: 160 27 | gosec: 28 | excludes: 29 | - G204 # Subprocess launched with variable 30 | - G304 # Potential file inclusion via variable 31 | depguard: 32 | rules: 33 | regular: 34 | files: 35 | - !$test 36 | allow: 37 | - $gostd 38 | - github.com/michurin/human-readable-json-logging/slogtotext 39 | - github.com/Masterminds/sprig/v3 40 | tests: 41 | files: 42 | - $test 43 | allow: 44 | - $gostd 45 | - github.com/michurin/human-readable-json-logging/slogtotext 46 | - github.com/stretchr/testify 47 | varnamelen: 48 | min-name-length: 2 49 | max-distance: 36 50 | exclusions: 51 | generated: lax 52 | warn-unused: true 53 | rules: 54 | - path: cmd/pplog/main.go 55 | linters: 56 | - dupword 57 | - errorlint 58 | - forbidigo 59 | - funlen 60 | - gochecknoglobals 61 | - gochecknoinits 62 | - mnd 63 | - path: slogtotext/example_slog_test.go 64 | linters: 65 | - exhaustruct 66 | formatters: 67 | enable: 68 | - gci 69 | - gofmt 70 | - gofumpt 71 | - goimports 72 | settings: 73 | gci: 74 | sections: 75 | - standard 76 | - default 77 | - prefix(github.com/michurin/human-readable-json-logging) 78 | -------------------------------------------------------------------------------- /slogtotext/example_slog_test.go: -------------------------------------------------------------------------------- 1 | //go:build go1.21 2 | 3 | package slogtotext_test 4 | 5 | import ( 6 | "context" 7 | "log/slog" 8 | "os" 9 | "time" 10 | 11 | "github.com/michurin/human-readable-json-logging/slogtotext" 12 | ) 13 | 14 | func panicIfError(err error) { 15 | if err != nil { 16 | panic(err) 17 | } 18 | } 19 | 20 | func tweakNowToMakeTestReproducible(_ []string, a slog.Attr) slog.Attr { 21 | if a.Key == slog.TimeKey { 22 | return slog.Attr{ 23 | Key: a.Key, 24 | Value: slog.TimeValue(time.Unix(186714000, 0).UTC()), 25 | } 26 | } 27 | return a 28 | } 29 | 30 | func Example_slog() { 31 | templateForJSONLogRecords := `{{ .time }} [{{ .level }}] {{ .msg }}{{ range .ALL | rm "time" "level" "msg" }} {{.K}}={{.V}}{{end}}` + "\n" 32 | templateForInvalidRecords := `INVALID JSON: {{ .TEXT | printf "%q" }}` + "\n" 33 | 34 | ctx, cancel := context.WithCancel(context.Background()) 35 | defer cancel() 36 | 37 | rd, wr, err := os.Pipe() 38 | panicIfError(err) 39 | 40 | defer func() { 41 | err = wr.Close() // The good idea is to close logging stream... 42 | panicIfError(err) 43 | <-ctx.Done() // ...and wait for all messages 44 | }() 45 | 46 | go func() { 47 | defer cancel() 48 | panicIfError(slogtotext.Read( 49 | rd, 50 | slogtotext.MustFormatter(os.Stdout, templateForJSONLogRecords), 51 | slogtotext.MustFormatter(os.Stdout, templateForInvalidRecords), 52 | 1024)) 53 | }() 54 | 55 | log := slog.New(slog.NewJSONHandler(wr, &slog.HandlerOptions{ReplaceAttr: tweakNowToMakeTestReproducible})) 56 | 57 | log.Info("Just log message") 58 | log.Error("Some error message", "customKey", "customValue") 59 | 60 | _, err = wr.WriteString("panic message\n") // emulate wrong json in stream 61 | panicIfError(err) 62 | // Output: 63 | // 1975-12-02T01:00:00Z [INFO] Just log message 64 | // 1975-12-02T01:00:00Z [ERROR] Some error message customKey=customValue 65 | // INVALID JSON: "panic message" 66 | } 67 | -------------------------------------------------------------------------------- /cmd/pplog/run.go: -------------------------------------------------------------------------------- 1 | //go:build !(windows || (linux && (arm64 || loong64 || riscv64))) 2 | 3 | package main 4 | 5 | import ( 6 | "flag" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "syscall" 11 | ) 12 | 13 | func runSubprocessModeChild() { 14 | deb("run child mode") 15 | 16 | target := flag.Args() 17 | binary, err := exec.LookPath(target[0]) 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | r, w, err := os.Pipe() 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | chFiles := make([]uintptr, 3) //nolint:mnd // in, out, err 28 | chFiles[0] = r.Fd() 29 | chFiles[1] = os.Stdout.Fd() 30 | chFiles[2] = os.Stderr.Fd() 31 | 32 | selfBinaty, err := exec.LookPath(os.Args[0]) 33 | if err != nil { 34 | panic(err) 35 | } 36 | cwd, err := os.Getwd() 37 | if err != nil { 38 | panic(err) 39 | } 40 | pid, err := syscall.ForkExec(selfBinaty, os.Args[:1], &syscall.ProcAttr{ 41 | Dir: cwd, 42 | Env: os.Environ(), 43 | Files: chFiles, 44 | Sys: nil, 45 | }) 46 | if err != nil { 47 | panic(selfBinaty + ": " + err.Error()) 48 | } 49 | deb(fmt.Sprintf("subprocess pid: %d", pid)) 50 | 51 | err = safeDup2(w.Fd(), safeStdout) // os.Stdout = w 52 | if err != nil { 53 | panic(err) 54 | } 55 | err = safeDup2(w.Fd(), safeStderr) // os.Stderr = w 56 | if err != nil { 57 | panic(err) 58 | } 59 | 60 | err = syscall.Exec(binary, target, os.Environ()) 61 | if err != nil { 62 | panic(binary + ": " + err.Error()) 63 | } 64 | } 65 | 66 | var ( 67 | safeStdout = uintptr(syscall.Stdout) //nolint:gochecknoglobals 68 | safeStderr = uintptr(syscall.Stderr) //nolint:gochecknoglobals 69 | ) 70 | 71 | func safeDup2(oldfd, newfd uintptr) error { 72 | // standard syscall.Dup2 is forcing us do unsafe uintptr->int->uintptr casting. It's security issue. 73 | _, _, errno := syscall.Syscall(syscall.SYS_DUP2, oldfd, newfd, 0) 74 | if errno != 0 { 75 | return fmt.Errorf("dup2: errno: %d", errno) //nolint:errorlint 76 | } 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /behavioral-tests/custom/pplog.env: -------------------------------------------------------------------------------- 1 | # PPLog config 2 | # 3 | # GitHub: https://github.com/michurin/human-readable-json-logging 4 | # Install: go install -v github.com/michurin/human-readable-json-logging/cmd/...@latest 5 | # 6 | # Configuration file syntax: systemd env-files https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#EnvironmentFile= 7 | # 8 | # Configuration variables: 9 | # - PPLOG_LOGLINE for JSON-lines 10 | # - PPLOG_ERRLINE for non-JSON-lines 11 | # 12 | # Templates: go standard test/template: https://pkg.go.dev/text/template 13 | # 14 | # Colors: terminal standard color sequences. `\e` considering as '\033' (escape) 15 | # 16 | # Color hints: 17 | # 18 | # Text colors Text High Background Hi Background Decoration 19 | # ------------------ ------------------ ------------------ ------------------- -------------------- 20 | # \e[30mBlack \e[0m \e[90mBlack \e[0m \e[40mBlack \e[0m \e[100mBlack \e[0m \e[1mBold \e[0m 21 | # \e[31mRed \e[0m \e[91mRed \e[0m \e[41mRed \e[0m \e[101mRed \e[0m \e[4mUnderline \e[0m 22 | # \e[32mGreen \e[0m \e[92mGreen \e[0m \e[42mGreen \e[0m \e[102mGreen \e[0m \e[7mReverse \e[0m 23 | # \e[33mYellow \e[0m \e[93mYellow \e[0m \e[43mYellow \e[0m \e[103mYellow \e[0m 24 | # \e[34mBlue \e[0m \e[94mBlue \e[0m \e[44mBlue \e[0m \e[104mBlue \e[0m Combinations 25 | # \e[35mMagenta\e[0m \e[95mMagenta\e[0m \e[45mMagenta\e[0m \e[105mMagenta\e[0m ----------------------- 26 | # \e[36mCyan \e[0m \e[96mCyan \e[0m \e[46mCyan \e[0m \e[106mCyan \e[0m \e[1;4;103;31mWARN\e[0m 27 | # \e[37mWhite \e[0m \e[97mWhite \e[0m \e[47mWhite \e[0m \e[107mWhite \e[0m 28 | 29 | PPLOG_LOGLINE=' 30 | {{- if .time }}{{ .time | tmf "2006-01-02T15:04:05Z07:00" "15:04:05" }}{{ end }} 31 | {{- if .level }} {{ if eq .level "INFO" }}\e[32m{{ end }}{{ if eq .level "ERROR" }}\e[91m{{ end }}{{ .level }}\e[0m{{ end }} 32 | {{- if .msg }} \e[97m{{ .msg }}\e[0m{{ end }} 33 | {{- range .ALL | rm "time" "level" "msg" }} \e[33m{{ .K }}\e[0m={{ .V }}{{ end }} 34 | ' 35 | 36 | PPLOG_ERRLINE='\e[7mNOJSON:\e[0m {{ if .BINARY }}{{ .TEXT }}{{ else }}\e[97m{{ .TEXT }}\e[0m{{ end }}' 37 | -------------------------------------------------------------------------------- /slogtotext/reader_test.go: -------------------------------------------------------------------------------- 1 | package slogtotext_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/michurin/human-readable-json-logging/slogtotext" 12 | ) 13 | 14 | func pairs(p []slogtotext.Pair) string { 15 | x := make([]string, len(p)) 16 | for i, v := range p { 17 | x[i] = v.K + "=" + v.V 18 | } 19 | return strings.Join(x, "\x20") 20 | } 21 | 22 | func collector(t *testing.T) ( 23 | func([]slogtotext.Pair) error, 24 | func([]slogtotext.Pair) error, 25 | func() string, 26 | ) { 27 | t.Helper() 28 | out := []string(nil) 29 | f := func(p []slogtotext.Pair) error { 30 | out = append(out, pairs(p)) 31 | return nil 32 | } 33 | g := func(p []slogtotext.Pair) error { 34 | t.Helper() 35 | require.Len(t, p, 2) 36 | require.Equal(t, "TEXT", p[0].K) 37 | require.Equal(t, "BINARY", p[1].K) 38 | out = append(out, fmt.Sprintf("%q (%q)", p[0].V, p[1].V)) 39 | return nil 40 | } 41 | c := func() string { 42 | return strings.Join(out, "|") 43 | } 44 | return f, g, c 45 | } 46 | 47 | func TestReader(t *testing.T) { 48 | t.Parallel() 49 | for _, cs := range []struct { 50 | name string 51 | in string 52 | exp string 53 | }{ 54 | {name: "json", in: `{"a":1}`, exp: `a=1 RAW_INPUT={"a":1}`}, 55 | {name: "empty", in: "", exp: ""}, // has no effect 56 | {name: "nl", in: "\n\n", exp: `"" ("")|"" ("")`}, // invalid json 57 | {name: "invalid_json", in: `{"a":1}x`, exp: `"{\"a\":1}x" ("")`}, // as is 58 | {name: "invalid_json_with_ctrl", in: `{"a":1}` + "\033" + `x`, exp: `"{\"a\":1}\x1bx" ("yes")`}, // as is with label binary=yes 59 | {name: "valid_but_too_long", in: `{"a":"123"}`, exp: `"{\"a\":\"123\"" ("")|"}" ("")`}, 60 | {name: "valid_but_too_long_and_ok", in: `{"a":"123"}` + "\n" + `{"a":1}`, exp: `"{\"a\":\"123\"" ("")|"}" ("")|a=1 RAW_INPUT={"a":1}`}, 61 | } { 62 | cs := cs 63 | t.Run(cs.name, func(t *testing.T) { 64 | buf := strings.NewReader(cs.in) 65 | f, g, c := collector(t) 66 | err := slogtotext.Read(buf, f, g, 10) 67 | require.NoError(t, err) 68 | assert.Equal(t, cs.exp, c()) 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: build # this string appears on badge 3 | on: 4 | - push 5 | - pull_request 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | timeout-minutes: 10 10 | name: "Go ${{ matrix.go }} build and behavioral tests" 11 | strategy: 12 | matrix: 13 | go: 14 | - "1.21" # minimum sprig compatible version 15 | - "1.22" 16 | - "1.23" 17 | - "1.24" 18 | - "1.25" 19 | steps: 20 | - uses: actions/checkout@v4 21 | - uses: actions/setup-go@v5 22 | with: 23 | go-version: "${{ matrix.go }}" 24 | - run: "go version" 25 | - run: "go build ./cmd/..." 26 | - run: "./behavioral-tests/run.sh" 27 | x-build-windows: 28 | runs-on: ubuntu-latest 29 | timeout-minutes: 10 30 | name: "Simple build binary for MS Windows" 31 | steps: 32 | - uses: actions/checkout@v4 33 | - uses: actions/setup-go@v5 34 | with: 35 | go-version: "1.25" 36 | - run: "go version" 37 | - run: "go build ./cmd/..." 38 | env: 39 | GOOS: windows 40 | GOARCH: amd64 41 | - run: "file pplog.exe" 42 | windows: 43 | runs-on: windows-latest 44 | timeout-minutes: 10 45 | name: "Build on MS Windows" 46 | steps: 47 | - uses: actions/checkout@v4 48 | - run: "go version" # go1.21.11 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md 49 | - run: "go build ./cmd/..." 50 | - run: "dir" 51 | - run: "./pplog.exe -v" 52 | - shell: bash # naked `echo` won't work on windows 53 | run: "./pplog.exe -d bash -c 'echo \\{\\\"time\\\":\\\"2024-12-02T12:00:00Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"Hello\\\"\\}'" 54 | test: 55 | runs-on: ubuntu-latest 56 | timeout-minutes: 10 57 | name: "Unit test and lint" 58 | steps: 59 | - uses: actions/checkout@v4 60 | - uses: actions/setup-go@v5 61 | with: 62 | go-version: "1.25" 63 | - uses: golangci/golangci-lint-action@v7 64 | with: 65 | version: "v2.7.1" 66 | - run: "go test -v -race -coverprofile=coverage.txt -covermode=atomic ./..." 67 | - run: "grep -v '^github.com/michurin/human-readable-json-logging/cmd/pplog/' coverage.txt >coverage.tmp" 68 | - run: "rm coverage.txt" 69 | - run: "mv coverage.tmp coverage.txt" 70 | - uses: codecov/codecov-action@v5 71 | with: 72 | files: ./coverage.txt 73 | verbose: true 74 | token: ${{ secrets.CODECOV_TOKEN }} # required 75 | -------------------------------------------------------------------------------- /slogtotext/reader.go: -------------------------------------------------------------------------------- 1 | package slogtotext 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "unicode" 8 | ) 9 | 10 | const ( 11 | allKey = "ALL" 12 | rawInputKey = "RAW_INPUT" 13 | invalidLineKey = "TEXT" 14 | binaryKey = "BINARY" 15 | ) 16 | 17 | func tryToParse(b []byte) (any, bool) { 18 | if !json.Valid(b) { 19 | return nil, false 20 | } 21 | d := json.NewDecoder(bytes.NewReader(b)) 22 | d.UseNumber() 23 | data := any(nil) 24 | err := d.Decode(&data) 25 | if err != nil { 26 | return nil, false 27 | } 28 | return data, true 29 | } 30 | 31 | func Read(input io.Reader, out func([]Pair) error, outStr func([]Pair) error, maxCap int) error { // TODO wrap errors 32 | // This implementation is slightly naive (extra reading/coping prone) and mimics bufio.Scan. 33 | // However, we do not use bufio.Scan because it consider too long taken as error (ErrTooLong). 34 | // Considering of this error makes code too ugly because 35 | // in our case we do not consider it as error, however it is special case. 36 | // The room for refactoring. 37 | buf := make([]byte, maxCap) 38 | line := []byte(nil) 39 | bufEnd := 0 40 | for { 41 | n, err := input.Read(buf[bufEnd:]) 42 | noMoreData := n == 0 && err != nil // it is the last data 43 | if noMoreData && bufEnd == 0 { 44 | // Read can return n > 0 and EOF, however it must return 0 and EOF next time 45 | // And we give the split function a chance, like bufio.Scon does (check bufEnd == 0) 46 | break 47 | } 48 | bufEnd += n 49 | for bufEnd > 0 { 50 | s := bytes.IndexByte(buf[:bufEnd], '\n') 51 | if s < 0 { 52 | if bufEnd == maxCap || noMoreData { // consider full buffer 53 | line = buf[:bufEnd] 54 | buf = make([]byte, maxCap) 55 | bufEnd = 0 // will cause end of iterations 56 | } else { 57 | break // the buffer is not full, however, it is not a complete line 58 | } 59 | } else { 60 | line = buf[:s] 61 | x := make([]byte, maxCap) 62 | copy(x, buf[s+1:]) 63 | buf = x 64 | bufEnd -= s + 1 65 | } 66 | data, ok := tryToParse(line) 67 | if ok { 68 | rec := flat(data) 69 | rec = append(rec, Pair{rawInputKey, string(line)}) 70 | err := out(rec) 71 | if err != nil { 72 | return err 73 | } 74 | } else { 75 | x := "" 76 | if bytes.IndexFunc(line, unicode.IsControl) >= 0 { // bytes.ContainsFunc shows up in go go1.21 77 | x = "yes" 78 | } 79 | err := outStr([]Pair{ 80 | {K: invalidLineKey, V: string(line)}, 81 | {K: binaryKey, V: x}, 82 | }) 83 | if err != nil { 84 | return err 85 | } 86 | } 87 | } 88 | if noMoreData { 89 | break 90 | } 91 | } 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /slogtotext/xxjson.go: -------------------------------------------------------------------------------- 1 | package slogtotext 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "sort" 8 | ) 9 | 10 | type kv struct { 11 | k string 12 | v any 13 | } 14 | 15 | type kvSlice []kv 16 | 17 | func (x kvSlice) Len() int { return len(x) } 18 | func (x kvSlice) Less(i, j int) bool { return x[i].k < x[j].k } 19 | func (x kvSlice) Swap(i, j int) { x[i], x[j] = x[j], x[i] } 20 | 21 | type writer struct { 22 | w io.Writer 23 | } 24 | 25 | func (w *writer) wr(x ...[]byte) { 26 | for _, e := range x { 27 | _, err := w.w.Write(e) 28 | if err != nil { 29 | panic(err.Error()) 30 | } 31 | } 32 | } 33 | 34 | func marshal(x any, buf *writer, clr *xxJSONColors) { // ACHTUNG we do not care about looping 35 | switch q := x.(type) { 36 | case map[string]any: 37 | kk := make(kvSlice, 0, len(q)) 38 | for k, v := range q { 39 | kk = append(kk, kv{k: k, v: v}) 40 | } 41 | sort.Sort(kk) 42 | buf.wr([]byte(`{`)) 43 | for i, e := range kk { 44 | if i > 0 { 45 | buf.wr([]byte(`,`)) 46 | } 47 | buf.wr(clr.KeyOpen, []byte(e.k), clr.KeyClose, []byte(`:`)) 48 | marshal(e.v, buf, clr) 49 | } 50 | buf.wr([]byte(`}`)) 51 | case []any: 52 | buf.wr([]byte(`[`)) 53 | for i, e := range q { 54 | if i > 0 { 55 | buf.wr([]byte(`,`)) 56 | } 57 | marshal(e, buf, clr) 58 | } 59 | buf.wr([]byte(`]`)) 60 | case string: 61 | buf.wr(clr.StringOpen, []byte(q), clr.StringClose) 62 | case bool: 63 | if q { 64 | buf.wr(clr.TrueOpen, []byte(`T`), clr.TrueClose) 65 | } else { 66 | buf.wr(clr.FalseOpen, []byte(`F`), clr.FalseClose) 67 | } 68 | case nil: 69 | buf.wr(clr.NullOpen, []byte(`N`), clr.NullClose) 70 | case json.Number: 71 | buf.wr(clr.NumberOpen, []byte(q.String()), clr.NumberClose) 72 | } 73 | } 74 | 75 | type xxJSONColors struct { 76 | KeyOpen []byte 77 | KeyClose []byte 78 | FalseOpen []byte 79 | FalseClose []byte 80 | TrueOpen []byte 81 | TrueClose []byte 82 | NullOpen []byte 83 | NullClose []byte 84 | StringOpen []byte 85 | StringClose []byte 86 | NumberOpen []byte 87 | NumberClose []byte 88 | } 89 | 90 | var defaultColors = xxJSONColors{ //nolint:gochecknoglobals 91 | KeyOpen: []byte("\033[93m"), 92 | KeyClose: []byte("\033[0m"), 93 | FalseOpen: []byte("\033[91m"), 94 | FalseClose: []byte("\033[0m"), 95 | TrueOpen: []byte("\033[92m"), 96 | TrueClose: []byte("\033[0m"), 97 | NullOpen: []byte("\033[95m"), 98 | NullClose: []byte("\033[0m"), 99 | StringOpen: []byte("\033[35m"), 100 | StringClose: []byte("\033[0m"), 101 | NumberOpen: []byte("\033[95m"), 102 | NumberClose: []byte("\033[0m"), 103 | } 104 | 105 | func tXXJson(x string) string { 106 | p, ok := strToAny(x) 107 | if !ok { 108 | return x 109 | } 110 | buf := bytes.NewBuffer(nil) 111 | marshal(p, &writer{w: buf}, &defaultColors) // TODO configurable colors 112 | return buf.String() 113 | } 114 | -------------------------------------------------------------------------------- /cmd/pplog/run_common.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | "os/exec" 10 | "os/signal" 11 | "strings" 12 | "syscall" 13 | "time" 14 | 15 | "github.com/michurin/human-readable-json-logging/slogtotext" 16 | ) 17 | 18 | func runSubprocessMode(lineFmt, errFmt func([]slogtotext.Pair) error) { //nolint:funlen,cyclop // TODO split it? 19 | deb("run subprocess mode") 20 | 21 | rd, wr := io.Pipe() 22 | 23 | done := make(chan struct{}) 24 | 25 | go func() { 26 | defer close(done) // must be closed whether error or not 27 | err := slogtotext.Read(rd, lineFmt, errFmt, buffSize) 28 | if err != nil { 29 | deb("reader is finished with err: " + err.Error()) 30 | return 31 | } 32 | deb("reader is finished") 33 | }() 34 | 35 | args := flag.Args()[1:] 36 | command := flag.Args()[0] 37 | 38 | deb("running subprocess: " + command + " " + strings.Join(args, " ")) 39 | 40 | cmd := exec.Command(command, args...) //nolint:noctx 41 | cmd.Stdout = wr 42 | cmd.Stderr = wr 43 | 44 | err := cmd.Start() 45 | if err != nil { 46 | panic(err) // TODO 47 | } 48 | 49 | syncWait := make(chan error, 1) 50 | go func() { 51 | syncWait <- cmd.Wait() 52 | }() 53 | 54 | syncTerm := make(chan os.Signal, 1) 55 | countTerm := 0 56 | signal.Notify(syncTerm, os.Interrupt, syscall.SIGTERM) 57 | 58 | syncSignal := make(chan os.Signal, 1) 59 | 60 | syncForceDone := make(chan struct{}) 61 | 62 | exitCode := 0 63 | 64 | LOOP: 65 | for { 66 | select { 67 | case err := <-syncWait: // have to break loop in this case 68 | deb("subprocess waiting is done") 69 | if err != nil { 70 | xerr := new(exec.ExitError) 71 | if errors.As(err, &xerr) { 72 | exitCode = xerr.ExitCode() 73 | if exitCode < 0 { 74 | exitCode = 1 75 | } 76 | break LOOP 77 | } 78 | panic(err) // TODO 79 | } 80 | break LOOP 81 | case sig := <-syncTerm: 82 | countTerm++ 83 | deb(fmt.Sprintf("pplog gets: %s (#%d)", sig.String(), countTerm)) 84 | if countTerm > 1 { 85 | deb("breaking loop") 86 | exitCode = 1 87 | break LOOP 88 | } 89 | syncSignal <- syscall.SIGINT 90 | case sig := <-syncSignal: 91 | deb("pplog sending " + sig.String() + " to subprocess") 92 | err := cmd.Process.Signal(sig) 93 | if err != nil { // it seems we send signal to already dead process 94 | deb("sending signal: " + sig.String() + ": " + err.Error()) 95 | break LOOP // the best idea? 96 | } 97 | switch sig { 98 | case syscall.SIGINT: 99 | go func() { 100 | <-time.After(time.Second) 101 | syncSignal <- syscall.SIGTERM 102 | }() 103 | case syscall.SIGTERM: 104 | go func() { 105 | <-time.After(time.Second) 106 | syncSignal <- syscall.SIGKILL 107 | }() 108 | case syscall.SIGKILL: 109 | go func() { 110 | <-time.After(time.Second) 111 | close(syncForceDone) 112 | }() 113 | } 114 | case <-syncForceDone: 115 | panic("it seems the process cannot be stopped") 116 | } 117 | } 118 | 119 | deb("stopping reader and waiting for it") 120 | 121 | err = wr.Close() // cause stop rendering 122 | if err != nil { 123 | deb("closing writer: " + err.Error()) 124 | } 125 | 126 | <-done // will be closed by reader 127 | 128 | os.Exit(exitCode) 129 | } 130 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= 2 | dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= 3 | github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= 4 | github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= 5 | github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= 6 | github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= 7 | github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= 8 | github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 12 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 13 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 14 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 15 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 16 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 17 | github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= 18 | github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= 19 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 20 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 21 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 22 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 23 | github.com/michurin/systemd-env-file v0.0.0-20250606021327-1e826f3c7c79 h1:WjFNDqyekxNgCFJ6oCgdOSljZ/BOc+8E+o2wlBS8ajw= 24 | github.com/michurin/systemd-env-file v0.0.0-20250606021327-1e826f3c7c79/go.mod h1:ao0K33zrZMAipygOiqEfWCHeu05zMN623X7D0QDB7JA= 25 | github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= 26 | github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= 27 | github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= 28 | github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 29 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 30 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 31 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 32 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 33 | github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= 34 | github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= 35 | github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= 36 | github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 37 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 38 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 39 | golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= 40 | golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= 41 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 42 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 43 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 44 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 45 | -------------------------------------------------------------------------------- /cmd/pplog/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io/fs" 8 | "os" 9 | "os/exec" 10 | "path" 11 | "runtime" 12 | "runtime/debug" 13 | "strings" 14 | "syscall" 15 | 16 | "github.com/michurin/systemd-env-file/sdenv" 17 | 18 | "github.com/michurin/human-readable-json-logging/slogtotext" 19 | ) 20 | 21 | var ( 22 | debugFlag = false 23 | showVersionFlag = false 24 | childModeFlag = false 25 | debugFilePrefix = "" 26 | ) 27 | 28 | func init() { 29 | _, file, _, ok := runtime.Caller(0) 30 | if ok { 31 | debugFilePrefix = strings.TrimSuffix(file, "cmd/pplog/main.go") 32 | } 33 | } 34 | 35 | func deb(m string) { 36 | if debugFlag { 37 | loc := "" 38 | _, file, line, ok := runtime.Caller(1) 39 | if ok { 40 | loc = fmt.Sprintf(" %s:%d", strings.TrimPrefix(file, debugFilePrefix), line) 41 | } 42 | fmt.Println("DEBUG" + loc + ": " + m) 43 | } 44 | } 45 | 46 | const configFile = "pplog.env" 47 | 48 | func lookupEnvFile() string { 49 | cwd, err := os.Getwd() 50 | if err != nil { 51 | deb(err.Error()) 52 | return "" 53 | } 54 | home, err := os.UserHomeDir() 55 | if err != nil { 56 | deb(err.Error()) 57 | home = cwd 58 | } 59 | for { 60 | fn := path.Join(cwd, configFile) 61 | fi, err := os.Stat(fn) 62 | if err != nil { 63 | deb(err.Error()) 64 | } 65 | if err == nil && fi.Mode()&fs.ModeType == 0 { 66 | deb("file found: " + fn) 67 | return fn 68 | } 69 | cwd = path.Dir(cwd) 70 | if len(cwd) < len(home) { 71 | break 72 | } 73 | } 74 | deb("no configuration file has been found") 75 | return "" 76 | } 77 | 78 | func normLine(t string) string { 79 | return strings.ReplaceAll(strings.TrimSpace(t), "\\e", "\033") + "\n" 80 | } 81 | 82 | func showBuildInfo() { 83 | info, ok := debug.ReadBuildInfo() 84 | if !ok { 85 | fmt.Println("Cannot get build info") 86 | return 87 | } 88 | fmt.Println(info.String()) 89 | } 90 | 91 | func readEnvs() (string, string, bool) { 92 | c := sdenv.NewCollectsion() 93 | c.PushStd(os.Environ()) 94 | envFile := lookupEnvFile() 95 | if envFile != "" { 96 | b, err := os.ReadFile(envFile) 97 | if err != nil { 98 | panic(err) // TODO 99 | } 100 | pairs, err := sdenv.Parser(b) 101 | if err != nil { 102 | panic(err) // TODO 103 | } 104 | c.Push(pairs) 105 | } 106 | 107 | logLine := `{{ .time }} [{{ .level }}] {{ .msg }}{{ range .ALL | rm "time" "level" "msg" }} {{.K}}={{.V}}{{end}}` 108 | errLine := `INVALID JSON: {{ .TEXT | printf "%q" }}` 109 | childMode := false 110 | for _, p := range c.Collection() { 111 | switch p[0] { 112 | case "PPLOG_LOGLINE": 113 | logLine = p[1] 114 | case "PPLOG_ERRLINE": 115 | errLine = p[1] 116 | case "PPLOG_CHILD_MODE": 117 | childMode = true 118 | } 119 | } 120 | logLine = normLine(logLine) 121 | errLine = normLine(errLine) 122 | return logLine, errLine, childMode 123 | } 124 | 125 | func runPipeMode(lineFmt, errFmt func([]slogtotext.Pair) error) { 126 | deb("run pipe mode") 127 | err := slogtotext.Read(os.Stdin, lineFmt, errFmt, buffSize) 128 | if err != nil { 129 | printError(err) 130 | return 131 | } 132 | } 133 | 134 | func main() { 135 | flag.BoolVar(&debugFlag, "d", false, "debug mode") 136 | flag.BoolVar(&showVersionFlag, "v", false, "show version and exit") 137 | flag.BoolVar(&childModeFlag, "c", false, "child mode. pplog runs as child of target process") 138 | flag.Parse() 139 | 140 | deb(fmt.Sprintf("flags: debug=%t, showVersion=%t, childMode=%t", debugFlag, showVersionFlag, childModeFlag)) 141 | 142 | if showVersionFlag { 143 | showBuildInfo() 144 | return 145 | } 146 | 147 | lineTemplate, errTemplate, childMode := readEnvs() 148 | outputStream := os.Stdout // TODO make it tunable 149 | 150 | if flag.NArg() >= 1 { 151 | if childModeFlag || childMode { 152 | runSubprocessModeChild() 153 | } else { 154 | runSubprocessMode(slogtotext.MustFormatter(outputStream, lineTemplate), slogtotext.MustFormatter(outputStream, errTemplate)) 155 | } 156 | } else { 157 | runPipeMode(slogtotext.MustFormatter(outputStream, lineTemplate), slogtotext.MustFormatter(outputStream, errTemplate)) 158 | } 159 | } 160 | 161 | func printError(err error) { // TODO reconsider 162 | pe := new(os.PathError) 163 | if errors.As(err, &pe) { 164 | if pe.Err == syscall.EBADF { // fragile code; somehow syscall.Errno.Is doesn't recognize EBADF, so we unable to use errors.As 165 | // maybe it is good idea just ignore SIGPIPE 166 | fmt.Fprintf(os.Stderr, "PPLog: It seems output descriptor has been closed\n") // trying to report it to stderr 167 | return 168 | } 169 | } 170 | xe := new(exec.ExitError) 171 | if errors.As(err, &xe) { 172 | fmt.Printf("exit code = %d: %s\n", xe.ExitCode(), xe.Error()) // just for information 173 | return 174 | } 175 | fmt.Printf("Error: %s\n", err.Error()) 176 | } 177 | -------------------------------------------------------------------------------- /slogtotext/formatter.go: -------------------------------------------------------------------------------- 1 | package slogtotext 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "sort" 9 | "strings" 10 | "sync/atomic" 11 | "text/template" 12 | "time" 13 | 14 | "github.com/Masterminds/sprig/v3" 15 | ) 16 | 17 | func tTimeFormatter(args ...any) string { 18 | // args: fromFormat1, fromFormat2, ... toFormat, timeString 19 | if len(args) < 3 { //nolint:mnd 20 | return fmt.Sprintf("Too few arguments: %d", len(args)) 21 | } 22 | sa := make([]string, len(args)) 23 | for i, v := range args { 24 | var ok bool 25 | sa[i], ok = v.(string) 26 | if !ok { 27 | return fmt.Sprintf("Invalid type: pos=%[1]d: %[2]T (%[2]v)", i+1, v) 28 | } 29 | } 30 | tm := sa[len(sa)-1] // the last argument is timeString 31 | tf := sa[len(sa)-2] // the before last argument is target format 32 | sa = sa[:len(sa)-2] // source formats to try 33 | for _, v := range sa { 34 | t, err := time.Parse(v, tm) 35 | if err != nil { 36 | e := new(time.ParseError) 37 | if errors.As(err, &e) { 38 | continue 39 | } 40 | return err.Error() // in fact, it is impossible case, time.Parse always returns *time.ParseError 41 | } 42 | return t.Format(tf) 43 | } 44 | return tm // return original time as fallback 45 | } 46 | 47 | func tRemoveByPfx(args ...any) []Pair { // TODO naive nested loop implementation 48 | nLast := len(args) - 1 49 | if nLast <= 0 { 50 | panic(fmt.Sprintf("Invalid number of args: %d: %v", len(args), args)) 51 | } 52 | c := make([]string, nLast) 53 | ok := false 54 | for i := 0; i < nLast; i++ { 55 | c[i], ok = args[i].(string) 56 | if !ok { 57 | panic(fmt.Sprintf("Invalid type: idx=%[1]d: %[2]T: %[2]v", i, args[i])) 58 | } 59 | } 60 | av := args[nLast] 61 | a, ok := av.([]Pair) 62 | if !ok { 63 | panic(fmt.Sprintf("Invalid type: %[1]T: %[1]v: only .ALL allows", av)) 64 | } 65 | r := []Pair(nil) 66 | for _, x := range a { 67 | found := false 68 | for _, p := range c { 69 | if strings.HasPrefix(x.K, p) { 70 | found = true 71 | break 72 | } 73 | } 74 | if !found { 75 | r = append(r, x) 76 | } 77 | } 78 | return r 79 | } 80 | 81 | func tRemove(args ...any) []Pair { 82 | nLast := len(args) - 1 83 | if nLast <= 0 { 84 | panic(fmt.Sprintf("Invalid number of args: %d: %v", len(args), args)) 85 | } 86 | c := make(map[string]struct{}, nLast) 87 | for i := 0; i < nLast; i++ { 88 | s, ok := args[i].(string) 89 | if !ok { 90 | panic(fmt.Sprintf("Invalid type: idx=%[1]d: %[2]T: %[2]v", i, args[i])) 91 | } 92 | c[s] = struct{}{} 93 | } 94 | av := args[nLast] 95 | a, ok := av.([]Pair) 96 | if !ok { 97 | panic(fmt.Sprintf("Invalid type: %[1]T: %[1]v: only .ALL allows", av)) 98 | } 99 | r := []Pair(nil) 100 | for _, x := range a { 101 | if _, ok := c[x.K]; !ok { 102 | r = append(r, x) 103 | } 104 | } 105 | return r 106 | } 107 | 108 | func tTrimSpace(args ...any) string { 109 | r := make([]string, len(args)) 110 | for i, v := range args { 111 | if s, ok := v.(string); ok { 112 | r[i] = strings.TrimSpace(s) 113 | } else { 114 | r[i] = fmt.Sprintf("%#v", v) 115 | } 116 | } 117 | return strings.Join(r, " ") 118 | } 119 | 120 | func tSkipLineIf(x *atomic.Bool, xor bool) func(args ...any) string { 121 | return func(args ...any) string { 122 | f := false 123 | for _, v := range args { 124 | switch x := v.(type) { 125 | case bool: 126 | f = f || x 127 | case string: 128 | f = f || (len(x) > 0) 129 | case int: 130 | f = f || (x != 0) 131 | } 132 | if f { 133 | break 134 | } 135 | } 136 | x.Store(f != xor) // xor operation 137 | return "" 138 | } 139 | } 140 | 141 | func Formatter(stream io.Writer, templateString string) (func([]Pair) error, error) { 142 | flag := new(atomic.Bool) 143 | tm, err := template. 144 | New("base"). 145 | Option("missingkey=zero"). 146 | Funcs(template.FuncMap{ 147 | "tmf": tTimeFormatter, 148 | "rm": tRemove, 149 | "rmByPfx": tRemoveByPfx, 150 | "xjson": tXJson, 151 | "xxjson": tXXJson, 152 | "trimSpace": tTrimSpace, 153 | "skipLineIf": tSkipLineIf(flag, false), 154 | "skipLineUnless": tSkipLineIf(flag, true), 155 | }). 156 | Funcs(sprig.FuncMap()). 157 | Parse(templateString) 158 | if err != nil { 159 | return nil, err // TODO wrap? 160 | } 161 | 162 | return func(p []Pair) error { 163 | kv := make(map[string]any, len(p)) 164 | for _, v := range p { 165 | kv[v.K] = v.V 166 | } 167 | q := make([]Pair, 0, len(p)) 168 | for _, v := range p { 169 | if v.K != rawInputKey { 170 | q = append(q, v) 171 | } 172 | } 173 | sort.Slice(q, func(i, j int) bool { return q[i].K < q[j].K }) 174 | kv[allKey] = q 175 | buff := new(bytes.Buffer) 176 | err := tm.Execute(buff, kv) 177 | if err != nil { 178 | return err // TODO wrap error? 179 | } 180 | if !flag.Load() { 181 | _, err = io.Copy(stream, buff) 182 | if err != nil { 183 | return err // TODO wrap error? 184 | } 185 | } 186 | return nil 187 | }, nil 188 | } 189 | 190 | func MustFormatter(stream io.Writer, templateString string) func([]Pair) error { 191 | f, err := Formatter(stream, templateString) 192 | if err != nil { 193 | panic(err.Error()) // TODO wrap? 194 | } 195 | return f 196 | } 197 | -------------------------------------------------------------------------------- /slogtotext/formatter_test.go: -------------------------------------------------------------------------------- 1 | //nolint:funlen // it's ok for tests 2 | package slogtotext_test 3 | 4 | import ( 5 | "bytes" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/michurin/human-readable-json-logging/slogtotext" 12 | ) 13 | 14 | func TestFormatter(t *testing.T) { 15 | for _, cs := range []struct { 16 | name string 17 | template string 18 | in []slogtotext.Pair 19 | out string 20 | }{ 21 | { 22 | name: "nil", 23 | template: "OK", 24 | in: nil, 25 | out: "OK", 26 | }, 27 | { 28 | name: "simple", 29 | template: "{{ .A }}", 30 | in: []slogtotext.Pair{{K: "A", V: "1"}}, 31 | out: "1", 32 | }, 33 | { 34 | name: "simple_not_value", 35 | template: "{{ .A }}", 36 | in: []slogtotext.Pair{}, 37 | out: "", 38 | }, 39 | { 40 | name: "time_formatter", 41 | template: `{{ .A | tmf "2006-01-02T15:04:05Z07:00" "15:04:05" }}`, 42 | in: []slogtotext.Pair{{K: "A", V: "1975-12-02T12:01:02Z"}}, 43 | out: "12:01:02", 44 | }, 45 | { 46 | name: "time_formatter_skip_format", 47 | template: `{{ .A | tmf "Z07:00" "2006-01-02T15:04:05Z07:00" "15:04:05" }}`, 48 | in: []slogtotext.Pair{{K: "A", V: "1975-12-02T12:01:02Z"}}, 49 | out: "12:01:02", 50 | }, 51 | { 52 | name: "time_formatter_too_few_args", 53 | template: `{{ .A | tmf "xxx" }}`, 54 | in: []slogtotext.Pair{{K: "A", V: "1975-12-02T12:01:02Z"}}, 55 | out: "Too few arguments: 2", 56 | }, 57 | { 58 | name: "time_formatter_invalid_fallback", 59 | template: `{{ .A | tmf "2006-01-02" "2006-01-02" }}`, 60 | in: []slogtotext.Pair{{K: "A", V: "1975-xii-02"}}, 61 | out: `1975-xii-02`, 62 | }, 63 | { 64 | name: "range", 65 | template: `{{ range .ALL }}{{.K}}={{.V}};{{end}}`, 66 | in: []slogtotext.Pair{{K: "A", V: "1"}, {K: "AA", V: "11"}, {K: "B", V: "2"}}, 67 | out: "A=1;AA=11;B=2;", 68 | }, 69 | { 70 | name: "range_rm", 71 | template: `{{ range .ALL | rm "A" }}{{.K}}={{.V}};{{end}}`, 72 | in: []slogtotext.Pair{{K: "A", V: "1"}, {K: "AA", V: "11"}, {K: "B", V: "2"}}, 73 | out: "AA=11;B=2;", 74 | }, 75 | { 76 | name: "range_rm_multi", 77 | template: `{{ range .ALL | rm "A" "B" }}{{.K}}={{.V}};{{end}}`, 78 | in: []slogtotext.Pair{{K: "A", V: "1"}, {K: "AA", V: "11"}, {K: "B", V: "2"}}, 79 | out: "AA=11;", 80 | }, 81 | { 82 | name: "range_rm_pfx", 83 | template: `{{ range .ALL | rmByPfx "A" }}{{.K}}={{.V}};{{end}}`, 84 | in: []slogtotext.Pair{{K: "A", V: "1"}, {K: "AA", V: "11"}, {K: "B", V: "2"}}, 85 | out: "B=2;", 86 | }, 87 | { 88 | name: "range_rm_pfx_multi", 89 | template: `{{ range .ALL | rmByPfx "A" "B" }}{{.K}}={{.V}};{{end}}`, 90 | in: []slogtotext.Pair{{K: "A", V: "1"}, {K: "AA", V: "11"}, {K: "B", V: "2"}, {K: "BB", V: "22"}, {K: "C", V: "3"}}, 91 | out: "C=3;", 92 | }, 93 | { 94 | name: "trim_space", 95 | template: `{{ .A | trimSpace }}`, 96 | in: []slogtotext.Pair{{K: "A", V: " X "}}, 97 | out: `X`, 98 | }, 99 | { 100 | name: "skip_string_true", 101 | template: `{{ skipLineUnless .A }}{{ .M }}`, 102 | in: []slogtotext.Pair{{K: "A", V: "not empty"}, {K: "M", V: "x"}}, 103 | out: `x`, 104 | }, 105 | { 106 | name: "skip_string_false", 107 | template: `{{ skipLineUnless .A }}{{ .M }}`, 108 | in: []slogtotext.Pair{{K: "A", V: ""}, {K: "M", V: "x"}}, 109 | out: ``, 110 | }, 111 | { 112 | name: "skip_int_true", 113 | template: `{{ skipLineUnless (len .A) }}{{ .M }}`, 114 | in: []slogtotext.Pair{{K: "A", V: "string"}, {K: "M", V: "x"}}, 115 | out: `x`, 116 | }, 117 | { 118 | name: "skip_int_false", 119 | template: `{{ skipLineUnless (len .A) }}{{ .M }}`, 120 | in: []slogtotext.Pair{{K: "A", V: ""}, {K: "M", V: "x"}}, 121 | out: ``, 122 | }, 123 | { 124 | name: "skip_bool_true", 125 | template: `{{ skipLineIf (eq 0 (len .A)) }}{{ .M }}`, 126 | in: []slogtotext.Pair{{K: "A", V: ""}, {K: "M", V: "x"}}, 127 | out: ``, 128 | }, 129 | { 130 | name: "sprig_function", // just to be sure that https://masterminds.github.io/sprig/ are on 131 | template: `{{ upper "hello" }}`, 132 | in: nil, 133 | out: "HELLO", 134 | }, 135 | } { 136 | cs := cs 137 | t.Run(cs.name, func(t *testing.T) { 138 | buf := new(bytes.Buffer) 139 | f := slogtotext.MustFormatter(buf, cs.template) 140 | err := f(cs.in) 141 | require.NoError(t, err) 142 | assert.Equal(t, cs.out, buf.String()) 143 | }) 144 | } 145 | } 146 | 147 | func TestFormatter_errors(t *testing.T) { 148 | for _, cs := range []struct { 149 | name string 150 | template string 151 | err string 152 | }{ 153 | { 154 | name: "range_rm_wrong_type", 155 | template: `{{ range .ALL | rm "A" true }}{{.K}}={{.V}};{{end}}`, 156 | err: `template: base:1:16: executing "base" at : error calling rm: Invalid type: idx=1: bool: true`, 157 | }, 158 | { 159 | name: "range_rm_wrong_input_type", 160 | template: `{{ range 1 | rm "A" }}{{.K}}={{.V}};{{end}}`, 161 | err: `template: base:1:13: executing "base" at : error calling rm: Invalid type: int: 1: only .ALL allows`, 162 | }, 163 | { 164 | name: "range_rm_noargs", 165 | template: `{{ range .ALL | rm }}{{.K}}={{.V}};{{end}}`, 166 | err: `template: base:1:16: executing "base" at : error calling rm: Invalid number of args: 1: [[]]`, 167 | }, 168 | { 169 | name: "range_rm_pfx_wrong_type", 170 | template: `{{ range .ALL | rmByPfx "A" true }}{{.K}}={{.V}};{{end}}`, 171 | err: `template: base:1:16: executing "base" at : error calling rmByPfx: Invalid type: idx=1: bool: true`, 172 | }, 173 | { 174 | name: "range_rm_pfx_wrong_input_type", 175 | template: `{{ range 1 | rmByPfx "A" }}{{.K}}={{.V}};{{end}}`, 176 | err: `template: base:1:13: executing "base" at : error calling rmByPfx: Invalid type: int: 1: only .ALL allows`, 177 | }, 178 | { 179 | name: "range_rm_pfx_noargs", 180 | template: `{{ range .ALL | rmByPfx }}{{.K}}={{.V}};{{end}}`, 181 | err: `template: base:1:16: executing "base" at : error calling rmByPfx: Invalid number of args: 1: [[]]`, 182 | }, 183 | } { 184 | cs := cs 185 | t.Run(cs.name, func(t *testing.T) { 186 | buf := new(bytes.Buffer) 187 | f := slogtotext.MustFormatter(buf, cs.template) 188 | err := f([]slogtotext.Pair{}) 189 | require.EqualError(t, err, cs.err) 190 | assert.Empty(t, buf.String()) 191 | }) 192 | } 193 | } 194 | 195 | func TestFormatter_invalidArgs(t *testing.T) { 196 | for _, cs := range []struct { 197 | name string 198 | template string 199 | out string 200 | }{ 201 | { 202 | name: "wrong_time", 203 | template: `{{ 1 | tmf "2006-01-02" "2006-01-02" }}`, 204 | out: `Invalid type: pos=3: int (1)`, 205 | }, 206 | { 207 | name: "wrong_string", 208 | template: `{{ 1 | trimSpace " ok " }}`, 209 | out: `ok 1`, 210 | }, 211 | } { 212 | cs := cs 213 | t.Run(cs.name, func(t *testing.T) { 214 | buf := new(bytes.Buffer) 215 | f := slogtotext.MustFormatter(buf, cs.template) 216 | err := f([]slogtotext.Pair{}) 217 | require.NoError(t, err) 218 | assert.Equal(t, cs.out, buf.String()) 219 | }) 220 | } 221 | } 222 | 223 | func TestMustFormatter_invalid(t *testing.T) { 224 | for _, cs := range []struct { 225 | name string 226 | template string 227 | value string 228 | }{ 229 | { 230 | name: "function", 231 | template: "{{ . | notExists }}", 232 | value: `template: base:1: function "notExists" not defined`, 233 | }, 234 | { 235 | name: "template", 236 | template: "{{", 237 | value: "template: base:1: unclosed action", 238 | }, 239 | } { 240 | cs := cs 241 | t.Run(cs.name, func(t *testing.T) { 242 | require.PanicsWithValue(t, cs.value, func() { 243 | slogtotext.MustFormatter(nil, cs.template) 244 | }) 245 | }) 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # human-readable-json-logging (pplog) 2 | 3 | [![build](https://github.com/michurin/human-readable-json-logging/actions/workflows/ci.yaml/badge.svg)](https://github.com/michurin/human-readable-json-logging/actions/workflows/ci.yaml) 4 | [![codecov](https://codecov.io/gh/michurin/human-readable-json-logging/graph/badge.svg?token=LDYMK3ZM06)](https://codecov.io/gh/michurin/human-readable-json-logging) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/michurin/human-readable-json-logging)](https://goreportcard.com/report/github.com/michurin/human-readable-json-logging) 6 | [![Go Reference](https://pkg.go.dev/badge/github.com/michurin/human-readable-json-logging.svg)](https://pkg.go.dev/github.com/michurin/human-readable-json-logging/slogtotext) 7 | 8 | It is library and binary CLI tool to make JSON logs readable including colorizing. 9 | The formatting is based on standard goland templating engine [text/template](https://pkg.go.dev/text/template). 10 | 11 | The CLI tool obtains templates from environment or configuration file. See examples below. 12 | 13 | You can find examples of using the library in [documentation](https://pkg.go.dev/github.com/michurin/human-readable-json-logging/slogtotext). 14 | Long story short, you are to direct output of your JSON logger (including modern [slog](https://pkg.go.dev/log/slog)) to magic reader and 15 | readable loglines shows up. 16 | 17 | ## Install and use 18 | 19 | ```sh 20 | go install -v github.com/michurin/human-readable-json-logging/cmd/...@latest 21 | ``` 22 | 23 | Running in subprocess mode: 24 | 25 | ```sh 26 | pplog ./service 27 | # or even 28 | pplog go run ./cmd/service/... 29 | ``` 30 | 31 | Running in pipe mode: 32 | 33 | ```sh 34 | ./service | pplog 35 | # or with redirections if you need to take both stderr and stdout 36 | ./service 2>&1 | pplog 37 | # or the same redirections in modern shells 38 | ./service |& pplog 39 | ``` 40 | 41 | ## Real life example 42 | 43 | One of my configuration file: 44 | 45 | ```sh 46 | # File pplog.env. The syntax of file is right the same as systemd env-files. 47 | # You can put it into your working dirrectory or any parrent. 48 | # You are free to set this variables in your .bashrc as well. 49 | 50 | PPLOG_LOGLINE=' 51 | {{- if .type }}{{ if eq .type "I" }}\e[92m{{ end }}{{if eq .type "E" }}\e[1;33;41m{{ end }}{{.type}}\e[0m {{ end }} 52 | {{- if .time }}{{ if eq .type "E" }}\e[93;41m{{ else }}\e[33m{{ end }}{{.time | tmf "2006-01-02T15:04:05Z07:00" "15:04:05"}}\e[0m {{ end }} 53 | {{- if .run }}\e[93m{{ .run | printf "%4.4s"}}\e[0m {{ end }} 54 | {{- if .comp }}\e[92m{{ .comp }}\e[0m {{ end }} 55 | {{- if .scope }}\e[32m{{ .scope }}\e[0m {{ end }} 56 | {{- if .ci_test_name }}\e[35;44;1m{{ .ci_test_name}}\e[0m {{ end }} 57 | {{- if .function }}\e[94m{{ .function }} \e[95m{{.lineno}}\e[0m {{ end }} 58 | {{- if .message }}\e[97m{{ .message }}\e[0m {{ end }} 59 | {{- if .error }}\e[91m{{ .error }}\e[0m {{ end }} 60 | {{- if .error_trace }}\e[93m{{ .error_trace }}\e[0m {{ end }} 61 | {{- range .ALL | rmByPfx 62 | "_tracing" 63 | "ci_test_name" 64 | "cluster_name" 65 | "comp" 66 | "env" 67 | "error" 68 | "error_trace" 69 | "function" 70 | "k8s_" 71 | "lineno" 72 | "message" 73 | "run" 74 | "scope" 75 | "tag" 76 | "time" 77 | "type" 78 | "xsource" 79 | }}\e[33m{{.K}}\e[0m={{.V}} {{ end }} 80 | ' 81 | 82 | PPLOG_ERRLINE='{{ if .BINARY }}{{ .TEXT }}{{ else }}\e[97m{{ .TEXT }}\e[0m{{ end }}' 83 | ``` 84 | 85 | My original logs look like this: 86 | 87 | ```json 88 | {"type":"I","time":"2024-01-01T07:33:44Z","message":"RPC call","k8s_node":"ix-x-kub114","k8s_pod":"booking-v64-64cf64db6d-gm9pc","cluster_name":"zeta","env":"prod","tag":"service.booking","lineno":39,"function":"xxx.xx/service-booking/internal/rpc/booking.(*Handler).Handle.func1","run":"578710a04dbb","comp":"rpc.booking","payload_resp":"{\"provider\":\"None\"}","payload_req":"{\"userId\":34664834}","xsource":"profile","_tracing":{"uber-trace-id":"669f:6a2c:c35c:1"}} 89 | ``` 90 | 91 | It turns to: 92 | 93 | ``` 94 | I 07:33:44 5787 rpc.booking xxx.xx/service-booking/internal/rpc/booking.(*Handler).Handle.func1 39 RPC call payload_req={"userId":34664834} payload_resp={"provider":"None"} 95 | ``` 96 | 97 | ## One more example: settings for gRPC+slog out of the box logging 98 | 99 | Basic settings for [github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging](https://github.com/grpc-ecosystem/go-grpc-middleware/) logs. 100 | 101 | ``` 102 | # file: pplog.env 103 | 104 | PPLOG_LOGLINE=' 105 | {{- .time | tmf "2006-01-02T15:04:05Z07:00" "15:04:05" }}{{" "}} 106 | {{- if .level }} 107 | {{- if eq .level "DEBUG"}}\e[90m 108 | {{- else if eq .level "INFO" }}\e[32m 109 | {{- else}}\e[91m 110 | {{- end }} 111 | {{- .level }}\e[0m 112 | {{- end }}{{" "}} 113 | {{- if (index . "grpc.code") }} 114 | {{- if eq "OK" (index . "grpc.code") }}\e[32mOK\e[0m {{else}}\e[91m{{ index . "grpc.code" }}\e[0m {{ end }} 115 | {{- else -}} 116 | {{"- "}} 117 | {{- end -}} 118 | \e[35m{{ index . "grpc.component" }}/\e[95m{{ index . "grpc.service" }}\e[35m/{{ index . "grpc.method" }}\e[0m{{" "}} 119 | {{- .msg }} 120 | {{- range .ALL | rm "msg" "time" "level" "grpc.component" "grpc.service" "grpc.method" "grpc.code"}} \e[33m{{.K}}\e[0m={{.V}}{{end}}' 121 | 122 | PPLOG_ERRLINE='{{ if .BINARY }}{{ .TEXT }}{{ else }}\e[97m{{.TEXT}}\e[0m{{ end }}' 123 | ``` 124 | 125 | ## Step by step customization 126 | 127 | First things first, I recommend you to prepare small file with your logs. Let's name it `example.log`. 128 | 129 | Now you can start to play with it by command like that: 130 | 131 | ```sh 132 | pplog cat example.log 133 | ``` 134 | 135 | You will see some formatted logs. 136 | 137 | Now create your first `pplog.env`. You can start from this universal one: 138 | 139 | ```sh 140 | PPLOG_LOGLINE='{{range .ALL}}{{.K}}={{.V}} {{end}}' 141 | PPLOG_ERRLINE='Invalid JONS: {{.TEXT}}' 142 | ``` 143 | 144 | You will see all your logs in KEY=VALUE format. Now look over all your keys and choose one you want 145 | to see in the first place. Say, `message`. Modify your `pplog.env` this way: 146 | 147 | ```sh 148 | PPLOG_LOGLINE='{{.message}}{{range .ALL | rm "message"}} {{.K}}={{.V}}{{end}}' 149 | PPLOG_ERRLINE='Invalid JONS: {{.TEXT}}' 150 | ``` 151 | 152 | You will see `message` in the first place and remove it from KEY=VALUE tail. 153 | 154 | Now, you are free to add colors: 155 | 156 | ```sh 157 | PPLOG_LOGLINE='\e[32m{{.message}}\e[m{{range .ALL | rm "message"}} {{.K}}={{.V}}{{end}}' 158 | PPLOG_ERRLINE='Invalid JONS: {{.TEXT}}' 159 | ``` 160 | 161 | We makes `message` green. Keep shaping your logs field by field. 162 | 163 | ## Template functions 164 | 165 | - All [`Masterminds/sprig/v3` functions](https://masterminds.github.io/sprig/) 166 | - `trimSpace` — example: `PPLOG_ERRLINE='INVALID: {{ .TEXT | trimSpace | printf "%q" }}'` 167 | - `tmf` — example: `{{ .A | tmf "2006-01-02T15:04:05Z07:00" "15:04:05" }}`. It is possible to specify number of alternative input formats: `tmf "inpfmt1" "inpfmt2" ... "outfmt"`. The first appropriate input format will be taken. 168 | - `rm` — example: `{{ range .ALL | rm "A" "B" "C" }}{{.K}}={{.V}};{{end}}` 169 | - `rmByPfx` 170 | - `xjson` 171 | - `xxjson` (experimental) 172 | - `skipLineIf` — evaluates to empty string, however has side effect: if one of arguments is true, or nonempty string or nonzero integer the line will be skipped and won't appear in output (see [discussion](https://github.com/michurin/human-readable-json-logging/issues/20) and [comment](https://github.com/michurin/human-readable-json-logging/pull/21)) 173 | - `skipLineUnless` — inverted `skipLineIf`, example: `PPLOG_ERRLINE='{{ skipLineUnless .TEXT }}Invalid JONS: {{ .TEXT }}'` — it skips empty lines 174 | 175 | ## Template special variables 176 | 177 | - In `PPLOG_LOGLINE` template: 178 | - `RAW_INPUT` 179 | - `ALL` — list of all pairs key-value 180 | - In `PPLOG_ERRLINE` template: 181 | - `TEXT` 182 | - `BINARY` — does TEXT contains control characters 183 | - If `PPLOG_CHILD_MODE` not empty `pplog` runs in child mode as if it has `-c` switch 184 | 185 | ## Most common colors 186 | 187 | ``` 188 | Text colors Text High Background Hi Background Decoration 189 | ------------------ ------------------ ------------------ ------------------- -------------------- 190 | \e[30mBlack \e[0m \e[90mBlack \e[0m \e[40mBlack \e[0m \e[100mBlack \e[0m \e[1mBold \e[0m 191 | \e[31mRed \e[0m \e[91mRed \e[0m \e[41mRed \e[0m \e[101mRed \e[0m \e[4mUnderline \e[0m 192 | \e[32mGreen \e[0m \e[92mGreen \e[0m \e[42mGreen \e[0m \e[102mGreen \e[0m \e[7mReverse \e[0m 193 | \e[33mYellow \e[0m \e[93mYellow \e[0m \e[43mYellow \e[0m \e[103mYellow \e[0m 194 | \e[34mBlue \e[0m \e[94mBlue \e[0m \e[44mBlue \e[0m \e[104mBlue \e[0m Combinations 195 | \e[35mMagenta\e[0m \e[95mMagenta\e[0m \e[45mMagenta\e[0m \e[105mMagenta\e[0m ----------------------- 196 | \e[36mCyan \e[0m \e[96mCyan \e[0m \e[46mCyan \e[0m \e[106mCyan \e[0m \e[1;4;103;31mWARN\e[0m 197 | \e[37mWhite \e[0m \e[97mWhite \e[0m \e[47mWhite \e[0m \e[107mWhite \e[0m 198 | ``` 199 | 200 | ## Run modes explanation 201 | 202 | ### Pipe mode 203 | 204 | The most confident mode. In this mode your shell cares about all your processes. Just do 205 | 206 | ```sh 207 | ./service | pplog 208 | # or with redirections if you need to take both stderr and stdout 209 | ./service 2>&1 | pplog 210 | # or the same redirections in modern shells 211 | ./service |& pplog 212 | ``` 213 | 214 | ### Simple subprocess mode 215 | 216 | If you say just like that: 217 | 218 | ```sh 219 | pplog ./service 220 | ``` 221 | 222 | `pplog` runs `./servcie` as a child process and tries to manage it. 223 | 224 | If you press Ctrl-C, `pplog` sends `SIGINT`, `SIGTERM`, `SIGKILL` to its child consequently with 1s delay in between. 225 | 226 | `pplog` tries to wait child process exited and returns its exit code transparently. 227 | 228 | Obvious disadvantage is that `pplog` doesn't try to manage children of child (if any), daemons etc. 229 | 230 | ### Child (or coprocess) mode 231 | 232 | In this mode `pplog` starts as a child of `./service` 233 | 234 | ```sh 235 | pplog -c ./service 236 | ``` 237 | 238 | So, `./service` itself obtains all signals and Ctrl-Cs directly. 239 | 240 | However, there are disadvantages here too. `pplog` can not get `./service`s exit code. And this mode unavailable under MS Windows. 241 | 242 | ## Similar projects 243 | 244 | - `jlv` (JSON Log Viewer) — [https://github.com/hedhyw/json-log-viewer](https://github.com/hedhyw/json-log-viewer) 245 | - `logdy` — [https://logdy.dev/](https://logdy.dev/) 246 | - `humanlog` — [https://humanlog.io/](https://humanlog.io/), [https://github.com/humanlogio/humanlog](https://github.com/humanlogio/humanlog) 247 | - `jq` — `echo '{"time":"12:00","msg":"OK"}' | jq -r '.time+" "+.msg'` produces `12:00 OK` — [https://jqlang.github.io/jq/](https://jqlang.github.io/jq/) 248 | - `grc/grcat` — [https://github.com/garabik/grc](https://github.com/garabik/grc) 249 | - `hl` — [https://github.com/pamburus/hl](https://github.com/pamburus/hl) — Powerful log viewer 250 | 251 | In fact, `jq` is really great. If you are brave enough, you can dive into things like that: 252 | 253 | ```sh 254 | cat log 255 | ``` 256 | 257 | Log file content: 258 | 259 | ``` 260 | {"msg": "ok1", "id": 1} 261 | INV 262 | {"msg": "ok2", "id": 2, "opt": "HERE"} 263 | ``` 264 | 265 | Formatting, using `jq`: 266 | 267 | ```sh 268 | cat log | jq -rR '. as $line | try ( fromjson | "\u001b[92m\(.msg)\u001b[m \(.id) \(.opt // "-")" ) catch "\u001b[1mInvalid JSON: \u001b[31m\($line)\u001b[m"' 269 | ``` 270 | 271 | The output will be colored: 272 | 273 | ``` 274 | ok1 1 - 275 | Invalid JSON: INV 276 | ok2 2 HERE 277 | ``` 278 | 279 | ## TODO 280 | 281 | - Usage: show templates in debug mode 282 | - Behavior tests: 283 | - `-c` 284 | - `PPLOG_CHILD_MODE` environment variable 285 | - basic `runs-on: windows-latest` 286 | - passing exit code 287 | - Docs: contributing guide: how to run behavior tests locally 288 | - Docs: godoc 289 | 290 | ## Known issues 291 | 292 | ### Not optimal integration with log/slog 293 | 294 | If you decided to use this code as library as part of your product, you have to keep in mind, that 295 | this tool provides `io.Writer` to pipe log stream. It is easiest way to modify behavior of logger, however 296 | it leads to overhead for extra marshaling/unmarshaling. However, as well as we use human readable logs in 297 | local environment only, it is acceptable to have a little overhead. 298 | 299 | ### Subprocesses handling issues 300 | 301 | The problem is that many processes have to be synchronized: shell-process, pplog-process, target-process with all its children. 302 | 303 | You are able to choose one of three modes: pipe-, subprocess- and child-mode. Each of them has its own disadvantages. 304 | 305 | ### Line-by-line processing 306 | 307 | This tool processes input stream line by line. It means, that it won't work with multiline JSONs in logs like that 308 | 309 | ```json 310 | { 311 | "level": "info", 312 | "message": "obtaining data" 313 | } 314 | { 315 | "level": "error", 316 | "message": "invalid data" 317 | } 318 | ``` 319 | 320 | as well as it won't work with mixed lines like this: 321 | 322 | ``` 323 | Raw message{"message": "valid json log record"} 324 | ``` 325 | 326 | all that cases will be considered and reported as wrong JSON. 327 | 328 | Honestly, I have tried to implement smarter scanner. It's not a big deal, however, 329 | in fact, it is not convenient. For instance, it consider message like this `code=200` 330 | in wearied way: `code=` is wrong JSON, however `200` is valid JSON. 331 | Things like that `0xc00016f0e1` get really awful. 332 | 333 | I have played with different approaches 334 | and decided just to split logs line by line first. 335 | 336 | ### Hard to reproduce issue 337 | 338 | It seems there is a problem appears in subprocess mode when subprocess going to die and its final 339 | output makes error (or panic?) in `text/template` package. 340 | 341 | This issue has to be solved by [this commit](https://github.com/michurin/human-readable-json-logging/commit/c8ce47a67812e8f616b0c23a7b1abc2fced15461), 342 | however please report if you find how to reproduce such things. 343 | 344 | ## Contributors 345 | 346 | - [vitalyshatskikh](https://github.com/vitalyshatskikh) 347 | --------------------------------------------------------------------------------