├── .gitignore ├── static └── screenshot.png ├── go.mod ├── .goreleaser.yml ├── main_test.go ├── README.md ├── go.sum └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | my.db 2 | dist/ 3 | wonitor 4 | -------------------------------------------------------------------------------- /static/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rverton/wonitor/HEAD/static/screenshot.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rverton/wonitor 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/dgraph-io/badger v1.6.1 7 | github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c 8 | github.com/gosimple/slug v1.9.0 9 | github.com/magiconair/properties v1.8.1 10 | github.com/pmezard/go-difflib v1.0.0 11 | github.com/sergi/go-diff v1.1.0 12 | github.com/stretchr/testify v1.4.0 13 | github.com/urfave/cli/v2 v2.2.0 14 | go.etcd.io/bbolt v1.3.5 15 | ) 16 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | builds: 5 | - env: 6 | - CGO_ENABLED=0 7 | goos: 8 | - linux 9 | - windows 10 | - darwin 11 | archives: 12 | - replacements: 13 | darwin: Darwin 14 | linux: Linux 15 | windows: Windows 16 | 386: i386 17 | amd64: x86_64 18 | checksum: 19 | name_template: 'checksums.txt' 20 | snapshot: 21 | name_template: "{{ .Tag }}-next" 22 | changelog: 23 | sort: asc 24 | filters: 25 | exclude: 26 | - '^docs:' 27 | - '^test:' 28 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | 11 | badger "github.com/dgraph-io/badger" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TempFileName(prefix, suffix string) string { 16 | randBytes := make([]byte, 16) 17 | rand.Read(randBytes) 18 | return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix) 19 | } 20 | 21 | func refreshDb() (string, *badger.DB, error) { 22 | fname := TempFileName("wonitor", "") 23 | db, err := initDb(fname) 24 | 25 | return fname, db, err 26 | } 27 | 28 | func bucketCount(db *badger.DB, name string) (int, error) { 29 | count := 0 30 | 31 | err := db.View(func(tx *badger.Txn) error { 32 | 33 | it := tx.NewIterator(badger.DefaultIteratorOptions) 34 | defer it.Close() 35 | for it.Rewind(); it.Valid(); it.Next() { 36 | count++ 37 | } 38 | 39 | return nil 40 | }) 41 | 42 | return count, err 43 | } 44 | 45 | func TestAdd(t *testing.T) { 46 | fname, db, err := refreshDb() 47 | fmt.Println(fname) 48 | if err != nil { 49 | panic(err) 50 | } 51 | defer os.RemoveAll(fname) 52 | 53 | cnt, err := bucketCount(db, "urls") 54 | assert.Nil(t, err) 55 | assert.Equal(t, 0, cnt) 56 | 57 | addUrl(db, "https://robinverton.de", false) 58 | 59 | cnt, err = bucketCount(db, "urls") 60 | assert.Nil(t, err) 61 | assert.Equal(t, 1, cnt) 62 | } 63 | 64 | func TestRemove(t *testing.T) { 65 | fname, db, err := refreshDb() 66 | if err != nil { 67 | panic(err) 68 | } 69 | defer os.RemoveAll(fname) 70 | 71 | cnt, err := bucketCount(db, "urls") 72 | assert.Nil(t, err) 73 | assert.Equal(t, cnt, 0) 74 | 75 | addUrl(db, "https://robinverton.de", false) 76 | 77 | cnt, _ = bucketCount(db, "urls") 78 | assert.Equal(t, cnt, 1) 79 | 80 | removeUrl(db, "https://robinverton.de") 81 | 82 | cnt, _ = bucketCount(db, "urls") 83 | assert.Equal(t, cnt, 0) 84 | } 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # web monitor 2 | 3 | fast, zero config web endpoint change monitor. for comparing responses, a selected 4 | list of http headers and the full response body is stored on a local key/value store file. 5 | no configuration needed. 6 | 7 | * to increase network throughput, a `--worker` flag allows to set the concurrency when monitoring. 8 | * endpoints returning a javascript content type will be beautified by default. 9 | * using `--headersOnly` when adding a URL allows to only monitor response headers. 10 | ![](./static/screenshot.png) 11 | 12 | ## installation 13 | 14 | Install via go or [binary release](https://github.com/rverton/wonitor/releases): 15 | 16 | go get -u github.com/rverton/wonitor 17 | 18 | ## usage 19 | 20 | ``` 21 | λ $ ./wonitor 22 | NAME: 23 | wonitor - web monitor 24 | 25 | USAGE: 26 | wonitor [global options] command [command options] [arguments...] 27 | 28 | COMMANDS: 29 | add, a add endpoint to monitor 30 | delete, d deletes an endpoint 31 | get, g get endpoint body 32 | list, l list all monitored endpoints and their body size in bytes 33 | monitor, m retrieve all urls and compare them 34 | help, h Shows a list of commands or help for one command 35 | 36 | GLOBAL OPTIONS: 37 | --help, -h show help (default: false) 38 | 39 | λ $ ./wonitor add --url https://unlink.io/ 40 | + https://unlink.io/ 41 | λ $ ./wonitor monitor --save 42 | [https://unlink.io/] 1576b diff: 43 | --- Original 44 | +++ Current 45 | @@ -1 +1,47 @@ 46 | +HTTP/1.1 200 OK 47 | +Content-Type: text/html 48 | +Server: nginx/1.10.3 (Ubuntu) 49 | +X-Frame-Options: DENY 50 | 51 | + 52 | + 53 | +
54 | [... snip ...]
55 | +
56 | + 57 | + 58 | + 59 | 60 | λ $ ./wonitor monitor --save 61 | λ $ # no output because no change detected 62 | ``` 63 | 64 | ## endpoint diffing 65 | 66 | The following headers are also included in the saved response and monitored for changes: 67 | 68 | ```go 69 | var headerToInclude = []string{ 70 | "Host", 71 | "Content-Length", 72 | "Content-Type", 73 | "Location", 74 | "Access-Control-Allow-Origin", 75 | "Access-Control-Allow-Methods", 76 | "Access-Control-Expose-Headers", 77 | "Access-Control-Allow-Credentials", 78 | "Allow", 79 | "Content-Security-Policy", 80 | "Proxy-Authenticate", 81 | "Server", 82 | "WWW-Authenticate", 83 | "X-Frame-Options", 84 | "X-Powered-By", 85 | } 86 | ``` 87 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= 2 | github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 5 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 6 | github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= 7 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 8 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 9 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 10 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 11 | github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= 12 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 13 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 14 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 15 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 17 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg= 19 | github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= 20 | github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= 21 | github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= 22 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 23 | github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c h1:+Zo5Ca9GH0RoeVZQKzFJcTLoAixx5s5Gq3pTIS+n354= 24 | github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c/go.mod h1:HJGU9ULdREjOcVGZVPB5s6zYmHi1RxzT71l2wQyLmnE= 25 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 26 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 27 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 28 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 29 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 30 | github.com/gosimple/slug v1.9.0 h1:r5vDcYrFz9BmfIAMC829un9hq7hKM4cHUrsv36LbEqs= 31 | github.com/gosimple/slug v1.9.0/go.mod h1:AMZ+sOVe65uByN3kgEyf9WEBKBCSS+dJjMX9x4vDJbg= 32 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 33 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 34 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 35 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 36 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 37 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 38 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 39 | github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= 40 | github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 41 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 42 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 43 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 44 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 45 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 46 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 47 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 48 | github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ= 49 | github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q= 50 | github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= 51 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 52 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 53 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 54 | github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= 55 | github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 56 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 57 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 58 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 59 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 60 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 61 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 62 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 63 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 64 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 65 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 66 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 67 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 68 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 69 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 70 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 71 | github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= 72 | github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= 73 | github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= 74 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 75 | go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= 76 | go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= 77 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 78 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 79 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 80 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 81 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 82 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 83 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= 85 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 86 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 87 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 88 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 89 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 90 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 91 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 92 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "crypto/tls" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | "os" 13 | "strings" 14 | "sync" 15 | "time" 16 | 17 | badger "github.com/dgraph-io/badger" 18 | "github.com/ditashi/jsbeautifier-go/jsbeautifier" 19 | "github.com/pmezard/go-difflib/difflib" 20 | 21 | "github.com/gosimple/slug" 22 | "github.com/urfave/cli/v2" 23 | ) 24 | 25 | const TIMEOUT = 8 * time.Second 26 | const RESPONSE_BODY_LIMIT = 1024 * 1024 * 3 //3MB 27 | const BUCKET_URLS = "urls" 28 | const DB_FILE = "my.db" 29 | 30 | // this headers will be included in the response which is stored 31 | var headerToInclude = []string{ 32 | "Host", 33 | "Content-Length", 34 | "Content-Type", 35 | "Location", 36 | "Access-Control-Allow-Origin", 37 | "Access-Control-Allow-Methods", 38 | "Access-Control-Expose-Headers", 39 | "Access-Control-Allow-Credentials", 40 | "Allow", 41 | "Content-Security-Policy", 42 | "Proxy-Authenticate", 43 | "Server", 44 | "WWW-Authenticate", 45 | "X-Frame-Options", 46 | "X-Powered-By", 47 | } 48 | 49 | type Bits uint8 50 | 51 | const MODE_HEADERS_ONLY Bits = 1 << iota 52 | 53 | func Set(b, flag Bits) Bits { return b | flag } 54 | func Clear(b, flag Bits) Bits { return b &^ flag } 55 | func Toggle(b, flag Bits) Bits { return b ^ flag } 56 | func Has(b, flag Bits) bool { return b&flag != 0 } 57 | 58 | func addUrl(db *badger.DB, url string, useStdin bool, headersOnly bool) error { 59 | 60 | var mode Bits = 0 61 | 62 | if headersOnly { 63 | mode = MODE_HEADERS_ONLY 64 | } 65 | 66 | return db.Update(func(tx *badger.Txn) error { 67 | 68 | if useStdin { 69 | 70 | scanner := bufio.NewScanner(os.Stdin) 71 | for scanner.Scan() { 72 | url = scanner.Text() 73 | e := badger.NewEntry([]byte(url), []byte("")).WithMeta(byte(mode)) 74 | err := tx.SetEntry(e) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | fmt.Printf("+ %v\n", url) 80 | } 81 | 82 | return nil 83 | } 84 | fmt.Printf("+ %v\n", url) 85 | 86 | e := badger.NewEntry([]byte(url), []byte("")).WithMeta(byte(mode)) 87 | return tx.SetEntry(e) 88 | }) 89 | } 90 | 91 | func removeUrl(db *badger.DB, url string) error { 92 | return db.Update(func(tx *badger.Txn) error { 93 | return tx.Delete([]byte(url)) 94 | }) 95 | } 96 | 97 | func list(db *badger.DB) error { 98 | return db.View(func(tx *badger.Txn) error { 99 | opts := badger.DefaultIteratorOptions 100 | opts.PrefetchSize = 10 101 | it := tx.NewIterator(opts) 102 | defer it.Close() 103 | 104 | for it.Rewind(); it.Valid(); it.Next() { 105 | item := it.Item() 106 | k := item.Key() 107 | 108 | mode := "" 109 | if Has(Bits(item.UserMeta()), MODE_HEADERS_ONLY) { 110 | mode = ", ONLY_HEADERS" 111 | } 112 | 113 | err := item.Value(func(v []byte) error { 114 | fmt.Printf("%v, %vB%v\n", string(k), len(v), mode) 115 | return nil 116 | }) 117 | if err != nil { 118 | return err 119 | } 120 | } 121 | 122 | return nil 123 | }) 124 | } 125 | 126 | func getUrl(db *badger.DB, url string) error { 127 | return db.View(func(tx *badger.Txn) error { 128 | item, err := tx.Get([]byte(url)) 129 | if err != nil { 130 | return err 131 | } 132 | 133 | valCopy, err := item.ValueCopy(nil) 134 | fmt.Println(string(valCopy)) 135 | return nil 136 | }) 137 | } 138 | 139 | func retrieve(url string) (*http.Response, error) { 140 | transport := &http.Transport{ 141 | MaxIdleConnsPerHost: -1, 142 | TLSClientConfig: &tls.Config{ 143 | InsecureSkipVerify: true, 144 | }, 145 | DisableKeepAlives: true, 146 | } 147 | 148 | var client = &http.Client{ 149 | Timeout: TIMEOUT, 150 | Transport: transport, 151 | CheckRedirect: func(redirectedRequest *http.Request, previousRequest []*http.Request) error { 152 | return http.ErrUseLastResponse 153 | }, 154 | } 155 | 156 | return client.Get(url) 157 | } 158 | 159 | func Abs(x int) int { 160 | if x < 0 { 161 | return -x 162 | } 163 | return x 164 | } 165 | 166 | func beautifyJs(s string) string { 167 | opts := jsbeautifier.DefaultOptions() 168 | splitted := strings.SplitN(s, "\n\n", 2) 169 | 170 | if len(splitted) != 2 { 171 | return s 172 | } 173 | 174 | beautified, err := jsbeautifier.Beautify(&splitted[1], opts) 175 | if err != nil { 176 | return s 177 | } 178 | 179 | return fmt.Sprintf("%v\n\n%v", splitted[0], beautified) 180 | } 181 | 182 | func handleDiff(url, bodyOld, bodyNew, outDir string, beautify bool) { 183 | diffLen := Abs(len(bodyOld) - len(bodyNew)) 184 | 185 | if beautify { 186 | bodyOld = beautifyJs(bodyOld) 187 | bodyNew = beautifyJs(bodyNew) 188 | } 189 | 190 | diff := difflib.UnifiedDiff{ 191 | A: difflib.SplitLines(bodyOld), 192 | B: difflib.SplitLines(bodyNew), 193 | FromFile: "Original", 194 | ToFile: "Current", 195 | Context: 3, 196 | } 197 | text, _ := difflib.GetUnifiedDiffString(diff) 198 | 199 | if outDir == "" { 200 | fmt.Printf("[%v] %vb diff:\n", url, diffLen) 201 | fmt.Printf("%v", text) 202 | } else { 203 | filename := fmt.Sprintf("%v/%v_%v.diff", outDir, time.Now().Format("20060201-150405"), slug.Make(url)) 204 | data := []byte(text) 205 | 206 | err := ioutil.WriteFile(filename, data, 0644) 207 | if err != nil { 208 | log.Printf("error saving output to %v: %v", filename, err) 209 | } 210 | } 211 | } 212 | 213 | // only leave a few interesting headers in the response 214 | func minifyResponse(resp *http.Response, onlyHeaders bool) ([]byte, error) { 215 | defer resp.Body.Close() 216 | 217 | var b bytes.Buffer 218 | 219 | b.WriteString(fmt.Sprintf("%v %v\n", resp.Proto, resp.Status)) 220 | 221 | for _, header := range headerToInclude { 222 | if onlyHeaders && header == "Content-Length" { 223 | continue 224 | } 225 | 226 | v := resp.Header.Get(header) 227 | if v != "" { 228 | b.WriteString(fmt.Sprintf("%v: %v\n", header, v)) 229 | } 230 | } 231 | 232 | if !onlyHeaders { 233 | limitedReader := &io.LimitedReader{R: resp.Body, N: RESPONSE_BODY_LIMIT} 234 | 235 | b.WriteString("\n") 236 | io.Copy(&b, limitedReader) 237 | } 238 | 239 | return b.Bytes(), nil 240 | } 241 | 242 | func retrieveAndCompare(db *badger.DB, url, outDir string, save bool, bodyOld []byte, beautify bool, onlyHeaders bool, wg *sync.WaitGroup) { 243 | defer wg.Done() 244 | 245 | resp, err := retrieve(url) 246 | if err != nil { 247 | log.Printf("err retrieving %v: %v", url, err) 248 | return 249 | } 250 | 251 | bodyNew, err := minifyResponse(resp, onlyHeaders) 252 | if err != nil { 253 | log.Printf("err minifying resp: %v", err) 254 | return 255 | } 256 | 257 | if bytes.Compare(bodyOld, bodyNew) == 0 { 258 | return 259 | } 260 | 261 | if beautify && strings.Contains(resp.Header.Get("Content-Type"), "javascript") { 262 | beautify = true 263 | } else { 264 | beautify = false 265 | } 266 | 267 | handleDiff(url, string(bodyOld), string(bodyNew), outDir, beautify) 268 | 269 | if save { 270 | err := db.Update(func(tx *badger.Txn) error { 271 | return tx.Set([]byte(url), bodyNew) 272 | }) 273 | 274 | if err != nil { 275 | log.Printf("err updating body: %v", err) 276 | } 277 | } 278 | } 279 | 280 | func monitor(db *badger.DB, save bool, outDir string, beautify bool, worker int) error { 281 | var wg sync.WaitGroup 282 | var sem = make(chan int, worker) 283 | 284 | err := db.Update(func(tx *badger.Txn) error { 285 | 286 | opts := badger.DefaultIteratorOptions 287 | opts.PrefetchSize = 10 288 | it := tx.NewIterator(opts) 289 | defer it.Close() 290 | 291 | for it.Rewind(); it.Valid(); it.Next() { 292 | sem <- 1 293 | 294 | item := it.Item() 295 | url := string(item.Key()) 296 | 297 | onlyHeaders := Has(Bits(item.UserMeta()), MODE_HEADERS_ONLY) 298 | 299 | bodyOld, err := item.ValueCopy(nil) 300 | if err != nil { 301 | return err 302 | } 303 | 304 | wg.Add(1) 305 | go func(db *badger.DB, url, outDIr string, save bool, bodyOld []byte, onlyHeaders bool, wg *sync.WaitGroup) { 306 | retrieveAndCompare(db, string(url), outDir, save, bodyOld, beautify, onlyHeaders, wg) 307 | <-sem 308 | }(db, string(url), outDir, save, bodyOld, onlyHeaders, &wg) 309 | } 310 | 311 | return nil 312 | }) 313 | 314 | wg.Wait() 315 | 316 | return err 317 | } 318 | 319 | func initDb(path string) (*badger.DB, error) { 320 | options := badger.DefaultOptions(path) 321 | options.Logger = nil 322 | 323 | return badger.Open(options) 324 | } 325 | 326 | func main() { 327 | db, err := initDb(DB_FILE) 328 | if err != nil { 329 | log.Fatal(err) 330 | } 331 | defer db.Close() 332 | 333 | app := &cli.App{ 334 | Name: "wonitor", 335 | Usage: "web monitor", 336 | Commands: []*cli.Command{ 337 | { 338 | Name: "add", 339 | Aliases: []string{"a"}, 340 | Usage: "add endpoint to monitor", 341 | Flags: []cli.Flag{ 342 | &cli.StringFlag{ 343 | Name: "url", 344 | Usage: "url to add", 345 | }, 346 | &cli.BoolFlag{ 347 | Name: "stdin", 348 | Usage: "read urls from stdin, line by line", 349 | }, 350 | &cli.BoolFlag{ 351 | Name: "headersOnly", 352 | Usage: "only retrieve headers and discard body", 353 | Value: false, 354 | }, 355 | }, 356 | Action: func(c *cli.Context) error { 357 | if c.String("url") == "" && !c.Bool("stdin") { 358 | fmt.Println("please use --url or --stdin") 359 | os.Exit(1) 360 | } 361 | 362 | return addUrl(db, c.String("url"), c.Bool("stdin"), c.Bool("headersOnly")) 363 | }, 364 | }, 365 | { 366 | Name: "delete", 367 | Aliases: []string{"d"}, 368 | Usage: "deletes an endpoint", 369 | Flags: []cli.Flag{ 370 | &cli.StringFlag{ 371 | Name: "url", 372 | Usage: "url to delete", 373 | Required: true, 374 | }, 375 | }, 376 | Action: func(c *cli.Context) error { 377 | return removeUrl(db, c.String("url")) 378 | }, 379 | }, 380 | { 381 | Name: "get", 382 | Aliases: []string{"g"}, 383 | Usage: "get endpoint body", 384 | Flags: []cli.Flag{ 385 | &cli.StringFlag{ 386 | Name: "url", 387 | Usage: "url to get from store", 388 | Required: true, 389 | }, 390 | }, 391 | Action: func(c *cli.Context) error { 392 | return getUrl(db, c.String("url")) 393 | }, 394 | }, 395 | { 396 | Name: "list", 397 | Aliases: []string{"l"}, 398 | Usage: "list all monitored endpoints and their body size in bytes", 399 | Action: func(c *cli.Context) error { 400 | return list(db) 401 | }, 402 | }, 403 | { 404 | Name: "monitor", 405 | Aliases: []string{"m"}, 406 | Usage: "retrieve all urls and compare them", 407 | Action: func(c *cli.Context) error { 408 | return monitor(db, c.Bool("save"), c.String("outDir"), c.Bool("jsbeautify"), c.Int("worker")) 409 | }, 410 | Flags: []cli.Flag{ 411 | &cli.BoolFlag{ 412 | Name: "save", 413 | Usage: "save updates to store", 414 | Value: false, 415 | }, 416 | &cli.StringFlag{ 417 | Name: "outDir", 418 | Usage: "save diffs as html to folder", 419 | }, 420 | &cli.IntFlag{ 421 | Name: "worker", 422 | Usage: "numbers of worker to retrieve data", 423 | Value: 20, 424 | }, 425 | &cli.BoolFlag{ 426 | Name: "jsbeautify", 427 | Usage: "beautify javascript if found", 428 | Value: true, 429 | }, 430 | }, 431 | }, 432 | }, 433 | } 434 | 435 | err = app.Run(os.Args) 436 | if err != nil { 437 | log.Fatal(err) 438 | } 439 | } 440 | --------------------------------------------------------------------------------