├── .gitignore ├── suite_test.go ├── noop_dest.go ├── LICENSE ├── Makefile ├── .travis.yml ├── interface.go ├── persist_test.go ├── file_dest_test.go ├── persist.go ├── file_dest.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | *.coverprofile 26 | -------------------------------------------------------------------------------- /suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 RightScale, Inc. - see LICENSE 2 | 3 | package persist 4 | 5 | import ( 6 | "testing" 7 | 8 | . "github.com/onsi/ginkgo" 9 | . "github.com/onsi/gomega" 10 | "github.com/onsi/gomega/format" 11 | "gopkg.in/inconshreveable/log15.v2" 12 | ) 13 | 14 | func TestUCA(t *testing.T) { 15 | //log.SetOutput(ginkgo.GinkgoWriter) 16 | log15.Root().SetHandler(log15.StreamHandler(GinkgoWriter, log15.TerminalFormat())) 17 | 18 | format.UseStringerRepresentation = true 19 | RegisterFailHandler(Fail) 20 | 21 | RunSpecs(t, "persist") 22 | } 23 | -------------------------------------------------------------------------------- /noop_dest.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 RightScale, Inc. - see LICENSE 2 | 3 | package persist 4 | 5 | import ( 6 | "io" 7 | 8 | "gopkg.in/inconshreveable/log15.v2" 9 | ) 10 | 11 | type noopDest struct { 12 | log log15.Logger 13 | } 14 | 15 | func NewNoopDest(log log15.Logger) (LogDestination, error) { 16 | nd := noopDest{log: log} 17 | return &nd, nil 18 | } 19 | 20 | func (nd *noopDest) Close() {} 21 | 22 | func (nd *noopDest) Write(p []byte) (int, error) { 23 | return len(p), nil 24 | } 25 | 26 | func (nd *noopDest) ReplayReaders() []io.ReadCloser { 27 | return nil 28 | } 29 | 30 | // StartRotate is called by persist in order to start a new log file. 31 | func (nd *noopDest) StartRotate() error { 32 | return nil 33 | } 34 | 35 | func (nd *noopDest) EndRotate() error { 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 RightScale, Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #! /usr/bin/make 2 | # 3 | # Makefile for Golang projects, v1 (simplified for libraries) 4 | # 5 | # Features: 6 | # - runs ginkgo tests recursively, computes code coverage report 7 | # - code coverage ready for travis-ci to upload and produce badges for README.md 8 | # - to include the build status and code coverage badge in CI use (replace NAME by what 9 | # you set $(NAME) to further down, and also replace magnum.travis-ci.com by travis-ci.org for 10 | # publicly accessible repos [sigh]): 11 | # [![Build Status](https://magnum.travis-ci.com/rightscale/NAME.svg?token=4Q13wQTY4zqXgU7Edw3B&branch=master)](https://magnum.travis-ci.com/rightscale/NAME) 12 | # ![Code Coverage](https://s3.amazonaws.com/rs-code-coverage/NAME/cc_badge_master.svg) 13 | # 14 | # Top-level targets: 15 | # default: compile the program, you can thus use make && ./NAME -options ... 16 | # test: runs unit tests recursively and produces code coverage stats and shows them 17 | # travis-test: just runs unit tests recursively 18 | # 19 | # ** GOPATH and import dependencies ** 20 | # - to compile or run ginkgo like in the Makefile: 21 | # export GOPATH=`pwd`/.vendor; export PATH="`pwd`/.vendor/bin:$PATH" 22 | 23 | #NAME=$(shell basename $$PWD) 24 | NAME=persist 25 | ACL=public-read 26 | # dependencies that are used by the build&test process 27 | DEPEND=golang.org/x/tools/cmd/cover github.com/onsi/ginkgo/ginkgo \ 28 | github.com/onsi/gomega github.com/rlmcpherson/s3gof3r/gof3r \ 29 | github.com/dkulchenko/bunch gopkg.in/inconshreveable/log15.v2 30 | 31 | #=== below this line ideally remains unchanged, add new targets at the end === 32 | 33 | TRAVIS_BRANCH?=dev 34 | DATE=$(shell date '+%F %T') 35 | TRAVIS_COMMIT?=$(shell git symbolic-ref HEAD | cut -d"/" -f 3) 36 | # we manually adjust the GOPATH instead of trying to prefix everything with `bunch go` 37 | ifeq ($(OS),Windows_NT) 38 | SHELL:=/bin/dash 39 | GOPATH:=$(shell cygpath --windows $(PWD))/.vendor;$(GOPATH) 40 | else 41 | GOPATH:=$(PWD)/.vendor:$(GOPATH) 42 | endif 43 | # we build $(DEPEND) binaries into the .vendor subdir 44 | PATH:=$(PWD)/.vendor/bin:$(PATH) 45 | 46 | .PHONY: depend clean default 47 | 48 | # the default target builds a binary in the top-level dir for whatever the local OS is 49 | default: $(NAME) 50 | $(NAME): *.go 51 | go build -o $(NAME) . 52 | 53 | gopath: 54 | @echo export GOPATH="$(GOPATH)" 55 | 56 | # Installing build dependencies is a bit of a mess. Don't want to spend lots of time in 57 | # Travis doing this. The folllowing just relies on go get no reinstalling when it's already 58 | # there, like your laptop. 59 | depend: 60 | go get -v $(DEPEND) 61 | 62 | clean: 63 | rm -rf build .vendor/pkg 64 | 65 | # gofmt uses the awkward *.go */*.go because gofmt -l . descends into the .vendor tree 66 | # and then pointlessly complains about bad formatting in imported packages, sigh 67 | lint: 68 | @if gofmt -l *.go | grep .go; then \ 69 | echo "^- Repo contains improperly formatted go files; run gofmt -w *.go" && exit 1; \ 70 | else echo "All .go files formatted correctly"; fi 71 | go tool vet -composites=false *.go 72 | # go tool vet -composites=false **/*.go 73 | 74 | travis-test: cover 75 | 76 | # running ginkgo twice, sadly, the problem is that -cover modifies the source code with the effect 77 | # that if there are errors the output of gingko refers to incorrect line numbers 78 | # tip: if you don't like colors use gingkgo -r -noColor 79 | test: lint gopath 80 | ginkgo -r --randomizeAllSpecs --randomizeSuites --failOnPending 81 | 82 | race: lint 83 | ginkgo -r --randomizeAllSpecs --randomizeSuites --failOnPending --race 84 | 85 | cover: lint 86 | ginkgo -r --randomizeAllSpecs --randomizeSuites --failOnPending -cover 87 | @echo 'mode: atomic' >_total 88 | @for f in `find . -name \*.coverprofile`; do tail -n +2 $$f >>_total; done 89 | @mv _total total.coverprofile 90 | @COVERAGE=$$(go tool cover -func=total.coverprofile | grep "^total:" | grep -o "[0-9\.]*");\ 91 | echo "*** Code Coverage is $$COVERAGE% ***" 92 | @echo Details: go tool cover -func=total.coverprofile 93 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Travis-CI for Golang projects 2 | # Runs the tests and produces a code coverage report. 3 | language: go 4 | go: 5 | - 1.4 6 | env: 7 | global: 8 | # GITHUB_TOKEN= to push code coverage comment to github 9 | - secure: "VENBT5SZ8r/nDHaRvyVCkvbs87UJdQw9YzrmKY4fJGlWR3GzVP1UuVl96vMWTCkJoonQSfpkllCdvnHUhm7OJIhd4+x3l53S1KNx2YbzOZ9T2h43xHfYlVhkWAikQkDVd/R3z1ykn+Jihh7uwqh266fyxn0XdBGVf1NlEbmcS4O7Yr3dWgctSK3CFHvCWuCSHPfg0JTNL35qoQ+z9RoHqxk/59/xNK5JF/MgzCAqTbfym8Hhl0uaBGzXXlEfmqluHemOpoyJP8EqNQaA6G5NbFYmIaBXIbFCHLopRr3KAypssZY/JEBl0SAt/4ZsV5OffqwvKJKHPwVAYD5w/XWPY938+Sud6H3+o2pcen8C9RUgaGOh5maN+619zXUMd2//PRv+seACrwa6dkZvZz5yWv4bdsT+eY7TktrWlD96Jl6m7+66D+HJhKBvwk8xKQjFk93nWen3xPLfap5M5E2l5P9tH3dOsZeEUwRR2axlDc7G5ZUx3cijpkNewsjwzJIfI03skJF26lgjmp6eaIuxkPR4waAUIkCJ695DZsXgFdCb2cxzULnImVRL6NahoGHQ2hGpp7O5/ZQeit9MUSZyTYeSJEkLsfrbZ8LNd/NELDkO9Zjmpm0ZHE5k7jj2aMEHMyG+u7pIgyHm2/t6VXZmmC26SBu7RazeOCoatRFAsX8=" 10 | # COV_KEY= code coverage upload keys 11 | - secure: "Fw17ZB9+OTT9w4XfmPBQJFcLkZduo/R0cx0ME0wxkNta0t4b8uADEevN12V22tYaWi7tzyMwIxJvBmXlPdOz6omfCJgp4Os4UegdtBjaXglaw040fK0y1FaTA6t47cQ7OtTpKOxIAq9mWRbFKdF5CRa1+O3s/IfPZySMOo9twe7c0T715dbsE0j5beOgvd9AKWA7p2rbfgi0+cewm6IhKHMT5LwPnoU+UlxKpE8ZJSilAyfKOtxxwhqMlsa7VRVf2XRwBPdV4pYgvJ+5cb+zodF3rLI17j5KdglZRrUKIpuyQ17r7R1dvKsIu5RAj0M04gmjJrKMue4SYuHR6A1u4NEsmMqefea2YCXQCQZrzpZXg5rAZO4UjgP/YXMHGEgB89mN14CvUJyaTNISue5gedsF4HKfnCdVyLzy2WgpxLiI6JTaf0PcqhAt+c5rEcuQwo6J/+asLhTAFtgMuAWNSoDRM41OLPzVT6VercCTsiZHNMmTv8OMCKEeHSjczJCPJFccA9tGnvs5h3L+1BJ6WLMh9XArGSa+8Q+1gUyOuqQ+hkxFfp8nmCfvyCL5ngBpFLrkntXl29KG0g12kf4XMc36EFhuGbszB/y81ieqO/jokObRsrpBwL79ySaIvwndU0Ga2JOO8raGpC2784oGEvRKhiYNF539TbKi5ADEVOc=" 12 | 13 | # sudo=false makes the build run using a container 14 | sudo: false 15 | 16 | # I'm putting as many tasks as possible into the Makefile, hence the make depend... 17 | install: 18 | - export PATH=$PATH:$HOME/gopath/bin # travis' worker doesn't seem to do this consistently 19 | - #(cd $GOROOT/src && GOOS=darwin GOARCH=amd64 ./make.bash --no-clean) >/dev/null 20 | - #(cd $GOROOT/src && GOOS=windows GOARCH=amd64 ./make.bash --no-clean) >/dev/null 21 | - #(cd $GOROOT/src && GOOS=linux GOARCH=arm ./make.bash --no-clean) >/dev/null 22 | - make depend 23 | 24 | # Everything else in here, we don't put the uploads into an after_success section because 25 | # that can cause the build to succeed even if the artifacts are not actually uploaded 26 | script: 27 | - export NAME=`basename $TRAVIS_BUILD_DIR` 28 | - echo NAME=$NAME 29 | - make travis-test 30 | - export PATH=./.vendor/bin:$PATH 31 | - which gof3r 32 | # Compute code coverage 33 | - go tool cover -func=total.coverprofile > coverage.txt 34 | - export COVERAGE=$(grep "^total:" coverage.txt | grep -o "[0-9\.]*") 35 | - export BUILD=${TRAVIS_BUILD_NUMBER} 36 | - if [[ "${TRAVIS_PULL_REQUEST}" != "false" ]]; then let BUILD=BUILD-1; fi 37 | - export FILENAME="$(date +%Y-%m-%d)_${BUILD}_Coverage-${COVERAGE}.txt" 38 | - mv coverage.txt $FILENAME 39 | # Post code coverage comment to github 40 | - export CODCOV_URL="https://s3.amazonaws.com/rs-code-coverage/${NAME}/${FILENAME}" 41 | - export JSON_COMMENT="{\"body\":\"Code Coverage is ${COVERAGE}%, details at $CODCOV_URL 42 | (sometimes the URL can take a few minutes to be available)\"}" 43 | - | 44 | if [[ "${TRAVIS_PULL_REQUEST}" != "false" ]]; then 45 | curl -XPOST -s -H "Authorization: token ${GITHUB_TOKEN}" \ 46 | -H "Content-Type: application/json" -d "${JSON_COMMENT}" \ 47 | "https://api.github.com/repos/rightscale/${NAME}/issues/${TRAVIS_PULL_REQUEST}/comments" 48 | fi 49 | # Deploy code coverage result to S3 50 | - export AWS_ACCESS_KEY_ID=AKIAI4RIGBPD3NP2RQ3Q # code coverage bucket access 51 | - export AWS_SECRET_ACCESS_KEY=${COV_KEY} 52 | - gof3r put -b rs-code-coverage -k ${NAME}/${FILENAME} -m x-amz-acl:public-read < ${FILENAME} 53 | # Deploy code coverage badge to S3 54 | - export COVERAGE_INT=$(echo $COVERAGE | cut -d. -f1) 55 | - export BADGE_FILENAME=cc_badge_${TRAVIS_BRANCH}.svg 56 | - export BADGE_COLOR=$( if [ ${COVERAGE_INT} -gt 80 ] ; then echo brightgreen ; elif [ ${COVERAGE_INT} -gt 40 ] ; then echo yellow ; else echo red ; fi ) 57 | - wget "http://img.shields.io/badge/coverage-${COVERAGE}%25-${BADGE_COLOR}.svg" -O ${BADGE_FILENAME} 58 | - gof3r put -b rs-code-coverage -k ${NAME}/${BADGE_FILENAME} -m x-amz-acl:public-read -m cache-control:no-cache -m content-type:image/svg+xml < ${BADGE_FILENAME} 59 | -------------------------------------------------------------------------------- /interface.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 RightScale, Inc. - see LICENSE 2 | 3 | package persist 4 | 5 | import ( 6 | "encoding/gob" 7 | "io" 8 | ) 9 | 10 | // LogClient is the interface the application needs to implement so the persist can call it back 11 | type LogClient interface { 12 | // Replay is called by the persistence layer during log replay in order to replay 13 | // an individual log event. If Replay returns an error the replay is aborted and 14 | // produces the error. 15 | // Beware that it it possible that log events are replayed that contain mutations to 16 | // resources that have not been created yet, i.e. for which the event produced by 17 | // Enumerate has not been replayed yet. In those cases, Replay must ignore the event 18 | // because a subsequent create will be Replayed with the correct value for the 19 | // resource. Note that if events contain updates to multiple resources then some 20 | // of them may have been created and need updating while others may not have been created 21 | // and shouldn't be created/updated. 22 | Replay(logEvent interface{}) error 23 | 24 | // PersistAll is called by the persistence layer in order to enumerate all live resources 25 | // and persist them by making calls to Log.Write(). 26 | // (If PersistAll encounters an error it's time to panic.) 27 | // PersistAll can run in parallel with new updates to resources however the application 28 | // must ensure that calls to Log.Write() are in the same order as PersistAll's reads 29 | // and other update's writes. 30 | PersistAll(pl Log) 31 | } 32 | 33 | type Log interface { 34 | // Output an event to the log, this uses gob serialization internally. If an error 35 | // occurs there is a serious problem with the log, for example, disk full or socket 36 | // disconnected from the log destination. If the application can unroll the mutations 37 | // it has performed it should do so and return its client and error. If, however, the 38 | // application cannot unroll it is recommended not to check for error here and continue 39 | // to operate optimistically. Once the log problem gets repaired the persist layer will 40 | // do a log rotation to ensure all live data is captured. 41 | Output(logEvent interface{}) error 42 | 43 | // SetSizeLimit determines when the persist layer should rotate logs. The default is 44 | // 10MB 45 | SetSizeLimit(bytes int) 46 | 47 | // AddDestination adds additional destinations to the Log (not yet implemented) 48 | SetSecondaryDestination(dest LogDestination) error 49 | 50 | // HealthCheck returns any persistent error encountered in persist that prevents it 51 | // from logging. If HealthCheck() returns an error then all Write() calls will return 52 | // the same error. If the problem is fixed the error will eventually go away again and 53 | // the log will be "repaired" by doing a rotation. The intent of the HealthCheck call 54 | // is for the application to be able to reject requests early if the logging is broken. 55 | HealthCheck() error 56 | 57 | // Stats returns a list of implementation dependent statistics as name->value 58 | Stats() map[string]float64 59 | } 60 | 61 | // Register a type being written to the log, this must be called for each type passed 62 | // to Write and for any type expected in an interface type inside an event. This calls 63 | // gob.Register() internally, please see the gob docs 64 | func Register(value interface{}) { gob.Register(value) } 65 | 66 | // A log destination represents something the persist layer can write log entries to, and then 67 | // replay them in the future. A "New" function is expected to exist for each type of log 68 | // destination in order to open/create it. At open time, the writer must work, and if there 69 | // is an old log to replay the reader must work too. 70 | type LogDestination interface { 71 | // StartRotate() tells the dest to open a fresh log dest and start writing all output to 72 | // this new destination 73 | StartRotate() error 74 | // EndRotate() tells the dest that the fresh log dest has a complete snapshot and 75 | // thus is now "stand-alone" and older logs are no longer needed; this is called after 76 | // StartRotate() *and* after the initial registration of the log destination 77 | EndRotate() error 78 | // Writer writes to current (new) log 79 | io.Writer 80 | // ReplayReaders returns an arra of readers for each log that needs to be replayed 81 | // in sequence 82 | ReplayReaders() []io.ReadCloser 83 | // Close ends the entire log writing and offers a way to cleanly flush and close 84 | Close() 85 | } 86 | -------------------------------------------------------------------------------- /persist_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 RightScale, Inc. - see LICENSE 2 | 3 | package persist 4 | 5 | // Omega: Alt+937 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | . "github.com/onsi/ginkgo" 12 | . "github.com/onsi/gomega" 13 | "gopkg.in/inconshreveable/log15.v2" 14 | ) 15 | 16 | // log client used for testing 17 | type testLogClient struct { 18 | i int // log generation, incremented to distinguish rotations 19 | n int // count number of objects replayed 20 | intr bool // interrupt persistAll 21 | } 22 | 23 | // verifies that what's being replayed is what we expect 24 | func (tlc *testLogClient) Replay(ev interface{}) error { 25 | log15.Info("Replay called", "ev", ev, "i", tlc.i, "n", tlc.n) 26 | switch tlc.n { 27 | case 0: 28 | Ω(ev).Should(Equal(&logEv1{S: fmt.Sprintf("hello world #%d!", tlc.i)})) 29 | case 1: 30 | Ω(ev).Should(Equal(&logEv2{A: 55 + tlc.i, B: "Hello Again"})) 31 | case 2: 32 | Ω(ev).Should(Equal(&logEv1{S: "not again!"})) 33 | case 10: 34 | Ω(ev).Should(Equal(&logEv2{A: tlc.n - 3, B: "A log event"})) 35 | //default: 36 | // Ω(ev).Should(Equal(&logEv2{A: tlc.n - 3, B: "A log event"})) 37 | } 38 | tlc.n++ 39 | return nil 40 | } 41 | 42 | // persist some new data to the new log... 43 | func (tlc *testLogClient) PersistAll(pl Log) { 44 | log15.Info("Populating log", "i", tlc.i) 45 | Ω(pl.Output(&logEv1{S: fmt.Sprintf("hello world #%d!", tlc.i+1)})).ShouldNot(HaveOccurred()) 46 | if tlc.intr { 47 | pl.(*pLog).rotating = false 48 | pl.(*pLog).Close() 49 | return 50 | } 51 | Ω(pl.Output(&logEv2{A: 55 + tlc.i + 1, B: "Hello Again"})).ShouldNot(HaveOccurred()) 52 | Ω(pl.Output(&logEv1{S: "not again!"})).ShouldNot(HaveOccurred()) 53 | } 54 | 55 | // custom even types written to the log 56 | type logEv1 struct { 57 | S string 58 | } 59 | type logEv2 struct { 60 | A int 61 | B string 62 | } 63 | 64 | func init() { 65 | Register(&logEv1{}) 66 | Register(&logEv2{}) 67 | } 68 | 69 | var _ = Describe("NewLog", func() { 70 | 71 | BeforeEach(func() { 72 | os.RemoveAll(PT) 73 | os.Mkdir(PT, 0777) 74 | }) 75 | AfterEach(func() { 76 | //os.RemoveAll(PT) 77 | }) 78 | 79 | startNewLog := func(i int, intr bool) (Log, LogClient) { 80 | fd, err := NewFileDest(PT+"/newfile", true, nil) 81 | Ω(err).ShouldNot(HaveOccurred()) 82 | Ω(fd).ShouldNot(BeNil()) 83 | 84 | logClient := testLogClient{i: i, intr: intr} 85 | pl, err := NewLog(fd, &logClient, log15.Root()) 86 | // if we interrupt the replay we're breaking the log 87 | // (intentionally) and so we do expect to get an error 88 | if !intr { 89 | Ω(err).ShouldNot(HaveOccurred()) 90 | Ω(pl).ShouldNot(BeNil()) 91 | } else { 92 | Ω(err).Should(HaveOccurred()) 93 | Ω(pl).Should(BeNil()) 94 | } 95 | 96 | return pl, &logClient 97 | } 98 | 99 | rereadLog := func(oldI, n int) Log { 100 | pl, lc := startNewLog(oldI, false) 101 | Ω(lc.(*testLogClient).n).Should(Equal(n)) 102 | return pl 103 | } 104 | 105 | rereadLogInterrupted := func(oldI int) { 106 | pl, lc := startNewLog(oldI, true) 107 | Ω(lc.(*testLogClient).n).Should(Equal(3)) 108 | Ω(pl).Should(BeNil()) 109 | } 110 | 111 | It("verifies a new log file", func() { 112 | By("starting a new log") 113 | pl, _ := startNewLog(0, false) 114 | pl.(*pLog).Close() 115 | 116 | By("re-reading the log") 117 | pl = rereadLog(1, 3) 118 | pl.(*pLog).Close() 119 | }) 120 | 121 | It("verifies a new incomplete log file", func() { 122 | By("starting a new log") 123 | pl, _ := startNewLog(0, false) 124 | pl.(*pLog).Close() 125 | 126 | By("re-reading the log") 127 | rereadLogInterrupted(1) 128 | 129 | By("re-reading the log again") 130 | pl = rereadLog(1, 4) 131 | pl.(*pLog).Close() 132 | 133 | By("re-reading the log yet again") 134 | pl = rereadLog(2, 3) 135 | pl.(*pLog).Close() 136 | }) 137 | 138 | It("verifies log rotation", func() { 139 | By("starting a new log") 140 | pl, _ := startNewLog(0, false) 141 | // set a small size limit so we get it to rotate 142 | pl.SetSizeLimit(100) 143 | for i := 0; i <= 100; i++ { 144 | data := logEv2{B: "A log event", A: i} 145 | pl.Output(&data) 146 | } 147 | pl.(*pLog).Close() 148 | 149 | By("re-reading the log") 150 | pl = rereadLog(1, 3) 151 | pl.(*pLog).Close() 152 | 153 | By("re-reading the log again") 154 | pl = rereadLog(2, 3) 155 | pl.(*pLog).Close() 156 | 157 | }) 158 | 159 | It("verifies interrupted log rotation", func() { 160 | By("starting a new log") 161 | pl, lc := startNewLog(0, false) 162 | // set a small size limit so we get it to rotate 163 | pl.SetSizeLimit(100) 164 | lc.(*testLogClient).intr = true 165 | for i := 0; i <= 100; i++ { 166 | data := logEv2{B: "A log event", A: i} 167 | pl.Output(&data) 168 | } 169 | pl.(*pLog).Close() 170 | 171 | By("re-reading the log") 172 | pl, lc = startNewLog(1, false) 173 | Ω(lc.(*testLogClient).n).Should(BeNumerically(">", 10)) 174 | pl.(*pLog).Close() 175 | 176 | By("re-reading the log again") 177 | pl = rereadLog(2, 3) 178 | pl.(*pLog).Close() 179 | 180 | }) 181 | 182 | }) 183 | -------------------------------------------------------------------------------- /file_dest_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 RightScale, Inc. - see LICENSE 2 | 3 | package persist 4 | 5 | // Omega: Alt+937 6 | 7 | import ( 8 | "io" 9 | "os" 10 | 11 | . "github.com/onsi/ginkgo" 12 | . "github.com/onsi/gomega" 13 | "gopkg.in/inconshreveable/log15.v2" 14 | ) 15 | 16 | const PT = "/tmp/persist_test" 17 | 18 | var _ = Describe("NewFileDest", func() { 19 | 20 | BeforeEach(func() { 21 | os.RemoveAll(PT) 22 | os.Mkdir(PT, 0777) 23 | }) 24 | AfterEach(func() { os.RemoveAll(PT) }) 25 | 26 | It("rejects invalid chars", func() { 27 | f, err := NewFileDest(PT+"/test.plog", true, nil) 28 | Ω(f).Should(BeNil()) 29 | Ω(err).Should(HaveOccurred()) 30 | Ω(err.Error()).Should(ContainSubstring("cannot contain")) 31 | }) 32 | 33 | It("handles an invalid directory", func() { 34 | f, err := NewFileDest(PT+"/xxx/test", true, nil) 35 | Ω(f).Should(BeNil()) 36 | Ω(err).Should(HaveOccurred()) 37 | Ω(err.Error()).Should(ContainSubstring("Cannot create")) 38 | }) 39 | 40 | It("does not create log without create=true", func() { 41 | f, err := NewFileDest(PT+"/test", false, nil) 42 | Ω(f).Should(BeNil()) 43 | Ω(err).Should(HaveOccurred()) 44 | Ω(err.Error()).Should(ContainSubstring("No existing")) 45 | }) 46 | }) 47 | 48 | var _ = Describe("FileDest", func() { 49 | 50 | BeforeEach(func() { 51 | os.RemoveAll(PT) 52 | os.Mkdir(PT, 0777) 53 | }) 54 | AfterEach(func() { 55 | //os.RemoveAll(PT) 56 | }) 57 | 58 | startNewLog := func() LogDestination { 59 | By("starting a new log") 60 | 61 | fd, err := NewFileDest(PT+"/newfile", true, nil) 62 | Ω(err).ShouldNot(HaveOccurred()) 63 | Ω(fd).ShouldNot(BeNil()) 64 | 65 | Ω(fd.ReplayReaders()).Should(BeNil()) 66 | 67 | Ω(fd.EndRotate()).ShouldNot(HaveOccurred()) 68 | 69 | n, err := fd.Write([]byte("Hello World")) 70 | Ω(err).ShouldNot(HaveOccurred()) 71 | Ω(n).Should(Equal(11)) 72 | 73 | n, err = fd.Write([]byte("Hello Again")) 74 | Ω(err).ShouldNot(HaveOccurred()) 75 | Ω(n).Should(Equal(11)) 76 | 77 | return fd 78 | } 79 | 80 | rereadLog := func() LogDestination { 81 | By("re-reading the log file") 82 | 83 | fd, err := NewFileDest(PT+"/newfile", false, nil) 84 | Ω(err).ShouldNot(HaveOccurred()) 85 | Ω(fd).ShouldNot(BeNil()) 86 | 87 | buf := make([]byte, 100) 88 | rr := fd.ReplayReaders() 89 | Ω(rr).Should(HaveLen(1)) 90 | 91 | n, err := rr[0].Read(buf) 92 | Ω(err).ShouldNot(HaveOccurred()) 93 | Ω(n).Should(Equal(22)) 94 | Ω(buf[:n]).Should(Equal([]byte("Hello WorldHello Again"))) 95 | 96 | n, err = rr[0].Read(buf) 97 | if err == nil { 98 | log15.Warn("Read not EOF", "n", n) 99 | } 100 | Ω(err).Should(Equal(io.EOF)) 101 | Ω(n).Should(Equal(0)) 102 | 103 | return fd 104 | } 105 | 106 | rereadLog2x := func() LogDestination { 107 | By("re-reading 2x the log file") 108 | 109 | fd, err := NewFileDest(PT+"/newfile", false, nil) 110 | Ω(err).ShouldNot(HaveOccurred()) 111 | Ω(fd).ShouldNot(BeNil()) 112 | 113 | buf := make([]byte, 100) 114 | rr := fd.ReplayReaders() 115 | Ω(rr).Should(HaveLen(2)) 116 | 117 | // replays the first log file 118 | n, err := rr[0].Read(buf) 119 | Ω(err).ShouldNot(HaveOccurred()) 120 | Ω(n).Should(Equal(22)) 121 | Ω(buf[:n]).Should(Equal([]byte("Hello WorldHello Again"))) 122 | 123 | n, err = rr[0].Read(buf) 124 | Ω(err).Should(Equal(io.EOF)) 125 | Ω(n).Should(Equal(0)) 126 | 127 | // replays the second log file 128 | n, err = rr[1].Read(buf) 129 | Ω(err).ShouldNot(HaveOccurred()) 130 | Ω(n).Should(Equal(22)) 131 | Ω(buf[:n]).Should(Equal([]byte("Hello WorldHello Again"))) 132 | 133 | n, err = rr[1].Read(buf) 134 | Ω(err).Should(Equal(io.EOF)) 135 | Ω(n).Should(Equal(0)) 136 | 137 | return fd 138 | } 139 | 140 | rotateLog := func(fd LogDestination) { 141 | By("rotating the log file") 142 | 143 | err := fd.StartRotate() 144 | Ω(err).ShouldNot(HaveOccurred()) 145 | 146 | n, err := fd.Write([]byte("Hello World")) 147 | Ω(err).ShouldNot(HaveOccurred()) 148 | Ω(n).Should(Equal(11)) 149 | 150 | n, err = fd.Write([]byte("Hello Again")) 151 | Ω(err).ShouldNot(HaveOccurred()) 152 | Ω(n).Should(Equal(11)) 153 | 154 | } 155 | 156 | It("verifies a new log file", func() { 157 | fd := startNewLog() 158 | fd.Close() 159 | 160 | fd = rereadLog() 161 | Ω(fd.EndRotate()).ShouldNot(HaveOccurred()) 162 | fd.Close() 163 | }) 164 | 165 | It("verifies a new log file twice", func() { 166 | fd := startNewLog() 167 | fd.Close() 168 | 169 | fd = rereadLog() 170 | 171 | n, err := fd.Write([]byte("Hello World")) 172 | Ω(err).ShouldNot(HaveOccurred()) 173 | Ω(n).Should(Equal(11)) 174 | 175 | Ω(fd.EndRotate()).ShouldNot(HaveOccurred()) 176 | 177 | n, err = fd.Write([]byte("Hello Again")) 178 | Ω(err).ShouldNot(HaveOccurred()) 179 | Ω(n).Should(Equal(11)) 180 | 181 | fd.Close() 182 | 183 | fd = rereadLog() 184 | Ω(fd.EndRotate()).ShouldNot(HaveOccurred()) 185 | fd.Close() 186 | }) 187 | 188 | It("verifies a new incomplete log file", func() { 189 | fd := startNewLog() 190 | fd.Close() 191 | 192 | fd = rereadLog() 193 | 194 | n, err := fd.Write([]byte("Hello World")) 195 | Ω(err).ShouldNot(HaveOccurred()) 196 | Ω(n).Should(Equal(11)) 197 | 198 | n, err = fd.Write([]byte("Hello Again")) 199 | Ω(err).ShouldNot(HaveOccurred()) 200 | Ω(n).Should(Equal(11)) 201 | 202 | fd.Close() 203 | 204 | fd = rereadLog2x() 205 | 206 | Ω(fd.EndRotate()).ShouldNot(HaveOccurred()) 207 | fd.Close() 208 | }) 209 | 210 | It("verifies log rotation", func() { 211 | fd := startNewLog() 212 | rotateLog(fd) 213 | Ω(fd.EndRotate()).ShouldNot(HaveOccurred()) 214 | fd = rereadLog() 215 | fd.Close() 216 | }) 217 | 218 | It("verifies log rotation with failure", func() { 219 | fd := startNewLog() 220 | rotateLog(fd) 221 | By("rotation fails here") 222 | fd = rereadLog2x() 223 | fd.Close() 224 | }) 225 | 226 | }) 227 | -------------------------------------------------------------------------------- /persist.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 RightScale, Inc. - see LICENSE 2 | 3 | // issues: 4 | // - need to rotate() in NewLog? not clear 5 | // - ordering of replay and persist-all in NewLog not clear 6 | 7 | package persist 8 | 9 | import ( 10 | "encoding/gob" 11 | "fmt" 12 | "io" 13 | "sync" 14 | "time" 15 | 16 | "gopkg.in/inconshreveable/log15.v2" 17 | ) 18 | 19 | type pLog struct { 20 | client LogClient // client which we make callbacks 21 | size int // size used to decide when to rotate 22 | sizeLimit int // size limit when to rotate 23 | sizeReplay int // size of the initial replay 24 | objects uint64 // number of objects output, purely for stats 25 | encoder *gob.Encoder 26 | priDest LogDestination // primary dest, where we initially replay from 27 | secDest LogDestination // secondary dest, no replay and OK if "down" 28 | rotating bool // avoid concurrent rotations 29 | errState error 30 | log log15.Logger 31 | sync.Mutex 32 | } 33 | 34 | // Return some statistics about the logging 35 | func (pl *pLog) Stats() map[string]float64 { 36 | pl.Lock() 37 | defer pl.Unlock() 38 | 39 | stats := make(map[string]float64) 40 | stats["LogSizeReplay"] = float64(pl.sizeReplay) 41 | stats["LogSize"] = float64(pl.size + pl.sizeReplay) 42 | stats["LogSizeLimit"] = float64(pl.sizeLimit) 43 | stats["ObjectOutputRate"] = float64(pl.objects) 44 | stats["ErrorState"] = 0.0 45 | if pl.errState != nil { 46 | stats["ErrorState"] = 1.0 47 | } 48 | return stats 49 | } 50 | 51 | // Close the log for test purposes 52 | func (pl *pLog) Close() { 53 | for { 54 | pl.Lock() 55 | if !pl.rotating { 56 | break 57 | } 58 | pl.Unlock() 59 | time.Sleep(1 * time.Millisecond) 60 | } 61 | 62 | pl.priDest.Close() 63 | if pl.secDest != nil { 64 | pl.secDest.Close() 65 | } 66 | pl.Unlock() 67 | } 68 | 69 | // SetSizeLimit sets the log size limit at which a rotation occurs, the size value is in 70 | // addition to the initial size produced by the initial snapshot, i.e., it doesn't count that 71 | func (pl *pLog) SetSizeLimit(bytes int) { pl.sizeLimit = bytes } 72 | 73 | // HealthCheck returns nil if everything is OK and an error if the log is in an error state 74 | func (pl *pLog) HealthCheck() error { return pl.errState } 75 | 76 | // hack... 77 | var pLogError bool 78 | 79 | // Output a log entry 80 | func (pl *pLog) Output(logEvent interface{}) error { 81 | pl.Lock() 82 | defer pl.Unlock() 83 | 84 | //pl.log.Debug("persist.Output", "ev", logEvent) 85 | 86 | if pl.errState != nil { 87 | if !pLogError { 88 | pl.log.Crit("Persistence log in error state: " + 89 | pl.errState.Error()) 90 | pLogError = true 91 | } 92 | return pl.errState 93 | } 94 | pLogError = false 95 | if pl.encoder == nil { 96 | return fmt.Errorf("uninitialized persistence log (nil encoder)") 97 | } 98 | // perverse stuff: we need to slap the event into an interface{} so gob later allows 99 | // us to decode into an interface{} 100 | pl.objects += 1 101 | var t interface{} = logEvent 102 | err := pl.encoder.Encode(&t) 103 | if err != nil { 104 | pl.errState = err 105 | } else if !pl.rotating && pl.size > pl.sizeLimit { 106 | pl.rotate() 107 | } 108 | return err 109 | } 110 | 111 | func (pl *pLog) SetSecondaryDestination(dest LogDestination) error { 112 | pl.Lock() 113 | defer pl.Unlock() 114 | 115 | return fmt.Errorf("not implemented yet!") 116 | 117 | /* 118 | if pl.secDest != nil { 119 | return fmt.Errorf("secondary destination is already set") 120 | } 121 | pl.secDest = dest 122 | 123 | return pl.rotate() 124 | */ 125 | } 126 | 127 | // perform a log rotation, must be called while holding the pl.Lock() 128 | func (pl *pLog) rotate() { 129 | if pl.rotating { 130 | return 131 | } 132 | pl.rotating = true 133 | pl.log.Info("Persist: starting rotation") 134 | go pl.finishRotate() 135 | } 136 | 137 | func (pl *pLog) finishRotate() { 138 | // tell all log destinations to start a rotation 139 | pl.Lock() 140 | defer pl.Unlock() 141 | pl.size = 0 142 | pl.sizeReplay = 0 143 | err := pl.priDest.StartRotate() 144 | if pl.secDest != nil { 145 | pl.secDest.StartRotate() // TODO: record error 146 | } 147 | if err != nil { 148 | pl.errState = err 149 | return 150 | } 151 | // we need a new encoder 'cause we start a fresh stream 152 | pl.encoder = gob.NewEncoder(pl) 153 | 154 | // now create a full snapshot, relinquish the lock while doing that 'cause otherwise 155 | // we end up with deadlocks since PersistAll will end up calling pl.Output() 156 | pl.Unlock() 157 | pl.client.PersistAll(pl) 158 | pl.Lock() 159 | 160 | // tell all log destinations that we're done with the rotation 161 | err = pl.priDest.EndRotate() 162 | if pl.secDest != nil { 163 | pl.secDest.EndRotate() // TODO: record error 164 | } 165 | pl.rotating = false 166 | if err != nil { 167 | pl.log.Crit("Finished rotation with error", 168 | "replay_size", pl.sizeReplay, "err", err) 169 | pl.errState = err 170 | } else { 171 | pl.log.Info("Finished rotation", "replay_size", pl.sizeReplay) 172 | } 173 | return 174 | } 175 | 176 | // replay a log file 177 | func (pl *pLog) replay() (err error) { 178 | for i, rr := range pl.priDest.ReplayReaders() { 179 | pl.log.Info("Starting replay", "log_num", i+1) 180 | dec := gob.NewDecoder(rr) 181 | // iterate reading one log entry after another until EOF is reached 182 | count := 0 183 | for { 184 | var ev interface{} 185 | err := dec.Decode(&ev) 186 | if err == io.EOF { 187 | break // done replaying 188 | } 189 | if err != nil { 190 | pl.log.Debug("replay decode failed", "err", err, "log_num", i+1, 191 | "count", count) 192 | return fmt.Errorf("replay decode failed in log %d after %d entries: %s", 193 | i+1, count, err.Error()) 194 | } 195 | //pl.log.Debug("replay decoded", "ev", ev) 196 | count += 1 197 | err = pl.client.Replay(ev) 198 | if err != nil { 199 | return fmt.Errorf("replay failed on entry %d: %s", count, err.Error()) 200 | } 201 | } 202 | rr.Close() 203 | } 204 | pl.log.Debug("Ending replay", "logs", len(pl.priDest.ReplayReaders())) 205 | return nil 206 | } 207 | 208 | // Write is called by the gob encoder and needs to write the bytes to all destinations 209 | func (pl *pLog) Write(p []byte) (int, error) { 210 | if pl.errState != nil { 211 | return 0, pl.errState // in error state don't move! 212 | } 213 | 214 | // track size for log rotation, initial snapshot doesn't count towards limit 215 | l := len(p) 216 | if !pl.rotating { 217 | pl.size += len(p) 218 | } else { 219 | pl.sizeReplay += len(p) 220 | } 221 | 222 | // write to primary destination 223 | n, err := pl.priDest.Write(p) 224 | if n != l || err != nil { 225 | pl.errState = err 226 | return n, err 227 | } 228 | 229 | // write to secondary destination 230 | if pl.secDest != nil { 231 | pl.secDest.Write(p) // TODO: record error 232 | } 233 | 234 | return n, nil 235 | } 236 | 237 | // NewLog reopens an existing log, replays all log entries, and then prepares to append 238 | // to it. The call to NewLog completes once any necessary replay has completed. 239 | func NewLog(priDest LogDestination, client LogClient, logger log15.Logger) (Log, error) { 240 | pl := &pLog{ 241 | client: client, 242 | sizeLimit: 1024 * 1024, // 1MB default 243 | priDest: priDest, 244 | log: logger.New("start", time.Now()), 245 | } 246 | pl.encoder = gob.NewEncoder(pl) 247 | 248 | pl.log.Debug("Starting replay") 249 | err := pl.replay() 250 | if err != nil { 251 | pl.errState = err 252 | return nil, err 253 | } 254 | pl.log.Info("Replay done") 255 | 256 | // now create a full snapshot 257 | pl.log.Debug("Starting snapshot") 258 | pl.rotating = true 259 | pl.client.PersistAll(pl) 260 | pl.rotating = false 261 | pl.log.Info("Snapshot done") 262 | 263 | // tell all log destinations that we're done with the rotation 264 | err = pl.priDest.EndRotate() 265 | if pl.secDest != nil { 266 | pl.secDest.EndRotate() // TODO: record error 267 | } 268 | if err != nil { 269 | pl.errState = err 270 | return nil, err 271 | } 272 | return pl, err 273 | } 274 | -------------------------------------------------------------------------------- /file_dest.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 RightScale, Inc. - see LICENSE 2 | 3 | package persist 4 | 5 | import ( 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "sort" 11 | "strings" 12 | "time" 13 | 14 | "gopkg.in/inconshreveable/log15.v2" 15 | ) 16 | 17 | type fileDest struct { 18 | basepath string 19 | replayReaders []io.ReadCloser 20 | outputFile *os.File 21 | outputFilename string 22 | oldFilename string // name of previous file (used at end of rotation) 23 | snapOK bool // true when the initial snapshot is completed 24 | log log15.Logger 25 | } 26 | 27 | const ( 28 | newExt = "-new.plog" // new log with incomplete initial snapshot 29 | currExt = "-curr.plog" // current log with complete initial snapshot 30 | oldExt = "-old.plog" // old log no longer needed 31 | dateFmt = "-20060102-150405" // format for timestamp added for log files 32 | ) 33 | 34 | // NewFileDest creates or opens a file for logging. The basepath must not contain any character 35 | // in the set '*', '?', '[', '\', or '.'. The individual log file names will have a - 36 | // and possibly a <-new>, <-curr>, and '.plog' extension appended. 37 | // The create argument determines whether it's OK to create a new set of log files or whether 38 | // an existing set is expected to be found. 39 | func NewFileDest(basepath string, create bool, log log15.Logger) (LogDestination, error) { 40 | if log == nil { 41 | log = log15.Root() 42 | } 43 | log = log.New("basepath", basepath) 44 | 45 | if strings.ContainsAny(basepath, "*?[\\.") { 46 | return nil, fmt.Errorf("basepath cannot contain '*', '?', '[', '\\' or '.'") 47 | } 48 | m, err := filepath.Glob(basepath + "*.plog") 49 | if err != nil { 50 | return nil, fmt.Errorf("basepath invalid: %s", err.Error()) 51 | } 52 | 53 | fd := &fileDest{basepath: basepath, log: log} 54 | 55 | if len(m) > 0 { 56 | sort.Strings(m) 57 | lm := len(m) - 1 58 | if strings.HasSuffix(m[lm], currExt) { 59 | // the most recent log file is current, i.e. it's all we need 60 | f0, err := os.Open(m[lm]) 61 | if err != nil { 62 | return nil, fmt.Errorf("error opening %s: %s", m[lm], err.Error()) 63 | } 64 | fd.replayReaders = []io.ReadCloser{f0} 65 | fd.oldFilename = m[lm] 66 | stat, _ := f0.Stat() 67 | log.Info("Opening existing log, replaying one file", 68 | "file1", m[lm], "len1", stat.Size()) 69 | } else if strings.HasSuffix(m[lm], newExt) && lm > 0 && 70 | strings.HasSuffix(m[lm-1], currExt) { 71 | // the most recent log is not a complete snapshot, we need it and 72 | // the prior log file (and we have both) 73 | // we create a multi-reader that reads from the prior log file and then 74 | // from the new one 75 | f0, err := os.Open(m[lm-1]) 76 | if err != nil { 77 | return nil, fmt.Errorf("error opening %s: %s", m[lm-1], err.Error()) 78 | } 79 | f1, err := os.Open(m[lm]) 80 | if err != nil { 81 | f0.Close() 82 | return nil, fmt.Errorf("error opening %s: %s", m[lm], err.Error()) 83 | } 84 | fd.replayReaders = []io.ReadCloser{f0, f1} 85 | fd.oldFilename = m[lm] 86 | log.Info("Opening existing log, replaying two files", "file1", m[lm-1], 87 | "file2", m[lm]) 88 | } else { 89 | return nil, fmt.Errorf( 90 | "Cannot determine current (&new) logs from basepath %s", basepath) 91 | } 92 | } else if !create { 93 | return nil, fmt.Errorf("No existing log file found at %s", basepath) 94 | } else { 95 | log.Info("No existing log found, creating a new one") 96 | } 97 | 98 | // Open new destination 99 | err = fd.startNew(len(m) > 0) 100 | if err != nil { 101 | if fd.replayReaders != nil { 102 | for _, rr := range fd.replayReaders { 103 | rr.Close() 104 | } 105 | fd.replayReaders = nil 106 | } 107 | return nil, err 108 | } 109 | return fd, nil 110 | } 111 | 112 | // createNewFile attempts to create a new file and keeps adding from 'a' to 'z' to ensure it 113 | // doesn't open an existing file 114 | // TODO: can't create foo-new.plog if foo-curr.plog exists! 115 | func createNewFile(name, ext string) (*os.File, string, error) { 116 | for i := '`'; i <= 'z'; i++ { 117 | n := name 118 | if i != '`' { // '`' is just before 'a', signifies no suffix 119 | n += string(i) 120 | } 121 | // check that we have no file with this suffix 122 | if m, _ := filepath.Glob(n + "*"); len(m) > 0 { 123 | continue 124 | } 125 | 126 | // try to create, making sure it does not exist 127 | n += ext 128 | fd, err := os.OpenFile(n, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) 129 | if err == nil { 130 | // success, return this one 131 | return fd, n, nil 132 | } 133 | if err != os.ErrExist { 134 | return nil, "", err 135 | } 136 | } 137 | return nil, "", fmt.Errorf("Too many log files in the same second") 138 | } 139 | 140 | // start a new log file and use either the newExt (normal case) or the currExt (when creating the 141 | // very first log file since there's no preceding currExt file) 142 | func (fd *fileDest) startNew(useNewExt bool) error { 143 | // work out filename 144 | date := time.Now().UTC().Format(dateFmt) 145 | name := fd.basepath + date 146 | ext := currExt 147 | if useNewExt { 148 | ext = newExt 149 | } 150 | 151 | // create it and deal with errors 152 | outF, outFn, err := createNewFile(name, ext) 153 | if err != nil { 154 | return fmt.Errorf("Cannot create new log file: %s", err.Error()) 155 | } 156 | fd.log.Info("Starting new log file", "file", outF.Name()) 157 | fd.outputFile = outF 158 | fd.outputFilename = outFn 159 | fd.snapOK = false 160 | return nil 161 | } 162 | 163 | func (fd *fileDest) Close() { 164 | if fd.replayReaders != nil { 165 | for _, rr := range fd.replayReaders { 166 | rr.Close() 167 | } 168 | fd.replayReaders = nil 169 | } 170 | if fd.outputFile != nil { 171 | fd.outputFile.Close() 172 | fd.outputFile = nil 173 | fd.outputFilename = "" 174 | } 175 | fd.basepath = "" 176 | } 177 | 178 | func (fd *fileDest) Write(p []byte) (int, error) { 179 | return fd.outputFile.Write(p) 180 | } 181 | 182 | func (fd *fileDest) ReplayReaders() []io.ReadCloser { 183 | return fd.replayReaders 184 | } 185 | 186 | // StartRotate is called by persist in order to start a new log file. 187 | func (fd *fileDest) StartRotate() error { 188 | if !fd.snapOK { 189 | return fmt.Errorf("Cannot rotate: initial snapshot incomplete") 190 | } 191 | fd.outputFile.Close() 192 | fd.outputFile = nil 193 | fd.oldFilename = fd.outputFilename 194 | fd.outputFilename = "" 195 | return fd.startNew(true) 196 | } 197 | 198 | // EndRotate is called by persist in order to signal the completion of the initial snapshot 199 | // on the new log file. It is called after StartRotate() and after NewFileDest(), i.e., there's 200 | // an implicit StartRotate() when the destination is initially created. 201 | func (fd *fileDest) EndRotate() error { 202 | if fd.snapOK { 203 | return fmt.Errorf("internal error: StartRotate not called") 204 | } 205 | 206 | // if we started a new log and there's no replay, then the first file has 207 | // currExt and there's nothing to do. If we opened an existing log, then the 208 | // current file has newExt and we need some renaming to make it currExt 209 | if fd.oldFilename == "" { 210 | if !strings.HasSuffix(fd.outputFilename, currExt) { 211 | return fmt.Errorf( 212 | "internal error: first log file (%s) should have %s suffix", 213 | fd.outputFilename, currExt) 214 | } 215 | fd.snapOK = true 216 | fd.log.Info("New log file now initialized") 217 | return nil 218 | } 219 | if !strings.HasSuffix(fd.outputFilename, newExt) { 220 | return fmt.Errorf("internal error: new log file (%s) does not have %s suffix !?", 221 | fd.outputFilename, newExt) 222 | } 223 | 224 | // Rename new log file 225 | newName := strings.TrimSuffix(fd.outputFilename, newExt) + currExt 226 | err := os.Rename(fd.outputFilename, newName) 227 | if err != nil { 228 | return err 229 | } 230 | fd.outputFilename = newName 231 | fd.log.Info("New log file now initialized & renamed", "file", newName) 232 | 233 | // Rename old file 234 | var oldName string // new name for old file... 235 | if strings.HasSuffix(fd.oldFilename, currExt) { 236 | oldName = strings.TrimSuffix(fd.oldFilename, currExt) + oldExt 237 | } else if strings.HasSuffix(fd.oldFilename, newExt) { 238 | oldName = strings.TrimSuffix(fd.oldFilename, newExt) + oldExt 239 | // TODO: should really also rename the log file prior to that, which must 240 | // have a currExt 241 | } else { 242 | return fmt.Errorf("internal error: old log file (%s) doesn't have %s or %s suffix", 243 | fd.oldFilename, currExt, newExt) 244 | } 245 | fd.log.Info("Old log file now superceded", "file", oldName) 246 | err = os.Rename(fd.oldFilename, oldName) 247 | if err != nil { 248 | return err 249 | } 250 | fd.oldFilename = "" 251 | fd.snapOK = true 252 | 253 | return nil 254 | } 255 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Persistence Log 2 | =============== 3 | [![Build Status](https://travis-ci.org/rightscale/persist.svg?branch=master)](https://travis-ci.org/rightscale/persist) 4 | ![Code Coverage](https://s3.amazonaws.com/rs-code-coverage/persist/cc_badge_master.svg) 5 | [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rightscale/persist/blob/master/LICENSE) 6 | [![Godoc](https://godoc.org/github.com/rightscale/persist?status.svg)](http://godoc.org/github.com/rightscale/persist) 7 | 8 | This Golang package implements a persistence log very similar to a database replay log 9 | or write-ahead-log (WAL): before committing a change to a resource (arbitrary data structure) 10 | the app writes the new state of the resource to a log. If the app crashes, the log can be replayed 11 | in order to recreate all the resources. The trick is to rotate the log periodically 12 | so it doesn't grow indefinitely. This is what this persistence package implements. 13 | 14 | One of the special features of this persistence log is that it does not define the set of 15 | operations that can be persisted to the log nor does it require storage beyond the typical 16 | streaming encoder/decoder buffers. In particular, it does not create a copy of the log or 17 | of the in-memory data structure in order to start a new log or create a snapshot. Instead 18 | it makes a callback into the app to enumerate the set of live objects. 19 | 20 | Goals 21 | ----- 22 | - persist changes that are committed to in-memory data structures/databases to a log 23 | - replay all the chanages in order to re-create the in-memory data structure 24 | - start a fresh stream periodically in order not to grow the log indefinitely 25 | - use the log in order to keep a replica server up-to-date 26 | - not be tied to disk and instead allow the log to go to a remote server/service 27 | 28 | Some things this persistence layer does not do: 29 | - multi-resource updates (transactions) where multiple resources are written atomically 30 | - attempt to guarantee durability in the sense that once the log write completes the data 31 | is guaranteed to be on stable storage, such as disk 32 | 33 | Model of operation 34 | ------------------ 35 | 36 | The way the persistence log functions is as follows. After opening the log, the application 37 | calls Update() for every change and passes it an opaque desrciption of the change. 38 | The Update function serializes the change and appends it to the log. 39 | 40 | When the time to rotate the log comes, the persistence layer stops writing to the 41 | current log and starts a fresh log. The fresh log begins with a mixture of additional 42 | updates and an enumeration of all current resources. 43 | To generate this enumeration, the persistence layer makes a callback into the application 44 | which must traverse all live resources and call Update() with a "create" descriptor. 45 | 46 | When/if the application asks the persistence layer to replay a log, the latter locates 47 | the last log for which the initialization callback has completed, sends all updates in 48 | that log to the application, and then opens any new incomplete log and replays the updates 49 | in that log as well. 50 | 51 | One critical question is how to handle concurrency. The persistence layer is designed to 52 | allow concurrency while keeping the model as simple as possible. The primary requirement 53 | on the application is as follows: 54 | - When an application calls Update() for a resource it must guarantee that no concurrent 55 | call to Update() can happen for the same resource. This guarantees strict ordering on 56 | updates to a resource, i.e., it prevents updates from getting out of order. The mutual 57 | exclusion guarantee made by the application must apply to the initialization callback 58 | during which the application enumerates all resources as well. 59 | 60 | Typically an application acquires a lock on a resource before mutating it. This makes it 61 | easy to satisfy this requirement by calling Update() while holding the lock on the resource. 62 | 63 | An additional requirement arises if an application makes resource mutations and calls 64 | Update() concurrently with enumerating all resources in a log initialization callback. 65 | The persistence layer writes the updates to the new log as they come in from the application 66 | which means that on a replay is is likely that the application will receive an update 67 | to a resource before it has received the initialization descriptor written in the 68 | init callback for that resource. There are several ways to deal with this issue: 69 | - the application can acquire a global lock for the duration of the initialization 70 | callback, thereby preventing concurrent updates 71 | - during replay, the application can ignore updates to resources that have not yet been 72 | created knowing that eventually it will encounter a full create descriptor as written 73 | during the initialization callback 74 | - if the application creates log entries that involve multiple resources, for example, 75 | a log entry to increment resource A and decrement resource B, then during a replay 76 | it must cope with the situation where one of the resources has been created and the 77 | other one hasn't. If this example, It may be that A has been created by a prior replayed 78 | log entry and the app has to increment A, but B may not yet have been created and thus 79 | the decrement B has to be skipped without producing an error (the correct value of B will 80 | be created later during the replay). 81 | 82 | Note that the persistence layer does not prescribe what the application passes it in an 83 | Update() call. One option is to pass a full copy of the resource being updated or a delete 84 | marker. In this case replaying the log consists of re-creating or updating each resource as 85 | it is fed back to the application and deleting existing resources when encountering a 86 | delete marker. 87 | 88 | Concepts 89 | -------- 90 | 91 | ### Resources 92 | 93 | A resource is an object that can be serialized to the log and that is 94 | the unit of atomic update in the application. 95 | 96 | ### Logs 97 | 98 | A log is the object to which changes to resources are persisted, and it has 99 | one or multiple destinations to which data is written. Each log has a primary destination, 100 | which is the destination from which a restore can be initiated. 101 | A log has the following operations: 102 | - create: creates a log object and names a primary destination 103 | - restore: reads the last log at the destination and replays all events, making 104 | callbacks into the application in order to recreate the state 105 | - update: records a change to a resource, i.e., writes the serialized version to the log 106 | - addDestination: adds a secondary destination, this will cause a log rotation 107 | 108 | Sample code 109 | ----------- 110 | 111 | ```go 112 | // A sample resource type 113 | type resourceType struct { 114 | Id int64 // unique resource ID 115 | Field string // sample data field 116 | } 117 | // The set of resources in the application 118 | var resources map[int64]resourceType // all resources indexed by ID 119 | var resourcesMutex mutex.Lock // exclusive access to the resources map 120 | var pLog persistence.Log // persistence log for the resources 121 | const ( 122 | ResourceUpsert = iota // insert or update 123 | ResourceDelete 124 | ) 125 | type ResourceLogEntry struct { 126 | Op int // ResourceUpsert or ResourceDelete 127 | Res resourceType 128 | } 129 | 130 | // Create function used within the application to create a new resource. It assumes 131 | // that a unique Id has already been generated. 132 | func createResource(data resourceType) error { 133 | // lock all resources 134 | resourcesMutex.Lock() 135 | defer resourcesMutex.Unlock() 136 | // check non-existence 137 | if _, ok := resources[data.Id]; ok { 138 | return fmt.Errorf("duplicate resource ID") 139 | } 140 | // write to log 141 | pLog.Update(ResourceLogEntry{Op: ResourceUpsert, Res: data}) 142 | // insert into map 143 | resources[data.Id] = data 144 | } 145 | 146 | // Delete function used within the application to delete an existing resource. 147 | func deleteResource(id int64) { 148 | // lock all resources 149 | resourcesMutex.Lock() 150 | defer resourcesMutex.Unlock() 151 | // check existence 152 | if _, ok := resources[id]; !ok { 153 | return fmt.Errorf("deleting non-existing resource") 154 | } 155 | // write to log 156 | pLog.Update(ResourceLogEntry{Op: ResourceDelete, Res: ResourceTpe{Id: id}}) 157 | // delete from map 158 | delete(resources, id) 159 | } 160 | 161 | // Update function used within the application to update an existing resource 162 | func updateResource(data resourceType) error { 163 | // lock all resources 164 | resourcesMutex.Lock() 165 | defer resourcesMutex.Unlock() 166 | // check existence 167 | if _, ok := resources[id]; !ok { 168 | return fmt.Errorf("updating non-existing resource") 169 | } 170 | // write to log 171 | pLog.Update(ResourceLogEntry{Op: ResourceUpsert, Res: data}) 172 | // update map 173 | resources[data.Id] = data 174 | } 175 | 176 | // Callback from persistence log to enumerate all resources in order to start a fresh log 177 | func enumerateResources() { 178 | // lock all resources 179 | resourcesMutex.Lock() 180 | defer resourcesMutex.Unlock() 181 | // iterate through the entire map 182 | for _, v := range resources { 183 | pLog.Update(v) 184 | } 185 | } 186 | 187 | // Callback from persistence log to replay a resource operation 188 | func replayResource(logEntry interface{}) error { 189 | // lock all resources 190 | resourcesMutex.Lock() 191 | defer resourcesMutex.Unlock() 192 | // type cast log operation 193 | op, ok := logEntry.(ResourceLogEntry) 194 | if !ok { 195 | return fmt.Errorf("invalid replay record type") 196 | } 197 | // perform operation 198 | switch op.Op { 199 | case ResourceUpsert: 200 | resources[op.Res.Id] = op.Res 201 | case ResourceDelete: 202 | delete(resources, op.Res.Id) 203 | } 204 | } 205 | ``` 206 | --------------------------------------------------------------------------------