├── .gitignore ├── logo └── logo.png ├── main.go ├── util ├── types.go └── util.go ├── go.mod ├── CONTRIBUTING.md ├── cmd ├── root.go ├── head.go ├── put.go ├── restore.go ├── list.go ├── init.go └── get.go ├── progress ├── types.go └── progress.go ├── Makefile ├── UNLICENSE ├── profile ├── types.go ├── profile_outer.go ├── profile.go └── profile_inner.go ├── object ├── types.go └── object.go ├── crypt └── crypt.go ├── go.sum ├── input └── input.go ├── doc └── ogive.1.templ └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.gitignore 3 | /ogive -------------------------------------------------------------------------------- /logo/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mgren/ogive/HEAD/logo/logo.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/awnumar/memguard" 6 | "github.com/mgren/ogive/cmd" 7 | ) 8 | 9 | func main() { 10 | memguard.CatchInterrupt(func() { 11 | fmt.Println("Exiting...") 12 | }) 13 | 14 | defer memguard.DestroyAll() 15 | cmd.Execute() 16 | } -------------------------------------------------------------------------------- /util/types.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // WriterAtFake is used to implement fake compliance with the io.WriteAt interface while also validating the first two bytes for sio 8 | type WriterAtFake struct { 9 | // w is the underlying io.Writer that WriterAtFake proxies writes to. 10 | w io.Writer 11 | 12 | // f is a flag used to indicate whether the writer had received no writes (true) or had its first byte wtitten (false). 13 | f *bool 14 | } 15 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mgren/ogive 2 | 3 | require ( 4 | github.com/InVisionApp/tabular v0.3.0 5 | github.com/awnumar/memguard v0.15.1 6 | github.com/aws/aws-sdk-go v1.19.28 7 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 8 | github.com/minio/sio v0.0.0-20190118043801-035b4ef8c449 9 | github.com/schollz/progressbar/v2 v2.12.1 10 | github.com/spf13/cobra v0.0.3 11 | github.com/spf13/pflag v1.0.3 // indirect 12 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 13 | golang.org/x/sys v0.0.0-20190412213103-97732733099d 14 | ) 15 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributions are always welcome. 2 | 3 | #### Code Setup 4 | 1. Fork and clone the repository 5 | 2. `make prepare` 6 | 3. Fix bugs, implement features 7 | 4. Update the `README.md` and the `doc/ogive.1` files if your changes impact the user interface 8 | 5. Format your code using `make format` 9 | 6. Submit a pull request 10 | 11 | #### Semantic Versioning 12 | https://semver.org/ 13 | 14 | Given a version number MAJOR.MINOR.PATCH, increment the: 15 | 16 | * MAJOR version when you make incompatible API changes, 17 | * MINOR version when you add functionality in a backwards-compatible manner, and 18 | * PATCH version when you make backwards-compatible bug fixes. 19 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/mgren/ogive/util" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | func init() { 9 | rootCmd.PersistentFlags().StringVarP(&profileFile, "profile", "p", util.GetDefaultProfileLoc(), "Location of ogive profile file.") 10 | cobra.MarkFlagFilename(rootCmd.PersistentFlags(), "profile") 11 | } 12 | 13 | var profileFile string 14 | 15 | var rootCmd = &cobra.Command{ 16 | Use: "ogive", 17 | Short: "secure backups with AWS S3 Glacier Deep Archive", 18 | Long: "ogive is a simple commandline tool for storing and retrieving cryptographically secure backups from AWS S3 Glacier Deep Archive.", 19 | } 20 | 21 | // Execute is the hook for main to start Cobra 22 | func Execute() { 23 | if err := rootCmd.Execute(); err != nil { 24 | util.Fail(err, "Critical error.") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /progress/types.go: -------------------------------------------------------------------------------- 1 | package progress 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // Reader extends io.Reader interface with a byte counter 8 | type Reader struct { 9 | // Reader is the underlying io.Reader to which all read calls are proxied. 10 | io.Reader 11 | 12 | // totalProgress indicates the total number of bytes read from reader. 13 | totalProgress int 14 | } 15 | 16 | // Writer extends io.Writer interface with a byte counter 17 | type Writer struct { 18 | // Writer is the underlying io.Writer to which all read calls are proxied. 19 | io.Writer 20 | 21 | // totalProgress indicates the total number of bytes written to writer. 22 | totalProgress int 23 | } 24 | 25 | // ProgressReporter is an interface implemented by progress-tracking readers and writers 26 | type ProgressReporter interface { 27 | // GetProgress returns the totalProgress of the r/w interface 28 | GetProgress() int 29 | } 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifndef VERSION 2 | VERSION = `git describe --always --long --dirty` 3 | endif 4 | 5 | # https://wiki.debian.org/ReproducibleBuilds/TimestampsProposal 6 | ifndef SOURCE_DATE_EPOCH 7 | SOURCE_DATE_EPOCH = `git log -1 --format=%ct` 8 | endif 9 | 10 | 11 | default: ogive 12 | 13 | dev: clean ogive-dev 14 | 15 | 16 | install: install-binary install-doc 17 | 18 | 19 | all: prepare ogive install-binary install-doc 20 | 21 | 22 | prepare: 23 | @go mod download 24 | 25 | ogive: 26 | @export GO111MODULE=on && go build -ldflags="\ 27 | -X main.version=${VERSION} \ 28 | -X main.sourceDateTs=${SOURCE_DATE_EPOCH} \ 29 | " ${GOFLAGS} 30 | 31 | ogive-dev: 32 | @export GO111MODULE=on && go build -a -race ${GOFLAGS} 33 | 34 | 35 | install-binary: ogive 36 | @mkdir -p /usr/local/bin/ && cp -a ogive /usr/local/bin/ 37 | 38 | 39 | install-doc: 40 | @sed "s/__VERSION__/${VERSION}/g" doc/ogive.1.templ > doc/ogive.1 && \ 41 | install -g 0 -o 0 -m 0644 doc/ogive.1 /usr/local/man/man1/ && \ 42 | gzip -f /usr/local/man/man1/ogive.1 && rm -f doc/ogive.1 43 | 44 | format: 45 | @go fmt ./... 46 | @go vet ./... 47 | 48 | clean: 49 | @go clean && rm -f doc/ogive.1 -------------------------------------------------------------------------------- /UNLICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to -------------------------------------------------------------------------------- /profile/types.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "github.com/awnumar/memguard" 5 | ) 6 | 7 | // InnerData is the actual profile data, stored in an encrypted format 8 | type InnerData struct { 9 | // Key is the master key used to encrypt files at rest 10 | Key *memguard.LockedBuffer 11 | 12 | // AWSKeyId ia the AWS Key ID used to upload/download files 13 | AWSKeyId *memguard.LockedBuffer 14 | 15 | // AWSSecret is the AWS Secret associated with the AWS Key ID 16 | AWSSecret *memguard.LockedBuffer 17 | 18 | // BucketName is the S3 bucket name 19 | BucketName string 20 | 21 | // Endpoint is the S3 endpoint to connect to 22 | Endpoint string 23 | 24 | // Region is the AWS region in which the S3 bucket is located 25 | Region string 26 | } 27 | 28 | // OuterData is a wrapper for InnerData that holds information needed to perform 29 | // encryption and decryption of the stored information 30 | type OuterData struct { 31 | // Magic is a string indicator of this indeed being an ogive struct 32 | Magic string 33 | 34 | // Version ia the profile file version, in case the spec changes 35 | Version uint32 36 | 37 | // Salt is the salt value used for password derivation to unseal the InnerData 38 | Salt []byte 39 | 40 | // Inner is the encrypted, serialized representation of InnerData 41 | Inner []byte 42 | } 43 | -------------------------------------------------------------------------------- /profile/profile_outer.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "github.com/awnumar/memguard" 5 | "github.com/mgren/ogive/crypt" 6 | ) 7 | 8 | func (od *OuterData) pack(in *InnerData, key *memguard.LockedBuffer) error { 9 | gcm, err := crypt.GetGCM(key, 0) 10 | if err != nil { 11 | return err 12 | } 13 | 14 | // No need to protect this, but it saves us importing rand 15 | nonce, err := memguard.NewImmutableRandom(gcm.NonceSize()) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | marshal, err := in.marshalBinaryLocked() 21 | if err != nil { 22 | return err 23 | } 24 | 25 | od.Inner = gcm.Seal(nonce.Buffer(), nonce.Buffer(), marshal.Buffer(), nil) 26 | marshal.Destroy() 27 | return nil 28 | } 29 | 30 | func (od *OuterData) unpack(key *memguard.LockedBuffer) (*InnerData, error) { 31 | gcm, err := crypt.GetGCM(key, 0) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | nonceSize := gcm.NonceSize() 37 | nonce, data := od.Inner[:nonceSize], od.Inner[nonceSize:] 38 | 39 | plain, err := gcm.Open(nil, nonce, data, nil) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | inner := InnerData{} 45 | locked, err := memguard.NewImmutableFromBytes(plain) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | err = inner.unmarshalBinaryLocked(locked) 51 | return &inner, err 52 | } 53 | -------------------------------------------------------------------------------- /object/types.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "github.com/awnumar/memguard" 5 | "time" 6 | ) 7 | 8 | // ResponseObject is an ogive-friendly representation of a HEAD result on a stored file 9 | type ResponseObject struct { 10 | // Restore indicates object restore status 11 | Restore string 12 | 13 | // Size is the objects size as indicated by Content-Length 14 | Size int 15 | 16 | // LastModified is the file creation date as indicated by Last-Modified 17 | LastModified time.Time 18 | 19 | // Nonce is the unique nonce used for key derivation 20 | Nonce []byte 21 | 22 | // Name is the original unencrypted filename 23 | Name string 24 | 25 | // Key is the unique derived key 26 | Key *memguard.LockedBuffer 27 | } 28 | 29 | // RequestObject is an ogive-friendly representation of object metadata needed to prepare a PUT request 30 | type RequestObject struct { 31 | // Nonce is the unique nonce used for key derivation 32 | Nonce []byte 33 | 34 | // Name is the encrypted filename, represented as AWS-key-safe version of base64 35 | // (no padding =, / replaced with . and + replaced with -) 36 | // see Characters That Might Require Special Handling 37 | // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html 38 | Name string 39 | 40 | // Key is the unique derived key 41 | Key *memguard.LockedBuffer 42 | } 43 | -------------------------------------------------------------------------------- /cmd/head.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/awnumar/memguard" 6 | "github.com/aws/aws-sdk-go/service/s3" 7 | "github.com/mgren/ogive/object" 8 | "github.com/mgren/ogive/profile" 9 | "github.com/mgren/ogive/util" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func init() { 14 | rootCmd.AddCommand(headCmd) 15 | } 16 | 17 | var headCmd = &cobra.Command{ 18 | Use: "head ", 19 | Short: "Head a specific file.", 20 | Long: "Head a specific ogive file on S3 and retrieve its current archival status. Prints out file status and exits with code: 0 - file available for download, 1 - error occurred, 2 - file not available for download.", 21 | Args: cobra.MinimumNArgs(1), 22 | Run: func(cmd *cobra.Command, args []string) { 23 | inner, err := profile.Open(profileFile) 24 | if err != nil { 25 | util.Fail(err, "Failed to open profile. Wrong password?") 26 | } 27 | inner.Key.Destroy() // Not needed here 28 | 29 | svc := s3.New(util.GetSession(inner)) 30 | 31 | res, err := svc.HeadObject(&s3.HeadObjectInput{ 32 | Bucket: &inner.BucketName, 33 | Key: &args[0], 34 | }) 35 | if err != nil { 36 | util.Fail(err, "Failed to head object.") 37 | } 38 | 39 | obj, err := object.Parse(res, nil, nil, nil) 40 | if err != nil { 41 | util.Fail(err, "Failed to parse response.") 42 | } 43 | 44 | fmt.Println(obj.Restore) 45 | 46 | if obj.Restore != "READY" { 47 | memguard.SafeExit(2) 48 | } 49 | 50 | memguard.SafeExit(0) 51 | }, 52 | } 53 | -------------------------------------------------------------------------------- /progress/progress.go: -------------------------------------------------------------------------------- 1 | package progress 2 | 3 | import ( 4 | "fmt" 5 | "github.com/schollz/progressbar/v2" 6 | "io" 7 | "os" 8 | "time" 9 | ) 10 | 11 | // TrackProgress monitors the passed ProgressReporter and draws a progress bar to stdout based on expected total. 12 | // Due to a 1 second resolution it uses a channel to wake the parent goroutine once it finishes. 13 | func TrackProgress(p ProgressReporter, total int, done chan<- bool) { 14 | theme := progressbar.Theme{Saucer: "█", SaucerPadding: "░", BarStart: "┨", BarEnd: "┠"} 15 | bar := progressbar.NewOptions(total, 16 | progressbar.OptionSetTheme(theme), 17 | progressbar.OptionSetRenderBlankState(true), 18 | progressbar.OptionSetWriter(os.Stderr)) 19 | 20 | t := time.NewTicker(time.Second) 21 | sum := 0 22 | 23 | for true { 24 | current := p.GetProgress() 25 | if current >= total { 26 | bar.Finish() 27 | fmt.Println("\nFinalizing, please wait for the process to exit...") 28 | break 29 | } 30 | bar.Add(current - sum) 31 | sum = current 32 | <-t.C 33 | } 34 | 35 | done <- true 36 | } 37 | 38 | // Read proxies all reads while tracking the totalProgress. 39 | func (r *Reader) Read(p []byte) (n int, err error) { 40 | n, err = r.Reader.Read(p) 41 | r.totalProgress += n 42 | return 43 | } 44 | 45 | // Write proxies all writes while tracking the totalProgress. 46 | func (w *Writer) Write(p []byte) (n int, err error) { 47 | n, err = w.Writer.Write(p) 48 | w.totalProgress += n 49 | return 50 | } 51 | 52 | // GetProgress returns the total number of bytes read. 53 | func (r *Reader) GetProgress() int { 54 | return r.totalProgress 55 | } 56 | 57 | // GetProgress returns the total number of bytes written. 58 | func (w *Writer) GetProgress() int { 59 | return w.totalProgress 60 | } 61 | 62 | // NewReader returns a new reader with progress reporting capability. 63 | func NewReader(r io.Reader) Reader { 64 | return Reader{r, 0} 65 | } 66 | 67 | // NewWriter returns a new writer with progress reporting capability. 68 | func NewWriter(w io.Writer) Writer { 69 | return Writer{w, 0} 70 | } 71 | -------------------------------------------------------------------------------- /cmd/put.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/awnumar/memguard" 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 8 | "github.com/mgren/ogive/crypt" 9 | "github.com/mgren/ogive/object" 10 | "github.com/mgren/ogive/profile" 11 | "github.com/mgren/ogive/progress" 12 | "github.com/mgren/ogive/util" 13 | "github.com/spf13/cobra" 14 | "path/filepath" 15 | ) 16 | 17 | func init() { 18 | rootCmd.AddCommand(putCmd) 19 | } 20 | 21 | var putCmd = &cobra.Command{ 22 | Use: "put ", 23 | Short: "Upload file.", 24 | Long: "Encrypt and upload file to S3 Glacier Deep Archive.", 25 | Args: cobra.MinimumNArgs(1), 26 | Run: func(cmd *cobra.Command, args []string) { 27 | inner, err := profile.Open(profileFile) 28 | if err != nil { 29 | util.Fail(err, "Failed to open profile. Wrong password?") 30 | } 31 | 32 | base := filepath.Base(args[0]) 33 | 34 | obj, err := object.Prepare(inner.Key, base) 35 | if err != nil { 36 | util.Fail(err, "Failed to prepare file for encryption.") 37 | } 38 | 39 | reader, size, err := crypt.GetCryptReader(obj.Key, args[0]) 40 | if err != nil { 41 | util.Fail(err, "Failed to encrypt file.") 42 | } 43 | 44 | fmt.Printf("Uploading %s as %s\n", base, obj.Name) 45 | 46 | proxyReader := progress.NewReader(reader) 47 | done := make(chan bool) 48 | go progress.TrackProgress(&proxyReader, size, done) 49 | 50 | _, err = s3manager.NewUploader(util.GetSession(inner), func(u *s3manager.Uploader) { 51 | u.PartSize = util.GetPartSize(int64(size)) 52 | }).Upload(&s3manager.UploadInput{ 53 | Body: &proxyReader, 54 | Bucket: &inner.BucketName, 55 | Key: &obj.Name, 56 | ContentType: aws.String("application/x-ogive"), 57 | StorageClass: aws.String("DEEP_ARCHIVE"), 58 | Metadata: map[string]*string{ 59 | "Nonce": aws.String(fmt.Sprintf("%x", obj.Nonce)), 60 | }, 61 | }) 62 | 63 | if err != nil { 64 | util.Fail(err, "Failed to upload file.") 65 | } 66 | 67 | <-done 68 | fmt.Printf("Successfully uploaded %s as %s\n", base, obj.Name) 69 | memguard.SafeExit(0) 70 | }, 71 | } 72 | -------------------------------------------------------------------------------- /cmd/restore.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/awnumar/memguard" 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/awserr" 8 | "github.com/aws/aws-sdk-go/service/s3" 9 | "github.com/mgren/ogive/profile" 10 | "github.com/mgren/ogive/util" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func init() { 15 | restoreCmd.Flags().IntVarP(&lifetime, "lifetime", "t", 1, "Specifies the number of days to retain the restored object before returning it to Deep Archive.") 16 | rootCmd.AddCommand(restoreCmd) 17 | } 18 | 19 | var lifetime int 20 | 21 | var restoreCmd = &cobra.Command{ 22 | Use: "restore ", 23 | Short: "Restore a specific file.", 24 | Long: "Initiate file recovery from Deep Archive. Bulk Restore is used. Use \"head\" command to verify when the file becomes ready for download.", 25 | Args: cobra.MinimumNArgs(1), 26 | Run: func(cmd *cobra.Command, args []string) { 27 | inner, err := profile.Open(profileFile) 28 | if err != nil { 29 | util.Fail(err, "Failed to open profile. Wrong password? Exiting...") 30 | memguard.SafeExit(1) 31 | } 32 | inner.Key.Destroy() // Not needed here 33 | 34 | svc := s3.New(util.GetSession(inner)) 35 | 36 | _, err = svc.RestoreObject(&s3.RestoreObjectInput{ 37 | Bucket: &inner.BucketName, 38 | Key: &args[0], 39 | RestoreRequest: &s3.RestoreRequest{ 40 | Days: aws.Int64(int64(lifetime)), 41 | GlacierJobParameters: &s3.GlacierJobParameters{ 42 | Tier: aws.String("Bulk"), 43 | }, 44 | }, 45 | }) 46 | 47 | if err != nil { 48 | if aerr, ok := err.(awserr.Error); ok { 49 | switch aerr.Code() { 50 | case s3.ErrCodeObjectAlreadyInActiveTierError: 51 | fmt.Println("Restoration already completed.") 52 | case "RestoreAlreadyInProgress": // Doesn't seem to be defined in current version of AWS SDK 53 | fmt.Println("Restoration already in progress.") 54 | default: 55 | util.Fail(err, "Failed to request restore.") 56 | } 57 | } else { 58 | util.Fail(err, "Failed to request restore.") 59 | } 60 | } else { 61 | fmt.Println("Restoration request sent.") 62 | } 63 | 64 | memguard.SafeExit(0) 65 | }, 66 | } 67 | -------------------------------------------------------------------------------- /profile/profile.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "errors" 7 | "github.com/awnumar/memguard" 8 | "github.com/mgren/ogive/input" 9 | "golang.org/x/crypto/argon2" 10 | "io/ioutil" 11 | ) 12 | 13 | const version = uint32(1) 14 | const magic = "OGPROF" 15 | 16 | // Open reads the profile file from provided location and returns decrypted InnerData. 17 | func Open(fname string) (in *InnerData, err error) { 18 | var pwd, derived *memguard.LockedBuffer 19 | var od *OuterData 20 | 21 | pwd, err = input.GetMaskedInput("Enter password", "", "", 64, 8) 22 | if err != nil { 23 | return 24 | } 25 | defer pwd.Destroy() 26 | 27 | var data []byte 28 | data, err = ioutil.ReadFile(fname) 29 | if err != nil { 30 | return 31 | } 32 | 33 | dec := gob.NewDecoder(bytes.NewBuffer(data)) 34 | err = dec.Decode(&od) 35 | if err != nil { 36 | return 37 | } 38 | 39 | if od.Magic != magic || od.Version != version { 40 | err = errors.New("Unsupported or corrupted profile file.") 41 | return 42 | } 43 | 44 | derived, err = memguard.NewImmutableFromBytes(argon2.Key(pwd.Buffer(), od.Salt, 3, 32*1024, 4, 32)) 45 | if err != nil { 46 | return 47 | } 48 | defer derived.Destroy() 49 | 50 | return od.unpack(derived) 51 | } 52 | 53 | // Save takes InnerData, encrypts it with the provided key and saves under the selected filename. 54 | func Save(key *memguard.LockedBuffer, in *InnerData, fname string) (err error) { 55 | var salt, derived *memguard.LockedBuffer 56 | salt, err = memguard.NewImmutableRandom(32) 57 | if err != nil { 58 | return 59 | } 60 | 61 | defer salt.Destroy() 62 | derived, err = memguard.NewImmutableFromBytes(argon2.Key(key.Buffer(), salt.Buffer(), 3, 32*1024, 4, 32)) 63 | if err != nil { 64 | return 65 | } 66 | 67 | od := OuterData{Magic: magic, Version: version, Salt: salt.Buffer()} 68 | 69 | err = od.pack(in, derived) 70 | if err != nil { 71 | return 72 | } 73 | 74 | var buf bytes.Buffer 75 | enc := gob.NewEncoder(&buf) 76 | 77 | err = enc.Encode(&od) 78 | if err != nil { 79 | return 80 | } 81 | 82 | return ioutil.WriteFile(fname, buf.Bytes(), 0600) 83 | } 84 | 85 | // NewInner creates a mew InnerData instance with a randomly generated master key. 86 | func NewInner() (in *InnerData, err error) { 87 | var i InnerData 88 | in = &i 89 | in.Key, err = memguard.NewImmutableRandom(32) 90 | return 91 | } 92 | -------------------------------------------------------------------------------- /cmd/list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/InVisionApp/tabular" 6 | "github.com/awnumar/memguard" 7 | "github.com/aws/aws-sdk-go/service/s3" 8 | "github.com/mgren/ogive/crypt" 9 | "github.com/mgren/ogive/object" 10 | "github.com/mgren/ogive/profile" 11 | "github.com/mgren/ogive/util" 12 | "github.com/spf13/cobra" 13 | "os" 14 | ) 15 | 16 | func init() { 17 | rootCmd.AddCommand(listCmd) 18 | 19 | tab = tabular.New() 20 | tab.Col("SIZE", "SIZE", 10) 21 | tab.Col("DATE", "DATE", 12) 22 | tab.Col("STAT", "STATUS", 6) 23 | tab.Col("ID", "STORAGE ID", 10) 24 | tab.Col("NAME", "FILENAME", 8) 25 | } 26 | 27 | var tab tabular.Table 28 | 29 | var listCmd = &cobra.Command{ 30 | Use: "list", 31 | Short: "List archives.", 32 | Long: "Lists all ogive archives in bucket. Lists entire bucket and HEADs each file.", 33 | Run: func(cmd *cobra.Command, args []string) { 34 | inner, err := profile.Open(profileFile) 35 | if err != nil { 36 | util.Fail(err, "Failed to open profile. Wrong password?") 37 | } 38 | 39 | gcm, err := crypt.GetGCM(inner.Key, 32) 40 | inner.Key.Destroy() 41 | if err != nil { 42 | util.Fail(err, "Failed to set up decryptors.") 43 | } 44 | 45 | svc := s3.New(util.GetSession(inner)) 46 | format := tab.Print(tabular.All) 47 | 48 | err = svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{ 49 | Bucket: &inner.BucketName, 50 | }, func(page *s3.ListObjectsV2Output, lastPage bool) bool { 51 | for _, key := range page.Contents { 52 | res, err := svc.HeadObject(&s3.HeadObjectInput{ 53 | Bucket: &inner.BucketName, 54 | Key: key.Key, 55 | }) 56 | if err != nil { 57 | fmt.Fprintln(os.Stderr, "Failed to head object", *key.Key, err) 58 | continue 59 | } 60 | 61 | // Just to make the list show less clutter in case the bucket is not ogive-exclusive 62 | if *res.ContentType != "application/x-ogive" { 63 | continue 64 | } 65 | 66 | obj, err := object.Parse(res, key.Key, gcm, nil) 67 | if err != nil { 68 | fmt.Fprintln(os.Stderr, "Invalid file metadata", *key.Key, err) 69 | continue 70 | } 71 | 72 | // https://golang.org/src/time/format.go 73 | fmt.Printf(format, 74 | util.SizeIEC(int64(obj.Size)), 75 | obj.LastModified.Format("2006-Jan-02"), 76 | obj.Restore, *key.Key, obj.Name) 77 | 78 | } 79 | return !lastPage 80 | }) 81 | 82 | if err != nil { 83 | util.Fail(err, "Failed to list bucket.") 84 | } 85 | 86 | memguard.SafeExit(0) 87 | }, 88 | } 89 | -------------------------------------------------------------------------------- /crypt/crypt.go: -------------------------------------------------------------------------------- 1 | package crypt 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "errors" 7 | "github.com/awnumar/memguard" 8 | "github.com/minio/sio" 9 | "golang.org/x/sys/unix" 10 | "io" 11 | "os" 12 | "path/filepath" 13 | ) 14 | 15 | // GetGCM returns a new AES GCM cipher with optional custom nonce size 16 | func GetGCM(key *memguard.LockedBuffer, size int) (gcm cipher.AEAD, err error) { 17 | var c cipher.Block 18 | 19 | c, err = aes.NewCipher(key.Buffer()) 20 | if err != nil { 21 | return 22 | } 23 | 24 | if size > 0 { 25 | return cipher.NewGCMWithNonceSize(c, size) 26 | } 27 | 28 | return cipher.NewGCM(c) 29 | } 30 | 31 | // GetCryptWriter returns a new io.WriteCloser that will decrypt data written to it. 32 | // Plaintext will be saved under the specified directory with chosen filename. 33 | // 34 | // This writer has 2 bytes already written to it in order to initialize the underlying AES instance. 35 | func GetCryptWriter(key *memguard.LockedBuffer, dir, fname string) (w io.WriteCloser, err error) { 36 | var f os.FileInfo 37 | f, err = os.Stat(dir) 38 | if err != nil { 39 | return 40 | } 41 | if !f.Mode().IsDir() { 42 | err = errors.New(dir + " is not a directory.") 43 | return 44 | } 45 | 46 | dstFname := filepath.Join(dir, fname) 47 | var dst *os.File 48 | 49 | dst, err = os.OpenFile(dstFname, os.O_RDWR|os.O_CREATE, 0600) 50 | if err != nil { 51 | return 52 | } 53 | 54 | w, err = sio.DecryptWriter(dst, sio.Config{Key: key.Buffer()}) 55 | 56 | var n int 57 | n, err = w.Write([]byte{sio.Version20, sio.AES_256_GCM}) 58 | if n != 2 { 59 | err = errors.New("Invalid write length " + string(n)) 60 | } 61 | 62 | return 63 | } 64 | 65 | // GetCryptReader returns a new io.Reader that reads and encrypts the specified filename 66 | func GetCryptReader(key *memguard.LockedBuffer, fname string) (r io.Reader, s int, err error) { 67 | var src *os.File 68 | src, err = os.Open(fname) 69 | if err != nil { 70 | return 71 | } 72 | 73 | var f os.FileInfo 74 | f, err = src.Stat() 75 | if err != nil { 76 | return 77 | } 78 | 79 | s = int(f.Size()) 80 | 81 | // Assume block device 82 | if s == 0 { 83 | // Ignoring the error is ok here, continue without size 84 | s, _ = unix.IoctlGetInt(int(src.Fd()), unix.BLKGETSIZE64) 85 | } 86 | 87 | r, err = sio.EncryptReader(src, sio.Config{ 88 | MinVersion: sio.Version20, 89 | MaxVersion: sio.Version20, 90 | CipherSuites: []byte{sio.AES_256_GCM}, 91 | Key: key.Buffer(), 92 | }) 93 | 94 | return 95 | } 96 | -------------------------------------------------------------------------------- /profile/profile_inner.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "github.com/awnumar/memguard" 8 | "reflect" 9 | ) 10 | 11 | func (id *InnerData) marshalBinaryLocked() (buf *memguard.LockedBuffer, err error) { 12 | defer id.wipe() 13 | var head, body bytes.Buffer 14 | var cnt int 15 | v := reflect.ValueOf(id).Elem() 16 | 17 | for i := 0; i < v.NumField(); i++ { 18 | f := v.Field(i) 19 | if f.CanInterface() { 20 | t := f.Interface() 21 | switch t := t.(type) { 22 | case *memguard.LockedBuffer: 23 | err = binary.Write(&head, binary.LittleEndian, uint32(t.Size())) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | cnt, err = body.Write(t.Buffer()) 29 | if err != nil { 30 | return nil, err 31 | } 32 | if cnt != t.Size() { 33 | return nil, errors.New("Incorrect number of bytes written.") 34 | } 35 | 36 | t.Destroy() 37 | case string: 38 | err = binary.Write(&head, binary.LittleEndian, uint32(len(t))) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | cnt, err = body.Write([]byte(t)) 44 | if err != nil { 45 | return nil, err 46 | } 47 | if cnt != len(t) { 48 | return nil, errors.New("Incorrect number of bytes written.") 49 | } 50 | default: 51 | return nil, errors.New("Incorrect field type.") 52 | } 53 | } 54 | } 55 | 56 | // bytes.Join copies the data, so create two LockedBuffers to zero out underlying memory regions 57 | headLocked, err := memguard.NewImmutableFromBytes(head.Bytes()) 58 | bodyLocked, err := memguard.NewImmutableFromBytes(body.Bytes()) 59 | 60 | return memguard.Concatenate(headLocked, bodyLocked) 61 | } 62 | 63 | func (id *InnerData) unmarshalBinaryLocked(data *memguard.LockedBuffer) error { 64 | v, total := reflect.ValueOf(id).Elem(), uint32(24) // four bytes per value times six fields 65 | data.MakeMutable() 66 | 67 | for i := 0; i < v.NumField(); i++ { 68 | f := v.Field(i) 69 | if f.CanInterface() { 70 | size := binary.LittleEndian.Uint32(data.Buffer()[4*i : 4*(1+i)]) 71 | if size == 0 { 72 | return errors.New("Corruped profile file.") 73 | } 74 | t := f.Interface() 75 | switch t.(type) { 76 | case *memguard.LockedBuffer: 77 | b, err := memguard.NewImmutableFromBytes(data.Buffer()[total : total+size]) 78 | if err != nil { 79 | return err 80 | } 81 | f.Set(reflect.ValueOf(b)) 82 | total += size 83 | case string: 84 | f.SetString(string(data.Buffer()[total : total+size])) 85 | total += size 86 | default: 87 | return errors.New("Incorrect field type.") 88 | } 89 | } 90 | } 91 | 92 | data.Destroy() 93 | return nil 94 | } 95 | 96 | func (id *InnerData) wipe() { 97 | v := reflect.ValueOf(id).Elem() 98 | 99 | for i := 0; i < v.NumField(); i++ { 100 | f := v.Field(i) 101 | if f.CanInterface() { 102 | t := f.Interface() 103 | // We don't care about string data here 104 | switch t := t.(type) { 105 | case *memguard.LockedBuffer: 106 | t.Destroy() 107 | default: 108 | } 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "github.com/awnumar/memguard" 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/awserr" 8 | "github.com/aws/aws-sdk-go/aws/credentials" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 11 | "github.com/mgren/ogive/profile" 12 | "github.com/minio/sio" 13 | "io" 14 | "os" 15 | "path/filepath" 16 | ) 17 | 18 | // GetDefaultProfileLoc returns the default ogive profile location 19 | func GetDefaultProfileLoc() string { 20 | return filepath.Join(os.Getenv("HOME"), ".ogive") 21 | } 22 | 23 | // SizeIEC transforms a bytesize into an approximate (1 decimal place) IEC-compliant representation. 24 | // It supports sizes up to 1000 TiB which is more than plenty for S3 maximum of 5 TB 25 | func SizeIEC(b int64) string { 26 | unit, div, exp := int64(1024), int64(1024), 0 27 | 28 | if b < unit { 29 | return fmt.Sprintf("%d B", b) 30 | } 31 | 32 | for n := b / unit; n >= unit; n /= unit { 33 | div *= unit 34 | exp++ 35 | } 36 | 37 | return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGT"[exp]) 38 | } 39 | 40 | // WriteAt is a dummy positional writer method. It ignores the offset and writes into the original Writer sequentially. 41 | // This implementation is ogive-specific and omits first two bytes, making sure they are 0x20 0x00. 42 | // See ogive/cmd/get.go source code for an explanation. 43 | func (w WriterAtFake) WriteAt(p []byte, offset int64) (s int, err error) { 44 | if offset == 0 && *w.f { 45 | if p[0] != byte(sio.Version20) || p[1] != byte(sio.AES_256_GCM) { 46 | return 2, fmt.Errorf("Wrong start of header %x %x", p[0], p[1]) 47 | } 48 | *w.f = false 49 | s, err = w.w.Write(p[2:]) 50 | s += 2 51 | return 52 | } 53 | return w.w.Write(p) 54 | } 55 | 56 | // NewWriterAtFake wraps an io.Writer into a dummy io.WriterAt interface. 57 | func NewWriterAtFake(w io.Writer) WriterAtFake { 58 | f := true 59 | return WriterAtFake{w, &f} 60 | } 61 | 62 | // GetSession uses ogive profile data to create a new AWS session. 63 | func GetSession(i *profile.InnerData) *session.Session { 64 | defer i.AWSKeyId.Destroy() 65 | defer i.AWSSecret.Destroy() 66 | return session.New(&aws.Config{ 67 | Region: &i.Region, 68 | Credentials: credentials.NewStaticCredentials(string(i.AWSKeyId.Buffer()), string(i.AWSSecret.Buffer()), ""), 69 | Endpoint: &i.Endpoint, 70 | }) 71 | } 72 | 73 | // GetPartSize returns the part size for multipart upload. 74 | // For files less than 500 MiB the part size is 5 MiB 75 | // For files between 500 MiB and 5 000 MiB part size grows dynamically to create a 100-part upload 76 | // For files between 5 000 MiB and 50 000 MiB the part size is 50 MiB and part count increases 77 | // For files more than 50 000 MiB part count is 10 000 and part size starts to grow again 78 | func GetPartSize(size int64) int64 { 79 | partSize := size / 100 80 | def := int64(50 << 20) 81 | 82 | if partSize < s3manager.MinUploadPartSize { 83 | return s3manager.MinUploadPartSize 84 | } 85 | 86 | if partSize > def { 87 | partSize = size / s3manager.MaxUploadParts 88 | if partSize < def { 89 | return def 90 | } 91 | 92 | return partSize 93 | } 94 | 95 | return partSize 96 | } 97 | 98 | // Fail prints out the error, additional message and exits the program with code 1 while also zeroing all memguard buffers. 99 | func Fail(err error, msg string) { 100 | if awserr, ok := err.(awserr.Error); ok { 101 | fmt.Fprintln(os.Stderr, awserr) 102 | } else { 103 | fmt.Fprintln(os.Stderr, err) 104 | } 105 | fmt.Fprintln(os.Stderr, msg+" Exiting...") 106 | memguard.SafeExit(1) 107 | } 108 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/InVisionApp/tabular v0.3.0 h1:4DGJoBZRTcgd/O+YgfG7/9bXAQy01tSJxrxWEuHVgnM= 2 | github.com/InVisionApp/tabular v0.3.0/go.mod h1:/G6t7qe0ZULisB+FjMsB0Qu0mtJ2CZldq92nXWjfHGI= 3 | github.com/awnumar/memguard v0.15.1 h1:RDPYo+e6rm65NLKJqmSVQpO9LuLh7R/hfC6Ed3ahbEU= 4 | github.com/awnumar/memguard v0.15.1/go.mod h1:77EUD6uwfgcd6zTmn++i5ujEFviGRQfE8ELbDJO1rpA= 5 | github.com/aws/aws-sdk-go v1.19.28 h1:u0KMC+Qv0YVyz8YR6mREEtslSPkdUMzXgDJFD5196O8= 6 | github.com/aws/aws-sdk-go v1.19.28/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 11 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 12 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 13 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 14 | github.com/minio/sio v0.0.0-20190118043801-035b4ef8c449 h1:p7L1eKiloAwHpDkurkmzaLuRYTReh0aWNxj0rrVVsF8= 15 | github.com/minio/sio v0.0.0-20190118043801-035b4ef8c449/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= 16 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= 17 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= 18 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 19 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 20 | github.com/schollz/progressbar/v2 v2.12.1 h1:0Ce7IBClG+s3lxXN1Noqwh7aToKGL5a3mnMfPJqDlv4= 21 | github.com/schollz/progressbar/v2 v2.12.1/go.mod h1:fBI3onORwtNtwCWJHsrXtjE3QnJOtqIZrvr3rDaF7L0= 22 | github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= 23 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 24 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 25 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 26 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 27 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 28 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 29 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 30 | golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 31 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 32 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= 33 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 34 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= 35 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 36 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 37 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 38 | golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= 39 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 40 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 41 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 42 | -------------------------------------------------------------------------------- /object/object.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "crypto/cipher" 5 | "encoding/base64" 6 | "encoding/hex" 7 | "errors" 8 | "github.com/awnumar/memguard" 9 | "github.com/aws/aws-sdk-go/service/s3" 10 | "github.com/mgren/ogive/crypt" 11 | "golang.org/x/crypto/argon2" 12 | "regexp" 13 | "strings" 14 | ) 15 | 16 | // Parse translates the output of an s3 HeadObject command into a robust ogive archive file representation 17 | // retrieving information such as original filename, unique file nonce, or the derived key (if possible). 18 | // 19 | // The reason this function requires an existing instance of cipher.AEAD is that it can be reused between 20 | // multiple object instances (in case of list command), which is more efficient than creating it every time. 21 | func Parse(res *s3.HeadObjectOutput, key *string, gcm cipher.AEAD, master *memguard.LockedBuffer) (o ResponseObject, err error) { 22 | if *res.ContentType != "application/x-ogive" { 23 | err = errors.New("Invalid content-type " + *res.ContentType) 24 | return 25 | } 26 | 27 | pattern := regexp.MustCompile("ongoing-request=\\\"(false|true)\\\"") 28 | 29 | o.Restore = "?????" 30 | 31 | if res.StorageClass != nil && *res.StorageClass == "DEEP_ARCHIVE" { 32 | o.Restore = "DEEPS" 33 | } 34 | 35 | if res.Restore != nil { 36 | match := pattern.FindStringSubmatch(*res.Restore) 37 | if match[1] == "true" { 38 | o.Restore = "RECOV" 39 | } else if match[1] == "false" { 40 | o.Restore = "READY" 41 | } else { 42 | o.Restore = "?????" 43 | } 44 | } 45 | 46 | o.Size = int(*res.ContentLength) 47 | o.LastModified = *res.LastModified 48 | 49 | if key == nil || gcm == nil { 50 | return 51 | } 52 | 53 | base := strings.Replace(strings.Replace(*key, ".", "/", -1), "-", "+", -1) 54 | var cryptName, name []byte 55 | 56 | cryptName, err = base64.RawStdEncoding.DecodeString(base) 57 | if err != nil { 58 | return 59 | } 60 | 61 | o.Nonce, err = hex.DecodeString(*res.Metadata["Nonce"]) 62 | if err != nil { 63 | return 64 | } 65 | if len(o.Nonce) != 32 { 66 | err = errors.New("Malformed nonce " + *res.Metadata["Nonce"]) 67 | return 68 | } 69 | 70 | name, err = gcm.Open(nil, o.Nonce, cryptName, nil) 71 | if err != nil { 72 | return 73 | } 74 | 75 | o.Name = string(name) 76 | 77 | if master == nil { 78 | return 79 | } 80 | 81 | o.Key, err = memguard.NewImmutableFromBytes(argon2.Key(master.Buffer(), o.Nonce, 3, 32*1024, 4, 32)) 82 | 83 | return 84 | } 85 | 86 | // Prepare is the inverse of Parse. It generates a unique nonce and encrypts the filename. 87 | func Prepare(master *memguard.LockedBuffer, fname string) (o RequestObject, err error) { 88 | var buf *memguard.LockedBuffer 89 | buf, err = memguard.NewImmutableRandom(32) 90 | if err != nil { 91 | return 92 | } 93 | 94 | // Assignment to o.Nonce must be done after this KDF (at least in GO 1.11.10). Otherwise, hilarity ensues: 95 | // If o.Nonce is used as salt parameter instead of buf.Buffer() the GC will prematurely decide to free 96 | // the memory region for memguard container which will then be immediately reused for o.Key... 97 | // o.Nonce and o.Key.Buffer() will refer to the same address and return identical data. 98 | o.Key, err = memguard.NewImmutableFromBytes(argon2.Key(master.Buffer(), buf.Buffer(), 3, 32*1024, 4, 32)) 99 | if err != nil { 100 | return 101 | } 102 | 103 | o.Nonce = buf.Buffer() 104 | 105 | // Use bare AES for filename, to save on sio overhead 106 | // Override default GCM nonce size, since a single nonce is shared between file content and file name 107 | gcm, err := crypt.GetGCM(master, 32) 108 | master.Destroy() 109 | if err != nil { 110 | return 111 | } 112 | 113 | encryptedBase := gcm.Seal(nil, o.Nonce, []byte(fname), nil) 114 | 115 | o.Name = strings.Replace(strings.Replace(base64.RawStdEncoding.EncodeToString(encryptedBase), "/", ".", -1), "+", "-", -1) 116 | 117 | if len(o.Name) > 1024 { 118 | err = errors.New("Storage filename too long.") 119 | } 120 | 121 | return 122 | } 123 | -------------------------------------------------------------------------------- /cmd/init.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/awnumar/memguard" 7 | "github.com/mgren/ogive/input" 8 | "github.com/mgren/ogive/profile" 9 | "github.com/mgren/ogive/util" 10 | "github.com/spf13/cobra" 11 | "os" 12 | ) 13 | 14 | func init() { 15 | initCmd.Flags().BoolVarP(&reinit, "reinit", "r", false, "Reinitialize an existing profile to change password and/or AWS keys. Old profile is stored as \".bak\".") 16 | rootCmd.AddCommand(initCmd) 17 | } 18 | 19 | var reinit bool 20 | 21 | var initCmd = &cobra.Command{ 22 | Use: "init", 23 | Short: "Set up an ogive profile.", 24 | Long: "Set up an ogive profile, including your cryptographic key and S3 bucket location.", 25 | Run: func(cmd *cobra.Command, args []string) { 26 | var profileInner *profile.InnerData 27 | var err error 28 | 29 | if reinit { 30 | profileInner, err = profile.Open(profileFile) 31 | } else { 32 | profileInner, err = profile.NewInner() 33 | } 34 | if err != nil { 35 | util.Fail(err, "Failed to generate profile.") 36 | } 37 | defer profileInner.Key.Destroy() 38 | 39 | pwd, err := input.GetMaskedInput("Enter password", "", "", 64, 8) 40 | if err != nil { 41 | util.Fail(err, "Failed to read password.") 42 | } 43 | defer pwd.Destroy() 44 | 45 | pwd2, err := input.GetMaskedInput("Confirm password", "", "", 64, 8) 46 | if err != nil { 47 | util.Fail(err, "Failed to read password.") 48 | } 49 | defer pwd2.Destroy() 50 | 51 | assertEqual(pwd, pwd2) 52 | 53 | id, secret := getMaskedInputs(reinit) 54 | 55 | if id != nil { 56 | profileInner.AWSKeyId = id 57 | } 58 | 59 | if secret != nil { 60 | profileInner.AWSSecret = secret 61 | } 62 | 63 | defer profileInner.AWSKeyId.Destroy() 64 | defer profileInner.AWSSecret.Destroy() 65 | 66 | if reinit { 67 | err = os.Rename(profileFile, profileFile+".bak") 68 | if err != nil { 69 | util.Fail(err, "Failed to back up profile.") 70 | } 71 | } else { 72 | // This remains unchanged on reinit 73 | profileInner.BucketName, profileInner.Region, profileInner.Endpoint = getInputs() 74 | } 75 | 76 | err = profile.Save(pwd, profileInner, profileFile) 77 | if err != nil { 78 | util.Fail(err, "Failed to generate profile.") 79 | } 80 | 81 | fmt.Println("Profile successfully created.") 82 | memguard.SafeExit(0) 83 | }, 84 | } 85 | 86 | func assertEqual(a, b *memguard.LockedBuffer) { 87 | eq, err := memguard.Equal(a, b) 88 | if err != nil { 89 | util.Fail(err, "Error reading password.") 90 | } 91 | if !eq { 92 | util.Fail(errors.New("Passwords must match."), "Passwords must match.") 93 | } 94 | b.Destroy() 95 | } 96 | 97 | func getInputs() (bucket, region, endpoint string) { 98 | var err error 99 | var buf *memguard.LockedBuffer 100 | 101 | // https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html 102 | buf, err = input.GetInput("Enter AWS S3 bucket name", "", "", 63, 3) 103 | if err != nil { 104 | util.Fail(err, "Failed to generate profile.") 105 | } 106 | bucket = string(buf.Buffer()) 107 | 108 | buf, err = input.GetInput("Enter AWS S3 Region", "eu-west-1", "", 64, 0) 109 | if err != nil { 110 | util.Fail(err, "Failed to generate profile.") 111 | } 112 | region = string(buf.Buffer()) 113 | 114 | buf, err = input.GetInput("Enter AWS S3 endpoint", "https://s3."+region+".amazonaws.com", "", 64, 0) 115 | if err != nil { 116 | util.Fail(err, "Failed to generate profile.") 117 | } 118 | endpoint = string(buf.Buffer()) 119 | 120 | return 121 | } 122 | 123 | func getMaskedInputs(allowBlank bool) (id, secret *memguard.LockedBuffer) { 124 | var err error 125 | min := 1 126 | 127 | if allowBlank { 128 | min = 0 129 | } 130 | 131 | id, err = input.GetMaskedInput("Enter AWS Key ID", "", "", 64, min) 132 | if err != nil { 133 | util.Fail(err, "Failed to generate profile.") 134 | } 135 | 136 | secret, err = input.GetMaskedInput("Enter AWS Key Secret", "", "", 64, min) 137 | if err != nil { 138 | util.Fail(err, "Failed to generate profile.") 139 | } 140 | 141 | return 142 | } 143 | -------------------------------------------------------------------------------- /input/input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/awnumar/memguard" 7 | "os" 8 | "syscall" 9 | ) 10 | 11 | // GetMaskedInput prompts the user for input and then reads a single newline-terminated line 12 | // from stdin and returns it as a memguard.LockedBuffer with the terminating newline removed. 13 | // User input is not displayed in the console. 14 | // 15 | // It also enforces the user to provide a limited number of characters bound by limitMax and limitMin. 16 | // If an empty input is allowed (indicated by lmitMin = 0) GetInput can optionally return a default value. 17 | // 18 | // If standard input is not interactive (ex. redirected from another process) all prompts, limit checks 19 | // and fallbacks are disabled and raw input is returned. 20 | func GetMaskedInput(prompt, def, after string, limitMax, limitMin int) (b *memguard.LockedBuffer, err error) { 21 | attrs := syscall.ProcAttr{ 22 | Dir: "", 23 | Env: []string{}, 24 | Files: []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd()}, 25 | Sys: nil} 26 | var ws syscall.WaitStatus 27 | var stat os.FileInfo 28 | var pid int 29 | 30 | // Ugly hack to hide even uglier warning messages when receiving password input from redirected stdin. 31 | stat, err = os.Stdin.Stat() 32 | if err != nil { 33 | return 34 | } 35 | if (stat.Mode() & os.ModeCharDevice) == 0 { 36 | b, err = readInputBare() 37 | if err != nil { 38 | return 39 | } 40 | 41 | return memguard.Trim(b, 0, b.Size()-1) 42 | } 43 | 44 | pid, err = syscall.ForkExec( 45 | "/bin/stty", 46 | []string{"stty", "-echo"}, 47 | &attrs) 48 | if err != nil { 49 | return 50 | } 51 | 52 | _, err = syscall.Wait4(pid, &ws, 0, nil) 53 | if err != nil { 54 | return 55 | } 56 | 57 | b, err = GetInput(prompt, def, after+"\n", limitMax, limitMin) 58 | if err != nil { 59 | return 60 | } 61 | 62 | pid, err = syscall.ForkExec( 63 | "/bin/stty", 64 | []string{"stty", "echo"}, 65 | &attrs) 66 | if err != nil { 67 | return 68 | } 69 | 70 | _, err = syscall.Wait4(pid, &ws, 0, nil) 71 | 72 | return 73 | } 74 | 75 | // GetInput prompts the user for input and then reads a single newline-terminated line 76 | // from stdin and returns it as a memguard.LockedBuffer with the terminating newline removed. 77 | // 78 | // It also enforces the user to provide a limited number of characters bound by limitMax and limitMin. 79 | // If an empty input is allowed (indicated by lmitMin = 0) GetInput can optionally return a default value. 80 | func GetInput(prompt, def, after string, limitMax, limitMin int) (b *memguard.LockedBuffer, err error) { 81 | if def != "" { 82 | prompt = fmt.Sprintf("%s (default is %s)", prompt, def) 83 | } 84 | 85 | b, err = readInput(prompt, after) 86 | if err != nil { 87 | return 88 | } 89 | 90 | for b.Size()-1 > limitMax || b.Size()-1 < limitMin { 91 | if b.Size()-1 > limitMax { 92 | fmt.Printf("Input is too long. Maximum of %d characters allowed.\n", limitMax) 93 | } else { 94 | fmt.Printf("Input must be at least %d characters.\n", limitMin) 95 | } 96 | b.Destroy() 97 | 98 | b, err = readInput(prompt, after) 99 | if err != nil { 100 | return 101 | } 102 | } 103 | 104 | if b.Size() == 1 && def != "" { 105 | return memguard.NewImmutableFromBytes([]byte(def)) 106 | } 107 | if b.Size() > 1 { 108 | return memguard.Trim(b, 0, b.Size()-1) 109 | } 110 | 111 | // If empty imput is allowed and no default is passed, return a nil pointer since memguard can't create an empty buffer. 112 | return nil, nil 113 | } 114 | 115 | // readInput is the prompting and reading primitive 116 | func readInput(prompt, after string) (b *memguard.LockedBuffer, err error) { 117 | fmt.Print(prompt + ": ") 118 | b, err = readInputBare() 119 | fmt.Print(after) 120 | return 121 | } 122 | 123 | // readInputBare can just read user input without any prompts/trailers 124 | func readInputBare() (b *memguard.LockedBuffer, err error) { 125 | var in []byte 126 | reader := bufio.NewReader(os.Stdin) 127 | 128 | in, err = reader.ReadBytes('\n') 129 | if err != nil { 130 | return 131 | } 132 | 133 | return memguard.NewImmutableFromBytes(in) 134 | } 135 | -------------------------------------------------------------------------------- /cmd/get.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/awnumar/memguard" 6 | "github.com/aws/aws-sdk-go/service/s3" 7 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 8 | "github.com/mgren/ogive/crypt" 9 | "github.com/mgren/ogive/object" 10 | "github.com/mgren/ogive/profile" 11 | "github.com/mgren/ogive/progress" 12 | "github.com/mgren/ogive/util" 13 | "github.com/spf13/cobra" 14 | ) 15 | 16 | func init() { 17 | getCmd.Flags().StringVarP(&output, "output", "o", "", "Override destination filename.") 18 | rootCmd.AddCommand(getCmd) 19 | } 20 | 21 | var output string 22 | 23 | var getCmd = &cobra.Command{ 24 | Use: "get ", 25 | Short: "Download file.", 26 | Long: "Download and decrypt file, saving it under the original filename.", 27 | Args: cobra.MinimumNArgs(2), 28 | Run: func(cmd *cobra.Command, args []string) { 29 | inner, err := profile.Open(profileFile) 30 | if err != nil { 31 | util.Fail(err, "Failed to open profile. Wrong password?") 32 | } 33 | 34 | gcm, err := crypt.GetGCM(inner.Key, 32) 35 | if err != nil { 36 | util.Fail(err, "Failed to set up decryptors.") 37 | } 38 | 39 | sess := util.GetSession(inner) 40 | svc := s3.New(sess) 41 | 42 | res, err := svc.HeadObject(&s3.HeadObjectInput{ 43 | Bucket: &inner.BucketName, 44 | Key: &args[0], 45 | }) 46 | if err != nil { 47 | util.Fail(err, "Failed to head object.") 48 | } 49 | 50 | obj, err := object.Parse(res, &args[0], gcm, inner.Key) 51 | inner.Key.Destroy() 52 | if err != nil { 53 | util.Fail(err, "Invalid file metadata") 54 | } 55 | if obj.Restore != "READY" { 56 | util.Fail(err, "File not restored, please run ogive restore first.") 57 | } 58 | 59 | if output == "" { 60 | output = obj.Name 61 | } 62 | 63 | fmt.Println("File will be saved as", output) 64 | 65 | writer, err := crypt.GetCryptWriter(obj.Key, args[1], output) 66 | if err != nil { 67 | util.Fail(err, "Failed to open file for writing.") 68 | } 69 | // This writer is initiated with 2 bytes already written. 70 | // Since sio supports two different ciphers, it lazily initiates only one of them when it knows which one 71 | // i.e. when the second byte is written to the writer. 72 | // The destruction must be delayed until that happens, otherwise the underlying AES asm code will run into a memory violation during key expansion. 73 | // Since there is no out-of-the-box way to notify this routine of when that happens, the writer is initialized via magic. 74 | // Both sio version and cipher are pinned for the sio.EncryptReader, so all uploaded files will always have the same header. 75 | // The first two bytes (0x20 0x00) are written manually using WriterAtFake which then omits first two bytes on the very first call. 76 | // As bad as it sounds, it relies on exported constants, it's just that they weren't supposed to be used this way. 77 | obj.Key.Destroy() 78 | defer writer.Close() 79 | 80 | proxyWriter := progress.NewWriter(writer) 81 | done := make(chan bool) 82 | go progress.TrackProgress(&proxyWriter, int(*res.ContentLength)-2, done) 83 | 84 | // WriterAtFake works, because s3manager.Downloader.Concurrency = 1 assures sequential write. 85 | // Using actual WriterAt with AES is technically possible but requires every At to be a multiple of block size. 86 | // This could also be implemented with an intermediate buffer of size s3manager.Download.Concurreny * s3manager.Download.PartSize, 87 | // but Download doesn't guarantee a write of size s3manager.Download.PartSize even for non-final parts, which makes it much more difficult to do. 88 | // Best way would be probably to request with s3.GetObjectInput.Range specified and implement concurrency locally. 89 | fake := util.NewWriterAtFake(&proxyWriter) 90 | 91 | _, err = s3manager.NewDownloader(sess, func(u *s3manager.Downloader) { 92 | u.PartSize = 50 << 20 93 | u.Concurrency = 1 94 | }).Download(&fake, &s3.GetObjectInput{ 95 | Bucket: &inner.BucketName, 96 | Key: &args[0], 97 | }) 98 | if err != nil { 99 | util.Fail(err, "Failed to download file.") 100 | } 101 | 102 | <-done 103 | fmt.Printf("Successfully downloaded %s as %s. Exiting...\n", args[0], output) 104 | 105 | // This is needed because memguard.SafeExit relies on os.Exit, which doesn't honour defer stack. 106 | writer.Close() 107 | memguard.SafeExit(0) 108 | }, 109 | } 110 | -------------------------------------------------------------------------------- /doc/ogive.1.templ: -------------------------------------------------------------------------------- 1 | .TH ogive 1 "12 May 2019" "version __VERSION__" 2 | . 3 | .SH NAME 4 | ogive - secure backups with AWS S3 Glacier Deep Archive 5 | . 6 | .SH SYNOPSIS 7 | .B ogive 8 | .I SUBCOMMAND 9 | .I [FLAGS] 10 | . 11 | .SH DESCRIPTION 12 | .B ogive 13 | is a simple commandline tool for storing and retrieving cryptographically secure 14 | backups from AWS S3 Glacier Deep Archive. 15 | . 16 | .SH OPTIONS 17 | .SS General Flags 18 | .TP 19 | .BR \-h ", " \-\^\-help\fP[=false] 20 | Help for 21 | .B ogive\fP. 22 | This flag is also available on all subcommands and prints subcommand-specific help. 23 | .TP 24 | .BR \-p ", " \-\^\-profile\fP[="$HOME/.ogive"] 25 | Location of the 26 | .B ogive 27 | profile file to be used with subcommands. 28 | . 29 | .SS Subcommands 30 | .TP 31 | .B get \fISOURCE_FILE DESTINATION_DIRECTORY 32 | Can be used to download individual stored files. By default, files are saved in the 33 | .I DESTINATION_DIRECTORY 34 | under the orignial filename. 35 | .RS 36 | .TP 37 | .BR \-o ", " \-\^\-output\fP[=""] 38 | Override destination filename. 39 | .RE 40 | .TP 41 | .B head \fISTORAGE_ID 42 | Can be used to head a single file and check if its recovery has completed. 43 | Following exit codes and file statuses are possible: 44 | .TS 45 | l l. 46 | CODE DESCRIPTION 47 | _ 48 | \fI0\fP file available for download, 49 | \fI1\fP error occurred, 50 | \fI2\fP file not available for download. 51 | .TE 52 | .TS 53 | l l. 54 | STATUS DESCRIPTION 55 | _ 56 | \fIDEEPS\fP file is in DEEP_ARCHIVE state with no restore job pending, 57 | \fIRECOV\fP file is currently being restored, 58 | \fIREADY\fP file has been restored into STANDARD storage and is ready for downloading, 59 | \fI?????\fP file state is unrecognized. 60 | .TE 61 | .TP 62 | .B init 63 | .RS 64 | Can be used to set up an ogive profile, including the cryptographic key, 65 | AWS credentials and S3 bucket location. 66 | .TP 67 | .BR \-r ", " \-\^\-reinit\fP[=false] 68 | Reinitialize an existing profile to change the profile password and/or AWS keys. 69 | Old profile is stored as "\fIORIGINAL_PROFILE\fP.bak". 70 | .RE 71 | .TP 72 | .B list 73 | .RS 74 | Lists all ogive archives in an S3 bucket. 75 | Lists entire bucket and HEADs each file to retrieve metadata. 76 | .RE 77 | .TP 78 | .B put \fISOURCE_FILE 79 | Encrypt and upload file to S3 Glacier Deep Archive. 80 | .TP 81 | .B restore \fISTORAGE_ID 82 | Initiate file recovery from Deep Archive. Bulk Restore is used. 83 | Use \fIhead\fP command to verify when the file becomes ready for download. 84 | .RS 85 | .TP 86 | .BR \-r ", " \-\^\-lifetime\fP[=1] 87 | Specifies the number of days to retain the restored object before returning it 88 | to Deep Archive. 89 | .RE 90 | . 91 | .SH NOTES 92 | .SS Progress Reporting 93 | When running the \fIget\fP or \fIput\fP commands, ogive will report an approximate 94 | progress. For file uploads this is highly inaccurate for objects smaller than 550 MiB. 95 | This is because aws-sdk-go lacks progress reporting in its s3manager, 96 | so this program relies on the amount of bytes read by the manager instead. 97 | Users should always wait for the program to exit gracefully instead of relying solely 98 | on the progress bar. 99 | .SS Multiple Backup Versions 100 | Since each \fIput\fP generates an unique nonce and derives an unique name, 101 | the probability of name collision in storage is basically zero. This allows to 102 | \fIput\fP the same file multiple times at different points in time to create 103 | multiple backups. 104 | .SS About the profile file 105 | Since the profile file stores the master key, its loss or corruption renders 106 | all backups created with it unrecoverable. A copy of the profile file on a separate 107 | medium is essential. An additional, physical backup of the profile file such as PaperBack 108 | .RB < http://ollydbg.de/Paperbak/ > 109 | is suggested. 110 | .SS Broken Downloads/uploads 111 | Currently ogive does not support any form of download/upload resumption. 112 | One file operation must complete in one run. This is unlikely to change, 113 | at least until sio supports WriteAt and ReadAt 114 | .RB < https://github.com/minio/sio/issues/13 > 115 | With that in place, the upload/download code would need to be rewritten to replace 116 | s3manager with manual control of part download/upload in order to ensue all operations 117 | are aligned to the underlying cipher block size. 118 | . 119 | .SH EXAMPLE 120 | .SS Basic Example 121 | .nf 122 | .RS 123 | ogive init 124 | // enter profile information 125 | ogive put example.dat 126 | ogive list 127 | // find example.dat on the list to determine its storage_id 128 | ogive restore 129 | ogive head 130 | // Bulk Restore usually completes within 48 hours 131 | ogive get /directory/to/save-in 132 | .RE 133 | .fi 134 | .SS Writing To/From a Block Device 135 | .nf 136 | .RS 137 | ogive put /dev/sdf 138 | ogive get /dev # --output=sdg to write to sdg instead of sdf 139 | .RE 140 | .fi 141 | .SS Supplying a Password From Stdin 142 | .nf 143 | .RS 144 | bash securely-retrieve-password-and-write-to-stdout.sh | ogive put example.dat 145 | .SS Restore All Archives 146 | .nf 147 | .RS 148 | bash securely-retrieve-password-and-write-to-stdout.sh | ogive list | \ 149 | awk 'NR>2 && $4 == "DEEPS" {print $5}' | while read id; 150 | do bash securely-retrieve-password-and-write-to-stdout.sh | ogive restore $id; 151 | done 152 | .RE 153 | .fi 154 | . 155 | .SH KNOWN ISSUES 156 | .TP 157 | .B panic: memguard.memcall.Lock() when running ogive init 158 | There seems to be a rare race condition in the Go compiler that makes 159 | the resulting binary panic each time when creating or reinitializng a profile file. 160 | Recompiling the program and replacing its binary fixes the issue. 161 | . 162 | .SH COPYRIGHT 163 | This is free and unencumbered software released into the public domain. 164 | .PP 165 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 166 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 167 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 168 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 169 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 170 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 171 | OTHER DEALINGS IN THE SOFTWARE. 172 | .PP 173 | For more information, please refer to 174 | .RB < http://unlicense.org/ > 175 | . 176 | .SH BUGS 177 | .SS "Reporting Bugs" 178 | Please report any bugs or issues you might encounter at 179 | .RB < https://github.com/mgren/ogive/issues >. 180 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | ogive logo 3 |
4 | Secure backups with AWS S3 Glacier Deep Archive 5 |
6 |
7 |

8 | 9 | goreportcard badge 10 | 11 |

12 |

13 |
14 | ---- 15 | 16 | Ogive is a simple commandline tool for storing and retrieving cryptographically secure backups from [AWS S3 Glacier Deep Archive](https://aws.amazon.com/blogs/aws/new-amazon-s3-storage-class-glacier-deep-archive/). 17 | 18 | Ogive encrypts all data and metadata (original filename) before uploading the file to S3 and decrypts it upon retrieval. Each file upload has its own, unique encryption key derived from the master key. Only the non-secret nonce used for key derivation is stored together with the encrypted file. Neither the master key nor the derived key are ever uploaded to any AWS service. The master key together with AWS credentials used for S3 operations is stored locally, in portable profile files. Those files are in turn indirectly (using [Argon2](https://www.argon2.com) KDF) secured with an user-provided password. Why not just use KMS? [No reason.](https://i.imgur.com/T5cKGDr.jpg) 19 | 20 | ## Installation 21 | This project requires go 1.11.0 or newer. Assuming the go binary is available in $PATH and a valid $GOPATH exists: 22 | 23 | ```sh 24 | $ go get -d github.com/mgren/ogive 25 | $ cd $GOPATH/src/github.com/mgren/ogive 26 | $ make 27 | $ make install 28 | ``` 29 | 30 | ## Configuring AWS 31 | * A separate, dedicated bucket for ogive is recommended, but not necessary. The list command will skip any files whose Content-Type is not application/x-ogive. 32 | * Ogive uploads objects with private ACLs. Nevertheless, bucket configuration should block uploading public objects and remove public access (those are the default and recommended settings when creating an S3 bucket in the AWS Console). 33 | * Enabling bucket encryption is not necessary, as stored data is already encrypted. There are, however, no arguments against doing it - the locally stored key and the key used by S3 will be different. 34 | * **It is essential to configure a lifecycle rule that automatically cancels incomplete multipart uploads.** Ogive will not keep track of failed uploads in case a critical network failure or other issues prevent upload completion. 35 | 36 | The following is a minimal IAM Policy for ogive: 37 | ``` 38 | { 39 | "Version": "2012-10-17", 40 | "Statement": [ 41 | { 42 | "Effect": "Allow", 43 | "Action": [ 44 | "s3:PutObject", 45 | "s3:GetObject", 46 | "s3:RestoreObject" 47 | ], 48 | "Resource": "arn:aws:s3:::BUCKET_NAME/*" 49 | }, 50 | { 51 | "Effect": "Allow", 52 | "Action": "s3:ListBucket", 53 | "Resource": "arn:aws:s3:::BUCKET_NAME" 54 | } 55 | ] 56 | } 57 | ``` 58 | 59 | ## Examples 60 | #### Basic Example 61 | ```sh 62 | $ ogive init 63 | ... 64 | # enter profile information 65 | $ ogive put example.dat 66 | ... 67 | $ ogive list 68 | SIZE DATE STATUS STORAGE ID FILENAME 69 | ---------- ------------ ------ ---------- -------- 70 | 123.4 MiB 2019-May-05 DEEPS 5PqBqHILQoIckevFn5EbXX1yGrgXIgAY2UvWT5ruYD-YOCGNMGEM Example1.dat 71 | 123.4 GiB 2019-May-03 READY 5hC4jbOhHGpF-j5wIO9aLkgRAAWw9wzpvpN9pvGdbjX.1wPAHJQe Example2.dat 72 | 123.4 KiB 2019-May-04 RECOV liSROji4FYcW6MVr0fzrdeNJnOHeOL7qRsHHY88cpTmnDQDRZ99M Example3.dat 73 | 74 | # find example.dat on the list to determine its storage_id 75 | $ ogive restore 76 | ... 77 | $ ogive head 78 | READY 79 | # Bulk Restore usually completes within 48 hours 80 | $ ogive get /directory/to/save-in 81 | ... 82 | ``` 83 | 84 | #### Writing To/From a Block Device 85 | ```sh 86 | $ ogive put /dev/sdf 87 | ... 88 | $ ogive restore /dev # --output=sdg to write to sdg instead of sdf 89 | ``` 90 | 91 | #### Supplying a Password From Stdin 92 | ```sh 93 | $ bash securely-retrieve-password-and-write-to-stdout.sh | ogive put example.dat 94 | ``` 95 | 96 | #### Restore All Archives 97 | ```sh 98 | $ bash securely-retrieve-password-and-write-to-stdout.sh | ogive list | \ 99 | > awk 'NR>2 && $4 == "DEEPS" {print $5}' | while read id; 100 | > do bash securely-retrieve-password-and-write-to-stdout.sh | ogive restore $id; 101 | > done 102 | ``` 103 | 104 | ## Usage 105 | All of the following information is also available as a manpage. 106 | 107 | Global flags available for any subcommand: 108 | ``` 109 | -h, --help help for ogive 110 | -p, --profile string Location of Ogive profile file. (default "$HOME/.ogive") 111 | ``` 112 | 113 | ### get 114 | Download and decrypt file, saving it under its original filename. 115 | 116 | ```sh 117 | $ ogive get [flags] 118 | ``` 119 | 120 | ##### flags 121 | ``` 122 | -o, --output string Override destination filename. 123 | ``` 124 | 125 | ### head 126 | Head a specific Ogive file on S3 and retrieve its current archival status. 127 | 128 | ```sh 129 | $ ogive head [flags] 130 | ``` 131 | 132 | Following exit codes and file statuses are possible for this command: 133 | 134 | | Code | Description | 135 | | ------ | ------ | 136 | | 0 | file available for download | 137 | | 1 | error occured | 138 | | 2 | file not available for download | 139 | 140 | | Status | Description | 141 | | ------ | ------ | 142 | | IDEEPS | file is in DEEP_ARCHIVE state with no restore job pending | 143 | | RECOV | file is currently being restored | 144 | | READY | file has been restored into STANDARD storage and is ready for downloading | 145 | | \?\?\?\?\? | file state is unrecognized | 146 | 147 | ### init 148 | Set up an Ogive profile, including generating the master key and providing the S3 bucket location. 149 | 150 | ```sh 151 | $ ogive init [flags] 152 | ``` 153 | 154 | ##### flags 155 | ``` 156 | -r, --reinit Reinitialize an existing profile to change password and/or AWS keys. Old profile is stored as ".bak". 157 | ``` 158 | 159 | ### list 160 | Lists all Ogive archives in an S3 bucket. Lists entire bucket and HEADs each file to retrieve metadata. 161 | 162 | ```sh 163 | $ ogive list [flags] 164 | ``` 165 | 166 | ### put 167 | Encrypt and upload file to S3 Glacier Deep Archive. 168 | 169 | ```sh 170 | $ ogive put [flags] 171 | ``` 172 | 173 | ### restore 174 | Initiate file recovery from Deep Archive. Bulk Restore is used. Use _head_ command to verify when the file becomes ready for download. 175 | 176 | ```sh 177 | $ ogive restore [flags] 178 | ``` 179 | 180 | ##### flags 181 | ``` 182 | -t, --lifetime int Specifies the number of days to retain the restored object before returning it to Deep Archive. (default 1) 183 | ``` 184 | 185 | ## Notes 186 | #### Progress Reporting 187 | When running the _get_ or _put_ commands, ogive will report an approximate progress. For file uploads this is highly inaccurate for objects smaller than 550 MiB. This is because aws-sdk-go lacks progress reporting in its s3manager, so this program relies on the amount of bytes read by the manager instead. Users should always wait for the program to exit gracefully instead of relying solely on the progress bar. 188 | 189 | #### Multiple Backup Versions 190 | Since each _put_ generates an unique nonce and derives an unique name, the probability of name collision in storage is basically zero. This allows to _put_ the same file multiple times at different points in time to create multiple backups. 191 | 192 | #### About the profile file 193 | Since the profile file stores the master key, its loss or corruption renders all backups created with it unrecoverable. A copy of the profile file on a separate medium is essential. An additional, physical backup of the profile file such as [PaperBack](http://ollydbg.de/Paperbak/) is suggested. 194 | 195 | #### Broken Downloads/uploads 196 | Currently ogive does not support any form of download/upload resumption. One file operation must complete in one run. This is unlikely to change, at least until [sio supports WriteAt and ReadAt](https://github.com/minio/sio/issues/13). With that in place, the upload/download code would need to be rewritten to replace s3manager with manual control of part download/upload in order to ensue all operations are aligned to the underlying cipher block size. 197 | 198 | ## Built With 199 | * [sio](https://github.com/minio/sio) - Go implementation of the Data At Rest Encryption (DARE) format 200 | * [memguard](https://github.com/awnumar/memguard) - Secure software enclave for storage of sensitive information in memory 201 | * [cobra](https://github.com/spf13/cobra) - A Commander for modern Go CLI interactions 202 | * [aws-sdk-go](https://aws.amazon.com/sdk-for-go/) - The official AWS SDK for the Go programming language 203 | 204 | ## Contributing 205 | Please refer to [CONTRIBUTING.md](CONTRIBUTING.md) 206 | 207 | ## License 208 | This software is released under [the Unlicense](https://unlicense.org). --------------------------------------------------------------------------------