├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── api.go ├── api_test.go ├── base_test.go ├── cli ├── birthday │ ├── birthday.log.xz │ └── main.go ├── seof │ └── seof.go └── soaktest │ └── main.go ├── crypto ├── misc.go ├── scrypt.go └── scrypt_test.go ├── crypto_test.go ├── go.mod ├── go.sum ├── structs.go └── structs_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.soak 3 | *.seof 4 | 5 | /soaktest 6 | /seof 7 | 8 | password 9 | coverage.out 10 | .idea/ 11 | 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v1.0.1 4 | 2023-06-30 5 | 6 | ### Notes 7 | - Updated x/crypto to latest version 8 | - Updated Hashicorp LRU library to latest version that does not require generics 9 | 10 | ## v1.0.0 11 | 2021-01-05 12 | 13 | ### Notes 14 | 15 | - Feature complete 16 | - Production ready (as per license, you are using it at your own risk.) 17 | - Multithreading safe 18 | - Quality assurance: 19 | - Unit tests covers ~ 87% of code 20 | - Extensive soak test, including multithreading tests and multiple (mis)aligned access patterns 21 | - No linting issues 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Eduardo ES Riccardi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | all: clean build test coverage release 3 | 4 | clean: 5 | go clean -testcache -testcache 6 | rm -f seof soaktest 7 | 8 | build: 9 | go build ./... 10 | 11 | test: 12 | go test ./... 13 | 14 | coverage: 15 | go test -cover -coverprofile=coverage.out ./... 16 | go tool cover -func=coverage.out 17 | 18 | release: 19 | go build -o seof cli/seof/seof.go 20 | go build -o soaktest cli/soaktest/main.go 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go seof: Simple Encrypted os.File 2 | 3 | Encrypted drop-in replacement of golang' [`os.File`](https://golang.org/pkg/os/#File), the file stored in disk will be 4 | encrypted, and the resulting type can be used anywhere an [`os.File`](https://golang.org/pkg/os/#File) could be used. 5 | i.e. it can be both sequentially and randomly read and write, at any file position for any amount of bytes, can be 6 | truncate, seek, stats, etc. i.e. [`Read`](https://golang.org/pkg/os/#File.Read), 7 | [`ReadAt`](https://golang.org/pkg/os/#File.ReadAt), 8 | [`WriteAt`](https://golang.org/pkg/os/#File.WriteAt), 9 | [`Seek`](https://golang.org/pkg/os/#File.Seek), 10 | [`Truncate`](https://golang.org/pkg/os/#File.Truncate), etc. 11 | 12 | It derives a file-wide key using [scrypt](http://www.tarsnap.com/scrypt.html) with a provided string password, the file 13 | is sliced into blocks of n bytes (decided at creation time.). Each block is encrypted and sealed using three AES256/GCM 14 | envelops, one inside the other, with three different keys and nonces achieving 15 | both [confidentiality and authenticity](https://en.wikipedia.org/wiki/Authenticated_encryption). File wide integrity is 16 | warrantied by signing blocks and avoiding empty sparse blocks. 17 | 18 | Current Version: [v1.0.0](https://github.com/kuking/seof/tree/v1.0.0), changelog [here](CHANGELOG.md). 19 | 20 | Example 21 | ------- 22 | 23 | Snippet taken from [base_test.go](base_test.go). Check the test files for more examples, i.e. Seek, Truncate, Stats, 24 | etc. 25 | 26 | ``` 27 | password := "this is a very long password nobody should know about" 28 | BEBlockSize := 1024 29 | data := crypto.RandBytes(BEBlockSize*10) 30 | 31 | // create, write, close. 32 | f, err := seof.CreateExt("encrypted.seof", password, BEBlockSize, 1) 33 | assertNoErr(err, t) 34 | 35 | n, err := f.Write(data) 36 | assertNoErr(err, t) 37 | if n != len(data) { 38 | t.Fatal("did not write the whole buffer") 39 | } 40 | err = f.Close() 41 | assertNoErr(err, t) 42 | 43 | // open, read, close. 44 | f, err = seof.OpenExt("encrypted.seof", password, 1) 45 | assertNoErr(err, t) 46 | readBuf := make([]byte, BEBlockSize*15) // bigger, purposely 47 | n, err = f.Read(readBuf) 48 | if n != len(data) { 49 | t.Fatal("It did not read fully") 50 | } 51 | if !bytes.Equal(data, readBuf[0:n]) { 52 | t.Fatal("read error, does not equals to initial write") 53 | } 54 | err = f.Close() 55 | assertNoErr(err, t) 56 | 57 | ``` 58 | 59 | CLI 60 | --- 61 | 62 | Usage, it can encrypt/decrypt/inspect files' metadata from CLI: 63 | 64 | ```$ ./seof ed@luxuriance 65 | Usage of ./seof: seof file utility 66 | 67 | -e encrypt (default: to decrypt) 68 | -h Show usage 69 | -i show seof encrypted file metadata 70 | -p string 71 | password file 72 | -s uint 73 | block size (default: 1024) 74 | -scrypt string 75 | Encrypting Scrypt parameters: min, default, better, max (default "default") 76 | 77 | NOTES: 78 | - Password must be provided in a file. Command line is not secure in a multi-user host. 79 | - When encrypting, contents have to be provided via stdin pipe, decrypted output will be via stdout. 80 | - Scrypt parameters target times in modern CPUs (2021): min>20ms, default>600ms, better>5s, max>9s 81 | 82 | Examples: 83 | $ cat file | seof -e -p @password_file file.seof 84 | $ seof -p @password_file file.seof > file 85 | $ seof -i -p @password_file file.seof 86 | ``` 87 | 88 | Inspecting metadata for an encrypted file: 89 | 90 | ``` 91 | $ /seof -p password -i file.seof ed@luxuriance 92 | File Name: file.seof 93 | Modification Time: 2021-01-03 13:53:55.698769333 +0000 GMT 94 | File Mode: -rw-r--r-- 95 | Content Size: 247086468 bytes 96 | File Size On Disk: 268321756 bytes 97 | Encryption Overhead: 8.59% 98 | Content Block Size: 1024 bytes 99 | Encrypted Block Size: 1112 bytes 100 | Total Blocks Writen: 241298 (= unique nonces) 101 | SCrypt Preset: Maximum (>9s) 102 | SCrypt Parameters: N=524288, R=64, P=1, keyLength=96, salt= 103 | e036b1c8443913266fa514404dc56fa2603e5215136dfe7b83cb2149eb924dc1 104 | 40cc023e94fcde57b4ca095e81b3ab94331a9defbb03187b4a1761ee37179402 105 | f206d9f768034a9cb7d42e9355f55876c4ffb8710da32d56c6b384101a3d13f4 106 | ``` 107 | 108 | Performance 109 | ----------- 110 | There is no performance overhead beyond the encryption primitives. Internally, `seof` holds multiple unencrypted blocks 111 | in memory, unbuffered reading and writing should not incur in any extra encryption work, and the typical sequential 112 | reads and writes should be performant independently of the access pattern. 113 | 114 | Finally, encryption occurs in blocks, so changing just one byte would require encrypting and storing a whole block (i.e. 115 | 10kb). You want to tune the quantity of in-memory blocks when opening the file; and the block size when creating it. 116 | 117 | ### CLI sequential encryption/decryption performance 118 | 119 | (MacBook Pro (13-inch, 2018, Four Thunderbolt 3 Ports), 2.7 GHz Quad-Core Intel Core i7) 120 | 121 | ``` 122 | $ cat ~/Downloads/debian-10.5.0-amd64-netinst.iso | pv | ./seof -p password -e debian-10.5.0-amd64-netinst.iso.seof 123 | 349MiB 0:00:06 [50.8MiB/s] [ <=> ] 124 | 125 | $ ./seof -p password debian-10.5.0-amd64-netinst.iso.seof | pv > debian-10.5.0-amd64-netinst.iso ed@luxuriance 126 | 349MiB 0:00:02 [ 132MiB/s] [ <=> ] 127 | ``` 128 | 129 | File Structure 130 | -------------- 131 | 132 | - Header: (128 bytes, 120 used) 133 | - uint64 Magic 134 | - [96]byte Script salt 135 | - uint32 Scrypt parameters: N, R, P. 136 | - uint32 Disk block size 137 | - [8]byte zeros (verified on open) 138 | - A block: 139 | - [36]byte: nonce 140 | - uint32: cipherText length 141 | - [disk-block-size]byte: CGM stream 142 | - the additional data for the AEAD is an uint64 holding the block number (verified) 143 | - Special block 0: 144 | - uint64: File size 145 | - uint32: Disk block size (must eq to the header) 146 | - uint32: un-encrypted block size 147 | - uint64: written blocks (as in number of unique nonces generated) 148 | - []byte: Further metadata expansion 149 | 150 | Testing 151 | ------- 152 | Code is extensively tested and there is a soak test suite that tests multiple access patterns (i.e. misaligned reads, 153 | writes, multi-blocks ops, sub-block ops, concurrency, etc). 154 | 155 | ``` 156 | $ ./soaktest ed@luxuriance 157 | soaktest: seof soak test, creates a native file and a seof encrypted file. 158 | applies many different IO operations equally to both files and verifies both behave similar. You want a fast disk (NVMe). 159 | 160 | 1. Creating 2 x 256MB files: native.soak, seof.soak 161 | 2. Writing 256MB of [0x00, 0x01, 0x02, ... 0xff] in: native.soak, seof.soak 162 | .................................................. done 163 | 3.1. Fully comparing files, using read_chunk_size=1 164 | .................................................. done 165 | 3.2. Fully comparing files, using read_chunk_size=2 166 | .................................................. done 167 | 3.3. Fully comparing files, using read_chunk_size=3 168 | .................................................. done 169 | 3.4. Fully comparing files, using read_chunk_size=4 170 | .................................................. done 171 | 172 | [...] 173 | 174 | .................................................. done 175 | 3.16. Fully comparing files, using read_chunk_size=16 176 | .................................................. done 177 | 3.17. Fully comparing files, using read_chunk_size=256 178 | .................................................. done 179 | 3.18. Fully comparing files, using read_chunk_size=512 180 | .................................................. done 181 | 3.19. Fully comparing files, using read_chunk_size=924 182 | .................................................. done 183 | 3.20. Fully comparing files, using read_chunk_size=1023 184 | .................................................. done 185 | 3.21. Fully comparing files, using read_chunk_size=1024 186 | .................................................. done 187 | 3.22. Fully comparing files, using read_chunk_size=1025 188 | .................................................. done 189 | 3.23. Fully comparing files, using read_chunk_size=1124 190 | .................................................. done 191 | 3.24. Fully comparing files, using read_chunk_size=2048 192 | .................................................. done 193 | 3.25. Fully comparing files, using read_chunk_size=3072 194 | .................................................. done 195 | 3.26. Fully comparing files, using read_chunk_size=4096 196 | .................................................. done 197 | 3.27. Fully comparing files, using read_chunk_size=4095 198 | .................................................. done 199 | 3.28. Fully comparing files, using read_chunk_size=4097 200 | .................................................. done 201 | 4.1.1. Rewriting wholy using chunk_size=1 202 | .................................................. done 203 | 4.1.2. Verifying (fast, using chunk_size=1024) 204 | .................................................. done 205 | 4.2.1. Rewriting wholy using chunk_size=2 206 | .................................................. done 207 | 4.2.2. Verifying (fast, using chunk_size=1024) 208 | .................................................. done 209 | 4.3.1. Rewriting wholy using chunk_size=3 210 | .................................................. done 211 | 212 | [...] 213 | 214 | 4.22.1. Rewriting wholy using chunk_size=1025 215 | .................................................. done 216 | 4.22.2. Verifying (fast, using chunk_size=1024) 217 | .................................................. done 218 | 4.23.1. Rewriting wholy using chunk_size=1124 219 | .................................................. done 220 | 4.23.2. Verifying (fast, using chunk_size=1024) 221 | .................................................. done 222 | 4.24.1. Rewriting wholy using chunk_size=2048 223 | .................................................. done 224 | 4.24.2. Verifying (fast, using chunk_size=1024) 225 | .................................................. done 226 | 4.25.1. Rewriting wholy using chunk_size=3072 227 | .................................................. done 228 | 4.25.2. Verifying (fast, using chunk_size=1024) 229 | .................................................. done 230 | 4.26.1. Rewriting wholy using chunk_size=4096 231 | .................................................. done 232 | 4.26.2. Verifying (fast, using chunk_size=1024) 233 | .................................................. done 234 | 4.27.1. Rewriting wholy using chunk_size=4095 235 | .................................................. done 236 | 4.27.2. Verifying (fast, using chunk_size=1024) 237 | .................................................. done 238 | 4.28.1. Rewriting wholy using chunk_size=4097 239 | .................................................. done 240 | 4.28.2. Verifying (fast, using chunk_size=1024) 241 | .................................................. done 242 | 5.1. Writing 262144 random chunks of miscelaneous sizes of up to 2048 bytes 243 | ................................................... done 244 | 5.2. Verifying (fast, using chunk_size=1024) 245 | .................................................. done 246 | 6.1. Reading 262144 random chunks of miscelaneous sizes of up to 2048 bytes 247 | ................................................... done 248 | 7.1 Synchronisation: reading native, writing encrypted 1048576 chunks of up to 2048 bytes within 64 concurrent threads 249 | .................................................. done 250 | 7.2. Verifying (fast, using chunk_size=1024) 251 | .................................................. done 252 | 7.3. Synchronisation: reading encryptede 1048576 chunks of up to 2048 bytes within 64 concurrent threads 253 | .................................................. done 254 | 255 | SUCCESS! 256 | 257 | ``` 258 | 259 | Syncronisation 260 | -------------- 261 | Concurrency safety is achieved with a global lock, do not expect optimal concurrent performance. It is safe to do 262 | operations on the same seof File object from multiple concurrent goroutines. 263 | 264 | Attack vectors 265 | -------------- 266 | 267 | - Each time a block is written, a new **random** nonce is generated. Internally, the implementation uses buffers and 268 | will write to disk only when the buffer needs to be flushed (file closed, sync or cache eviction.). It is a 269 | requirement for GCM to never reuse a nonce, or the key can be compromised. We have calculated the odds of duplicating 270 | a nonce (consisting of 12 bytes of randomness), see the details in the 271 | ticket [here](https://github.com/kuking/seof/issues/1) and 272 | the [calculator](https://github.com/kuking/seof/blob/master/cli/birthday/main.go). Long story short, with triple-AES 273 | is practically impossible, with single AES, the chance is 1 in a billion after writing 37TiB into one single file. If 274 | you are worried about those odds, create multiple smaller files, the password can be reused as the scrypt will be 275 | initialised with different salts in each file. To put this number in perspective, the average write-life expectancy 276 | for a modern SSD disk is 500TiB. Finally, special block 0 holds a counter with the number of unique nonces ever 277 | generated. This value can be inspected using the `seof -i` CLI command or via the `Stats` function. 278 | 279 | - The weakest encryption-link is the password string used for generating the 768 bits (96 bytes) of key. A string in 280 | latin characters should have to be approx. 150 characters in order to hold 768 bits of entropy. You have to keep that 281 | in mind. 282 | 283 | - Blocks within the same file can not be shuffled or moved to another block (or even another file) as the AEAD seals 284 | hold the block number in the signed plaintext. This is verified. 285 | 286 | - Replacing a ciphertext block with a previous copy of the same block will not be detected. Like replacing the whole 287 | encrypted file with a previous version of itself. For the user this will experienced as if the file as lost written 288 | data (as the previously stored data would come back). An attacker has to have read/write access to the filesystem, but 289 | will not be able to generate new arbitrary plaintext. It is possible to prevent this attack, at both performance cost 290 | and higher risk of corrupting the file on failure scenarios (i.e. block is flushed, but reference to block 291 | high-water-mark is lost). 292 | 293 | - Most filesystems can handle [sparse files](https://en.wikipedia.org/wiki/Sparse_file). seof supports sparse files, but 294 | read of never written/zeroed blocks is disabled by default to avoid a possible attack (see: XXX flag). User can create 295 | a new file and [`Seek`](https://golang.org/pkg/os/#File.Seek) to any part of it, write a byte, and later read it. 296 | Reading outside the block boundaries of the unique written byte will fail unless explicitly enabled. This is not a 297 | very typical user case. 298 | 299 | __Long explanation__: in order to keep track of blocks holding data, seof should keep a block-written-bitmap. So when 300 | a block is read from the disk and comes completely empty (zeroed, no AEAD seal present), but the block-written-bitmap 301 | accuses it was written previously, it is fair to assume the data has been lost, therefore deemed inconsistent, an IO 302 | error should be raised (it could have been zeroed by a malicious actor, too.). Without this block-written-bitmap, a 303 | zeroed block by a malicious actor and an honest empty blob in a sparse file are indistinguishable, potentially 304 | allowing a "selective block zero-ing attack." and failing the integrity assurances. 305 | 306 | If you really need this assurance, let me know, the block-written-bitmap can be done. 307 | 308 | USAGE 309 | ----- 310 | 311 | - Storing passwords and secrets using auto-generated system+app+user derived key 312 | - Encrypting distributable assets and you need random access reads. (i.e. reading a ZIP File) 313 | - Enhancing encryption in traditional file formats (i.e. 314 | golang' [zip reader](https://golang.org/pkg/archive/zip/#NewReader)) 315 | - Secure long-term storing of files (some people might want to use GPG as it is "proven" to work) 316 | - Keeping usage data away from user's eyes 317 | - Random access on *very* big files, seof supports 64bits files. i.e. efficient and fast random access of >4gb files. 318 | 319 | - Any of the above and you really want to make it future proof, i.e. scenario where AES is degraded. 320 | 321 | TODO 322 | ---- 323 | 324 | - Flag to allow reading empty holes in sparse files as no errors 325 | - Crypto analysis 326 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package seof 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "encoding/binary" 7 | "errors" 8 | "fmt" 9 | lru "github.com/hashicorp/golang-lru" 10 | "github.com/kuking/seof/crypto" 11 | "golang.org/x/crypto/scrypt" 12 | "io" 13 | "os" 14 | "sync" 15 | "time" 16 | ) 17 | 18 | const nonceSize int = 36 19 | 20 | type File struct { 21 | mutex sync.Mutex 22 | file *os.File 23 | pendingErr *error 24 | header Header 25 | blockZero BlockZero 26 | aead [3]cipher.AEAD 27 | cache *lru.Cache 28 | cursor int64 29 | } 30 | 31 | type inMemoryBlock struct { 32 | modified bool 33 | plainText []byte 34 | } 35 | 36 | func (f *File) initialiseCiphers(password string, header *Header) error { 37 | err := header.Verify() 38 | if err != nil { 39 | return err 40 | } 41 | var key []byte 42 | key, err = scrypt.Key([]byte(password), header.ScriptSalt[:], int(header.ScriptN), int(header.ScriptR), int(header.ScriptP), 96) 43 | if err != nil { 44 | return err 45 | } 46 | var block cipher.Block 47 | keySize := 32 48 | for i := 0; i < 3; i++ { 49 | block, err = aes.NewCipher(key[keySize*i : keySize*(i+1)]) 50 | if err != nil { 51 | return err 52 | } 53 | f.aead[i], err = cipher.NewGCM(block) 54 | if err != nil { 55 | return err 56 | } 57 | } 58 | return nil 59 | } 60 | 61 | func (f *File) initialiseCache(size int) error { 62 | var err error 63 | f.cache, err = lru.NewWithEvict(size, f.flushBlock) 64 | return err 65 | } 66 | 67 | func (f *File) flushBlock(blockI interface{}, dataI interface{}) { 68 | blockNo := blockI.(int64) 69 | imb := dataI.(*inMemoryBlock) 70 | if !imb.modified { 71 | return 72 | } 73 | 74 | blockOffset := int64(HeaderLength) + int64(f.blockZero.DiskBlockSize)*blockNo 75 | newOfs, err := f.file.Seek(blockOffset, 0) 76 | if newOfs != blockOffset { 77 | err := errors.New("failed to fseek") 78 | f.pendingErr = &err 79 | return 80 | } 81 | if err != nil { 82 | f.pendingErr = &err 83 | return 84 | } 85 | 86 | if len(imb.plainText) > int(f.blockZero.BEncBlockSize) { 87 | panic(fmt.Sprintf("block %v plainText too big: %v > %v\n", blockNo, len(imb.plainText), int(f.blockZero.BEncBlockSize))) 88 | } 89 | 90 | cipherText, nonce := f.seal(imb.plainText, uint64(blockNo)) 91 | if len(cipherText) > int(f.blockZero.DiskBlockSize) { 92 | panic(fmt.Sprintf("cipherText encoded size too big: %v > %v\n", len(cipherText), f.blockZero.DiskBlockSize)) 93 | } 94 | n, err := f.file.Write(nonce) 95 | if err != nil { 96 | f.pendingErr = &err 97 | return 98 | } 99 | if n != len(nonce) { 100 | err := errors.New("could not write fully to disk") 101 | f.pendingErr = &err 102 | return 103 | } 104 | err = binary.Write(f.file, binary.LittleEndian, uint32(len(cipherText))) 105 | if err != nil { 106 | f.pendingErr = &err 107 | return 108 | } 109 | n, err = f.file.Write(cipherText) 110 | if n != len(cipherText) { 111 | err := errors.New("could not write fully to disk") 112 | f.pendingErr = &err 113 | return 114 | } 115 | if err != nil { 116 | f.pendingErr = &err 117 | return 118 | } 119 | f.blockZero.BlocksWritten++ 120 | } 121 | 122 | func (f *File) flushBlockZero() { 123 | f.flushBlock(int64(0), &inMemoryBlock{ 124 | modified: true, 125 | plainText: f.blockZero.Bytes(), 126 | }) 127 | } 128 | 129 | func (f *File) getOrLoadBlock(blockNo int64) (*inMemoryBlock, error) { 130 | 131 | if imb, ok := f.cache.Get(blockNo); ok { 132 | return imb.(*inMemoryBlock), nil 133 | } 134 | 135 | blockOffset := int64(HeaderLength) + int64(f.blockZero.DiskBlockSize)*blockNo 136 | seekOfs, err := f.file.Seek(blockOffset, 0) 137 | if err != nil { 138 | return nil, err 139 | } 140 | if seekOfs != blockOffset { 141 | return nil, errors.New("could not seek to block start") 142 | } 143 | 144 | nonce := make([]byte, nonceSize) 145 | n, err := f.file.Read(nonce) 146 | if err == io.EOF { // to help detection of EOF (so new blocks can be created.) 147 | return nil, err 148 | } 149 | if n != nonceSize { 150 | return nil, errors.New("could not read nonce bytes") 151 | } 152 | if err != nil { 153 | return nil, err 154 | } 155 | var cipherTextLen uint32 156 | err = binary.Read(f.file, binary.LittleEndian, &cipherTextLen) 157 | if err != nil { 158 | return nil, err 159 | } 160 | 161 | cipherText := make([]byte, cipherTextLen) 162 | n, err = f.file.Read(cipherText) 163 | if n != int(cipherTextLen) { 164 | return nil, errors.New("could not read cipherText from file") 165 | } 166 | if err != nil { 167 | return nil, err 168 | } 169 | 170 | plainText, err := f.unseal(cipherText, uint64(blockNo), nonce) 171 | if err != nil { 172 | return nil, err 173 | } 174 | imb := inMemoryBlock{ 175 | modified: false, 176 | plainText: plainText, 177 | } 178 | 179 | f.cache.Add(blockNo, &imb) 180 | 181 | return &imb, nil 182 | } 183 | 184 | func (f *File) seal(plainText []byte, blockNo uint64) (cipherText []byte, nonce []byte) { 185 | additional := make([]byte, 8) 186 | binary.LittleEndian.PutUint64(additional, blockNo) 187 | if f.aead[0].NonceSize() * 3 != nonceSize { 188 | panic("unexpected nonce size") 189 | } 190 | nonce = crypto.RandBytes(nonceSize) 191 | cipherText = f.aead[0].Seal(nil, nonce[0:12], plainText, additional) 192 | cipherText = f.aead[1].Seal(nil, nonce[12:24], cipherText, additional) 193 | cipherText = f.aead[2].Seal(nil, nonce[24:36], cipherText, additional) 194 | return 195 | } 196 | 197 | func (f *File) unseal(cipherText []byte, blockNo uint64, nonce []byte) (plainText []byte, err error) { 198 | additional := make([]byte, 8) 199 | binary.LittleEndian.PutUint64(additional, blockNo) 200 | // 3 201 | cipherText, err = f.aead[2].Open(nil, nonce[24:36], cipherText, additional) 202 | if err != nil { 203 | return 204 | } 205 | cipherText, err = f.aead[1].Open(nil, nonce[12:24], cipherText, additional) 206 | if err != nil { 207 | return 208 | } 209 | plainText, err = f.aead[0].Open(nil, nonce[0:12], cipherText, additional) 210 | return 211 | } 212 | 213 | func Create(_ string) (*File, error) { 214 | return nil, errors.New("use CreateExt") 215 | } 216 | 217 | func Open(_ string) (*File, error) { 218 | return nil, errors.New("use OpenExt") 219 | } 220 | 221 | func OpenFile(_ string, _ int, _ os.FileMode) (*File, error) { 222 | return nil, errors.New("use OpenExt") 223 | } 224 | 225 | func OpenExt(name string, password string, memoryBuffers int) (*File, error) { 226 | if memoryBuffers < 1 || memoryBuffers > 1024 { 227 | return nil, errors.New("memory buffers can be between 1 and 1024") 228 | } 229 | 230 | var err error 231 | file := File{} 232 | header := Header{} 233 | file.file, err = os.Open(name) 234 | if err != nil { 235 | return nil, err 236 | } 237 | err = binary.Read(file.file, binary.LittleEndian, &header) 238 | if err != nil { 239 | return nil, err 240 | } 241 | file.header = header 242 | 243 | err = file.initialiseCiphers(password, &header) 244 | if err != nil { 245 | return nil, err 246 | } 247 | 248 | err = file.initialiseCache(memoryBuffers) 249 | if err != nil { 250 | return nil, err 251 | } 252 | 253 | file.blockZero = BlockZero{ 254 | BEncBlockSize: 0, 255 | DiskBlockSize: header.DiskBlockSize, // only used for block 0 256 | BEncFileSize: 0, 257 | BlocksWritten: 0, 258 | } 259 | imb, err := file.getOrLoadBlock(0) //FIXME: blockZero should not be cached 260 | if err != nil { 261 | return nil, err 262 | } 263 | bz, err := BlockZeroFromBytes(imb.plainText) 264 | if err != nil { 265 | return nil, err 266 | } 267 | file.blockZero = *bz 268 | 269 | return &file, nil 270 | } 271 | 272 | func CreateExt(name string, password string, scryptParams crypto.SCryptParameters, BEBlockSize int, memoryBuffers int) (*File, error) { 273 | if len(password) < 20 { 274 | return nil, errors.New("password should be at least 20 characters long") 275 | } 276 | if BEBlockSize < 1024 || BEBlockSize > 128*1024 { 277 | return nil, errors.New("before encryption block size has to be between 1KB and 128KB") 278 | } 279 | if memoryBuffers < 1 || memoryBuffers > 128 { 280 | return nil, errors.New("memory buffers can be between 1 and 128") 281 | } 282 | 283 | var err error 284 | file := File{} 285 | 286 | // header 287 | header := Header{ 288 | Magic: HeaderMagic, 289 | ScriptSalt: [96]byte{}, 290 | ScriptN: scryptParams.N, 291 | ScriptR: scryptParams.R, 292 | ScriptP: scryptParams.P, 293 | DiskBlockSize: 0, 294 | TailOfZeros: [8]byte{}, 295 | } 296 | copy(header.ScriptSalt[:], crypto.RandBytes(len(header.ScriptSalt))) 297 | header.DiskBlockSize = 2000 // temporarily fixed for initialising ciphers 298 | 299 | err = file.initialiseCiphers(password, &header) 300 | if err != nil { 301 | return nil, err 302 | } 303 | 304 | err = file.initialiseCache(memoryBuffers) 305 | if err != nil { 306 | return nil, err 307 | } 308 | 309 | // calculates encrypted block size 310 | plainTextBlock := crypto.RandBytes(BEBlockSize) 311 | cipherText, _ := file.seal(plainTextBlock, 1) 312 | header.DiskBlockSize = uint32(nonceSize + 4 + len(cipherText)) // 4=length of uint32 for cipherTextLength 313 | file.header = header 314 | 315 | // blockZero 316 | file.blockZero = BlockZero{ 317 | BEncBlockSize: uint32(BEBlockSize), 318 | DiskBlockSize: header.DiskBlockSize, 319 | BEncFileSize: 0, 320 | BlocksWritten: 1, 321 | } 322 | 323 | // writes common headers 324 | file.file, err = os.Create(name) 325 | if err != nil { 326 | return nil, err 327 | } 328 | err = binary.Write(file.file, binary.LittleEndian, &header) 329 | if err != nil { // possible left open file 330 | return nil, err 331 | } 332 | 333 | imb := inMemoryBlock{ 334 | modified: true, 335 | plainText: file.blockZero.Bytes(), 336 | } 337 | file.flushBlock(int64(0), &imb) 338 | 339 | return &file, nil 340 | } 341 | 342 | func (f *File) blockNoForOffset(offset int64) int64 { 343 | block := offset / int64(f.blockZero.BEncBlockSize) 344 | return block + 1 // because block zero is special, so everything is offset +1 345 | } 346 | func (f *File) blockNoForCursor() int64 { 347 | return f.blockNoForOffset(f.cursor) 348 | } 349 | 350 | func (f *File) Write(b []byte) (n int, err error) { 351 | f.mutex.Lock() 352 | defer f.mutex.Unlock() 353 | return f.writeLocked(b) 354 | } 355 | 356 | func (f *File) writeLocked(b []byte) (n int, err error) { 357 | if f.pendingErr != nil { 358 | return 0, *f.pendingErr 359 | } 360 | if len(b) == 0 { 361 | return 0, nil 362 | } 363 | blockNo := f.blockNoForCursor() 364 | var imb *inMemoryBlock 365 | imb, err = f.getOrLoadBlock(blockNo) 366 | 367 | if err != nil && err == io.EOF { 368 | // at the tail of the file, a new block is created 369 | newImb := inMemoryBlock{ 370 | modified: false, 371 | plainText: make([]byte, 0, f.blockZero.BEncBlockSize), 372 | } 373 | imb = &newImb 374 | f.cache.Add(blockNo, imb) 375 | } else if err != nil { 376 | return 0, err 377 | } 378 | 379 | imb.modified = true 380 | // appends zeroes if not intend to write at the beginning of the block, and the block is empty 381 | ofsStart := int(f.cursor % int64(f.blockZero.BEncBlockSize)) 382 | for i := len(imb.plainText); i < ofsStart; i++ { 383 | imb.plainText = append(imb.plainText, 0) 384 | } 385 | 386 | availableInBlock := int(f.blockZero.BEncBlockSize) - ofsStart 387 | if len(b) < availableInBlock { 388 | // b fits wholly in this block 389 | // appends bytes to an potentially half-allocated block 390 | for i := len(imb.plainText); i < ofsStart+len(b); i++ { 391 | imb.plainText = append(imb.plainText, 0) 392 | } 393 | copy(imb.plainText[ofsStart:], b) 394 | f.cursor += int64(len(b)) 395 | if uint64(f.cursor) > f.blockZero.BEncFileSize { 396 | f.blockZero.BEncFileSize = uint64(f.cursor) 397 | } 398 | return len(b), nil 399 | } else { 400 | // B won't fit wholly in this block 401 | imb.plainText = append(imb.plainText[0:ofsStart], b[0:availableInBlock]...) 402 | f.cursor += int64(availableInBlock) 403 | if uint64(f.cursor) > f.blockZero.BEncFileSize { 404 | f.blockZero.BEncFileSize = uint64(f.cursor) 405 | } 406 | n, err := f.writeLocked(b[availableInBlock:]) 407 | return availableInBlock + n, err 408 | } 409 | } 410 | 411 | func (f *File) WriteAt(b []byte, off int64) (n int, err error) { 412 | f.mutex.Lock() 413 | defer f.mutex.Unlock() 414 | _, err = f.seekLocked(off, 0) 415 | if err != nil { 416 | return 417 | } 418 | return f.writeLocked(b) 419 | } 420 | 421 | func (f *File) WriteString(s string) (n int, err error) { 422 | return f.Write([]byte(s)) 423 | } 424 | 425 | func (f *File) Read(b []byte) (n int, err error) { 426 | f.mutex.Lock() 427 | defer f.mutex.Unlock() 428 | return f.readLocked(b) 429 | } 430 | 431 | func (f *File) readLocked(b []byte) (n int, err error) { 432 | if f.pendingErr != nil { 433 | return 0, *f.pendingErr 434 | } 435 | if len(b) == 0 { 436 | return 0, nil 437 | } 438 | if f.cursor >= int64(f.blockZero.BEncFileSize) { 439 | return 0, io.EOF 440 | } 441 | blockNo := f.blockNoForCursor() 442 | imb, err := f.getOrLoadBlock(blockNo) 443 | if err != nil { 444 | return 0, err 445 | } 446 | ofsStart := int(f.cursor % int64(f.blockZero.BEncBlockSize)) 447 | 448 | if len(imb.plainText) != int(f.blockZero.BEncBlockSize) { 449 | // at end of file, we read what we can --- or end of block-ish //XXX: potential bug here 450 | copy(b[:], imb.plainText[ofsStart:]) 451 | if len(b) < len(imb.plainText[ofsStart:]) { 452 | n = len(b) 453 | } else { 454 | n = len(imb.plainText[ofsStart:]) 455 | } 456 | f.cursor += int64(n) 457 | return n, nil 458 | } 459 | 460 | if len(b) < len(imb.plainText)-ofsStart { 461 | // smaller chunk 462 | copy(b[:], imb.plainText[ofsStart:]) 463 | f.cursor += int64(len(b)) 464 | return len(b), nil 465 | } 466 | 467 | // bigger chunk 468 | partial := int(f.blockZero.BEncBlockSize) - ofsStart 469 | copy(b[:], imb.plainText[ofsStart:]) 470 | f.cursor += int64(partial) 471 | if f.cursor == int64(f.blockZero.BEncFileSize) { 472 | // don't recurse, we know we are at the end of the file it will trigger an EOF as trying to read at EOF 473 | return partial, err 474 | } 475 | n, err = f.readLocked(b[partial:]) 476 | return n + partial, err 477 | } 478 | 479 | func (f *File) ReadAt(b []byte, off int64) (n int, err error) { 480 | f.mutex.Lock() 481 | defer f.mutex.Unlock() 482 | _, err = f.seekLocked(off, 0) 483 | if err != nil { 484 | return 485 | } 486 | return f.readLocked(b) 487 | } 488 | 489 | //func (f *File) ReadFrom(r io.Reader) (n int64, err error) { 490 | // panic("i dunno") 491 | //} 492 | 493 | func (f *File) Seek(offset int64, whence int) (ret int64, err error) { 494 | f.mutex.Lock() 495 | defer f.mutex.Unlock() 496 | return f.seekLocked(offset, whence) 497 | } 498 | 499 | func (f *File) seekLocked(offset int64, whence int) (ret int64, err error) { 500 | if f.pendingErr != nil { 501 | return 0, *f.pendingErr 502 | } 503 | newCursor := int64(0) 504 | 505 | switch whence { 506 | case 0: // absolute 507 | newCursor = offset 508 | case 1: // relative 509 | newCursor = f.cursor + offset 510 | case 2: // from EoF 511 | newCursor = int64(f.blockZero.BEncFileSize) - offset 512 | default: 513 | return 0, errors.New("invalid whence value") 514 | } 515 | 516 | if newCursor < 0 { 517 | return 0, errors.New("absolute negative offset position not allowed") 518 | } 519 | f.cursor = newCursor 520 | return f.cursor, nil 521 | } 522 | 523 | func (f *File) Sync() error { 524 | f.mutex.Lock() 525 | defer f.mutex.Unlock() 526 | for _, blockNoI := range f.cache.Keys() { 527 | imbI, ok := f.cache.Get(blockNoI) 528 | if ok { 529 | f.flushBlock(blockNoI, imbI) 530 | } 531 | } 532 | f.flushBlockZero() 533 | return f.file.Sync() 534 | } 535 | 536 | func (f *File) Truncate(size int64) error { 537 | f.mutex.Lock() 538 | defer f.mutex.Unlock() 539 | if f.pendingErr != nil { 540 | return *f.pendingErr 541 | } 542 | if size < 0 || uint64(size) > f.blockZero.BEncFileSize { 543 | return os.ErrInvalid 544 | } 545 | 546 | blockNo := f.blockNoForOffset(size) 547 | 548 | partial := size%int64(f.blockZero.BEncBlockSize) != 0 549 | if partial { 550 | imb, err := f.getOrLoadBlock(blockNo) 551 | if err != nil { 552 | return err 553 | } 554 | imb.plainText = imb.plainText[0 : size%int64(f.blockZero.BEncBlockSize)] 555 | imb.modified = true 556 | } 557 | 558 | if partial { 559 | blockNo++ 560 | } 561 | for _, k := range f.cache.Keys() { 562 | cacheBlockNo := k.(int64) 563 | if blockNo <= cacheBlockNo { 564 | f.cache.Remove(k) 565 | } 566 | } 567 | 568 | f.blockZero.BEncFileSize = uint64(size) 569 | return f.file.Truncate(int64(HeaderLength) + int64(f.blockZero.DiskBlockSize)*blockNo) 570 | } 571 | 572 | func (f *File) Stat() (*FileInfo, error) { 573 | f.mutex.Lock() 574 | defer f.mutex.Unlock() 575 | stats, err := f.file.Stat() 576 | if err != nil { 577 | return nil, err 578 | } 579 | 580 | return &FileInfo{ 581 | name: f.Name(), 582 | size: int64(f.blockZero.BEncFileSize), 583 | eSize: stats.Size(), 584 | mode: stats.Mode(), 585 | time: stats.ModTime(), 586 | sys: stats.Sys(), 587 | diskBlockSize: f.blockZero.DiskBlockSize, 588 | bEncBlockSize: f.blockZero.BEncBlockSize, 589 | blocksWritten: f.blockZero.BlocksWritten, 590 | scryptSalt: f.header.ScriptSalt[:], 591 | scryptN: f.header.ScriptN, 592 | scryptR: f.header.ScriptR, 593 | scryptP: f.header.ScriptP, 594 | }, nil 595 | } 596 | 597 | type FileInfo struct { 598 | name string 599 | size int64 600 | eSize int64 601 | mode os.FileMode 602 | time time.Time 603 | sys interface{} 604 | diskBlockSize uint32 605 | bEncBlockSize uint32 606 | blocksWritten uint64 607 | scryptSalt []byte 608 | scryptN uint32 609 | scryptR uint32 610 | scryptP uint32 611 | } 612 | 613 | func (s FileInfo) Name() string { 614 | return s.name 615 | } 616 | func (s FileInfo) Size() int64 { 617 | return s.size 618 | } 619 | func (s FileInfo) EncryptedSize() int64 { 620 | return s.eSize 621 | } 622 | func (s FileInfo) Mode() os.FileMode { 623 | return s.mode 624 | } 625 | func (s FileInfo) ModTime() time.Time { 626 | return s.time 627 | } 628 | func (_ FileInfo) IsDir() bool { 629 | return false 630 | } 631 | func (s FileInfo) Sys() interface{} { 632 | return s.sys 633 | } 634 | func (s FileInfo) DiskBlockSize() uint32 { 635 | return s.diskBlockSize 636 | } 637 | func (s FileInfo) BEBlockSize() uint32 { 638 | return s.bEncBlockSize 639 | } 640 | func (s FileInfo) BlocksWritten() uint64 { 641 | return s.blocksWritten 642 | } 643 | func (s FileInfo) SCryptParameters() (salt []byte, N, R, P uint32) { 644 | return s.scryptSalt, s.scryptN, s.scryptR, s.scryptP 645 | } 646 | 647 | func (f *File) Close() error { 648 | f.mutex.Lock() 649 | defer f.mutex.Unlock() 650 | if f.pendingErr != nil && f.pendingErr != &os.ErrClosed { 651 | return *f.pendingErr 652 | } 653 | f.cache.Purge() 654 | f.flushBlockZero() 655 | closedErr := os.ErrClosed 656 | f.pendingErr = &closedErr 657 | return f.file.Close() 658 | } 659 | 660 | func (f *File) Name() string { 661 | return f.file.Name() 662 | } 663 | -------------------------------------------------------------------------------- /api_test.go: -------------------------------------------------------------------------------- 1 | package seof 2 | 3 | import ( 4 | "github.com/kuking/seof/crypto" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | const ( 10 | password = "4f760f9e67c61ff63044fa97e0b00fdc96c744a3" 11 | BEBlockSize = 1024 12 | ) 13 | 14 | func TestNoUsableMethods(t *testing.T) { 15 | if _, err := Create("any"); err == nil { 16 | t.Fatal("Create should not work, use CreateExt") 17 | } 18 | if _, err := Open("any"); err == nil { 19 | t.Fatal("Open should not work, use OpenExt") 20 | } 21 | if _, err := OpenFile("any", 0, os.ModeAppend); err == nil { 22 | t.Fatal("OpenFile should not work, use OpenExt") 23 | } 24 | } 25 | 26 | func TestCreateExt_InvalidArguments(t *testing.T) { 27 | for _, password := range []string{"", "1234567890", "1234567890123456789"} { 28 | if _, err := CreateExt("file", password, crypto.MinSCryptParameters, BEBlockSize, 1); err == nil { 29 | t.Fatal("password should be at least 20 characters long") 30 | } 31 | } 32 | for _, blockSize := range []int{-123, -1, 0, 10, 1023, 128*1024 + 1, 256 * 1024} { 33 | if _, err := CreateExt("file", password, crypto.MinSCryptParameters, blockSize, 1); err == nil { 34 | t.Fatal("block size should be: 1kb<=block_size<128kb") 35 | } 36 | } 37 | for _, memBuffers := range []int{-123, -1, 0, 129, 65535} { 38 | if _, err := CreateExt("file", password, crypto.MinSCryptParameters, BEBlockSize, memBuffers); err == nil { 39 | t.Fatal("memory buffers be: 1<=buffers<128") 40 | } 41 | } 42 | _, err := CreateExt("", password, crypto.MinSCryptParameters, BEBlockSize, 1) 43 | if err == nil { 44 | t.Fatal("invalid filename should fail") 45 | } 46 | } 47 | 48 | func TestOpenExt_InvalidArguments(t *testing.T) { 49 | for _, buffers := range []int{-123, -1, 0, 1025, 128 * 1024} { 50 | _, err := OpenExt("file", password, buffers) 51 | if err == nil { 52 | t.Fatal("memory buffers should be checked for bounds") 53 | } 54 | } 55 | _, err := OpenExt("", password, 1) 56 | if err == nil { 57 | t.Fatal("invalid filename should fail") 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /base_test.go: -------------------------------------------------------------------------------- 1 | package seof 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/hex" 7 | "fmt" 8 | "github.com/kuking/seof/crypto" 9 | "io" 10 | "io/ioutil" 11 | "os" 12 | "testing" 13 | ) 14 | 15 | func Test_HappySequentialWriteRead(t *testing.T) { 16 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 17 | defer deferredCleanup(tempFile) 18 | 19 | data := crypto.RandBytes(BEBlockSize*3 + BEBlockSize/3) 20 | 21 | // create, write, close. 22 | f, err := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 1) 23 | assertNoErr(err, t) 24 | n, err := f.Write(data) 25 | assertNoErr(err, t) 26 | if n != len(data) { 27 | t.Fatal("did not write the whole buffer") 28 | } 29 | err = f.Close() 30 | assertNoErr(err, t) 31 | 32 | // open, read, close. 33 | f, err = OpenExt(tempFile.Name(), password, 1) 34 | assertNoErr(err, t) 35 | readBuf := make([]byte, BEBlockSize*5) // bigger, purposely 36 | n, err = f.Read(readBuf) 37 | if n != len(data) { 38 | t.Fatal("It did not read fully") 39 | } 40 | if !bytes.Equal(data, readBuf[0:n]) { 41 | t.Fatal("read error, does not equals to initial write") 42 | } 43 | err = f.Close() 44 | assertNoErr(err, t) 45 | } 46 | 47 | // "trivial" test but necessary during the implementation, also maybe a good safety guard to leave around 48 | func Test_NoPlainTextInDisk(t *testing.T) { 49 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 50 | defer deferredCleanup(tempFile) 51 | 52 | data := crypto.RandBytes(128) 53 | f, _ := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 1) 54 | for i := 0; i < 100; i++ { 55 | _, _ = f.Write(data) 56 | } 57 | _ = f.Close() 58 | 59 | rf, _ := os.Open(tempFile.Name()) 60 | raw, _ := ioutil.ReadAll(rf) 61 | if bytes.Contains(raw, data) { 62 | t.Fatal("encryption might not be working") 63 | } 64 | } 65 | 66 | func Test_ChunkedBigWrite(t *testing.T) { 67 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 68 | defer deferredCleanup(tempFile) 69 | 70 | data := crypto.RandBytes(256) 71 | 72 | // create, write, close. 73 | f, _ := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 1) 74 | 75 | for i := 0; i < 20; i++ { 76 | _, _ = f.Write(data) 77 | } 78 | _ = f.Close() 79 | 80 | // open, read, close. 81 | f, _ = OpenExt(tempFile.Name(), password, 1) 82 | readBuf := make([]byte, 256*20) // the whole thing should fit 83 | n, err := f.Read(readBuf) 84 | assertNoErr(err, t) 85 | if n != len(readBuf) { 86 | t.Fatal("It did not read fully") 87 | } 88 | for i := 0; i < 20; i++ { 89 | if !bytes.Equal(data, readBuf[i*256:(i+1)*256]) { 90 | fmt.Println("BLK:", i, "EXP:", hex.EncodeToString(data)) 91 | fmt.Println("BLK:", i, "GOT:", hex.EncodeToString(readBuf[i*256:(i+1)*256])) 92 | t.Fatal("What was read was not correct what was initially written at chunk", i) 93 | } 94 | } 95 | err = f.Close() 96 | assertNoErr(err, t) 97 | } 98 | 99 | func Test_AnythingOnClosedFileFails(t *testing.T) { 100 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 101 | defer deferredCleanup(tempFile) 102 | 103 | f, err := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 1) 104 | assertNoErr(err, t) 105 | 106 | err = f.Close() 107 | assertNoErr(err, t) 108 | 109 | _, err = f.WriteString("hola") 110 | if err != os.ErrClosed { 111 | t.Fatal() 112 | } 113 | 114 | _, err = f.Write([]byte{1, 2}) 115 | if err != os.ErrClosed { 116 | t.Fatal() 117 | } 118 | _, err = f.WriteAt([]byte{1, 2}, 123) 119 | if err != os.ErrClosed { 120 | t.Fatal() 121 | } 122 | 123 | _, err = f.Read([]byte{1, 2}) 124 | if err != os.ErrClosed { 125 | t.Fatal() 126 | } 127 | 128 | _, err = f.ReadAt([]byte{1, 2}, 123) 129 | if err != os.ErrClosed { 130 | t.Fatal() 131 | } 132 | 133 | _, err = f.Seek(0, 0) 134 | if err != os.ErrClosed { 135 | t.Fatal() 136 | } 137 | 138 | err = f.Close() 139 | if err != os.ErrClosed { 140 | t.Fatal() 141 | } 142 | 143 | err = f.Truncate(0) 144 | if err != os.ErrClosed { 145 | t.Fatal() 146 | } 147 | } 148 | 149 | func Test_ClosingAnErroredSoefIsOK(t *testing.T) { 150 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 151 | defer deferredCleanup(tempFile) 152 | 153 | f, err := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 1) 154 | assertNoErr(err, t) 155 | for i := 0; i < 1024; i++ { // so it is bigger than 1 buffer 156 | _, err = f.WriteString("HELLO") 157 | assertNoErr(err, t) 158 | } 159 | 160 | _ = f.file.Close() // this will trigger an error on the following read as the underlying file is close 161 | 162 | _, err = f.Seek(0, 0) 163 | assertNoErr(err, t) 164 | b := make([]byte, 128) 165 | _, err = f.Read(b) 166 | if err == nil { 167 | t.Fatal() 168 | } 169 | 170 | f.file, _ = os.Open(tempFile.Name()) // but won't trigger a second error on close. .. a bit hacky 171 | 172 | err = f.Close() 173 | assertNoErr(err, t) 174 | } 175 | 176 | func TestFile_Name(t *testing.T) { 177 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 178 | defer deferredCleanup(tempFile) 179 | f, _ := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 1) 180 | 181 | if f.Name() != tempFile.Name() { 182 | t.Fatal() 183 | } 184 | _ = f.Close() 185 | } 186 | 187 | func TestFile_Seek(t *testing.T) { 188 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 189 | defer deferredCleanup(tempFile) 190 | 191 | f, err := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 10) // 10 is important to buffers are left in memory 192 | assertNoErr(err, t) 193 | for i := 0; i < 1024; i++ { // so it is bigger than 1 buffer 194 | _, err = f.WriteString("HELLO") 195 | assertNoErr(err, t) 196 | } 197 | n, err := f.Seek(1000, 0) 198 | assertNoErr(err, t) 199 | if n != 1000 || n != f.cursor { 200 | t.Fatal() 201 | } 202 | n, err = f.Seek(50, 1) 203 | assertNoErr(err, t) 204 | if n != 1050 || n != f.cursor { 205 | t.Fatal() 206 | } 207 | n, err = f.Seek(50, 2) 208 | assertNoErr(err, t) 209 | if n != (5*1024)-50 || n != f.cursor { 210 | t.Fatal() 211 | } 212 | n, err = f.Seek(-25, 1) 213 | assertNoErr(err, t) 214 | if n != (5*1024)-75 || n != f.cursor { 215 | t.Fatal() 216 | } 217 | n, err = f.Seek(1_000_000_000_000, 0) 218 | assertNoErr(err, t) 219 | if n != f.cursor { 220 | t.Fatal() 221 | } 222 | 223 | n, err = f.Seek(-25, 0) 224 | if err == nil { 225 | t.Fatal() 226 | } 227 | n, err = f.Seek(12, 123) 228 | if err == nil { 229 | t.Fatal() 230 | } 231 | n, err = f.Seek(-1_000_000_000_001, 1) 232 | if err == nil { 233 | t.Fatal() 234 | } 235 | } 236 | 237 | func TestFile_Truncate(t *testing.T) { 238 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 239 | defer deferredCleanup(tempFile) 240 | 241 | f, err := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 10) // 10 is important to buffers are left in memory 242 | assertNoErr(err, t) 243 | for i := 0; i < 1024; i++ { // so it is bigger than 1 buffer 244 | _, err = f.WriteString("HELLO") 245 | assertNoErr(err, t) 246 | } 247 | 248 | // too far away 249 | err = f.Truncate(1024 * 1024) 250 | if err != os.ErrInvalid { 251 | t.Fatal() 252 | } 253 | 254 | // negative 255 | err = f.Truncate(-123) 256 | if err != os.ErrInvalid { 257 | t.Fatal() 258 | } 259 | 260 | // block aligned 261 | err = f.Truncate(BEBlockSize * 4) 262 | if err != nil { 263 | t.Fatal(err) 264 | } 265 | 266 | stats, err := tempFile.Stat() 267 | assertNoErr(err, t) 268 | if stats == nil { 269 | t.Fatal() 270 | } 271 | exp := int64(HeaderLength) + int64(5*f.blockZero.DiskBlockSize) // 4+1=5 because block-zero 272 | if stats.Size() != exp { 273 | t.Fatal("seems it did not truncate at the right place", stats.Size(), "!=", exp) 274 | } 275 | 276 | big := make([]byte, BEBlockSize*10) 277 | _, _ = f.Seek(0, 0) 278 | n, err := f.Read(big) 279 | assertNoErr(err, t) 280 | if n != 4*BEBlockSize { 281 | t.Fatal() 282 | } 283 | 284 | // Cut half-way block 285 | newLength := BEBlockSize + BEBlockSize/2 286 | err = f.Truncate(int64(newLength)) 287 | _, _ = f.Seek(0, 0) 288 | n, err = f.Read(big) 289 | assertNoErr(err, t) 290 | if n != newLength { 291 | t.Fatal() 292 | } 293 | 294 | // for sure, after closing the file ... no buffers left hanging 295 | err = f.Close() 296 | assertNoErr(err, t) 297 | stats, err = tempFile.Stat() 298 | assertNoErr(err, t) 299 | if stats == nil { 300 | t.Fatal() 301 | } 302 | if stats.Size() != int64(HeaderLength)+int64(3*f.blockZero.DiskBlockSize) { // +1 for blockzero 303 | t.Fatal("seems it did not truncate at the right place") 304 | } 305 | } 306 | 307 | func TestFile_SparseFile(t *testing.T) { 308 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 309 | defer deferredCleanup(tempFile) 310 | 311 | f, err := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 10) // 10 is important to buffers are left in memory 312 | assertNoErr(err, t) 313 | 314 | // seeking far away, reading and writting ... works 315 | s, err := f.Seek(1_000_000_000, 0) 316 | assertNoErr(err, t) 317 | if s != 1_000_000_000 { 318 | t.Fatal() 319 | } 320 | n, err := f.WriteString("Hello") 321 | assertNoErr(err, t) 322 | if n != 5 { 323 | t.Fatal() 324 | } 325 | s, err = f.Seek(1_000_000_000, 0) 326 | assertNoErr(err, t) 327 | if s != 1_000_000_000 { 328 | t.Fatal() 329 | } 330 | b := make([]byte, 100) 331 | n, err = f.Read(b) 332 | assertNoErr(err, t) 333 | if n != 5 { 334 | t.Fatal() 335 | } 336 | err = f.Sync() 337 | assertNoErr(err, t) 338 | 339 | // seeking in the middle, where there is no data, should fail with crypto 340 | s, err = f.Seek(-500_000_000, 1) 341 | assertNoErr(err, t) 342 | if s != 500_000_005 { 343 | t.Fatal() 344 | } 345 | n, err = f.Read(b) 346 | if n != 0 || err == nil { 347 | t.Fatal() 348 | } 349 | if err.Error() != "cipher: message authentication failed" { 350 | t.Fatal() 351 | } 352 | } 353 | 354 | func TestFile_Stat(t *testing.T) { 355 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 356 | defer deferredCleanup(tempFile) 357 | 358 | f, err := CreateExt(tempFile.Name(), password, crypto.MinSCryptParameters, BEBlockSize, 10) // 10 is important to buffers are left in memory 359 | assertNoErr(err, t) 360 | for i := 0; i < 1024; i++ { // so it is bigger than 1 buffer 361 | _, err = f.WriteString("HELLO") 362 | assertNoErr(err, t) 363 | } 364 | 365 | stats, err := f.Stat() 366 | assertNoErr(err, t) 367 | 368 | if stats.Size() != 1024*5 { 369 | t.Fatal("stats.size should return before encryption size") 370 | } 371 | 372 | if stats.Name() != f.Name() || stats.IsDir() { 373 | t.Fatal() 374 | } 375 | 376 | if stats.DiskBlockSize() != 1112 || stats.BEBlockSize() != 1024 || stats.BlocksWritten() != 2 || stats.EncryptedSize() != 240 { 377 | t.Fatal() 378 | } 379 | 380 | salt, n, r, p := stats.SCryptParameters() 381 | if len(salt) != 96 || 382 | crypto.MinSCryptParameters.N != n || 383 | crypto.MinSCryptParameters.P != p || 384 | crypto.MinSCryptParameters.R != r { 385 | t.Fatal() 386 | } 387 | 388 | tempFileStats, err := tempFile.Stat() 389 | assertNoErr(err, t) 390 | 391 | // || tempFileStats.Sys() != stats.Sys() can't be compared 392 | if tempFileStats.Mode() != stats.Mode() || tempFileStats.ModTime() != stats.ModTime() { 393 | t.Fatal() 394 | } 395 | } 396 | 397 | func TestOpenExt_EmptyFile(t *testing.T) { 398 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 399 | defer deferredCleanup(tempFile) 400 | 401 | f, err := os.Create(tempFile.Name()) 402 | assertNoErr(err, t) 403 | assertNoErr(f.Close(), t) 404 | 405 | _, err = OpenExt(tempFile.Name(), password, 1) 406 | if err != io.EOF { 407 | t.Fatal() 408 | } 409 | } 410 | 411 | func TestOpenExt_InvalidHeader(t *testing.T) { 412 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 413 | defer deferredCleanup(tempFile) 414 | 415 | header := Header{ 416 | Magic: HeaderMagic, 417 | ScriptSalt: [96]byte{}, 418 | ScriptN: crypto.MinSCryptParameters.N, 419 | ScriptR: crypto.MinSCryptParameters.R, 420 | ScriptP: crypto.MinSCryptParameters.P, 421 | DiskBlockSize: 0, 422 | TailOfZeros: [8]byte{}, 423 | } 424 | f, err := os.Create(tempFile.Name()) 425 | assertNoErr(err, t) 426 | assertNoErr(binary.Write(f, binary.LittleEndian, header), t) 427 | assertNoErr(f.Close(), t) 428 | 429 | _, err = OpenExt(tempFile.Name(), password, 1) 430 | if err != nil && err.Error() != "header: invalid disk_block_size" { 431 | t.Fatal() 432 | } 433 | } 434 | 435 | func TestOpenExt_ValidHeader_TruncatedFile(t *testing.T) { 436 | tempFile, _ := ioutil.TempFile(os.TempDir(), "lala") 437 | defer deferredCleanup(tempFile) 438 | 439 | header := Header{ 440 | Magic: HeaderMagic, 441 | ScriptSalt: [96]byte{}, 442 | ScriptN: crypto.MinSCryptParameters.N, 443 | ScriptR: crypto.MinSCryptParameters.R, 444 | ScriptP: crypto.MinSCryptParameters.P, 445 | DiskBlockSize: 1211, 446 | TailOfZeros: [8]byte{}, 447 | } 448 | header.ScriptSalt[0] = 1 449 | f, err := os.Create(tempFile.Name()) 450 | assertNoErr(err, t) 451 | assertNoErr(binary.Write(f, binary.LittleEndian, header), t) 452 | assertNoErr(f.Close(), t) 453 | 454 | _, err = OpenExt(tempFile.Name(), password, 1) 455 | if err != io.EOF { 456 | t.Fatal() 457 | } 458 | } 459 | 460 | func assertNoErr(err error, t *testing.T) { 461 | if err != nil { 462 | t.Fatal(err) 463 | } 464 | } 465 | 466 | func deferredCleanup(file *os.File) { 467 | if file != nil { 468 | _ = os.Remove(file.Name()) 469 | } 470 | } 471 | -------------------------------------------------------------------------------- /cli/birthday/birthday.log.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuking/seof/2307924a307857dafc3c703bf194b35fbe4ce686/cli/birthday/birthday.log.xz -------------------------------------------------------------------------------- /cli/birthday/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "math/big" 7 | ) 8 | 9 | // The following calculates the probability of duplicating a nonce when picked at random (as the current implementation 10 | // does). The current implementation uses 3 nonces of 12 bytes each (for each AES256/GCM cipher). 11 | // 12 | // The probability of picking the same nonce twice (birthday paradox) is calculated for one nonce of 12 bytes (96 bits). 13 | // 14 | // 100M, 0.999999999999936891126951232322755154461737176572294203867910474082896986505554273464571044238582672950519... 15 | // lower than 0.00000000000001, or 1 in 100.000.000.000.000 16 | // 100M blocks of 1K is approx. 95GiB written. 17 | // 18 | // 1258M, 0.99999999999001261703522929514448885033012604685025065706759067812784429829344821159506777018445782306879... 19 | // lower than 0.000000000001, or 1 in 1.000.000.000.000 20 | // 1258M blocks of 1K is approx. 1.17TiB written. 21 | // 22 | // 39805M, 0.9999999900007903323961553855263092697380031246078278537831527269974242016478462970070809347116269575979... 23 | // lower than 0.000000001 or 1 in 1.000.000.000 (1 billion) 24 | // 39805M block of 1K is approx. 37.07TiB written. 25 | // 26 | // You will have to write 37TiB into one file to have a chance of 1 in a billion to have a duplicated nonce, you also 27 | // have to consider it would compromise the first layer of AES256/GCM, there will be two more layers to break. 28 | // 29 | // see: `birthday.log.xz` for the dump of this program after running for about 50hs 30 | 31 | func main() { 32 | const prec = 5000 // bits 33 | const bits = 12 * 8 // the first nonce for the first AES256 34 | const n = 1_000_000_000_000 35 | 36 | days := new(big.Float).SetPrec(prec).SetFloat64(math.Pow(2, bits)) 37 | 38 | fmt.Printf("Precision: %v mantissa bits\n", prec) 39 | fmt.Printf("Days: %.0f\n", days) 40 | fmt.Printf("N: %v\n", n) 41 | 42 | p := new(big.Float).SetPrec(prec).SetInt64(1) 43 | 44 | work := new(big.Float).SetPrec(prec) 45 | iFlt := new(big.Float).SetPrec(prec) 46 | one := new(big.Float).SetPrec(prec).SetInt64(1) 47 | for i := 0; i < n; i++ { 48 | // p *= (days - i) / days 49 | work.Sub(days, iFlt) 50 | work.Quo(work, days) 51 | p.Mul(p, work) 52 | iFlt.Add(iFlt, one) 53 | if i%1_000_000 == 0 { 54 | fmt.Printf("%vM, %.200f\n", i/1000/1000, p) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /cli/seof/seof.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/hex" 5 | "flag" 6 | "fmt" 7 | pwe "github.com/kuking/go-pwentropy" 8 | "github.com/kuking/seof" 9 | "github.com/kuking/seof/crypto" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | ) 14 | 15 | var doEncrypt bool 16 | var passwordFile string 17 | var doHelp bool 18 | var doInfo bool 19 | var blockSize uint 20 | var scryptParamsCli string 21 | 22 | func doArgsParsing() bool { 23 | flag.BoolVar(&doEncrypt, "e", false, "encrypt (default: to decrypt)") 24 | flag.StringVar(&scryptParamsCli, "scrypt", "default", "Encrypting Scrypt parameters: min, default, better, max") 25 | flag.BoolVar(&doInfo, "i", false, "show seof encrypted file metadata") 26 | flag.StringVar(&passwordFile, "p", "", "password file") 27 | flag.UintVar(&blockSize, "s", 1024, "block size") 28 | flag.BoolVar(&doHelp, "h", false, "Show usage") 29 | flag.Parse() 30 | if doHelp || flag.NArg() != 1 { 31 | fmt.Printf("Usage of %v: seof file utility\n\n", os.Args[0]) 32 | flag.PrintDefaults() 33 | fmt.Print(` 34 | NOTES: 35 | - Password must be provided in a file. Command line is not secure in a multi-user host. 36 | - When encrypting, contents have to be provided via stdin pipe, decrypted output will be via stdout. 37 | - Scrypt parameters target times in modern CPUs (2021): min>20ms, default>600ms, better>5s, max>9s 38 | 39 | Examples: 40 | $ cat file | seof -e -p @password_file file.seof 41 | $ seof -p @password_file file.seof > file 42 | $ seof -i -p @password_file file.seof 43 | `) 44 | return false 45 | } 46 | return true 47 | } 48 | 49 | func main() { 50 | 51 | if !doArgsParsing() { 52 | os.Exit(-1) 53 | } 54 | 55 | if passwordFile == "" { 56 | _, _ = os.Stderr.WriteString("password not provided.\n") 57 | os.Exit(-1) 58 | } 59 | 60 | if len(passwordFile) > 1 && passwordFile[0] == '@' { 61 | passwordFile = passwordFile[1:] 62 | } 63 | passwordBytes, err := ioutil.ReadFile(passwordFile) 64 | if err != nil { 65 | panic(err) 66 | } 67 | password := string(passwordBytes) 68 | 69 | entropy := pwe.FairEntropy(password) 70 | if entropy < 96 { 71 | _, _ = os.Stderr.WriteString(fmt.Sprintf("FATAL: Est. entropy for provided password is not enough: %2.2f (minimum: 96)\n\n", entropy)) 72 | password = pwe.PwGen(pwe.FormatEasy, pwe.Strength256) 73 | entropy = pwe.FairEntropy(password) 74 | _, _ = os.Stderr.WriteString(fmt.Sprintf("We have created a password for you with %2.2f bits of entropy \n"+ 75 | "+-------------------------------------------------------+\n"+ 76 | "| %52v |\n"+ 77 | "+-------------------------------------------------------+\n", entropy, password)) 78 | os.Exit(-1) 79 | } 80 | 81 | var scryptParams crypto.SCryptParameters 82 | if doEncrypt { 83 | switch scryptParamsCli { 84 | case "min": 85 | scryptParams = crypto.MinSCryptParameters 86 | case "default": 87 | scryptParams = crypto.RecommendedSCryptParameters 88 | case "better": 89 | scryptParams = crypto.BetterSCryptParameters 90 | case "max": 91 | scryptParams = crypto.MaxSCryptParameters 92 | default: 93 | fmt.Println("SCrypt parameter not recognised:", scryptParamsCli) 94 | os.Exit(-1) 95 | } 96 | 97 | } 98 | 99 | filename := os.Args[len(os.Args)-1] 100 | var ef *seof.File 101 | if doInfo || !doEncrypt { 102 | ef, err = seof.OpenExt(filename, password, 10) 103 | } else { 104 | ef, err = seof.CreateExt(filename, password, scryptParams, int(blockSize), 10) 105 | } 106 | assertNoError(err, "Failed to open file: "+filename+" -- %v") 107 | 108 | if doInfo { 109 | stats, err := ef.Stat() 110 | assertNoError(err, "FATAL: problems doing file stats %v") 111 | 112 | fmt.Printf(" File Name: %v\n", stats.Name()) 113 | fmt.Printf(" Modification Time: %v\n", stats.ModTime()) 114 | fmt.Printf(" File Mode: %v \n", stats.Mode()) 115 | fmt.Printf(" Content Size: %v bytes\n", stats.Size()) 116 | fmt.Printf(" File Size On Disk: %v bytes\n", stats.EncryptedSize()) 117 | fmt.Printf(" Encryption Overhead: %2.2f%%\n", float32(stats.EncryptedSize())*100/float32(stats.Size())-100) 118 | fmt.Printf(" Content Block Size: %v bytes\n", stats.BEBlockSize()) 119 | fmt.Printf("Encrypted Block Size: %v bytes\n", stats.DiskBlockSize()) 120 | fmt.Printf(" Total Blocks Writen: %v (= unique nonces)\n", stats.BlocksWritten()) 121 | var scryptLevel string 122 | salt, n, r, p := stats.SCryptParameters() 123 | if n == crypto.MinSCryptParameters.N && r == crypto.MinSCryptParameters.R && p == crypto.MinSCryptParameters.P { 124 | scryptLevel = "Minimal (>20ms)" 125 | } else if n == crypto.MaxSCryptParameters.N && r == crypto.MaxSCryptParameters.R && p == crypto.MaxSCryptParameters.P { 126 | scryptLevel = "Maximum (>9s)" 127 | } else if n == crypto.RecommendedSCryptParameters.N && r == crypto.RecommendedSCryptParameters.R && p == crypto.RecommendedSCryptParameters.P { 128 | scryptLevel = "Recommended (>600ms)" 129 | } else if n == crypto.BetterSCryptParameters.N && r == crypto.BetterSCryptParameters.R && p == crypto.BetterSCryptParameters.P { 130 | scryptLevel = "Better (>5s)" 131 | } else { 132 | scryptLevel = "Unknown" 133 | } 134 | fmt.Printf(" SCrypt Preset: %v\n", scryptLevel) 135 | fmt.Printf(" SCrypt Parameters: N=%v, R=%v, P=%v, keyLength=96, salt=\n", n, r, p) 136 | hexa := hex.EncodeToString(salt) 137 | fmt.Printf("%69v\n%69v\n%69v\n", hexa[:64], hexa[64:128], hexa[128:]) 138 | 139 | } else if doEncrypt { 140 | _, err = io.Copy(ef, os.Stdin) 141 | } else { 142 | _, err = io.Copy(os.Stdout, ef) 143 | } 144 | assertNoError(err, "FATAL: io error: %v") 145 | 146 | err = ef.Close() 147 | assertNoError(err, "FATAL: could not close the seof file: %v") 148 | } 149 | 150 | func assertNoError(err error, pattern string) { 151 | if err != nil { 152 | _, _ = os.Stderr.WriteString(fmt.Sprintf(pattern+"\n", err)) 153 | os.Exit(-1) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /cli/soaktest/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/hex" 6 | "fmt" 7 | "github.com/kuking/seof" 8 | "github.com/kuking/seof/crypto" 9 | "math/rand" 10 | "os" 11 | "runtime" 12 | ) 13 | 14 | var password = "e924a81d0abd80b4c2ded664c7881a75575d9e45" 15 | 16 | var wholeSize = 256 * 1024 * 1024 17 | var seofBlockSize = 1024 18 | 19 | var misalignedBlockSizes = []int{ 20 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 21 | 256, 22 | 512, 23 | seofBlockSize - 100, 24 | seofBlockSize - 1, 25 | seofBlockSize, 26 | seofBlockSize + 1, 27 | seofBlockSize + 100, 28 | seofBlockSize * 2, 29 | seofBlockSize * 3, 30 | seofBlockSize * 4, 31 | (seofBlockSize * 4) - 1, 32 | (seofBlockSize * 4) + 1} 33 | 34 | var nat *os.File 35 | var enc *seof.File 36 | 37 | func main() { 38 | 39 | fmt.Println(`soaktest: seof soak test, creates a native file and a seof encrypted file. 40 | applies many different IO operations equally to both files and verifies both behave similar. You want a fast disk (NVMe).`) 41 | 42 | if len(os.Args) == 2 && os.Args[1] == "quick" { 43 | fmt.Println("\nNOTE: Running a quick test as per run parameters.") 44 | wholeSize = 1024 * 1024 45 | } 46 | fmt.Println() 47 | 48 | var err error 49 | fmt.Printf("1. Creating 2 x %vMB files: native.soak, seof.soak\n", wholeSize/1024/1024) 50 | nat, err = os.Create("native.soak") 51 | assertErr(err, "creating native.soak") 52 | enc, err = seof.CreateExt("seof.soak", password, crypto.RecommendedSCryptParameters, seofBlockSize, 5) 53 | assertErr(err, "creating seof.soak") 54 | 55 | fmt.Printf("2. Writing %vMB of [0x00, 0x01, 0x02, ... 0xff] in: native.soak, seof.soak\n", wholeSize/1024/1024) 56 | writeFullyUsingChunkSize(seofBlockSize) 57 | 58 | for i, bs := range misalignedBlockSizes { 59 | fmt.Printf("3.%v. Fully comparing files, using read_chunk_size=%v\n", i+1, bs) 60 | fullyCompare(bs) 61 | } 62 | 63 | for i, bs := range misalignedBlockSizes { 64 | fmt.Printf("4.%v.1. Rewriting wholy using chunk_size=%v\n", i+1, bs) 65 | writeFullyUsingChunkSize(bs) 66 | fmt.Printf("4.%v.2. Verifying (fast, using chunk_size=%v)\n", i+1, seofBlockSize) 67 | fullyCompare(seofBlockSize) 68 | } 69 | 70 | fmt.Printf("5.1. Writing %v random chunks of miscelaneous sizes of up to %v bytes\n", wholeSize/1024, seofBlockSize*2) 71 | writeRandomChunks(wholeSize/1024, seofBlockSize*2) 72 | fmt.Printf("5.2. Verifying (fast, using chunk_size=%v)\n", seofBlockSize) 73 | fullyCompare(seofBlockSize) 74 | 75 | fmt.Printf("6.1. Reading %v random chunks of miscelaneous sizes of up to %v bytes\n", wholeSize/1024, seofBlockSize*2) 76 | readingRandomChunks(wholeSize/1024, seofBlockSize*2) 77 | 78 | chunks := wholeSize / 1024 / 16 79 | threads := 64 80 | fmt.Printf("7.1 Synchronisation: reading native, writing encrypted %v chunks of up to %v bytes within %v concurrent threads\n", chunks*threads, seofBlockSize*2, threads) 81 | multithreadingWriteTest(chunks, seofBlockSize*2, threads) 82 | fmt.Printf("7.2. Verifying (fast, using chunk_size=%v)\n", seofBlockSize) 83 | fullyCompare(seofBlockSize) 84 | fmt.Printf("7.3. Synchronisation: reading %v encrypted chunks of up to %v bytes within %v concurrent threads\n", chunks*threads, seofBlockSize*2, threads) 85 | multithreadingReadTest(chunks, seofBlockSize*2, threads) 86 | 87 | fmt.Println("\nSUCCESS!") 88 | _ = nat.Close() 89 | _ = os.Remove(nat.Name()) 90 | _ = enc.Close() 91 | _ = os.Remove(enc.Name()) 92 | } 93 | 94 | func fullyCompare(readChunkSize int) { 95 | natb := make([]byte, readChunkSize) 96 | encb := make([]byte, readChunkSize) 97 | _, _ = nat.Seek(0, 0) 98 | _, _ = enc.Seek(0, 0) 99 | lastDot := 0 100 | ofs := 0 101 | for { 102 | n, err := nat.Read(natb) 103 | if err != nil { 104 | fmt.Println("nat", ofs, n, wholeSize) 105 | } 106 | assertErr(err, "reading native file") 107 | m, err := enc.Read(encb) 108 | if err != nil { 109 | fmt.Println("enc", ofs, m, wholeSize) 110 | } 111 | assertErr(err, "reading encrypted file") 112 | if n != m { 113 | fmt.Printf("\nERROR: It did not read the same quantity of bytes, native=%v seof=%v, ofs=%v\n", n, m, ofs) 114 | os.Exit(-1) 115 | } 116 | ofs += n 117 | if !bytes.Equal(natb, encb) { 118 | fmt.Println("ERROR: Files are not equal.") 119 | fmt.Println("native:", hex.EncodeToString(natb)) 120 | fmt.Println(" seof:", hex.EncodeToString(encb)) 121 | os.Exit(-1) 122 | } 123 | if lastDot < ofs/(wholeSize/50) { 124 | _, _ = os.Stdout.WriteString(".") 125 | _ = os.Stdout.Sync() 126 | lastDot = ofs / (wholeSize / 50) 127 | } 128 | if ofs == wholeSize { 129 | fmt.Println(" done") 130 | break 131 | } 132 | } 133 | } 134 | 135 | func writeFullyUsingChunkSize(cs int) { 136 | _, _ = nat.Seek(0, 0) 137 | _, _ = enc.Seek(0, 0) 138 | lastDot := 0 139 | ofs := 0 140 | for ofs < wholeSize { 141 | 142 | toWrite := cs 143 | if ofs+cs > wholeSize { 144 | toWrite = wholeSize - ofs 145 | } 146 | b := crypto.RandBytes(toWrite) 147 | n, err := nat.Write(b) 148 | assertWritten(n, toWrite) 149 | assertErr(err, "writing native file") 150 | m, err := enc.Write(b) 151 | assertWritten(m, toWrite) 152 | assertErr(err, "writing encrypted file") 153 | if n != m || n != toWrite || m != toWrite { 154 | fmt.Printf( 155 | "\nERROR: It did not write the expected quantity, written native=%v, seof=%v, expected=%v, ofs=%v\n", 156 | n, m, toWrite, ofs) 157 | os.Exit(-1) 158 | } 159 | ofs += toWrite 160 | 161 | if lastDot < ofs/(wholeSize/50) { 162 | _, _ = os.Stdout.WriteString(".") 163 | _ = os.Stdout.Sync() 164 | lastDot = ofs / (wholeSize / 50) 165 | } 166 | } 167 | fmt.Println(" done") 168 | } 169 | 170 | func writeRandomChunks(chunks int, maxSize int) { 171 | lastDot := 0 172 | for chunk := 0; chunk < chunks; chunk++ { 173 | 174 | b := crypto.RandBytes(rand.Int() % maxSize) 175 | ofs := int64(rand.Int() % (wholeSize - len(b))) 176 | 177 | nOfs, err := nat.Seek(ofs, 0) 178 | assertErr(err, "seeking native file") 179 | if nOfs != ofs { 180 | fmt.Printf("ERROR: Couldn't seek to %v in native file.", ofs) 181 | os.Exit(-1) 182 | } 183 | 184 | n, err := nat.Write(b) 185 | assertWritten(n, len(b)) 186 | assertErr(err, "writing native file") 187 | 188 | mOfs, err := enc.Seek(ofs, 0) 189 | assertErr(err, "seeking encrypted file") 190 | if mOfs != ofs { 191 | fmt.Printf("ERROR: Couldn't seek to %v in encrypted file.", ofs) 192 | os.Exit(-1) 193 | } 194 | 195 | m, err := enc.Write(b) 196 | assertWritten(m, len(b)) 197 | assertErr(err, "writing encrypted file") 198 | if n != m || n != len(b) || m != len(b) { 199 | fmt.Printf( 200 | "\nERROR: It did not write the expected quantity, written native=%v, seof=%v, expected=%v, ofs=%v\n", 201 | n, m, len(b), ofs) 202 | os.Exit(-1) 203 | } 204 | 205 | if lastDot < chunk/(chunks/50) { 206 | _, _ = os.Stdout.WriteString(".") 207 | _ = os.Stdout.Sync() 208 | lastDot = chunk / (chunks / 50) 209 | } 210 | } 211 | fmt.Println(" done") 212 | } 213 | 214 | func readingRandomChunks(chunks int, maxSize int) { 215 | lastDot := 0 216 | for chunk := 0; chunk < chunks; chunk++ { 217 | 218 | size := rand.Int() % maxSize 219 | nb := make([]byte, size) 220 | mb := make([]byte, size) 221 | ofs := int64(rand.Int() % (wholeSize - len(nb))) 222 | 223 | n, err := nat.ReadAt(nb, ofs) 224 | assertErr(err, "reading native file") 225 | 226 | m, err := enc.ReadAt(mb, ofs) 227 | assertErr(err, "reading encrypted file") 228 | if n != m || n != len(nb) || m != len(mb) { 229 | fmt.Printf( 230 | "\nERROR: It did not read the expected quantity, read native=%v, seof=%v, expected=%v, ofs=%v\n", 231 | n, m, len(nb), ofs) 232 | os.Exit(-1) 233 | } 234 | 235 | if !bytes.Equal(nb, mb) { 236 | fmt.Println("ERROR: Files are not equal.") 237 | fmt.Println("native:", hex.EncodeToString(nb)) 238 | fmt.Println(" seof:", hex.EncodeToString(mb)) 239 | os.Exit(-1) 240 | } 241 | 242 | if lastDot < chunk/(chunks/50) { 243 | _, _ = os.Stdout.WriteString(".") 244 | _ = os.Stdout.Sync() 245 | lastDot = chunk / (chunks / 50) 246 | } 247 | } 248 | fmt.Println(" done") 249 | } 250 | 251 | func multithreadingWriteTest(chunks int, maxSize int, threads int) { 252 | runtime.GOMAXPROCS(threads * 2) 253 | chunkRead := make(chan int, 5) 254 | for t := 0; t < threads; t++ { 255 | go concurrentReadWriter(chunkRead, chunks, maxSize, t) 256 | } 257 | 258 | lastDot := 0 259 | for t := 0; t < threads*chunks; t++ { 260 | <-chunkRead 261 | if lastDot < t/(threads*chunks/50) { 262 | _, _ = os.Stdout.WriteString(".") 263 | _ = os.Stdout.Sync() 264 | lastDot = t / (threads * chunks / 50) 265 | } 266 | } 267 | fmt.Println(" done") 268 | } 269 | 270 | func concurrentReadWriter(chunkRead chan int, chunks int, maxSize int, threadNo int) { 271 | for chunk := 0; chunk < chunks; chunk++ { 272 | size := rand.Int() % maxSize 273 | nb := make([]byte, size) 274 | ofs := int64(rand.Int() % (wholeSize - len(nb))) 275 | 276 | n, err := nat.ReadAt(nb, ofs) 277 | assertErr(err, "reading native file") 278 | 279 | m, err := enc.WriteAt(nb, ofs) 280 | assertErr(err, "writing encrypted file") 281 | if n != m || n != len(nb) { 282 | fmt.Printf( 283 | "\nERROR: It did not read/write the expected quantity, read native=%v, write seof=%v, expected=%v, ofs=%v (thread no %v)\n", 284 | n, m, len(nb), ofs, threadNo) 285 | os.Exit(-1) 286 | } 287 | chunkRead <- threadNo 288 | } 289 | } 290 | 291 | func multithreadingReadTest(chunks int, maxSize int, threads int) { 292 | runtime.GOMAXPROCS(threads * 2) 293 | chunkRead := make(chan int, 5) 294 | for t := 0; t < threads; t++ { 295 | go concurrentRead(chunkRead, chunks, maxSize, t) 296 | } 297 | 298 | lastDot := 0 299 | for t := 0; t < threads*chunks; t++ { 300 | <-chunkRead 301 | if lastDot < t/(threads*chunks/50) { 302 | _, _ = os.Stdout.WriteString(".") 303 | _ = os.Stdout.Sync() 304 | lastDot = t / (threads * chunks / 50) 305 | } 306 | } 307 | fmt.Println(" done") 308 | } 309 | 310 | func concurrentRead(chunkRead chan int, chunks int, maxSize int, threadNo int) { 311 | for chunk := 0; chunk < chunks; chunk++ { 312 | size := rand.Int() % maxSize 313 | nb := make([]byte, size) 314 | mb := make([]byte, size) 315 | ofs := int64(rand.Int() % (wholeSize - len(nb))) 316 | 317 | n, err := nat.ReadAt(nb, ofs) 318 | assertErr(err, "reading native file") 319 | 320 | m, err := enc.ReadAt(mb, ofs) 321 | assertErr(err, "reading encrypted file") 322 | if n != m || n != len(nb) || m != len(mb) { 323 | fmt.Printf( 324 | "\nERROR: It did not read the expected quantity, read native=%v, read seof=%v, expected=%v, ofs=%v (thread no %v)\n", 325 | n, m, len(nb), ofs, threadNo) 326 | os.Exit(-1) 327 | } 328 | if !bytes.Equal(nb, mb) { 329 | fmt.Println("ERROR: Files are not equal.") 330 | fmt.Println("native:", hex.EncodeToString(nb)) 331 | fmt.Println(" seof:", hex.EncodeToString(mb)) 332 | os.Exit(-1) 333 | } 334 | 335 | chunkRead <- threadNo 336 | } 337 | } 338 | 339 | func assertWritten(n, exp int) { 340 | if n != exp { 341 | fmt.Println("\nERROR: expected to write", exp, "but wrote", n) 342 | os.Exit(-1) 343 | } 344 | } 345 | 346 | func assertErr(err error, desc string) { 347 | if err != nil { 348 | fmt.Println("\nERROR:", desc, "err:", err) 349 | err = nat.Close() 350 | if err != nil { 351 | fmt.Println(err) 352 | } 353 | _ = os.Remove(nat.Name()) 354 | 355 | err = enc.Close() 356 | if err != nil { 357 | fmt.Println(err) 358 | } 359 | _ = os.Remove(enc.Name()) 360 | 361 | os.Exit(-1) 362 | } 363 | } 364 | -------------------------------------------------------------------------------- /crypto/misc.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import "crypto/rand" 4 | 5 | 6 | func RandBytes(size int) []byte { 7 | res := make([]byte, size) 8 | n, err := rand.Read(res) 9 | if n != size || err != nil { 10 | panic("could not generate randomness") 11 | } 12 | return res 13 | } 14 | -------------------------------------------------------------------------------- /crypto/scrypt.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | // Script parameters configuration valid at Dec 2020 for an AMD Ryzen 7 3800X 8-Core Processor 4 | // 5 | // Minimum accepted: a key spec with lower parameters shall not be accepted; processing in less than 21ms 6 | // Currently recommended: parameters targeting at least 250ms for deriving a key 7 | // Maximum accepted: an upper limit defined to avoid potential DoS attacks; maximum set at 4s 8 | 9 | type SCryptParameters struct { 10 | N uint32 11 | R uint32 12 | P uint32 13 | } 14 | 15 | var RecommendedSCryptParameters = SCryptParameters{ //>600ms 16 | N: 1 << 16, 17 | R: 1 << 6, 18 | P: 1 << 0, 19 | } 20 | 21 | var BetterSCryptParameters = SCryptParameters{ //>5s 22 | N: 1 << 18, 23 | R: 1 << 7, 24 | P: 1 << 0, 25 | } 26 | 27 | var MaxSCryptParameters = SCryptParameters{ //>15s 28 | N: 1 << 18, 29 | R: 1 << 8, 30 | P: 1 << 0, 31 | } 32 | 33 | var MinSCryptParameters = SCryptParameters{ //21ms 34 | N: 1 << 14, 35 | R: 1 << 2, 36 | P: 1 << 0, 37 | } 38 | -------------------------------------------------------------------------------- /crypto/scrypt_test.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "golang.org/x/crypto/scrypt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestScryptParameters(t *testing.T) { 10 | keyLen := 32 * 3 11 | password := []byte("some password") 12 | salt := RandBytes(keyLen) 13 | start := time.Now() 14 | count := 2 15 | for i := 0; i < count; i++ { 16 | _, err := scrypt.Key(password, salt, 17 | int(RecommendedSCryptParameters.N), int(RecommendedSCryptParameters.R), int(RecommendedSCryptParameters.P), keyLen) 18 | if err != nil { 19 | t.Error(err) 20 | } 21 | } 22 | duration := time.Now().Sub(start) 23 | scryptMs := duration.Milliseconds() / int64(count) 24 | //fmt.Println("Scrypt parameters taking on average in this CPU:", scryptMs, "ms") 25 | if scryptMs < 600 { 26 | t.Errorf("Scrypt should take at least 600ms, it took %vms -- Time to increase its parameters", scryptMs) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /crypto_test.go: -------------------------------------------------------------------------------- 1 | package seof 2 | 3 | import ( 4 | "github.com/kuking/seof/crypto" 5 | "testing" 6 | ) 7 | 8 | func TestSealOpen(t *testing.T) { 9 | f := File{} 10 | h := givenValidHeader() 11 | _ = f.initialiseCiphers(password, &h) 12 | 13 | plainText := "This is a secret" 14 | cipherText, nonce := f.seal([]byte(plainText), 1234) 15 | recoveredText, err := f.unseal(cipherText, 1234, nonce) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | if string(recoveredText) != plainText { 20 | t.Fatal("recovered plaintext not equal") 21 | } 22 | } 23 | 24 | func TestSealOpen_InvalidBlockNo(t *testing.T) { 25 | f := File{} 26 | h := givenValidHeader() 27 | _ = f.initialiseCiphers(password, &h) 28 | plainText := "This is a secret" 29 | cipherText, nonce := f.seal([]byte(plainText), 1234) 30 | _, err := f.unseal(cipherText, 5432, nonce) 31 | if err == nil { 32 | t.Fatal(err) 33 | } 34 | } 35 | 36 | func TestSealOpen_Sizes(t *testing.T) { 37 | f := File{} 38 | h := givenValidHeader() 39 | _ = f.initialiseCiphers(password, &h) 40 | plainText := "This is a secret" 41 | cipherText, nonce := f.seal([]byte(plainText), 1234) 42 | 43 | if len(nonce) != 36 || len(nonce) != nonceSize { 44 | t.Fatal("nonce has to be 12*3 bytes") 45 | } 46 | if float32(len(plainText))*1.5 > float32(len(cipherText)) { 47 | t.Fatal("cipherText seems too short") 48 | } 49 | } 50 | 51 | // of course we don't intend to test the crypto primitives here, we want to assert without any doubt we did not "f.up" 52 | // the integration with the crypto primitives, now or in the future. 53 | func TestSealOpen_AnyByteChangeShouldFail(t *testing.T) { 54 | f := File{} 55 | h := givenValidHeader() 56 | _ = f.initialiseCiphers(password, &h) 57 | plainText := "This is a secret" 58 | cipherText, nonce := f.seal([]byte(plainText), 1234) 59 | // cipher-text 60 | for i := 0; i < len(cipherText); i++ { 61 | orig := cipherText[i] 62 | cipherText[i] = crypto.RandBytes(1)[0] 63 | if cipherText[i] == orig { 64 | cipherText[i]++ 65 | } 66 | _, err := f.unseal(cipherText, 1234, nonce) 67 | if err == nil { 68 | t.Fatal("this should have failed after changing one byte in the cipherText") 69 | } 70 | cipherText[i] = orig 71 | } 72 | // nonce 73 | for i := 0; i < len(nonce); i++ { 74 | orig := nonce[i] 75 | nonce[i] = orig ^ crypto.RandBytes(1)[0] 76 | if nonce[i] == orig { 77 | nonce[i]++ 78 | } 79 | _, err := f.unseal(cipherText, 1234, nonce) 80 | if err == nil { 81 | t.Fatal("this should have failed after changing one byte in the cipherText") 82 | } 83 | nonce[i] = orig 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kuking/seof 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/hashicorp/golang-lru v0.6.0 7 | github.com/kuking/go-pwentropy v0.0.0-20200622162422-156827dab9e6 8 | golang.org/x/crypto v0.1.0 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= 4 | github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 5 | github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0= 6 | github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= 7 | github.com/kuking/go-pwentropy v0.0.0-20200622162422-156827dab9e6 h1:eQ1o0FggV9J3XsMhugjN3NPRJmi+F9t6jvAKgw8zw28= 8 | github.com/kuking/go-pwentropy v0.0.0-20200622162422-156827dab9e6/go.mod h1:52dL15phHig0Gcjwn4FgCtwpWk5IpTFHvZtnyZfkK1c= 9 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 10 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 11 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 12 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 13 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 14 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 15 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 16 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 17 | golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= 18 | golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= 19 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 20 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 21 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 22 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 23 | golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= 24 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 25 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 26 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 27 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 28 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 29 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 30 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 31 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 32 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 33 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 34 | golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 35 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 36 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 37 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 38 | golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 39 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 40 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 41 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 42 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 43 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 44 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 45 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 46 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 47 | -------------------------------------------------------------------------------- /structs.go: -------------------------------------------------------------------------------- 1 | package seof 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "github.com/kuking/seof/crypto" 8 | ) 9 | 10 | const HeaderMagic uint64 = 0xb0a713c 11 | const HeaderLength int = 128 12 | 13 | type Header struct { 14 | Magic uint64 15 | ScriptSalt [96]byte 16 | ScriptN uint32 17 | ScriptR uint32 18 | ScriptP uint32 19 | DiskBlockSize uint32 20 | TailOfZeros [8]byte 21 | } 22 | 23 | func (h *Header) Verify() error { 24 | if h.Magic != HeaderMagic { 25 | return errors.New("header: invalid magic") 26 | } 27 | if h.DiskBlockSize < 1112 || h.DiskBlockSize > 196608 { 28 | return errors.New("header: invalid disk_block_size") 29 | } 30 | for i := 0; i < len(h.ScriptSalt); i++ { 31 | if h.ScriptSalt[i] != 0 { 32 | break 33 | } 34 | if i == len(h.ScriptSalt)-1 { 35 | return errors.New("header: zero salt") 36 | } 37 | } 38 | 39 | if h.ScriptN > crypto.MaxSCryptParameters.N || h.ScriptN < crypto.MinSCryptParameters.N || 40 | h.ScriptR > crypto.MaxSCryptParameters.R || h.ScriptR < crypto.MinSCryptParameters.R || 41 | h.ScriptP > crypto.MaxSCryptParameters.P || h.ScriptP < crypto.MinSCryptParameters.P { 42 | return errors.New("header: invalid scrypt parameters") 43 | } 44 | 45 | for i := 0; i < len(h.TailOfZeros); i++ { 46 | if h.TailOfZeros[i] != 0 { 47 | return errors.New("header: tail of zeros not correct") 48 | } 49 | } 50 | 51 | return nil 52 | } 53 | 54 | type BlockEnvelop struct { 55 | Nonce [nonceSize]byte 56 | CipherTextLen uint32 57 | CipherText []byte 58 | } 59 | 60 | type BlockZero struct { 61 | BEncBlockSize uint32 //BEnc as 'Before Encryption' 62 | DiskBlockSize uint32 63 | BEncFileSize uint64 //reported file size 64 | BlocksWritten uint64 65 | } 66 | 67 | func (z *BlockZero) Bytes() []byte { 68 | buf := new(bytes.Buffer) 69 | _ = binary.Write(buf, binary.LittleEndian, z) 70 | return buf.Bytes() 71 | } 72 | 73 | func BlockZeroFromBytes(b []byte) (*BlockZero, error) { 74 | r := bytes.NewReader(b) 75 | bz := BlockZero{} 76 | err := binary.Read(r, binary.LittleEndian, &bz) 77 | if err != nil { 78 | return nil, err 79 | } 80 | return &bz, nil 81 | } 82 | -------------------------------------------------------------------------------- /structs_test.go: -------------------------------------------------------------------------------- 1 | package seof 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "github.com/kuking/seof/crypto" 7 | "testing" 8 | ) 9 | 10 | func TestHeaderStruct(t *testing.T) { 11 | h := Header{ 12 | Magic: 0xdeadbeef, 13 | ScriptSalt: [96]byte{}, 14 | ScriptN: crypto.RecommendedSCryptParameters.N, 15 | ScriptR: crypto.RecommendedSCryptParameters.R, 16 | ScriptP: crypto.RecommendedSCryptParameters.P, 17 | DiskBlockSize: 1024, 18 | TailOfZeros: [8]byte{}, 19 | } 20 | 21 | buf := make([]byte, 0, 1024) 22 | p := bytes.NewBuffer(buf) 23 | if binary.Write(p, binary.LittleEndian, &h) != nil { 24 | t.Fatal() 25 | } 26 | if p.Len() != 128 || p.Len() != HeaderLength { 27 | t.Fatal("header should be 128 bytes") 28 | } 29 | } 30 | 31 | func TestHeader_Verify(t *testing.T) { 32 | h := givenValidHeader() 33 | err := h.Verify() 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | } 38 | 39 | func TestHeader_WrongMagic(t *testing.T) { 40 | h := givenValidHeader() 41 | h.Magic = 123 42 | if h.Verify() == nil { 43 | t.Fatal("header should not be valid with that magic") 44 | } 45 | } 46 | 47 | func TestHeader_ZeroSalt(t *testing.T) { 48 | h := givenValidHeader() 49 | for i := 0; i < len(h.ScriptSalt); i++ { 50 | h.ScriptSalt[i] = 0 51 | } 52 | if h.Verify() == nil { 53 | t.Fatal("A header with an 'empty' (zeroed) salt is wrong") 54 | } 55 | } 56 | 57 | func TestHeader_ScriptParams(t *testing.T) { 58 | h := givenValidHeader() 59 | // N 60 | h.ScriptN = crypto.MinSCryptParameters.N - 1 61 | if h.Verify() == nil { 62 | t.Fatal() 63 | } 64 | h.ScriptN = crypto.MaxSCryptParameters.N + 1 65 | if h.Verify() == nil { 66 | t.Fatal() 67 | } 68 | h.ScriptN = crypto.MaxSCryptParameters.N 69 | 70 | // N 71 | h.ScriptP = crypto.MinSCryptParameters.P - 1 72 | if h.Verify() == nil { 73 | t.Fatal() 74 | } 75 | h.ScriptP = crypto.MaxSCryptParameters.P + 1 76 | if h.Verify() == nil { 77 | t.Fatal() 78 | } 79 | h.ScriptP = crypto.MaxSCryptParameters.P 80 | 81 | // N 82 | h.ScriptR = crypto.MinSCryptParameters.R - 1 83 | if h.Verify() == nil { 84 | t.Fatal() 85 | } 86 | h.ScriptR = crypto.MaxSCryptParameters.R + 1 87 | if h.Verify() == nil { 88 | t.Fatal() 89 | } 90 | h.ScriptR = crypto.MaxSCryptParameters.R 91 | } 92 | 93 | func TestHeader_DiskBlockSize(t *testing.T) { 94 | h := givenValidHeader() 95 | h.DiskBlockSize = 1111 96 | if h.Verify() == nil { 97 | t.Fatal("disk block size should be at least 1500 bytes") 98 | } 99 | h.DiskBlockSize = 196608 + 1 100 | if h.Verify() == nil { 101 | t.Fatal("disk block size should be smaller than 196608 bytes") 102 | } 103 | } 104 | 105 | func TestHeader_TailOfZeros(t *testing.T) { 106 | h := givenValidHeader() 107 | h.TailOfZeros[7] = 1 108 | if h.Verify() == nil { 109 | t.Fatal("tailOfZeros should be all zeroes") 110 | } 111 | } 112 | 113 | func TestBlockZero_Serialising(t *testing.T) { 114 | bz := BlockZero{ 115 | BEncBlockSize: 1, 116 | DiskBlockSize: 2, 117 | BEncFileSize: 3, 118 | BlocksWritten: 4, 119 | } 120 | bz2, err := BlockZeroFromBytes(bz.Bytes()) 121 | if err != nil { 122 | t.Fatal(err) 123 | } 124 | if bz2 == nil { 125 | t.Fatal() 126 | } 127 | if bz != *bz2 { 128 | t.Fatal() 129 | } 130 | if len(bz.Bytes()) != 4+4+8+8 { 131 | t.Fatal() 132 | } 133 | 134 | } 135 | 136 | func givenValidHeader() Header { 137 | h := Header{ 138 | Magic: HeaderMagic, 139 | ScriptSalt: [96]byte{}, 140 | ScriptN: crypto.RecommendedSCryptParameters.N, 141 | ScriptR: crypto.RecommendedSCryptParameters.R, 142 | ScriptP: crypto.RecommendedSCryptParameters.P, 143 | DiskBlockSize: 1524, 144 | TailOfZeros: [8]byte{}, 145 | } 146 | copy(h.ScriptSalt[:], crypto.RandBytes(96)) 147 | return h 148 | } 149 | --------------------------------------------------------------------------------