├── assets ├── .gitignore ├── swaggerui │ ├── assets.go │ ├── dist │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── swagger-ui.css.map │ │ ├── absolute-path.js │ │ ├── package.json │ │ ├── index.js │ │ ├── README.md │ │ ├── index.html │ │ └── oauth2-redirect.html │ ├── update.sh │ ├── assets_generate.go │ └── LICENSE └── webui │ ├── dist │ ├── domhelper.js │ ├── index.otaru-fe.html │ ├── infobar.js │ ├── commonsubstr.js │ ├── format.js │ ├── app.js │ ├── loglevel.js │ └── app-fe.js │ └── handler.go ├── out └── .placeholder ├── gen.go ├── prometheus └── const.go ├── testutils ├── testca │ ├── generate.go │ ├── wrong_ca.yml │ ├── clientauth_issue_admin.yml │ ├── clientauth_issue_readonly.yml │ ├── ca.yml │ ├── clientauth_issue_invalid.yml │ ├── clientauth_ca.yml │ ├── cert-key.pem │ ├── issue_wrong.yml │ ├── wrong_cert-key.pem │ ├── clientauth_admin-key.pem │ ├── clientauth_invalid-key.pem │ ├── clientauth_readonly-key.pem │ ├── kmgmbasedir │ │ ├── default │ │ │ ├── cakey.pem │ │ │ └── cacert.pem │ │ ├── wrong │ │ │ ├── cakey.pem │ │ │ ├── cacert.pem │ │ │ └── issuedb.json │ │ └── clientauth │ │ │ ├── cakey.pem │ │ │ ├── cacert.pem │ │ │ └── issuedb.json │ ├── issue_server.yml │ ├── wrong_cacert.pem │ ├── cacert.pem │ ├── clientauth_admin.pem │ ├── clientauth_readonly.pem │ ├── clientauth_invalid.pem │ ├── wrong_cert.pem │ ├── clientauth_cacert.pem │ ├── cert.pem │ └── mktestcerts.sh ├── testdata.go ├── cipher.go ├── ensurelogger.go ├── testfilesystem.go ├── rwinterceptblobstore.go └── blobhandle.go ├── util ├── syncer.go ├── countfds │ ├── countfds_darwin.go │ └── countfds_linux.go ├── unlocker.go ├── implname.go ├── ugname.go ├── path.go ├── guaranteedpool.go ├── gzip.go ├── err.go ├── cancellable │ └── io.go └── util.go ├── doc ├── cachedblobentry_fsm.png ├── Makefile ├── testca │ └── Makefile ├── cachedblobstore.md ├── cachedblobentry_fsm.gv └── config.toml.example ├── scripts ├── gcloud_setup.bash ├── update_version.bash ├── gen_self_signed_cert.bash └── resources │ ├── index.yaml │ └── selfsign-csr.json ├── buf.yaml ├── version ├── consts.go └── version.go ├── blobstore ├── version │ └── version.go ├── randomaccessblobstore.go ├── blobstore.go ├── cachedblobstore │ ├── invalidatecachetask.go │ ├── cacheusagestats_test.go │ ├── cachedblobhandle.go │ ├── cachedbackendversion_test.go │ ├── cacheusagestats.go │ └── reducecachetask.go ├── genblobpath_test.go ├── genblobpath.go ├── positionedio.go ├── mockblobstore.go ├── memblobstore.go └── mux.go ├── NOTICE ├── gc ├── blobstoregc │ ├── gctask_test.go │ ├── gctask.go │ ├── gc_test.go │ └── gc.go ├── inodedbssgc │ └── inodedbssgc.go └── inodedbtxloggc │ ├── inodedbtxloggctask.go │ ├── inodedbtxloggc.go │ └── inodedbtxloggc_test.go ├── cli ├── path │ ├── local.go │ ├── path_test.go │ └── path.go ├── cert.go ├── options.go ├── put.go ├── conn.go └── get.go ├── btncrypt ├── key.go ├── key_test.go └── btncrypt_test.go ├── pb └── gen.go ├── .gitignore ├── buf.gen.yaml ├── .devcontainer ├── postCreate.sh └── devcontainer.json ├── scheduler ├── idgen.go └── repetitive_test.go ├── buf.lock ├── cmd └── otaru │ ├── main.go │ ├── webdav │ └── command.go │ ├── mkfs │ └── command.go │ ├── fe │ └── main.go │ ├── serve │ └── serve.go │ ├── fscli │ └── commands.go │ ├── dumpblob │ └── command.go │ └── globallock │ └── command.go ├── facade ├── hostname.go └── apiserver.go ├── go-fuzz ├── chunkstore │ ├── repro │ │ └── main.go │ └── chunkstore.go ├── filesystem │ └── repro │ │ └── main.go └── filewritecache │ └── repro │ └── main.go ├── logger ├── mux.go ├── categorylogger.go ├── http.go ├── writerlogger.go ├── config │ └── config.go ├── registry.go └── logger.go ├── inodedb ├── dbtransaction.go ├── inodedbsyncer │ └── synctask.go ├── blobstoredbstatesnapshotio │ └── simplesslocator.go ├── simpledbstatesnapshotio.go ├── simpledbtransactionlogio.go ├── dboperation_serdes_test.go ├── cacheddbtransactionlogio_test.go └── cacheddbtransactionlogio.go ├── metadata ├── metadata.go └── statesnapshot │ └── statesnapshot.go ├── webdav ├── entry.go ├── error.go └── serve.go ├── apiserver ├── clientauth │ ├── role.go │ ├── userinfo.go │ └── interceptor.go └── server_test.go ├── basicauth └── basicauth.go ├── filewritecache └── filewritecache_test.go ├── gcloud ├── auth │ ├── auth.go │ └── testutils │ │ └── testutils.go ├── util │ ├── checkerr_test.go │ └── checkerr.go └── datastore │ └── config.go ├── .github └── workflows │ └── go.yml ├── filesystem ├── inodedbchunksarrayio.go ├── filesystem_test.go └── fsutil.go ├── otaruapiserver ├── inodedbservice.go └── systemservice.go ├── fuse ├── common.go ├── filesystem_linux_test.go └── filesystem.go ├── chunkstore └── chunkheader_test.go ├── debugcmd ├── otaru-btncrypt-benchmark │ └── main.go ├── otaru-txlogio │ └── main.go └── otaru-inodedbsslocator │ └── main.go ├── extra └── fe │ ├── apiserver │ ├── proxyhandler.go │ └── server.go │ └── preview │ ├── httpsrv.go │ └── zip.go ├── go.mod └── flags └── flags.go /assets/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /out/.placeholder: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gen.go: -------------------------------------------------------------------------------- 1 | //go:generate buf generate 2 | package otaru 3 | -------------------------------------------------------------------------------- /prometheus/const.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | const Namespace = "otaru" 4 | -------------------------------------------------------------------------------- /testutils/testca/generate.go: -------------------------------------------------------------------------------- 1 | package testca 2 | 3 | //go:generate ./mktestcerts.sh 4 | -------------------------------------------------------------------------------- /util/syncer.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | type Syncer interface { 4 | Sync() error 5 | } 6 | -------------------------------------------------------------------------------- /assets/swaggerui/assets.go: -------------------------------------------------------------------------------- 1 | //go:generate go run assets_generate.go 2 | package swaggerui 3 | -------------------------------------------------------------------------------- /doc/cachedblobentry_fsm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyaxt/otaru/HEAD/doc/cachedblobentry_fsm.png -------------------------------------------------------------------------------- /util/countfds/countfds_darwin.go: -------------------------------------------------------------------------------- 1 | package countfds 2 | 3 | func CountFds() int { 4 | return 0 5 | } 6 | -------------------------------------------------------------------------------- /assets/swaggerui/dist/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyaxt/otaru/HEAD/assets/swaggerui/dist/favicon-16x16.png -------------------------------------------------------------------------------- /assets/swaggerui/dist/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyaxt/otaru/HEAD/assets/swaggerui/dist/favicon-32x32.png -------------------------------------------------------------------------------- /assets/swaggerui/dist/swagger-ui.css.map: -------------------------------------------------------------------------------- 1 | {"version":3,"sources":[],"names":[],"mappings":"","file":"swagger-ui.css","sourceRoot":""} -------------------------------------------------------------------------------- /scripts/gcloud_setup.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.bash 4 | otaru::gcloud_setup 5 | -------------------------------------------------------------------------------- /scripts/update_version.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.bash 4 | otaru::update_version 5 | -------------------------------------------------------------------------------- /testutils/testdata.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | var ( 4 | HelloWorld = []byte("Hello, world") 5 | HogeFugaPiyo = []byte("hogefugapiyo") 6 | ) 7 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | version: v2 2 | breaking: 3 | use: 4 | - FILE 5 | lint: 6 | use: 7 | - STANDARD 8 | deps: 9 | - buf.build/googleapis/googleapis 10 | -------------------------------------------------------------------------------- /scripts/gen_self_signed_cert.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.bash 4 | otaru::gen_self_signed_cert 5 | -------------------------------------------------------------------------------- /scripts/resources/index.yaml: -------------------------------------------------------------------------------- 1 | indexes: 2 | 3 | - kind: OtaruINodeDBSS 4 | ancestor: yes 5 | properties: 6 | - name: TxID 7 | direction: desc 8 | -------------------------------------------------------------------------------- /version/consts.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | const GIT_COMMIT = "" 4 | const BUILD_HOST = "" 5 | const BUILD_TIME = 1453465836 6 | -------------------------------------------------------------------------------- /blobstore/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import "io" 4 | 5 | // FIXME: handle overflows 6 | type Version int64 7 | type QueryFunc func(r io.Reader) (Version, error) 8 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Otaru includes software from: 2 | - https://github.com/philips/grpc-gateway-example (under Apache License 2.0) 3 | - https://github.com/swagger-api/swagger-ui (under Apache License 2.0) 4 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean 2 | 3 | cachedblobentry_fsm.png: cachedblobentry_fsm.gv 4 | dot -Tpng cachedblobentry_fsm.gv -ocachedblobentry_fsm.png 5 | 6 | clean: 7 | rm cachedblobentry_fsm.png 8 | -------------------------------------------------------------------------------- /testutils/testca/wrong_ca.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # kmgm PKI CA config 3 | setup: 4 | subject: 5 | commonName: wrong CA 6 | country: US 7 | validity: farfuture 8 | keyType: ecdsa 9 | noDefault: true 10 | -------------------------------------------------------------------------------- /gc/blobstoregc/gctask_test.go: -------------------------------------------------------------------------------- 1 | package blobstoregc_test 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/gc/blobstoregc" 5 | "github.com/nyaxt/otaru/scheduler" 6 | ) 7 | 8 | var _ = scheduler.Task(&blobstoregc.Task{}) 9 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_issue_admin.yml: -------------------------------------------------------------------------------- 1 | --- 2 | issue: 3 | subject: 4 | commonName: admin alice 5 | validity: farfuture 6 | keyType: ecdsa 7 | keyUsage: 8 | preset: tlsClient 9 | noDefault: true 10 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_issue_readonly.yml: -------------------------------------------------------------------------------- 1 | --- 2 | issue: 3 | subject: 4 | commonName: readonly bob 5 | validity: farfuture 6 | keyType: ecdsa 7 | keyUsage: 8 | preset: tlsClient 9 | noDefault: true 10 | -------------------------------------------------------------------------------- /cli/path/local.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | import ( 4 | "path/filepath" 5 | ) 6 | 7 | func ResolveLocalPath(localRootPath, relPath string) string { 8 | return filepath.Join(localRootPath, filepath.Clean("/"+relPath)) 9 | } 10 | -------------------------------------------------------------------------------- /testutils/testca/ca.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # kmgm PKI CA config 3 | setup: 4 | subject: 5 | commonName: otaru dev CA 6 | country: JP 7 | province: Tokyo 8 | validity: farfuture 9 | keyType: ecdsa 10 | noDefault: true 11 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_issue_invalid.yml: -------------------------------------------------------------------------------- 1 | --- 2 | issue: 3 | subject: 4 | commonName: invalid charlie 5 | validity: farfuture 6 | keyType: ecdsa 7 | keyUsage: 8 | preset: tlsClient 9 | noDefault: true 10 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_ca.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # kmgm PKI CA config 3 | setup: 4 | subject: 5 | commonName: otaru dev clientauth CA 6 | country: JP 7 | province: Tokyo 8 | validity: farfuture 9 | keyType: ecdsa 10 | noDefault: true 11 | -------------------------------------------------------------------------------- /btncrypt/key.go: -------------------------------------------------------------------------------- 1 | package btncrypt 2 | 3 | import ( 4 | "crypto/sha1" 5 | 6 | "golang.org/x/crypto/pbkdf2" 7 | ) 8 | 9 | func KeyFromPassword(password string) []byte { 10 | return pbkdf2.Key([]byte(password), []byte("otaru"), 4096, 32, sha1.New) 11 | } 12 | -------------------------------------------------------------------------------- /pb/gen.go: -------------------------------------------------------------------------------- 1 | //go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative --grpc-gateway_out=. --grpc-gateway_opt=logtostderr=true --grpc-gateway_opt=paths=source_relative -I. -I./third_party otaru.proto 2 | package pb 3 | -------------------------------------------------------------------------------- /util/unlocker.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type EnsureUnlocker struct{ L sync.Locker } 8 | 9 | func (eu *EnsureUnlocker) Unlock() { 10 | if eu.L == nil { 11 | return 12 | } 13 | 14 | eu.L.Unlock() 15 | eu.L = nil 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | dep.png 4 | out/* 5 | otaru-cli 6 | otaru-fe 7 | otaru-fuzzymv 8 | otaru-globallock 9 | otaru-mkfs 10 | otaru-server 11 | otaru-webdav-fe 12 | *-fuzz.zip 13 | 14 | version/consts.go 15 | 16 | doc/testca/*.pem 17 | doc/testca/*.csr 18 | -------------------------------------------------------------------------------- /testutils/testca/cert-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIPjAGtZXprNnQDiR1Mrpuo/adIXbJE4chsTZLPPtHfy5oAoGCCqGSM49 3 | AwEHoUQDQgAE+Zf/2PGVxu0cuR1uUcte1SDKX6sBPv1TkZfJmq+dY2VoE7Ol7grY 4 | nJtXsvkVoGgi5qMaBX/xVgQozHkXEgWnRQ== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /scripts/resources/selfsign-csr.json: -------------------------------------------------------------------------------- 1 | { 2 | "hosts": [ 3 | "localhost" 4 | ], 5 | "CN": "localhost", 6 | "key": { 7 | "algo": "rsa", 8 | "size": 2048 9 | }, 10 | "names": [{ 11 | "O": "autogenerated", 12 | "OU": "otaru", 13 | "L": "lan" 14 | }] 15 | } 16 | -------------------------------------------------------------------------------- /testutils/testca/issue_wrong.yml: -------------------------------------------------------------------------------- 1 | --- 2 | issue: 3 | subject: 4 | commonName: otaru test server wrong ca 5 | subjectAltNames: 6 | - localhost 7 | validity: farfuture 8 | keyType: ecdsa 9 | keyUsage: 10 | preset: tlsClientServer 11 | noDefault: true 12 | -------------------------------------------------------------------------------- /testutils/testca/wrong_cert-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEICiazsY5kKDXQ07hyhXAZzqEhnqLEkBiiHogyUaIvSI9oAoGCCqGSM49 3 | AwEHoUQDQgAEo6ehJCJ0eE2Q4is6dl8yQ1E+zhdtIh8gAUuEcnkedySMNesu/lex 4 | Sh7CN7f59WGHpNo4kvqRsra3TengPrmaig== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_admin-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIJNIr3OfF0QFx0jYXJFVfUY5SgyRov+ZZIk2zmr+jduFoAoGCCqGSM49 3 | AwEHoUQDQgAESX1fChKzhTEzXiAmIFulXl6ITMlywAJA4AlaADXJ6jT8UaBJBlMn 4 | jQ7rekYqcGKapMHK9bAdHltSgycdIxbMfg== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_invalid-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIEiv93cTY220vghIhJU3oF0sCJ/DF7Zyo/sqWV/MSs9roAoGCCqGSM49 3 | AwEHoUQDQgAEGL+QT+rCJl8VAe0A2rHArg51tmXay4on+LGfdvCxsKi3g4lJ4j2m 4 | wRpbGPePqbr8dh79K69xZw34LYwjnJlL/w== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_readonly-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIHlYPN/FB7ehZKfXR6ADjBKTOhtZmU3+0BW5JZCp39m8oAoGCCqGSM49 3 | AwEHoUQDQgAE7kFF8CsSz0uPigXJhTZQdoXCJ2mz/dpKupDMpBZqTmiUxqeIokW9 4 | lRxh+6MQGcS7s9VkfqeTlPYrd88pVe38Yg== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/default/cakey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIGoJfwaRStsI40D9Oud/oKeezqSR05s6L/OlGU/LDazAoAoGCCqGSM49 3 | AwEHoUQDQgAEqdxtzz0RYgmquKuU3yj8wodn6g0EdXRzlkFEOMVAVW2yI+/I5WZ8 4 | 4/4hbcPDKHu68VQw9MUYQfkKS4Hz/nKX6g== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/wrong/cakey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIGnrr9KEyKbyagpvnVuiEMg3RtzhQ0l2QO1oYo9hDzT4oAoGCCqGSM49 3 | AwEHoUQDQgAEpqROrnm8K99pn16goKGSmBXEEkaI3Yh6UAqegZomxYVKeY5Mg4rC 4 | yBKdrVmuNFaXZOWP6AT47hA7sGmUxhVS9A== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/clientauth/cakey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIPEcaRfgDDdaR8UPS4lbj+Ed/vXYJtcfOhdtkrowm17soAoGCCqGSM49 3 | AwEHoUQDQgAEJn/2W2P1uJCm7ENPrXf//xUyyrQ094QczeKiwAkCdgD0Z8WejpgD 4 | RLJBvThMDj4aGgufL6sC7yBpMz+8l5AYhw== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | plugins: 3 | - remote: buf.build/grpc/go 4 | out: . 5 | opt: paths=source_relative 6 | - remote: buf.build/grpc-ecosystem/gateway 7 | out: . 8 | opt: paths=source_relative 9 | - remote: protocolbuffers/go 10 | out: . 11 | opt: paths=source_relative 12 | -------------------------------------------------------------------------------- /testutils/testca/issue_server.yml: -------------------------------------------------------------------------------- 1 | --- 2 | issue: 3 | subject: 4 | commonName: otaru test server 5 | organization: otaru 6 | subjectAltNames: 7 | - localhost 8 | validity: farfuture 9 | keyType: ecdsa 10 | keyUsage: 11 | preset: tlsClientServer 12 | noDefault: true 13 | -------------------------------------------------------------------------------- /util/countfds/countfds_linux.go: -------------------------------------------------------------------------------- 1 | package countfds 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | func CountFds() int { 10 | es, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())) 11 | if err != nil { 12 | return 0 13 | } 14 | return len(es) 15 | } 16 | -------------------------------------------------------------------------------- /.devcontainer/postCreate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | PREFIX="/usr/local" && \ 5 | VERSION="1.41.0" && \ 6 | curl -sSL \ 7 | "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m).tar.gz" | \ 8 | sudo tar -xvzf - -C "${PREFIX}" --strip-components 1 9 | -------------------------------------------------------------------------------- /assets/webui/dist/domhelper.js: -------------------------------------------------------------------------------- 1 | const $ = document.querySelector.bind(document); 2 | const $$ = document.querySelectorAll.bind(document); 3 | const removeAllChildNodes = par => { 4 | while(par.hasChildNodes()) 5 | par.removeChild(par.lastChild); 6 | }; 7 | 8 | export {$, $$, removeAllChildNodes}; 9 | -------------------------------------------------------------------------------- /scheduler/idgen.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "sync/atomic" 5 | ) 6 | 7 | type ID uint32 8 | 9 | const allJobs ID = 0 10 | 11 | type idGen struct { 12 | lastID ID 13 | } 14 | 15 | func (g *idGen) genID() ID { 16 | return ID(atomic.AddUint32((*uint32)(&g.lastID), 1)) 17 | } 18 | -------------------------------------------------------------------------------- /buf.lock: -------------------------------------------------------------------------------- 1 | # Generated by buf. DO NOT EDIT. 2 | version: v2 3 | deps: 4 | - name: buf.build/googleapis/googleapis 5 | commit: e7f8d366f5264595bcc4cd4139af9973 6 | digest: b5:0cd69a689ee320ed815663d57d1bc3a1d6823224a7a717d46fee3a68197c25a6f5f932c0b0e49f8370c70c247a6635969a6a54af5345cafd51e0667298768aca 7 | -------------------------------------------------------------------------------- /cmd/otaru/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | func main() { 10 | if err := NewApp().Run(os.Args); err != nil { 11 | // omit stacktrace 12 | zap.L().WithOptions(zap.AddStacktrace(zap.FatalLevel)).Error(err.Error()) 13 | os.Exit(1) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /btncrypt/key_test.go: -------------------------------------------------------------------------------- 1 | package btncrypt_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/nyaxt/otaru/btncrypt" 7 | ) 8 | 9 | func TestKeyFromPassword(t *testing.T) { 10 | key := btncrypt.KeyFromPassword("hogefuga") 11 | if len(key) != 32 { 12 | t.Errorf("invalid key length: %d", len(key)) 13 | } 14 | // t.Errorf("gen key: %v", key) 15 | } 16 | -------------------------------------------------------------------------------- /facade/hostname.go: -------------------------------------------------------------------------------- 1 | package facade 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "go.uber.org/zap" 8 | ) 9 | 10 | func GenHostName() string { 11 | hostname, err := os.Hostname() 12 | if err != nil { 13 | zap.S().Panicf("Failed to query local hostname: %v", err) 14 | } 15 | pid := os.Getpid() 16 | return fmt.Sprintf("%s-%d", hostname, pid) 17 | } 18 | -------------------------------------------------------------------------------- /go-fuzz/chunkstore/repro/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | 8 | "github.com/nyaxt/otaru/go-fuzz/chunkstore" 9 | ) 10 | 11 | func main() { 12 | blob, err := ioutil.ReadFile(os.Args[1]) 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | out := chunkstore.Fuzz(blob) 18 | fmt.Printf("Fuzz out: %d\n", out) 19 | } 20 | -------------------------------------------------------------------------------- /go-fuzz/filesystem/repro/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | 8 | "github.com/nyaxt/otaru/go-fuzz/filesystem" 9 | ) 10 | 11 | func main() { 12 | blob, err := ioutil.ReadFile(os.Args[1]) 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | out := filesystem.Fuzz(blob) 18 | fmt.Printf("Fuzz out: %d\n", out) 19 | } 20 | -------------------------------------------------------------------------------- /testutils/cipher.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/btncrypt" 5 | ) 6 | 7 | var ( 8 | Key = []byte("0123456789abcdef0123456789abcdef") 9 | ) 10 | 11 | func TestCipher() *btncrypt.Cipher { 12 | c, err := btncrypt.NewCipher(Key) 13 | if err != nil { 14 | panic("Failed to init Cipher for testing") 15 | } 16 | return c 17 | } 18 | -------------------------------------------------------------------------------- /assets/swaggerui/update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | VERSION=3.17.1 3 | curl -o /tmp/swagger.tgz https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-${VERSION}.tgz 4 | mkdir /tmp/swagger 5 | tar zxvf /tmp/swagger.tgz -C /tmp/swagger 6 | rm dist/* 7 | mv /tmp/swagger/package/* dist/ 8 | sed -i'' -e "s,https://petstore.swagger.io/v2/swagger.json,/otaru.swagger.json," dist/index.html 9 | -------------------------------------------------------------------------------- /go-fuzz/filewritecache/repro/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | 8 | "github.com/nyaxt/otaru/go-fuzz/filewritecache" 9 | ) 10 | 11 | func main() { 12 | blob, err := ioutil.ReadFile(os.Args[1]) 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | out := filewritecache.Fuzz(blob) 18 | fmt.Printf("Fuzz out: %d\n", out) 19 | } 20 | -------------------------------------------------------------------------------- /logger/mux.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | type Mux struct { 4 | Ls []Logger 5 | } 6 | 7 | func (m *Mux) Log(lv Level, data map[string]interface{}) { 8 | for _, l := range m.Ls { 9 | l.Log(lv, data) 10 | } 11 | } 12 | 13 | func (m *Mux) WillAccept(lv Level) bool { 14 | for _, l := range m.Ls { 15 | if l.WillAccept(lv) { 16 | return true 17 | } 18 | } 19 | return false 20 | } 21 | -------------------------------------------------------------------------------- /logger/categorylogger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | type CategoryLogger struct { 4 | BE Logger 5 | Category string 6 | Level 7 | } 8 | 9 | func (l *CategoryLogger) Log(lv Level, data map[string]interface{}) { 10 | if !l.WillAccept(lv) { 11 | return 12 | } 13 | data["category"] = l.Category 14 | l.BE.Log(lv, data) 15 | } 16 | 17 | func (l *CategoryLogger) WillAccept(lv Level) bool { 18 | return lv >= l.Level 19 | } 20 | -------------------------------------------------------------------------------- /inodedb/dbtransaction.go: -------------------------------------------------------------------------------- 1 | package inodedb 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type DBTransaction struct { 8 | TxID `json:"txid"` 9 | Ops []DBOperation `json:"ops"` 10 | } 11 | 12 | func (tx DBTransaction) String() string { 13 | opsJson, err := EncodeDBOperationsToJson(tx.Ops) 14 | if err != nil { 15 | opsJson = []byte("*ENC_ERR*") 16 | } 17 | 18 | return fmt.Sprintf("{TxID: %s, Ops: %s}", tx.TxID, string(opsJson)) 19 | } 20 | -------------------------------------------------------------------------------- /blobstore/randomaccessblobstore.go: -------------------------------------------------------------------------------- 1 | package blobstore 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/nyaxt/otaru/flags" 7 | ) 8 | 9 | type SizeQueryable interface { 10 | Size() int64 11 | } 12 | 13 | type BlobHandle interface { 14 | RandomAccessIO 15 | SizeQueryable 16 | Truncate(int64) error 17 | io.Closer 18 | } 19 | 20 | type RandomAccessBlobStore interface { 21 | Open(blobpath string, flags int) (BlobHandle, error) 22 | flags.FlagsReader 23 | } 24 | -------------------------------------------------------------------------------- /util/implname.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | type ImplNamed interface { 4 | ImplName() string 5 | } 6 | 7 | func TryGetImplName(v interface{}) string { 8 | named, ok := v.(ImplNamed) 9 | if !ok { 10 | return "" 11 | } 12 | return named.ImplName() 13 | } 14 | 15 | func Describe(v interface{}) string { 16 | type stringifier interface { 17 | String() string 18 | } 19 | if sf, ok := v.(stringifier); ok { 20 | return sf.String() 21 | } 22 | return TryGetImplName(v) 23 | } 24 | -------------------------------------------------------------------------------- /metadata/metadata.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | const INodeDBSnapshotBlobpathPrefix = "META_INODEDB_SNAPSHOT" 10 | const VersionCacheBlobpath = "META_VERSION_CACHE" 11 | 12 | func IsMetadataBlobpath(blobpath string) bool { 13 | return strings.HasPrefix(blobpath, "META_") 14 | } 15 | 16 | func GenINodeDBSnapshotBlobpath() string { 17 | return fmt.Sprintf("%s.%s", 18 | INodeDBSnapshotBlobpathPrefix, 19 | time.Now().Format("2006-01-02.150405.000")) 20 | } 21 | -------------------------------------------------------------------------------- /util/ugname.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os/user" 5 | "strconv" 6 | ) 7 | 8 | // FIXME: cache? 9 | 10 | func TryUserName(id uint32) string { 11 | idstr := strconv.FormatUint(uint64(id), 10) 12 | u, err := user.LookupId(idstr) 13 | if err != nil { 14 | return idstr 15 | } 16 | return u.Username 17 | } 18 | 19 | func TryGroupName(id uint32) string { 20 | idstr := strconv.FormatUint(uint64(id), 10) 21 | g, err := user.LookupGroupId(idstr) 22 | if err != nil { 23 | return idstr 24 | } 25 | return g.Name 26 | } 27 | -------------------------------------------------------------------------------- /assets/swaggerui/assets_generate.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "log" 7 | "net/http" 8 | 9 | "github.com/shurcooL/vfsgen" 10 | ) 11 | 12 | const filenameVfsGen = "assets_vfsgen.go" 13 | 14 | func main() { 15 | fs := http.Dir("dist") 16 | 17 | log.Printf("swaggerui assets_generate.go") 18 | err := vfsgen.Generate(fs, vfsgen.Options{ 19 | Filename: filenameVfsGen, 20 | PackageName: "swaggerui", 21 | VariableName: "Assets", 22 | }) 23 | if err != nil { 24 | log.Fatalln(err) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /cmd/otaru/webdav/command.go: -------------------------------------------------------------------------------- 1 | package webdav 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | 6 | ocli "github.com/nyaxt/otaru/cli" 7 | "github.com/nyaxt/otaru/webdav" 8 | ) 9 | 10 | var Command = &cli.Command{ 11 | Name: "webdav", 12 | Usage: "Run otaru webdav server", 13 | Action: func(c *cli.Context) error { 14 | cfg, err := ocli.NewConfig(c.String("configDir")) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | if err = webdav.Serve(c.Context, cfg); err != nil { 20 | return err 21 | } 22 | return nil 23 | }, 24 | } 25 | -------------------------------------------------------------------------------- /blobstore/blobstore.go: -------------------------------------------------------------------------------- 1 | package blobstore 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type BlobStore interface { 8 | OpenWriter(blobpath string) (io.WriteCloser, error) 9 | OpenReader(blobpath string) (io.ReadCloser, error) 10 | } 11 | 12 | type BlobLister interface { 13 | ListBlobs() ([]string, error) 14 | } 15 | 16 | type BlobSizer interface { 17 | BlobSize(blobpath string) (int64, error) 18 | } 19 | 20 | type BlobRemover interface { 21 | RemoveBlob(blobpath string) error 22 | } 23 | 24 | type TotalSizer interface { 25 | TotalSize() (int64, error) 26 | } 27 | -------------------------------------------------------------------------------- /doc/testca/Makefile: -------------------------------------------------------------------------------- 1 | cert.pem: cert.csr ca.pem ca-key.pem 2 | openssl x509 -req -days 3650 -in cert.csr -CA ca.pem -CAkey ca-key.pem -out $@ -set_serial 1 3 | 4 | cert.csr: cert-key.pem 5 | openssl req -new -sha256 -key $< -out $@ -subj '/CN=otaru-server/O=Test/C=JP' 6 | 7 | cert-key.pem: 8 | openssl ecparam -out $@ -name prime256v1 -genkey 9 | 10 | 11 | ca.pem: ca-key.pem 12 | openssl req -key $< -new -x509 -days 3650 -sha256 -out $@ -extensions v3_ca --subj '/C=JP' 13 | 14 | ca-key.pem: 15 | openssl ecparam -out $@ -name prime256v1 -genkey 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /logger/http.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "net/http" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | func HttpHandler(l *zap.Logger, h http.Handler) http.Handler { 10 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 11 | l.Info( 12 | "http request", 13 | zap.String("uri", r.URL.RequestURI()), 14 | zap.String("remote_addr", r.RemoteAddr), 15 | zap.String("method", r.Method), 16 | zap.String("referer", r.Referer()), 17 | zap.String("user_agent", r.UserAgent()), 18 | ) 19 | 20 | h.ServeHTTP(w, r) 21 | }) 22 | } 23 | -------------------------------------------------------------------------------- /webdav/entry.go: -------------------------------------------------------------------------------- 1 | package webdav 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/nyaxt/otaru/pb" 7 | ) 8 | 9 | type Entry struct { 10 | Id uint64 11 | Name string 12 | Size int64 13 | ModifiedTime time.Time 14 | IsDir bool 15 | } 16 | 17 | func INodeViewToEntry(v *pb.INodeView) *Entry { 18 | return &Entry{ 19 | Id: v.Id, 20 | Name: v.Name, 21 | Size: v.Size, 22 | ModifiedTime: time.Unix(v.ModifiedTime, 0), 23 | IsDir: v.Type == pb.INodeType_DIR, 24 | } 25 | } 26 | 27 | type Listing []*Entry 28 | -------------------------------------------------------------------------------- /assets/swaggerui/dist/absolute-path.js: -------------------------------------------------------------------------------- 1 | /* 2 | * getAbsoluteFSPath 3 | * @return {string} When run in NodeJS env, returns the absolute path to the current directory 4 | * When run outside of NodeJS, will return an error message 5 | */ 6 | const getAbsoluteFSPath = function () { 7 | // detect whether we are running in a browser or nodejs 8 | if (typeof module !== "undefined" && module.exports) { 9 | return require("path").resolve(__dirname) 10 | } 11 | throw new Error('getAbsoluteFSPath can only be called within a Nodejs environment'); 12 | } 13 | 14 | module.exports = getAbsoluteFSPath 15 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "runtime/debug" 7 | ) 8 | 9 | var BuildVersion string = "" 10 | var BuildSum string = "" 11 | 12 | func init() { 13 | if bi, ok := debug.ReadBuildInfo(); ok { 14 | BuildVersion = bi.Main.Version 15 | BuildSum = bi.Main.Sum 16 | } 17 | } 18 | 19 | func DumpBuildInfo() string { 20 | return fmt.Sprintf(""+ 21 | "Version: %s\n"+ 22 | "Sum: %s\n"+ 23 | "Go version: %s\n"+ 24 | "OS/Arch: %s/%s\n", 25 | BuildVersion, 26 | BuildSum, 27 | runtime.Version(), 28 | runtime.GOOS, runtime.GOARCH, 29 | ) 30 | } 31 | -------------------------------------------------------------------------------- /assets/swaggerui/dist/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "swagger-ui-dist", 3 | "version": "3.17.1", 4 | "main": "index.js", 5 | "repository": "git@github.com:swagger-api/swagger-ui.git", 6 | "contributors": [ 7 | "(in alphabetical order)", 8 | "Anna Bodnia ", 9 | "Buu Nguyen ", 10 | "Josh Ponelat ", 11 | "Kyle Shockey ", 12 | "Robert Barnwell ", 13 | "Sahar Jafari " 14 | ], 15 | "license": "Apache-2.0", 16 | "dependencies": {}, 17 | "devDependencies": {} 18 | } 19 | -------------------------------------------------------------------------------- /testutils/testca/wrong_cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBdjCCARygAwIBAgIBATAKBggqhkjOPQQDAjAgMQswCQYDVQQGEwJVUzERMA8G 3 | A1UEAxMId3JvbmcgQ0EwIBcNMjIwNTA0MDUxOTE1WhgPMjA5OTEyMzEyMzU5MDBa 4 | MCAxCzAJBgNVBAYTAlVTMREwDwYDVQQDEwh3cm9uZyBDQTBZMBMGByqGSM49AgEG 5 | CCqGSM49AwEHA0IABKakTq55vCvfaZ9eoKChkpgVxBJGiN2IelAKnoGaJsWFSnmO 6 | TIOKwsgSna1ZrjRWl2Tlj+gE+O4QO7BplMYVUvSjRTBDMA4GA1UdDwEB/wQEAwIC 7 | pDASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBRqvS7LeYfYLKzWXoVIrYun 8 | FWyxeTAKBggqhkjOPQQDAgNIADBFAiBTA56avNnSe7DSVJjp3seB3d6ndIkbgpmm 9 | cQ+FpbCrIgIhALnqOtZ7SJWwaidMAso7Ue+kRkAv3EPUx8oHowvh/+4V 10 | -----END CERTIFICATE----- 11 | -------------------------------------------------------------------------------- /cmd/otaru/mkfs/command.go: -------------------------------------------------------------------------------- 1 | package mkfs 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/urfave/cli/v2" 7 | "go.uber.org/zap" 8 | 9 | "github.com/nyaxt/otaru/facade" 10 | ) 11 | 12 | var Command = &cli.Command{ 13 | Name: "mkfs", 14 | Usage: "Prep new otaru filesystem instance.", 15 | Action: func(c *cli.Context) error { 16 | cfg, err := facade.NewConfig(c.Path("configDir")) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | if err := facade.Mkfs(cfg); err != nil { 22 | return fmt.Errorf("facade.Mkfs: %w", err) 23 | } 24 | zap.S().Infof("mkfs finished successfully!") 25 | 26 | return nil 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /blobstore/cachedblobstore/invalidatecachetask.go: -------------------------------------------------------------------------------- 1 | package cachedblobstore 2 | 3 | import ( 4 | "fmt" 5 | 6 | "context" 7 | 8 | "github.com/nyaxt/otaru/scheduler" 9 | ) 10 | 11 | type InvalidateCacheTask struct { 12 | BE *CachedBlobEntry 13 | } 14 | 15 | func (t *InvalidateCacheTask) Run(ctx context.Context) scheduler.Result { 16 | err := t.BE.invalidate(ctx) 17 | return scheduler.ErrorResult{err} 18 | } 19 | 20 | func (*InvalidateCacheTask) ImplName() string { return "InvalidateCacheTask" } 21 | 22 | func (t *InvalidateCacheTask) String() string { 23 | return fmt.Sprintf("InvalidateCacheTask{blobpath: %s}", t.BE.blobpath) 24 | } 25 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/wrong/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBdjCCARygAwIBAgIBATAKBggqhkjOPQQDAjAgMQswCQYDVQQGEwJVUzERMA8G 3 | A1UEAxMId3JvbmcgQ0EwIBcNMjIwNTA0MDUxOTE1WhgPMjA5OTEyMzEyMzU5MDBa 4 | MCAxCzAJBgNVBAYTAlVTMREwDwYDVQQDEwh3cm9uZyBDQTBZMBMGByqGSM49AgEG 5 | CCqGSM49AwEHA0IABKakTq55vCvfaZ9eoKChkpgVxBJGiN2IelAKnoGaJsWFSnmO 6 | TIOKwsgSna1ZrjRWl2Tlj+gE+O4QO7BplMYVUvSjRTBDMA4GA1UdDwEB/wQEAwIC 7 | pDASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBRqvS7LeYfYLKzWXoVIrYun 8 | FWyxeTAKBggqhkjOPQQDAgNIADBFAiBTA56avNnSe7DSVJjp3seB3d6ndIkbgpmm 9 | cQ+FpbCrIgIhALnqOtZ7SJWwaidMAso7Ue+kRkAv3EPUx8oHowvh/+4V 10 | -----END CERTIFICATE----- 11 | -------------------------------------------------------------------------------- /gc/blobstoregc/gctask.go: -------------------------------------------------------------------------------- 1 | package blobstoregc 2 | 3 | import ( 4 | "fmt" 5 | 6 | "context" 7 | 8 | "github.com/nyaxt/otaru/inodedb" 9 | "github.com/nyaxt/otaru/scheduler" 10 | "github.com/nyaxt/otaru/util" 11 | ) 12 | 13 | type Task struct { 14 | BS GCableBlobStore 15 | IDB inodedb.DBFscker 16 | DryRun bool 17 | } 18 | 19 | func (t *Task) Run(ctx context.Context) scheduler.Result { 20 | err := GC(ctx, t.BS, t.IDB, t.DryRun) 21 | return scheduler.ErrorResult{err} 22 | } 23 | 24 | func (t *Task) String() string { 25 | return fmt.Sprintf("blobstoregc.Task{%s, %s}", util.TryGetImplName(t.BS), util.TryGetImplName(t.IDB)) 26 | } 27 | -------------------------------------------------------------------------------- /gc/inodedbssgc/inodedbssgc.go: -------------------------------------------------------------------------------- 1 | package inodedbssgc 2 | 3 | import ( 4 | "fmt" 5 | 6 | "context" 7 | 8 | "github.com/nyaxt/otaru/scheduler" 9 | "github.com/nyaxt/otaru/util" 10 | ) 11 | 12 | type INodeDBSSGCer interface { 13 | DeleteOldSnapshots(ctx context.Context, dryRun bool) error 14 | } 15 | 16 | type Task struct { 17 | Impl INodeDBSSGCer 18 | DryRun bool 19 | } 20 | 21 | func (t *Task) Run(ctx context.Context) scheduler.Result { 22 | err := t.Impl.DeleteOldSnapshots(ctx, t.DryRun) 23 | return scheduler.ErrorResult{err} 24 | } 25 | 26 | func (t *Task) String() string { 27 | return fmt.Sprintf("inodedbssgc.Task{%s}", util.Describe(t.Impl)) 28 | } 29 | -------------------------------------------------------------------------------- /gc/inodedbtxloggc/inodedbtxloggctask.go: -------------------------------------------------------------------------------- 1 | package inodedbtxloggc 2 | 3 | import ( 4 | "fmt" 5 | 6 | "context" 7 | 8 | "github.com/nyaxt/otaru/scheduler" 9 | "github.com/nyaxt/otaru/util" 10 | ) 11 | 12 | type Task struct { 13 | ThresFinder UnneededTxIDThresholdFinder 14 | LogDeleter TransactionLogDeleter 15 | DryRun bool 16 | } 17 | 18 | func (t *Task) Run(ctx context.Context) scheduler.Result { 19 | err := GC(ctx, t.ThresFinder, t.LogDeleter, t.DryRun) 20 | return scheduler.ErrorResult{err} 21 | } 22 | 23 | func (t *Task) String() string { 24 | return fmt.Sprintf("inodedbtxloggc.Task{%s, %s}", util.Describe(t.ThresFinder), util.Describe(t.LogDeleter)) 25 | } 26 | -------------------------------------------------------------------------------- /inodedb/inodedbsyncer/synctask.go: -------------------------------------------------------------------------------- 1 | package inodedbsyncer 2 | 3 | import ( 4 | "time" 5 | 6 | "context" 7 | 8 | "github.com/nyaxt/otaru/inodedb" 9 | "github.com/nyaxt/otaru/scheduler" 10 | ) 11 | 12 | type syncTask struct { 13 | ts inodedb.TriggerSyncer 14 | } 15 | 16 | var _ = scheduler.Task(&syncTask{}) 17 | var tzero time.Time 18 | 19 | func NewSyncTask(ts inodedb.TriggerSyncer) *syncTask { 20 | return &syncTask{ts: ts} 21 | } 22 | 23 | func (st *syncTask) Run(ctx context.Context) scheduler.Result { 24 | err := <-st.ts.TriggerSync() 25 | return scheduler.ErrorResult{err} 26 | } 27 | 28 | func (st *syncTask) ImplName() string { return "inodedbsyncer.syncTask" } 29 | -------------------------------------------------------------------------------- /testutils/testca/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBnjCCAUSgAwIBAgIBATAKBggqhkjOPQQDAjA0MQswCQYDVQQGEwJKUDEOMAwG 3 | A1UECBMFVG9reW8xFTATBgNVBAMTDG90YXJ1IGRldiBDQTAgFw0yMjA0MTcxNDM4 4 | MDlaGA8yMDk5MTIzMTIzNTkwMFowNDELMAkGA1UEBhMCSlAxDjAMBgNVBAgTBVRv 5 | a3lvMRUwEwYDVQQDEwxvdGFydSBkZXYgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMB 6 | BwNCAASp3G3PPRFiCaq4q5TfKPzCh2fqDQR1dHOWQUQ4xUBVbbIj78jlZnzj/iFt 7 | w8Moe7rxVDD0xRhB+QpLgfP+cpfqo0UwQzAOBgNVHQ8BAf8EBAMCAqQwEgYDVR0T 8 | AQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUbGsObM3asfV6fhyLb4YDQ4fcQ2cwCgYI 9 | KoZIzj0EAwIDSAAwRQIhANTrxIW0w2hhranrwTgmOZ2utsE51ObqflmcJDQdz1Cj 10 | AiAHhp1zE/PY8KuWwpH+XD4OyJX+IY80XuwYW7kH4bfN6g== 11 | -----END CERTIFICATE----- 12 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_admin.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBojCCAUmgAwIBAgIIQS10fo+eVE0wCgYIKoZIzj0EAwIwPzELMAkGA1UEBhMC 3 | SlAxDjAMBgNVBAgTBVRva3lvMSAwHgYDVQQDExdvdGFydSBkZXYgY2xpZW50YXV0 4 | aCBDQTAgFw0yMjA1MDQwNTE5MTVaGA8yMDk5MTIzMTIzNTkwMFowFjEUMBIGA1UE 5 | AxMLYWRtaW4gYWxpY2UwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARJfV8KErOF 6 | MTNeICYgW6VeXohMyXLAAkDgCVoANcnqNPxRoEkGUyeNDut6RipwYpqkwcr1sB0e 7 | W1KDJx0jFsx+o1YwVDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUH 8 | AwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBQ6TxkyIOt4rqprYdAORlV5gYfq 9 | ujAKBggqhkjOPQQDAgNHADBEAiBj9xEhh43CNdkrILOqv/xEoEVnn/XBfgNagkuS 10 | 9ceWcAIgHV3xjS5CCuvjLOz1wNI9Da8u4RyHcGx/+4EqPI2x5Kk= 11 | -----END CERTIFICATE----- 12 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_readonly.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBozCCAUqgAwIBAgIIaQ3XfSccMZowCgYIKoZIzj0EAwIwPzELMAkGA1UEBhMC 3 | SlAxDjAMBgNVBAgTBVRva3lvMSAwHgYDVQQDExdvdGFydSBkZXYgY2xpZW50YXV0 4 | aCBDQTAgFw0yMjA1MDQwNTE5MTVaGA8yMDk5MTIzMTIzNTkwMFowFzEVMBMGA1UE 5 | AxMMcmVhZG9ubHkgYm9iMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE7kFF8CsS 6 | z0uPigXJhTZQdoXCJ2mz/dpKupDMpBZqTmiUxqeIokW9lRxh+6MQGcS7s9VkfqeT 7 | lPYrd88pVe38YqNWMFQwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUF 8 | BwMCMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUOk8ZMiDreK6qa2HQDkZVeYGH 9 | 6rowCgYIKoZIzj0EAwIDRwAwRAIgNF4NuSrfWSg8u2aDlKpwquxRLkW7xkmVRABD 10 | 9/yRB4gCIAKkDdEHtMUjTlRPvrEFE9jmGW4HuvLDhp+ycLANOYwz 11 | -----END CERTIFICATE----- 12 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "otaru", 3 | "image": "mcr.microsoft.com/devcontainers/go", 4 | "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], 5 | "postCreateCommand": "./.devcontainer/postCreate.sh", 6 | "remoteUser": "vscode", 7 | "customizations": { 8 | "vscode": { 9 | "settings": { 10 | "terminal.integrated.defaultProfile.linux": "zsh", 11 | "terminal.integrated.profiles.linux": { 12 | "zsh": { 13 | "path": "/usr/bin/zsh", 14 | }, 15 | }, 16 | }, 17 | 18 | "extensions": [ 19 | "golang.go" 20 | ] 21 | } 22 | }, 23 | "mounts": [ 24 | // "source=/var/otaru,target=/var/otaru,type=bind", 25 | ], 26 | } 27 | -------------------------------------------------------------------------------- /assets/swaggerui/LICENSE: -------------------------------------------------------------------------------- 1 | dist/ contains swagger-ui, licensed as follows: 2 | 3 | Copyright 2017 SmartBear Software 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at [apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /cmd/otaru/fe/main.go: -------------------------------------------------------------------------------- 1 | package fe 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | 6 | "github.com/nyaxt/otaru/apiserver" 7 | ocli "github.com/nyaxt/otaru/cli" 8 | fe_apiserver "github.com/nyaxt/otaru/extra/fe/apiserver" 9 | ) 10 | 11 | var Command = &cli.Command{ 12 | Name: "frontend", 13 | Aliases: []string{"fe"}, 14 | Usage: "Run otaru frontend web server", 15 | Action: func(c *cli.Context) error { 16 | cfg, err := ocli.NewConfig(c.Path("configDir")) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | opts, err := fe_apiserver.BuildApiServerOptions(cfg) 22 | if err != nil { 23 | return err 24 | } 25 | return apiserver.Serve(c.Context, opts...) 26 | }, 27 | } 28 | -------------------------------------------------------------------------------- /testutils/ensurelogger.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "sync" 5 | 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | var ensureLoggerOnce sync.Once 11 | 12 | var ObservedFatalLog bool 13 | 14 | type testFatalHook struct{} 15 | 16 | var _ zapcore.CheckWriteHook = testFatalHook{} 17 | 18 | func (testFatalHook) OnWrite(*zapcore.CheckedEntry, []zap.Field) { 19 | ObservedFatalLog = true 20 | } 21 | 22 | func EnsureLogger() { 23 | ObservedFatalLog = false 24 | 25 | ensureLoggerOnce.Do(func() { 26 | logger, err := zap.NewDevelopment(zap.WithFatalHook(testFatalHook{})) 27 | if err != nil { 28 | panic(err) 29 | } 30 | zap.ReplaceGlobals(logger) 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/default/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBnjCCAUSgAwIBAgIBATAKBggqhkjOPQQDAjA0MQswCQYDVQQGEwJKUDEOMAwG 3 | A1UECBMFVG9reW8xFTATBgNVBAMTDG90YXJ1IGRldiBDQTAgFw0yMjA0MTcxNDM4 4 | MDlaGA8yMDk5MTIzMTIzNTkwMFowNDELMAkGA1UEBhMCSlAxDjAMBgNVBAgTBVRv 5 | a3lvMRUwEwYDVQQDEwxvdGFydSBkZXYgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMB 6 | BwNCAASp3G3PPRFiCaq4q5TfKPzCh2fqDQR1dHOWQUQ4xUBVbbIj78jlZnzj/iFt 7 | w8Moe7rxVDD0xRhB+QpLgfP+cpfqo0UwQzAOBgNVHQ8BAf8EBAMCAqQwEgYDVR0T 8 | AQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUbGsObM3asfV6fhyLb4YDQ4fcQ2cwCgYI 9 | KoZIzj0EAwIDSAAwRQIhANTrxIW0w2hhranrwTgmOZ2utsE51ObqflmcJDQdz1Cj 10 | AiAHhp1zE/PY8KuWwpH+XD4OyJX+IY80XuwYW7kH4bfN6g== 11 | -----END CERTIFICATE----- 12 | -------------------------------------------------------------------------------- /util/path.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func IsDir(path string) error { 9 | fi, err := os.Stat(path) 10 | if err != nil { 11 | if os.IsNotExist(err) { 12 | return err 13 | } 14 | return fmt.Errorf("Error os.Stat: %v", err) 15 | } 16 | if !fi.IsDir() { 17 | return fmt.Errorf("Is not a dir") 18 | } 19 | return nil 20 | } 21 | 22 | func IsRegular(path string) error { 23 | fi, err := os.Stat(path) 24 | if err != nil { 25 | if os.IsNotExist(err) { 26 | return err 27 | } 28 | return fmt.Errorf("Error os.Stat: %v", err) 29 | } 30 | if !fi.Mode().IsRegular() { 31 | return fmt.Errorf("Is not a regular file") 32 | } 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_invalid.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBqDCCAU2gAwIBAgIIWUsYawDjJ4AwCgYIKoZIzj0EAwIwPzELMAkGA1UEBhMC 3 | SlAxDjAMBgNVBAgTBVRva3lvMSAwHgYDVQQDExdvdGFydSBkZXYgY2xpZW50YXV0 4 | aCBDQTAgFw0yMjA1MDQwNTE5MTVaGA8yMDk5MTIzMTIzNTkwMFowGjEYMBYGA1UE 5 | AxMPaW52YWxpZCBjaGFybGllMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEGL+Q 6 | T+rCJl8VAe0A2rHArg51tmXay4on+LGfdvCxsKi3g4lJ4j2mwRpbGPePqbr8dh79 7 | K69xZw34LYwjnJlL/6NWMFQwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsG 8 | AQUFBwMCMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUOk8ZMiDreK6qa2HQDkZV 9 | eYGH6rowCgYIKoZIzj0EAwIDSQAwRgIhAO0gsOeNiJ/5Cmd0OptybTDjNNo/i1IY 10 | 2VveXQyVfAGHAiEA3MzywChstntqaTQDAAK7ZnffBU5EC6iA2o24TsjZH24= 11 | -----END CERTIFICATE----- 12 | -------------------------------------------------------------------------------- /util/guaranteedpool.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type GuaranteedPool struct { 8 | bufC chan interface{} 9 | pool *sync.Pool 10 | } 11 | 12 | func NewGuaranteedPool(new func() interface{}, guaranteed int) *GuaranteedPool { 13 | return &GuaranteedPool{ 14 | bufC: make(chan interface{}, guaranteed), 15 | pool: &sync.Pool{New: new}, 16 | } 17 | } 18 | 19 | func (p *GuaranteedPool) Get() interface{} { 20 | select { 21 | case x := <-p.bufC: 22 | return x 23 | default: 24 | return p.pool.Get() 25 | } 26 | } 27 | 28 | func (p *GuaranteedPool) Put(x interface{}) { 29 | select { 30 | case p.bufC <- x: 31 | return 32 | default: 33 | p.pool.Put(x) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /testutils/testca/wrong_cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBsjCCAVmgAwIBAgIIOO7xcpBeOewwCgYIKoZIzj0EAwIwIDELMAkGA1UEBhMC 3 | VVMxETAPBgNVBAMTCHdyb25nIENBMCAXDTIyMDUwNDA1MTkxNVoYDzIwOTkxMjMx 4 | MjM1OTAwWjAlMSMwIQYDVQQDExpvdGFydSB0ZXN0IHNlcnZlciB3cm9uZyBjYTBZ 5 | MBMGByqGSM49AgEGCCqGSM49AwEHA0IABKOnoSQidHhNkOIrOnZfMkNRPs4XbSIf 6 | IAFLhHJ5HnckjDXrLv5XsUoewje3+fVhh6TaOJL6kbK2t03p4D65moqjdjB0MA4G 7 | A1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYD 8 | VR0TAQH/BAIwADAfBgNVHSMEGDAWgBRqvS7LeYfYLKzWXoVIrYunFWyxeTAUBgNV 9 | HREEDTALgglsb2NhbGhvc3QwCgYIKoZIzj0EAwIDRwAwRAIgaV25xNdYgiZWxFe8 10 | CikLkw5g3jq9sVn4zmGs8BNXd9gCIAoD0Yb6IcAlAh/fmMDdlAiPjc9t7INby0q2 11 | fw4Miu6i 12 | -----END CERTIFICATE----- 13 | -------------------------------------------------------------------------------- /blobstore/cachedblobstore/cacheusagestats_test.go: -------------------------------------------------------------------------------- 1 | package cachedblobstore_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/nyaxt/otaru/blobstore/cachedblobstore" 8 | fl "github.com/nyaxt/otaru/flags" 9 | ) 10 | 11 | func TestCacheUsageStats_FindLeastUsed(t *testing.T) { 12 | s := cachedblobstore.NewCacheUsageStats() 13 | s.ImportBlobList([]string{"nevertouched", "touched", "removed"}) 14 | s.ObserveOpen("touched", fl.O_RDONLY) 15 | s.ObserveRemoveBlob("removed") 16 | s.ObserveOpen("new", fl.O_WRONLY) 17 | 18 | leastUsed := s.FindLeastUsed() 19 | if !reflect.DeepEqual([]string{"nevertouched", "touched", "new"}, leastUsed) { 20 | t.Errorf("Unexpected result: %v", leastUsed) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /testutils/testca/clientauth_cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBtDCCAVqgAwIBAgIBATAKBggqhkjOPQQDAjA/MQswCQYDVQQGEwJKUDEOMAwG 3 | A1UECBMFVG9reW8xIDAeBgNVBAMTF290YXJ1IGRldiBjbGllbnRhdXRoIENBMCAX 4 | DTIyMDUwNDA1MTkxNFoYDzIwOTkxMjMxMjM1OTAwWjA/MQswCQYDVQQGEwJKUDEO 5 | MAwGA1UECBMFVG9reW8xIDAeBgNVBAMTF290YXJ1IGRldiBjbGllbnRhdXRoIENB 6 | MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJn/2W2P1uJCm7ENPrXf//xUyyrQ0 7 | 94QczeKiwAkCdgD0Z8WejpgDRLJBvThMDj4aGgufL6sC7yBpMz+8l5AYh6NFMEMw 8 | DgYDVR0PAQH/BAQDAgKkMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFDpP 9 | GTIg63iuqmth0A5GVXmBh+q6MAoGCCqGSM49BAMCA0gAMEUCIQDKInodwVhxNu93 10 | C5d0pLAS+Hhqj7DcZCWbfm6SfrMXmAIgX9WYVMxTrNZ7SNBxJctR0KOjKvOoj+tR 11 | tOsId2SH6cg= 12 | -----END CERTIFICATE----- 13 | -------------------------------------------------------------------------------- /testutils/testfilesystem.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "log" 5 | 6 | "go.uber.org/zap" 7 | 8 | "github.com/nyaxt/otaru/filesystem" 9 | "github.com/nyaxt/otaru/inodedb" 10 | ) 11 | 12 | func TestINodeDB() inodedb.DBHandler { 13 | snapshotio := inodedb.NewSimpleDBStateSnapshotIO() 14 | txio := inodedb.NewSimpleDBTransactionLogIO() 15 | idb, err := inodedb.NewEmptyDB(snapshotio, txio) 16 | if err != nil { 17 | log.Panicf("NewEmptyDB failed: %v", err) 18 | } 19 | 20 | return idb 21 | } 22 | 23 | func TestFileSystem() *filesystem.FileSystem { 24 | EnsureLogger() 25 | 26 | idb := TestINodeDB() 27 | bs := TestFileBlobStore() 28 | return filesystem.NewFileSystem(idb, bs, TestCipher(), zap.L()) 29 | } 30 | -------------------------------------------------------------------------------- /util/gzip.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "io/ioutil" 7 | ) 8 | 9 | func Gzip(p []byte) ([]byte, error) { 10 | var b bytes.Buffer 11 | w := gzip.NewWriter(&b) 12 | if _, err := w.Write(p); err != nil { 13 | return nil, err 14 | } 15 | if err := w.Close(); err != nil { 16 | return nil, err 17 | } 18 | return b.Bytes(), nil 19 | } 20 | 21 | func Gunzip(p []byte) ([]byte, error) { 22 | b := bytes.NewBuffer(p) 23 | r, err := gzip.NewReader(b) 24 | if err != nil { 25 | return nil, err 26 | } 27 | u, err := ioutil.ReadAll(r) 28 | if err != nil { 29 | return nil, err 30 | } 31 | if err := r.Close(); err != nil { 32 | return nil, err 33 | } 34 | return u, nil 35 | } 36 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/clientauth/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBtDCCAVqgAwIBAgIBATAKBggqhkjOPQQDAjA/MQswCQYDVQQGEwJKUDEOMAwG 3 | A1UECBMFVG9reW8xIDAeBgNVBAMTF290YXJ1IGRldiBjbGllbnRhdXRoIENBMCAX 4 | DTIyMDUwNDA1MTkxNFoYDzIwOTkxMjMxMjM1OTAwWjA/MQswCQYDVQQGEwJKUDEO 5 | MAwGA1UECBMFVG9reW8xIDAeBgNVBAMTF290YXJ1IGRldiBjbGllbnRhdXRoIENB 6 | MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJn/2W2P1uJCm7ENPrXf//xUyyrQ0 7 | 94QczeKiwAkCdgD0Z8WejpgDRLJBvThMDj4aGgufL6sC7yBpMz+8l5AYh6NFMEMw 8 | DgYDVR0PAQH/BAQDAgKkMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFDpP 9 | GTIg63iuqmth0A5GVXmBh+q6MAoGCCqGSM49BAMCA0gAMEUCIQDKInodwVhxNu93 10 | C5d0pLAS+Hhqj7DcZCWbfm6SfrMXmAIgX9WYVMxTrNZ7SNBxJctR0KOjKvOoj+tR 11 | tOsId2SH6cg= 12 | -----END CERTIFICATE----- 13 | -------------------------------------------------------------------------------- /cmd/otaru/serve/serve.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | 6 | "github.com/nyaxt/otaru/facade" 7 | ) 8 | 9 | var Command = &cli.Command{ 10 | Name: "serve", 11 | Usage: "Run otaru gRPC server", 12 | Flags: []cli.Flag{ 13 | &cli.BoolFlag{ 14 | Name: "readonly", 15 | Usage: "Mount as read-only mode. No changes to the filesystem is allowed.", 16 | }, 17 | }, 18 | Action: func(c *cli.Context) error { 19 | cfg, err := facade.NewConfig(c.Path("configDir")) 20 | if err != nil { 21 | return err 22 | } 23 | if c.Bool("readonly") { 24 | cfg.ReadOnly = true 25 | } 26 | 27 | if err := facade.Serve(c.Context, cfg); err != nil { 28 | return err 29 | } 30 | 31 | return nil 32 | }, 33 | } 34 | -------------------------------------------------------------------------------- /testutils/testca/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBzzCCAXSgAwIBAgIIJIzA1Xm/x0owCgYIKoZIzj0EAwIwNDELMAkGA1UEBhMC 3 | SlAxDjAMBgNVBAgTBVRva3lvMRUwEwYDVQQDEwxvdGFydSBkZXYgQ0EwIBcNMjIw 4 | NTA0MDUxOTE0WhgPMjA5OTEyMzEyMzU5MDBaMCwxDjAMBgNVBAoTBW90YXJ1MRow 5 | GAYDVQQDExFvdGFydSB0ZXN0IHNlcnZlcjBZMBMGByqGSM49AgEGCCqGSM49AwEH 6 | A0IABPmX/9jxlcbtHLkdblHLXtUgyl+rAT79U5GXyZqvnWNlaBOzpe4K2JybV7L5 7 | FaBoIuajGgV/8VYEKMx5FxIFp0WjdjB0MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUE 8 | FjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAW 9 | gBRsaw5szdqx9Xp+HItvhgNDh9xDZzAUBgNVHREEDTALgglsb2NhbGhvc3QwCgYI 10 | KoZIzj0EAwIDSQAwRgIhAIRVnPZUk2NooS2l2w1nhryniPfXwzidR0j5jZ0ty+Ch 11 | AiEAw+Vd08U5QWin+41AGtGfsuMpi8a6y1uyJ2oKiFvHKc0= 12 | -----END CERTIFICATE----- 13 | -------------------------------------------------------------------------------- /apiserver/clientauth/role.go: -------------------------------------------------------------------------------- 1 | package clientauth 2 | 3 | type Role int 4 | 5 | const ( 6 | RoleAnonymous Role = iota 7 | RoleReadOnly 8 | RoleAdmin 9 | ) 10 | 11 | var strToRole = map[string]Role{ 12 | "anonymous": RoleAnonymous, 13 | "readonly": RoleReadOnly, 14 | "admin": RoleAdmin, 15 | } 16 | 17 | var roleToStr = map[Role]string{ 18 | RoleAnonymous: "anonymous", 19 | RoleReadOnly: "readonly", 20 | RoleAdmin: "admin", 21 | } 22 | 23 | func IsValidRoleStr(s string) bool { 24 | _, ok := strToRole[s] 25 | return ok 26 | } 27 | 28 | func RoleFromStr(s string) Role { 29 | if r, ok := strToRole[s]; ok { 30 | return r 31 | } 32 | return RoleAnonymous 33 | } 34 | 35 | func (r Role) String() string { 36 | return roleToStr[r] 37 | } 38 | -------------------------------------------------------------------------------- /basicauth/basicauth.go: -------------------------------------------------------------------------------- 1 | package basicauth 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type Handler struct { 8 | User string 9 | Password string 10 | Handler http.Handler 11 | } 12 | 13 | func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 14 | user, password, ok := r.BasicAuth() 15 | if !ok || user != h.User || password != h.Password { 16 | // zap.S().Infof("Authentication failed. User: %q Password: %q Header: %+v", user, password, r.Header) 17 | 18 | w.Header().Set("WWW-Authenticate", "Basic realm=\"otaru\"") 19 | w.Header().Set("Content-Type", "text/plain") 20 | w.WriteHeader(http.StatusUnauthorized) 21 | _, _ = w.Write([]byte("Authentication Required.")) 22 | return 23 | } 24 | 25 | h.Handler.ServeHTTP(w, r) 26 | } 27 | -------------------------------------------------------------------------------- /assets/webui/handler.go: -------------------------------------------------------------------------------- 1 | package webui 2 | 3 | import ( 4 | "embed" 5 | "net/http" 6 | ) 7 | 8 | //go:embed dist/* 9 | var fs embed.FS 10 | 11 | var Handler http.Handler 12 | 13 | func init() { 14 | fs := http.FileServer(http.FS(fs)) 15 | Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | r.URL.Path = "/dist" + r.URL.Path 17 | fs.ServeHTTP(w, r) 18 | }) 19 | } 20 | 21 | func WebUIHandler(override, indexpath string) http.Handler { 22 | handler := Handler 23 | if override != "" { 24 | handler = http.FileServer(http.Dir(override)) 25 | } 26 | 27 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 28 | if r.URL.Path == "/" { 29 | r.URL.Path = indexpath 30 | } 31 | handler.ServeHTTP(w, r) 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /assets/webui/dist/index.otaru-fe.html: -------------------------------------------------------------------------------- 1 | 2 | Otaru WebUI: Initializing 3 | 4 | 5 | 6 | 10 | 11 | 12 |
13 | 14 |
15 | 16 |
17 | 18 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/wrong/issuedb.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "sn": 4102481784931760620, 4 | "state": "active", 5 | "certPem": "-----BEGIN CERTIFICATE-----\nMIIBsjCCAVmgAwIBAgIIOO7xcpBeOewwCgYIKoZIzj0EAwIwIDELMAkGA1UEBhMC\nVVMxETAPBgNVBAMTCHdyb25nIENBMCAXDTIyMDUwNDA1MTkxNVoYDzIwOTkxMjMx\nMjM1OTAwWjAlMSMwIQYDVQQDExpvdGFydSB0ZXN0IHNlcnZlciB3cm9uZyBjYTBZ\nMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKOnoSQidHhNkOIrOnZfMkNRPs4XbSIf\nIAFLhHJ5HnckjDXrLv5XsUoewje3+fVhh6TaOJL6kbK2t03p4D65moqjdjB0MA4G\nA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYD\nVR0TAQH/BAIwADAfBgNVHSMEGDAWgBRqvS7LeYfYLKzWXoVIrYunFWyxeTAUBgNV\nHREEDTALgglsb2NhbGhvc3QwCgYIKoZIzj0EAwIDRwAwRAIgaV25xNdYgiZWxFe8\nCikLkw5g3jq9sVn4zmGs8BNXd9gCIAoD0Yb6IcAlAh/fmMDdlAiPjc9t7INby0q2\nfw4Miu6i\n-----END CERTIFICATE-----\n" 6 | } 7 | ] -------------------------------------------------------------------------------- /cli/path/path_test.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestParse(t *testing.T) { 9 | testcases := []struct { 10 | input string 11 | expected Path 12 | }{ 13 | {"otaru://vhost/fuga/hoge.txt", Path{"vhost", "/fuga/hoge.txt"}}, 14 | {"//vhost/fuga/hoge.txt", Path{"vhost", "/fuga/hoge.txt"}}, 15 | {"otaru:/fuga/hoge.txt", Path{"default", "/fuga/hoge.txt"}}, 16 | {"/1/2/3.txt", Path{"default", "/1/2/3.txt"}}, 17 | } 18 | 19 | for _, tc := range testcases { 20 | p, err := Parse(tc.input) 21 | if err != nil { 22 | t.Errorf("parse input: \"%s\" err: %v", tc.input, err) 23 | } 24 | if !reflect.DeepEqual(p, tc.expected) { 25 | t.Errorf("parse input: \"%s\" exp: %+v act: %+v", tc.input, tc.expected, p) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /filewritecache/filewritecache_test.go: -------------------------------------------------------------------------------- 1 | package filewritecache_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/nyaxt/otaru/blobstore" 7 | "github.com/nyaxt/otaru/filewritecache" 8 | tu "github.com/nyaxt/otaru/testutils" 9 | ) 10 | 11 | func init() { tu.EnsureLogger() } 12 | 13 | func TestRegression_PWriteAfterSync(t *testing.T) { 14 | bh := blobstore.NewMockBlobHandle() 15 | 16 | wc := filewritecache.New() 17 | if err := wc.PWrite([]byte{1, 2, 3}, 0); err != nil { 18 | t.Errorf("PWrite failed: %v", err) 19 | return 20 | } 21 | 22 | if err := wc.Sync(bh); err != nil { 23 | t.Errorf("Sync failed: %v", err) 24 | } 25 | 26 | if err := wc.PWrite([]byte{4, 5, 6}, 3); err != nil { 27 | t.Errorf("PWrite failed: %v", err) 28 | return 29 | } 30 | // Test PASS if no panic 31 | } 32 | -------------------------------------------------------------------------------- /cli/cert.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | 8 | "go.uber.org/zap" 9 | ) 10 | 11 | func TLSConfigFromCert(cert *x509.Certificate) (*tls.Config, error) { 12 | cp := x509.NewCertPool() 13 | cp.AddCert(cert) 14 | 15 | // Subject Alt Name DNSNames are preferred over CommonName, since CommonName is ignored when SAN available. 16 | var serverNames []string 17 | serverNames = append(serverNames, cert.DNSNames...) 18 | serverNames = append(serverNames, cert.Subject.CommonName) 19 | 20 | if len(serverNames) == 0 { 21 | return nil, fmt.Errorf("Failed to find any valid server name from given certs.") 22 | } 23 | 24 | zap.S().Infof("Found server names %v. Using the first entry for grpc loopback connection.", serverNames) 25 | return &tls.Config{ServerName: serverNames[0], RootCAs: cp}, nil 26 | } 27 | -------------------------------------------------------------------------------- /gcloud/auth/auth.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | 8 | "cloud.google.com/go/datastore" 9 | "cloud.google.com/go/storage" 10 | "golang.org/x/oauth2" 11 | "golang.org/x/oauth2/google" 12 | ) 13 | 14 | func GetGCloudTokenSource(credentialsFilePath string) (oauth2.TokenSource, error) { 15 | credentialsJson, err := ioutil.ReadFile(credentialsFilePath) 16 | if err != nil { 17 | return nil, fmt.Errorf("failed to read google cloud client-secret file: %v", err) 18 | } 19 | 20 | conf, err := google.JWTConfigFromJSON( 21 | credentialsJson, 22 | storage.ScopeFullControl, 23 | datastore.ScopeDatastore, 24 | ) 25 | if err != nil { 26 | return nil, fmt.Errorf("invalid google cloud key json \"%v\" err: %v", credentialsFilePath, err) 27 | } 28 | 29 | return conf.TokenSource(context.Background()), nil 30 | } 31 | -------------------------------------------------------------------------------- /scheduler/repetitive_test.go: -------------------------------------------------------------------------------- 1 | package scheduler_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/nyaxt/otaru/scheduler" 8 | ) 9 | 10 | func TestRepetitiveJobRunner_RunEveryPeriod(t *testing.T) { 11 | s := scheduler.NewScheduler() 12 | r := scheduler.NewRepetitiveJobRunner(s) 13 | 14 | counter = 0 15 | rid := r.RunEveryPeriod(HogeTask{}, 50*time.Millisecond) 16 | 17 | if counter != 0 { 18 | t.Errorf("err") 19 | } 20 | time.Sleep(1 * time.Second) 21 | 22 | if counter < 2 { 23 | t.Errorf("Should have run at least 2 times: counter %d", counter) 24 | } 25 | 26 | if err := r.Abort(rid); err != nil { 27 | t.Errorf("Abort err: %v", err) 28 | } 29 | time.Sleep(500 * time.Millisecond) 30 | 31 | ss := counter 32 | time.Sleep(500 * time.Millisecond) 33 | if ss != counter { 34 | t.Errorf("Detected task run after Abort(): %d != %d", ss, counter) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /assets/swaggerui/dist/index.js: -------------------------------------------------------------------------------- 1 | try { 2 | module.exports.SwaggerUIBundle = require("./swagger-ui-bundle.js") 3 | module.exports.SwaggerUIStandalonePreset = require("./swagger-ui-standalone-preset.js") 4 | } catch(e) { 5 | // swallow the error if there's a problem loading the assets. 6 | // allows this module to support providing the assets for browserish contexts, 7 | // without exploding in a Node context. 8 | // 9 | // see https://github.com/swagger-api/swagger-ui/issues/3291#issuecomment-311195388 10 | // for more information. 11 | } 12 | 13 | // `absolutePath` and `getAbsoluteFSPath` are both here because at one point, 14 | // we documented having one and actually implemented the other. 15 | // They were both retained so we don't break anyone's code. 16 | module.exports.absolutePath = require("./absolute-path.js") 17 | module.exports.getAbsoluteFSPath = require("./absolute-path.js") 18 | -------------------------------------------------------------------------------- /webdav/error.go: -------------------------------------------------------------------------------- 1 | package webdav 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | gwruntime "github.com/grpc-ecosystem/grpc-gateway/runtime" 8 | "google.golang.org/grpc/status" 9 | ) 10 | 11 | type Error struct { 12 | HttpStatusCode int 13 | Context string 14 | Err error 15 | } 16 | 17 | func (e Error) Error() string { 18 | return fmt.Sprintf("%s: %v", e.Context, e.Err) 19 | } 20 | 21 | func ErrorFromGrpc(e error, context string) Error { 22 | s, ok := status.FromError(e) 23 | if !ok { 24 | return Error{http.StatusInternalServerError, context, e} 25 | } 26 | return Error{gwruntime.HTTPStatusFromCode(s.Code()), context, e} 27 | } 28 | 29 | func WriteError(w http.ResponseWriter, err error) { 30 | if mye, ok := err.(Error); ok { 31 | http.Error(w, mye.Error(), mye.HttpStatusCode) 32 | return 33 | } 34 | http.Error(w, err.Error(), http.StatusInternalServerError) 35 | } 36 | -------------------------------------------------------------------------------- /inodedb/blobstoredbstatesnapshotio/simplesslocator.go: -------------------------------------------------------------------------------- 1 | package blobstoredbstatesnapshotio 2 | 3 | import ( 4 | "fmt" 5 | 6 | "context" 7 | 8 | "github.com/nyaxt/otaru/metadata" 9 | ) 10 | 11 | func generateBlobpath() string { 12 | return fmt.Sprintf("%s_SimpleSSLocator", metadata.INodeDBSnapshotBlobpathPrefix) 13 | } 14 | 15 | var simplesslocatorTxID int64 16 | 17 | type SimpleSSLocator struct{} 18 | 19 | func (SimpleSSLocator) Locate(history int) (string, int64, error) { 20 | return generateBlobpath(), simplesslocatorTxID, nil 21 | } 22 | 23 | func (SimpleSSLocator) GenerateBlobpath() string { 24 | return generateBlobpath() 25 | } 26 | 27 | func (SimpleSSLocator) Put(blobpath string, txid int64) error { 28 | simplesslocatorTxID = txid 29 | return nil 30 | } 31 | 32 | func (SimpleSSLocator) DeleteOld(ctx context.Context, threshold int, dryRun bool) ([]string, error) { 33 | return []string{}, nil 34 | } 35 | -------------------------------------------------------------------------------- /doc/cachedblobstore.md: -------------------------------------------------------------------------------- 1 | # ChunkIO and CachedBlobStore 2 | 3 | - ChunkedFileIO.PWrite() 4 | -- cio := NewChunkIO() 5 | --- bh := cbs.Open() 6 | -- cio.PWrite() 7 | --- bh.PWrite() 8 | -- cio.Close() -> cio.Sync() 9 | --- bh.PWrite() (updateHeader) 10 | 11 | # State 12 | 13 | writeBackWithLock is called from: 14 | - Sync 15 | - Close 16 | 17 | and we have to ensure that: 18 | - forbid new write during writeBack, or at least 19 | - know that new write occured during the writeBack 20 | 21 | Which is better? 22 | - allow new write 23 | - simpler impl 24 | - less blocking 25 | - forbid new write during writeback 26 | - ??? 27 | 28 | Q: Should we really sync during chunk write? Good chance where chunk may become unreadable. 29 | A: Yes 30 | - Supporting transactional IO in cachedblobstore makes it even more complicated. 31 | - ChooseSyncEntry would do best to avoid the situation. 32 | - We should make ChunkedIO tolerant to bad blocks anyway. 33 | -------------------------------------------------------------------------------- /util/err.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "syscall" 5 | 6 | "github.com/nyaxt/fuse" 7 | ) 8 | 9 | type Error syscall.Errno 10 | 11 | const ( 12 | EACCES = Error(syscall.EACCES) 13 | EBADF = Error(syscall.EBADF) 14 | EEXIST = Error(syscall.EEXIST) 15 | EISDIR = Error(syscall.EISDIR) 16 | ENFILE = Error(syscall.ENFILE) 17 | ENOENT = Error(syscall.ENOENT) 18 | ENOTDIR = Error(syscall.ENOTDIR) 19 | ENOTEMPTY = Error(syscall.ENOTEMPTY) 20 | EPERM = Error(syscall.EPERM) 21 | ) 22 | 23 | func (e Error) Errno() fuse.Errno { 24 | return fuse.Errno(e) 25 | } 26 | 27 | func (e Error) Error() string { 28 | return syscall.Errno(e).Error() 29 | } 30 | 31 | func IsExist(e error) bool { 32 | e2, ok := e.(Error) 33 | if !ok { 34 | return false 35 | } 36 | return e2 == EEXIST 37 | } 38 | 39 | func IsNotExist(e error) bool { 40 | e2, ok := e.(Error) 41 | if !ok { 42 | return false 43 | } 44 | return e2 == ENOENT 45 | } 46 | -------------------------------------------------------------------------------- /logger/writerlogger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "time" 7 | ) 8 | 9 | type WriterLogger struct { 10 | W io.Writer 11 | } 12 | 13 | type syncer interface { // avoid using github.com/nyaxt/otaru/util.Syncer for refcycle 14 | Sync() error 15 | } 16 | 17 | func (l WriterLogger) Log(lv Level, data map[string]interface{}) { 18 | var b bytes.Buffer 19 | t := data["time"].(time.Time) 20 | b.WriteString(t.Format("2006/01/02 15:04:05 ")) 21 | b.WriteRune(lv.Rune()) 22 | b.WriteRune(' ') 23 | if c, ok := data["category"]; ok { 24 | b.WriteString("[") 25 | b.WriteString(c.(string)) 26 | b.WriteString("] ") 27 | } 28 | 29 | b.WriteString(data["location"].(string)) 30 | b.WriteString(": ") 31 | b.WriteString(data["log"].(string)) 32 | b.WriteString("\n") 33 | l.W.Write(b.Bytes()) 34 | 35 | if s, ok := l.W.(syncer); ok { 36 | s.Sync() 37 | } 38 | } 39 | 40 | func (l WriterLogger) WillAccept(lv Level) bool { return true } 41 | -------------------------------------------------------------------------------- /assets/swaggerui/dist/README.md: -------------------------------------------------------------------------------- 1 | # Swagger UI Dist 2 | [![NPM version](https://badge.fury.io/js/swagger-ui-dist.svg)](http://badge.fury.io/js/swagger-ui-dist) 3 | 4 | # API 5 | 6 | This module, `swagger-ui-dist`, exposes Swagger-UI's entire dist folder as a dependency-free npm module. 7 | Use `swagger-ui` instead, if you'd like to have npm install dependencies for you. 8 | 9 | `SwaggerUIBundle` and `SwaggerUIStandalonePreset` can be imported: 10 | ```javascript 11 | import { SwaggerUIBundle, SwaggerUIStandalonePreset } from "swagger-ui-dist" 12 | ``` 13 | 14 | To get an absolute path to this directory for static file serving, use the exported `getAbsoluteFSPath` method: 15 | 16 | ```javascript 17 | const swaggerUiAssetPath = require("swagger-ui-dist").getAbsoluteFSPath() 18 | 19 | // then instantiate server that serves files from the swaggerUiAssetPath 20 | ``` 21 | 22 | For anything else, check the [Swagger-UI](https://github.com/swagger-api/swagger-ui) repository. 23 | -------------------------------------------------------------------------------- /util/cancellable/io.go: -------------------------------------------------------------------------------- 1 | package cancellable 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "context" 8 | ) 9 | 10 | type CancelledErr struct { 11 | Orig error 12 | } 13 | 14 | func (e CancelledErr) Error() string { 15 | return fmt.Sprintf("Context was cancelled during IO: %v", e.Orig) 16 | } 17 | 18 | func IsCancelledErr(e error) bool { 19 | if e == nil { 20 | return false 21 | } 22 | if e == context.Canceled { 23 | return true 24 | } 25 | 26 | _, ok := e.(CancelledErr) 27 | return ok 28 | } 29 | 30 | func Read(ctx context.Context, r io.Reader, p []byte) (int, error) { 31 | if err := ctx.Err(); err != nil { 32 | return 0, CancelledErr{err} 33 | } 34 | 35 | var n int 36 | var err error 37 | complete := make(chan struct{}) 38 | go func() { 39 | n, err = r.Read(p) 40 | close(complete) 41 | }() 42 | 43 | select { 44 | case <-complete: 45 | return n, err 46 | 47 | case <-ctx.Done(): 48 | return 0, CancelledErr{ctx.Err()} 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcloud/util/checkerr_test.go: -------------------------------------------------------------------------------- 1 | package util_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/nyaxt/otaru/gcloud/util" 8 | ) 9 | 10 | func TestIsShouldRetryError(t *testing.T) { 11 | if util.IsShouldRetryError(nil) { 12 | t.Errorf("IsShouldRetryError should have returned false on nil") 13 | } 14 | 15 | eShouldRetry := errors.New("error during call, http status code: 502") 16 | if !util.IsShouldRetryError(eShouldRetry) { 17 | t.Errorf("IsShouldRetryError should have returned true on: %v", eShouldRetry) 18 | } 19 | 20 | eTooManyRequests := errors.New("status code: 429") 21 | if !util.IsShouldRetryError(eTooManyRequests) { 22 | t.Errorf("IsShouldRetryError should have returned true on: %v", eTooManyRequests) 23 | } 24 | 25 | eShouldNotRetry := errors.New("error during call, http status code: 400") 26 | if util.IsShouldRetryError(eShouldNotRetry) { 27 | t.Errorf("IsShouldRetryError should have returned false on: %v", eShouldNotRetry) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /inodedb/simpledbstatesnapshotio.go: -------------------------------------------------------------------------------- 1 | package inodedb 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "encoding/gob" 8 | ) 9 | 10 | type SimpleDBStateSnapshotIO struct { 11 | Buf bytes.Buffer 12 | } 13 | 14 | var _ = DBStateSnapshotIO(&SimpleDBStateSnapshotIO{}) 15 | 16 | func NewSimpleDBStateSnapshotIO() *SimpleDBStateSnapshotIO { 17 | return &SimpleDBStateSnapshotIO{} 18 | } 19 | 20 | func (io *SimpleDBStateSnapshotIO) SaveSnapshot(s *DBState) <-chan error { 21 | errC := make(chan error, 1) 22 | 23 | io.Buf.Reset() 24 | 25 | enc := gob.NewEncoder(&io.Buf) 26 | if err := s.EncodeToGob(enc); err != nil { 27 | errC <- fmt.Errorf("Failed to encode DBState: %v", err) 28 | } 29 | 30 | close(errC) 31 | return errC 32 | } 33 | 34 | func (io *SimpleDBStateSnapshotIO) RestoreSnapshot() (*DBState, error) { 35 | dec := gob.NewDecoder(&io.Buf) 36 | return DecodeDBStateFromGob(dec) 37 | } 38 | 39 | func (*SimpleDBStateSnapshotIO) ImplName() string { return "SimpleDBStateSnapshotIO" } 40 | -------------------------------------------------------------------------------- /inodedb/simpledbtransactionlogio.go: -------------------------------------------------------------------------------- 1 | package inodedb 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/util" 5 | ) 6 | 7 | type SimpleDBTransactionLogIO struct { 8 | readOnly bool 9 | txs []DBTransaction 10 | } 11 | 12 | var _ = DBTransactionLogIO(&SimpleDBTransactionLogIO{}) 13 | 14 | func NewSimpleDBTransactionLogIO() *SimpleDBTransactionLogIO { 15 | return &SimpleDBTransactionLogIO{readOnly: false} 16 | } 17 | 18 | func (io *SimpleDBTransactionLogIO) SetReadOnly(b bool) { 19 | io.readOnly = b 20 | } 21 | 22 | func (io *SimpleDBTransactionLogIO) AppendTransaction(tx DBTransaction) error { 23 | if io.readOnly { 24 | return util.EACCES 25 | } 26 | io.txs = append(io.txs, tx) 27 | return nil 28 | } 29 | 30 | func (io *SimpleDBTransactionLogIO) QueryTransactions(minID TxID) ([]DBTransaction, error) { 31 | result := []DBTransaction{} 32 | for _, tx := range io.txs { 33 | if tx.TxID >= minID { 34 | result = append(result, tx) 35 | } 36 | } 37 | return result, nil 38 | } 39 | -------------------------------------------------------------------------------- /blobstore/genblobpath_test.go: -------------------------------------------------------------------------------- 1 | package blobstore_test 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/blobstore" 5 | "github.com/nyaxt/otaru/flags" 6 | . "github.com/nyaxt/otaru/testutils" 7 | 8 | "testing" 9 | ) 10 | 11 | func TestGenerateNewBlobPath_Unique(t *testing.T) { 12 | n := 200 13 | bs := blobstore.NewMockBlobStore() 14 | 15 | for i := 0; i < n; i++ { 16 | bpath, err := blobstore.GenerateNewBlobPath(bs) 17 | if err != nil { 18 | t.Errorf("Failed to GenerateNewBlobPath on %d iter: %v", i, err) 19 | } 20 | 21 | bh, err := bs.Open(bpath, flags.O_RDONLY) 22 | if err != nil { 23 | t.Errorf("open bpath \"%s\" failed: %v", bpath, err) 24 | } 25 | if err := bh.PWrite(HelloWorld, 0); err != nil { 26 | t.Errorf("write helloworld to bpath \"%s\" failed: %v", bpath, err) 27 | } 28 | if err := bh.Close(); err != nil { 29 | t.Errorf("close bpath \"%s\" failed: %v", bpath, err) 30 | } 31 | } 32 | 33 | if len(bs.Paths) != n { 34 | t.Errorf("Expected %d unique entries, but found %d entries", n, len(bs.Paths)) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /gcloud/util/checkerr.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "regexp" 5 | "time" 6 | 7 | "cloud.google.com/go/datastore" 8 | "go.uber.org/zap" 9 | 10 | "github.com/nyaxt/otaru/logger" 11 | ) 12 | 13 | var shouldRetryErrorRegexp = regexp.MustCompile(`status code: (429|5\d\d)`) 14 | 15 | func IsShouldRetryError(e error) bool { 16 | if e == nil { 17 | return false 18 | } 19 | 20 | if e == datastore.ErrConcurrentTransaction { 21 | return true 22 | } 23 | 24 | estr := e.Error() 25 | return shouldRetryErrorRegexp.MatchString(estr) 26 | } 27 | 28 | func RetryIfNeeded(f func() error, mylog logger.Logger) (err error) { 29 | const numRetries = 3 30 | for i := 0; i < numRetries; i++ { 31 | start := time.Now() 32 | err = f() 33 | if err == nil { 34 | return 35 | } 36 | if !IsShouldRetryError(err) { 37 | return 38 | } 39 | if i < numRetries { 40 | zap.S().Infof("A Google Cloud API operation has failed after %s. Retrying %d / %d...", time.Since(start), i+1, numRetries) 41 | time.Sleep(time.Duration(i) * time.Second) 42 | } 43 | } 44 | return 45 | } 46 | -------------------------------------------------------------------------------- /gcloud/datastore/config.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/btncrypt" 5 | 6 | "context" 7 | 8 | "cloud.google.com/go/datastore" 9 | "golang.org/x/oauth2" 10 | "google.golang.org/api/option" 11 | ) 12 | 13 | var ctxNoNamespace = context.Background() 14 | 15 | type Config struct { 16 | projectName string 17 | rootKeyStr string 18 | c *btncrypt.Cipher 19 | tsrc oauth2.TokenSource 20 | } 21 | 22 | func NewConfig(projectName, rootKeyStr string, c *btncrypt.Cipher, tsrc oauth2.TokenSource) *Config { 23 | if len(projectName) == 0 { 24 | panic("empty projectName") 25 | } 26 | if len(rootKeyStr) == 0 { 27 | panic("empty rootKeyStr") 28 | } 29 | if tsrc == nil { 30 | panic("nil tokensource") 31 | } 32 | 33 | return &Config{ 34 | projectName: projectName, 35 | rootKeyStr: rootKeyStr, 36 | c: c, 37 | tsrc: tsrc, 38 | } 39 | } 40 | 41 | func (cfg *Config) getClient(ctx context.Context) (*datastore.Client, error) { 42 | return datastore.NewClient(ctx, cfg.projectName, option.WithTokenSource(cfg.tsrc)) 43 | } 44 | -------------------------------------------------------------------------------- /assets/webui/dist/infobar.js: -------------------------------------------------------------------------------- 1 | import {$} from './domhelper.js'; 2 | 3 | const kHiddenClass = 'hidden'; 4 | const kInnerHTMLSource = 5 | ` 6 |
7 |
hogefuga dayo-
8 |
✗
9 |
10 | `; 11 | 12 | // TODO: multi-line support 13 | class Infobar extends HTMLElement { 14 | constructor() { 15 | super(); 16 | } 17 | 18 | connectedCallback() { 19 | this.hide(); 20 | this.innerHTML = kInnerHTMLSource; 21 | 22 | this.textDiv_ = this.querySelector('.infobar__text'); 23 | this.closeBtn_ = this.querySelector('.infobar__button'); 24 | this.closeBtn_.addEventListener('click', _ => { 25 | this.hide(); 26 | }); 27 | } 28 | 29 | showMessage(msg) { 30 | this.textDiv_.innerText = msg; 31 | this.classList.remove(kHiddenClass); 32 | } 33 | 34 | hide() { 35 | this.classList.add(kHiddenClass); 36 | } 37 | }; 38 | window.customElements.define('otaru-infobar', Infobar); 39 | 40 | const infobar = $('otaru-infobar'); 41 | export {infobar}; 42 | -------------------------------------------------------------------------------- /blobstore/genblobpath.go: -------------------------------------------------------------------------------- 1 | package blobstore 2 | 3 | import ( 4 | "encoding/hex" 5 | "fmt" 6 | 7 | "github.com/nyaxt/otaru/flags" 8 | "github.com/nyaxt/otaru/util" 9 | ) 10 | 11 | // GenerateNewBlobPath tries to return a new unique blob path. 12 | // Note that this may return an already used blobpath in high contention, although it is highly unlikely it will happen. 13 | func GenerateNewBlobPath(bs RandomAccessBlobStore) (string, error) { 14 | const MaxTrial = 256 15 | const BlobPathLen = 16 16 | 17 | for i := 0; i < MaxTrial; i++ { 18 | randbin := util.RandomBytes(BlobPathLen) 19 | candidate := hex.EncodeToString(randbin) 20 | 21 | bh, err := bs.Open(candidate, flags.O_RDONLY) 22 | if err != nil { 23 | if err == util.ENOENT { 24 | return candidate, nil 25 | } 26 | return "", err 27 | } 28 | seemsNotUsed := bh.Size() == 0 29 | if err := bh.Close(); err != nil { 30 | return "", err 31 | } 32 | 33 | if seemsNotUsed { 34 | return candidate, nil 35 | } 36 | } 37 | return "", fmt.Errorf("Failed to generate unique blobpath within %d trials", MaxTrial) 38 | } 39 | -------------------------------------------------------------------------------- /facade/apiserver.go: -------------------------------------------------------------------------------- 1 | package facade 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/apiserver" 5 | "github.com/nyaxt/otaru/assets/webui" 6 | "github.com/nyaxt/otaru/otaruapiserver" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | func (o *Otaru) buildApiServerOptions(cfg *ApiServerConfig) ([]apiserver.Option, error) { 11 | override := cfg.WebUIRootPath 12 | if override != "" { 13 | zap.S().Infof("Overriding embedded WebUI and serving WebUI at %s", override) 14 | } 15 | 16 | options := []apiserver.Option{ 17 | apiserver.ListenAddr(cfg.ListenAddr), 18 | apiserver.TLSCertKey(cfg.Certs, cfg.Key), 19 | apiserver.ClientCACert(cfg.ClientCACert), 20 | apiserver.CORSAllowedOrigins(cfg.CORSAllowedOrigins), 21 | apiserver.SetDefaultHandler(webui.WebUIHandler(override, "/index.otaru-server.html")), 22 | otaruapiserver.InstallBlobstoreService(o.S, o.DefaultBS, o.CBS), 23 | otaruapiserver.InstallFileHandler(o.FS), 24 | otaruapiserver.InstallFileSystemService(o.FS), 25 | otaruapiserver.InstallINodeDBService(o.IDBS), 26 | otaruapiserver.InstallSystemService(), 27 | } 28 | 29 | return options, nil 30 | } 31 | -------------------------------------------------------------------------------- /cli/options.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type options struct { 8 | cfg *CliConfig 9 | ctx context.Context 10 | forceGrpc bool 11 | allowOverwrite bool 12 | } 13 | 14 | var defaultOptions = options{ 15 | ctx: context.Background(), 16 | forceGrpc: false, 17 | allowOverwrite: false, 18 | } 19 | 20 | type Option func(*options) 21 | 22 | func WithCliConfig(cfg *CliConfig) Option { 23 | return func(o *options) { o.cfg = cfg } 24 | } 25 | 26 | func WithContext(ctx context.Context) Option { 27 | return func(o *options) { o.ctx = ctx } 28 | } 29 | 30 | // AllowOverwrite allows NewWriter to open an existing file. 31 | func AllowOverwrite(b bool) Option { 32 | return func(o *options) { o.allowOverwrite = b } 33 | } 34 | 35 | func ForceGrpc() Option { 36 | return func(o *options) { o.forceGrpc = true } 37 | } 38 | 39 | func (o *options) QueryConnectionInfo(vhost string) (*ConnectionInfo, error) { 40 | ci, err := QueryConnectionInfo(o.cfg, vhost) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return ci, nil 46 | } 47 | -------------------------------------------------------------------------------- /apiserver/clientauth/userinfo.go: -------------------------------------------------------------------------------- 1 | package clientauth 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "google.golang.org/grpc" 8 | "google.golang.org/grpc/codes" 9 | ) 10 | 11 | type UserInfo struct { 12 | Role 13 | User string 14 | } 15 | 16 | func (ui UserInfo) String() string { 17 | return fmt.Sprintf("%s [role=%v]", ui.User, ui.Role) 18 | } 19 | 20 | var ( 21 | AnonymousUserInfo = UserInfo{Role: RoleAnonymous, User: "anonymous"} 22 | NoauthUserInfo = UserInfo{Role: RoleAdmin, User: "auth-disabled"} 23 | ) 24 | 25 | type userInfoKey struct{} 26 | 27 | func ContextWithUserInfo(ctx context.Context, ui UserInfo) context.Context { 28 | return context.WithValue(ctx, userInfoKey{}, ui) 29 | } 30 | 31 | func UserInfoFromContext(ctx context.Context) UserInfo { 32 | ui, ok := ctx.Value(userInfoKey{}).(UserInfo) 33 | if !ok { 34 | return AnonymousUserInfo 35 | } 36 | return ui 37 | } 38 | 39 | func RequireRoleGRPC(ctx context.Context, req Role) error { 40 | ui := UserInfoFromContext(ctx) 41 | if ui.Role < req { 42 | return grpc.Errorf(codes.PermissionDenied, "Action requires role %v, but you are %v", req, ui) 43 | } 44 | 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Test and Release 3 | jobs: 4 | unit_tests: 5 | strategy: 6 | matrix: 7 | platform: [ubuntu-latest] 8 | runs-on: ${{ matrix.platform }} 9 | steps: 10 | - name: Install Go 11 | uses: actions/setup-go@v2 12 | with: 13 | go-version: '>= 1.20' 14 | - name: Checkout code 15 | uses: actions/checkout@v2 16 | - name: Unpack test GCP service account 17 | env: 18 | OTARU_GITHUB_CI: ${{ secrets.OTARU_GITHUB_CI }} 19 | run: | 20 | echo -n $OTARU_GITHUB_CI | base64 -d > /tmp/otaru-github-ci.json 21 | - name: Test 22 | run: go test ./... 23 | env: 24 | SKIP_FUSE_TEST: "1" 25 | GOOGLE_APPLICATION_CREDENTIALS: "/tmp/otaru-github-ci.json" 26 | release_docker: 27 | needs: [unit_tests] 28 | if: github.event_name == 'push' 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Set up Go 32 | uses: actions/setup-go@v2 33 | with: 34 | go-version: '>= 1.20' 35 | - name: Checkout 36 | uses: actions/checkout@v2 37 | - uses: imjasonh/setup-ko@v0.4 38 | - run: ko publish --bare ./cmd/otaru 39 | -------------------------------------------------------------------------------- /gcloud/auth/testutils/testutils.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "log" 5 | 6 | "golang.org/x/oauth2" 7 | 8 | "github.com/nyaxt/otaru/facade" 9 | "github.com/nyaxt/otaru/gcloud/auth" 10 | "github.com/nyaxt/otaru/gcloud/datastore" 11 | "github.com/nyaxt/otaru/testutils" 12 | ) 13 | 14 | var CredentialsFilePath string 15 | 16 | func init() { 17 | CredentialsFilePath = facade.FindGCPServiceAccountJSON("") 18 | if CredentialsFilePath == "" { 19 | panic("Failed to find Google Cloud service account json file.") 20 | } 21 | } 22 | 23 | const TestBucketName = "otaru-dev-unittest" 24 | 25 | func TestConfig() *facade.Config { 26 | return &facade.Config{ 27 | ProjectName: "otaru-dev", 28 | CredentialsFilePath: CredentialsFilePath, 29 | } 30 | } 31 | 32 | func TestTokenSource() oauth2.TokenSource { 33 | clisrc, err := auth.GetGCloudTokenSource(CredentialsFilePath) 34 | if err != nil { 35 | log.Fatalf("Failed to create TestTokenSource: %v", err) 36 | } 37 | return clisrc 38 | } 39 | 40 | func TestDSConfig(rootKeyStr string) *datastore.Config { 41 | projectName := TestConfig().ProjectName 42 | return datastore.NewConfig(projectName, rootKeyStr, testutils.TestCipher(), TestTokenSource()) 43 | } 44 | -------------------------------------------------------------------------------- /cli/put.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | ) 10 | 11 | func Put(ctx context.Context, cfg *CliConfig, args []string) error { 12 | fset := flag.NewFlagSet("put", flag.ExitOnError) 13 | fset.Usage = func() { 14 | fmt.Printf("Usage of %s put:\n", os.Args[0]) 15 | fmt.Printf(" %s put LOCAL_PATH OTARU_PATH\n", os.Args[0]) 16 | fset.PrintDefaults() 17 | } 18 | fset.Parse(args[1:]) 19 | 20 | if fset.NArg() != 2 { 21 | fset.Usage() 22 | return fmt.Errorf("Invalid number of arguments") 23 | } 24 | localpathstr, pathstr := fset.Arg(0), fset.Arg(1) 25 | // FIXME: pathstr may end in /, in which case should join(pathstr, base(localpathstr)) 26 | 27 | f, err := os.Open(localpathstr) 28 | if err != nil { 29 | return fmt.Errorf("Failed to open source file: \"%s\". err: %v", localpathstr, err) 30 | } 31 | 32 | w, err := NewWriter(pathstr, WithCliConfig(cfg), WithContext(ctx)) 33 | if err != nil { 34 | f.Close() 35 | return err 36 | } 37 | 38 | if _, err := io.Copy(w, f); err != nil { 39 | return err 40 | } 41 | if err := w.Close(); err != nil { 42 | return err 43 | } 44 | if err := f.Close(); err != nil { 45 | return err 46 | } 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /blobstore/positionedio.go: -------------------------------------------------------------------------------- 1 | package blobstore 2 | 3 | // PReader implements positioned read 4 | type PReader interface { 5 | PRead(p []byte, offset int64) error 6 | } 7 | 8 | type ZeroFillPReader struct{} 9 | 10 | func (ZeroFillPReader) PRead(p []byte, offset int64) error { 11 | for i := range p { 12 | p[i] = 0 13 | } 14 | return nil 15 | } 16 | 17 | // PWriter implements positioned write 18 | type PWriter interface { 19 | PWrite(p []byte, offset int64) error 20 | } 21 | 22 | type RandomAccessIO interface { 23 | PReader 24 | PWriter 25 | } 26 | 27 | // OffsetReader provides io.Reader from PReader 28 | type OffsetReader struct { 29 | PReader 30 | Offset int64 31 | } 32 | 33 | func (r *OffsetReader) Read(p []byte) (int, error) { 34 | if err := r.PReader.PRead(p, r.Offset); err != nil { 35 | return 0, err 36 | } 37 | 38 | r.Offset += int64(len(p)) 39 | return len(p), nil 40 | } 41 | 42 | // OffsetWriter provides io.Reader from PWriter 43 | type OffsetWriter struct { 44 | PWriter 45 | Offset int64 46 | } 47 | 48 | func (w *OffsetWriter) Write(p []byte) (int, error) { 49 | if err := w.PWriter.PWrite(p, w.Offset); err != nil { 50 | return 0, err 51 | } 52 | 53 | w.Offset += int64(len(p)) 54 | return len(p), nil 55 | } 56 | -------------------------------------------------------------------------------- /testutils/testca/mktestcerts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname $0) 4 | OUTDIR=$(pwd) 5 | BASEDIR=$(pwd)/kmgmbasedir 6 | mkdir -p ${BASEDIR} 7 | 8 | kmgm --basedir ${BASEDIR} --config ca.yml setup 9 | kmgm --basedir ${BASEDIR} show --output pem ca > \ 10 | ${OUTDIR}/cacert.pem 11 | 12 | kmgm --basedir ${BASEDIR} --config issue_server.yml issue \ 13 | --cert ${OUTDIR}/cert.pem \ 14 | --priv ${OUTDIR}/cert-key.pem 15 | 16 | kmgm --basedir ${BASEDIR} --profile clientauth --config clientauth_ca.yml setup 17 | kmgm --basedir ${BASEDIR} --profile clientauth show --output pem ca > \ 18 | ${OUTDIR}/clientauth_cacert.pem 19 | 20 | for role in admin readonly invalid; do 21 | set -x 22 | kmgm --basedir ${BASEDIR} --profile clientauth --config clientauth_issue_${role}.yml issue \ 23 | --cert ${OUTDIR}/clientauth_${role}.pem \ 24 | --priv ${OUTDIR}/clientauth_${role}-key.pem 25 | set +x 26 | done 27 | 28 | kmgm --basedir ${BASEDIR} --profile wrong --config wrong_ca.yml setup 29 | kmgm --basedir ${BASEDIR} --profile wrong show --output pem ca > \ 30 | ${OUTDIR}/wrong_cacert.pem 31 | 32 | kmgm --basedir ${BASEDIR} --profile wrong --config issue_wrong.yml issue \ 33 | --cert ${OUTDIR}/wrong_cert.pem \ 34 | --priv ${OUTDIR}/wrong_cert-key.pem 35 | -------------------------------------------------------------------------------- /testutils/rwinterceptblobstore.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/nyaxt/otaru/blobstore" 7 | ) 8 | 9 | type RWInterceptBlobStore struct { 10 | BE blobstore.BlobStore 11 | WrapWriter func(orig io.WriteCloser) (io.WriteCloser, error) 12 | WrapReader func(orig io.ReadCloser) (io.ReadCloser, error) 13 | } 14 | 15 | var _ = blobstore.BlobStore(RWInterceptBlobStore{}) 16 | 17 | func (bs RWInterceptBlobStore) OpenWriter(blobpath string) (io.WriteCloser, error) { 18 | orig, err := bs.BE.OpenWriter(blobpath) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | return bs.WrapWriter(orig) 24 | } 25 | 26 | func (bs RWInterceptBlobStore) OpenReader(blobpath string) (io.ReadCloser, error) { 27 | orig, err := bs.BE.OpenReader(blobpath) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | return bs.WrapReader(orig) 33 | } 34 | 35 | var _ = blobstore.BlobLister(RWInterceptBlobStore{}) 36 | 37 | func (bs RWInterceptBlobStore) ListBlobs() ([]string, error) { 38 | return bs.BE.(blobstore.BlobLister).ListBlobs() 39 | } 40 | 41 | var _ = blobstore.BlobSizer(RWInterceptBlobStore{}) 42 | 43 | func (bs RWInterceptBlobStore) BlobSize(blobpath string) (int64, error) { 44 | return bs.BE.(blobstore.BlobSizer).BlobSize(blobpath) 45 | } 46 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "strings" 10 | ) 11 | 12 | func Int64Min(a, b int64) int64 { 13 | if a < b { 14 | return a 15 | } 16 | return b 17 | } 18 | 19 | func Int64Max(a, b int64) int64 { 20 | if a < b { 21 | return b 22 | } 23 | return a 24 | } 25 | 26 | func IntMin(a, b int) int { 27 | if a < b { 28 | return a 29 | } 30 | return b 31 | } 32 | 33 | func IntMax(a, b int) int { 34 | if a < b { 35 | return b 36 | } 37 | return a 38 | } 39 | 40 | func RandomBytes(size int) []byte { 41 | p := make([]byte, size) 42 | ReadRandomBytes(p) 43 | return p 44 | } 45 | 46 | func ReadRandomBytes(p []byte) { 47 | if _, err := io.ReadFull(rand.Reader, p); err != nil { 48 | panic(err) 49 | } 50 | } 51 | 52 | func StringFromFile(filename string) (string, error) { 53 | b, err := ioutil.ReadFile(filename) 54 | if err != nil { 55 | return "", fmt.Errorf("Failed to read file \"%s\": %v", filename, err) 56 | } 57 | return strings.TrimRight(string(b), "\n"), nil 58 | } 59 | 60 | func StringFromFileOrDie(filename string, usage string) string { 61 | s, err := StringFromFile(filename) 62 | if err != nil { 63 | log.Fatalf("While fetching %s: %v", usage, err) 64 | } 65 | return s 66 | } 67 | -------------------------------------------------------------------------------- /testutils/blobhandle.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | ) 7 | 8 | type TestBlobHandle struct { 9 | Buf []byte 10 | } 11 | 12 | func (bh *TestBlobHandle) PRead(p []byte, offset int64) error { 13 | if offset < 0 || int64(len(bh.Buf)) < offset+int64(len(p)) { 14 | return fmt.Errorf("PRead offset out of bound. buf len: %d while given offset: %d and len: %d", len(bh.Buf), offset, len(p)) 15 | } 16 | 17 | copy(p, bh.Buf[offset:]) 18 | return nil 19 | } 20 | 21 | func (bh *TestBlobHandle) PWrite(p []byte, offset int64) error { 22 | if offset < 0 || math.MaxInt32 < offset+int64(len(p)) { 23 | return fmt.Errorf("PWrite offset out of bound. buf len: %d while given offset: %d and len: %d", len(bh.Buf), offset, len(p)) 24 | } 25 | if int64(len(bh.Buf)) < offset+int64(len(p)) { 26 | newsize := offset + int64(len(p)) 27 | buf := make([]byte, newsize) 28 | copy(buf[:len(bh.Buf)], bh.Buf) 29 | bh.Buf = buf 30 | } 31 | 32 | copy(bh.Buf[offset:], p) 33 | return nil 34 | } 35 | 36 | func (bh *TestBlobHandle) Truncate(size int64) error { 37 | if size < int64(len(bh.Buf)) { 38 | bh.Buf = bh.Buf[:int(size)] 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func (bh *TestBlobHandle) Size() int64 { 45 | return int64(len(bh.Buf)) 46 | } 47 | 48 | func (TestBlobHandle) Close() error { 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /filesystem/inodedbchunksarrayio.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/nyaxt/otaru/chunkstore" 7 | "github.com/nyaxt/otaru/inodedb" 8 | ) 9 | 10 | type INodeDBChunksArrayIO struct { 11 | db inodedb.DBHandler 12 | nlock inodedb.NodeLock 13 | } 14 | 15 | var _ = chunkstore.ChunksArrayIO(&INodeDBChunksArrayIO{}) 16 | 17 | func NewINodeDBChunksArrayIO(db inodedb.DBHandler, nlock inodedb.NodeLock) *INodeDBChunksArrayIO { 18 | return &INodeDBChunksArrayIO{db: db, nlock: nlock} 19 | } 20 | 21 | func (caio *INodeDBChunksArrayIO) Read() ([]inodedb.FileChunk, error) { 22 | v, _, err := caio.db.QueryNode(caio.nlock.ID, false) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | fn, ok := v.(*inodedb.FileNodeView) 28 | if !ok { 29 | return nil, fmt.Errorf("Target node view is not a file.") 30 | } 31 | 32 | return fn.Chunks, nil 33 | } 34 | 35 | func (caio *INodeDBChunksArrayIO) Write(cs []inodedb.FileChunk) error { 36 | if !caio.nlock.HasTicket() { 37 | return fmt.Errorf("No ticket lock is acquired.") 38 | } 39 | 40 | tx := inodedb.DBTransaction{Ops: []inodedb.DBOperation{ 41 | &inodedb.UpdateChunksOp{NodeLock: caio.nlock, Chunks: cs}, 42 | }} 43 | if _, err := caio.db.ApplyTransaction(tx); err != nil { 44 | return fmt.Errorf("Failed to apply tx for updating cs: %v", err) 45 | } 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /logger/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/nyaxt/otaru/logger" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type Config struct { 12 | LogLevel map[string]string 13 | } 14 | 15 | func Str2Level(s string) (logger.Level, error) { 16 | switch strings.ToLower(s) { 17 | case "debug": 18 | return logger.Debug, nil 19 | case "info": 20 | return logger.Info, nil 21 | case "warn", "warning": 22 | return logger.Warning, nil 23 | case "critical": 24 | return logger.Critical, nil 25 | case "panic": 26 | return logger.Panic, nil 27 | default: 28 | return logger.Debug, fmt.Errorf("Unknown log level \"%s\"", s) 29 | } 30 | } 31 | 32 | func Apply(l logger.Logger, c Config) error { 33 | r := logger.Registry() 34 | 35 | if v, ok := c.LogLevel["*"]; ok { 36 | lv, err := Str2Level(v) 37 | if err != nil { 38 | return err 39 | } 40 | for _, e := range r.Categories() { 41 | c := r.Category(e.Category) 42 | c.Level = lv 43 | } 44 | } 45 | for k, v := range c.LogLevel { 46 | if k == "*" { 47 | continue 48 | } 49 | c := r.CategoryIfExist(k) 50 | if c == nil { 51 | zap.S().Warnf("Log category \"%s\" does not exist.", k) 52 | continue 53 | } 54 | lv, err := Str2Level(v) 55 | if err != nil { 56 | return err 57 | } 58 | c.Level = lv 59 | } 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /doc/cachedblobentry_fsm.gv: -------------------------------------------------------------------------------- 1 | digraph finite_state_machine { 2 | rankdir=TB; 3 | size="8,5"; 4 | graph [ dpi = 300 ]; 5 | 6 | Uninitialized -> Invalidating [ label = "OpenHandle (old)" ]; 7 | Uninitialized -> Clean [ label = "OpenHandle (up-to-date)" ]; 8 | Uninitialized -> Errored [ label = "OpenHandle (error)" ]; 9 | Invalidating -> Clean [ label = "succeed" ]; 10 | Invalidating -> Errored [ label = "error" ]; 11 | Errored -> ErroredClosed [ label = "cleaned up" ]; 12 | Clean -> WriteInProgress [ label = "write/truncate started" ]; 13 | Clean -> Closing [ label = "close started" ]; 14 | WriteInProgress -> Dirty [ label = "write/truncate done" ]; 15 | WriteInProgress -> Errored [ label = "write/truncate errored" ]; 16 | Dirty -> WritebackInProgress [ label = "writeback started" ]; 17 | Dirty -> WriteInProgress [ label = "write/truncate started" ]; 18 | Dirty -> DirtyClosing [ label = "closing started" ]; 19 | WritebackInProgress -> Clean [ label = "writeback done" ]; 20 | WritebackInProgress -> StaleWritebackInProgress [ label = "write/truncate started" ]; 21 | StaleWritebackInProgress -> Dirty [ label = "write/truncate done" ]; 22 | DirtyClosing -> Closed [ label = "writeback done+close end" ]; 23 | Closing -> Closed [ label = "close end" ]; 24 | Closed -> Clean [ label = "OpenHandle" ]; 25 | Closed -> Errored [ label = "OpenHandle (error)" ]; 26 | } 27 | -------------------------------------------------------------------------------- /apiserver/server_test.go: -------------------------------------------------------------------------------- 1 | package apiserver_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | _ "embed" 7 | "io/ioutil" 8 | "testing" 9 | "time" 10 | 11 | "github.com/nyaxt/otaru/apiserver" 12 | "github.com/nyaxt/otaru/testutils" 13 | "github.com/nyaxt/otaru/testutils/testca" 14 | ) 15 | 16 | const testListenAddr = "localhost:20246" 17 | 18 | func init() { testutils.EnsureLogger() } 19 | 20 | func TestServe_Healthz(t *testing.T) { 21 | ctx, cancel := context.WithCancel(context.Background()) 22 | joinC := make(chan struct{}) 23 | go func() { 24 | if err := apiserver.Serve(ctx, 25 | apiserver.ListenAddr(testListenAddr), 26 | apiserver.TLSCertKey(testca.Certs, testca.Key.Parsed), 27 | apiserver.ClientCACert(testca.ClientAuthCACert), 28 | ); err != nil { 29 | t.Errorf("Serve failed: %v", err) 30 | } 31 | close(joinC) 32 | }() 33 | 34 | // FIXME: wait until Serve to actually start accepting conns 35 | time.Sleep(100 * time.Millisecond) 36 | 37 | resp, err := testca.TLSHTTPClient.Get("https://" + testListenAddr + "/healthz") 38 | if err != nil { 39 | t.Errorf("http.Get: %v", err) 40 | t.FailNow() 41 | } 42 | cont, err := ioutil.ReadAll(resp.Body) 43 | if err != nil { 44 | t.Errorf("ReadAll(http.Get resp.Body): %v", err) 45 | t.FailNow() 46 | } 47 | if !bytes.Equal(cont, []byte("ok\n")) { 48 | t.Errorf("unexpected content: %v", cont) 49 | } 50 | resp.Body.Close() 51 | 52 | cancel() 53 | <-joinC 54 | } 55 | -------------------------------------------------------------------------------- /otaruapiserver/inodedbservice.go: -------------------------------------------------------------------------------- 1 | package otaruapiserver 2 | 3 | import ( 4 | "context" 5 | 6 | "google.golang.org/grpc" 7 | "google.golang.org/grpc/codes" 8 | 9 | "github.com/nyaxt/otaru/apiserver" 10 | "github.com/nyaxt/otaru/apiserver/clientauth" 11 | "github.com/nyaxt/otaru/inodedb" 12 | "github.com/nyaxt/otaru/pb" 13 | ) 14 | 15 | type inodedbService struct { 16 | h inodedb.DBHandler 17 | pb.UnimplementedINodeDBServiceServer 18 | } 19 | 20 | func (svc *inodedbService) GetINodeDBStats(ctx context.Context, req *pb.GetINodeDBStatsRequest) (*pb.GetINodeDBStatsResponse, error) { 21 | if err := clientauth.RequireRoleGRPC(ctx, clientauth.RoleAdmin); err != nil { 22 | return nil, err 23 | } 24 | 25 | prov, ok := svc.h.(inodedb.DBServiceStatsProvider) 26 | if !ok { 27 | return nil, grpc.Errorf(codes.Unimplemented, "inodedb doesn't support providing stats.") 28 | } 29 | stats := prov.GetStats() 30 | 31 | return &pb.GetINodeDBStatsResponse{ 32 | LastSync: stats.LastSync.Unix(), 33 | LastTx: stats.LastTx.Unix(), 34 | LastId: uint64(stats.LastID), 35 | Version: uint64(stats.Version), 36 | LastTicket: uint64(stats.LastTicket), 37 | NumberOfNodeLocks: uint32(stats.NumberOfNodeLocks), 38 | }, nil 39 | } 40 | 41 | func InstallINodeDBService(h inodedb.DBHandler) apiserver.Option { 42 | return apiserver.RegisterService( 43 | func(s *grpc.Server) { pb.RegisterINodeDBServiceServer(s, &inodedbService{h: h}) }, 44 | pb.RegisterINodeDBServiceHandlerFromEndpoint, 45 | ) 46 | } 47 | -------------------------------------------------------------------------------- /assets/webui/dist/commonsubstr.js: -------------------------------------------------------------------------------- 1 | function findCommonSubStrs2(a, b) { 2 | let lastcol = null; 3 | let best = new Array(a.length); 4 | for (let i = 0; i < a.length; ++ i) { 5 | let col = new Array(b.length); 6 | best[i] = 0; 7 | for (let j = 0; j < b.length; ++ j) { 8 | if (a[i] === b[j]) { 9 | const lu = (i > 0 && j > 0) ? lastcol[j-1] : 0; 10 | col[j] = lu + 1; 11 | const off = i - col[j] + 1; 12 | if (col[j] > best[off]) { 13 | best[off] = col[j]; 14 | } 15 | } else { 16 | col[j] = 0; 17 | } 18 | } 19 | lastcol = col; 20 | } 21 | 22 | let substrs = []; 23 | for (let i = 0; i < a.length; ++ i) { 24 | if (best[i] >= 2) { 25 | substrs.push(a.substr(i, best[i])); 26 | } 27 | } 28 | return substrs; 29 | } 30 | 31 | function findCommonSubStrs(ss) { 32 | if (ss.length < 2) 33 | return ss; 34 | 35 | let substrs = findCommonSubStrs2(ss[0], ss[1]); 36 | for (let i = 2; i < ss.length; ++ i) { 37 | let newsubstrs = []; 38 | for (let s of substrs) { 39 | newsubstrs = newsubstrs.concat(findCommonSubStrs2(s, ss[i])); 40 | } 41 | substrs = newsubstrs; 42 | } 43 | return substrs; 44 | } 45 | 46 | function findLongestCommonSubStr(ss) { 47 | const substrs = findCommonSubStrs(ss); 48 | 49 | let longest = ""; 50 | for (let s of substrs) { 51 | if (s.length > longest.length) 52 | longest = s; 53 | } 54 | return longest; 55 | } 56 | 57 | export {findCommonSubStrs, findLongestCommonSubStr}; 58 | -------------------------------------------------------------------------------- /blobstore/cachedblobstore/cachedblobhandle.go: -------------------------------------------------------------------------------- 1 | package cachedblobstore 2 | 3 | import ( 4 | "fmt" 5 | 6 | fl "github.com/nyaxt/otaru/flags" 7 | "github.com/nyaxt/otaru/util" 8 | ) 9 | 10 | type CachedBlobHandle struct { 11 | be *CachedBlobEntry 12 | flags int 13 | stacktrace []byte 14 | } 15 | 16 | func (bh *CachedBlobHandle) Flags() int { return bh.flags } 17 | 18 | func (bh *CachedBlobHandle) PRead(p []byte, offset int64) error { 19 | if !fl.IsReadAllowed(bh.flags) { 20 | return util.EACCES 21 | } 22 | 23 | return bh.be.PRead(p, offset) 24 | } 25 | 26 | func (bh *CachedBlobHandle) PWrite(p []byte, offset int64) error { 27 | if !fl.IsWriteAllowed(bh.flags) { 28 | return util.EACCES 29 | } 30 | 31 | return bh.be.PWrite(p, offset) 32 | } 33 | 34 | func (bh *CachedBlobHandle) Size() int64 { 35 | return bh.be.Size() 36 | } 37 | 38 | func (bh *CachedBlobHandle) Truncate(newsize int64) error { 39 | if !fl.IsWriteAllowed(bh.flags) { 40 | return util.EACCES 41 | } 42 | 43 | return bh.be.Truncate(newsize) 44 | } 45 | 46 | var _ = util.Syncer(&CachedBlobHandle{}) 47 | 48 | func (bh *CachedBlobHandle) Sync() error { 49 | if !fl.IsWriteAllowed(bh.flags) { 50 | return nil 51 | } 52 | 53 | return bh.be.Sync() 54 | } 55 | 56 | func (bh *CachedBlobHandle) Close() error { 57 | bh.be.CloseHandle(bh) 58 | 59 | return nil 60 | } 61 | 62 | func (bh *CachedBlobHandle) String() string { 63 | return fmt.Sprintf("CachedBlobHandle{be.blobpath: \"%s\", %v, stack: %v}", bh.be.blobpath, fl.FlagsToString(bh.flags), string(bh.stacktrace)) 64 | } 65 | -------------------------------------------------------------------------------- /cli/path/path.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "path" 8 | "strings" 9 | ) 10 | 11 | type Path struct { 12 | Vhost string 13 | FsPath string 14 | } 15 | 16 | type State int 17 | 18 | const ( 19 | Initial State = iota 20 | AfterOtaruScheme 21 | BeforeVhost 22 | BeforeFsPath 23 | End 24 | ) 25 | 26 | const OtaruScheme = "otaru:" 27 | const VhostLocal = "[local]" 28 | 29 | func advance(st *State, p *Path, s string) (string, error) { 30 | switch *st { 31 | case Initial: 32 | if strings.HasPrefix(s, OtaruScheme) { 33 | *st = AfterOtaruScheme 34 | return s[len(OtaruScheme):], nil 35 | } 36 | fallthrough 37 | 38 | case AfterOtaruScheme: 39 | if strings.HasPrefix(s, "//") { 40 | *st = BeforeVhost 41 | return s[2:], nil 42 | } 43 | *st = BeforeFsPath 44 | return s, nil 45 | 46 | case BeforeVhost: 47 | i := strings.Index(s, "/") 48 | if i < 0 { 49 | *st = End 50 | return s, fmt.Errorf("parser: Expected vhost/path, but got \"%s\"", s) 51 | } 52 | *st = BeforeFsPath 53 | p.Vhost, s = s[:i], s[i:] 54 | return s, nil 55 | 56 | case BeforeFsPath: 57 | p.FsPath = path.Clean(s) 58 | *st = End 59 | return "", nil 60 | 61 | case End: 62 | return "", nil 63 | 64 | default: 65 | log.Panicf("Unknown state: %q", *st) 66 | } 67 | return "", errors.New("NOTREACHED") 68 | } 69 | 70 | func Parse(s string) (Path, error) { 71 | p := Path{Vhost: "default", FsPath: "/"} 72 | var err error 73 | for st := Initial; st != End; s, err = advance(&st, &p, s) { 74 | // fmt.Printf("state: %v, path: %+v, left: \"%s\"\n", st, p, s) 75 | } 76 | return p, err 77 | } 78 | -------------------------------------------------------------------------------- /inodedb/dboperation_serdes_test.go: -------------------------------------------------------------------------------- 1 | package inodedb_test 2 | 3 | import ( 4 | "testing" 5 | 6 | i "github.com/nyaxt/otaru/inodedb" 7 | ) 8 | 9 | func TestEncodeDBOperationToJson_InitializeFileSystemOp(t *testing.T) { 10 | json, err := i.EncodeDBOperationsToJson([]i.DBOperation{&i.InitializeFileSystemOp{}}) 11 | if err != nil { 12 | t.Errorf("EncodeDBOperationToJson failed: %v", err) 13 | return 14 | } 15 | 16 | // t.Errorf("%v", string(json)) 17 | ops, err := i.DecodeDBOperationsFromJson(json) 18 | if err != nil { 19 | t.Errorf("DecodeDBOperationToJson failed: %v", err) 20 | return 21 | } 22 | if _, ok := ops[0].(*i.InitializeFileSystemOp); !ok { 23 | t.Errorf("Decode failed to recover original type") 24 | } 25 | } 26 | 27 | func TestEncodeDBOperationToJson_CreateNodeOp(t *testing.T) { 28 | json, err := i.EncodeDBOperationsToJson([]i.DBOperation{&i.CreateNodeOp{ 29 | NodeLock: i.NodeLock{ID: 123, Ticket: 456}, 30 | OrigPath: "/foo/bar", 31 | Type: i.DirNodeT, 32 | }}) 33 | if err != nil { 34 | t.Errorf("EncodeDBOperationToJson failed: %v", err) 35 | return 36 | } 37 | 38 | ops, err := i.DecodeDBOperationsFromJson(json) 39 | if err != nil { 40 | t.Errorf("DecodeDBOperationToJson failed: %v", err) 41 | return 42 | } 43 | 44 | dirop, ok := ops[0].(*i.CreateNodeOp) 45 | if !ok { 46 | t.Errorf("Decode failed to recover original type") 47 | } 48 | 49 | if dirop.NodeLock.ID != 123 { 50 | t.Errorf("encode/decode data mismatch") 51 | } 52 | if dirop.NodeLock.Ticket != 456 { 53 | t.Errorf("encode/decode data mismatch") 54 | } 55 | if dirop.OrigPath != "/foo/bar" { 56 | t.Errorf("encode/decode data mismatch") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /logger/registry.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type registry struct { 8 | mu sync.Mutex 9 | 10 | mux Mux 11 | catmap map[string]*CategoryLogger 12 | } 13 | 14 | var registryInstance *registry 15 | var muRegistryInstance sync.Mutex 16 | 17 | func Registry() *registry { 18 | muRegistryInstance.Lock() 19 | defer muRegistryInstance.Unlock() 20 | 21 | if registryInstance == nil { 22 | registryInstance = ®istry{ 23 | catmap: make(map[string]*CategoryLogger), 24 | } 25 | } 26 | 27 | return registryInstance 28 | } 29 | 30 | func (r *registry) AddOutput(l Logger) { 31 | r.mu.Lock() 32 | defer r.mu.Unlock() 33 | 34 | r.mux.Ls = append(r.mux.Ls, l) 35 | } 36 | 37 | func (r *registry) Category(c string) *CategoryLogger { 38 | r.mu.Lock() 39 | defer r.mu.Unlock() 40 | 41 | l, ok := r.catmap[c] 42 | if !ok { 43 | l = &CategoryLogger{ 44 | BE: &r.mux, 45 | Category: c, 46 | Level: Debug, 47 | } 48 | r.catmap[c] = l 49 | } 50 | return l 51 | } 52 | 53 | func (r *registry) CategoryIfExist(c string) *CategoryLogger { 54 | r.mu.Lock() 55 | defer r.mu.Unlock() 56 | 57 | return r.catmap[c] 58 | } 59 | 60 | type CategoryEntry struct { 61 | Category string `json:"category"` 62 | Level `json:"level"` 63 | } 64 | 65 | func (cl *CategoryLogger) View() CategoryEntry { 66 | return CategoryEntry{Category: cl.Category, Level: cl.Level} 67 | } 68 | 69 | func (r *registry) Categories() []CategoryEntry { 70 | r.mu.Lock() 71 | defer r.mu.Unlock() 72 | 73 | ret := make([]CategoryEntry, 0, len(r.catmap)) 74 | for _, cl := range r.catmap { 75 | ret = append(ret, cl.View()) 76 | } 77 | return ret 78 | } 79 | -------------------------------------------------------------------------------- /inodedb/cacheddbtransactionlogio_test.go: -------------------------------------------------------------------------------- 1 | package inodedb_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | idb "github.com/nyaxt/otaru/inodedb" 8 | ) 9 | 10 | func testTx(id idb.TxID) idb.DBTransaction { 11 | return idb.DBTransaction{TxID: id, Ops: []idb.DBOperation{ 12 | &idb.CreateNodeOp{}, 13 | }} 14 | } 15 | 16 | func TestCachedDBTransactionLogIO_SingleTx(t *testing.T) { 17 | be := idb.NewSimpleDBTransactionLogIO() 18 | ctxio := idb.NewCachedDBTransactionLogIO(be) 19 | 20 | if err := ctxio.AppendTransaction(testTx(1)); err != nil { 21 | t.Errorf("AppendTransaction failed: %v", err) 22 | } 23 | 24 | txs, err := ctxio.QueryTransactions(1) 25 | if err != nil { 26 | t.Errorf("QueryTransactions failed: %v", err) 27 | } 28 | 29 | txs2, err := be.QueryTransactions(1) 30 | if err != nil { 31 | t.Errorf("be.QueryTransactions failed: %v", err) 32 | } 33 | 34 | if !reflect.DeepEqual(txs, txs2) { 35 | t.Errorf("mismatch %+v != %+v", txs, txs2) 36 | } 37 | } 38 | 39 | func TestCachedDBTransactionLogIO_1000Tx(t *testing.T) { 40 | be := idb.NewSimpleDBTransactionLogIO() 41 | ctxio := idb.NewCachedDBTransactionLogIO(be) 42 | 43 | for i := idb.TxID(1); i <= 1000; i++ { 44 | if err := ctxio.AppendTransaction(testTx(i)); err != nil { 45 | t.Errorf("AppendTransaction failed: %v", err) 46 | } 47 | } 48 | 49 | txs, err := ctxio.QueryTransactions(800) 50 | if err != nil { 51 | t.Errorf("QueryTransactions failed: %v", err) 52 | } 53 | 54 | txs2, err := be.QueryTransactions(800) 55 | if err != nil { 56 | t.Errorf("be.QueryTransactions failed: %v", err) 57 | } 58 | 59 | if !reflect.DeepEqual(txs, txs2) { 60 | t.Errorf("mismatch %+v != %+v", txs, txs2) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /assets/webui/dist/format.js: -------------------------------------------------------------------------------- 1 | const formatBlobSize = val => { 2 | if (val === undefined) { 3 | return '0B'; 4 | } else if (val > 1024 * 1024 * 1024) { 5 | return (val / (1024 * 1024 * 1024)).toFixed(2) + 'GiB'; 6 | } else if (val > 1024 * 1024) { 7 | return (val / (1024 * 1024)).toFixed(2) + 'MiB'; 8 | } else if (val > 1024) { 9 | return (val / 1024).toFixed(2) + 'KiB'; 10 | } else { 11 | return val + 'B'; 12 | } 13 | } 14 | 15 | const formatTimestamp = (t, opts={}) => { 16 | const diff = new Date() - t; 17 | 18 | const pad = n => (n < 10 ? '0' : '') + n; 19 | if (opts.relative !== false) { 20 | const startOfToday = new Date(); 21 | startOfToday.setHours(0); 22 | startOfToday.setMinutes(0); 23 | startOfToday.setSeconds(0); 24 | startOfToday.setMilliseconds(0); 25 | 26 | if (diff < 60 * 1000) { 27 | return `${(diff / (1000)).toFixed(0)}s ago`; 28 | } else if (diff < 1 * 60 * 60 * 1000) { 29 | return `${(diff / (60 * 1000)).toFixed(0)}m ago`; 30 | } else if (diff < 6 * 60 * 60 * 1000) { 31 | return `${(diff / (60 * 60 * 1000)).toFixed(0)}h ago`; 32 | } else if (t > startOfToday) { 33 | return `${pad(t.getHours())}:${pad(t.getMinutes())}`; 34 | } 35 | } 36 | const ymd = `${pad(t.getFullYear()-2000)}/${pad(t.getMonth()+1)}/${pad(t.getDate())}`; 37 | if (!opts.full) { 38 | return ymd; 39 | } 40 | 41 | return `${ymd} ${pad(t.getHours())}:${pad(t.getMinutes())}:${pad(t.getSeconds())}` 42 | } 43 | 44 | const formatTimestampRPC = n => { 45 | if (n < 0) 46 | return "-"; 47 | else 48 | return formatTimestamp(new Date(n*1000)); 49 | }; 50 | 51 | export {formatBlobSize, formatTimestamp, formatTimestampRPC}; 52 | -------------------------------------------------------------------------------- /gc/blobstoregc/gc_test.go: -------------------------------------------------------------------------------- 1 | package blobstoregc_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/nyaxt/otaru/gc/blobstoregc" 8 | 9 | "context" 10 | ) 11 | 12 | type MockGCBlobStore struct { 13 | bs []string 14 | removedbs []string 15 | } 16 | 17 | var _ = blobstoregc.GCableBlobStore(&MockGCBlobStore{}) 18 | 19 | func (bs *MockGCBlobStore) ListBlobs() ([]string, error) { return bs.bs, nil } 20 | func (bs *MockGCBlobStore) RemoveBlob(b string) error { 21 | bs.removedbs = append(bs.removedbs, b) 22 | return nil 23 | } 24 | 25 | type MockFscker struct { 26 | usedbs []string 27 | } 28 | 29 | func (idb *MockFscker) Fsck() ([]string, []error) { return idb.usedbs, nil } 30 | 31 | func TestGC_Basic(t *testing.T) { 32 | bs := &MockGCBlobStore{ 33 | bs: []string{"a", "b", "x", "y", "z", "META_INODEDB_SNAPSHOT"}, 34 | removedbs: []string{}, 35 | } 36 | idb := &MockFscker{ 37 | usedbs: []string{"x", "y", "z"}, 38 | } 39 | 40 | if err := blobstoregc.GC(context.TODO(), bs, idb, false); err != nil { 41 | t.Errorf("GC err: %v", err) 42 | } 43 | 44 | if !reflect.DeepEqual([]string{"a", "b"}, bs.removedbs) { 45 | t.Errorf("GC removed unexpected blobs: %v", bs.removedbs) 46 | } 47 | } 48 | 49 | func TestGC_EmptyRun(t *testing.T) { 50 | bs := &MockGCBlobStore{ 51 | bs: []string{"x", "y", "z"}, 52 | removedbs: []string{}, 53 | } 54 | idb := &MockFscker{ 55 | usedbs: []string{"x", "y", "z"}, 56 | } 57 | 58 | // vvv should not panic. 59 | if err := blobstoregc.GC(context.TODO(), bs, idb, false); err != nil { 60 | t.Errorf("GC err: %v", err) 61 | } 62 | if len(bs.removedbs) > 0 { 63 | t.Errorf("GC removed unexpected blobs: %v", bs.removedbs) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /cmd/otaru/fscli/commands.go: -------------------------------------------------------------------------------- 1 | package fscli 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/urfave/cli/v2" 7 | 8 | ocli "github.com/nyaxt/otaru/cli" 9 | ) 10 | 11 | var Commands = []*cli.Command{ 12 | { 13 | Name: "ls", 14 | Aliases: []string{"list"}, 15 | ArgsUsage: "otaru://vhost/path", 16 | Action: func(c *cli.Context) error { 17 | cfg, err := ocli.NewConfig(c.String("configDir")) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | if err := ocli.Ls(c.Context, os.Stdout, cfg, c.Args().Slice()); err != nil { 23 | return err 24 | } 25 | 26 | return nil 27 | }, 28 | }, 29 | { 30 | Name: "attr", 31 | ArgsUsage: "otaru://vhost/path", 32 | Action: func(c *cli.Context) error { 33 | cfg, err := ocli.NewConfig(c.String("configDir")) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if err := ocli.Attr(c.Context, cfg, c.Args().Slice()); err != nil { 39 | return err 40 | } 41 | 42 | return nil 43 | }, 44 | }, 45 | { 46 | Name: "get", 47 | ArgsUsage: "otaru://vhost/path", 48 | Action: func(c *cli.Context) error { 49 | cfg, err := ocli.NewConfig(c.String("configDir")) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | if err := ocli.Get(c.Context, cfg, c.Args().Slice()); err != nil { 55 | return err 56 | } 57 | 58 | return nil 59 | }, 60 | }, 61 | { 62 | Name: "put", 63 | ArgsUsage: "hello.txt otaru://vhost/path", 64 | Action: func(c *cli.Context) error { 65 | cfg, err := ocli.NewConfig(c.String("configDir")) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | if err := ocli.Put(c.Context, cfg, c.Args().Slice()); err != nil { 71 | return err 72 | } 73 | 74 | return nil 75 | }, 76 | }, 77 | } 78 | -------------------------------------------------------------------------------- /assets/swaggerui/dist/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Swagger UI 7 | 8 | 9 | 10 | 11 | 32 | 33 | 34 | 35 |
36 | 37 | 38 | 39 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /cmd/otaru/dumpblob/command.go: -------------------------------------------------------------------------------- 1 | package dumpblob 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path" 8 | 9 | "github.com/urfave/cli/v2" 10 | "go.uber.org/zap" 11 | 12 | "github.com/nyaxt/otaru/btncrypt" 13 | "github.com/nyaxt/otaru/chunkstore" 14 | "github.com/nyaxt/otaru/facade" 15 | "github.com/nyaxt/otaru/util" 16 | ) 17 | 18 | var Command = &cli.Command{ 19 | Name: "dumpblob", 20 | Usage: "dump a chunkstore blob file content", 21 | ArgsUsage: "OTARU_BLOBFILE", 22 | Flags: []cli.Flag{ 23 | &cli.BoolFlag{ 24 | Name: "header", 25 | Usage: "Show header", 26 | }, 27 | &cli.PathFlag{ 28 | Name: "passwordFile", 29 | Value: path.Join(facade.DefaultConfigDir(), "password.txt"), 30 | Usage: "Path to a text file storing password", 31 | }, 32 | }, 33 | Action: func(c *cli.Context) error { 34 | s := zap.S().Named("dumpblob") 35 | if !c.Args().Present() { 36 | return fmt.Errorf("No file path specified on the commandline argument") 37 | } 38 | 39 | filepath := c.Args().First() 40 | 41 | f, err := os.Open(filepath) 42 | if err != nil { 43 | return fmt.Errorf("Failed to read file %q: %w", filepath, err) 44 | } 45 | defer f.Close() 46 | 47 | password := util.StringFromFileOrDie(c.Path("passwordFile"), "password") 48 | key := btncrypt.KeyFromPassword(password) 49 | cipher, err := btncrypt.NewCipher(key) 50 | if err != nil { 51 | return fmt.Errorf("Failed to init Cipher: %w", err) 52 | } 53 | 54 | cr, err := chunkstore.NewChunkReader(f, cipher) 55 | if err != nil { 56 | return fmt.Errorf("Failed to init ChunkReader: %v", err) 57 | } 58 | defer cr.Close() 59 | 60 | if c.Bool("header") { 61 | s.Infof("Header: %+v", cr.Header()) 62 | } 63 | 64 | _, err = io.Copy(os.Stdout, cr) 65 | return err 66 | }, 67 | } 68 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "runtime" 7 | "time" 8 | ) 9 | 10 | type Level int 11 | 12 | const ( 13 | Debug Level = iota // Debugging logs 14 | Info // What is happening right now 15 | Warning // Recoverable errors 16 | Critical // Non-recoverable errors which will start graceful shutdown 17 | Panic // Non-recoverable errors with immediate crash 18 | ) 19 | 20 | func (lv Level) String() string { 21 | switch lv { 22 | case Debug: 23 | return "Debug" 24 | case Info: 25 | return "Info" 26 | case Warning: 27 | return "Warning" 28 | case Critical: 29 | return "Critical" 30 | case Panic: 31 | return "Panic" 32 | default: 33 | return "" 34 | } 35 | } 36 | 37 | func (lv Level) Rune() rune { 38 | switch lv { 39 | case Debug: 40 | return 'D' 41 | case Info: 42 | return 'I' 43 | case Warning: 44 | return 'W' 45 | case Critical: 46 | return 'C' 47 | case Panic: 48 | return 'P' 49 | default: 50 | return '?' 51 | } 52 | } 53 | 54 | type Logger interface { 55 | Log(lv Level, data map[string]interface{}) 56 | WillAccept(lv Level) bool 57 | } 58 | 59 | func genLocation() string { 60 | const skip = 3 61 | _, fullpath, line, ok := runtime.Caller(skip) 62 | if !ok { 63 | return ":0" 64 | } 65 | return fmt.Sprintf("%s:%d", filepath.Base(fullpath), line) 66 | 67 | } 68 | 69 | func Logf(l Logger, lv Level, format string, v ...interface{}) { 70 | if l == nil || !l.WillAccept(lv) { 71 | return 72 | } 73 | 74 | logstr := fmt.Sprintf(format, v...) 75 | l.Log(lv, map[string]interface{}{ 76 | "log": logstr, 77 | "time": time.Now(), 78 | "location": genLocation(), 79 | }) 80 | 81 | if lv >= Panic { 82 | panic(logstr) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /gc/inodedbtxloggc/inodedbtxloggc.go: -------------------------------------------------------------------------------- 1 | package inodedbtxloggc 2 | 3 | import ( 4 | "fmt" 5 | "sync/atomic" 6 | "time" 7 | 8 | "context" 9 | 10 | "github.com/nyaxt/otaru/inodedb" 11 | "github.com/nyaxt/otaru/logger" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | type UnneededTxIDThresholdFinder interface { 16 | FindUnneededTxIDThreshold() (inodedb.TxID, error) 17 | } 18 | 19 | type TransactionLogDeleter interface { 20 | DeleteTransactions(smallerThanID inodedb.TxID) error 21 | } 22 | 23 | var mylog = logger.Registry().Category("inodedbtxloggc") 24 | 25 | var gcRunning uint32 26 | 27 | func GC(ctx context.Context, thresfinder UnneededTxIDThresholdFinder, logdeleter TransactionLogDeleter, dryrun bool) error { 28 | start := time.Now() 29 | 30 | if !atomic.CompareAndSwapUint32(&gcRunning, 0, 1) { 31 | return fmt.Errorf("Another inodedbtxloggc is already running.") 32 | } 33 | defer atomic.StoreUint32(&gcRunning, 0) 34 | 35 | zap.S().Infof("GC start. Dryrun: %t. Trying to find UnneededTxIDThreshold.", dryrun) 36 | 37 | txid, err := thresfinder.FindUnneededTxIDThreshold() 38 | if err != nil { 39 | return fmt.Errorf("Failed to find UnneededTxIDThreshold: %v", err) 40 | } 41 | if txid == inodedb.AnyVersion { 42 | zap.S().Infof("UnneededTxIDThreshold was AnyVersion. No TxID log to be deleted") 43 | return nil 44 | } 45 | zap.S().Infof("Found UnneededTxIDThreshold: %v", txid) 46 | 47 | if err := ctx.Err(); err != nil { 48 | zap.S().Infof("Detected cancel. Bailing out.") 49 | return err 50 | } 51 | 52 | if dryrun { 53 | zap.S().Infof("Dry run. Not actually deleting txlog.") 54 | } else { 55 | if err := logdeleter.DeleteTransactions(txid); err != nil { 56 | return err 57 | } 58 | } 59 | zap.S().Infof("GC success. Dryrun: %t. The whole GC took %v.", dryrun, time.Since(start)) 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /cli/conn.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "errors" 8 | "fmt" 9 | 10 | "github.com/nyaxt/otaru/util/readpem" 11 | "go.uber.org/zap" 12 | "google.golang.org/grpc" 13 | "google.golang.org/grpc/credentials" 14 | ) 15 | 16 | var ErrUnknownVhost = errors.New("Unknown vhost.") 17 | 18 | type ConnectionInfo struct { 19 | ApiEndpoint string 20 | TLSConfig *tls.Config 21 | } 22 | 23 | func QueryConnectionInfo(cfg *CliConfig, vhost string) (*ConnectionInfo, error) { 24 | h, ok := cfg.Host[vhost] 25 | if !ok { 26 | return nil, ErrUnknownVhost 27 | } 28 | 29 | ci := ConnectionInfoFromHost(h) 30 | return ci, nil 31 | } 32 | 33 | func ConnectionInfoFromHost(h *Host) *ConnectionInfo { 34 | var tc tls.Config 35 | 36 | if h.CACert != nil { 37 | cp := x509.NewCertPool() 38 | cp.AddCert(h.CACert) 39 | 40 | tc.RootCAs = cp 41 | } 42 | if len(h.Certs) != 0 { 43 | zap.S().Infof("Configuring client cert: cn=%s", h.Certs[0].Subject.CommonName) 44 | tlscert := readpem.TLSCertificate(h.Certs, h.Key) 45 | tc.Certificates = []tls.Certificate{tlscert} 46 | } 47 | if h.OverrideServerName != "" { 48 | tc.ServerName = h.OverrideServerName 49 | } 50 | 51 | return &ConnectionInfo{ 52 | ApiEndpoint: h.ApiEndpoint, 53 | TLSConfig: &tc, 54 | } 55 | } 56 | 57 | func (ci *ConnectionInfo) DialGrpc(ctx context.Context) (*grpc.ClientConn, error) { 58 | zap.S().Infof("about to dial %s with len(tlsc.Certificates)=%d", ci.ApiEndpoint, len(ci.TLSConfig.Certificates)) 59 | 60 | opts := []grpc.DialOption{ 61 | grpc.WithTransportCredentials(credentials.NewTLS(ci.TLSConfig)), 62 | } 63 | conn, err := grpc.DialContext(ctx, ci.ApiEndpoint, opts...) 64 | if err != nil { 65 | return nil, fmt.Errorf("Failed to grpc.Dial(%q). err: %v", ci.ApiEndpoint, err) 66 | } 67 | return conn, nil 68 | } 69 | -------------------------------------------------------------------------------- /fuse/common.go: -------------------------------------------------------------------------------- 1 | package fuse 2 | 3 | import ( 4 | "os" 5 | 6 | bfuse "github.com/nyaxt/fuse" 7 | "go.uber.org/zap" 8 | 9 | "github.com/nyaxt/otaru/filesystem" 10 | oflags "github.com/nyaxt/otaru/flags" 11 | "github.com/nyaxt/otaru/inodedb" 12 | ) 13 | 14 | func otaruSetattr(fs *filesystem.FileSystem, id inodedb.ID, req *bfuse.SetattrRequest) error { 15 | var valid filesystem.ValidAttrFields 16 | var a filesystem.Attr 17 | 18 | if req.Valid.Uid() { 19 | valid |= filesystem.UidValid 20 | a.Uid = req.Uid 21 | } 22 | if req.Valid.Gid() { 23 | valid |= filesystem.GidValid 24 | a.Gid = req.Gid 25 | } 26 | if req.Valid.Mode() { 27 | valid |= filesystem.PermModeValid 28 | a.PermMode = uint16(req.Mode & os.ModePerm) 29 | } 30 | if req.Valid.Atime() { 31 | // otaru fs doesn't keep atime. set mtime instead. 32 | valid |= filesystem.ModifiedTValid 33 | a.ModifiedT = req.Atime 34 | } 35 | if req.Valid.Mtime() { 36 | valid |= filesystem.ModifiedTValid 37 | a.ModifiedT = req.Mtime 38 | } 39 | 40 | if valid != 0 { 41 | if err := fs.SetAttr(id, a, valid); err != nil { 42 | return err 43 | } 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func Bazil2OtaruFlags(bf bfuse.OpenFlags) int { 50 | ret := 0 51 | if bf.IsReadOnly() { 52 | ret = oflags.O_RDONLY 53 | } else if bf.IsWriteOnly() { 54 | ret = oflags.O_WRONLY 55 | } else if bf.IsReadWrite() { 56 | ret = oflags.O_RDWR 57 | } 58 | 59 | if bf&bfuse.OpenAppend != 0 { 60 | ret |= oflags.O_APPEND 61 | } 62 | if bf&bfuse.OpenCreate != 0 { 63 | ret |= oflags.O_CREATE 64 | } 65 | if bf&bfuse.OpenExclusive != 0 { 66 | ret |= oflags.O_EXCL 67 | } 68 | if bf&bfuse.OpenSync != 0 { 69 | zap.S().Errorf("FIXME: OpenSync not supported yet !!!!!!!!!!!") 70 | } 71 | if bf&bfuse.OpenTruncate != 0 { 72 | ret |= oflags.O_TRUNCATE 73 | } 74 | 75 | return ret 76 | } 77 | -------------------------------------------------------------------------------- /blobstore/mockblobstore.go: -------------------------------------------------------------------------------- 1 | package blobstore 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/flags" 5 | ) 6 | 7 | type MockBlobStoreOperation struct { 8 | Type rune 9 | Offset int64 10 | Length int 11 | FirstByte byte 12 | } 13 | 14 | type MockBlobHandle struct { 15 | Log []MockBlobStoreOperation 16 | PayloadLen int64 17 | } 18 | 19 | func NewMockBlobHandle() *MockBlobHandle { 20 | return &MockBlobHandle{ 21 | Log: []MockBlobStoreOperation{}, 22 | PayloadLen: 0, 23 | } 24 | } 25 | 26 | func (bh *MockBlobHandle) PRead(p []byte, offset int64) error { 27 | if len(p) == 0 { 28 | return nil 29 | } 30 | bh.Log = append(bh.Log, MockBlobStoreOperation{'R', offset, len(p), p[0]}) 31 | return nil 32 | } 33 | 34 | func (bh *MockBlobHandle) PWrite(p []byte, offset int64) error { 35 | if len(p) == 0 { 36 | return nil 37 | } 38 | bh.Log = append(bh.Log, MockBlobStoreOperation{'W', offset, len(p), p[0]}) 39 | 40 | right := offset + int64(len(p)) 41 | if right > bh.PayloadLen { 42 | bh.PayloadLen = right 43 | } 44 | return nil 45 | } 46 | 47 | func (bh *MockBlobHandle) Size() int64 { 48 | return bh.PayloadLen 49 | } 50 | 51 | func (bh *MockBlobHandle) Truncate(size int64) error { 52 | bh.PayloadLen = size 53 | return nil 54 | } 55 | 56 | func (bh *MockBlobHandle) Close() error { 57 | return nil 58 | } 59 | 60 | type MockBlobStore struct { 61 | Paths map[string]*MockBlobHandle 62 | } 63 | 64 | func NewMockBlobStore() *MockBlobStore { 65 | return &MockBlobStore{make(map[string]*MockBlobHandle)} 66 | } 67 | 68 | func (bs *MockBlobStore) Open(blobpath string, flags int) (BlobHandle, error) { 69 | bh := bs.Paths[blobpath] 70 | if bh == nil { 71 | bh = NewMockBlobHandle() 72 | bs.Paths[blobpath] = bh 73 | } 74 | return bh, nil 75 | } 76 | 77 | func (bs *MockBlobStore) Flags() int { 78 | return flags.O_RDWR 79 | } 80 | -------------------------------------------------------------------------------- /chunkstore/chunkheader_test.go: -------------------------------------------------------------------------------- 1 | package chunkstore_test 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/nyaxt/otaru/chunkstore" 8 | . "github.com/nyaxt/otaru/testutils" 9 | ) 10 | 11 | func TestChunkHeader_SerDes(t *testing.T) { 12 | // ser 13 | var b bytes.Buffer 14 | { 15 | h := chunkstore.ChunkHeader{ 16 | FrameEncapsulation: 0x02, 17 | PayloadLen: 0x0dedbeef, 18 | PayloadVersion: 0x12345678, 19 | OrigFilename: "/home/otaru/foobar.txt", 20 | OrigOffset: 0x0123456789abcdef, 21 | } 22 | 23 | if err := h.WriteTo(&b, TestCipher()); err != nil { 24 | t.Errorf("WriteTo failed: %v", err) 25 | } 26 | if !bytes.Equal(b.Bytes()[:3], []byte{0x05, 0xa6, chunkstore.CurrentFormat}) { 27 | t.Errorf("Unexpected ChunkHeader bytestream: %v", b) 28 | } 29 | } 30 | 31 | // des 32 | { 33 | var h chunkstore.ChunkHeader 34 | if err := h.ReadFrom(&b, TestCipher()); err != nil { 35 | t.Errorf("ReadFrom failed: %v", err) 36 | } 37 | 38 | if h.FrameEncapsulation != chunkstore.CurrentFrameEncapsulation { 39 | t.Errorf("Failed to unmarshal FrameEncapsulation") 40 | } 41 | if h.PayloadLen != 0x0dedbeef { 42 | t.Errorf("Failed to unmarshal PayloadLen") 43 | } 44 | if h.PayloadVersion != 0x12345678 { 45 | t.Errorf("Failed to unmarshal PayloadVersion") 46 | } 47 | if h.OrigFilename != "/home/otaru/foobar.txt" { 48 | t.Errorf("Failed to unmarshal OrigFilename") 49 | } 50 | if h.OrigOffset != 0x0123456789abcdef { 51 | t.Errorf("Failed to unmarshal OrigOffset") 52 | } 53 | } 54 | } 55 | 56 | func TestChunkHeader_Read_BadMagic(t *testing.T) { 57 | b := []byte{0xba, 0xad, chunkstore.CurrentFormat, 0x02, 0xcd, 0xab, 0x21, 0x43, 0x01, 0x02, 0x03, 0x04} 58 | var h chunkstore.ChunkHeader 59 | if err := h.ReadFrom(bytes.NewBuffer(b), TestCipher()); err == nil { 60 | t.Errorf("UnmarshalBinary passed on bad magic!: %v", err) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /blobstore/memblobstore.go: -------------------------------------------------------------------------------- 1 | package blobstore 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/nyaxt/otaru/flags" 7 | ) 8 | 9 | type MemBlobHandle struct { 10 | Content []byte 11 | } 12 | 13 | func NewMemBlobHandle() *MemBlobHandle { 14 | return &MemBlobHandle{ 15 | Content: []byte{}, 16 | } 17 | } 18 | 19 | func (bh *MemBlobHandle) PRead(p []byte, offset int64) error { 20 | if offset < 0 || offset > bh.Size() { 21 | return io.EOF 22 | } 23 | if offset+int64(len(p)) > int64(len(bh.Content)) { 24 | return io.EOF 25 | } 26 | copy(p, bh.Content[offset:offset+int64(len(p))]) 27 | return nil 28 | } 29 | 30 | func (bh *MemBlobHandle) PWrite(p []byte, offset int64) error { 31 | if len(p) == 0 { 32 | return nil 33 | } 34 | 35 | right := offset + int64(len(p)) 36 | if right > int64(len(bh.Content)) { 37 | bh.Truncate(right) 38 | } 39 | copy(bh.Content[offset:int(offset)+len(p)], p) 40 | return nil 41 | } 42 | 43 | func (bh *MemBlobHandle) Size() int64 { 44 | return int64(len(bh.Content)) 45 | } 46 | 47 | func (bh *MemBlobHandle) Truncate(newSize int64) error { 48 | if newSize > bh.Size() { 49 | newContent := make([]byte, newSize) 50 | copy(newContent[:len(bh.Content)], bh.Content) 51 | bh.Content = newContent 52 | } else if newSize < bh.Size() { 53 | bh.Content = bh.Content[:int(newSize)] 54 | } 55 | return nil 56 | } 57 | 58 | func (bh *MemBlobHandle) Close() error { 59 | return nil 60 | } 61 | 62 | type MemBlobStore struct { 63 | Paths map[string]*MemBlobHandle 64 | } 65 | 66 | func NewMemBlobStore() *MemBlobStore { 67 | return &MemBlobStore{make(map[string]*MemBlobHandle)} 68 | } 69 | 70 | func (bs *MemBlobStore) Open(blobpath string, flags int) (BlobHandle, error) { 71 | bh := bs.Paths[blobpath] 72 | if bh == nil { 73 | bh = NewMemBlobHandle() 74 | bs.Paths[blobpath] = bh 75 | } 76 | return bh, nil 77 | } 78 | 79 | func (bs *MemBlobStore) Flags() int { 80 | return flags.O_RDWR 81 | } 82 | -------------------------------------------------------------------------------- /go-fuzz/chunkstore/chunkstore.go: -------------------------------------------------------------------------------- 1 | package chunkstore 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | 8 | "github.com/nyaxt/otaru/blobstore" 9 | "github.com/nyaxt/otaru/chunkstore" 10 | tu "github.com/nyaxt/otaru/testutils" 11 | ) 12 | 13 | func init() { tu.EnsureLogger() } 14 | 15 | const ( 16 | InvalidInput = -1 17 | NeutralInput = 0 18 | InterestingInput = 1 19 | ) 20 | 21 | type cmdpack struct { 22 | IsWrite uint8 23 | Offset uint32 24 | OpLen uint32 25 | } 26 | 27 | func Fuzz(data []byte) int { 28 | chunkstore.ChunkSplitSize = 1 * 1024 * 1024 29 | const AbsoluteMaxLen uint32 = 4 * 1024 * 1024 30 | 31 | caio := chunkstore.NewSimpleDBChunksArrayIO() 32 | bs := blobstore.NewMemBlobStore() 33 | cfio := chunkstore.NewChunkedFileIO(bs, tu.TestCipher(), caio) 34 | 35 | currLen := uint32(0) 36 | 37 | reader := bytes.NewBuffer(data) 38 | cmdp := cmdpack{} 39 | iobuf := make([]byte, AbsoluteMaxLen) 40 | for n := byte(0); true; n++ { 41 | if err := binary.Read(reader, binary.BigEndian, &cmdp); err != nil { 42 | if n < 4 { 43 | return InvalidInput 44 | } else { 45 | return NeutralInput 46 | } 47 | } 48 | fmt.Printf("===================== Cmd %d\n", n) 49 | 50 | isWrite := (cmdp.IsWrite & 1) == 1 51 | offset := uint32(0) 52 | if currLen > 0 { 53 | offset = cmdp.Offset % currLen 54 | } 55 | for i, _ := range iobuf { 56 | iobuf[i] = n 57 | } 58 | if isWrite { 59 | opLen := cmdp.OpLen % (AbsoluteMaxLen - offset) 60 | if err := cfio.PWrite(iobuf[:opLen], int64(offset)); err != nil { 61 | panic(err) 62 | } 63 | if currLen < offset+opLen { 64 | currLen = offset + opLen 65 | } 66 | } else { 67 | maxLen := currLen - offset 68 | if maxLen == 0 { 69 | return InvalidInput 70 | } 71 | opLen := cmdp.OpLen % maxLen 72 | 73 | if _, err := cfio.ReadAt(iobuf[:opLen], int64(offset)); err != nil { 74 | panic(err) 75 | } 76 | } 77 | } 78 | return NeutralInput 79 | } 80 | -------------------------------------------------------------------------------- /blobstore/cachedblobstore/cachedbackendversion_test.go: -------------------------------------------------------------------------------- 1 | package cachedblobstore_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/nyaxt/otaru/blobstore/cachedblobstore" 7 | tu "github.com/nyaxt/otaru/testutils" 8 | ) 9 | 10 | func TestCachedBackendVersion_UseCached(t *testing.T) { 11 | bs := tu.TestFileBlobStore() 12 | cbv := cachedblobstore.NewCachedBackendVersion(bs, tu.TestQueryVersion) 13 | 14 | cbv.Set("foobar", 123) 15 | v, err := cbv.Query("foobar") 16 | if err != nil { 17 | t.Errorf("Unexpected Query() err: %v", err) 18 | return 19 | } 20 | if v != 123 { 21 | t.Errorf("Unexpected Query() result. v: %d", v) 22 | return 23 | } 24 | } 25 | 26 | func TestCachedBackendVersion_FillCache(t *testing.T) { 27 | bs := tu.TestFileBlobStore() 28 | if err := tu.WriteVersionedBlob(bs, "uncached", 42); err != nil { 29 | t.Errorf("%v", err) 30 | return 31 | } 32 | 33 | cbv := cachedblobstore.NewCachedBackendVersion(bs, tu.TestQueryVersion) 34 | v, err := cbv.Query("uncached") 35 | if err != nil { 36 | t.Errorf("Unexpected Query() err: %v", err) 37 | return 38 | } 39 | if v != 42 { 40 | t.Errorf("Unexpected Query() result. v: %d", v) 41 | return 42 | } 43 | } 44 | 45 | func TestCachedBackedVersion_SaveRestore(t *testing.T) { 46 | bs := tu.TestFileBlobStore() 47 | 48 | cbv := cachedblobstore.NewCachedBackendVersion(bs, tu.TestQueryVersion) 49 | cbv.Set("foobar", 123) 50 | if err := cbv.SaveStateToBlobstore(tu.TestCipher(), bs); err != nil { 51 | t.Errorf("Failed to save state: %v", err) 52 | return 53 | } 54 | 55 | cbv2 := cachedblobstore.NewCachedBackendVersion(bs, tu.TestQueryVersion) 56 | if err := cbv2.RestoreStateFromBlobstore(tu.TestCipher(), bs); err != nil { 57 | t.Errorf("Failed to restore state: %v", err) 58 | } 59 | v, err := cbv2.Query("foobar") 60 | if err != nil { 61 | t.Errorf("Unexpected Query() err: %v", err) 62 | return 63 | } 64 | if v != 123 { 65 | t.Errorf("Unexpected Query() result. v: %d", v) 66 | return 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /debugcmd/otaru-btncrypt-benchmark/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/rand" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "time" 9 | 10 | tu "github.com/nyaxt/otaru/testutils" 11 | "go.uber.org/zap" 12 | 13 | "github.com/dustin/go-humanize" 14 | 15 | "github.com/nyaxt/otaru/btncrypt" 16 | ) 17 | 18 | var ( 19 | flagSize = flag.String("size", "100MB", "Test target blob size") 20 | ) 21 | 22 | func Usage() { 23 | fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) 24 | fmt.Fprintf(os.Stderr, " %s\n", os.Args[0]) 25 | flag.PrintDefaults() 26 | } 27 | 28 | func main() { 29 | panic("migrate to urfave/cli") 30 | 31 | flag.Usage = Usage 32 | flag.Parse() 33 | 34 | size, err := humanize.ParseBytes(*flagSize) 35 | if err != nil { 36 | zap.S().Errorf("Failed to parse size: %s", *flagSize) 37 | } 38 | zap.S().Infof("Target blob size: %d", size) 39 | tstart := time.Now() 40 | buf := make([]byte, size) 41 | if _, err := rand.Read(buf); err != nil { 42 | zap.S().Errorf("Failed to generate random seq: %v", err) 43 | } 44 | zap.S().Infof("Preparing target took: %v", time.Since(tstart)) 45 | 46 | var envelope []byte 47 | { 48 | tstart = time.Now() 49 | envelope, err = btncrypt.Encrypt(tu.TestCipher(), buf) 50 | elapsed := time.Since(tstart) 51 | 52 | if err != nil { 53 | zap.S().Errorf("Failed to encrypt: %v", err) 54 | } 55 | zap.S().Infof("Encrypt took %v", elapsed) 56 | sizeMB := (float64)(size) / (1000 * 1000) 57 | MBps := sizeMB / elapsed.Seconds() 58 | Mbps := MBps * 8 59 | zap.S().Infof("%v MB/s %v Mbps", MBps, Mbps) 60 | } 61 | 62 | { 63 | tstart = time.Now() 64 | _, err = btncrypt.Decrypt(tu.TestCipher(), envelope, len(buf)) 65 | elapsed := time.Since(tstart) 66 | 67 | if err != nil { 68 | zap.S().Errorf("Failed to decrypt: %v", err) 69 | } 70 | zap.S().Infof("Decrypt took %v", elapsed) 71 | sizeMB := (float64)(size) / (1000 * 1000) 72 | MBps := sizeMB / elapsed.Seconds() 73 | Mbps := MBps * 8 74 | zap.S().Infof("%v MB/s %v Mbps", MBps, Mbps) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /fuse/filesystem_linux_test.go: -------------------------------------------------------------------------------- 1 | package fuse_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path" 7 | "syscall" 8 | "testing" 9 | 10 | tu "github.com/nyaxt/otaru/testutils" 11 | ) 12 | 13 | // Note: invoke test by: docker build -t otaru-dev . && docker run -ti --rm -privileged=true otaru-dev go test ./fuse -run TestServe_ChUGid -v 14 | func TestServe_ChUGid(t *testing.T) { 15 | maybeSkipTest(t) 16 | fs := fusetestFileSystem() 17 | 18 | if os.Getuid() != 0 { 19 | t.Skip("ChUGid test is only possible when root") 20 | return 21 | } 22 | 23 | fusetestCommon(t, fs, func(mountpoint string) { 24 | dirpath := path.Join(mountpoint, "hokkaido") 25 | if err := os.Mkdir(dirpath, 0755); err != nil { 26 | t.Errorf("Failed to mkdir: %v", err) 27 | return 28 | } 29 | 30 | filepath := path.Join(dirpath, "otaru.txt") 31 | if err := ioutil.WriteFile(filepath, tu.HelloWorld, 0644); err != nil { 32 | t.Errorf("failed to write file: %v", err) 33 | return 34 | } 35 | 36 | if err := os.Chown(filepath, 123, 456); err != nil { 37 | t.Errorf("Failed to chown file: %v", err) 38 | return 39 | } 40 | if err := os.Chown(dirpath, 234, 567); err != nil { 41 | t.Errorf("Failed to chown dir: %v", err) 42 | return 43 | } 44 | 45 | fi, err := os.Stat(filepath) 46 | if err != nil { 47 | t.Errorf("Failed to stat dir: %v", err) 48 | return 49 | } 50 | if fi.IsDir() { 51 | t.Errorf("file is dir!") 52 | } 53 | if uid := fi.Sys().(*syscall.Stat_t).Uid; uid != 123 { 54 | t.Errorf("Invalid UID: %d", uid) 55 | } 56 | if gid := fi.Sys().(*syscall.Stat_t).Gid; gid != 456 { 57 | t.Errorf("Invalid GID: %d", gid) 58 | } 59 | 60 | fi, err = os.Stat(dirpath) 61 | if err != nil { 62 | t.Errorf("Failed to stat dir: %v", err) 63 | return 64 | } 65 | if !fi.IsDir() { 66 | t.Errorf("dir isn't dir!") 67 | } 68 | if uid := fi.Sys().(*syscall.Stat_t).Uid; uid != 234 { 69 | t.Errorf("Invalid UID: %d", uid) 70 | } 71 | if gid := fi.Sys().(*syscall.Stat_t).Gid; gid != 567 { 72 | t.Errorf("Invalid GID: %d", gid) 73 | } 74 | }) 75 | } 76 | -------------------------------------------------------------------------------- /webdav/serve.go: -------------------------------------------------------------------------------- 1 | package webdav 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "errors" 7 | "fmt" 8 | "net" 9 | "net/http" 10 | 11 | "github.com/nyaxt/otaru/basicauth" 12 | "github.com/nyaxt/otaru/cli" 13 | "github.com/nyaxt/otaru/logger" 14 | "github.com/nyaxt/otaru/util/readpem" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | func Serve(ctx context.Context, cfg *cli.CliConfig) error { 19 | s := zap.S().Named("webdav.Serve") 20 | 21 | wcfg := cfg.Webdav 22 | 23 | var handler http.Handler 24 | handler = &Handler{cfg} 25 | 26 | if wcfg.ListenAddr == "" { 27 | return errors.New("Webdav server listen addr must be configured.") 28 | } 29 | 30 | lis, err := net.Listen("tcp", wcfg.ListenAddr) 31 | if err != nil { 32 | return fmt.Errorf("Failed to listen \"%s\": %v", wcfg.ListenAddr, err) 33 | } 34 | 35 | if wcfg.BasicAuthPassword == "" { 36 | s.Warnf("Basic auth not enabled!") 37 | } else { 38 | s.Infof("Basic auth enabled.") 39 | handler = &basicauth.Handler{ 40 | User: wcfg.BasicAuthUser, 41 | Password: wcfg.BasicAuthPassword, 42 | Handler: handler, 43 | } 44 | } 45 | 46 | mux := http.NewServeMux() 47 | mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { 48 | w.Header().Set("Content-Type", "text/plain") 49 | _, _ = w.Write([]byte("ok\n")) 50 | }) 51 | mux.Handle("/", handler) 52 | 53 | loghandler := logger.HttpHandler(s.Desugar(), mux) 54 | 55 | httpsrv := http.Server{ 56 | Addr: wcfg.ListenAddr, 57 | Handler: loghandler, 58 | } 59 | if !wcfg.NoTls { 60 | // Note: This doesn't enable h2. Reconsider this if there is a webdav client w/ h2 support. 61 | tc := readpem.TLSCertificate(wcfg.Certs, wcfg.Key) 62 | httpsrv.TLSConfig = &tls.Config{ 63 | Certificates: []tls.Certificate{tc}, 64 | NextProtos: []string{"http/1.1"}, 65 | } 66 | lis = tls.NewListener(lis, httpsrv.TLSConfig) 67 | } 68 | 69 | go func() { 70 | <-ctx.Done() 71 | httpsrv.Close() 72 | lis.Close() 73 | }() 74 | 75 | if err := httpsrv.Serve(lis); !errors.Is(err, http.ErrServerClosed) { 76 | return err 77 | } 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /testutils/testca/kmgmbasedir/clientauth/issuedb.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "sn": 4696538073341514829, 4 | "state": "active", 5 | "certPem": "-----BEGIN CERTIFICATE-----\nMIIBojCCAUmgAwIBAgIIQS10fo+eVE0wCgYIKoZIzj0EAwIwPzELMAkGA1UEBhMC\nSlAxDjAMBgNVBAgTBVRva3lvMSAwHgYDVQQDExdvdGFydSBkZXYgY2xpZW50YXV0\naCBDQTAgFw0yMjA1MDQwNTE5MTVaGA8yMDk5MTIzMTIzNTkwMFowFjEUMBIGA1UE\nAxMLYWRtaW4gYWxpY2UwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARJfV8KErOF\nMTNeICYgW6VeXohMyXLAAkDgCVoANcnqNPxRoEkGUyeNDut6RipwYpqkwcr1sB0e\nW1KDJx0jFsx+o1YwVDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUH\nAwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBQ6TxkyIOt4rqprYdAORlV5gYfq\nujAKBggqhkjOPQQDAgNHADBEAiBj9xEhh43CNdkrILOqv/xEoEVnn/XBfgNagkuS\n9ceWcAIgHV3xjS5CCuvjLOz1wNI9Da8u4RyHcGx/+4EqPI2x5Kk=\n-----END CERTIFICATE-----\n" 6 | }, 7 | { 8 | "sn": 7569943481206714778, 9 | "state": "active", 10 | "certPem": "-----BEGIN CERTIFICATE-----\nMIIBozCCAUqgAwIBAgIIaQ3XfSccMZowCgYIKoZIzj0EAwIwPzELMAkGA1UEBhMC\nSlAxDjAMBgNVBAgTBVRva3lvMSAwHgYDVQQDExdvdGFydSBkZXYgY2xpZW50YXV0\naCBDQTAgFw0yMjA1MDQwNTE5MTVaGA8yMDk5MTIzMTIzNTkwMFowFzEVMBMGA1UE\nAxMMcmVhZG9ubHkgYm9iMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE7kFF8CsS\nz0uPigXJhTZQdoXCJ2mz/dpKupDMpBZqTmiUxqeIokW9lRxh+6MQGcS7s9VkfqeT\nlPYrd88pVe38YqNWMFQwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUF\nBwMCMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUOk8ZMiDreK6qa2HQDkZVeYGH\n6rowCgYIKoZIzj0EAwIDRwAwRAIgNF4NuSrfWSg8u2aDlKpwquxRLkW7xkmVRABD\n9/yRB4gCIAKkDdEHtMUjTlRPvrEFE9jmGW4HuvLDhp+ycLANOYwz\n-----END CERTIFICATE-----\n" 11 | }, 12 | { 13 | "sn": 6434263340484339584, 14 | "state": "active", 15 | "certPem": "-----BEGIN CERTIFICATE-----\nMIIBqDCCAU2gAwIBAgIIWUsYawDjJ4AwCgYIKoZIzj0EAwIwPzELMAkGA1UEBhMC\nSlAxDjAMBgNVBAgTBVRva3lvMSAwHgYDVQQDExdvdGFydSBkZXYgY2xpZW50YXV0\naCBDQTAgFw0yMjA1MDQwNTE5MTVaGA8yMDk5MTIzMTIzNTkwMFowGjEYMBYGA1UE\nAxMPaW52YWxpZCBjaGFybGllMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEGL+Q\nT+rCJl8VAe0A2rHArg51tmXay4on+LGfdvCxsKi3g4lJ4j2mwRpbGPePqbr8dh79\nK69xZw34LYwjnJlL/6NWMFQwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsG\nAQUFBwMCMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUOk8ZMiDreK6qa2HQDkZV\neYGH6rowCgYIKoZIzj0EAwIDSQAwRgIhAO0gsOeNiJ/5Cmd0OptybTDjNNo/i1IY\n2VveXQyVfAGHAiEA3MzywChstntqaTQDAAK7ZnffBU5EC6iA2o24TsjZH24=\n-----END CERTIFICATE-----\n" 16 | } 17 | ] -------------------------------------------------------------------------------- /gc/inodedbtxloggc/inodedbtxloggc_test.go: -------------------------------------------------------------------------------- 1 | package inodedbtxloggc_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "context" 7 | 8 | "github.com/nyaxt/otaru/gc/inodedbtxloggc" 9 | "github.com/nyaxt/otaru/inodedb" 10 | tu "github.com/nyaxt/otaru/testutils" 11 | ) 12 | 13 | func init() { tu.EnsureLogger() } 14 | 15 | type MockUnneededTxIDThresholdFinder inodedb.TxID 16 | 17 | func (n MockUnneededTxIDThresholdFinder) FindUnneededTxIDThreshold() (inodedb.TxID, error) { 18 | return inodedb.TxID(n), nil 19 | } 20 | 21 | type MockTransactionLogDeleter struct { 22 | called bool 23 | id inodedb.TxID 24 | } 25 | 26 | func (logdeleter *MockTransactionLogDeleter) DeleteTransactions(id inodedb.TxID) error { 27 | logdeleter.called = true 28 | logdeleter.id = id 29 | return nil 30 | } 31 | 32 | func TestINodeDBTxLogGC_DryRun(t *testing.T) { 33 | thresfinder := MockUnneededTxIDThresholdFinder(inodedb.TxID(345)) 34 | logdeleter := &MockTransactionLogDeleter{called: false, id: inodedb.AnyVersion} 35 | 36 | if err := inodedbtxloggc.GC(context.TODO(), thresfinder, logdeleter, true); err != nil { 37 | t.Errorf("GC err: %v", err) 38 | } 39 | 40 | if logdeleter.called { 41 | t.Errorf("Dry run invoked log deleter!") 42 | } 43 | } 44 | 45 | func TestINodeDBTxLogGC_RealRun(t *testing.T) { 46 | thresfinder := MockUnneededTxIDThresholdFinder(inodedb.TxID(345)) 47 | logdeleter := &MockTransactionLogDeleter{called: false, id: inodedb.AnyVersion} 48 | 49 | if err := inodedbtxloggc.GC(context.TODO(), thresfinder, logdeleter, false); err != nil { 50 | t.Errorf("GC err: %v", err) 51 | } 52 | 53 | if !logdeleter.called { 54 | t.Errorf("Log deleter was not invoked!") 55 | } 56 | if logdeleter.id != inodedb.TxID(345) { 57 | t.Errorf("Log deleter was invoked with unexpected txid: %v", logdeleter.id) 58 | } 59 | } 60 | 61 | func TestINodeDBTxLogGC_AnyVersion(t *testing.T) { 62 | thresfinder := MockUnneededTxIDThresholdFinder(inodedb.AnyVersion) 63 | logdeleter := &MockTransactionLogDeleter{called: false, id: 123} 64 | 65 | if err := inodedbtxloggc.GC(context.TODO(), thresfinder, logdeleter, false); err != nil { 66 | t.Errorf("GC err: %v", err) 67 | } 68 | 69 | if logdeleter.called { 70 | t.Errorf("Log deleter should not be invoked!") 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /inodedb/cacheddbtransactionlogio.go: -------------------------------------------------------------------------------- 1 | package inodedb 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/logger" 5 | "go.uber.org/zap" 6 | ) 7 | 8 | var clog = logger.Registry().Category("cacheddbtxlogio") 9 | 10 | type CachedDBTransactionLogIO struct { 11 | be DBTransactionLogIO 12 | 13 | ringbuf []DBTransaction 14 | next int 15 | oldestTxID TxID 16 | } 17 | 18 | var _ = DBTransactionLogIO(&CachedDBTransactionLogIO{}) 19 | 20 | const ringbufLen = 256 // FIXME: this should be >2k 21 | 22 | func NewCachedDBTransactionLogIO(be DBTransactionLogIO) *CachedDBTransactionLogIO { 23 | txio := &CachedDBTransactionLogIO{ 24 | be: be, 25 | ringbuf: make([]DBTransaction, ringbufLen), 26 | next: 0, 27 | oldestTxID: LatestVersion, 28 | } 29 | 30 | for i, _ := range txio.ringbuf { 31 | txio.ringbuf[i].TxID = 0 32 | } 33 | 34 | return txio 35 | } 36 | 37 | func (txio *CachedDBTransactionLogIO) AppendTransaction(tx DBTransaction) error { 38 | if err := txio.be.AppendTransaction(tx); err != nil { 39 | return err 40 | } 41 | 42 | txidToBeDeleted := txio.ringbuf[txio.next].TxID 43 | if txidToBeDeleted == txio.oldestTxID { 44 | txio.oldestTxID = txidToBeDeleted + 1 45 | } 46 | 47 | txio.ringbuf[txio.next] = tx 48 | if txio.oldestTxID == LatestVersion { 49 | txio.oldestTxID = tx.TxID 50 | } 51 | txio.next++ 52 | if txio.next == ringbufLen { 53 | txio.next = 0 54 | } 55 | return nil 56 | } 57 | 58 | func (txio *CachedDBTransactionLogIO) QueryTransactions(minID TxID) ([]DBTransaction, error) { 59 | if minID < txio.oldestTxID { 60 | zap.S().Debugf("Queried id range of \">= %d\" is not cached. Falling back to backend.", minID) 61 | return txio.be.QueryTransactions(minID) 62 | } 63 | 64 | return txio.QueryCachedTransactions(minID) 65 | } 66 | 67 | func (txio *CachedDBTransactionLogIO) QueryCachedTransactions(minID TxID) ([]DBTransaction, error) { 68 | // FIXME: Optimize? no need to scan over all ringbuf contents 69 | 70 | result := []DBTransaction{} 71 | for _, tx := range txio.ringbuf[txio.next:] { 72 | if tx.TxID >= minID { 73 | result = append(result, tx) 74 | } 75 | } 76 | for _, tx := range txio.ringbuf[:txio.next] { 77 | if tx.TxID >= minID { 78 | result = append(result, tx) 79 | } 80 | } 81 | return result, nil 82 | } 83 | -------------------------------------------------------------------------------- /doc/config.toml.example: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Otaru config file example. 3 | ################################################################################ 4 | 5 | # GCP settings 6 | 7 | # - Google Cloud Platform project name for storing otaru blobs/metadata. 8 | project_name = "example-com" 9 | 10 | # - Google Cloud Storage bucket for storing otaru blobs. 11 | bucket_name = "otaru-my-foobar" 12 | # - If set to true, use [bucket_name]+"-meta" for storing metadata. 13 | use_separate_bucket_for_metadata = true 14 | 15 | # - Service account private key json file path 16 | # credentials_file_path = "${OTARUDIR}/credentials.json" 17 | 18 | # Blob cache config 19 | 20 | # - Directory for storing cache. 21 | cache_dir = "/var/cache/otaru" 22 | # - Cache directory high water mark: 23 | # cache discard will run if cache dir usage is above this threshold. 24 | cache_high_watermark = "25GB" 25 | # - Cache directory low water mark: 26 | # cache discard will try to keep cache dir usage below this threshold. 27 | cache_low_watermark = "18GB" 28 | 29 | # - If true, forbid any modificatino to the filesystem. 30 | # read_only = false 31 | 32 | # - If specified, FUSE mount to specified point. 33 | # fuse_mount_point = "/mnt/otaru" 34 | # - Run GC once per specified seconds. Set -1 to disable auto GC. 35 | # gc_period = 900 36 | 37 | # API server config 38 | [api_server] 39 | # - API server listen addr. Defaults to ":10246". 40 | # listen_addr = ":10246" 41 | 42 | # - Enable debug apis. Makes otaru insecure. Defaults to false. 43 | # enable_debug = false 44 | 45 | # - If specified, serve webui from the specified directory instead of using embedded one. 46 | # webui_root_path = "/home/kouhei/go/src/github.com/nyaxt/otaru/webui/dist" 47 | 48 | # - TLS certificate file. Defaults to "${OTARUDIR}/cert.pem" 49 | # cert_file = "cert.pem" 50 | 51 | # - TLS key file. Defaults to "${OTARUDIR}/cert-key.pem" 52 | # key_file = "cert-key.pem" 53 | 54 | # - CORS allowed origins. 55 | # Domains listed here will be added to "Access-Control-Allow-Origin" HTTP header. 56 | # cors_allowed_origins = ["https://localhost:9000"] 57 | 58 | # Logger config 59 | [logger] 60 | 61 | # - Log levels at startup. The log levels can be dynamically configured via webui. 62 | log_level = {"*" = "Debug", "bfuse" = "Info", "scheduler" = "Info"} 63 | -------------------------------------------------------------------------------- /cli/get.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path" 10 | 11 | "github.com/nyaxt/otaru/logger" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | var Log = logger.Registry().Category("cli") 16 | 17 | func Get(ctx context.Context, cfg *CliConfig, args []string) error { 18 | fset := flag.NewFlagSet("get", flag.ExitOnError) 19 | flagO := fset.String("o", "", "destination file path") 20 | flagC := fset.String("C", "", "destination dir path") 21 | fset.Usage = func() { 22 | fmt.Printf("Usage of %s get:\n", os.Args[0]) 23 | fmt.Printf(" %s get OTARU_PATH...\n", os.Args[0]) 24 | fset.PrintDefaults() 25 | } 26 | fset.Parse(args[1:]) 27 | 28 | if fset.NArg() == 0 { 29 | fset.Usage() 30 | return fmt.Errorf("Invalid number of arguments.") 31 | } 32 | if *flagO != "" && fset.NArg() != 1 { 33 | fset.Usage() 34 | return fmt.Errorf("Only one path is allowed when specified -o option.") 35 | } 36 | 37 | var destdir string 38 | if *flagC != "" { 39 | destdir = *flagC 40 | } else { 41 | var err error 42 | destdir, err = os.Getwd() 43 | if err != nil { 44 | return fmt.Errorf("Failed to query current dir: %v", err) 45 | } 46 | } 47 | fi, err := os.Stat(destdir) 48 | if err != nil { 49 | return fmt.Errorf("Failed to stat target dir: %v", err) 50 | } 51 | if !fi.IsDir() { 52 | return fmt.Errorf("Specified destination is not a dir") 53 | } 54 | 55 | for _, srcstr := range fset.Args() { 56 | r, err := NewReader(srcstr, WithCliConfig(cfg), WithContext(ctx)) 57 | if err != nil { 58 | return fmt.Errorf("%v", err) 59 | } 60 | 61 | var dest string 62 | if *flagO != "" { 63 | dest = *flagO 64 | } else { 65 | dest = path.Join(destdir, path.Base(srcstr)) 66 | } 67 | w, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0644) // FIXME: r.Stat().FileMode 68 | if err != nil { 69 | r.Close() 70 | return fmt.Errorf("Failed to open dest file: %v", err) 71 | } 72 | zap.S().Infof("Remote %s -> Local %s", srcstr, dest) 73 | 74 | if _, err := io.Copy(w, r); err != nil { 75 | r.Close() 76 | w.Close() 77 | return fmt.Errorf("io.Copy failed: %v", err) 78 | } 79 | if err := r.Close(); err != nil { 80 | return fmt.Errorf("Failed to Close(src): %v", err) 81 | } 82 | if err := w.Close(); err != nil { 83 | return fmt.Errorf("Failed to Close(dest): %v", err) 84 | } 85 | } 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /blobstore/cachedblobstore/cacheusagestats.go: -------------------------------------------------------------------------------- 1 | package cachedblobstore 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | "time" 7 | 8 | fl "github.com/nyaxt/otaru/flags" 9 | ) 10 | 11 | type usageStatEntry struct { 12 | lastUsed time.Time 13 | readCount int 14 | writeCount int 15 | } 16 | 17 | type CacheUsageStats struct { 18 | mu sync.Mutex 19 | entries map[string]usageStatEntry 20 | } 21 | 22 | func NewCacheUsageStats() *CacheUsageStats { 23 | return &CacheUsageStats{entries: make(map[string]usageStatEntry)} 24 | } 25 | 26 | func (s *CacheUsageStats) ObserveOpen(blobpath string, flags int) { 27 | s.mu.Lock() 28 | defer s.mu.Unlock() 29 | 30 | e := s.entries[blobpath] 31 | if fl.IsReadAllowed(flags) { 32 | e.readCount++ 33 | } 34 | if fl.IsWriteAllowed(flags) { 35 | e.writeCount++ 36 | } 37 | e.lastUsed = time.Now() 38 | 39 | s.entries[blobpath] = e 40 | } 41 | 42 | func (s *CacheUsageStats) ObserveRemoveBlob(blobpath string) { 43 | s.mu.Lock() 44 | defer s.mu.Unlock() 45 | 46 | delete(s.entries, blobpath) 47 | } 48 | 49 | func (s *CacheUsageStats) ImportBlobList(blobpaths []string) { 50 | s.mu.Lock() 51 | defer s.mu.Unlock() 52 | 53 | var tzero time.Time 54 | 55 | for _, bp := range blobpaths { 56 | s.entries[bp] = usageStatEntry{lastUsed: tzero} 57 | } 58 | } 59 | 60 | type lastUsedSorter struct { 61 | bps []string 62 | entries map[string]usageStatEntry 63 | } 64 | 65 | func (s lastUsedSorter) Len() int { return len(s.bps) } 66 | 67 | func (s lastUsedSorter) Swap(i, j int) { 68 | s.bps[i], s.bps[j] = s.bps[j], s.bps[i] 69 | } 70 | 71 | func (s lastUsedSorter) Less(i, j int) bool { 72 | a := s.entries[s.bps[i]].lastUsed 73 | b := s.entries[s.bps[j]].lastUsed 74 | 75 | return a.Before(b) 76 | } 77 | 78 | func (s *CacheUsageStats) FindLeastUsed() []string { 79 | s.mu.Lock() 80 | defer s.mu.Unlock() 81 | 82 | bps := make([]string, 0, len(s.entries)) 83 | for bp, _ := range s.entries { 84 | bps = append(bps, bp) 85 | } 86 | sort.Sort(lastUsedSorter{bps, s.entries}) 87 | 88 | return bps 89 | } 90 | 91 | type CacheUsageStatsView struct { 92 | NumEntries int `json:"num_entries"` 93 | } 94 | 95 | func (s *CacheUsageStats) View() CacheUsageStatsView { 96 | s.mu.Lock() 97 | defer s.mu.Unlock() 98 | 99 | return CacheUsageStatsView{ 100 | NumEntries: len(s.entries), 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /filesystem/filesystem_test.go: -------------------------------------------------------------------------------- 1 | package filesystem_test 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/filesystem" 5 | "github.com/nyaxt/otaru/flags" 6 | "github.com/nyaxt/otaru/inodedb" 7 | "github.com/nyaxt/otaru/testutils" 8 | "go.uber.org/zap" 9 | 10 | "bytes" 11 | "testing" 12 | ) 13 | 14 | func init() { testutils.EnsureLogger() } 15 | 16 | func TestFileWriteRead(t *testing.T) { 17 | snapshotio := inodedb.NewSimpleDBStateSnapshotIO() 18 | txio := inodedb.NewSimpleDBTransactionLogIO() 19 | idb, err := inodedb.NewEmptyDB(snapshotio, txio) 20 | if err != nil { 21 | t.Errorf("NewEmptyDB failed: %v", err) 22 | return 23 | } 24 | 25 | bs := testutils.TestFileBlobStore() 26 | fs := filesystem.NewFileSystem(idb, bs, testutils.TestCipher(), zap.L()) 27 | h, err := fs.OpenFileFullPath("/hello.txt", flags.O_RDWRCREATE, 0666) 28 | if err != nil { 29 | t.Errorf("OpenFileFullPath failed: %v", err) 30 | return 31 | } 32 | defer h.Close() 33 | 34 | err = h.PWrite(testutils.HelloWorld, 0) 35 | if err != nil { 36 | t.Errorf("PWrite failed: %v", err) 37 | } 38 | 39 | buf := make([]byte, 32) 40 | n, err := h.ReadAt(buf, 0) 41 | if err != nil { 42 | t.Errorf("PRead failed: %v", err) 43 | } 44 | buf = buf[:n] 45 | if n != len(testutils.HelloWorld) { 46 | t.Errorf("n: %d", n) 47 | } 48 | if !bytes.Equal(testutils.HelloWorld, buf) { 49 | t.Errorf("PRead content != PWrite content: %v", buf) 50 | } 51 | } 52 | 53 | func TestFile_CloseOpenFile(t *testing.T) { 54 | snapshotio := inodedb.NewSimpleDBStateSnapshotIO() 55 | txio := inodedb.NewSimpleDBTransactionLogIO() 56 | idb, err := inodedb.NewEmptyDB(snapshotio, txio) 57 | if err != nil { 58 | t.Errorf("NewEmptyDB failed: %v", err) 59 | return 60 | } 61 | 62 | bs := testutils.TestFileBlobStore() 63 | fs := filesystem.NewFileSystem(idb, bs, testutils.TestCipher(), zap.L()) 64 | h, err := fs.OpenFileFullPath("/hello.txt", flags.O_CREATE|flags.O_RDWR, 0666) 65 | if err != nil { 66 | t.Errorf("OpenFileFullPath failed: %v", err) 67 | return 68 | } 69 | 70 | if err = h.PWrite(testutils.HelloWorld, 0); err != nil { 71 | t.Errorf("PWrite failed: %v", err) 72 | return 73 | } 74 | 75 | if stats := fs.GetStats(); stats.NumOpenFiles != 1 { 76 | t.Errorf("NumOpenFiles %d != 1", stats.NumOpenFiles) 77 | } 78 | 79 | h.Close() 80 | 81 | if stats := fs.GetStats(); stats.NumOpenFiles != 0 { 82 | t.Errorf("NumOpenFiles %d != 0", stats.NumOpenFiles) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /extra/fe/apiserver/proxyhandler.go: -------------------------------------------------------------------------------- 1 | package apiserver 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "regexp" 9 | 10 | "github.com/nyaxt/otaru/apiserver" 11 | "github.com/nyaxt/otaru/basicauth" 12 | "github.com/nyaxt/otaru/cli" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | type apiproxy struct { 17 | cfg *cli.CliConfig 18 | } 19 | 20 | var reMatch = regexp.MustCompile(`^/proxy/(\w+)(/.*)$`) 21 | 22 | func copyHeader(d, s http.Header) { 23 | for k, v := range s { 24 | d[k] = v 25 | } 26 | } 27 | 28 | func (ap *apiproxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { 29 | path := r.URL.Path 30 | ms := reMatch.FindStringSubmatch(path) 31 | if len(ms) != 3 { 32 | http.Error(w, "Invalid proxy URL.", http.StatusBadRequest) 33 | return 34 | } 35 | // fmt.Printf("path %v match %v", path, ms) 36 | 37 | hname, tgtpath := ms[1], ms[2] 38 | 39 | ci, err := cli.QueryConnectionInfo(ap.cfg, hname) 40 | if err != nil { 41 | http.Error(w, "Failed to construct connection info.", http.StatusInternalServerError) 42 | return 43 | } 44 | 45 | hcli := &http.Client{ 46 | Transport: &http.Transport{ 47 | TLSClientConfig: ci.TLSConfig, 48 | }, 49 | } 50 | zap.S().Debugf("tc.RootCAs: %+v", ci.TLSConfig.RootCAs) 51 | url := &url.URL{ 52 | Scheme: "https", 53 | Host: ci.ApiEndpoint, 54 | Path: tgtpath, 55 | RawQuery: r.URL.RawQuery, 56 | } 57 | zap.S().Debugf("URL: %v", url) 58 | 59 | // FIXME: filter tgtpath 60 | // FIXME: add forwarded-for 61 | 62 | preq := &http.Request{ 63 | Method: r.Method, 64 | Header: make(http.Header), 65 | URL: url, 66 | Body: r.Body, 67 | } 68 | copyHeader(preq.Header, r.Header) 69 | 70 | presp, err := hcli.Do(preq) 71 | if err != nil { 72 | zap.S().Warnf("Failed to issue request: %v", err) 73 | http.Error(w, "Failed to issue request.", http.StatusInternalServerError) 74 | return 75 | } 76 | defer presp.Body.Close() 77 | 78 | // zap.S().Debugf("resp st: %d", presp.StatusCode) 79 | 80 | copyHeader(w.Header(), presp.Header) 81 | w.WriteHeader(presp.StatusCode) 82 | io.Copy(w, presp.Body) 83 | } 84 | 85 | func InstallProxyHandler(cfg *cli.CliConfig, basicuser, basicpassword string) apiserver.Option { 86 | return apiserver.AddMuxHook(func(_ context.Context, mux *http.ServeMux) error { 87 | mux.Handle("/proxy/", basicauth.Handler{ 88 | User: basicuser, 89 | Password: basicpassword, 90 | Handler: &apiproxy{ 91 | cfg: cfg, 92 | }, 93 | }) 94 | return nil 95 | }) 96 | } 97 | -------------------------------------------------------------------------------- /extra/fe/apiserver/server.go: -------------------------------------------------------------------------------- 1 | package apiserver 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" 9 | "go.uber.org/zap" 10 | "google.golang.org/grpc" 11 | 12 | "github.com/nyaxt/otaru/apiserver" 13 | "github.com/nyaxt/otaru/assets/webui" 14 | "github.com/nyaxt/otaru/cli" 15 | "github.com/nyaxt/otaru/extra/fe/preview" 16 | "github.com/nyaxt/otaru/logger" 17 | "github.com/nyaxt/otaru/pb" 18 | ) 19 | 20 | var mylog = logger.Registry().Category("fe-apiserver") 21 | var accesslog = logger.Registry().Category("http-fe") 22 | 23 | type registerServiceHandlerFunc func(ctx context.Context, mux *gwruntime.ServeMux, conn *grpc.ClientConn) error 24 | 25 | var serviceHandlers = []registerServiceHandlerFunc{ 26 | pb.RegisterBlobstoreServiceHandler, 27 | pb.RegisterFileSystemServiceHandler, 28 | pb.RegisterINodeDBServiceHandler, 29 | pb.RegisterSystemInfoServiceHandler, 30 | } 31 | 32 | func InstallApiGatewayProxy(hostmap map[string]*cli.Host) apiserver.Option { 33 | return apiserver.AddMuxHook(func(ctx context.Context, mux *http.ServeMux) error { 34 | for vhost, h := range hostmap { 35 | ci := cli.ConnectionInfoFromHost(h) 36 | conn, err := ci.DialGrpc(ctx) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | gwmux := gwruntime.NewServeMux() 42 | 43 | for _, sh := range serviceHandlers { 44 | if err := sh(ctx, gwmux, conn); err != nil { 45 | return fmt.Errorf("Failed to register a grpc-gateway handler for host %q: %w", vhost, err) 46 | } 47 | } 48 | 49 | prefix := fmt.Sprintf("/apigw/%s", vhost) 50 | mux.Handle(prefix+"/", http.StripPrefix(prefix, gwmux)) 51 | } 52 | 53 | return nil 54 | }) 55 | } 56 | 57 | func BuildApiServerOptions(cfg *cli.CliConfig) ([]apiserver.Option, error) { 58 | override := cfg.Fe.WebUIRootPath 59 | if override != "" { 60 | zap.S().Infof("Overriding embedded WebUI and serving WebUI at %s", override) 61 | } 62 | 63 | opts := []apiserver.Option{ 64 | apiserver.ListenAddr(cfg.Fe.ListenAddr), 65 | apiserver.TLSCertKey(cfg.Fe.Certs, cfg.Fe.Key), 66 | // apiserver.SetSwaggerJson(json.Assets, "/otaru-fe.swagger.json"), 67 | apiserver.ServeApiGateway(true), 68 | apiserver.SetDefaultHandler(webui.WebUIHandler(override, "/index.otaru-fe.html")), 69 | preview.Install(cfg), 70 | InstallFeService(cfg), 71 | InstallProxyHandler(cfg, cfg.Fe.BasicAuthUser, cfg.Fe.BasicAuthPassword), 72 | InstallApiGatewayProxy(cfg.Host), 73 | } 74 | 75 | return opts, nil 76 | } 77 | -------------------------------------------------------------------------------- /extra/fe/preview/httpsrv.go: -------------------------------------------------------------------------------- 1 | package preview 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "path" 9 | "strconv" 10 | 11 | "github.com/nyaxt/otaru/apiserver" 12 | "github.com/nyaxt/otaru/cli" 13 | "github.com/nyaxt/otaru/logger" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | const MaxArchiveSize = 512 * 1024 * 1024 // 512MiB 18 | const MaxPreviewSize = 8 * 1024 * 1024 // 8MiB 19 | const MaxTextPreviewSize = 32 * 1024 // 32KiB 20 | 21 | var mylog = logger.Registry().Category("fe-preview") 22 | 23 | type server struct { 24 | cfg *cli.CliConfig 25 | zp *zipPreviewer 26 | } 27 | 28 | func (s *server) ServeHTTP(w http.ResponseWriter, req *http.Request) { 29 | if req.Method != "GET" { 30 | http.Error(w, "Invalid method", http.StatusMethodNotAllowed) 31 | return 32 | } 33 | 34 | q := req.URL.Query() 35 | opath := q.Get("opath") 36 | idx, err := strconv.Atoi(q.Get("i")) 37 | if err != nil { 38 | idx = -1 39 | } 40 | 41 | ctx := req.Context() 42 | 43 | ext := path.Ext(opath) 44 | switch ext { 45 | case ".zip": 46 | if err := s.zp.Serve(ctx, opath, idx, w); err != nil { 47 | zap.S().Warnf("zip preview failed: %v", err) 48 | http.Error(w, "zip preview failed", http.StatusInternalServerError) 49 | return 50 | } 51 | 52 | case ".txt", ".json", ".c", ".h", ".md": 53 | if err := s.dumpAsText(ctx, opath, w); err != nil { 54 | zap.S().Warnf("text preview failed: %v", err) 55 | http.Error(w, "text preview failed", http.StatusInternalServerError) 56 | return 57 | } 58 | 59 | default: 60 | http.Error(w, fmt.Sprintf("no preview for %q", ext), http.StatusUnsupportedMediaType) 61 | } 62 | } 63 | 64 | func (s *server) dumpAsText(ctx context.Context, opath string, w http.ResponseWriter) error { 65 | r, err := cli.NewReader(opath, cli.WithCliConfig(s.cfg), cli.WithContext(ctx)) 66 | if err != nil { 67 | return fmt.Errorf("Failed to start read of given opath %q. err: %v", opath, err) 68 | } 69 | defer r.Close() 70 | 71 | if r.Size() > MaxTextPreviewSize { 72 | return fmt.Errorf("File too large for text preview.") 73 | } 74 | 75 | w.Header().Set("Content-Type", "text/plain") 76 | if _, err := io.Copy(w, r); err != nil { 77 | return err 78 | } 79 | return nil 80 | } 81 | 82 | func Install(cfg *cli.CliConfig) apiserver.Option { 83 | s := &server{ 84 | cfg: cfg, 85 | zp: &zipPreviewer{cfg}, 86 | } 87 | 88 | return apiserver.AddMuxHook(func(_ context.Context, mux *http.ServeMux) error { 89 | mux.Handle("/preview", s) 90 | return nil 91 | }) 92 | } 93 | -------------------------------------------------------------------------------- /apiserver/clientauth/interceptor.go: -------------------------------------------------------------------------------- 1 | package clientauth 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "errors" 8 | "strings" 9 | 10 | "google.golang.org/grpc" 11 | "google.golang.org/grpc/codes" 12 | "google.golang.org/grpc/credentials" 13 | "google.golang.org/grpc/peer" 14 | ) 15 | 16 | const ( 17 | AuthorizationKey = "authorization" 18 | BearerPrefix = "Bearer " 19 | ) 20 | 21 | type AuthProvider struct { 22 | Disabled bool 23 | } 24 | 25 | func UserInfoFromClientCert(cert *x509.Certificate) UserInfo { 26 | cn := cert.Subject.CommonName 27 | 28 | a := strings.SplitN(cn, " ", 2) 29 | rolestr := a[0] 30 | user := rolestr 31 | if len(a) > 1 { 32 | user = a[1] 33 | } 34 | 35 | return UserInfo{Role: RoleFromStr(rolestr), User: user} 36 | } 37 | 38 | var ErrZeroVerifiedChains = errors.New("AuthProvider could not find a client cert.") 39 | var ErrZeroVerifiedChains2 = errors.New("AuthProvider requires len(VerifiedChains[0]) > 0.") 40 | 41 | func UserInfoFromTLSConnectionState(tcs *tls.ConnectionState) (UserInfo, error) { 42 | vcs := tcs.VerifiedChains 43 | if len(vcs) == 0 { 44 | return AnonymousUserInfo, ErrZeroVerifiedChains 45 | } 46 | vc := vcs[0] 47 | if len(vc) == 0 { 48 | return AnonymousUserInfo, ErrZeroVerifiedChains2 49 | } 50 | 51 | return UserInfoFromClientCert(vc[0]), nil 52 | } 53 | 54 | func (p AuthProvider) UnaryServerInterceptor() grpc.UnaryServerInterceptor { 55 | if p.Disabled { 56 | return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 57 | ctx = ContextWithUserInfo(ctx, NoauthUserInfo) 58 | return handler(ctx, req) 59 | } 60 | } 61 | 62 | return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 63 | p, ok := peer.FromContext(ctx) 64 | if !ok { 65 | return nil, grpc.Errorf(codes.Unauthenticated, "AuthProvider requires metadata.") 66 | } 67 | if p.AuthInfo == nil { 68 | return nil, grpc.Errorf(codes.Unauthenticated, "AuthProvider requires grpc Peer with AuthInfo.") 69 | } 70 | ti, ok := p.AuthInfo.(credentials.TLSInfo) 71 | if !ok { 72 | return nil, grpc.Errorf(codes.Unauthenticated, "AuthProvider requires grpc Peer with credentails.TLSInfo.") 73 | } 74 | 75 | ui, err := UserInfoFromTLSConnectionState(&ti.State) 76 | if err != nil { 77 | return nil, grpc.Errorf(codes.Unauthenticated, "%v", err) 78 | } 79 | 80 | ctx = ContextWithUserInfo(ctx, ui) 81 | return handler(ctx, req) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /assets/webui/dist/app.js: -------------------------------------------------------------------------------- 1 | import {$} from './domhelper.js'; 2 | import {contentSection, isSectionSelected} from './nav.js'; 3 | import {fillRemoteContent, kHostNoProxy} from './api.js'; 4 | import {formatTimestampRPC} from './format.js'; 5 | import {setHostList} from './browsefs.js'; 6 | import './logview.js'; 7 | import './loglevel.js'; 8 | 9 | const updateInterval = 3000; 10 | 11 | (() => { 12 | const bfs = $('browse-fs'); 13 | setHostList([kHostNoProxy]); 14 | contentSection('browsefs').addEventListener('shown', e => { 15 | bfs.hasFocus = true; 16 | }); 17 | contentSection('browsefs').addEventListener('hidden', e => { 18 | bfs.hasFocus = false; 19 | }); 20 | })(); 21 | 22 | (() => { 23 | const triggerUpdate = async () => { 24 | if (!isSectionSelected('inodedbstats')) 25 | return; 26 | 27 | try { 28 | await fillRemoteContent('api/v1/inodedb/stats', '#inodedbstats-', { 29 | 'last_sync': formatTimestampRPC, 30 | 'last_tx': formatTimestampRPC, 31 | 'last_id': null, 32 | 'version': null, 33 | 'last_ticket': null, 34 | 'number_of_node_locks': null, 35 | }); 36 | } catch (e) { 37 | console.log(e); 38 | } 39 | window.setTimeout(triggerUpdate, updateInterval); 40 | } 41 | contentSection('inodedbstats').addEventListener('shown', e => { 42 | triggerUpdate(); 43 | }); 44 | })(); 45 | 46 | (() => { 47 | const triggerUpdate = async () => { 48 | if (!isSectionSelected('blobstore')) 49 | return; 50 | 51 | try { 52 | await fillRemoteContent('api/v1/blobstore/config', '#blobstore-', [ 53 | 'backend_impl_name', 'backend_flags', 54 | 'cache_impl_name', 'cache_flags']); 55 | } catch (e) { 56 | console.log(e); 57 | } 58 | window.setTimeout(triggerUpdate, updateInterval); 59 | } 60 | contentSection('blobstore').addEventListener('shown', e => { 61 | triggerUpdate(); 62 | }); 63 | })(); 64 | 65 | (() => { 66 | const triggerUpdate = async () => { 67 | if (!isSectionSelected('settings')) 68 | return; 69 | 70 | try { 71 | await fillRemoteContent("api/v1/system/info", "#settings-", [ 72 | 'go_version', 'os', 'arch', 'num_goroutine', 'hostname', 'pid', 'uid', 73 | 'mem_alloc', 'mem_sys', 'num_gc', 'num_fds']); 74 | } catch (e) { 75 | console.log(e); 76 | } 77 | window.setTimeout(triggerUpdate, updateInterval); 78 | } 79 | contentSection('settings').addEventListener('shown', e => { 80 | triggerUpdate(); 81 | }); 82 | })(); 83 | -------------------------------------------------------------------------------- /blobstore/cachedblobstore/reducecachetask.go: -------------------------------------------------------------------------------- 1 | package cachedblobstore 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "time" 7 | 8 | "context" 9 | 10 | "github.com/dustin/go-humanize" 11 | "go.uber.org/zap" 12 | 13 | "github.com/nyaxt/otaru/blobstore" 14 | "github.com/nyaxt/otaru/scheduler" 15 | "github.com/nyaxt/otaru/util" 16 | ) 17 | 18 | type ReduceCacheTask struct { 19 | CBS *CachedBlobStore 20 | DesiredSize int64 21 | DryRun bool 22 | } 23 | 24 | func (t *ReduceCacheTask) Run(ctx context.Context) scheduler.Result { 25 | err := t.CBS.ReduceCache(ctx, t.DesiredSize, t.DryRun) 26 | return scheduler.ErrorResult{err} 27 | } 28 | 29 | const autoReduceCachePeriod = 10 * time.Second 30 | 31 | type AutoReduceCacheTask struct { 32 | CBS *CachedBlobStore 33 | HighWatermark int64 34 | LowWatermark int64 35 | } 36 | 37 | func (t AutoReduceCacheTask) main(ctx context.Context) error { 38 | cachebs := t.CBS.cachebs 39 | tsizer, ok := cachebs.(blobstore.TotalSizer) 40 | if !ok { 41 | return fmt.Errorf("Cache backend \"%s\" doesn't support TotalSize() method, required for AutoReduceCacheTask. aborting.", util.TryGetImplName(cachebs)) 42 | } 43 | currentSize, err := tsizer.TotalSize() 44 | if err != nil { 45 | return fmt.Errorf("Failed to query current total cache size: %v", err) 46 | } 47 | 48 | if currentSize <= t.HighWatermark { 49 | zap.S().Infof("AutoReduceCacheTask: Current size %s < high watermark %s. No-op.", 50 | humanize.Bytes(uint64(currentSize)), 51 | humanize.Bytes(uint64(t.HighWatermark))) 52 | return nil 53 | } 54 | 55 | return t.CBS.ReduceCache(ctx, t.LowWatermark, false) 56 | } 57 | 58 | func (t AutoReduceCacheTask) Run(ctx context.Context) scheduler.Result { 59 | return scheduler.ErrorResult{t.main(ctx)} 60 | } 61 | 62 | func (t AutoReduceCacheTask) String() string { 63 | return fmt.Sprintf("AutoReduceCacheTask{highwm: %s, lowwm: %s}", 64 | humanize.Bytes(uint64(t.HighWatermark)), 65 | humanize.Bytes(uint64(t.LowWatermark))) 66 | } 67 | 68 | func SetupAutoReduceCache(cbs *CachedBlobStore, r *scheduler.RepetitiveJobRunner, highwm, lowwm int64) scheduler.ID { 69 | if highwm == math.MaxInt64 || lowwm == math.MaxInt64 || lowwm > highwm { 70 | zap.S().Infof("No automatic cache discards due to bad watermarks") 71 | return 0 72 | } 73 | 74 | zap.S().Infof("Setting up automatic cache discards. high/low watermark: %s/%s", 75 | humanize.Bytes(uint64(highwm)), humanize.Bytes(uint64(lowwm))) 76 | 77 | return r.RunEveryPeriod(AutoReduceCacheTask{cbs, highwm, lowwm}, autoReduceCachePeriod) 78 | } 79 | -------------------------------------------------------------------------------- /metadata/statesnapshot/statesnapshot.go: -------------------------------------------------------------------------------- 1 | package statesnapshot 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/gob" 7 | "fmt" 8 | "io" 9 | 10 | "go.uber.org/multierr" 11 | "go.uber.org/zap" 12 | 13 | "github.com/nyaxt/otaru/btncrypt" 14 | "github.com/nyaxt/otaru/chunkstore" 15 | "github.com/nyaxt/otaru/logger" 16 | ) 17 | 18 | var mylog = logger.Registry().Category("statess") 19 | 20 | func SaveBytes(w io.Writer, c *btncrypt.Cipher, p []byte) error { 21 | cw, err := chunkstore.NewChunkWriter(w, c, chunkstore.ChunkHeader{ 22 | PayloadLen: uint32(len(p)), 23 | PayloadVersion: 1, 24 | OrigFilename: "statesnapshot", 25 | OrigOffset: 0, 26 | }) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | var me error 32 | if _, err := cw.Write(p); err != nil { 33 | me = multierr.Append(me, fmt.Errorf("Failed to write to ChunkWriter: %w", err)) 34 | } 35 | if err := cw.Close(); err != nil { 36 | me = multierr.Append(me, fmt.Errorf("Failed to close ChunkWriter: %w", err)) 37 | } 38 | return me 39 | } 40 | 41 | type EncodeCallback func(enc *gob.Encoder) error 42 | 43 | func EncodeBytes(cb EncodeCallback) ([]byte, error) { 44 | var buf bytes.Buffer 45 | zw := zlib.NewWriter(&buf) 46 | enc := gob.NewEncoder(zw) 47 | 48 | var me error 49 | if err := cb(enc); err != nil { 50 | me = multierr.Append(me, fmt.Errorf("Failed to encode state: %w", err)) 51 | } 52 | if err := zw.Close(); err != nil { 53 | me = multierr.Append(me, fmt.Errorf("Failed to close zlib Writer: %w", err)) 54 | } 55 | 56 | if me != nil { 57 | return nil, me 58 | } 59 | return buf.Bytes(), nil 60 | } 61 | 62 | func Save(w io.Writer, c *btncrypt.Cipher, cb EncodeCallback) error { 63 | p, err := EncodeBytes(cb) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | return SaveBytes(w, c, p) 69 | } 70 | 71 | type DecodeCallback func(dec *gob.Decoder) error 72 | 73 | func Restore(r io.Reader, c *btncrypt.Cipher, cb DecodeCallback) error { 74 | cr, err := chunkstore.NewChunkReader(r, c) 75 | if err != nil { 76 | return err 77 | } 78 | defer cr.Close() 79 | 80 | zap.S().Debugf("serialized blob size: %d", cr.Length()) 81 | zr, err := zlib.NewReader(&io.LimitedReader{cr, int64(cr.Length())}) 82 | if err != nil { 83 | return err 84 | } 85 | zap.S().Debugf("statesnapshot.Restore: zlib init success!") 86 | dec := gob.NewDecoder(zr) 87 | 88 | var me error 89 | if err := cb(dec); err != nil { 90 | me = multierr.Append(me, fmt.Errorf("Failed to decode state: %w", err)) 91 | } 92 | if err := zr.Close(); err != nil { 93 | me = multierr.Append(me, fmt.Errorf("Failed to close zlib Reader: %w", err)) 94 | } 95 | return err 96 | } 97 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nyaxt/otaru 2 | 3 | require ( 4 | cloud.google.com/go/datastore v1.6.0 5 | cloud.google.com/go/storage v1.22.1 6 | github.com/dustin/go-humanize v1.0.0 7 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 8 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 9 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 10 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 11 | github.com/naoina/toml v0.1.1 12 | github.com/nyaxt/fuse v0.0.0-20171213112031-b89602e08173 13 | github.com/prometheus/client_golang v1.12.1 14 | github.com/rs/cors v1.9.0 15 | github.com/urfave/cli/v2 v2.8.1 16 | go.uber.org/multierr v1.8.0 17 | go.uber.org/zap v1.21.1-0.20220617042904-2e615d88d0eb 18 | golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 19 | golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb 20 | google.golang.org/api v0.84.0 21 | google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad 22 | google.golang.org/grpc v1.47.0 23 | google.golang.org/protobuf v1.28.0 24 | ) 25 | 26 | require ( 27 | cloud.google.com/go v0.102.1 // indirect 28 | cloud.google.com/go/compute v1.7.0 // indirect 29 | cloud.google.com/go/iam v0.3.0 // indirect 30 | github.com/beorn7/perks v1.0.1 // indirect 31 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 32 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 33 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 34 | github.com/golang/protobuf v1.5.2 // indirect 35 | github.com/google/go-cmp v0.5.8 // indirect 36 | github.com/google/uuid v1.3.0 // indirect 37 | github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect 38 | github.com/googleapis/gax-go/v2 v2.4.0 // indirect 39 | github.com/googleapis/go-type-adapters v1.0.0 // indirect 40 | github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect 41 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 42 | github.com/naoina/go-stringutil v0.1.0 // indirect 43 | github.com/prometheus/client_model v0.2.0 // indirect 44 | github.com/prometheus/common v0.33.0 // indirect 45 | github.com/prometheus/procfs v0.7.3 // indirect 46 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 47 | github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect 48 | go.opencensus.io v0.23.0 // indirect 49 | go.uber.org/atomic v1.9.0 // indirect 50 | golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect 51 | golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect 52 | golang.org/x/text v0.3.7 // indirect 53 | golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect 54 | google.golang.org/appengine v1.6.7 // indirect 55 | ) 56 | 57 | go 1.20 58 | -------------------------------------------------------------------------------- /assets/swaggerui/dist/oauth2-redirect.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 68 | -------------------------------------------------------------------------------- /flags/flags.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | import ( 4 | "bytes" 5 | "syscall" 6 | ) 7 | 8 | const ( 9 | O_RDONLY int = syscall.O_RDONLY 10 | O_WRONLY int = syscall.O_WRONLY 11 | O_RDWR int = syscall.O_RDWR 12 | O_CREATE int = syscall.O_CREAT 13 | O_EXCL int = syscall.O_EXCL 14 | O_TRUNCATE int = syscall.O_TRUNC 15 | O_APPEND int = syscall.O_APPEND 16 | O_RDWRCREATE int = O_RDWR | O_CREATE 17 | O_VALIDMASK int = O_RDONLY | O_WRONLY | O_RDWR | O_CREATE | O_EXCL | O_TRUNCATE | O_APPEND 18 | ) 19 | 20 | type FlagsReader interface { 21 | Flags() int 22 | } 23 | 24 | func IsReadAllowed(flags int) bool { 25 | mode := flags & syscall.O_ACCMODE 26 | return mode == O_RDONLY || mode == O_RDWR 27 | } 28 | 29 | func IsWriteAllowed(flags int) bool { 30 | mode := flags & syscall.O_ACCMODE 31 | return mode == O_WRONLY || mode == O_RDWR 32 | } 33 | 34 | func IsReadWriteAllowed(flags int) bool { 35 | mode := flags & syscall.O_ACCMODE 36 | return mode == O_RDWR 37 | } 38 | 39 | func IsCreateAllowed(flags int) bool { 40 | return flags&O_CREATE != 0 41 | } 42 | 43 | func IsCreateExclusive(flags int) bool { 44 | return IsCreateAllowed(flags) && flags&O_EXCL != 0 45 | } 46 | 47 | func IsWriteTruncate(flags int) bool { 48 | return IsWriteAllowed(flags) && flags&O_TRUNCATE != 0 49 | } 50 | 51 | func IsWriteAppend(flags int) bool { 52 | return IsWriteAllowed(flags) && flags&O_APPEND != 0 53 | } 54 | 55 | func FlagsToString(flags int) string { 56 | var b bytes.Buffer 57 | if IsReadAllowed(flags) { 58 | b.WriteString("R") 59 | } 60 | if IsWriteAllowed(flags) { 61 | b.WriteString("W") 62 | } 63 | if IsCreateAllowed(flags) { 64 | b.WriteString("C") 65 | } 66 | if IsCreateExclusive(flags) { 67 | b.WriteString("X") 68 | } 69 | if IsWriteTruncate(flags) { 70 | b.WriteString("T") 71 | } 72 | if IsWriteAppend(flags) { 73 | b.WriteString("A") 74 | } 75 | 76 | return b.String() 77 | } 78 | 79 | func Mask(a, b int) int { 80 | rok := IsReadAllowed(a) && IsReadAllowed(b) 81 | wok := IsWriteAllowed(a) && IsWriteAllowed(b) 82 | cok := IsCreateAllowed(a) && IsCreateAllowed(b) 83 | 84 | ret := 0 85 | if rok && wok { 86 | ret = O_RDWR 87 | } else if rok { 88 | ret = O_RDONLY 89 | } else if wok { 90 | ret = O_WRONLY 91 | } 92 | 93 | if cok { 94 | ret |= O_CREATE 95 | } 96 | 97 | return ret 98 | } 99 | 100 | func MaskPermMode(pm uint16, flags int) uint16 { 101 | if !IsReadAllowed(flags) { 102 | pm &= ^uint16(0444) 103 | } 104 | if !IsWriteAllowed(flags) { 105 | pm &= ^uint16(0222) 106 | } 107 | /* 108 | if !IsCreateAllowed(flags) { 109 | pm = pm & ^uint16(0111) 110 | } 111 | */ 112 | 113 | return pm 114 | } 115 | -------------------------------------------------------------------------------- /assets/webui/dist/loglevel.js: -------------------------------------------------------------------------------- 1 | import {contentSection, isSectionSelected} from './nav.js'; 2 | import {$, $$, removeAllChildNodes} from './domhelper.js'; 3 | import {rpc} from './api.js'; 4 | 5 | const listDiv = $('.loglevel--list'); 6 | const levels = ['debug', 'info', 'warning', 'critical']; 7 | Object.freeze(levels); 8 | 9 | const triggerUpdate = async () => { 10 | if (!isSectionSelected('loglevel')) 11 | return; 12 | 13 | try { 14 | removeAllChildNodes(listDiv); 15 | 16 | const result = await rpc('api/v1/logger/categories'); 17 | const categories = result['category'].sort( 18 | (a, b) => a['category'].localeCompare(b['category'])); 19 | for (let category of categories) { 20 | const name = category['category']; 21 | let currLevel = category['level'] || 0; 22 | const inputName = `loglevel-${name}`; 23 | const onchange = async (ev) => { 24 | const selectedInput = $(`.loglevel__radio[name='${inputName}']:checked`); 25 | const selectedValue = parseInt(selectedInput.value); 26 | if (currLevel != selectedValue) { 27 | await rpc(`api/v1/logger/category/${name}`, {method: 'post', body: selectedValue}); 28 | currLevel = selectedValue; 29 | } 30 | }; 31 | 32 | const itemDiv = document.createElement('div'); 33 | itemDiv.classList.add('kvview__item'); 34 | listDiv.appendChild(itemDiv); 35 | 36 | const labelDiv = document.createElement('div'); 37 | labelDiv.classList.add('kvview__label'); 38 | labelDiv.textContent = name; 39 | itemDiv.appendChild(labelDiv); 40 | 41 | const valueDiv = document.createElement('div'); 42 | valueDiv.classList.add('kvview__value'); 43 | valueDiv.classList.add('loglevel__level'); 44 | valueDiv.id = `loglevel-${name}`; 45 | itemDiv.appendChild(valueDiv); 46 | 47 | for (let i = 0; i < levels.length; ++ i) { 48 | const inputId = `loglevel-${name}-${i}`; 49 | 50 | const input = document.createElement('input'); 51 | input.type = 'radio'; 52 | input.classList.add('loglevel__radio'); 53 | input.name = inputName; 54 | input.id = inputId; 55 | input.value = i; 56 | input.checked = (currLevel == i); 57 | input.addEventListener('change', onchange); 58 | valueDiv.appendChild(input); 59 | 60 | const label = document.createElement('label'); 61 | label.classList.add('loglevel__label'); 62 | label.setAttribute('for', inputId); 63 | label.textContent = levels[i]; 64 | valueDiv.appendChild(label); 65 | } 66 | } 67 | } catch (e) { 68 | console.log(e); 69 | } 70 | } 71 | contentSection('loglevel').addEventListener('shown', e => { 72 | triggerUpdate(); 73 | }); 74 | 75 | export { levels }; 76 | -------------------------------------------------------------------------------- /cmd/otaru/globallock/command.go: -------------------------------------------------------------------------------- 1 | package globallock 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/urfave/cli/v2" 7 | 8 | "github.com/nyaxt/otaru/btncrypt" 9 | "github.com/nyaxt/otaru/facade" 10 | "github.com/nyaxt/otaru/gcloud/auth" 11 | "github.com/nyaxt/otaru/gcloud/datastore" 12 | ) 13 | 14 | type Action int 15 | 16 | const ( 17 | QueryAction Action = iota 18 | LockAction 19 | UnlockAction 20 | ) 21 | 22 | var Command = &cli.Command{ 23 | Name: "globallock", 24 | Usage: "Inspect or modify otaru global lock", 25 | ArgsUsage: "{lock,unlock,query}", 26 | Flags: []cli.Flag{ 27 | &cli.BoolFlag{ 28 | Name: "force", 29 | Usage: "Force Unlock currently active global lock when specified with unlock cmd", 30 | }, 31 | &cli.StringFlag{ 32 | Name: "info", 33 | Usage: "Custom info string", 34 | }, 35 | }, 36 | Action: func(c *cli.Context) error { 37 | cfg, err := facade.NewConfig(c.Path("configDir")) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | a := QueryAction 43 | if c.Args().Present() { 44 | actionstr := c.Args().First() 45 | switch actionstr { 46 | case "query": 47 | a = QueryAction 48 | case "lock": 49 | a = LockAction 50 | case "unlock": 51 | a = UnlockAction 52 | default: 53 | return fmt.Errorf("Unknown action %q", actionstr) 54 | } 55 | } 56 | 57 | tsrc, err := auth.GetGCloudTokenSource(cfg.CredentialsFilePath) 58 | if err != nil { 59 | return fmt.Errorf("Failed to init GCloudClientSource: %v", err) 60 | } 61 | 62 | nullCipher := &btncrypt.Cipher{} // Null cipher is fine, as GlobalLocker doesn't make use of it. 63 | dscfg := datastore.NewConfig(cfg.ProjectName, cfg.BucketName, nullCipher, tsrc) 64 | info := c.String("info") 65 | if info == "" { 66 | info = "otaru-globallock-cli cmdline debug tool" 67 | } 68 | l := datastore.NewGlobalLocker(dscfg, "otaru-globallock-cli", info) 69 | 70 | ctx := c.Context 71 | 72 | switch a { 73 | case LockAction: 74 | readOnly := false 75 | if err := l.Lock(ctx, readOnly); err != nil { 76 | return fmt.Errorf("Lock failed: %w", err) 77 | } 78 | case UnlockAction: 79 | if c.Bool("force") { 80 | if err := l.ForceUnlock(ctx); err != nil { 81 | return fmt.Errorf("ForceUnlock failed: %w", err) 82 | } 83 | } else { 84 | if err := l.UnlockIgnoreCreatedAt(ctx); err != nil { 85 | return fmt.Errorf("Unlock failed: %w", err) 86 | } 87 | } 88 | case QueryAction: 89 | entry, err := l.Query(ctx) 90 | if err != nil { 91 | return fmt.Errorf("Query failed: %w", err) 92 | } 93 | fmt.Printf("%+v\n", entry) 94 | 95 | default: 96 | panic("should not be reached") 97 | } 98 | return nil 99 | }, 100 | } 101 | -------------------------------------------------------------------------------- /fuse/filesystem.go: -------------------------------------------------------------------------------- 1 | package fuse 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "time" 8 | 9 | bfuse "github.com/nyaxt/fuse" 10 | bfs "github.com/nyaxt/fuse/fs" 11 | "go.uber.org/zap" 12 | 13 | "github.com/nyaxt/otaru/filesystem" 14 | "github.com/nyaxt/otaru/inodedb" 15 | "github.com/nyaxt/otaru/logger" 16 | ) 17 | 18 | var mylog = logger.Registry().Category("fuse") 19 | var bfuseLogger = logger.Registry().Category("bfuse") 20 | 21 | func init() { 22 | bfuse.Debug = func(msg interface{}) { zap.S().Debugf("%v", msg) } 23 | } 24 | 25 | type FileSystem struct { 26 | ofs *filesystem.FileSystem 27 | } 28 | 29 | func (fs FileSystem) Root() (bfs.Node, error) { 30 | return DirNode{fs: fs.ofs, id: inodedb.RootDirID}, nil 31 | } 32 | 33 | func (fs FileSystem) Statfs(ctx context.Context, req *bfuse.StatfsRequest, resp *bfuse.StatfsResponse) error { 34 | // fill dummy 35 | resp.Blocks = 0 36 | if tsize, err := fs.ofs.TotalSize(); err != nil { 37 | resp.Blocks = uint64(tsize) 38 | } 39 | resp.Bfree = math.MaxUint64 40 | resp.Bavail = math.MaxUint64 41 | resp.Files = 0 42 | resp.Ffree = 0 43 | resp.Bsize = 32 * 1024 44 | resp.Namelen = 32 * 1024 45 | resp.Frsize = 1 46 | 47 | return nil 48 | } 49 | 50 | func Serve(ctx context.Context, bucketName string, mountpoint string, ofs *filesystem.FileSystem, ready chan<- bool) error { 51 | fsName := fmt.Sprintf("otaru+gs://%s", bucketName) 52 | volName := fmt.Sprintf("Otaru %s", bucketName) 53 | 54 | c, err := bfuse.Mount( 55 | mountpoint, 56 | bfuse.FSName(fsName), 57 | bfuse.Subtype("otarufs"), 58 | bfuse.VolumeName(volName), 59 | bfuse.MaxReadahead(math.MaxUint32), 60 | ) 61 | if err != nil { 62 | return fmt.Errorf("bfuse.Mount failed: %v", err) 63 | } 64 | defer c.Close() 65 | 66 | serveC := make(chan error) 67 | go func() { 68 | if err := bfs.Serve(c, FileSystem{ofs}); err != nil { 69 | serveC <- err 70 | } 71 | close(serveC) 72 | }() 73 | 74 | // check if the mount process has an error to report 75 | <-c.Ready 76 | if err := c.MountError; err != nil { 77 | return err 78 | } 79 | 80 | zap.S().Infof("Mountpoint \"%s\" should be ready now!", mountpoint) 81 | if ready != nil { 82 | close(ready) 83 | } 84 | 85 | go func() { 86 | <-ctx.Done() 87 | Unmount(mountpoint) 88 | }() 89 | 90 | return <-serveC 91 | } 92 | 93 | func Unmount(mountpoint string) { 94 | doneC := make(chan struct{}) 95 | go func() { 96 | bfuse.Unmount(mountpoint) 97 | close(doneC) 98 | }() 99 | timeoutC := time.After(time.Second * 3) 100 | select { 101 | case <-doneC: 102 | zap.S().Infof("Successfully unmounted: %v", mountpoint) 103 | case <-timeoutC: 104 | zap.S().Warnf("Timeout reached while trying to unmount: %v", mountpoint) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /gc/blobstoregc/gc.go: -------------------------------------------------------------------------------- 1 | package blobstoregc 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "context" 8 | 9 | "github.com/nyaxt/otaru/blobstore" 10 | "github.com/nyaxt/otaru/inodedb" 11 | "github.com/nyaxt/otaru/logger" 12 | "github.com/nyaxt/otaru/metadata" 13 | "github.com/nyaxt/otaru/util" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | var mylog = logger.Registry().Category("blobstoregc") 18 | 19 | type GCableBlobStore interface { 20 | blobstore.BlobLister 21 | blobstore.BlobRemover 22 | } 23 | 24 | func GC(ctx context.Context, bs GCableBlobStore, idb inodedb.DBFscker, dryrun bool) error { 25 | start := time.Now() 26 | 27 | zap.S().Infof("GC start. Dryrun: %t. Listing blobs.", dryrun) 28 | allbs, err := bs.ListBlobs() 29 | if err != nil { 30 | return fmt.Errorf("ListBlobs failed: %v", err) 31 | } 32 | zap.S().Infof("List blobs done. %d blobs found.", len(allbs)) 33 | if err := ctx.Err(); err != nil { 34 | zap.S().Infof("Detected cancel. Bailing out.") 35 | return err 36 | } 37 | zap.S().Infof("Starting INodeDB fsck.") 38 | usedbs, errs := idb.Fsck() 39 | if len(errs) != 0 { 40 | return fmt.Errorf("Fsck returned err: %v", err) 41 | } 42 | zap.S().Infof("Fsck done. %d used blobs found.", len(usedbs)) 43 | if err := ctx.Err(); err != nil { 44 | zap.S().Infof("Detected cancel. Bailing out.") 45 | return err 46 | } 47 | 48 | zap.S().Infof("Converting used blob list to a hashset") 49 | usedbset := make(map[string]struct{}) 50 | for _, b := range usedbs { 51 | usedbset[b] = struct{}{} 52 | } 53 | zap.S().Infof("Convert used blob list to a hashset: Done.") 54 | 55 | if err := ctx.Err(); err != nil { 56 | zap.S().Infof("Detected cancel. Bailing out.") 57 | return err 58 | } 59 | 60 | zap.S().Infof("Listing unused blobpaths.") 61 | unusedbs := make([]string, 0, util.IntMax(len(allbs)-len(usedbs), 0)) 62 | for _, b := range allbs { 63 | if _, ok := usedbset[b]; ok { 64 | continue 65 | } 66 | 67 | if metadata.IsMetadataBlobpath(b) { 68 | zap.S().Infof("Marking metadata blobpath as used: %s", b) 69 | continue 70 | } 71 | 72 | unusedbs = append(unusedbs, b) 73 | } 74 | 75 | traceend := time.Now() 76 | zap.S().Infof("GC Found %d unused blobpaths. (Trace took %v)", len(unusedbs), traceend.Sub(start)) 77 | 78 | for _, b := range unusedbs { 79 | if err := ctx.Err(); err != nil { 80 | zap.S().Infof("Detected cancel. Bailing out.") 81 | return err 82 | } 83 | 84 | if dryrun { 85 | zap.S().Infof("Dryrun found unused blob: %s", b) 86 | } else { 87 | zap.S().Infof("Removing unused blob: %s", b) 88 | if err := bs.RemoveBlob(b); err != nil { 89 | return fmt.Errorf("Removing unused blob \"%s\" failed: %v", b, err) 90 | } 91 | } 92 | } 93 | sweepend := time.Now() 94 | zap.S().Infof("GC success. Dryrun: %t. (Sweep took %v. The whole GC took %v.)", dryrun, sweepend.Sub(traceend), sweepend.Sub(start)) 95 | 96 | return err 97 | } 98 | -------------------------------------------------------------------------------- /debugcmd/otaru-txlogio/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | 9 | "github.com/nyaxt/otaru/btncrypt" 10 | "github.com/nyaxt/otaru/facade" 11 | "github.com/nyaxt/otaru/flags" 12 | "github.com/nyaxt/otaru/gcloud/auth" 13 | "github.com/nyaxt/otaru/gcloud/datastore" 14 | "github.com/nyaxt/otaru/inodedb" 15 | "github.com/nyaxt/otaru/logger" 16 | "go.uber.org/zap" 17 | ) 18 | 19 | var mylog = logger.Registry().Category("otaru-txlogio") 20 | 21 | var ( 22 | flagConfigDir = flag.String("configDir", facade.DefaultConfigDir(), "Config dirpath") 23 | ) 24 | 25 | func Usage() { 26 | fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) 27 | fmt.Fprintf(os.Stderr, " %s [options] purge\n", os.Args[0]) 28 | fmt.Fprintf(os.Stderr, " %s [options] query [minID]\n", os.Args[0]) 29 | flag.PrintDefaults() 30 | } 31 | 32 | func main() { 33 | panic("migrate to urfave/cli") 34 | 35 | flag.Usage = Usage 36 | flag.Parse() 37 | 38 | cfg, err := facade.NewConfig(*flagConfigDir) 39 | if err != nil { 40 | zap.S().Infof("%v", err) 41 | Usage() 42 | os.Exit(2) 43 | } 44 | minID := inodedb.LatestVersion 45 | if flag.NArg() < 1 { 46 | Usage() 47 | os.Exit(2) 48 | } 49 | switch flag.Arg(0) { 50 | case "purge": 51 | if flag.NArg() != 1 { 52 | Usage() 53 | os.Exit(2) 54 | } 55 | case "query": 56 | switch flag.NArg() { 57 | case 1: 58 | break 59 | case 2: 60 | n, err := strconv.ParseInt(flag.Arg(1), 10, 64) 61 | if err != nil { 62 | Usage() 63 | os.Exit(2) 64 | } 65 | minID = inodedb.TxID(n) 66 | break 67 | } 68 | break 69 | default: 70 | zap.S().Infof("Unknown cmd: %v", flag.Arg(0)) 71 | Usage() 72 | os.Exit(2) 73 | } 74 | 75 | tsrc, err := auth.GetGCloudTokenSource(cfg.CredentialsFilePath) 76 | if err != nil { 77 | zap.S().Errorf("Failed to init GCloudClientSource: %v", err) 78 | } 79 | 80 | key := btncrypt.KeyFromPassword(cfg.Password) 81 | c, err := btncrypt.NewCipher(key) 82 | if err != nil { 83 | zap.S().Errorf("Failed to init *btncrypt.Cipher: %v", err) 84 | } 85 | dscfg := datastore.NewConfig(cfg.ProjectName, cfg.BucketName, c, tsrc) 86 | 87 | txlogio := datastore.NewDBTransactionLogIO(dscfg, flags.O_RDWRCREATE) 88 | 89 | switch flag.Arg(0) { 90 | case "purge": 91 | if err := txlogio.DeleteAllTransactions(); err != nil { 92 | zap.S().Infof("DeleteAllTransactions() failed: %v", err) 93 | } 94 | 95 | case "query": 96 | zap.S().Infof("Start QueryTransactions(%v)", minID) 97 | txs, err := txlogio.QueryTransactions(minID) 98 | if err != nil { 99 | zap.S().Infof("QueryTransactions() failed: %v", err) 100 | } 101 | for _, tx := range txs { 102 | fmt.Printf("%s\n", tx) 103 | } 104 | 105 | default: 106 | zap.S().Infof("Unknown cmd: %v", flag.Arg(0)) 107 | os.Exit(1) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /otaruapiserver/systemservice.go: -------------------------------------------------------------------------------- 1 | package otaruapiserver 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | 7 | "context" 8 | 9 | "google.golang.org/grpc" 10 | 11 | "github.com/nyaxt/otaru/apiserver" 12 | "github.com/nyaxt/otaru/apiserver/clientauth" 13 | "github.com/nyaxt/otaru/pb" 14 | "github.com/nyaxt/otaru/util/countfds" 15 | "github.com/nyaxt/otaru/version" 16 | ) 17 | 18 | type systemService struct { 19 | pb.UnimplementedSystemInfoServiceServer 20 | } 21 | 22 | func (*systemService) GetSystemInfo(ctx context.Context, in *pb.GetSystemInfoRequest) (*pb.SystemInfoResponse, error) { 23 | if err := clientauth.RequireRoleGRPC(ctx, clientauth.RoleAdmin); err != nil { 24 | return nil, err 25 | } 26 | 27 | var m runtime.MemStats 28 | runtime.ReadMemStats(&m) 29 | 30 | hostname, err := os.Hostname() 31 | if err != nil { 32 | hostname = "" 33 | } 34 | 35 | return &pb.SystemInfoResponse{ 36 | GoVersion: runtime.Version(), 37 | Os: runtime.GOOS, 38 | Arch: runtime.GOARCH, 39 | 40 | NumGoroutine: uint32(runtime.NumGoroutine()), 41 | 42 | Hostname: hostname, 43 | Pid: uint64(os.Getpid()), 44 | Uid: uint64(os.Getuid()), 45 | 46 | MemAlloc: m.Alloc, 47 | MemSys: m.Sys, 48 | 49 | NumGc: m.NumGC, 50 | 51 | NumFds: uint32(countfds.CountFds()), 52 | }, nil 53 | } 54 | 55 | func (*systemService) GetVersion(ctx context.Context, in *pb.GetVersionRequest) (*pb.VersionResponse, error) { 56 | return &pb.VersionResponse{ 57 | GitCommit: version.GIT_COMMIT, 58 | BuildHost: version.BUILD_HOST, 59 | }, nil 60 | } 61 | 62 | func (*systemService) Whoami(ctx context.Context, in *pb.WhoamiRequest) (*pb.WhoamiResponse, error) { 63 | ui := clientauth.UserInfoFromContext(ctx) 64 | 65 | return &pb.WhoamiResponse{ 66 | Role: ui.Role.String(), 67 | User: ui.User, 68 | }, nil 69 | } 70 | 71 | func (*systemService) AuthTestAnonymous(ctx context.Context, in *pb.AuthTestRequest) (*pb.AuthTestResponse, error) { 72 | if err := clientauth.RequireRoleGRPC(ctx, clientauth.RoleAnonymous); err != nil { 73 | return nil, err 74 | } 75 | 76 | return &pb.AuthTestResponse{}, nil 77 | } 78 | 79 | func (*systemService) AuthTestReadOnly(ctx context.Context, in *pb.AuthTestRequest) (*pb.AuthTestResponse, error) { 80 | if err := clientauth.RequireRoleGRPC(ctx, clientauth.RoleReadOnly); err != nil { 81 | return nil, err 82 | } 83 | 84 | return &pb.AuthTestResponse{}, nil 85 | } 86 | 87 | func (*systemService) AuthTestAdmin(ctx context.Context, in *pb.AuthTestRequest) (*pb.AuthTestResponse, error) { 88 | if err := clientauth.RequireRoleGRPC(ctx, clientauth.RoleAdmin); err != nil { 89 | return nil, err 90 | } 91 | 92 | return &pb.AuthTestResponse{}, nil 93 | } 94 | 95 | func InstallSystemService() apiserver.Option { 96 | return apiserver.RegisterService( 97 | func(s *grpc.Server) { pb.RegisterSystemInfoServiceServer(s, &systemService{}) }, 98 | pb.RegisterSystemInfoServiceHandlerFromEndpoint, 99 | ) 100 | } 101 | -------------------------------------------------------------------------------- /debugcmd/otaru-inodedbsslocator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | "context" 10 | 11 | "github.com/nyaxt/otaru/btncrypt" 12 | "github.com/nyaxt/otaru/facade" 13 | "github.com/nyaxt/otaru/flags" 14 | "github.com/nyaxt/otaru/gcloud/auth" 15 | "github.com/nyaxt/otaru/gcloud/datastore" 16 | "github.com/nyaxt/otaru/inodedb" 17 | "github.com/nyaxt/otaru/logger" 18 | "go.uber.org/zap" 19 | ) 20 | 21 | var mylog = logger.Registry().Category("otaru-globallock") 22 | 23 | var ( 24 | flagConfigDir = flag.String("configDir", facade.DefaultConfigDir(), "Config dirpath") 25 | flagDryRun = flag.Bool("dryRun", false, "Don't actually make a change if set true") 26 | ) 27 | 28 | func Usage() { 29 | fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) 30 | fmt.Fprintf(os.Stderr, " %s [options] {list,purge}\n", os.Args[0]) 31 | flag.PrintDefaults() 32 | } 33 | 34 | func main() { 35 | panic("migrate to urfave/cli") 36 | 37 | flag.Usage = Usage 38 | flag.Parse() 39 | if flag.NArg() != 1 { 40 | Usage() 41 | os.Exit(1) 42 | } 43 | switch flag.Arg(0) { 44 | case "list", "purge": 45 | break 46 | default: 47 | zap.S().Infof("Unknown cmd: %v", flag.Arg(0)) 48 | Usage() 49 | os.Exit(2) 50 | } 51 | 52 | cfg, err := facade.NewConfig(*flagConfigDir) 53 | if err != nil { 54 | zap.S().Infof("%v", err) 55 | Usage() 56 | os.Exit(1) 57 | } 58 | 59 | tsrc, err := auth.GetGCloudTokenSource(cfg.CredentialsFilePath) 60 | if err != nil { 61 | zap.S().Errorf("Failed to init GCloudTokenSource: %v", err) 62 | } 63 | key := btncrypt.KeyFromPassword(cfg.Password) 64 | c, err := btncrypt.NewCipher(key) 65 | if err != nil { 66 | zap.S().Errorf("Failed to init *btncrypt.Cipher: %v", err) 67 | } 68 | 69 | dscfg := datastore.NewConfig(cfg.ProjectName, cfg.BucketName, c, tsrc) 70 | ssloc := datastore.NewINodeDBSSLocator(dscfg, flags.O_RDWRCREATE) 71 | switch flag.Arg(0) { 72 | case "purge": 73 | fmt.Printf("Do you really want to proceed with deleting all inodedbsslocator entry for %s?\n", cfg.BucketName) 74 | fmt.Printf("Type \"deleteall\" to proceed: ") 75 | sc := bufio.NewScanner(os.Stdin) 76 | if !sc.Scan() { 77 | return 78 | } 79 | if sc.Text() != "deleteall" { 80 | zap.S().Infof("Cancelled.\n") 81 | os.Exit(1) 82 | } 83 | 84 | es, err := ssloc.DeleteAll(context.Background(), *flagDryRun) 85 | if err != nil { 86 | zap.S().Infof("DeleteAll failed: %v", err) 87 | } 88 | zap.S().Infof("DeleteAll deleted entries for blobpath: %v", es) 89 | // FIXME: delete the entries from blobpath too 90 | 91 | case "list": 92 | history := 0 93 | histloop: 94 | for { 95 | bp, txid, err := ssloc.Locate(history) 96 | if err != nil { 97 | if err == datastore.EEMPTY { 98 | zap.S().Infof("Locate(%d): no entry", history) 99 | } else { 100 | zap.S().Infof("Locate(%d) err: %v", history, err) 101 | } 102 | break histloop 103 | } 104 | zap.S().Infof("Locate(%d) txid %v blobpath %v", history, inodedb.TxID(txid), bp) 105 | 106 | history++ 107 | } 108 | default: 109 | panic("NOT REACHED") 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /btncrypt/btncrypt_test.go: -------------------------------------------------------------------------------- 1 | package btncrypt_test 2 | 3 | import ( 4 | "github.com/nyaxt/otaru/btncrypt" 5 | tu "github.com/nyaxt/otaru/testutils" 6 | "github.com/nyaxt/otaru/util" 7 | 8 | "bytes" 9 | "io" 10 | "testing" 11 | ) 12 | 13 | func init() { tu.EnsureLogger() } 14 | 15 | func TestEncrypt_Short(t *testing.T) { 16 | payload := []byte("short string") 17 | envelope, err := btncrypt.Encrypt(tu.TestCipher(), payload) 18 | if err != nil { 19 | t.Errorf("Failed to encrypt: %v", err) 20 | } 21 | 22 | plain, err := btncrypt.Decrypt(tu.TestCipher(), envelope, len(payload)) 23 | if err != nil { 24 | t.Errorf("Failed to decrypt: %v", err) 25 | } 26 | 27 | if !bytes.Equal(plain, payload) { 28 | t.Errorf("Failed to restore original payload") 29 | } 30 | } 31 | 32 | func TestEncrypt_Long(t *testing.T) { 33 | payload := util.RandomBytes(1024 * 1024) 34 | 35 | envelope, err := btncrypt.Encrypt(tu.TestCipher(), payload) 36 | if err != nil { 37 | t.Errorf("Failed to encrypt: %v", err) 38 | } 39 | 40 | plain, err := btncrypt.Decrypt(tu.TestCipher(), envelope, len(payload)) 41 | if err != nil { 42 | t.Errorf("Failed to decrypt: %v", err) 43 | } 44 | 45 | if !bytes.Equal(payload, plain) { 46 | t.Errorf("Failed to restore original payload") 47 | } 48 | } 49 | 50 | func verifyWrite(t *testing.T, w io.Writer, payload []byte) { 51 | n, err := w.Write(payload) 52 | if err != nil { 53 | t.Errorf("Failed to write data to BtnEncryptWriter: %v", err) 54 | } 55 | if n != len(payload) { 56 | t.Errorf("bew.Write returned n != len(p)") 57 | } 58 | } 59 | 60 | func TestBtnEncryptWriter_WriteAtOnce(t *testing.T) { 61 | //payload := RandomBytes(1024 * 1024) 62 | payload := []byte("short string") 63 | 64 | var b bytes.Buffer 65 | bew, err := tu.TestCipher().NewWriteCloser(&b, len(payload)) 66 | if err != nil { 67 | t.Errorf("Failed to create BtnEncryptWriter: %v", err) 68 | } 69 | 70 | verifyWrite(t, bew, payload) 71 | if err := bew.Close(); err != nil { 72 | t.Errorf("bew.Close failed: %v", err) 73 | } 74 | 75 | plain, err := btncrypt.Decrypt(tu.TestCipher(), b.Bytes(), len(payload)) 76 | if err != nil { 77 | t.Errorf("Failed to decrypt: %v", err) 78 | } 79 | 80 | if !bytes.Equal(payload, plain) { 81 | t.Errorf("Failed to restore original payload") 82 | } 83 | } 84 | 85 | func TestBtnEncryptWriter_PartialWrite(t *testing.T) { 86 | payload := util.RandomBytes(1024 * 1024) 87 | 88 | var b bytes.Buffer 89 | bew, err := tu.TestCipher().NewWriteCloser(&b, len(payload)) 90 | if err != nil { 91 | t.Errorf("Failed to create BtnEncryptWriter: %v", err) 92 | } 93 | 94 | verifyWrite(t, bew, payload[:3]) 95 | verifyWrite(t, bew, payload[3:1024]) 96 | verifyWrite(t, bew, payload[1024:4096]) 97 | verifyWrite(t, bew, payload[4096:]) 98 | 99 | if err := bew.Close(); err != nil { 100 | t.Errorf("bew.Close failed: %v", err) 101 | } 102 | 103 | plain, err := btncrypt.Decrypt(tu.TestCipher(), b.Bytes(), len(payload)) 104 | if err != nil { 105 | t.Errorf("Failed to decrypt: %v", err) 106 | } 107 | 108 | if !bytes.Equal(payload, plain) { 109 | t.Errorf("Failed to restore original payload") 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /extra/fe/preview/zip.go: -------------------------------------------------------------------------------- 1 | package preview 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "mime" 12 | "net/http" 13 | "path" 14 | "sort" 15 | "strings" 16 | 17 | "github.com/nyaxt/otaru/cli" 18 | ) 19 | 20 | var allowedExt map[string]struct{} 21 | 22 | func init() { 23 | allowedExtList := []string{ 24 | ".jpg", ".jpeg", ".png", 25 | } 26 | 27 | allowedExt = make(map[string]struct{}) 28 | for _, e := range allowedExtList { 29 | allowedExt[e] = struct{}{} 30 | } 31 | } 32 | 33 | type zipPreviewer struct { 34 | cfg *cli.CliConfig 35 | } 36 | 37 | func (p *zipPreviewer) Serve(ctx context.Context, opath string, idx int, w http.ResponseWriter) error { 38 | r, err := cli.NewReader(opath, cli.WithCliConfig(p.cfg), cli.WithContext(ctx)) 39 | if err != nil { 40 | return fmt.Errorf("Failed to start read of given opath %q. err: %v", opath, err) 41 | } 42 | 43 | size := r.Size() 44 | if size > MaxArchiveSize { 45 | return fmt.Errorf("Refusing to read large archive %.02f MB", float64(size)/1024/1024) 46 | } 47 | 48 | ra, ok := r.(io.ReaderAt) 49 | if ok { 50 | defer r.Close() 51 | } else { 52 | // FIXME: cache! 53 | bs, err := ioutil.ReadAll(r) 54 | if err != nil { 55 | return fmt.Errorf("ioutil.ReadAll err: %v", err) 56 | } 57 | r.Close() 58 | ra = bytes.NewReader(bs) 59 | } 60 | 61 | z, err := zip.NewReader(ra, size) 62 | if err != nil { 63 | return fmt.Errorf("Failed to open zip reader: %v", err) 64 | } 65 | sort.Slice(z.File, func(i, j int) bool { 66 | n, m := z.File[i].Name, z.File[j].Name 67 | return strings.Compare(n, m) < 0 68 | }) 69 | 70 | if idx < 0 { 71 | return listEntries(w, z) 72 | } 73 | return extract(w, z, idx) 74 | } 75 | 76 | func listEntries(w http.ResponseWriter, z *zip.Reader) error { 77 | type jentry struct { 78 | Name string `json:"name"` 79 | Size float64 `json:"size"` 80 | } 81 | 82 | jes := make([]jentry, 0, len(z.File)) 83 | for _, f := range z.File { 84 | jes = append(jes, jentry{Name: f.Name, Size: float64(f.UncompressedSize64)}) 85 | } 86 | 87 | w.Header().Set("X-Otaru-Preview", "archive-listing") 88 | w.Header().Set("Content-Type", "application/json") 89 | enc := json.NewEncoder(w) 90 | if err := enc.Encode(jes); err != nil { 91 | return err 92 | } 93 | return nil 94 | } 95 | 96 | func extract(w http.ResponseWriter, z *zip.Reader, idx int) error { 97 | if idx >= len(z.File) { 98 | return fmt.Errorf("out of bounds") 99 | } 100 | 101 | f := z.File[idx] 102 | 103 | if f.UncompressedSize64 > MaxPreviewSize { 104 | return fmt.Errorf("File too large for preview.") 105 | } 106 | 107 | ext := path.Ext(z.File[idx].Name) 108 | ext = strings.ToLower(ext) 109 | if _, ok := allowedExt[ext]; !ok { 110 | return fmt.Errorf("Refusing to serve ext %q", ext) 111 | } 112 | mtype := mime.TypeByExtension(ext) 113 | w.Header().Set("Content-Type", mtype) 114 | 115 | r, err := f.Open() 116 | if err != nil { 117 | return fmt.Errorf("Failed to open entry: %v", err) 118 | } 119 | defer r.Close() 120 | if _, err := io.Copy(w, r); err != nil { 121 | return err 122 | } 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /assets/webui/dist/app-fe.js: -------------------------------------------------------------------------------- 1 | import {contentSection, updateContentIfNeeded} from './nav.js'; 2 | import {fillRemoteContent} from './api.js'; 3 | import {$} from './domhelper.js'; 4 | import './browsefs.js'; 5 | import {preview} from './preview.js'; 6 | import {infobar} from './infobar.js'; 7 | 8 | const leftfs = $("#leftfs"); 9 | const rightfs = $("#rightfs"); 10 | 11 | leftfs.path = '//[local]/'; 12 | 13 | leftfs.counterpart = rightfs; 14 | rightfs.counterpart = leftfs; 15 | 16 | contentSection('browsefs').addEventListener('shown', e => { 17 | if (e.browsefsPath !== undefined) 18 | rightfs.path = e.browsefsPath; 19 | rightfs.triggerUpdate(); 20 | }); 21 | contentSection('browsefs').addEventListener('hidden', () => { 22 | return rightfs.clear(); 23 | }); 24 | rightfs.addEventListener('pathChanged', e => { 25 | updateContentIfNeeded({currBrowsefsPath: e.newPath}); 26 | }); 27 | 28 | const splitbar = $('.splitbar'); 29 | const noophandler = () => { return false; }; 30 | splitbar.addEventListener('mousedown', md => { 31 | const pn = splitbar.parentNode; 32 | const offX = pn.offsetLeft; 33 | const offW = pn.offsetWidth; 34 | const leftpane = pn.querySelector('.split--leftpane'); 35 | const rightpane = pn.querySelector('.split--rightpane'); 36 | 37 | const mmhandler = mm => { 38 | const l = (event.pageX - offX) / offW; 39 | leftpane.style.width = `${l * 100}%`; 40 | rightpane.style.width =`${(1.0 - l) * 100}%`; 41 | }; 42 | const muhandler = mu => { 43 | pn.removeEventListener('mousemove', mmhandler); 44 | pn.removeEventListener('mouseup', muhandler); 45 | pn.removeEventListener('selectstart', noophandler); 46 | pn.classList.remove('drag_parent'); 47 | }; 48 | pn.addEventListener('mousemove', mmhandler); 49 | pn.addEventListener('mouseup', muhandler); 50 | pn.addEventListener('selectstart', noophandler); 51 | pn.classList.add('drag_parent'); 52 | }); 53 | 54 | window.addEventListener('DOMContentLoaded', () => { 55 | leftfs.cursorIndex = 0; 56 | leftfs.hasFocus = true; 57 | 58 | document.addEventListener('keydown', e => { 59 | console.log(`keydown ${e.key}`); 60 | }); 61 | document.addEventListener('keypress', e => { 62 | if (e.target instanceof HTMLInputElement) 63 | return true; 64 | 65 | console.log(`doc keypress ${e.key}`); 66 | if (preview.isOpen) { 67 | if (e.key === 'j') { 68 | ++ preview.cursorIndex; 69 | } else if (e.key === 'k') { 70 | preview.cursorIndex = Math.max(preview.cursorIndex - 1, 0); 71 | } else if (e.key === 'Enter') { 72 | preview.toggleView(); 73 | } 74 | return false; 75 | } 76 | 77 | if (e.key === 'l') { 78 | rightfs.cursorIndex = leftfs.cursorIndex; 79 | leftfs.clearCursor(); 80 | leftfs.hasFocus = false; 81 | rightfs.hasFocus = true; 82 | } else if (e.key === 'h') { 83 | leftfs.cursorIndex = rightfs.cursorIndex; 84 | rightfs.clearCursor(); 85 | rightfs.hasFocus = false; 86 | leftfs.hasFocus = true; 87 | } else { 88 | console.log(`keypress ${e.key}`); 89 | } 90 | }); 91 | document.addEventListener('keyup', e => { 92 | if (e.key === 'Escape') { 93 | if (preview.isOpen) 94 | preview.close(); 95 | } else { 96 | console.log(`keyup ${e.key}`); 97 | } 98 | }); 99 | }); 100 | -------------------------------------------------------------------------------- /filesystem/fsutil.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | fl "github.com/nyaxt/otaru/flags" 10 | "github.com/nyaxt/otaru/inodedb" 11 | "github.com/nyaxt/otaru/util" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | func (fs *FileSystem) FindNodeFullPath(fullpath string) (inodedb.ID, error) { 16 | fullpath = filepath.Clean(fullpath) 17 | if len(fullpath) < 1 || fullpath[0] != '/' { 18 | return 0, fmt.Errorf("Path must start with /, but given: %v", fullpath) 19 | } 20 | 21 | if fullpath == "/" { 22 | return inodedb.ID(1), nil 23 | } 24 | 25 | parentPath := filepath.Dir(fullpath) 26 | parentId, err := fs.FindNodeFullPath(parentPath) 27 | if err != nil { 28 | return 0, err 29 | } 30 | 31 | entries, err := fs.DirEntries(parentId) 32 | base := filepath.Base(fullpath) 33 | id, ok := entries[base] 34 | if !ok { 35 | return 0, util.ENOENT 36 | } 37 | 38 | return id, nil 39 | } 40 | 41 | func (fs *FileSystem) CreateFileFullPath(fullpath string, perm uint16, uid, gid uint32, modifiedT time.Time) (inodedb.ID, error) { 42 | perm &= 0777 43 | 44 | if len(fullpath) < 1 || fullpath[0] != '/' { 45 | return 0, fmt.Errorf("Path must start with /, but given: %v", fullpath) 46 | } 47 | 48 | dirname := filepath.Dir(fullpath) 49 | basename := filepath.Base(fullpath) 50 | 51 | dirID, err := fs.FindNodeFullPath(dirname) 52 | if err != nil { 53 | return 0, err 54 | } 55 | 56 | return fs.CreateFile(dirID, basename, uint16(perm), uid, gid, modifiedT) 57 | } 58 | 59 | func (fs *FileSystem) OpenFileFullPath(fullpath string, flags int, perm uint16) (*FileHandle, error) { 60 | perm &= 0777 61 | 62 | if len(fullpath) < 1 || fullpath[0] != '/' { 63 | return nil, fmt.Errorf("Path must start with /, but given: %v", fullpath) 64 | } 65 | 66 | dirname := filepath.Dir(fullpath) 67 | basename := filepath.Base(fullpath) 68 | 69 | dirID, err := fs.FindNodeFullPath(dirname) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | entries, err := fs.DirEntries(dirID) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | var id inodedb.ID 80 | id, ok := entries[basename] 81 | if !ok { 82 | if flags|os.O_CREATE != 0 { 83 | id, err = fs.CreateFile(dirID, basename, uint16(perm), 0, 0, time.Now()) 84 | if err != nil { 85 | return nil, err 86 | } 87 | } else { 88 | return nil, util.ENOENT 89 | } 90 | } 91 | 92 | if id == 0 { 93 | zap.S().Panicf("inode id must != 0 here, but got %v", id) 94 | } 95 | 96 | fh, err := fs.OpenFile(id, flags) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | return fh, nil 102 | } 103 | 104 | func (fs *FileSystem) WriteFile(fullpath string, content []byte, perm uint16) error { 105 | h, err := fs.OpenFileFullPath(fullpath, fl.O_RDWRCREATE, perm) 106 | if err != nil { 107 | return err 108 | } 109 | defer h.Close() 110 | 111 | return h.PWrite(content, 0) 112 | } 113 | 114 | func (fs *FileSystem) CreateDirFullPath(fullpath string, perm uint16, uid, gid uint32, modifiedT time.Time) (inodedb.ID, error) { 115 | parent := filepath.Dir(fullpath) 116 | id, err := fs.FindNodeFullPath(parent) 117 | if err != nil { 118 | return 0, fmt.Errorf("Failed to find parent \"%s\": %v", parent, err) 119 | } 120 | 121 | name := filepath.Base(fullpath) 122 | return fs.CreateDir(id, name, perm, uid, gid, modifiedT) 123 | } 124 | -------------------------------------------------------------------------------- /blobstore/mux.go: -------------------------------------------------------------------------------- 1 | package blobstore 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | 8 | fl "github.com/nyaxt/otaru/flags" 9 | "github.com/nyaxt/otaru/util" 10 | ) 11 | 12 | var ( 13 | ErrEmptyMux = errors.New("blobstore.Mux is empty.") 14 | ) 15 | 16 | type BlobMatcher func(blobpath string) bool 17 | 18 | type MuxEntry struct { 19 | BlobMatcher 20 | BlobStore 21 | } 22 | 23 | type Mux []MuxEntry 24 | 25 | var _ = BlobStore(Mux{}) 26 | 27 | func (m Mux) findBlobStoreFor(blobpath string) BlobStore { 28 | for i, e := range m { 29 | lastEntry := i == len(m)-1 30 | if lastEntry || e.BlobMatcher(blobpath) { 31 | return e.BlobStore 32 | } 33 | } 34 | return nil 35 | } 36 | 37 | func (m Mux) OpenWriter(blobpath string) (io.WriteCloser, error) { 38 | bs := m.findBlobStoreFor(blobpath) 39 | if bs == nil { 40 | return nil, ErrEmptyMux 41 | } 42 | return bs.OpenWriter(blobpath) 43 | } 44 | 45 | func (m Mux) OpenReader(blobpath string) (io.ReadCloser, error) { 46 | bs := m.findBlobStoreFor(blobpath) 47 | if bs == nil { 48 | return nil, ErrEmptyMux 49 | } 50 | return bs.OpenReader(blobpath) 51 | } 52 | 53 | var _ = RandomAccessBlobStore(Mux{}) 54 | 55 | func (m Mux) Open(blobpath string, flags int) (BlobHandle, error) { 56 | bs := m.findBlobStoreFor(blobpath) 57 | if bs == nil { 58 | return nil, ErrEmptyMux 59 | } 60 | rabs, ok := bs.(RandomAccessBlobStore) 61 | if !ok { 62 | return nil, fmt.Errorf("Backend blobstore \"%s\" don't support Open()", util.TryGetImplName(bs)) 63 | } 64 | return rabs.Open(blobpath, flags) 65 | } 66 | 67 | var _ = fl.FlagsReader(Mux{}) 68 | 69 | func (m Mux) Flags() int { 70 | flags := fl.O_RDWRCREATE 71 | 72 | for _, e := range m { 73 | if flagsreader, ok := e.BlobStore.(fl.FlagsReader); ok { 74 | flags = fl.Mask(flags, flagsreader.Flags()) 75 | } 76 | } 77 | 78 | return flags 79 | } 80 | 81 | var _ = BlobLister(Mux{}) 82 | 83 | func (m Mux) ListBlobs() ([]string, error) { 84 | ret := make([]string, 0) 85 | for _, e := range m { 86 | blobLister, ok := e.BlobStore.(BlobLister) 87 | if !ok { 88 | return nil, fmt.Errorf("Backend blobstore \"%s\" don't support ListBlobs()", util.TryGetImplName(e.BlobStore)) 89 | } 90 | entries, err := blobLister.ListBlobs() 91 | if err != nil { 92 | return nil, fmt.Errorf("Backend blobstore \"%s\" failed to ListBlobs: %v", util.TryGetImplName(e.BlobStore), err) 93 | } 94 | ret = append(ret, entries...) 95 | } 96 | return ret, nil 97 | } 98 | 99 | var _ = BlobSizer(Mux{}) 100 | 101 | func (m Mux) BlobSize(blobpath string) (int64, error) { 102 | bs := m.findBlobStoreFor(blobpath) 103 | if bs == nil { 104 | return -1, ErrEmptyMux 105 | } 106 | sizer, ok := bs.(BlobSizer) 107 | if !ok { 108 | return -1, fmt.Errorf("Backend blobstore \"%s\" don't support BlobSize()", util.TryGetImplName(bs)) 109 | } 110 | return sizer.BlobSize(blobpath) 111 | } 112 | 113 | var _ = BlobRemover(Mux{}) 114 | 115 | func (m Mux) RemoveBlob(blobpath string) error { 116 | bs := m.findBlobStoreFor(blobpath) 117 | if bs == nil { 118 | return ErrEmptyMux 119 | } 120 | remover, ok := bs.(BlobRemover) 121 | if !ok { 122 | return fmt.Errorf("Backend blobstore \"%s\" don't support RemoveBlob()", util.TryGetImplName(bs)) 123 | } 124 | return remover.RemoveBlob(blobpath) 125 | } 126 | 127 | var _ = util.ImplNamed(Mux{}) 128 | 129 | func (Mux) ImplName() string { return "blobstore.Mux" } 130 | --------------------------------------------------------------------------------