├── docs ├── comparison-to-similar-software │ └── index.md ├── vision.png ├── logo-readme.png ├── screenshots │ ├── clients.png │ ├── service-view.png │ ├── topology-view.png │ ├── health-monitoring.png │ ├── volumes-and-mounts.png │ └── varasto-qnap-tr-004.jpg ├── content │ ├── movies │ │ ├── adoption.png │ │ ├── endresult.png │ │ ├── directoryid.png │ │ ├── directorytype.png │ │ ├── pull-metadata.png │ │ └── tmdb-apikey.png │ ├── games │ │ ├── screenshot.png │ │ └── index.md │ ├── photos │ │ ├── screenshot.png │ │ └── index.md │ ├── tvshows │ │ ├── endresult.png │ │ ├── enter-imdb-id.png │ │ └── create-season-directory.png │ ├── magazines-comics │ │ ├── screenshot.png │ │ └── index.md │ └── generic-files │ │ ├── screenshot-web-ui.png │ │ ├── varasto-with-git-repos.png │ │ ├── index.md │ │ └── varasto-with-git-repos.drawio ├── security │ ├── encryption │ │ ├── dial.png │ │ ├── ciphertext.bin │ │ ├── plaintext.jpg │ │ ├── key-encryption-keys.png │ │ ├── encrypted-integrity-verification.png │ │ └── encrypted-integrity-verification.drawio │ ├── ransomware-protection │ │ ├── 1pc.png │ │ ├── 1pc-infected.png │ │ ├── 2pc-infected.png │ │ └── separate-security-boundaries.png │ └── privacy │ │ └── index.md ├── data-interfaces │ ├── fuse │ │ ├── shell.png │ │ ├── architecture-1pc.png │ │ └── architecture-2pc.png │ ├── network-folders │ │ ├── screenshot.png │ │ ├── architecture-1pc.png │ │ ├── architecture-2pc.png │ │ └── architecture.xml │ ├── web-ui │ │ └── index.md │ └── index.md ├── developers │ └── codebase │ │ ├── drawing.png │ │ ├── tag-add.png │ │ ├── tageditor.png │ │ ├── tag-add-modal.png │ │ ├── index.md │ │ └── drawing.drawio ├── using │ ├── smart-monitoring │ │ ├── smart.png │ │ └── serverhealth.png │ ├── metadata-backup │ │ └── backuplist.png │ ├── observability │ │ ├── grafana-metrics.png │ │ ├── prometheus-grafana.png │ │ ├── prometheus-grafana.drawio │ │ └── index.md │ ├── replication-policies │ │ ├── screenshot.png │ │ ├── replication-policies.png │ │ ├── replication-queue-status.png │ │ ├── index.md │ │ └── replication-policies.drawio │ └── background-integrity-verification │ │ └── screenshot.png ├── storage │ ├── googledrive │ │ ├── folder-id.png │ │ └── drive-consent-screen.png │ ├── local-fs │ │ ├── architecture.png │ │ └── architecture.drawio │ └── naming-your-volumes.md ├── concepts-ideas-architecture │ ├── architecture.png │ ├── cas-deduplication.png │ ├── cas-write-first-file.png │ ├── cas-write-second-file.png │ ├── cas-integrity-violation.png │ ├── one-computer-client-and-server.png │ ├── cas.drawio │ └── architecture.xml ├── assets │ ├── mkdocs-additional-styles.css │ └── Download-install-green.svg └── install │ ├── windows.md │ ├── index.md │ ├── mac.md │ └── linux-manual.md ├── misc ├── varasto-logo │ ├── color.txt │ ├── Turbologo.url │ ├── raster │ │ ├── logo.xcf │ │ ├── preview.png │ │ └── logo_height512.png │ ├── font │ │ └── baron_neue.zip │ └── varasto-github.png ├── seed-data │ ├── varasto.db │ ├── blob-volumes.tar │ ├── varastoclient-config.json │ ├── import.sh │ └── README.md ├── varasto-updateserver │ └── manifest.json └── docs-website-deployerspec │ └── manifest.json ├── public ├── robots.txt ├── favicon.ico ├── filetypes │ ├── pdf.png │ ├── audio.png │ ├── excel.png │ ├── video.png │ ├── word.png │ ├── archive.png │ ├── generic.png │ ├── picture.png │ ├── powerpoint.png │ ├── spreadsheet.png │ └── Filetypes icons by Igor Verizub.url └── embed.go ├── .dockerignore ├── pkg ├── frontend │ ├── doc.go │ └── ui-routes.json ├── stomediascanner │ ├── types.json │ ├── importsideeffects.go │ ├── thumbnailer_test.go │ └── stateandchangefeed.go ├── stofuse │ ├── stofusetypes │ │ └── types.json │ ├── os_linux.go │ ├── stofuseentrypoint │ │ ├── entrypoint_windows.go │ │ └── entrypoint_notwin.go │ ├── os_darwin.go │ ├── collectionbyidquery_test.go │ ├── stofuseclient │ │ └── client.go │ ├── utils_test.go │ ├── types.go │ ├── restapi.go │ └── dirbyidquery.go ├── stotypes │ ├── errors.go │ ├── getters.go │ ├── apptypes_test.go │ └── apptypes.go ├── fssnapshot │ ├── factorylinux.go │ ├── factorywindows.go │ ├── utils.go │ ├── interface.go │ ├── null.go │ ├── lvm_test.go │ └── windows_test.go ├── stoserver │ ├── stoservertypes │ │ └── customizations.go │ ├── stodiskaccess │ │ ├── writecounter_test.go │ │ ├── writecounter.go │ │ └── support.go │ ├── stodbimportexport │ │ └── dbexportimport_test.go │ ├── stohealth │ │ ├── staticnode.go │ │ ├── healthinterface_test.go │ │ ├── ivchecker.go │ │ └── healthinterface.go │ ├── stodb │ │ ├── configkeys.go │ │ ├── schemaversion.go │ │ ├── scheduledjobseeddata.go │ │ └── configaccessor.go │ ├── commandhandlersmetadata_test.go │ ├── server_test.go │ ├── restapichangefeed.go │ ├── commandhandlerskek.go │ └── commandhandlersscheduledjobs.go ├── docreference │ ├── refs.go │ └── refs_test.go ├── gokitbp │ └── backports.go ├── stoutils │ ├── tcpordomainsocketlistener_test.go │ ├── tcpordomainsocketlistener.go │ ├── utils_test.go │ └── utils.go ├── tui │ ├── progressbar_test.go │ └── progressbar.go ├── blobstore │ ├── localfsblobstore │ │ └── localfs_test.go │ ├── interface.go │ ├── googledriveblobstore │ │ └── googledrive_test.go │ └── s3blobstore │ │ └── s3_test.go ├── stomvu │ ├── custompattern_test.go │ ├── entrypoint.go │ ├── photorenamer_test.go │ ├── tv.go │ ├── custompattern.go │ ├── photorenamer.go │ └── filemodificationtime.go ├── stoclient │ ├── utils.go │ ├── rm.go │ ├── bulkuploadscript.go │ ├── uploadprogessui_test.go │ ├── localstate.go │ └── adopt.go ├── byteshuman │ ├── humanize_test.go │ └── humanize.go ├── duration │ ├── humanize_test.go │ └── humanize.go ├── mutexmap │ ├── mutexmap_test.go │ └── mutexmap.go ├── stodebug │ └── entrypoint.go ├── logtee │ ├── stringtail.go │ ├── logtee_test.go │ └── linesplittertee.go ├── seasonepisodedetector │ ├── detector_test.go │ └── detector.go ├── smart │ ├── jsonformat.go │ └── smart.go ├── blorm │ └── interface.go ├── stodupremover │ ├── entrypoint.go │ ├── removeemptydirs.go │ ├── scanner.go │ └── actioners.go ├── restartcontroller │ └── restartcontroller.go ├── stateresolver │ ├── dirpeek.go │ └── dirpeek_test.go ├── igdbapi │ └── externalidextractor.go └── sslca │ └── sslca.go ├── frontend ├── modules.d.ts ├── component │ ├── numberformatter.ts │ ├── refreshbutton.tsx │ ├── doclink.tsx │ ├── assetimg.tsx │ ├── tabcontroller.tsx │ ├── collectiondropdown.tsx │ ├── tags.tsx │ └── filetypes.ts ├── tslint.json ├── package.json ├── webpack.config.js ├── pages │ ├── RootRedirectPage.tsx │ ├── LogsPage.tsx │ └── MetricsPage.tsx └── layout │ ├── HelpLayout.tsx │ ├── appdefaultlayout.tsx │ └── AdminLayout.tsx ├── bin ├── build-frontend.sh ├── build.sh └── codegenerate.go ├── .github └── workflows │ └── build.yml ├── Dockerfile ├── cmd └── sto │ └── main.go └── turbobob.json /docs/comparison-to-similar-software/index.md: -------------------------------------------------------------------------------- 1 | TODO 2 | -------------------------------------------------------------------------------- /misc/varasto-logo/color.txt: -------------------------------------------------------------------------------- 1 | #337ab7 2 | (from Bootstrap) 3 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # empty robots.txt so we won't get lots of 404s 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # ignore everyhing ... 2 | ** 3 | 4 | # ... except 5 | !/rel/** 6 | -------------------------------------------------------------------------------- /docs/vision.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/vision.png -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /docs/logo-readme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/logo-readme.png -------------------------------------------------------------------------------- /misc/varasto-logo/Turbologo.url: -------------------------------------------------------------------------------- 1 | [InternetShortcut] 2 | URL=https://turbologo.com/logos/9067876 3 | -------------------------------------------------------------------------------- /pkg/frontend/doc.go: -------------------------------------------------------------------------------- 1 | // Definitions (like UI routes) for Varasto's frontend 2 | package frontend 3 | -------------------------------------------------------------------------------- /public/filetypes/pdf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/pdf.png -------------------------------------------------------------------------------- /misc/seed-data/varasto.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/misc/seed-data/varasto.db -------------------------------------------------------------------------------- /public/filetypes/audio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/audio.png -------------------------------------------------------------------------------- /public/filetypes/excel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/excel.png -------------------------------------------------------------------------------- /public/filetypes/video.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/video.png -------------------------------------------------------------------------------- /public/filetypes/word.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/word.png -------------------------------------------------------------------------------- /docs/screenshots/clients.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/screenshots/clients.png -------------------------------------------------------------------------------- /public/filetypes/archive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/archive.png -------------------------------------------------------------------------------- /public/filetypes/generic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/generic.png -------------------------------------------------------------------------------- /public/filetypes/picture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/picture.png -------------------------------------------------------------------------------- /docs/content/movies/adoption.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/movies/adoption.png -------------------------------------------------------------------------------- /misc/seed-data/blob-volumes.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/misc/seed-data/blob-volumes.tar -------------------------------------------------------------------------------- /public/embed.go: -------------------------------------------------------------------------------- 1 | package public 2 | 3 | import ( 4 | "embed" 5 | ) 6 | 7 | //go:embed * 8 | var Content embed.FS 9 | -------------------------------------------------------------------------------- /public/filetypes/powerpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/powerpoint.png -------------------------------------------------------------------------------- /public/filetypes/spreadsheet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/public/filetypes/spreadsheet.png -------------------------------------------------------------------------------- /docs/content/games/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/games/screenshot.png -------------------------------------------------------------------------------- /docs/content/movies/endresult.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/movies/endresult.png -------------------------------------------------------------------------------- /docs/content/photos/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/photos/screenshot.png -------------------------------------------------------------------------------- /docs/content/tvshows/endresult.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/tvshows/endresult.png -------------------------------------------------------------------------------- /docs/screenshots/service-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/screenshots/service-view.png -------------------------------------------------------------------------------- /docs/screenshots/topology-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/screenshots/topology-view.png -------------------------------------------------------------------------------- /docs/security/encryption/dial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/encryption/dial.png -------------------------------------------------------------------------------- /misc/varasto-logo/raster/logo.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/misc/varasto-logo/raster/logo.xcf -------------------------------------------------------------------------------- /docs/content/movies/directoryid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/movies/directoryid.png -------------------------------------------------------------------------------- /docs/content/movies/directorytype.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/movies/directorytype.png -------------------------------------------------------------------------------- /docs/content/movies/pull-metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/movies/pull-metadata.png -------------------------------------------------------------------------------- /docs/content/movies/tmdb-apikey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/movies/tmdb-apikey.png -------------------------------------------------------------------------------- /docs/data-interfaces/fuse/shell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/data-interfaces/fuse/shell.png -------------------------------------------------------------------------------- /docs/developers/codebase/drawing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/developers/codebase/drawing.png -------------------------------------------------------------------------------- /docs/developers/codebase/tag-add.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/developers/codebase/tag-add.png -------------------------------------------------------------------------------- /docs/using/smart-monitoring/smart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/smart-monitoring/smart.png -------------------------------------------------------------------------------- /misc/varasto-logo/font/baron_neue.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/misc/varasto-logo/font/baron_neue.zip -------------------------------------------------------------------------------- /misc/varasto-logo/raster/preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/misc/varasto-logo/raster/preview.png -------------------------------------------------------------------------------- /misc/varasto-logo/varasto-github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/misc/varasto-logo/varasto-github.png -------------------------------------------------------------------------------- /public/filetypes/Filetypes icons by Igor Verizub.url: -------------------------------------------------------------------------------- 1 | [InternetShortcut] 2 | URL=https://www.iconfinder.com/iconsets/file-8 3 | -------------------------------------------------------------------------------- /docs/content/tvshows/enter-imdb-id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/tvshows/enter-imdb-id.png -------------------------------------------------------------------------------- /docs/developers/codebase/tageditor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/developers/codebase/tageditor.png -------------------------------------------------------------------------------- /docs/screenshots/health-monitoring.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/screenshots/health-monitoring.png -------------------------------------------------------------------------------- /docs/screenshots/volumes-and-mounts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/screenshots/volumes-and-mounts.png -------------------------------------------------------------------------------- /docs/security/encryption/ciphertext.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/encryption/ciphertext.bin -------------------------------------------------------------------------------- /docs/security/encryption/plaintext.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/encryption/plaintext.jpg -------------------------------------------------------------------------------- /docs/storage/googledrive/folder-id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/storage/googledrive/folder-id.png -------------------------------------------------------------------------------- /docs/storage/local-fs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/storage/local-fs/architecture.png -------------------------------------------------------------------------------- /docs/developers/codebase/tag-add-modal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/developers/codebase/tag-add-modal.png -------------------------------------------------------------------------------- /docs/screenshots/varasto-qnap-tr-004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/screenshots/varasto-qnap-tr-004.jpg -------------------------------------------------------------------------------- /docs/using/metadata-backup/backuplist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/metadata-backup/backuplist.png -------------------------------------------------------------------------------- /docs/content/magazines-comics/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/magazines-comics/screenshot.png -------------------------------------------------------------------------------- /docs/security/ransomware-protection/1pc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/ransomware-protection/1pc.png -------------------------------------------------------------------------------- /docs/using/observability/grafana-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/observability/grafana-metrics.png -------------------------------------------------------------------------------- /docs/using/smart-monitoring/serverhealth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/smart-monitoring/serverhealth.png -------------------------------------------------------------------------------- /misc/varasto-logo/raster/logo_height512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/misc/varasto-logo/raster/logo_height512.png -------------------------------------------------------------------------------- /docs/data-interfaces/fuse/architecture-1pc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/data-interfaces/fuse/architecture-1pc.png -------------------------------------------------------------------------------- /docs/data-interfaces/fuse/architecture-2pc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/data-interfaces/fuse/architecture-2pc.png -------------------------------------------------------------------------------- /docs/using/observability/prometheus-grafana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/observability/prometheus-grafana.png -------------------------------------------------------------------------------- /docs/using/replication-policies/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/replication-policies/screenshot.png -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/concepts-ideas-architecture/architecture.png -------------------------------------------------------------------------------- /docs/content/generic-files/screenshot-web-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/generic-files/screenshot-web-ui.png -------------------------------------------------------------------------------- /docs/content/tvshows/create-season-directory.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/tvshows/create-season-directory.png -------------------------------------------------------------------------------- /docs/security/encryption/key-encryption-keys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/encryption/key-encryption-keys.png -------------------------------------------------------------------------------- /docs/storage/googledrive/drive-consent-screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/storage/googledrive/drive-consent-screen.png -------------------------------------------------------------------------------- /docs/assets/mkdocs-additional-styles.css: -------------------------------------------------------------------------------- 1 | 2 | /* Fix bad color in table heading links */ 3 | .md-typeset thead a { 4 | color: hsl(231, 43%, 72%); 5 | } 6 | -------------------------------------------------------------------------------- /docs/data-interfaces/network-folders/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/data-interfaces/network-folders/screenshot.png -------------------------------------------------------------------------------- /docs/security/ransomware-protection/1pc-infected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/ransomware-protection/1pc-infected.png -------------------------------------------------------------------------------- /docs/security/ransomware-protection/2pc-infected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/ransomware-protection/2pc-infected.png -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/cas-deduplication.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/concepts-ideas-architecture/cas-deduplication.png -------------------------------------------------------------------------------- /docs/content/generic-files/varasto-with-git-repos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/content/generic-files/varasto-with-git-repos.png -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/cas-write-first-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/concepts-ideas-architecture/cas-write-first-file.png -------------------------------------------------------------------------------- /docs/data-interfaces/network-folders/architecture-1pc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/data-interfaces/network-folders/architecture-1pc.png -------------------------------------------------------------------------------- /docs/data-interfaces/network-folders/architecture-2pc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/data-interfaces/network-folders/architecture-2pc.png -------------------------------------------------------------------------------- /docs/using/replication-policies/replication-policies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/replication-policies/replication-policies.png -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/cas-write-second-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/concepts-ideas-architecture/cas-write-second-file.png -------------------------------------------------------------------------------- /docs/using/background-integrity-verification/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/background-integrity-verification/screenshot.png -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/cas-integrity-violation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/concepts-ideas-architecture/cas-integrity-violation.png -------------------------------------------------------------------------------- /docs/security/encryption/encrypted-integrity-verification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/encryption/encrypted-integrity-verification.png -------------------------------------------------------------------------------- /docs/using/replication-policies/replication-queue-status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/using/replication-policies/replication-queue-status.png -------------------------------------------------------------------------------- /frontend/modules.d.ts: -------------------------------------------------------------------------------- 1 | // https://stackoverflow.com/a/12695001 2 | 3 | // badly behaving external modules with no TypeScript defs 4 | declare module 'react-autocomplete'; 5 | -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/one-computer-client-and-server.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/concepts-ideas-architecture/one-computer-client-and-server.png -------------------------------------------------------------------------------- /docs/security/ransomware-protection/separate-security-boundaries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/function61/varasto/HEAD/docs/security/ransomware-protection/separate-security-boundaries.png -------------------------------------------------------------------------------- /pkg/stomediascanner/types.json: -------------------------------------------------------------------------------- 1 | { 2 | "endpoints": [ 3 | { "chain": "public", "method": "GET", "path": "/api/mediascanner/collections/{id}?mode={mode}", "name": "triggerScan" } 4 | ] 5 | } -------------------------------------------------------------------------------- /pkg/stofuse/stofusetypes/types.json: -------------------------------------------------------------------------------- 1 | { 2 | "endpoints": [ 3 | { "chain": "public", "method": "POST", "path": "/api/fuse/unmount_all", "name": "fuseUnmountAll" } 4 | ], 5 | "types": [] 6 | } 7 | -------------------------------------------------------------------------------- /pkg/stotypes/errors.go: -------------------------------------------------------------------------------- 1 | package stotypes 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | ErrBlobNotAccessibleOnThisNode = errors.New("blob not accessible on this node") 9 | ) 10 | -------------------------------------------------------------------------------- /frontend/component/numberformatter.ts: -------------------------------------------------------------------------------- 1 | // thanks https://stackoverflow.com/a/2901298 2 | export function thousandSeparate(x: number): string { 3 | return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ' '); 4 | } 5 | -------------------------------------------------------------------------------- /frontend/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | // contains per-project overrides on top of the buildkit's own strict config 3 | "extends": [ 4 | "/etc/tslint.json" 5 | ], 6 | "linterOptions": { 7 | "exclude": [] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /pkg/fssnapshot/factorylinux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package fssnapshot 4 | 5 | import ( 6 | "log" 7 | ) 8 | 9 | func PlatformSpecificSnapshotter(logger *log.Logger) Snapshotter { 10 | return LvmSnapshotter("1GB", logger) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/fssnapshot/factorywindows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package fssnapshot 4 | 5 | import ( 6 | "log" 7 | ) 8 | 9 | func PlatformSpecificSnapshotter(logger *log.Logger) Snapshotter { 10 | return WindowsSnapshotter(logger) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/stoserver/stoservertypes/customizations.go: -------------------------------------------------------------------------------- 1 | package stoservertypes 2 | 3 | // customizations on top of generated code 4 | 5 | // needed since we can't take the address of const 6 | func (h HealthKind) Ptr() *HealthKind { 7 | return &h 8 | } 9 | -------------------------------------------------------------------------------- /misc/seed-data/varastoclient-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "server_addr": "https://localhost", 3 | "auth_token": "fdspdptPaMjyi-6l-fLgQwlin6rClEYSJ6_LFzWQsRE", 4 | "fuse_mount_path": "/mnt/stofuse/varasto", 5 | "tls_insecure_skip_validation": true 6 | } 7 | -------------------------------------------------------------------------------- /pkg/stotypes/getters.go: -------------------------------------------------------------------------------- 1 | package stotypes 2 | 3 | func FindDekEnvelope(keyId string, kenvs []KeyEnvelope) *KeyEnvelope { 4 | for _, kenv := range kenvs { 5 | if kenv.KeyId == keyId { 6 | return &kenv 7 | } 8 | } 9 | 10 | return nil 11 | } 12 | -------------------------------------------------------------------------------- /pkg/stofuse/os_linux.go: -------------------------------------------------------------------------------- 1 | package stofuse 2 | 3 | // os-specific abstractions (Linux) 4 | 5 | import ( 6 | "syscall" 7 | "time" 8 | ) 9 | 10 | func accessTimeFromStatt(stat *syscall.Stat_t, _ time.Time) time.Time { 11 | return timespecToTime(stat.Atim) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/docreference/refs.go: -------------------------------------------------------------------------------- 1 | package docreference 2 | 3 | import ( 4 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 5 | ) 6 | 7 | func GitHubMaster(ref stoservertypes.DocRef) string { 8 | return "https://github.com/function61/varasto/blob/master/" + string(ref) 9 | } 10 | -------------------------------------------------------------------------------- /pkg/stofuse/stofuseentrypoint/entrypoint_windows.go: -------------------------------------------------------------------------------- 1 | package stofuseentrypoint 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | func Entrypoint() *cobra.Command { 8 | return &cobra.Command{ 9 | Use: "fuse", 10 | Short: "Varasto-FUSE does not work in Windows", 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /pkg/stomediascanner/importsideeffects.go: -------------------------------------------------------------------------------- 1 | package stomediascanner 2 | 3 | // below side effects have to be imported to transparently support their decoding 4 | 5 | import ( 6 | _ "image/gif" 7 | _ "image/jpeg" 8 | _ "image/png" 9 | 10 | _ "golang.org/x/image/bmp" 11 | _ "golang.org/x/image/webp" 12 | ) 13 | -------------------------------------------------------------------------------- /pkg/stofuse/os_darwin.go: -------------------------------------------------------------------------------- 1 | package stofuse 2 | 3 | // os-specific abstractions (Darwin) 4 | 5 | import ( 6 | "syscall" 7 | "time" 8 | ) 9 | 10 | // darwin doesn't seem to have field `Atim` in `syscall.Stat_t` 11 | func accessTimeFromStatt(_ *syscall.Stat_t, modTime time.Time) time.Time { 12 | return modTime 13 | } 14 | -------------------------------------------------------------------------------- /bin/build-frontend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | buildFrontend() { 4 | source /build-common.sh 5 | 6 | standardBuildProcess "frontend" 7 | } 8 | 9 | copyF61uiStaticFiles() { 10 | rm -rf public/f61ui/ 11 | cp -r frontend/f61ui/public/ public/f61ui/ 12 | } 13 | 14 | (cd frontend/ && buildFrontend) 15 | 16 | copyF61uiStaticFiles 17 | -------------------------------------------------------------------------------- /misc/seed-data/import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | # Imports seed data to be used for testing 4 | 5 | cp misc/seed-data/varasto.db /tmp/varasto.db 6 | 7 | # and client config (needed for server subsystems as well) 8 | cp misc/seed-data/varastoclient-config.json /root/ 9 | 10 | # and also with sample data 11 | tar -C /mnt -xf "misc/seed-data/blob-volumes.tar" 12 | -------------------------------------------------------------------------------- /pkg/stofuse/stofuseentrypoint/entrypoint_notwin.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | // This entrypoint is in own package, so we don't need to sprinkle conditional compilation 4 | // all around the base "stofuse" package because it doesn't compile on Windows 5 | package stofuseentrypoint 6 | 7 | import ( 8 | "github.com/function61/varasto/pkg/stofuse" 9 | ) 10 | 11 | var Entrypoint = stofuse.Entrypoint 12 | -------------------------------------------------------------------------------- /pkg/stotypes/apptypes_test.go: -------------------------------------------------------------------------------- 1 | package stotypes 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestEqual(t *testing.T) { 10 | a, _ := BlobRefFromHex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") 11 | b, _ := BlobRefFromHex("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") 12 | 13 | assert.Assert(t, a.Equal(*a)) 14 | assert.Assert(t, !a.Equal(*b)) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/gokitbp/backports.go: -------------------------------------------------------------------------------- 1 | // `gokit` backports. things that exist in newer version of gokit, but which we cannot update to yet. 2 | package gokitbp 3 | 4 | import ( 5 | "time" 6 | ) 7 | 8 | var ( 9 | DefaultReadHeaderTimeout = 60 * time.Second 10 | ) 11 | 12 | func Pointer[T any](input T) *T { 13 | return &input 14 | } 15 | 16 | func Must[T any](value T, err error) T { 17 | if err != nil { 18 | panic(err) 19 | } 20 | 21 | return value 22 | } 23 | -------------------------------------------------------------------------------- /docs/data-interfaces/web-ui/index.md: -------------------------------------------------------------------------------- 1 | Screenshots 2 | ----------- 3 | 4 | ### Generic files 5 | 6 | !!! note "" 7 | ![](../../content/generic-files/screenshot-web-ui.png) 8 | 9 | 10 | ### Movies 11 | 12 | !!! note "" 13 | ![](../../content/movies/endresult.png) 14 | 15 | 16 | ### TV shows 17 | 18 | !!! note "" 19 | ![](../../content/tvshows/endresult.png) 20 | 21 | 22 | ### Photos 23 | 24 | !!! note "" 25 | ![](../../content/photos/screenshot.png) 26 | -------------------------------------------------------------------------------- /pkg/fssnapshot/utils.go: -------------------------------------------------------------------------------- 1 | package fssnapshot 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "github.com/function61/gokit/cryptorandombytes" 7 | ) 8 | 9 | func randomSnapId() string { 10 | return "snap-" + cryptorandombytes.Hex(4) 11 | } 12 | 13 | // see tests for what this does 14 | func originPathInSnapshot(originPath string, mountPoint string, snapshotPath string) string { 15 | return filepath.Join( 16 | snapshotPath, 17 | originPath[len(mountPoint):]) 18 | } 19 | -------------------------------------------------------------------------------- /frontend/component/refreshbutton.tsx: -------------------------------------------------------------------------------- 1 | import { Glyphicon } from 'f61ui/component/bootstrap'; 2 | import * as React from 'react'; 3 | 4 | interface RefreshButtonProps { 5 | refresh: () => void; 6 | } 7 | 8 | export class RefreshButton extends React.Component { 9 | render() { 10 | return ( 11 | 12 | 13 | 14 | ); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /pkg/stoutils/tcpordomainsocketlistener_test.go: -------------------------------------------------------------------------------- 1 | package stoutils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestParseDomainSocketPath(t *testing.T) { 10 | assert.EqualString(t, ParseDomainSocketPath("domainsocket:///var/run/docker.sock"), "/var/run/docker.sock") 11 | assert.EqualString(t, ParseDomainSocketPath("domainsocket:/var/run/docker.sock"), "") 12 | assert.EqualString(t, ParseDomainSocketPath(":80"), "") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/stofuse/collectionbyidquery_test.go: -------------------------------------------------------------------------------- 1 | package stofuse 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestEncodeAndParseDirRef(t *testing.T) { 10 | combined := encodeDirRef("r-iZ5J_lXUI", "Dankest memes") 11 | 12 | assert.EqualString(t, combined, "Dankest memes - r-iZ5J_lXUI") 13 | 14 | assert.EqualString(t, parseDirRef(combined), "r-iZ5J_lXUI") 15 | 16 | assert.EqualString(t, parseDirRef("r-iZ5J_lXUI"), "r-iZ5J_lXUI") 17 | } 18 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@types/bootstrap": "3.4.0", 4 | "@types/jquery": "3.5.0", 5 | "@types/react-dom": "16.9.5", 6 | "react": "16.13.1", 7 | "jquery": "3.5.0", 8 | "date-fns": "2.0.0-alpha.27", 9 | "react-autocomplete": "1.8.1", 10 | "react-dom": "16.13.1", 11 | "bootstrap": "3.4.1" 12 | }, 13 | "devDependencies": {}, 14 | "scripts": { 15 | "test": "echo \"no test specified\" && exit 0" 16 | }, 17 | "private": true 18 | } 19 | -------------------------------------------------------------------------------- /docs/assets/Download-install-green.svg: -------------------------------------------------------------------------------- 1 | DOWNLOADINSTALL -------------------------------------------------------------------------------- /docs/content/magazines-comics/index.md: -------------------------------------------------------------------------------- 1 | Screenshot 2 | ---------- 3 | 4 | Varasto automatically shows thumbnails for 5 | [.cbz, .cbr](https://wiki.mobileread.com/wiki/CBR_and_CBZ) and `.pdf` files: 6 | 7 | ![](screenshot.png) 8 | 9 | 10 | No configuration 11 | ---------------- 12 | 13 | !!! info "Ready out-of-the-box" 14 | There is nothing you need to configure. 15 | 16 | 17 | Reading 18 | ------- 19 | 20 | Varasto doesn't implement reading - all it does is display the thumbnail. You can use any 21 | reader apps you're used to, to open the files. 22 | 23 | -------------------------------------------------------------------------------- /pkg/fssnapshot/interface.go: -------------------------------------------------------------------------------- 1 | // Cross-platform filesystem snapshotting library 2 | package fssnapshot 3 | 4 | type Snapshot struct { 5 | ID string // opaque platform-specific string (do not use for anything) 6 | OriginInSnapshotPath string // path used to access origin in snapshot 7 | OriginPath string // snapshot taken from 8 | SnapshotRootMountPath string // path used to access the snapshotted root 9 | } 10 | 11 | type Snapshotter interface { 12 | Snapshot(path string) (*Snapshot, error) 13 | Release(Snapshot) error 14 | } 15 | -------------------------------------------------------------------------------- /pkg/stoserver/stodiskaccess/writecounter_test.go: -------------------------------------------------------------------------------- 1 | package stodiskaccess 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/function61/gokit/assert" 9 | ) 10 | 11 | func TestWriteCounter(t *testing.T) { 12 | counter := &writeCounter{} 13 | 14 | msg, err := ioutil.ReadAll(counter.Tee(strings.NewReader("The quick brown fox jumps over the lazy dog"))) 15 | assert.Assert(t, err == nil) 16 | 17 | assert.EqualString(t, string(msg), "The quick brown fox jumps over the lazy dog") 18 | assert.Assert(t, counter.BytesWritten() == 43) 19 | } 20 | -------------------------------------------------------------------------------- /docs/content/photos/index.md: -------------------------------------------------------------------------------- 1 | Screenshot 2 | ---------- 3 | 4 | Varasto automatically generates and shows thumbnails for your photos and images: 5 | 6 | ![](screenshot.png) 7 | 8 | 9 | No configuration 10 | ---------------- 11 | 12 | !!! info "Ready out-of-the-box" 13 | There is nothing you need to configure. 14 | 15 | 16 | Upcoming feature: commenting 17 | ---------------------------- 18 | 19 | In the future, you'll be 20 | [able to comment on photos](https://github.com/function61/varasto/issues/169) (and other 21 | content) - even with friends that you have chosen to share your content with. 22 | -------------------------------------------------------------------------------- /pkg/stofuse/stofuseclient/client.go: -------------------------------------------------------------------------------- 1 | // Client for FUSE server's API 2 | package stofuseclient 3 | 4 | import ( 5 | "context" 6 | 7 | "github.com/function61/gokit/ezhttp" 8 | "github.com/function61/varasto/pkg/stofuse/stofusetypes" 9 | ) 10 | 11 | type Client struct { 12 | urls *stofusetypes.RestClientUrlBuilder 13 | } 14 | 15 | func New(baseUrl string) *Client { 16 | return &Client{stofusetypes.NewRestClientUrlBuilder(baseUrl)} 17 | } 18 | 19 | func (c *Client) UnmountAll(ctx context.Context) error { 20 | _, err := ezhttp.Post( 21 | ctx, 22 | c.urls.FuseUnmountAll()) 23 | return err 24 | } 25 | -------------------------------------------------------------------------------- /frontend/component/doclink.tsx: -------------------------------------------------------------------------------- 1 | import { DocsLink } from 'f61ui/component/docslink'; 2 | import { DocRef } from 'generated/stoserver/stoservertypes_types'; 3 | import * as React from 'react'; 4 | 5 | interface DocLinkProps { 6 | doc: DocRef; 7 | title?: string; 8 | } 9 | export class DocLink extends React.Component { 10 | render() { 11 | return ; 12 | } 13 | } 14 | 15 | export function DocUrlLatest(doc: DocRef): string { 16 | return 'https://function61.com/varasto/' + doc.replace('index.md', '').replace('.md', '/'); 17 | } 18 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | 11 | - uses: docker/setup-buildx-action@v1 12 | 13 | - name: Build 14 | run: | 15 | curl --fail --location --silent --output bob https://function61.com/go/turbobob-latest-stable-linux-amd64 && chmod +x bob 16 | ./bob build in-ci-autodetect-settings 17 | env: 18 | DOCKER_CREDS: ${{ secrets.DOCKER_CREDS }} 19 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | EVENTHORIZON: ${{ secrets.EVENTHORIZON }} 21 | -------------------------------------------------------------------------------- /pkg/stoserver/stodbimportexport/dbexportimport_test.go: -------------------------------------------------------------------------------- 1 | package stodbimportexport 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestBackupHeaderWritingAndParsing(t *testing.T) { 10 | backupHeader := makeBackupHeader(backupHeaderJson{NodeId: "RH7j", SchemaVersion: 314}) 11 | 12 | assert.EqualString(t, backupHeader, `# Varasto-DB-snapshot{"node_id":"RH7j","schema_version":314}`) 13 | 14 | details, err := parseBackupHeader(backupHeader) 15 | assert.Assert(t, err == nil) 16 | 17 | assert.EqualString(t, details.NodeId, "RH7j") 18 | assert.Assert(t, details.SchemaVersion == 314) 19 | } 20 | -------------------------------------------------------------------------------- /frontend/component/assetimg.tsx: -------------------------------------------------------------------------------- 1 | import { globalConfig } from 'f61ui/globalconfig'; 2 | import * as React from 'react'; 3 | 4 | interface AssetImgProps { 5 | // TODO: have this backed by enum like: 6 | // src: '/collection.png' | '/directory.png' | '/file.png'; 7 | src: string; 8 | width?: number; 9 | height?: number; 10 | } 11 | 12 | export class AssetImg extends React.Component { 13 | render() { 14 | // FIXME: path descension is a hack 15 | return ( 16 | 22 | ); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /pkg/stoserver/stodiskaccess/writecounter.go: -------------------------------------------------------------------------------- 1 | package stodiskaccess 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // simply keeps track of how many bytes were written to it. 8 | // one use case is to tee a io.Reader to this counter so a consumer of the reader does not 9 | // have to report the count back to you, but you can capture it out-of-band 10 | type writeCounter struct { 11 | count int64 12 | } 13 | 14 | func (c *writeCounter) BytesWritten() int64 { 15 | return c.count 16 | } 17 | 18 | func (c *writeCounter) Write(data []byte) (int, error) { 19 | l := len(data) 20 | c.count += int64(l) 21 | return l, nil 22 | } 23 | 24 | func (c *writeCounter) Tee(source io.Reader) io.Reader { 25 | return io.TeeReader(source, c) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/tui/progressbar_test.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestProgressBar(t *testing.T) { 10 | assert.EqualString(t, ProgressBar(0, 20, ProgressBarDefaultTheme()), "░░░░░░░░░░░░░░░░░░░░") 11 | assert.EqualString(t, ProgressBar(50, 20, ProgressBarDefaultTheme()), "██████████░░░░░░░░░░") 12 | assert.EqualString(t, ProgressBar(100, 20, ProgressBarDefaultTheme()), "████████████████████") 13 | } 14 | 15 | func TestProgressBarThemes(t *testing.T) { 16 | assert.EqualString(t, ProgressBar(13, 20, ProgressBarDefaultTheme()), "██░░░░░░░░░░░░░░░░░░") 17 | assert.EqualString(t, ProgressBar(13, 20, ProgressBarCirclesTheme()), "⬤⬤○○○○○○○○○○○○○○○○○○") 18 | } 19 | -------------------------------------------------------------------------------- /pkg/blobstore/localfsblobstore/localfs_test.go: -------------------------------------------------------------------------------- 1 | package localfsblobstore 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | "github.com/function61/varasto/pkg/stotypes" 8 | ) 9 | 10 | func TestPath(t *testing.T) { 11 | driver := New("APvMjudT4IQ", "/tmp/", nil) 12 | 13 | blobRef, err := stotypes.BlobRefFromHex("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592") 14 | assert.Ok(t, err) 15 | 16 | // base32Decode("qukfnco7qu098qeajaub021e9u6lckf4dkudmthd0b8budu9sm90") 17 | // = hex("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592") 18 | assert.EqualString(t, 19 | driver.getPath(*blobRef), 20 | "/tmp/q/uk/fnco7qu098qeajaub021e9u6lckf4dkudmthd0b8budu9sm90") 21 | } 22 | -------------------------------------------------------------------------------- /pkg/tui/progressbar.go: -------------------------------------------------------------------------------- 1 | // Utils for text-based UIs 2 | package tui 3 | 4 | func ProgressBar(pct int, barLength int, theme ProgressBarTheme) string { 5 | r := make([]rune, barLength) 6 | 7 | ratio := float64(barLength) * float64(pct) / 100.0 8 | 9 | for i := 0; i < barLength; i++ { 10 | ch := theme.Vacant 11 | if float64(i+1) <= ratio { 12 | ch = theme.Filled 13 | } 14 | 15 | r[i] = ch 16 | } 17 | 18 | return string(r) 19 | } 20 | 21 | type ProgressBarTheme struct { 22 | Filled rune 23 | Vacant rune 24 | } 25 | 26 | func ProgressBarDefaultTheme() ProgressBarTheme { 27 | return ProgressBarTheme{'█', '░'} 28 | } 29 | 30 | func ProgressBarCirclesTheme() ProgressBarTheme { 31 | return ProgressBarTheme{'⬤', '○'} 32 | } 33 | -------------------------------------------------------------------------------- /pkg/stoserver/stohealth/staticnode.go: -------------------------------------------------------------------------------- 1 | package stohealth 2 | 3 | import ( 4 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 5 | ) 6 | 7 | func NewStaticHealthNode( 8 | title string, 9 | healthStatus stoservertypes.HealthStatus, 10 | descr string, 11 | kind *stoservertypes.HealthKind, 12 | ) HealthChecker { 13 | return &staticNode{title, healthStatus, descr, kind} 14 | } 15 | 16 | type staticNode struct { 17 | title string 18 | healthStatus stoservertypes.HealthStatus 19 | descr string 20 | kind *stoservertypes.HealthKind 21 | } 22 | 23 | func (s *staticNode) CheckHealth() (*stoservertypes.Health, error) { 24 | return mkHealthWithChildren(s.title, s.healthStatus, s.descr, []HealthChecker{}, s.kind) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/stomvu/custompattern_test.go: -------------------------------------------------------------------------------- 1 | package stomvu 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestCustomMonthlyPattern(t *testing.T) { 10 | phoneCallPattern := customMonthlyPattern("^[0-9a-f]{2}([0-9]{14})p", "20060102150405") 11 | 12 | tcs := []struct { 13 | input string 14 | expect string 15 | }{ 16 | {"0d20190620121528p+358504123456.m4a", "2019/06"}, 17 | {"0d20181220121528p+358504123456.m4a", "2018/12"}, 18 | {"0d20181320121528p+358504123456.m4a", ""}, // there is no 13th month => invalid 19 | } 20 | 21 | for _, tc := range tcs { 22 | tc := tc // pin 23 | t.Run(tc.input, func(t *testing.T) { 24 | assert.EqualString(t, phoneCallPattern(tc.input), tc.expect) 25 | }) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /docs/content/games/index.md: -------------------------------------------------------------------------------- 1 | Varasto supports fetching metadata for games from [IGDB](https://www.igdb.com/) for 2 | [pretty much all platforms](https://www.igdb.com/platforms) like PC, game consoles (both 3 | retro & modern), mobile devices etc. 4 | 5 | Metadata includes: 6 | 7 | - Screenshot 8 | - YouTube video 9 | - Links to external websites 10 | - Publish date 11 | 12 | Note that Varasto's also includes support for ratings for all content, so you can also rate 13 | your games in your collection! 14 | 15 | 16 | Screenshot 17 | ---------- 18 | 19 | ![](screenshot.png) 20 | 21 | 22 | Configuration 23 | ------------- 24 | 25 | You have to configure an API key to access IGDB (it doesn't cost anything). There's a link 26 | in Varasto's UI to the registration. 27 | -------------------------------------------------------------------------------- /pkg/stoclient/utils.go: -------------------------------------------------------------------------------- 1 | package stoclient 2 | 3 | import ( 4 | "io/fs" 5 | "net/http" 6 | 7 | "github.com/function61/gokit/ezhttp" 8 | "github.com/function61/varasto/pkg/stotypes" 9 | ) 10 | 11 | func BlobIdxFromOffset(offset int64) (int, int64) { 12 | blobIdx := int(offset / stotypes.BlobSize) 13 | return blobIdx, offset - (int64(blobIdx) * stotypes.BlobSize) 14 | } 15 | 16 | func boolToStr(input bool) string { 17 | if input { 18 | return "true" 19 | } else { 20 | return "false" 21 | } 22 | } 23 | 24 | func translate404ToFSErrNotExist(err error) error { 25 | if err != nil { 26 | if ezhttp.ErrorIs(err, http.StatusNotFound) { 27 | return fs.ErrNotExist 28 | } else { // some other error 29 | return err 30 | } 31 | } 32 | 33 | return nil // no error 34 | } 35 | -------------------------------------------------------------------------------- /pkg/byteshuman/humanize_test.go: -------------------------------------------------------------------------------- 1 | package byteshuman 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestHumanize(t *testing.T) { 10 | for _, tc := range []struct { 11 | input uint64 12 | output string 13 | }{ 14 | {0, "0 B"}, 15 | {1024, "1.00 kiB"}, 16 | {1536.0, "1.50 kiB"}, 17 | {1048576, "1.00 MiB"}, 18 | {1572864, "1.50 MiB"}, 19 | {1073741824, "1.00 GiB"}, 20 | {1610612736, "1.50 GiB"}, 21 | {1099511627776, "1.00 TiB"}, 22 | {1649267441664, "1.50 TiB"}, 23 | {1125899906842624, "1.00 PiB"}, 24 | {1688849860263936, "1.50 PiB"}, 25 | {1152921504606846976, "1024.00 PiB"}, 26 | } { 27 | t.Run(tc.output, func(t *testing.T) { 28 | assert.EqualString(t, Humanize(tc.input), tc.output) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /docs/install/windows.md: -------------------------------------------------------------------------------- 1 | Windows 2 | ======= 3 | 4 | Follow same instructions as for [Linux (manual installation)](linux-manual.md), but 5 | there's no autostart yet (the `server install` thing), so you have to just run the .exe 6 | file directly from command line. 7 | 8 | 9 | Supported Windows versions 10 | -------------------------- 11 | 12 | Only Windows 10 works, because it introduced some 13 | [features that we need](https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/). 14 | 15 | 16 | Future of our Windows support 17 | ----------------------------- 18 | 19 | In the future I think we should research targeting 20 | [Windows Subsystem for Linux](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) 21 | (present since Win10) via Docker to have less moving parts. 22 | 23 | -------------------------------------------------------------------------------- /pkg/byteshuman/humanize.go: -------------------------------------------------------------------------------- 1 | // Formats byte amounts into human readable format 2 | package byteshuman 3 | 4 | import ( 5 | "fmt" 6 | ) 7 | 8 | const ( 9 | B = 1 10 | kiB = 1024 * B 11 | MiB = 1024 * kiB 12 | GiB = 1024 * MiB 13 | TiB = 1024 * GiB 14 | PiB = 1024 * TiB 15 | ) 16 | 17 | func Humanize(num uint64) string { 18 | switch { 19 | case num >= PiB: 20 | return fmt.Sprintf("%.02f PiB", float64(num)/PiB) 21 | case num >= TiB: 22 | return fmt.Sprintf("%.02f TiB", float64(num)/TiB) 23 | case num >= GiB: 24 | return fmt.Sprintf("%.02f GiB", float64(num)/GiB) 25 | case num >= MiB: 26 | return fmt.Sprintf("%.02f MiB", float64(num)/MiB) 27 | case num >= kiB: 28 | return fmt.Sprintf("%.02f kiB", float64(num)/kiB) 29 | default: 30 | return fmt.Sprintf("%d B", num) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/stomvu/entrypoint.go: -------------------------------------------------------------------------------- 1 | package stomvu 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | func Entrypoint() *cobra.Command { 10 | cmd := &cobra.Command{ 11 | Use: "mvu", 12 | Short: `Renaming utils ("mv utils") for photos, TV series etc.`, 13 | } 14 | 15 | cmd.AddCommand(tvEntrypoint()) 16 | cmd.AddCommand(photoEntrypoint()) 17 | cmd.AddCommand(customMonthlyPatternEntrypoint()) 18 | cmd.AddCommand(fileModificationTimeEntrypoint()) 19 | 20 | return cmd 21 | } 22 | 23 | func runOrExplainPlan(targetFn func(string) string, doIt bool) error { 24 | plan, err := ComputePlan("./", targetFn) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | if doIt { 30 | return ExecutePlan(plan) 31 | } else { 32 | explainPlan(plan, os.Stdout) 33 | return nil 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pkg/fssnapshot/null.go: -------------------------------------------------------------------------------- 1 | package fssnapshot 2 | 3 | // you can use NullSnapshotter when your application gives the option of using snapshots. 4 | // in the cases where snapshotting is not available (or user doesn't want it), you can do 5 | // your file accessing using the same logic (take snapshot, read files, release snapshot) 6 | // regardless of if snapshotting is actually used or not. 7 | 8 | func NullSnapshotter() Snapshotter { 9 | return &nullSnapshotter{} 10 | } 11 | 12 | type nullSnapshotter struct{} 13 | 14 | func (l *nullSnapshotter) Snapshot(path string) (*Snapshot, error) { 15 | return &Snapshot{ 16 | ID: "No snapshotting was used", 17 | OriginPath: path, 18 | OriginInSnapshotPath: path, 19 | SnapshotRootMountPath: path, 20 | }, nil 21 | } 22 | 23 | func (l *nullSnapshotter) Release(Snapshot) error { 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /frontend/component/tabcontroller.tsx: -------------------------------------------------------------------------------- 1 | import { getCurrentLocation } from 'f61ui/browserutils'; 2 | import * as React from 'react'; 3 | 4 | export interface Tab { 5 | url: string; 6 | title: string; 7 | } 8 | 9 | interface TabControllerProps { 10 | tabs: Tab[]; 11 | children: React.ReactNode; 12 | } 13 | 14 | export class TabController extends React.Component { 15 | render() { 16 | const currentLoc = getCurrentLocation(); 17 | 18 | return ( 19 |
20 |
    21 | {this.props.tabs.map((tab) => ( 22 |
  • 23 | {tab.title} 24 |
    • 25 | ))} 26 |
27 |
{this.props.children}
28 |
29 | ); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pkg/stomvu/photorenamer_test.go: -------------------------------------------------------------------------------- 1 | package stomvu 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestDetectPhotoVideoDate(t *testing.T) { 10 | tcs := []struct { 11 | input string 12 | expect string 13 | }{ 14 | {"IMG_20180526_151345.jpg", "2018-05 - Unsorted"}, 15 | {"VID_20190626_151345.jpg", "2019-06 - Unsorted"}, 16 | {"20170429_194919.mp4", "2017-04 - Unsorted"}, 17 | {"20170627_203226.jpg", "2017-06 - Unsorted"}, 18 | {"IMG_20180526666_151345.jpg", "nomatch"}, 19 | } 20 | 21 | for _, tc := range tcs { 22 | tc := tc // pin 23 | t.Run(tc.input, func(t *testing.T) { 24 | res := detectPhotoVideoDate(tc.input) 25 | 26 | var output string 27 | if res != nil { 28 | output = res.String() 29 | } else { 30 | output = "nomatch" 31 | } 32 | 33 | assert.EqualString(t, output, tc.expect) 34 | }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/stoserver/stodb/configkeys.go: -------------------------------------------------------------------------------- 1 | package stodb 2 | 3 | import ( 4 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 5 | ) 6 | 7 | var ( 8 | CfgNodeId = configAccessor("nodeId") 9 | CfgTheMovieDbApikey = configAccessor(stoservertypes.CfgTheMovieDbApikey) 10 | CfgIgdbApikey = configAccessor(stoservertypes.CfgIgdbApikey) 11 | CfgFuseServerBaseUrl = configAccessor(stoservertypes.CfgFuseServerBaseUrl) 12 | CfgNetworkShareBaseUrl = configAccessor(stoservertypes.CfgNetworkShareBaseUrl) 13 | CfgUbackupConfig = configAccessor(stoservertypes.CfgUbackupConfig) 14 | CfgUpdateStatusAt = configAccessor(stoservertypes.CfgUpdateStatusAt) 15 | CfgNodeTlsCertKey = configAccessor(stoservertypes.CfgNodeTlsCertKey) 16 | CfgGrafanaUrl = configAccessor(stoservertypes.CfgGrafanaUrl) 17 | CfgMediascannerState = configAccessor(stoservertypes.CfgMediascannerState) 18 | ) 19 | -------------------------------------------------------------------------------- /pkg/duration/humanize_test.go: -------------------------------------------------------------------------------- 1 | package duration 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/function61/gokit/assert" 8 | ) 9 | 10 | func TestHumanize(t *testing.T) { 11 | tcs := []struct { 12 | input string 13 | output string 14 | }{ 15 | {"0ms", "0 milliseconds"}, 16 | {"1ms", "1 millisecond"}, 17 | {"499ms", "499 milliseconds"}, 18 | {"500ms", "1 second"}, 19 | {"1s", "1 second"}, 20 | {"29s", "29 seconds"}, 21 | {"30s", "1 minute"}, 22 | {"29m", "29 minutes"}, 23 | {"36m34.20996749s", "1 hour"}, 24 | {"89m", "1 hour"}, 25 | {"90m", "2 hours"}, 26 | {"12h", "1 day"}, 27 | {"36h", "2 days"}, 28 | } 29 | 30 | for _, tc := range tcs { 31 | tc := tc // pin 32 | 33 | t.Run(tc.input, func(t *testing.T) { 34 | dur, err := time.ParseDuration(tc.input) 35 | assert.Assert(t, err == nil) 36 | 37 | assert.EqualString(t, Humanize(dur), tc.output) 38 | }) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pkg/mutexmap/mutexmap_test.go: -------------------------------------------------------------------------------- 1 | package mutexmap 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/function61/gokit/assert" 8 | ) 9 | 10 | func TestMutexMap(t *testing.T) { 11 | mm := New() 12 | 13 | unlockFoo := mm.Lock("foo") 14 | unlockFoo() 15 | 16 | unlockFoo, fooOk := mm.TryLock("foo") 17 | assert.Assert(t, fooOk) 18 | 19 | _, fooConcurrentOk := mm.TryLock("foo") 20 | assert.Assert(t, !fooConcurrentOk) 21 | 22 | unlockFoo() 23 | 24 | unlockFoo, fooOk = mm.TryLock("foo") 25 | assert.Assert(t, fooOk) 26 | 27 | lockAcquireDuration := make(chan time.Duration) 28 | 29 | go func() { 30 | startedAcquiringLock := time.Now() 31 | 32 | unlock := mm.Lock("foo") 33 | defer unlock() 34 | 35 | lockAcquireDuration <- time.Since(startedAcquiringLock) 36 | }() 37 | 38 | time.Sleep(11 * time.Millisecond) 39 | 40 | unlockFoo() 41 | 42 | assert.Assert(t, <-lockAcquireDuration > 10*time.Millisecond) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/stodebug/entrypoint.go: -------------------------------------------------------------------------------- 1 | package stodebug 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/function61/gokit/osutil" 7 | "github.com/function61/varasto/pkg/blobstore/localfsblobstore" 8 | "github.com/function61/varasto/pkg/stodupremover" 9 | "github.com/function61/varasto/pkg/stotypes" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func Entrypoint() *cobra.Command { 14 | debug := &cobra.Command{ 15 | Use: "debug", 16 | Short: `Debug utilities`, 17 | } 18 | 19 | debug.AddCommand(&cobra.Command{ 20 | Use: "localfsblobstore-path [blobRef]", 21 | Short: "Format FS path from BlobRef", 22 | Args: cobra.ExactArgs(1), 23 | Run: func(cmd *cobra.Command, args []string) { 24 | ref, err := stotypes.BlobRefFromHex(args[0]) 25 | osutil.ExitIfError(err) 26 | 27 | fmt.Println(localfsblobstore.RefToPath(*ref, "/")) 28 | }, 29 | }) 30 | 31 | debug.AddCommand(stodupremover.Entrypoint()) 32 | 33 | return debug 34 | } 35 | -------------------------------------------------------------------------------- /pkg/stomediascanner/thumbnailer_test.go: -------------------------------------------------------------------------------- 1 | package stomediascanner 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/function61/gokit/assert" 8 | ) 9 | 10 | func TestResizedDimensions300x533(t *testing.T) { 11 | tcs := []struct { 12 | width int 13 | height int 14 | expected string 15 | }{ 16 | { 17 | 16, 18 | 16, 19 | "300x300", 20 | }, 21 | { 22 | 3264, 23 | 1836, 24 | "300x168", 25 | }, 26 | { 27 | 1836, 28 | 3264, 29 | "299x533", 30 | }, 31 | { 32 | 400, 33 | 200, 34 | "300x150", // 2:1 ratio 35 | }, 36 | { 37 | 250, 38 | 1000, 39 | "133x533", // 1:4 ratio 40 | }, 41 | } 42 | 43 | for _, tc := range tcs { 44 | tc := tc // pin 45 | t.Run(tc.expected, func(t *testing.T) { 46 | w, h := resizedDimensions(tc.width, tc.height, 300, 533) 47 | result := fmt.Sprintf("%dx%d", w, h) 48 | 49 | assert.EqualString(t, result, tc.expected) 50 | }) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pkg/stomvu/tv.go: -------------------------------------------------------------------------------- 1 | package stomvu 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "github.com/function61/gokit/osutil" 7 | "github.com/function61/varasto/pkg/seasonepisodedetector" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | func tvEntrypoint() *cobra.Command { 12 | doIt := false 13 | 14 | cmd := &cobra.Command{ 15 | Use: "tv", 16 | Short: "Renames TV episodes", 17 | Args: cobra.NoArgs, 18 | Run: func(cmd *cobra.Command, args []string) { 19 | osutil.ExitIfError(runOrExplainPlan(episodeFromFilename, doIt)) 20 | }, 21 | } 22 | 23 | cmd.Flags().BoolVarP(&doIt, "do", "", doIt, "Whether to execute the plan or run a dry run") 24 | 25 | return cmd 26 | } 27 | 28 | // logic is already tested in seasonepisodedetector package 29 | func episodeFromFilename(input string) string { 30 | result := seasonepisodedetector.Detect(input) 31 | if result == nil { 32 | return "" 33 | } 34 | 35 | // "S03/S03E07" 36 | return filepath.Join(result.SeasonDesignation(), result.String()) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/stotypes/apptypes.go: -------------------------------------------------------------------------------- 1 | package stotypes 2 | 3 | import ( 4 | "bytes" 5 | "encoding/hex" 6 | "fmt" 7 | ) 8 | 9 | const ( 10 | BlobSize = 4 * mebibyte 11 | mebibyte = 1024 * 1024 12 | NoParentId = "" 13 | ) 14 | 15 | type BlobRef []byte 16 | 17 | func BlobRefFromHex(serialized string) (*BlobRef, error) { 18 | bytes, err := hex.DecodeString(serialized) 19 | if err != nil { 20 | return nil, fmt.Errorf("bad blob ref: %w", err) 21 | } 22 | 23 | return BlobRefFromBytes(bytes) 24 | } 25 | 26 | func BlobRefFromBytes(bytes []byte) (*BlobRef, error) { 27 | if len(bytes) != 32 { 28 | return nil, fmt.Errorf("bad blob ref: expecting 32 bytes (got %d)", len(bytes)) 29 | } 30 | 31 | br := BlobRef(bytes) 32 | return &br, nil 33 | } 34 | 35 | func (b *BlobRef) Equal(other BlobRef) bool { 36 | return bytes.Equal(*b, other) 37 | } 38 | 39 | func (b *BlobRef) AsHex() string { 40 | return hex.EncodeToString([]byte(*b)) 41 | } 42 | 43 | func (b *BlobRef) AsSha256Sum() []byte { 44 | return []byte(*b) 45 | } 46 | -------------------------------------------------------------------------------- /docs/install/index.md: -------------------------------------------------------------------------------- 1 | Server installation 2 | ------------------- 3 | 4 | | Installation | Fully supported[^1] | Notes | 5 | |-----------------------------|-----------------|-------| 6 | | [Linux (Docker)](linux-docker.md) | ☑️ | **Recommended, easiest option**. For PCs, Raspberry Pis etc. | 7 | | [Linux (manual installation)](linux-manual.md) | ☑️ | For users not wanting to use Docker | 8 | | [Windows](windows.md) | ☐ | | 9 | | [macOS](mac.md) | ☐ | | 10 | 11 | 12 | The client? 13 | ----------- 14 | 15 | !!! info "Not familiar with the differences of Varasto server and client?" 16 | [Read about it first](../concepts-ideas-architecture/index.md#client-vs-server)! 17 | 18 | !!! tip "Don't worry about this" 19 | Once you install the server, the UI will have client download links and help. 20 | 21 | If you want to dig in anyway, there's a [separate document](../data-interfaces/client/index.md). 22 | 23 | 24 | [^1]: Extensively tested by us and most likely to work as intended. 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | # NOTE: because of these args, if you want to build this manually you've to add 4 | # e.g. --build-arg TARGETARCH=amd64 to $ docker build ... 5 | 6 | # "amd64" | "arm" | ... 7 | ARG TARGETARCH 8 | # usually empty. for "linux/arm/v7" => "v7" 9 | ARG TARGETVARIANT 10 | 11 | WORKDIR /varasto 12 | 13 | # stores Varasto state (files' metadata) 14 | VOLUME /varasto-db 15 | 16 | ENTRYPOINT ["sto"] 17 | 18 | CMD ["server"] 19 | 20 | # symlink /root/varastoclient-config.json to /varasto-db/.. because it's stateful. 21 | # this config is used for server subsystems (thumbnailing, FUSE projector) to communicate 22 | # with the server. 23 | 24 | RUN mkdir -p /varasto /root/.config/varasto \ 25 | && ln -s /varasto/sto /bin/sto \ 26 | && ln -s /varasto-db/client-config.json /root/.config/varasto/client-config.json \ 27 | && apk add --update smartmontools fuse \ 28 | && echo '{"db_location": "/varasto-db/varasto.db"}' > /varasto/config.json 29 | 30 | COPY "rel/sto_linux-$TARGETARCH" /varasto/sto 31 | -------------------------------------------------------------------------------- /pkg/blobstore/interface.go: -------------------------------------------------------------------------------- 1 | // Interface for writing blob store adapters to Varasto 2 | package blobstore 3 | 4 | import ( 5 | "context" 6 | "io" 7 | 8 | "github.com/function61/varasto/pkg/stotypes" 9 | ) 10 | 11 | type Driver interface { 12 | // backing store must be idempotent, i.e. writing same blob again must not change outcome. 13 | // write also must be atomic. Fetch() must not return anything before store is completed succesfully. 14 | RawStore(ctx context.Context, ref stotypes.BlobRef, content io.Reader) error 15 | 16 | // raw = driver doesn't do any encryption/compression/integrity verifications, 17 | // they are done at a higher level. 18 | // if blob is not found, error must report os.IsNotExist(err) == true 19 | RawFetch(ctx context.Context, ref stotypes.BlobRef) (io.ReadCloser, error) 20 | 21 | // if blob is stored in multiple volumes, disk access controller fetches from the volume 22 | // (that is mounted) with the lowest routing cost. 23 | // currently 10 for local disks, 20 for cloud services. 24 | RoutingCost() int 25 | } 26 | -------------------------------------------------------------------------------- /pkg/logtee/stringtail.go: -------------------------------------------------------------------------------- 1 | package logtee 2 | 3 | import ( 4 | "container/ring" 5 | "sync" 6 | ) 7 | 8 | type StringTail struct { 9 | lines *ring.Ring 10 | mu sync.Mutex 11 | } 12 | 13 | // keeps only "capacity" last Write() calls (which you can retrieve with Snapshot() ) 14 | func NewStringTail(capacity int) *StringTail { 15 | r := ring.New(capacity) 16 | for i := 0; i < capacity; i++ { // init items 17 | r.Value = "" 18 | r = r.Next() 19 | } 20 | 21 | return &StringTail{ 22 | lines: r, 23 | } 24 | } 25 | 26 | func (t *StringTail) Snapshot() []string { 27 | t.mu.Lock() 28 | defer t.mu.Unlock() 29 | 30 | ret := []string{} 31 | 32 | r := t.lines 33 | 34 | ll := r.Len() 35 | for i := 0; i < ll; i++ { 36 | val := r.Value.(string) 37 | if val != "" { // not sure about this check 38 | ret = append(ret, val) 39 | } 40 | r = r.Next() 41 | } 42 | 43 | return ret 44 | } 45 | 46 | func (t *StringTail) Write(line string) { 47 | t.mu.Lock() 48 | defer t.mu.Unlock() 49 | 50 | t.lines.Value = line 51 | t.lines = t.lines.Next() 52 | } 53 | -------------------------------------------------------------------------------- /pkg/docreference/refs_test.go: -------------------------------------------------------------------------------- 1 | package docreference 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | "github.com/function61/gokit/fileexists" 8 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 9 | ) 10 | 11 | // tests that each for each member of DocRef (e.g. "docs/example.md") a file exists. that 12 | // makes it possible for us to link to markdown view in GitHub with confidence that the URL 13 | // will not 404 if we move files around later and forget to update the ref 14 | func TestDocsExistForDocRefs(t *testing.T) { 15 | for _, member := range stoservertypes.DocRefMembers { 16 | member := member // pin 17 | t.Run(string(member), func(t *testing.T) { 18 | exists, err := fileexists.Exists("../../" + string(member)) 19 | assert.Ok(t, err) 20 | assert.Assert(t, exists) 21 | }) 22 | } 23 | } 24 | 25 | func TestGitHubMaster(t *testing.T) { 26 | assert.EqualString( 27 | t, 28 | GitHubMaster(stoservertypes.DocRefDocsIndexMd), 29 | "https://github.com/function61/varasto/blob/master/docs/index.md") 30 | } 31 | -------------------------------------------------------------------------------- /pkg/duration/humanize.go: -------------------------------------------------------------------------------- 1 | package duration 2 | 3 | import ( 4 | "math" 5 | "strconv" 6 | "time" 7 | ) 8 | 9 | func Humanize(dur time.Duration) string { 10 | milliseconds := float64(dur.Milliseconds()) 11 | seconds := int(math.Round(milliseconds / (1.0 * 1000.0))) 12 | minutes := int(math.Round(milliseconds / (60.0 * 1000.0))) 13 | hours := int(math.Round(milliseconds / (3600.0 * 1000.0))) 14 | days := int(math.Round(milliseconds / (86400.0 * 1000.0))) 15 | 16 | plural := func(num int, singular string, plural string) string { 17 | if num == 1 { 18 | return strconv.Itoa(num) + " " + singular 19 | } else { 20 | return strconv.Itoa(num) + " " + plural 21 | } 22 | } 23 | 24 | switch { 25 | case days > 0: 26 | return plural(days, "day", "days") 27 | case hours > 0: 28 | return plural(hours, "hour", "hours") 29 | case minutes > 0: 30 | return plural(minutes, "minute", "minutes") 31 | case seconds > 0: 32 | return plural(seconds, "second", "seconds") 33 | default: 34 | return plural(int(milliseconds), "millisecond", "milliseconds") 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/logtee/logtee_test.go: -------------------------------------------------------------------------------- 1 | package logtee 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/function61/gokit/assert" 9 | ) 10 | 11 | func TestComposite(t *testing.T) { 12 | sink := &bytes.Buffer{} 13 | 14 | tail := NewStringTail(4) 15 | 16 | // writes to upstream all end up in the sink, but Snapshot() only returns the last 4 lines 17 | upstream := NewLineSplitterTee(sink, func(line string) { 18 | tail.Write(line) 19 | }) 20 | 21 | _, _ = upstream.Write([]byte("line 1\nline 2\nline 3 left open")) 22 | 23 | assert.EqualString(t, fmt.Sprintf("%v", tail.Snapshot()), "[line 1 line 2]") 24 | 25 | _, _ = upstream.Write([]byte("\n")) // close line 3 26 | 27 | assert.EqualString(t, fmt.Sprintf("%v", tail.Snapshot()), "[line 1 line 2 line 3 left open]") 28 | 29 | _, _ = upstream.Write([]byte("line 4\nline 5\nline 6\n")) 30 | 31 | assert.EqualString(t, fmt.Sprintf("%v", tail.Snapshot()), "[line 3 left open line 4 line 5 line 6]") 32 | 33 | assert.EqualString(t, sink.String(), "line 1\nline 2\nline 3 left open\nline 4\nline 5\nline 6\n") 34 | } 35 | -------------------------------------------------------------------------------- /misc/varasto-updateserver/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version_major": 1, 3 | "deployer_image": "fn61/edgerouter:20200422_0646_c28967d0", 4 | "deploy_command": [ 5 | "edgerouter", 6 | "s3", 7 | "deploy", 8 | "${_.env.edgerouterAppId}", 9 | "${_.version.friendly}", 10 | "updateserver.tar.gz" 11 | ], 12 | "deploy_interactive_command": ["/bin/sh"], 13 | "env_vars": [ 14 | { 15 | "key": "edgerouterAppId", 16 | "optional": false, 17 | "placeholder": "hq.example.com", 18 | "help": "" 19 | }, 20 | { 21 | "key": "AWS_ACCESS_KEY_ID", 22 | "optional": false, 23 | "placeholder": "AKI..", 24 | "help": "Needs to be able to update S3 static websites and write to EventHorizon" 25 | }, 26 | { 27 | "key": "AWS_SECRET_ACCESS_KEY", 28 | "optional": false, 29 | "placeholder": "yPId..", 30 | "help": "" 31 | }, 32 | { 33 | "key": "EVENTHORIZON_TENANT", 34 | "optional": false, 35 | "placeholder": "prod:1", 36 | "help": "" 37 | } 38 | ], 39 | "software_unique_id": "41bd58f8-dbeb-43e6-b269-cfd56b4cac56" 40 | } 41 | -------------------------------------------------------------------------------- /pkg/seasonepisodedetector/detector_test.go: -------------------------------------------------------------------------------- 1 | package seasonepisodedetector 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestSeasonDesignation(t *testing.T) { 10 | assert.EqualString(t, Detect("Simpsons 07x01 - Who Shot Mr Burns (Part 2)").SeasonDesignation(), "S07") 11 | } 12 | 13 | func TestDetect(t *testing.T) { 14 | tcs := []struct { 15 | input string 16 | expect string 17 | }{ 18 | { 19 | input: "Grand.Designs.S12E06.720p.HDTV.x264", 20 | expect: "S12E06", 21 | }, 22 | { 23 | input: "Grand.Designs.s12e6.720p.HDTV.x264", 24 | expect: "S12E6", 25 | }, 26 | { 27 | input: "Grand.Designs.s12e.720p.HDTV.x264", 28 | expect: "nomatch", 29 | }, 30 | { 31 | input: "Simpsons 07x01 - Who Shot Mr Burns (Part 2) [rl]", 32 | expect: "S07E01", 33 | }, 34 | } 35 | 36 | for _, tc := range tcs { 37 | tc := tc // pin 38 | t.Run(tc.input, func(t *testing.T) { 39 | res := Detect(tc.input) 40 | 41 | var output string 42 | if res != nil { 43 | output = res.String() 44 | } else { 45 | output = "nomatch" 46 | } 47 | 48 | assert.EqualString(t, output, tc.expect) 49 | }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/logtee/linesplittertee.go: -------------------------------------------------------------------------------- 1 | // Plumbing for teeing/tailing log messages. Plop this between your root logger and stderr 2 | // and you'll be tailing log messages efficiently and programmatically for e.g. your GUI 3 | package logtee 4 | 5 | import ( 6 | "io" 7 | "strings" 8 | "sync" 9 | ) 10 | 11 | type lineSplitterTee struct { 12 | buf []byte // buffer before receiving \n 13 | lineCompleted func(string) 14 | mu sync.Mutex 15 | } 16 | 17 | // returns io.Writer that tees full lines to lineCompleted callback 18 | func NewLineSplitterTee(sink io.Writer, lineCompleted func(string)) io.Writer { 19 | return io.MultiWriter(sink, &lineSplitterTee{ 20 | buf: []byte{}, 21 | lineCompleted: lineCompleted, 22 | }) 23 | } 24 | 25 | func (l *lineSplitterTee) Write(data []byte) (int, error) { 26 | l.mu.Lock() 27 | defer l.mu.Unlock() 28 | 29 | l.buf = append(l.buf, data...) 30 | 31 | // as long as we have lines, chop the buffer down 32 | for { 33 | idx := strings.IndexByte(string(l.buf), '\n') 34 | if idx == -1 { 35 | break 36 | } 37 | 38 | l.lineCompleted(string(l.buf[0:idx])) 39 | 40 | l.buf = l.buf[idx+1:] 41 | } 42 | 43 | return len(data), nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/stoclient/rm.go: -------------------------------------------------------------------------------- 1 | package stoclient 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/function61/gokit/osutil" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func rm(ctx context.Context, path string) error { 14 | dir, err := filepath.Abs(path) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // will error out if not a workdir 20 | wd, err := NewWorkdirLocation(dir) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | ch, err := computeChangeset(ctx, wd, NewBlobDiscoveredNoopListener()) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | if ch.AnyChanges() { 31 | fmt.Printf("Refusing to delete workdir '%s' because it has changes\n", path) 32 | os.Exit(1) 33 | } 34 | 35 | return os.RemoveAll(dir) 36 | } 37 | 38 | func rmEntrypoint() *cobra.Command { 39 | return &cobra.Command{ 40 | Use: "rm ", 41 | Short: "Removes a local clone of collection, but only if remote has full state", 42 | Args: cobra.ExactArgs(1), 43 | Run: func(cmd *cobra.Command, args []string) { 44 | osutil.ExitIfError(wrapWithStopSupport(func(ctx context.Context) error { 45 | return rm(ctx, args[0]) 46 | })) 47 | }, 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/stoutils/tcpordomainsocketlistener.go: -------------------------------------------------------------------------------- 1 | package stoutils 2 | 3 | import ( 4 | "net" 5 | "os" 6 | "strings" 7 | 8 | "github.com/function61/gokit/fileexists" 9 | "github.com/function61/gokit/logex" 10 | ) 11 | 12 | func CreateTcpOrDomainSocketListener(addr string, logl *logex.Leveled) (net.Listener, error) { 13 | domainSocketPath := ParseDomainSocketPath(addr) 14 | 15 | if domainSocketPath != "" { 16 | return createDomainSocketListener(domainSocketPath, logl) 17 | } else { 18 | return net.Listen("tcp", addr) 19 | } 20 | } 21 | 22 | func createDomainSocketListener(domainSocketPath string, logl *logex.Leveled) (net.Listener, error) { 23 | exists, err := fileexists.Exists(domainSocketPath) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | if exists { 29 | logl.Info.Println("removing previous socket") 30 | 31 | if err := os.Remove(domainSocketPath); err != nil { 32 | return nil, err 33 | } 34 | } 35 | 36 | return net.Listen("unix", domainSocketPath) 37 | } 38 | 39 | func ParseDomainSocketPath(baseUrl string) string { 40 | if strings.HasPrefix(baseUrl, "domainsocket://") { 41 | return baseUrl[len("domainsocket://"):] 42 | } else { 43 | return "" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pkg/stoserver/stodiskaccess/support.go: -------------------------------------------------------------------------------- 1 | package stodiskaccess 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/function61/varasto/pkg/stotypes" 7 | ) 8 | 9 | type BlobMeta struct { 10 | Ref stotypes.BlobRef 11 | RealSize int32 12 | SizeOnDisk int32 // after optional compression 13 | IsCompressed bool 14 | EncryptionKeyId string 15 | EncryptionKey []byte // this is set when read from QueryBlobMetadata(), but not when given to WriteBlobCreated() 16 | ExpectedCrc32 []byte 17 | } 18 | 19 | type MetadataStore interface { 20 | // returns os.ErrNotExist if ref does not exist 21 | QueryBlobMetadata(ref stotypes.BlobRef, encryptionKeys []stotypes.KeyEnvelope) (*BlobMeta, error) 22 | QueryBlobCrc32(ref stotypes.BlobRef) ([]byte, error) 23 | QueryBlobExists(ref stotypes.BlobRef) (bool, error) 24 | QueryCollectionEncryptionKeyForNewBlobs(collId string) (string, []byte, error) 25 | WriteBlobCreated(meta *BlobMeta, volumeId int) error 26 | WriteBlobReplicated(ref stotypes.BlobRef, volumeId int) error 27 | } 28 | 29 | type readCloseWrapper struct { 30 | io.Reader 31 | closer io.Closer 32 | } 33 | 34 | func (r *readCloseWrapper) Close() error { 35 | return r.closer.Close() 36 | } 37 | -------------------------------------------------------------------------------- /pkg/stoutils/utils_test.go: -------------------------------------------------------------------------------- 1 | package stoutils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestIsMaybeCompressible(t *testing.T) { 10 | tcs := []struct { 11 | filename string 12 | expectedMaybeCompressible bool 13 | }{ 14 | { 15 | "hello.txt", 16 | true, 17 | }, 18 | { 19 | "main.go", 20 | true, 21 | }, 22 | { 23 | "Unknown.fileformat", 24 | true, 25 | }, 26 | { 27 | "movie.mp4", 28 | false, 29 | }, 30 | { 31 | "DCIM20190930_331.jpg", 32 | false, 33 | }, 34 | { 35 | "DCIM20190930_331.JPG", 36 | false, 37 | }, 38 | { 39 | "DCIM20190930_331.jpeg", 40 | false, 41 | }, 42 | { 43 | "killitwithfire.gif", 44 | false, 45 | }, 46 | { 47 | "Dexter-S03E02.mkv", 48 | false, 49 | }, 50 | { 51 | "2001: A Space Odyssey.AVI", 52 | false, 53 | }, 54 | { 55 | "Rick Astley - Never Gonna Give You Up.mp3", 56 | false, 57 | }, 58 | } 59 | 60 | for _, tc := range tcs { 61 | tc := tc // pin 62 | t.Run(tc.filename, func(t *testing.T) { 63 | assert.Assert(t, IsMaybeCompressible(tc.filename) == tc.expectedMaybeCompressible) 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/smart/jsonformat.go: -------------------------------------------------------------------------------- 1 | package smart 2 | 3 | type SmartCtlJsonReport struct { 4 | JsonFormatVersion []int `json:"json_format_version"` 5 | AtaSmartAttributes struct { 6 | Revision int `json:"revision"` 7 | Table []AtaSmartAttribute `json:"table"` 8 | } `json:"ata_smart_attributes"` 9 | SmartStatus struct { 10 | Passed bool `json:"passed"` 11 | } `json:"smart_status"` 12 | PowerCycleCount int `json:"power_cycle_count"` 13 | PowerOnTime struct { 14 | Hours int `json:"hours"` 15 | } `json:"power_on_time"` 16 | Temperature struct { 17 | Current int `json:"current"` 18 | } `json:"temperature"` 19 | } 20 | 21 | type AtaSmartAttribute struct { 22 | Id int `json:"id"` 23 | Name string `json:"name"` 24 | Value int `json:"value"` 25 | Worst int `json:"worst"` 26 | Thresh int `json:"thresh"` 27 | Raw struct { 28 | Value int `json:"value"` 29 | String string `json:"string"` 30 | } `json:"raw"` 31 | } 32 | 33 | func (s *SmartCtlJsonReport) FindSmartAttributeByName(name string) *AtaSmartAttribute { 34 | for _, item := range s.AtaSmartAttributes.Table { 35 | if item.Name == name { 36 | return &item 37 | } 38 | } 39 | 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /pkg/stoserver/stodb/schemaversion.go: -------------------------------------------------------------------------------- 1 | package stodb 2 | 3 | import ( 4 | "encoding/binary" 5 | 6 | "github.com/function61/varasto/pkg/blorm" 7 | "go.etcd.io/bbolt" 8 | ) 9 | 10 | const ( 11 | CurrentSchemaVersion = 6 12 | ) 13 | 14 | var ( 15 | metaBucketKey = []byte("_meta") 16 | schemaVersionKey = []byte("schemaVersion") 17 | ) 18 | 19 | // returns blorm.ErrBucketNotFound if version not found 20 | func ReadSchemaVersion(tx *bbolt.Tx) (uint32, error) { 21 | metaBucket := tx.Bucket(metaBucketKey) 22 | if metaBucket == nil { 23 | return 0, blorm.ErrBucketNotFound 24 | } 25 | 26 | schemaVersionInDb := binary.LittleEndian.Uint32(metaBucket.Get(schemaVersionKey)) 27 | return schemaVersionInDb, nil 28 | } 29 | 30 | func WriteSchemaVersion(version uint32, tx *bbolt.Tx) error { 31 | metaBucket, err := tx.CreateBucketIfNotExists(metaBucketKey) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | schemaVersionInDb := make([]byte, 4) 37 | binary.LittleEndian.PutUint32(schemaVersionInDb, version) 38 | 39 | return metaBucket.Put(schemaVersionKey, schemaVersionInDb) 40 | } 41 | 42 | func writeSchemaVersionCurrent(tx *bbolt.Tx) error { 43 | return WriteSchemaVersion(CurrentSchemaVersion, tx) 44 | } 45 | -------------------------------------------------------------------------------- /misc/docs-website-deployerspec/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version_major": 1, 3 | "deployer_image": "fn61/edgerouter:20200422_0646_c28967d0", 4 | "deploy_command": [ 5 | "edgerouter", 6 | "s3", 7 | "deploy", 8 | "${_.env.edgerouterAppId}", 9 | "${_.version.friendly}", 10 | "docs-website.tar.gz" 11 | ], 12 | "deploy_interactive_command": ["/bin/sh"], 13 | "download_artefacts": [ 14 | "docs-website.tar.gz" 15 | ], 16 | "env_vars": [ 17 | { 18 | "key": "edgerouterAppId", 19 | "optional": false, 20 | "placeholder": "hq.example.com", 21 | "help": "" 22 | }, 23 | { 24 | "key": "AWS_ACCESS_KEY_ID", 25 | "optional": false, 26 | "placeholder": "AKI..", 27 | "help": "Needs to be able to update S3 static websites and write to EventHorizon" 28 | }, 29 | { 30 | "key": "AWS_SECRET_ACCESS_KEY", 31 | "optional": false, 32 | "placeholder": "yPId..", 33 | "help": "" 34 | }, 35 | { 36 | "key": "EVENTHORIZON_TENANT", 37 | "optional": false, 38 | "placeholder": "prod:1", 39 | "help": "" 40 | } 41 | ], 42 | "software_unique_id": "e364dac3-c1fc-41cd-9d22-663fab88352b" 43 | } 44 | -------------------------------------------------------------------------------- /docs/storage/naming-your-volumes.md: -------------------------------------------------------------------------------- 1 | Choose a naming scheme for your volumes 2 | --------------------------------------- 3 | 4 | !!! warning "Name your disks - not their content!" 5 | Don't use "Movies" or "Work" because with Varasto you don't need to stress about which 6 | disk is used for storing which type of data - your e.g. movies can span many disks. The data 7 | can even move between your volumes later as your needs change. 8 | 9 | You can name your volumes anything you like - a few examples: 10 | 11 | - Your favourite TV show characters (I used Futurama) 12 | - "Disk A", "Disk B", ... etc. 13 | - Disk serial number 14 | 15 | 16 | You can rename volumes later 17 | ---------------------------- 18 | 19 | If you only have one disk or don't have a lot of disks, don't worry about this and just 20 | use anything you like. You can rename volumes later if your needs change. 21 | 22 | !!! tip 23 | While you can easily rename volumes later, if you followed our advice and named the 24 | subdirectory for Varasto blobs after the volume name, you need to also rename the 25 | subdirectory to avoid confusion. For local disks this is easy, but renaming the directory 26 | varies for cloud disks (for S3 it's impossible, while for Google Drive it's easy). 27 | -------------------------------------------------------------------------------- /docs/install/mac.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: macOS 3 | --- 4 | 5 | | Option | User experience | Occasionally tested by us | 6 | |-------------------|-----------------|---------------------------| 7 | | Docker | highest | ☑️ | 8 | | Linux VM on macOS | medium | ☐ | 9 | | Native Varasto server for mac | [lowest](#use-natively) | ☐ | 10 | 11 | 12 | Use via Docker 13 | -------------- 14 | 15 | Docker works on macOS so you may have success: 16 | 17 | - Installing [Docker for macOS](https://docs.docker.com/docker-for-mac/) 18 | - Continuing with the [Linux (Docker)](linux-docker.md) instructions. 19 | * Linux because internally `Docker for macOS` uses a Linux VM 20 | 21 | 22 | Use via Linux VM 23 | ---------------- 24 | 25 | If you already have a Linux VM (or want to set up one for this), then then go back to 26 | [Installation](index.md) and follow the Linux instructions. 27 | 28 | 29 | Use natively 30 | ------------ 31 | 32 | Varasto server has native compilation to macOS, so this might work. We haven't tested it, 33 | and we don't have service autostart on macOS, so you'll have to start the process manually 34 | each time you start your computer or make an auto-start script yourself. 35 | -------------------------------------------------------------------------------- /frontend/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin'); 3 | const CircularDependencyPlugin = require('circular-dependency-plugin') 4 | const webpack = require('webpack'); 5 | 6 | module.exports = { 7 | mode: 'development', // overridden by CLI flag on prod build 8 | entry: './main.tsx', 9 | plugins: [ 10 | new CircularDependencyPlugin({ 11 | exclude: /node_modules/, 12 | failOnError: true, 13 | }), 14 | new webpack.ProvidePlugin({ 15 | jQuery: 'jquery/dist/jquery.slim.js', // for stupid Bootstrap 16 | u2f: 'u2f-api/dist/lib/generated-google-u2f-api.js', 17 | }), 18 | ], 19 | module: { 20 | rules: [ 21 | { 22 | test: /\.tsx?$/, 23 | use: 'ts-loader', 24 | exclude: /node_modules/ 25 | } 26 | ] 27 | }, 28 | optimization: { 29 | // defaults: with prod -> true, with dev -> false 30 | // minify: false 31 | }, 32 | performance: { 33 | hints: false 34 | }, 35 | resolve: { 36 | extensions: [ '.tsx', '.ts', '.js' ], 37 | plugins: [new TsconfigPathsPlugin({ /*configFile: "./path/to/tsconfig.json" */ })] 38 | }, 39 | output: { 40 | filename: 'build.js', 41 | library: 'main', 42 | path: path.resolve(__dirname, '../public') 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /pkg/stomvu/custompattern.go: -------------------------------------------------------------------------------- 1 | package stomvu 2 | 3 | import ( 4 | "path/filepath" 5 | "regexp" 6 | "time" 7 | 8 | "github.com/function61/gokit/osutil" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | func customMonthlyPattern(reString string, dateformat string) func(name string) string { 13 | re := regexp.MustCompile(reString) 14 | 15 | return func(name string) string { 16 | result := re.FindStringSubmatch(name) 17 | if result == nil { 18 | return "" 19 | } 20 | 21 | ts, err := time.Parse(dateformat, result[1]) 22 | if err != nil { 23 | return "" 24 | } 25 | 26 | return filepath.Join( 27 | ts.Format("2006"), 28 | ts.Format("01")) 29 | } 30 | } 31 | 32 | func customMonthlyPatternEntrypoint() *cobra.Command { 33 | doIt := false 34 | 35 | cmd := &cobra.Command{ 36 | Use: "custom-monthly [regexp] [dateformat]", 37 | Short: "Custom date pattern for moving to monthly folders. The first capture group must be the timestamp", 38 | Args: cobra.ExactArgs(2), 39 | Run: func(cmd *cobra.Command, args []string) { 40 | osutil.ExitIfError(runOrExplainPlan(customMonthlyPattern(args[0], args[1]), doIt)) 41 | }, 42 | } 43 | 44 | cmd.Flags().BoolVarP(&doIt, "do", "", doIt, "Whether to execute the plan or run a dry run") 45 | 46 | return cmd 47 | } 48 | -------------------------------------------------------------------------------- /docs/install/linux-manual.md: -------------------------------------------------------------------------------- 1 | Linux (manual) 2 | ============== 3 | 4 | Download suitable binary from the 5 | [newest release](https://github.com/function61/varasto/releases) (you don't need anything 6 | else). 7 | 8 | Rename `sto_linux-amd64` -> `sto` and `chmod +x` it. 9 | 10 | Make `config.json` in the same directory with content: 11 | 12 | ```json 13 | { 14 | "db_location": "varasto.db" 15 | } 16 | ``` 17 | 18 | Now start the server (you may need to use sudo): 19 | 20 | ```console 21 | $ ./sto server 22 | 2019/08/02 12:35:04 bootstrap [INFO] generated nodeId: LCb0 23 | 2019/08/02 12:35:04 [INFO] node LCb0 (ver. dev) started 24 | ``` 25 | 26 | If everything seems to work, now stop it by pressing `ctrl+c`. 27 | 28 | Now make it start on system boot (you may need to run this with `sudo`): 29 | 30 | ```console 31 | $ ./sto server install 32 | Wrote unit file to /etc/systemd/system/varasto.service 33 | Run to enable on boot & to start now: 34 | $ systemctl enable --now varasto 35 | $ systemctl status varasto 36 | ``` 37 | 38 | Just follow above instructions (again, you might need `sudo`). 39 | 40 | 41 | After Varasto is started 42 | ------------------------ 43 | 44 | Now you can navigate your browser to [https://localhost/](https://localhost/). 45 | (You'll have to approve the "insecure certificate" warning.) 46 | -------------------------------------------------------------------------------- /docs/developers/codebase/index.md: -------------------------------------------------------------------------------- 1 | Drawing 2 | ------- 3 | 4 | ![](drawing.png) 5 | 6 | 7 | Example of implementing an entire feature 8 | ----------------------------------------- 9 | 10 | To easier understand the codebase and its layout, let's look at a commit that implements 11 | [adding tagging support to collections](https://github.com/function61/varasto/commit/5117244c57547f21b51cfa548151359ed436dd69). 12 | 13 | Points of interest in the commit: 14 | 15 | - `dbtypes.go` is the "column" in the database for keeping track of collection's assigned tags 16 | - `types.json` shows the REST API model addition for adding `Tags []string` to both backend 17 | (Go) and frontend (TypeScript) 18 | - `commands.json` adds the command for adding/removing tags: 19 | * backend/frontend REST endpoint struct 20 | * autogenerates modal dialog for the commands 21 | - `commandhandlers.go` has the implementation for adding/removing tags from DB 22 | - `CollectionPage.tsx` adds tag editor (add / view / remove tags) to UI (see fig. 1). 23 | "Add" and "remove" both invoke autogenerated modal dialog (see fig. 2). 24 | - `collectiondropdown.tsx` adds "add tag" modal dialog to collection dropdown (see fig. 3) 25 | 26 | Fig. 1: 27 | 28 | ![](tageditor.png) 29 | 30 | Fig. 2: 31 | 32 | ![](tag-add-modal.png) 33 | 34 | Fig. 3: 35 | 36 | ![](tag-add.png) 37 | -------------------------------------------------------------------------------- /pkg/stoserver/commandhandlersmetadata_test.go: -------------------------------------------------------------------------------- 1 | package stoserver 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | "github.com/function61/varasto/pkg/themoviedbapi" 8 | ) 9 | 10 | func TestEncodeTmdbRef(t *testing.T) { 11 | assert.EqualString(t, encodeTmdbRef(themoviedbapi.MediaTypeMovie, "tt7207398"), "tmdb:movie:tt7207398") 12 | assert.EqualString(t, encodeTmdbRef(themoviedbapi.MediaTypeTv, "tt0904208"), "tmdb:tv:tt0904208") 13 | } 14 | 15 | func TestDecodeInvalidTmdbRef(t *testing.T) { 16 | typ, _, err := decodeTmdbRef("foo") 17 | assert.Assert(t, err == nil) 18 | assert.EqualString(t, typ, "") 19 | 20 | _, _, err = decodeTmdbRef("tmdb:invalidtype:123") 21 | assert.EqualString(t, err.Error(), "unsupported tmdb type: 123") 22 | } 23 | 24 | func TestDecodeTmdbMovieRef(t *testing.T) { 25 | typ, id, err := decodeTmdbRef(encodeTmdbRef(themoviedbapi.MediaTypeMovie, "tt7207398")) 26 | assert.Assert(t, err == nil) 27 | assert.EqualString(t, typ, themoviedbapi.MediaTypeMovie) 28 | assert.EqualString(t, id, "tt7207398") 29 | } 30 | 31 | func TestDecodeTmdbTvRef(t *testing.T) { 32 | typ, id, err := decodeTmdbRef(encodeTmdbRef(themoviedbapi.MediaTypeTv, "tt0904208")) 33 | assert.Assert(t, err == nil) 34 | assert.EqualString(t, typ, themoviedbapi.MediaTypeTv) 35 | assert.EqualString(t, id, "tt0904208") 36 | } 37 | -------------------------------------------------------------------------------- /pkg/stoserver/stodb/scheduledjobseeddata.go: -------------------------------------------------------------------------------- 1 | package stodb 2 | 3 | import ( 4 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 5 | "github.com/function61/varasto/pkg/stotypes" 6 | ) 7 | 8 | // shared here b/c most of the definitions need to be in two places: 9 | // 1) bootstrapper (= starting from fresh slate) 10 | // 2) schema migration (adding scheduled job to existing installation) 11 | 12 | func scheduledJobSeedSmartPoller() *stotypes.ScheduledJob { 13 | return &stotypes.ScheduledJob{ 14 | ID: "ocKgpTHU3Sk", 15 | Description: "SMART poller", 16 | Schedule: "@every 5m", 17 | Kind: stoservertypes.ScheduledJobKindSmartpoll, 18 | Enabled: true, 19 | } 20 | } 21 | 22 | func scheduledJobSeedMetadataBackup() *stotypes.ScheduledJob { 23 | return &stotypes.ScheduledJob{ 24 | ID: "h-cPYsYtFzM", 25 | Description: "Metadata backup", 26 | Schedule: "@midnight", 27 | Kind: stoservertypes.ScheduledJobKindMetadatabackup, 28 | Enabled: true, 29 | } 30 | } 31 | 32 | func ScheduledJobSeedVersionUpdateCheck() *stotypes.ScheduledJob { 33 | return &stotypes.ScheduledJob{ 34 | ID: "EQi_3OhROUs", 35 | Description: "Varasto software update check", 36 | Schedule: "@midnight", 37 | Kind: stoservertypes.ScheduledJobKindVersionupdatecheck, 38 | Enabled: true, 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /cmd/sto/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/function61/gokit/dynversion" 7 | "github.com/function61/gokit/osutil" 8 | "github.com/function61/varasto/pkg/stoclient" 9 | "github.com/function61/varasto/pkg/stodebug" 10 | "github.com/function61/varasto/pkg/stofuse/stofuseentrypoint" 11 | "github.com/function61/varasto/pkg/stomvu" 12 | "github.com/function61/varasto/pkg/stoserver" 13 | "github.com/spf13/cobra" 14 | ) 15 | 16 | func main() { 17 | rootCmd := &cobra.Command{ 18 | Use: os.Args[0], 19 | Short: `Varasto CLI: sto ("STOrage without the rage")`, 20 | Version: dynversion.Version, 21 | // hide the default "completion" subcommand from polluting UX (it can still be used). https://github.com/spf13/cobra/issues/1507 22 | CompletionOptions: cobra.CompletionOptions{HiddenDefaultCmd: true}, 23 | } 24 | 25 | // client's commands are at the root level somewhat unhygienically for convenience's 26 | // sake (since the client CLI commands are used most often). 27 | for _, entrypoint := range stoclient.Entrypoints() { 28 | rootCmd.AddCommand(entrypoint) 29 | } 30 | 31 | rootCmd.AddCommand(stoserver.Entrypoint()) 32 | rootCmd.AddCommand(stofuseentrypoint.Entrypoint()) 33 | rootCmd.AddCommand(stomvu.Entrypoint()) 34 | rootCmd.AddCommand(stodebug.Entrypoint()) 35 | 36 | osutil.ExitIfError(rootCmd.Execute()) 37 | } 38 | -------------------------------------------------------------------------------- /misc/seed-data/README.md: -------------------------------------------------------------------------------- 1 | This directory contains the persisted data in the first generation format, that we'll 2 | forever support migrating from, to newer formats. 3 | 4 | These files are intended for Varasto development, testing and ensuring that our migrations 5 | work from the very first generation of data. 6 | 7 | The files are: 8 | 9 | ``` 10 | . 11 | |-- blob-volumes.tar 12 | |-- client-config.json 13 | `-- varasto.db 14 | ``` 15 | 16 | Client config (`client-config.json`) is needed for some server subsystems as well. 17 | 18 | The tar contains two volumes. It has all the collection's blobs in volume A, and volume B 19 | is empty (it has the volume UUID though, so it can be mounted): 20 | 21 | ``` 22 | . 23 | |-- vol-a 24 | | |-- 0 25 | | | `-- 00 26 | | | `-- 0000000000000000000000000000000000000000000000000 27 | | |-- 8 28 | | | `-- 0s 29 | | | `-- 30qfrbdvet2aq8qdmsr1tkvqgf68d4edc5a83r2sm0293lsng 30 | | |-- c 31 | | | `-- sq 32 | | | `-- j1hn93jkm496k2b019fkl625h47e4lgr2sbukdg0l42ss65qg 33 | | |-- d 34 | | | `-- d8 35 | | | `-- 5vtg38dc3qp4v55qknbluao4sfi4adtuq44pc63aqfi3er5j0 36 | | `-- k 37 | | `-- r8 38 | | `-- m8sd6e7e90c8k70lr0ib84rsasc1l69k8ad83ajv38vbf1thg 39 | `-- vol-b 40 | `-- 0 41 | `-- 00 42 | `-- 0000000000000000000000000000000000000000000000000 43 | ``` 44 | -------------------------------------------------------------------------------- /pkg/blorm/interface.go: -------------------------------------------------------------------------------- 1 | // "Bolt Light ORM", doesn't do much else than persist structs into Bolt.. 2 | // this was born because: https://github.com/asdine/storm/issues/222#issuecomment-472791001 3 | // 4 | // Warning: don't Each() and Delete() at the same time. Deletion messes with the iteration 5 | // 6 | // order somehow, and I observed half of the records was deleted when I tried to delete all. 7 | package blorm 8 | 9 | import ( 10 | "errors" 11 | 12 | "go.etcd.io/bbolt" 13 | ) 14 | 15 | var ( 16 | ErrNotFound = errors.New("database: record not found") 17 | ErrBucketNotFound = errors.New("bucket not found") 18 | StopIteration = errors.New("blorm: stop iteration") 19 | ) 20 | 21 | type Repository interface { 22 | Bootstrap(tx *bbolt.Tx) error 23 | // returns ErrNotFound if record not found 24 | // returns ErrBucketNotFound if bootstrap not done for bucket 25 | OpenByPrimaryKey(id []byte, record interface{}, tx *bbolt.Tx) error 26 | Update(record interface{}, tx *bbolt.Tx) error 27 | Delete(record interface{}, tx *bbolt.Tx) error 28 | // return blorm.StopIteration from "fn" to stop iteration. that error is not returned 29 | // to the API caller 30 | Each(fn func(record interface{}) error, tx *bbolt.Tx) error 31 | // rules of Each() also apply here 32 | EachFrom(from []byte, fn func(record interface{}) error, tx *bbolt.Tx) error 33 | Alloc() interface{} 34 | } 35 | -------------------------------------------------------------------------------- /pkg/stoserver/stodb/configaccessor.go: -------------------------------------------------------------------------------- 1 | package stodb 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/function61/varasto/pkg/blorm" 7 | "github.com/function61/varasto/pkg/stotypes" 8 | "go.etcd.io/bbolt" 9 | ) 10 | 11 | type ConfigRequiredError struct { 12 | error 13 | } 14 | 15 | type ConfigAccessor struct { 16 | key string 17 | } 18 | 19 | func configAccessor(key string) *ConfigAccessor { 20 | return &ConfigAccessor{key} 21 | } 22 | 23 | func (c *ConfigAccessor) GetOptional(tx *bbolt.Tx) (string, error) { 24 | return c.getWithRequired(false, tx) 25 | } 26 | 27 | // returns descriptive error message if value not set 28 | func (c *ConfigAccessor) GetRequired(tx *bbolt.Tx) (string, error) { 29 | return c.getWithRequired(true, tx) 30 | } 31 | 32 | func (c *ConfigAccessor) getWithRequired(required bool, tx *bbolt.Tx) (string, error) { 33 | conf := &stotypes.Config{} 34 | if err := configRepository.OpenByPrimaryKey([]byte(c.key), conf, tx); err != nil && err != blorm.ErrNotFound { 35 | return "", err 36 | } 37 | 38 | if conf.Value == "" && required { 39 | return "", &ConfigRequiredError{fmt.Errorf("config value %s not set", c.key)} 40 | } 41 | 42 | return conf.Value, nil 43 | } 44 | 45 | func (c *ConfigAccessor) Set(value string, tx *bbolt.Tx) error { 46 | return configRepository.Update(&stotypes.Config{ 47 | Key: c.key, 48 | Value: value, 49 | }, tx) 50 | } 51 | -------------------------------------------------------------------------------- /pkg/seasonepisodedetector/detector.go: -------------------------------------------------------------------------------- 1 | // Extracts season & episode numbers for TV show filenames 2 | package seasonepisodedetector 3 | 4 | import ( 5 | "regexp" 6 | "strconv" 7 | ) 8 | 9 | type Result struct { 10 | Season string 11 | Episode string 12 | } 13 | 14 | func (d *Result) SeasonDesignation() string { 15 | return "S" + d.Season 16 | } 17 | 18 | func (d *Result) String() string { 19 | return d.SeasonDesignation() + "E" + d.Episode 20 | } 21 | 22 | func (d *Result) LaxEqual(other Result) bool { 23 | var err error 24 | 25 | // converts string to int while keeping track of any error occurring 26 | c := func(in string) int { 27 | parsed, errAtoi := strconv.Atoi(in) 28 | if errAtoi != nil { 29 | err = errAtoi 30 | } 31 | 32 | return parsed 33 | } 34 | 35 | return c(d.Season) == c(other.Season) && c(d.Episode) == c(other.Episode) && err == nil 36 | } 37 | 38 | var detectSeasonEpisodeRe = regexp.MustCompile("[Ss]([0-9]+)[Ee]([0-9]+)") 39 | 40 | var detectSeasonEpisodeStupidRe = regexp.MustCompile("([0-9]+)[Xx]([0-9]+)") 41 | 42 | func Detect(filename string) *Result { 43 | result := detectSeasonEpisodeRe.FindStringSubmatch(filename) 44 | if result == nil { 45 | result = detectSeasonEpisodeStupidRe.FindStringSubmatch(filename) 46 | } 47 | if result == nil { 48 | return nil 49 | } 50 | 51 | return &Result{ 52 | Season: result[1], 53 | Episode: result[2], 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /docs/using/observability/prometheus-grafana.drawio: -------------------------------------------------------------------------------- 1 | 5VhdU+IwFP01PLrTUlrwUVCXcXSHHd3V3bfQpm007e2kqYC/fm9oWloDyI64oltmIDn5aHLO/UjoOKNk/lWQLL6CgPJO1wrmHee00+0O+l38VsBCA65XApFgQQnZK+CaPVENWhotWEDzVkcJwCXL2qAPaUp92cKIEDBrdwuBt9+akYgawLVPuIneskDGGrW941XDmLIo1q8edPtlQ0KqznoneUwCmDUg56zjjASALEvJfES54q7ipRx3vqG1XpigqdxlwP2dPxiTU+/bRX6UfR/Pfl/0n460Oo+EF3rDP4kguQS9ZrmoiBBQpAFVc1kdZziLmaTXGfFV6wyVRyyWCceajcVHKiRDEk84i1LEJKgOIeN8BBzEckYndNUH8VwKeKCNFm/5qBGQygZePojrVeNb6HwjHXZNMhonhYRKscAuekBPy7JoV2cNkfsaixv62pYGiTasqJ55xT0WNP1/IYVjSHGF8zI/R3B8czPBH5oGGTDc36vEeSZDQOgg9NfK4A/oNNwP3a7bonuwhu7eGrp7b8W2a7A9EWrqmBb5p7d9xz004/e2GD9GI6Gi9Mczesc7MKvvGzRj1g5JSj69yXveoZn8YHPqtQKSx1MgIviARu/1D8zobTO4XAI8qNBC5FayaYBnQV0FIWOIICX8bIUO23Ks+uALMi3CPZVyoQ+2pEB5WxLROZN3avgXV9d+6clU+XTerCwalQkVDOmhQmPl2tWCt8uG+4NC+HQLX/pkK4mIqHzJgE0zEJQTyR7b69i7qMeGpj9yJOO5gnjozlSxSPiJL5W518Hqkkwpn0DOJAMVtKYgJSTYgauGIfEfoqW6zRC2fDYGvIasUEjOUnSx6lZi7ce5+k47jHUd07ucNc7lvZlzWYYQphulwYm6imEthVS5jXK5pdfYbdp2tWKTn8b+3TX7r7CdjVO/YVIetyv662C2eFavpiida1If0s9fmqgaWHqbMXCpSb3NV8hk/08yucd7kqme6F/J1DNkal5LPI48DacY5rxIlbKCc5XGkuqs/H75y25mrzqXrc9fe8xVzo65ynvPXGWbl8362G2FVPoxXSNuLaoVogmY7Vvuq59fd+99zyhYXf1xVzr/6t9P5+wP -------------------------------------------------------------------------------- /docs/using/observability/index.md: -------------------------------------------------------------------------------- 1 | Observe how Varasto is doing from key metrics like: 2 | 3 | - Per-volume: 4 | * Used and free space 5 | * Blob count 6 | * Read/write requests, errors and byte counts 7 | * Replication progress 8 | - HTTP server request counts partitioned over status code and method 9 | - Scheduled job durations 10 | 11 | 12 | ![](grafana-metrics.png) 13 | 14 | 15 | Prometheus 16 | ---------- 17 | 18 | Varasto has [Prometheus](https://prometheus.io/)-compatible metrics. Prometheus does not 19 | offer built-in dashboards, but there are many dashboarding solutions that can read data off 20 | of Prometheus. 21 | 22 | 23 | Grafana 24 | ------- 25 | 26 | [Grafana](https://grafana.com/) is our recommendation for building dashboards from metrics 27 | in Prometheus (the example screenshot is from Grafana). 28 | 29 | Here's how the whole looks: 30 | 31 | ![](prometheus-grafana.png) 32 | 33 | 34 | Embed Grafana in Varasto 35 | ------------------------ 36 | 37 | We also support embedding Grafana dashboard in Varasto's admin panel for quick access: 38 | 39 | 40 | 41 | 42 | !!! tip 43 | You need to enable 44 | [allow_embedding](https://grafana.com/docs/grafana/latest/installation/configuration/#allow-embedding) 45 | in Grafana's config. 46 | -------------------------------------------------------------------------------- /pkg/stoserver/stohealth/healthinterface_test.go: -------------------------------------------------------------------------------- 1 | package stohealth 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/function61/gokit/assert" 8 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 9 | ) 10 | 11 | func TestBasic(t *testing.T) { 12 | g, _ := getTestGraph() 13 | 14 | jsonBytes, _ := json.MarshalIndent(g, "", " ") 15 | 16 | assert.EqualString(t, string(jsonBytes), `{ 17 | "Children": [ 18 | { 19 | "Children": [ 20 | { 21 | "Children": [], 22 | "Details": "", 23 | "Health": "pass", 24 | "Kind": null, 25 | "Title": "Dummy 1" 26 | }, 27 | { 28 | "Children": [], 29 | "Details": "", 30 | "Health": "warn", 31 | "Kind": null, 32 | "Title": "Dummy 2" 33 | } 34 | ], 35 | "Details": "", 36 | "Health": "warn", 37 | "Kind": null, 38 | "Title": "SMART" 39 | } 40 | ], 41 | "Details": "", 42 | "Health": "warn", 43 | "Kind": null, 44 | "Title": "Varasto" 45 | }`) 46 | } 47 | 48 | func getTestGraph() (*stoservertypes.Health, error) { 49 | root := NewHealthFolder( 50 | "Varasto", 51 | nil, 52 | NewHealthFolder("SMART", 53 | nil, 54 | NewStaticHealthNode("Dummy 1", stoservertypes.HealthStatusPass, "", nil), 55 | NewStaticHealthNode("Dummy 2", stoservertypes.HealthStatusWarn, "", nil))) 56 | 57 | return root.CheckHealth() 58 | } 59 | -------------------------------------------------------------------------------- /pkg/stoserver/server_test.go: -------------------------------------------------------------------------------- 1 | package stoserver 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | "github.com/function61/gokit/cryptoutil" 8 | ) 9 | 10 | func TestMkWrappedKeypair(t *testing.T) { 11 | certPem := `-----BEGIN CERTIFICATE----- 12 | MIIBjTCCATOgAwIBAgIQcA8FmTXCBv38IhLOKVOiOzAKBggqhkjOPQQDAjAkMSIw 13 | IAYDVQQKExlFdmVudCBIb3Jpem9uIGludGVybmFsIENBMB4XDTE5MTIxNTE0MDIy 14 | NVoXDTM5MTIxNTE0MDIyNVowJDEiMCAGA1UEChMZRXZlbnQgSG9yaXpvbiBpbnRl 15 | cm5hbCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABMREdYpF+A54nz0SdYvU 16 | WvR/SBquwmimwRYKHwKOXOt5fqWHfv+2ayvAV/9v8eLQSUdcPGMinqW28ELV981S 17 | O5ujRzBFMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYB 18 | BQUHAwEwFAYDVR0RBA0wC4IJbG9jYWxob3N0MAoGCCqGSM49BAMCA0gAMEUCIDw1 19 | HpwVUGEkVsDp0Kl556XftcOJcKLkjgeMLERt4TUiAiEAqJZvB40TFLrAShtovcc5 20 | /FwjIqnJX8kT6Pox3QYSspI= 21 | -----END CERTIFICATE-----` 22 | 23 | //nolint:gosec // intentionally insecure key 24 | keyPem := `-----BEGIN EC PRIVATE KEY----- 25 | MHcCAQEEINTwTX5Xt26rkBv44y2dXEwetcT54HZr6v20FBFhW7hboAoGCCqGSM49 26 | AwEHoUQDQgAExER1ikX4DnifPRJ1i9Ra9H9IGq7CaKbBFgofAo5c63l+pYd+/7Zr 27 | K8BX/2/x4tBJR1w8YyKepbbwQtX3zVI7mw== 28 | -----END EC PRIVATE KEY-----` 29 | 30 | cw, err := mkWrappedKeypair([]byte(certPem), []byte(keyPem)) 31 | assert.Assert(t, err == nil) 32 | 33 | assert.EqualString(t, cryptoutil.Identity(cw.cert), "localhost") 34 | assert.EqualString(t, cryptoutil.Issuer(cw.cert), "Event Horizon internal CA") 35 | } 36 | -------------------------------------------------------------------------------- /frontend/pages/RootRedirectPage.tsx: -------------------------------------------------------------------------------- 1 | import { navigateTo } from 'f61ui/browserutils'; 2 | import * as React from 'react'; 3 | import * as r from 'generated/frontend_uiroutes'; 4 | import { Result } from 'f61ui/component/result'; 5 | import { RootFolderId } from 'generated/stoserver/stoservertypes_types'; 6 | import { getKeyEncryptionKeys } from 'generated/stoserver/stoservertypes_endpoints'; 7 | 8 | interface RootRedirectPageState { 9 | setupCheck: Result; 10 | } 11 | 12 | // redirects user to either the app or to the setup wizard (if things have not been set up) 13 | export default class RootRedirectPage extends React.Component<{}, RootRedirectPageState> { 14 | state: RootRedirectPageState = { 15 | setupCheck: new Result((_) => { 16 | this.setState({ setupCheck: _ }); 17 | }), 18 | }; 19 | 20 | componentDidMount() { 21 | this.fetchData(); 22 | } 23 | 24 | render() { 25 | // user should not be able to see this, or at least it should flash really fast 26 | return this.state.setupCheck.draw(() =>

Redirecting

); 27 | } 28 | 29 | private fetchData() { 30 | // as a setup check, see if user has KEKs set up 31 | this.state.setupCheck.load(async () => { 32 | if ((await getKeyEncryptionKeys()).length > 0) { 33 | // Varasto has been set up 34 | navigateTo(r.browseUrl({ dir: RootFolderId })); 35 | } else { 36 | // Varasto not set up 37 | navigateTo(r.gettingStartedUrl({ section: 'welcome' })); 38 | } 39 | }); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /bin/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | if [ ! -L "/usr/local/bin/sto" ]; then 4 | ln -s /workspace/rel/sto_linux-amd64 /usr/local/bin/sto 5 | fi 6 | 7 | function docsDeployerSpec { 8 | if [ -n "${FASTBUILD:-}" ]; then 9 | return # skip non-essential step 10 | fi 11 | 12 | cd misc/docs-website-deployerspec/ 13 | 14 | deployer package "$FRIENDLY_REV_ID" ../../rel/docs-website-deployerspec.zip 15 | } 16 | 17 | function updateServer { 18 | if [ -n "${FASTBUILD:-}" ]; then 19 | return # skip non-essential step 20 | fi 21 | 22 | cd misc/varasto-updateserver/ 23 | 24 | # this is a file that will be deployed (by function61/deployer) at: 25 | # https://function61.com/varasto/updateserver/latest-version.json 26 | # this won't be deployed on all commits however - deployments will be initiated manually 27 | # when we want users to update to the released version 28 | echo -n "{\"LatestVersion\": \"$FRIENDLY_REV_ID\"}" > latest-version.json 29 | 30 | tar -czf "updateserver.tar.gz" latest-version.json 31 | 32 | # so it won't end up in deployerspec zip 33 | rm latest-version.json 34 | 35 | deployer package "$FRIENDLY_REV_ID" ../../rel/updateserver-deployerspec.zip 36 | 37 | # clean up generated files 38 | rm updateserver.tar.gz 39 | } 40 | 41 | # make sure parent dir exits, under which FUSE projector will mount itself 42 | mkdir -p /mnt/stofuse 43 | 44 | build-go-project.sh --directory=cmd/sto/ --binary-basename=sto 45 | 46 | (docsDeployerSpec) 47 | 48 | (updateServer) 49 | -------------------------------------------------------------------------------- /pkg/stodupremover/entrypoint.go: -------------------------------------------------------------------------------- 1 | package stodupremover 2 | 3 | import ( 4 | "github.com/function61/gokit/osutil" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | func Entrypoint() *cobra.Command { 9 | dupremover := &cobra.Command{ 10 | Use: "dupremover", 11 | Short: `Remove duplicate files (files already in Varasto server)`, 12 | } 13 | 14 | dupremover.AddCommand(&cobra.Command{ 15 | Use: "refresh-db", 16 | Short: `Refresh the duplicate detector database`, 17 | Args: cobra.NoArgs, 18 | Run: func(cmd *cobra.Command, args []string) { 19 | osutil.ExitIfError(refreshDatabase( 20 | osutil.CancelOnInterruptOrTerminate(nil))) 21 | }, 22 | }) 23 | 24 | dupremover.AddCommand(scanEntry()) 25 | 26 | dupremover.AddCommand(removeEmptyDirsEntrypoint()) 27 | 28 | return dupremover 29 | } 30 | 31 | func scanEntry() *cobra.Command { 32 | acceptOutdatedDb := false 33 | removeDuplicates := false 34 | 35 | cmd := &cobra.Command{ 36 | Use: "scan", 37 | Short: `Scan current directory tree for items existing in Varasto`, 38 | Args: cobra.NoArgs, 39 | Run: func(cmd *cobra.Command, args []string) { 40 | osutil.ExitIfError(scan(removeDuplicates, acceptOutdatedDb)) 41 | }, 42 | } 43 | 44 | cmd.Flags().BoolVarP(&acceptOutdatedDb, "accept-outdated-db", "a", acceptOutdatedDb, "Accept dangerously outdated DB") 45 | cmd.Flags().BoolVarP(&removeDuplicates, "rm", "", removeDuplicates, "Actually remove duplicates, instead of only reporting") 46 | 47 | return cmd 48 | } 49 | -------------------------------------------------------------------------------- /pkg/stomvu/photorenamer.go: -------------------------------------------------------------------------------- 1 | package stomvu 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/function61/gokit/osutil" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | type PhotoResult struct { 11 | DateString string 12 | } 13 | 14 | func (p *PhotoResult) String() string { 15 | // - 16 | return p.DateString[0:4] + "-" + p.DateString[4:6] + " - Unsorted" 17 | } 18 | 19 | // IMG_20180526_151345.jpg 20 | // VID_20180526_151345.mp4 21 | var detectPhotoVideoDateRe = regexp.MustCompile("^((?:IMG|VID)_)?([0-9]{8})_") 22 | 23 | func detectPhotoVideoDate(filename string) *PhotoResult { 24 | result := detectPhotoVideoDateRe.FindStringSubmatch(filename) 25 | if result == nil { 26 | return nil 27 | } 28 | 29 | return &PhotoResult{ 30 | DateString: result[2], 31 | } 32 | } 33 | 34 | func PhotoOrVideoDateFromFilename(name string) string { 35 | result := detectPhotoVideoDate(name) 36 | if result == nil { 37 | return "" 38 | } 39 | 40 | return result.String() 41 | } 42 | 43 | func photoEntrypoint() *cobra.Command { 44 | doIt := false 45 | 46 | cmd := &cobra.Command{ 47 | Use: "photo", 48 | Short: "Organize photos & videos to '- - Unsorted' subdirectories", 49 | Args: cobra.NoArgs, 50 | Run: func(cmd *cobra.Command, args []string) { 51 | osutil.ExitIfError(runOrExplainPlan(PhotoOrVideoDateFromFilename, doIt)) 52 | }, 53 | } 54 | 55 | cmd.Flags().BoolVarP(&doIt, "do", "", doIt, "Whether to execute the plan or run a dry run") 56 | 57 | return cmd 58 | } 59 | -------------------------------------------------------------------------------- /pkg/stofuse/utils_test.go: -------------------------------------------------------------------------------- 1 | package stofuse 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/function61/gokit/assert" 9 | "github.com/function61/varasto/pkg/stotypes" 10 | ) 11 | 12 | func TestAlignReads(t *testing.T) { 13 | serialize := func(brs []alignedBlobRead) string { 14 | lines := []string{} 15 | 16 | for _, br := range brs { 17 | lines = append(lines, fmt.Sprintf( 18 | "blob<%d> offset<%d> len<%d>", 19 | br.blobIdx, 20 | br.offsetInBlob, 21 | br.lenInBlob)) 22 | } 23 | 24 | return strings.Join(lines, "\n") 25 | } 26 | 27 | assert.EqualString(t, serialize(alignReads(0, 100)), "blob<0> offset<0> len<100>") 28 | assert.EqualString(t, serialize(alignReads(stotypes.BlobSize-1, 2)), "blob<0> offset<4194303> len<1>\nblob<1> offset<0> len<1>") 29 | assert.EqualString(t, serialize(alignReads(stotypes.BlobSize-1, stotypes.BlobSize+1)), "blob<0> offset<4194303> len<1>\nblob<1> offset<0> len<4194304>") 30 | assert.EqualString(t, serialize(alignReads(stotypes.BlobSize-1, stotypes.BlobSize+2)), "blob<0> offset<4194303> len<1>\nblob<1> offset<0> len<4194304>\nblob<2> offset<0> len<1>") 31 | } 32 | 33 | func TestMkFsSafe(t *testing.T) { 34 | assert.EqualString(t, mkFsSafe("Police Academy: Mission to Moscow"), "Police Academy_ Mission to Moscow") 35 | 36 | assert.EqualString( 37 | t, 38 | mkFsSafe(`All special chars = \ and / and : and * and ? and " and < and > and |`), 39 | "All special chars = _ and _ and _ and _ and _ and _ and _ and _ and _") 40 | } 41 | -------------------------------------------------------------------------------- /docs/storage/local-fs/architecture.drawio: -------------------------------------------------------------------------------- 1 | 7VnbcpswEP0aHpMxFvjyGN/StGmnncykzaMCAtQIRIXAuF9fyYibRdy0Bced1n4wOlqt0DlarTYxwDLMrxmMg/fURcQYj9zcACtjPDZNyxY/EtkVyHQ8KgCfYVcZ1cAd/o4UWJql2EVJy5BTSjiO26BDowg5vIVBxui2beZR0p41hj7SgDsHEh39jF0elOsajeqONwj7gZp6ZquOEJbGCkgC6NJtAwJrAywZpbx4CvMlIpK8kpdi3OaZ3urFGIr4Swbcf8quZ/HtW7K5WaySD+bymt1cKC8ZJKla8JKGccoRUy/NdyUTjKaRi6SzkQEW2wBzdBdDR/ZuhfYCC3hIRMsUjxliHAsWrwj2I4FxKg3UZKIP5c+uwqy4EZsK0RBxthMm5YCpolPtJ9NS7W2tjgUUFjSEAXMFQrUj/Mp3TZp4ULz9AodjjcMVTp4EEsFQAFeGFGbyLZVCLzZywrLRL8ePlHMaig4PE7KkhLK9U+DZ8ivwhDP6hBo9k/1HjqARb+DFR+BQuWYFj30IOB61BQQdAlYiNwU0J0MJaHUIyMRhQoXvA/mE/wwymHB64Z1eSReimed0KunM0KPXk0Km3VbI7lDI6lBoNpRAtibQfSGCnCyOhziq+ouhAWKmio+GIqAzZsZDSTI5IklGSRqKpD28LCcKCNCmv7qcNOk3O+ifDsW+CTT6Q0Eu1yhHrrjNqCZlPKA+jSBZ1+iiLUptc0sl4XspviLOd+pqBlMhcUsolGP+RQ6/tFXrodGzypXnfWNXNiLBQmOQbD40++ph+1Y5rlifXNRxKQUHNGUOOkLhtLDjkPmIH7Gbd28NhgjkOGu/R+86TzWZN8rrb8dVH/FgH8QD0OOh6zSyhgqHmUbTO+y9Ok3WudE0P3Jmi4P1T0/sPnaWdXA5nOiUzU9JmalfPTSSYoojUTCtM7HIRHFRFXuSNxcmQUVig7B20opohPQMZwHLsuQZCZO4qG89nEtnCwIfEflIE8wxlQnSQfItGpnz9sCgulV2p9byrl/5oSknOBLvV1bWo2JhsVx3mPuy4L/MMvfSleXOIFdPq6M4ALauvz2Y/vo950hxtz91/uXizjq/4s7Uc+hLqrun00t5msusdXbVnamn73PLS9bZ5SU9l//PS8PlpcOgefW8VNb1f2sBaPRXyKnT46eFXHHM9F/JiWb9R/x9X+NfIWD9Aw== -------------------------------------------------------------------------------- /docs/content/generic-files/index.md: -------------------------------------------------------------------------------- 1 | Varasto is great for storing any type of file, whether they're large or small files. 2 | 3 | 4 | Screenshot from web UI 5 | ---------------------- 6 | 7 | ![](screenshot-web-ui.png) 8 | 9 | 10 | Automatic backups 11 | ----------------- 12 | 13 | Each time you change a file in Varasto, its previous version is also stored so you can 14 | always recover from accidental modifications or deletions. 15 | 16 | Varasto breaks files down to smaller chunks so if you only change part of a large file, the 17 | entire file will not be stored twice. 18 | 19 | 20 | Software projects 21 | ----------------- 22 | 23 | Even though Varasto collections work similar to a Git repository (changesets, cloning etc.), 24 | Varasto is not meant to replace software revision control - they can coexist peacefully and 25 | augment each other. 26 | 27 | Some people think of e.g. GitHub as a type of backup - and it kind of is, because if your 28 | drive fails, a copy exists at a remote server that you can download ("clone") to your new 29 | drive again. But there is a non-insignificant window of time where you can lose data: 30 | 31 | ![Varasto use with Git repos](varasto-with-git-repos.png) 32 | 33 | To summarize: 34 | 35 | | System | Commit/push interval | Window of losing data | 36 | |---------|--------------------------|-------------------------------| 37 | | Git | Whenever user decides to | Undefined - may be large | 38 | | Varasto | Automatic (maybe hourly) | Hour (or whatever configured) | 39 | -------------------------------------------------------------------------------- /docs/using/replication-policies/index.md: -------------------------------------------------------------------------------- 1 | What does it do? 2 | ---------------- 3 | 4 | A replication policy specifies on which volumes a collection's files should be stored on. 5 | 6 | ![Drawing](replication-policies.png) 7 | 8 | NOTE: Above drawing's policy differs from screenshots below. 9 | 10 | 11 | Screenshots 12 | ----------- 13 | 14 | ### Replication policies 15 | 16 | ![](screenshot.png) 17 | 18 | 19 | ### Replication queue 20 | 21 | ![](replication-queue-status.png) 22 | 23 | 24 | Reconciliation process 25 | ---------------------- 26 | 27 | When you make changes to your policies, your existing data can conflict with the desired 28 | state of the updated policy. This will cause policy conflicts, which Varasto will fix by 29 | replicating after confirming the fixes with you. 30 | 31 | This process is currently better explained in the 32 | [moving large amounts of data](../moving-data/index.md) -guide. 33 | 34 | 35 | Do I need multiple policies? 36 | ---------------------------- 37 | 38 | If all of your data is equally important, then you need only one policy. 39 | 40 | If you have data with varying types of importance, you can have for example: 41 | 42 | - A default replication policy that saves data to three disks 43 | * The default policy is specified for root directory and is used for all file collections 44 | unless a directory subtree explicitly specifies a different policy 45 | - A policy for one subdirectory three (e.g. "All movies") for less important 46 | data that will be saved onto just one disk 47 | 48 | -------------------------------------------------------------------------------- /pkg/stoclient/bulkuploadscript.go: -------------------------------------------------------------------------------- 1 | package stoclient 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/function61/gokit/osutil" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func bulkUploadScriptEntrypoint() *cobra.Command { 14 | rm := false 15 | 16 | cmd := &cobra.Command{ 17 | Use: "bulk [parentDirectory]", 18 | Short: "Generates a shell script to adopt & push all subdirectories as collections", 19 | Args: cobra.ExactArgs(1), 20 | Run: func(cmd *cobra.Command, args []string) { 21 | osutil.ExitIfError(bulkUploadScriptGenerate(args[0], rm, os.Stdout)) 22 | }, 23 | } 24 | 25 | cmd.Flags().BoolVarP(&rm, "rm", "", rm, "Whether to remove uploaded collections") 26 | 27 | return cmd 28 | } 29 | 30 | func bulkUploadScriptGenerate(parentDirectory string, rm bool, out io.Writer) error { 31 | maybeRm := "" 32 | if rm { 33 | maybeRm = `sto rm "$dir"` 34 | } 35 | 36 | if _, err := fmt.Fprintf(out, `set -eu 37 | 38 | parentDirId="%s" 39 | 40 | one() { 41 | local dir="$1" 42 | 43 | (cd "$dir" && sto adopt -- "$parentDirId" && sto push) 44 | 45 | %s 46 | } 47 | 48 | `, parentDirectory, maybeRm); err != nil { 49 | return err 50 | } 51 | 52 | dentries, err := ioutil.ReadDir(".") 53 | if err != nil { 54 | return err 55 | } 56 | 57 | for _, dentry := range dentries { 58 | if !dentry.IsDir() { 59 | continue 60 | } 61 | 62 | if _, err := fmt.Fprintf(out, "one \"%s\"\n", dentry.Name()); err != nil { 63 | return err 64 | } 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/stofuse/types.go: -------------------------------------------------------------------------------- 1 | package stofuse 2 | 3 | import ( 4 | "context" 5 | stdfs "io/fs" 6 | 7 | "bazil.org/fuse" 8 | "bazil.org/fuse/fs" 9 | ) 10 | 11 | type sigFabric struct { 12 | unmountAll chan interface{} 13 | } 14 | 15 | func newSigs() *sigFabric { 16 | return &sigFabric{ 17 | unmountAll: make(chan interface{}), 18 | } 19 | } 20 | 21 | // couples together a directory entry ("node name") and a node 22 | type stoEntry struct { 23 | dirent fuse.Dirent // name found from here 24 | node fs.Node 25 | } 26 | 27 | func newStoEntry(dirent fuse.Dirent, node fs.Node) stoEntry { 28 | return stoEntry{dirent, node} 29 | } 30 | 31 | type stoSymlink struct { 32 | target string 33 | inode uint64 34 | } 35 | 36 | func (s *stoSymlink) MakeDirent(name string) fuse.Dirent { 37 | return fuse.Dirent{ 38 | Inode: s.inode, 39 | Name: name, 40 | Type: fuse.DT_Link, 41 | } 42 | } 43 | 44 | func (s *stoSymlink) MakeStoEntry(name string) stoEntry { 45 | return stoEntry{s.MakeDirent(name), s} 46 | } 47 | 48 | func newStoSymlink(target string) *stoSymlink { 49 | return &stoSymlink{ 50 | target: target, 51 | inode: nextInode(), 52 | } 53 | } 54 | 55 | func (s *stoSymlink) Readlink(_ context.Context, req *fuse.ReadlinkRequest) (string, error) { 56 | return s.target, nil 57 | } 58 | 59 | func (s *stoSymlink) Attr(_ context.Context, attr *fuse.Attr) error { 60 | attr.Inode = s.inode 61 | attr.Mode = stdfs.ModeSymlink | 0555 62 | 63 | return nil 64 | } 65 | 66 | var _ interface { 67 | fs.NodeReadlinker 68 | fs.Node 69 | } = (*stoSymlink)(nil) 70 | -------------------------------------------------------------------------------- /pkg/stodupremover/removeemptydirs.go: -------------------------------------------------------------------------------- 1 | package stodupremover 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/function61/gokit/osutil" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func removeEmptyDirs(path string, dry bool) error { 14 | return filepath.Walk(path, func(path string, info os.FileInfo, err error) error { 15 | if err != nil { 16 | return err 17 | } 18 | 19 | if info.IsDir() { 20 | files, err := ioutil.ReadDir(path) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | if len(files) == 0 { 26 | if dry { 27 | fmt.Printf("would remove empty dir: %s\n", path) 28 | } else { 29 | fmt.Printf("removing empty dir: %s\n", path) 30 | 31 | // TODO: not sure if Walk() will be ok with removing currently walking item? 32 | if err := os.Remove(path); err != nil { 33 | return fmt.Errorf("Rmdir: %v", err) 34 | } 35 | } 36 | } 37 | } 38 | 39 | return nil 40 | }) 41 | } 42 | 43 | func removeEmptyDirsEntrypoint() *cobra.Command { 44 | really := false 45 | 46 | cmd := &cobra.Command{ 47 | Use: "removeemptydirs [path]", 48 | Short: "Remove empty leaf directories (run this multiple times to remove one level at a time)", 49 | Args: cobra.ExactArgs(1), 50 | Run: func(cmd *cobra.Command, args []string) { 51 | osutil.ExitIfError(removeEmptyDirs(args[0], !really)) 52 | }, 53 | } 54 | 55 | cmd.Flags().BoolVarP(&really, "really", "", really, "Really remove the files. Without this a dry run is performed.") 56 | 57 | return cmd 58 | } 59 | -------------------------------------------------------------------------------- /pkg/restartcontroller/restartcontroller.go: -------------------------------------------------------------------------------- 1 | // Wrapper for running a restartable fn. it gets its restart signal via context cancellation 2 | package restartcontroller 3 | 4 | import ( 5 | "context" 6 | "errors" 7 | "log" 8 | 9 | "github.com/function61/gokit/logex" 10 | ) 11 | 12 | type Controller struct { 13 | restart chan interface{} 14 | logl *logex.Leveled 15 | } 16 | 17 | func New(logger *log.Logger) *Controller { 18 | return &Controller{ 19 | restart: make(chan interface{}), 20 | logl: logex.Levels(logger), 21 | } 22 | } 23 | 24 | // returns immediately 25 | func (r *Controller) Restart() error { 26 | select { 27 | case r.restart <- nil: 28 | return nil 29 | default: 30 | return errors.New("unable to send restart signal - runner busy or exited?") 31 | } 32 | } 33 | 34 | func (r *Controller) Run(ctx context.Context, run func(ctx context.Context) error) error { 35 | stopped := make(chan error) 36 | 37 | var cancelFn context.CancelFunc 38 | 39 | start := func() { 40 | var subCtx context.Context 41 | subCtx, cancelFn = context.WithCancel(ctx) 42 | 43 | go func() { 44 | stopped <- run(subCtx) 45 | }() 46 | } 47 | 48 | start() 49 | 50 | for { 51 | select { 52 | case <-r.restart: 53 | r.logl.Info.Println("stopping due to restart request") 54 | 55 | cancelFn() 56 | 57 | if err := <-stopped; err != nil { 58 | r.logl.Error.Printf("stopped but with error (will start anyway): %v", err) 59 | } else { 60 | r.logl.Info.Println("graceful stop; starting") 61 | } 62 | 63 | start() 64 | case err := <-stopped: 65 | return err 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /docs/data-interfaces/network-folders/architecture.xml: -------------------------------------------------------------------------------- 1 | 7Vlbc9o6EP41zHlqxvINeGxo0j6cnskMvaRPHWELo0b2Ulnczq/vKpZvkUNoCTHtlGRAWl1sfd9+q7U88Cbp9q2ky8V7iJkYuE68HXhvBq5LnNEQf7RlV1jGrlMYEslj06k2TPn/rBxprCses7zVUQEIxZdtYwRZxiLVslEpYdPuNgfRvuqSJswyTCMqbOtnHquFsRLHqRveMZ4szKVHgWlIadnZGPIFjWHTMHlXA28iAVRRSrcTJjR4JS7FuOtHWqsbkyxTBw2gd3e3k/zzVzn2NuFb9en71++vyKiYZk3Fyqz4Y87kPznabib4BVKvBWYcAUEW2ZpHzKxH7UqQJKyymOnrOAPvcrPgik2XNNKtG3QLtC1UKrBGsLhmUnEE+LXgSYa2GSgFKTbMuRATEHhFPak3D/Qf2nMl4Y41WsL7jx4BmWrYiw/azZrwQmz7KFqk4gCdl0HKlNxhFzPAN6wZt/VKvjcNJxgb26LBPynppsbxkmrqmhssGHp+hiqLqQmky5VimiHyvJwoWJ45Ib7NR+B08IHMnYgPWzlTms7oCxARUzaaR51ERCM2mz8P4EHYAnzcIYAg6BAAORXg7j4BuH+8AELHf0oBrjN8SQV4FiGfqKS5Ak0Ek2tNzDGk9OL2oT+8CFpAk5FbWprO73ZEm/BkwT+woGQxJiqmClItIIGMiqvaetkGu+7zL2jvvof4G1NqZ7IuukLmWgSwLVe3evhFYGpfzGS6/GbbrOzKSobrbQzS1S/lfLpSD7uvleOK9elF7ScOMYCVjNgesIxOFJUJU/tAfcQTJBNU8XX7Rp6dUn+Peq4/Tq/04iV8w6QWjtTRGW0qvtPeVYh/oLJGwYloCCwarjHXzXe5YukfAzvuRA+CWtd27nflsyfbzonXa0hrBrQqvD0R0kgzoDXi2+lDWnhoSHO6/eBlQlpoaUlPg2t1r9dlbDtGUc+hBP+hEqpw09/mbme20/eXRfxXEIH4u/cfLJTxkUIxbvLKufCDcm86WDxmuhvg2uerLjCf53gzD92kuuqve87Ychx9eoOWVc6zBH8zpjYg79ARirMvJvPjJNjL9jV8qFnP612zjgX9fwXWFdIDNxR4G5czXUp0aRBM8D8yT60Ey+cSFi2EzyAq+n/zg4PDHinPxp+Ke2Gf+UF5l81k+/5J57eLSL7bfo4Z2o8xow6x+CcTy9BC9t2HDzd9Koj8ioKc/hR0aIrtPZU4EK986XW+iQOxU/X69CESXMNnbV+CzyQ1ztrnZuWXQ6oDumHHAV3X25ng5/WH1fodXQF//abTu/oB -------------------------------------------------------------------------------- /frontend/component/collectiondropdown.tsx: -------------------------------------------------------------------------------- 1 | import { tmdbMovieAutocomplete, igdbAutocomplete } from 'component/autocompletes'; 2 | import { CommandLink } from 'f61ui/component/CommandButton'; 3 | import { Dropdown } from 'f61ui/component/dropdown'; 4 | import { 5 | CollectionChangeDescription, 6 | CollectionChangeSensitivity, 7 | CollectionDelete, 8 | CollectionMove, 9 | CollectionTriggerMediaScan, 10 | CollectionPullTmdbMetadata, 11 | CollectionPullIgdbMetadata, 12 | CollectionRename, 13 | CollectionTag, 14 | } from 'generated/stoserver/stoservertypes_commands'; 15 | import { CollectionSubset } from 'generated/stoserver/stoservertypes_types'; 16 | import * as React from 'react'; 17 | 18 | export function collectionDropdown(coll: CollectionSubset) { 19 | return ( 20 | 21 | 22 | 23 | 24 | 25 | 28 | 31 | 32 | 33 | 34 | 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /pkg/stomvu/filemodificationtime.go: -------------------------------------------------------------------------------- 1 | package stomvu 2 | 3 | // Sorts files based on their modification time. 4 | 5 | import ( 6 | "os" 7 | 8 | "github.com/function61/gokit/osutil" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | func fileModificationTimeEntrypoint() *cobra.Command { 13 | doIt := false 14 | yearMonth := false 15 | 16 | cmd := &cobra.Command{ 17 | Use: "file-modtime", 18 | Short: "Sort files based on their modification time", 19 | Args: cobra.NoArgs, 20 | Run: func(cmd *cobra.Command, args []string) { 21 | osutil.ExitIfError(func() error { 22 | if yearMonth { 23 | return runOrExplainPlan(fileModificationTimeYearMonth, doIt) 24 | } else { 25 | return runOrExplainPlan(fileModificationTimeYear, doIt) 26 | } 27 | }()) 28 | }, 29 | } 30 | 31 | cmd.Flags().BoolVarP(&doIt, "do", "", doIt, "Whether to execute the plan or run a dry run") 32 | cmd.Flags().BoolVarP(&yearMonth, "year-month", "", yearMonth, "Sort //") 33 | 34 | return cmd 35 | } 36 | 37 | // sort / 38 | func fileModificationTimeYear(name string) string { 39 | return fileModificationTimeInternal(name, "2006") 40 | } 41 | 42 | // sort // 43 | func fileModificationTimeYearMonth(name string) string { 44 | return fileModificationTimeInternal(name, "2006/01") 45 | } 46 | 47 | func fileModificationTimeInternal(name string, datePattern string) string { 48 | stat, err := os.Stat(name) 49 | if err != nil { 50 | panic(err) 51 | } 52 | 53 | if stat.IsDir() { // directories don't make sense here 54 | return "" 55 | } 56 | 57 | // intentionally not UTC 58 | return stat.ModTime().Format(datePattern) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/blobstore/googledriveblobstore/googledrive_test.go: -------------------------------------------------------------------------------- 1 | package googledriveblobstore 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | "github.com/function61/varasto/pkg/stotypes" 8 | "golang.org/x/oauth2" 9 | ) 10 | 11 | func TestToGoogleDriveName(t *testing.T) { 12 | ref, _ := stotypes.BlobRefFromHex("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592") 13 | 14 | assert.EqualString(t, toGoogleDriveName(*ref), "16j7swfXgJRpypq8sAguT41WUeRtPNt2LQLQvzfJ5ZI") 15 | } 16 | 17 | func TestSerializeAndDeserializeConfig(t *testing.T) { 18 | serialized, err := (&Config{ 19 | VarastoDirectoryId: "dummyDirId", 20 | ClientId: "dummyClientId", 21 | ClientSecret: "dummyClientSecret", 22 | Token: &oauth2.Token{}, 23 | }).Serialize() 24 | assert.Assert(t, err == nil) 25 | 26 | assert.EqualString(t, serialized, `{"directory_id":"dummyDirId","oauth2_client_id":"dummyClientId","oauth2_client_secret":"dummyClientSecret","oauth2_token":{"access_token":"","expiry":"0001-01-01T00:00:00Z"}}`) 27 | 28 | conf, err := deserializeConfig(serialized) 29 | assert.Assert(t, err == nil) 30 | 31 | assert.EqualString(t, conf.VarastoDirectoryId, "dummyDirId") 32 | assert.EqualString(t, conf.ClientId, "dummyClientId") 33 | assert.EqualString(t, conf.ClientSecret, "dummyClientSecret") 34 | 35 | oauth2Conf := Oauth2Config(conf.ClientId, conf.ClientSecret) 36 | 37 | assert.EqualString(t, Oauth2AuthCodeUrl(oauth2Conf), "https://accounts.google.com/o/oauth2/auth?access_type=offline&client_id=dummyClientId&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&response_type=code&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&state=state-token") 38 | } 39 | -------------------------------------------------------------------------------- /pkg/stoutils/utils.go: -------------------------------------------------------------------------------- 1 | package stoutils 2 | 3 | import ( 4 | "io" 5 | "path" 6 | "strings" 7 | 8 | "github.com/function61/gokit/cryptorandombytes" 9 | "github.com/function61/gokit/hashverifyreader" 10 | "github.com/function61/varasto/pkg/stotypes" 11 | "github.com/minio/sha256-simd" 12 | ) 13 | 14 | // this should not be called from anywhere other than DiskAccessManager and varastoclient 15 | func BlobHashVerifier(reader io.Reader, br stotypes.BlobRef) io.Reader { 16 | return hashverifyreader.New(reader, sha256.New(), br.AsSha256Sum()) 17 | } 18 | 19 | // there's gonna be lots of these 20 | var NewCollectionId = longId 21 | var NewDirectoryId = longId 22 | 23 | // there's going to be comparatively few of these 24 | // (changeset IDs are unique within a collection) 25 | var NewCollectionChangesetId = shortId 26 | var NewVolumeMountId = shortId 27 | var NewVolumeUuid = longId 28 | var NewNodeId = shortId 29 | var NewClientId = shortId 30 | var NewIntegrityVerificationJobId = shortId 31 | var NewReplicationPolicyId = shortId 32 | var NewEncryptionKeyId = longId 33 | var NewKeyEncryptionKeyId = shortId 34 | var NewApiKeySecret = cryptoLongId 35 | 36 | func shortId() string { 37 | return cryptorandombytes.Base64UrlWithoutLeadingDash(3) 38 | } 39 | 40 | func longId() string { 41 | return cryptorandombytes.Base64UrlWithoutLeadingDash(8) 42 | } 43 | 44 | func cryptoLongId() string { 45 | return cryptorandombytes.Base64UrlWithoutLeadingDash(32) 46 | } 47 | 48 | func IsMaybeCompressible(filename string) bool { 49 | switch strings.ToLower(path.Ext(filename)) { 50 | case ".jpg", ".jpeg", ".gif", ".png", ".mp4", ".mkv", ".avi", ".mp3": 51 | return false 52 | default: 53 | return true 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/stodupremover/scanner.go: -------------------------------------------------------------------------------- 1 | package stodupremover 2 | 3 | import ( 4 | "crypto/sha256" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | func scan(removeDuplicates bool, acceptOutdatedDb bool) error { 12 | db, err := loadDatabase(acceptOutdatedDb) 13 | if err != nil { 14 | return err 15 | } 16 | 17 | actioner := func() Actioner { 18 | loggerActioner := &LoggerActioner{} 19 | 20 | if removeDuplicates { 21 | return &TeeActioner{loggerActioner, &RemoveDuplicatesActioner{}} 22 | } else { 23 | return loggerActioner 24 | } 25 | }() 26 | 27 | if err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { 28 | if err != nil { 29 | return err 30 | } 31 | 32 | if info.IsDir() { 33 | return nil 34 | } 35 | 36 | fileContentHashHex, err := hashFileContent(path) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | if duplicateFilename, isDuplicate := db.hashes[fileContentHashHex]; isDuplicate { 42 | if err := actioner.Duplicate(Item{path}, duplicateFilename); err != nil { 43 | return err 44 | } 45 | } else { 46 | if err := actioner.NotDuplicate(Item{path}); err != nil { 47 | return err 48 | } 49 | } 50 | 51 | return nil 52 | }); err != nil { 53 | return err 54 | } 55 | 56 | return actioner.Finish() 57 | } 58 | 59 | // TODO: this is awfully duplicated-like (is there anything we can use in stoclient?) 60 | func hashFileContent(path string) (string, error) { 61 | fil, err := os.Open(path) 62 | if err != nil { 63 | return "", err 64 | } 65 | defer fil.Close() 66 | 67 | fileContentHash := sha256.New() 68 | if _, err := io.Copy(fileContentHash, fil); err != nil { 69 | return "", err 70 | } 71 | 72 | fileContentHashHex := fmt.Sprintf("%x", fileContentHash.Sum(nil)) 73 | 74 | return fileContentHashHex, nil 75 | } 76 | -------------------------------------------------------------------------------- /frontend/pages/LogsPage.tsx: -------------------------------------------------------------------------------- 1 | import { RefreshButton } from 'component/refreshbutton'; 2 | import { Result } from 'f61ui/component/result'; 3 | import { Panel, tableClassStripedHover } from 'f61ui/component/bootstrap'; 4 | import { getLogs } from 'generated/stoserver/stoservertypes_endpoints'; 5 | import { AdminLayout } from 'layout/AdminLayout'; 6 | import * as React from 'react'; 7 | 8 | interface LogsPageState { 9 | logs: Result; 10 | } 11 | 12 | export default class LogsPage extends React.Component<{}, LogsPageState> { 13 | state: LogsPageState = { 14 | logs: new Result((_) => { 15 | this.setState({ logs: _ }); 16 | }), 17 | }; 18 | 19 | componentDidMount() { 20 | this.fetchData(); 21 | } 22 | 23 | componentWillReceiveProps() { 24 | this.fetchData(); 25 | } 26 | 27 | render() { 28 | const [logs, loadingOrError] = this.state.logs.unwrap(); 29 | 30 | return ( 31 | 32 | 33 | { 35 | this.fetchData(); 36 | }} 37 | /> 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | {(logs || []).map((line) => ( 47 | 48 | 49 | 50 | ))} 51 | 52 | 53 | 54 | 55 | 56 | 57 |
Line
{line}
{loadingOrError}
58 | 59 | { 61 | this.fetchData(); 62 | }} 63 | /> 64 | 65 | 66 | ); 67 | } 68 | 69 | private fetchData() { 70 | this.state.logs.loadWhileKeepingOldResult(() => getLogs()); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /pkg/stofuse/restapi.go: -------------------------------------------------------------------------------- 1 | package stofuse 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/function61/gokit/httpauth" 8 | "github.com/function61/gokit/httputils" 9 | "github.com/function61/gokit/logex" 10 | "github.com/function61/gokit/taskrunner" 11 | "github.com/function61/pi-security-module/pkg/httpserver/muxregistrator" 12 | "github.com/function61/varasto/pkg/gokitbp" 13 | "github.com/function61/varasto/pkg/stofuse/stofusetypes" 14 | "github.com/function61/varasto/pkg/stoutils" 15 | "github.com/gorilla/mux" 16 | ) 17 | 18 | type handlers struct { 19 | sigs *sigFabric 20 | } 21 | 22 | func (h *handlers) FuseUnmountAll(rctx *httpauth.RequestContext, w http.ResponseWriter, r *http.Request) { 23 | h.sigs.unmountAll <- nil 24 | } 25 | 26 | func rpcStart(addr string, sigs *sigFabric, tasks *taskrunner.Runner) error { 27 | router := mux.NewRouter() 28 | 29 | var han stofusetypes.HttpHandlers = &handlers{sigs} 30 | 31 | stofusetypes.RegisterRoutes(han, createDummyMiddlewares(), muxregistrator.New(router)) 32 | 33 | listener, err := stoutils.CreateTcpOrDomainSocketListener(addr, logex.Levels(logex.Discard)) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | srv := &http.Server{ 39 | Handler: router, 40 | ReadHeaderTimeout: gokitbp.DefaultReadHeaderTimeout, 41 | } 42 | 43 | tasks.Start("rpc "+addr, func(_ context.Context) error { 44 | return httputils.RemoveGracefulServerClosedError(srv.Serve(listener)) 45 | }) 46 | 47 | tasks.Start("rpcshutdowner", httputils.ServerShutdownTask(srv)) 48 | 49 | return nil 50 | } 51 | 52 | func createDummyMiddlewares() httpauth.MiddlewareChainMap { 53 | return httpauth.MiddlewareChainMap{ 54 | "public": func(w http.ResponseWriter, r *http.Request) *httpauth.RequestContext { 55 | return &httpauth.RequestContext{} 56 | }, 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /bin/codegenerate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/function61/eventkit/codegen" 7 | "github.com/function61/eventkit/codegen/codegentemplates" 8 | "github.com/function61/gokit/dynversion/precompilationversion" 9 | "github.com/function61/gokit/osutil" 10 | ) 11 | 12 | //go:generate go run codegenerate.go 13 | 14 | // FIXME: this is a dirty hack for fixing non-compiling generated code 15 | //go:generate rm ../frontend/generated/stofuse/stofusetypes_endpoints.ts 16 | 17 | func main() { 18 | osutil.ExitIfError(logic()) 19 | } 20 | 21 | func logic() error { 22 | // normalize to root of the project 23 | if err := os.Chdir(".."); err != nil { 24 | return err 25 | } 26 | 27 | modules := []*codegen.Module{ 28 | codegen.NewModule("stomediascanner/stomediascantypes", "pkg/stomediascanner/types.json", "", "", ""), 29 | codegen.NewModule("stoserver/stoservertypes", "pkg/stoserver/stoservertypes/types.json", "", "pkg/stoserver/stoservertypes/commands.json", ""), 30 | codegen.NewModule("stofuse/stofusetypes", "pkg/stofuse/stofusetypes/types.json", "", "", ""), 31 | codegen.NewModule("frontend", "", "", "", "pkg/frontend/ui-routes.json"), 32 | } 33 | 34 | opts := codegen.Opts{ 35 | BackendModulePrefix: "github.com/function61/varasto/pkg/", 36 | FrontendModulePrefix: "generated/", 37 | AutogenerateModuleDocs: true, 38 | } 39 | 40 | if err := codegen.ProcessModules(modules, opts); err != nil { 41 | return err 42 | } 43 | 44 | // PreCompilationVersion = code generation doesn't have access to version via regular method 45 | if err := codegen.ProcessFile( 46 | codegen.Inline("frontend/generated/version.ts", codegentemplates.FrontendVersion), 47 | codegen.NewVersionData(precompilationversion.PreCompilationVersion()), 48 | ); err != nil { 49 | return err 50 | } 51 | 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /frontend/component/tags.tsx: -------------------------------------------------------------------------------- 1 | import { DefaultLabel, Glyphicon } from 'f61ui/component/bootstrap'; 2 | import { CommandIcon } from 'f61ui/component/CommandButton'; 3 | import { CollectionTag, CollectionUntag } from 'generated/stoserver/stoservertypes_commands'; 4 | import { CollectionSubset } from 'generated/stoserver/stoservertypes_types'; 5 | import * as React from 'react'; 6 | 7 | interface CollectionTagViewProps { 8 | collection: CollectionSubset; 9 | } 10 | 11 | export class CollectionTagView extends React.Component { 12 | render() { 13 | const coll = this.props.collection; // shorthand 14 | 15 | return ( 16 | 17 | {coll.Tags.map((tag) => ( 18 | 19 | 20 | 21 |   22 | {tag} 23 | 24 | 25 | ))} 26 | 27 | ); 28 | } 29 | } 30 | 31 | interface CollectionTagEditorProps { 32 | collection: CollectionSubset; 33 | } 34 | 35 | export class CollectionTagEditor extends React.Component { 36 | render() { 37 | const coll = this.props.collection; // shorthand 38 | 39 | return ( 40 | 41 | {coll.Tags.map((tag) => ( 42 | 43 | 44 | 45 |   46 | {tag} 47 |   48 | 51 | 52 | 53 | ))} 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | ); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /frontend/layout/HelpLayout.tsx: -------------------------------------------------------------------------------- 1 | import { getCurrentLocation } from 'f61ui/browserutils'; 2 | import { Panel, GlyphiconIcon } from 'f61ui/component/bootstrap'; 3 | import { Breadcrumb } from 'f61ui/component/breadcrumbtrail'; 4 | import { NavLink, renderNavLink } from 'f61ui/component/navigation'; 5 | import { AppDefaultLayout } from 'layout/appdefaultlayout'; 6 | import * as React from 'react'; 7 | import * as r from 'generated/frontend_uiroutes'; 8 | 9 | interface HelpLayoutProps { 10 | title: string; 11 | breadcrumbs: Breadcrumb[]; 12 | children: React.ReactNode; 13 | } 14 | 15 | export class HelpLayout extends React.Component { 16 | render() { 17 | const currLoc = getCurrentLocation(); 18 | 19 | function mkLink(title: string, icon: GlyphiconIcon, url: string): NavLink { 20 | return { 21 | title, 22 | glyphicon: icon, 23 | url, 24 | active: url === currLoc, 25 | }; 26 | } 27 | 28 | const helpDocs: NavLink[] = [ 29 | mkLink('Getting started', 'home', r.gettingStartedUrl({ section: 'welcome' })), 30 | mkLink('Download client app', 'download-alt', r.downloadClientAppUrl()), 31 | mkLink('Documentation site', 'book', 'https://function61.com/varasto/docs/'), 32 | ]; 33 | 34 | return ( 35 | 43 |
44 | 45 |
    46 | {helpDocs.map(renderNavLink)} 47 |
48 | 49 |
50 |
{this.props.children}
51 | 52 | } 53 | /> 54 | ); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /docs/content/generic-files/varasto-with-git-repos.drawio: -------------------------------------------------------------------------------- 1 | 5Vpbc9soFP41nt19aEYSuvkxdtN2pt2Ztple9mkHS1giwUKLUGzvr1+Q0BWSuI1lJ13nweJwEXzn4ztw4hlYbnZvGczTP2mMyMyx4t0MvJ45jm0FlviSln1tCeygNiQMx6pRZ7jG/6Kmp7KWOEbFoCGnlHCcD40RzTIU8YENMka3w2ZrSoZvzWGCNMN1BIlu/YZjniqr7c+7incIJ6l6deio9W1g01itpEhhTLc9E7iagSWjlNdPm90SEQleg0vd7809te3EGMr4IR0+JZ+/7v557928f/MtyD7tPq/8v1+BepQ7SEq14LeYvytXasp83+DAaJnFSA5lzcBim2KOrnMYydqt8LywpXxDRMkWj3eIcSwwvCQ4yYSNU9lAvUrUod29a7BbZASlEN0gzvaiieoAGjopNs1Vcdu5pmVO2vOK3RihokPSDt0hJh4UaD8AoK0B+BUyWHAq4UNMrPaFQNny9DEsnXAqLB0Ny4+M3shdLTY9xGSLs3jm+ERMZLESuPqJfGIop0/DeI0JWVJCWdUXxBCF60jYC87oLerV+FGIVuvj4O/a3gB/24B/a+vj708Fv6vBv6Cr34pKXDd5yZ8tlQ8QBctEZHcqJP17kdxSdiujIWaC1lS96siIjui89uSfkc7VR/agGe/Z68+RZGZuXfjOUGl8A9P98CLwDD5yJhPu4H7lHktMSktG9rodlpyKOI8jvarIYF6klBeah1EsDhiqSBlPaUIzSK4662LIga7NByr9W3n+BnG+V6clOYshL9AO8++957/kUBLfuvh6p4auCvumkAlYv/cLdTevKXbdqlLTr16gXNXDFBEgCBgj9IBPlEs4ZAniD/kOmDnHEBHeuBtO5Phhaq4xZwOzEhKdBXlZpOgZUeDnGBA8Qwo4D6rOK+vCDv2B6ihxOZgiauyPFGe8G3gYs5uQ2fSn63UhZj0mWDu/n+ecLlaHH42Egi5U6BFx5wQR5zQHKNsahn0bGMKKa7oMeFPFlFDz0pdMnJ02mHMBsDhHpTBLDHrwlNPqOoxQZAR7FXquZx0JbHsEdnjgaXWyM5Yuwl+ySm4N+6D2wa+Au2Mi+Ulxt/WUwSFSFFFCRBtMM73ud5NO/fHLCJVruRfz+dCNhjuKWauCyfyoX/emF6vYQ2HsmgAPnRWo7iLHAHx0tQbe2TeNd2a1Og/wrnVu4JuX9YB/gVFglOBwDakiYEAVTIaqnvX8gBIkdB9cavCKZfMhhkOsMpqhEbDKBJV2RwImxAyivsFxXF2XTE4buvX4XgCBq3nBJOGTZTAcXcGvOeRITpGWVSQtiVRyQgtplKHXipm8/DjWWkTqkqFq+uJyWcrovYLRbZnrwvN0DxK05mf3X6s87TbydHXyT+pB/Vr3AlV/rE7BudXp4GuYJV/kWGUbiq029laboTKV+f9mQwSGDRGcckMAPVxHRCLnLOXKSjnC+XJpTcKszoo5BybT7GEyzTlhMk2pyaPJtMBMkx/LlV0yBve9BrnMnBW9kUepNDBOKQCvz5/H2/vhiG/1DA7Mu4li97/3unn3CwZw9R8= -------------------------------------------------------------------------------- /pkg/stoserver/restapichangefeed.go: -------------------------------------------------------------------------------- 1 | package stoserver 2 | 3 | import ( 4 | "encoding/binary" 5 | "net/http" 6 | "strconv" 7 | 8 | "github.com/function61/gokit/httpauth" 9 | "github.com/function61/varasto/pkg/stoserver/stodb" 10 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 11 | ) 12 | 13 | func (h *handlers) CollectionChangefeed(rctx *httpauth.RequestContext, w http.ResponseWriter, r *http.Request) *[]stoservertypes.CollectionChangefeedItem { 14 | httpErr := func(err error, errCode int) *[]stoservertypes.CollectionChangefeedItem { // shorthand 15 | http.Error(w, err.Error(), errCode) 16 | return nil 17 | } 18 | 19 | cursor, err := func() (uint64, error) { 20 | after := r.URL.Query().Get("after") 21 | if after != "" { 22 | return strconv.ParseUint(r.URL.Query().Get("after"), 10, 64) 23 | } else { 24 | return 0, nil // just start from beginning 25 | } 26 | }() 27 | if err != nil { 28 | return httpErr(err, http.StatusBadRequest) 29 | } 30 | 31 | tx, rollback, err := readTx(h.db) 32 | if err != nil { 33 | return httpErr(err, http.StatusInternalServerError) 34 | } 35 | defer rollback() 36 | 37 | // +1 so we exclude the items we already processed 38 | globalVersionStart := make([]byte, 8) 39 | binary.BigEndian.PutUint64(globalVersionStart, cursor+1) 40 | 41 | results := []stoservertypes.CollectionChangefeedItem{} 42 | err = stodb.CollectionsGlobalVersionIndex.Query(globalVersionStart, func(sortKey []byte, value []byte) error { 43 | globalVersion := binary.BigEndian.Uint64(sortKey) 44 | 45 | results = append(results, stoservertypes.CollectionChangefeedItem{ 46 | Cursor: strconv.FormatUint(globalVersion, 10), 47 | CollectionId: string(value), 48 | }) 49 | 50 | if len(results) >= 50 { 51 | return stodb.StopIteration 52 | } else { 53 | return nil 54 | } 55 | }, tx) 56 | if err != nil { 57 | return httpErr(err, http.StatusInternalServerError) 58 | } 59 | 60 | return &results 61 | } 62 | -------------------------------------------------------------------------------- /pkg/fssnapshot/lvm_test.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | // must exclude from Windows build due to syscall.Mount(), syscall.Unmount() 4 | 5 | package fssnapshot 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/function61/gokit/assert" 11 | "github.com/prometheus/procfs" 12 | ) 13 | 14 | func TestOriginPathInSnapshot(t *testing.T) { 15 | sp := "/mnt/snap1" 16 | 17 | assert.EqualString(t, originPathInSnapshot("/home/vagrant/snaptest", "/", sp), "/mnt/snap1/home/vagrant/snaptest") 18 | assert.EqualString(t, originPathInSnapshot("/home/vagrant/snaptest", "/home", sp), "/mnt/snap1/vagrant/snaptest") 19 | assert.EqualString(t, originPathInSnapshot("/home/vagrant/snaptest", "/home/vagrant", sp), "/mnt/snap1/snaptest") 20 | assert.EqualString(t, originPathInSnapshot("/home/vagrant/snaptest", "/home/vagrant/snaptest", sp), "/mnt/snap1") 21 | } 22 | 23 | func TestMountForPath(t *testing.T) { 24 | mounts := []*procfs.Mount{ 25 | {Mount: "/home"}, 26 | {Mount: "/"}, 27 | {Mount: "/var/logs"}, 28 | } 29 | 30 | assert.EqualString(t, mountForPath("/home/vagrant", mounts).Mount, "/home") 31 | assert.EqualString(t, mountForPath("/home", mounts).Mount, "/home") 32 | assert.EqualString(t, mountForPath("/root/.ssh/authorized_keys", mounts).Mount, "/") 33 | assert.EqualString(t, mountForPath("/var/logs/httpd/access.log", mounts).Mount, "/var/logs") 34 | assert.Assert(t, mountForPath("x", mounts) == nil) 35 | } 36 | 37 | func TestDevicePathFromLvsOutput(t *testing.T) { 38 | output := []byte(` root /dev/vagrant-vg/root 39 | snap1 /dev/vagrant-vg/snap1 40 | swap_1 /dev/vagrant-vg/swap_1 41 | `) 42 | 43 | assert.EqualString(t, devicePathFromLvsOutput("root", output), "/dev/vagrant-vg/root") 44 | assert.EqualString(t, devicePathFromLvsOutput("snap1", output), "/dev/vagrant-vg/snap1") 45 | assert.EqualString(t, devicePathFromLvsOutput("swap_1", output), "/dev/vagrant-vg/swap_1") 46 | assert.EqualString(t, devicePathFromLvsOutput("notfound", output), "") 47 | } 48 | -------------------------------------------------------------------------------- /pkg/stateresolver/dirpeek.go: -------------------------------------------------------------------------------- 1 | package stateresolver 2 | 3 | import ( 4 | "path" 5 | "strings" 6 | 7 | "github.com/function61/gokit/sliceutil" 8 | "github.com/function61/varasto/pkg/stotypes" 9 | ) 10 | 11 | type DirPeekResult struct { 12 | Path string 13 | Files []stotypes.File 14 | ParentDirs []string // doesn't include root 15 | SubDirs []string 16 | } 17 | 18 | // given a bunch of files with paths, we can create a directory model that lets us look 19 | // at one directory at a time, listing its sub- and parent dirs 20 | func DirPeek(files []stotypes.File, dirToPeek string) *DirPeekResult { 21 | res := &DirPeekResult{ 22 | Path: dirToPeek, 23 | Files: []stotypes.File{}, 24 | ParentDirs: parents(dirToPeek), 25 | SubDirs: []string{}, 26 | } 27 | 28 | // "foo" => 1 29 | // "foo/bar/baz" => 3 30 | levelOfSubDirToPeek := strings.Count(dirToPeek, "/") 31 | 32 | dirToPeekWithSlash := dirToPeek + "/" 33 | if dirToPeekWithSlash == "./" { 34 | levelOfSubDirToPeek-- 35 | dirToPeekWithSlash = "" 36 | } 37 | 38 | for _, file := range files { 39 | // "foo/bar/baz.txt" => "foo/bar" 40 | dir := path.Dir(file.Path) 41 | 42 | if dir == dirToPeek { 43 | res.Files = append(res.Files, file) 44 | } else if strings.HasPrefix(dir, dirToPeekWithSlash) { 45 | // "foo/bar" => ["foo", "bar"] 46 | components := strings.Split(dir, "/") 47 | if len(components) < levelOfSubDirToPeek+1 { 48 | continue 49 | } 50 | 51 | subDir := strings.Join(components[0:levelOfSubDirToPeek+2], "/") 52 | 53 | if !sliceutil.ContainsString(res.SubDirs, subDir) { 54 | res.SubDirs = append(res.SubDirs, subDir) 55 | } 56 | } 57 | } 58 | 59 | return res 60 | } 61 | 62 | // doesn't include root 63 | func parents(dirPath string) []string { 64 | ret := []string{} 65 | 66 | curr := path.Dir(dirPath) 67 | 68 | for curr != "." && curr != "/" { 69 | ret = append(ret, curr) 70 | 71 | curr = path.Dir(curr) 72 | } 73 | 74 | return ret 75 | } 76 | -------------------------------------------------------------------------------- /pkg/stoserver/commandhandlerskek.go: -------------------------------------------------------------------------------- 1 | package stoserver 2 | 3 | import ( 4 | "crypto/rand" 5 | "crypto/rsa" 6 | "crypto/x509" 7 | 8 | "github.com/function61/eventkit/command" 9 | "github.com/function61/gokit/cryptoutil" 10 | "github.com/function61/varasto/pkg/stoserver/stodb" 11 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 12 | "github.com/function61/varasto/pkg/stotypes" 13 | "github.com/function61/varasto/pkg/stoutils" 14 | "go.etcd.io/bbolt" 15 | ) 16 | 17 | func (c *cHandlers) KekGenerateOrImport(cmd *stoservertypes.KekGenerateOrImport, ctx *command.Ctx) error { 18 | data := cmd.Data 19 | 20 | if data == "" { 21 | var err error 22 | data, err = generateKek() 23 | if err != nil { 24 | return err 25 | } 26 | } 27 | 28 | privateKey, err := cryptoutil.ParsePemPkcs1EncodedRsaPrivateKey([]byte(data)) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | fingerprint, err := cryptoutil.Sha256FingerprintForPublicKey(&privateKey.PublicKey) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | kek := stotypes.KeyEncryptionKey{ 39 | ID: stoutils.NewKeyEncryptionKeyId(), 40 | Kind: "rsa", 41 | Bits: privateKey.PublicKey.Size() * 8, 42 | Created: ctx.Meta.Timestamp, 43 | Label: cmd.Label, 44 | Fingerprint: fingerprint, 45 | PublicKey: string(cryptoutil.MarshalPemBytes(x509.MarshalPKCS1PublicKey(&privateKey.PublicKey), cryptoutil.PemTypeRsaPublicKey)), 46 | PrivateKey: string(cryptoutil.MarshalPemBytes(x509.MarshalPKCS1PrivateKey(privateKey), cryptoutil.PemTypeRsaPrivateKey)), 47 | } 48 | 49 | return c.confreload(c.db.Update(func(tx *bbolt.Tx) error { 50 | return stodb.KeyEncryptionKeyRepository.Update(&kek, tx) 51 | })) 52 | } 53 | 54 | func generateKek() (string, error) { 55 | privateKey, err := rsa.GenerateKey(rand.Reader, 4096) 56 | if err != nil { 57 | return "", err 58 | } 59 | 60 | return string(cryptoutil.MarshalPemBytes(x509.MarshalPKCS1PrivateKey(privateKey), cryptoutil.PemTypeRsaPrivateKey)), nil 61 | } 62 | -------------------------------------------------------------------------------- /pkg/stodupremover/actioners.go: -------------------------------------------------------------------------------- 1 | package stodupremover 2 | 3 | import ( 4 | "log" 5 | "os" 6 | ) 7 | 8 | type Item struct { 9 | Filename string 10 | } 11 | 12 | type Actioner interface { 13 | Duplicate(Item, string) error 14 | NotDuplicate(Item) error 15 | Finish() error 16 | } 17 | 18 | type LoggerActioner struct { 19 | duplicates int 20 | notDuplicates int 21 | } 22 | 23 | func (l *LoggerActioner) Duplicate(item Item, duplicateFilename string) error { 24 | l.duplicates++ 25 | 26 | log.Printf("DUP %s (%s)", item.Filename, duplicateFilename) 27 | 28 | return nil 29 | } 30 | 31 | func (l *LoggerActioner) NotDuplicate(item Item) error { 32 | l.notDuplicates++ 33 | 34 | log.Printf("NOT %s", item.Filename) 35 | 36 | return nil 37 | } 38 | 39 | func (l *LoggerActioner) Finish() error { 40 | log.Printf( 41 | "Finished; duplicates=%d notDuplicates=%d", 42 | l.duplicates, 43 | l.notDuplicates) 44 | 45 | return nil 46 | } 47 | 48 | type RemoveDuplicatesActioner struct{} 49 | 50 | func (l *RemoveDuplicatesActioner) Duplicate(item Item, _ string) error { 51 | return os.Remove(item.Filename) 52 | } 53 | 54 | func (l *RemoveDuplicatesActioner) NotDuplicate(item Item) error { 55 | return nil 56 | } 57 | 58 | func (l *RemoveDuplicatesActioner) Finish() error { 59 | return nil 60 | } 61 | 62 | // can be used to, fox example, log AND remove duplicates 63 | type TeeActioner struct { 64 | a Actioner 65 | b Actioner 66 | } 67 | 68 | func (t *TeeActioner) Duplicate(item Item, duplicateFilename string) error { 69 | if err := t.a.Duplicate(item, duplicateFilename); err != nil { 70 | return err 71 | } 72 | 73 | return t.b.Duplicate(item, duplicateFilename) 74 | } 75 | 76 | func (t *TeeActioner) NotDuplicate(item Item) error { 77 | if err := t.a.NotDuplicate(item); err != nil { 78 | return err 79 | } 80 | 81 | return t.b.NotDuplicate(item) 82 | } 83 | 84 | func (t *TeeActioner) Finish() error { 85 | if err := t.a.Finish(); err != nil { 86 | return err 87 | } 88 | 89 | return t.b.Finish() 90 | } 91 | -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/cas.drawio: -------------------------------------------------------------------------------- 1 | 7Vpbb6M4FP4t+xBp96ERmGsem07beZiRuupDO08rBwx4BnDGOG2yv35tMAFjaJKWNG20qVThgzH2+c7l8zET6ypb31K4TL6TEKUTYITrifVlAoBvevy/EGwqgVcLYorDSmQ2gnv8L5JCQ0pXOESF0pERkjK8VIUByXMUMEUGKSXPareIpOpblzBGmuA+gKkufcAhS6TUdGfNja8Ix4l8tQ/k+jJYd5YrKRIYkueWyLqeWFeUEFZdZesrlArd1XqpnrsZuLudGEU52+eBu9+Lx/vZg7X4O12Ztz+jpHj850KO8gTTlVzw/ddL4Lh/ToCb8mHnC8qvYnG1j+QvuVS2qfXH0FrcSFiWcoHJLwtGyS90RVJCuSQnOe85j3CadkQwxXHOmwFfH+Ly+ROiDHNkLuWNDIeheM38OcEM3S9hIN75zM2QyyhZ5SESSzfKmZar5AOg9aD6zC0o3JgRyRCjG95FPgBciaM0ZNOS7efGLGwpSloGUcMPpSHG25EbrPiFhOsA6BwNugfK9VA6iI5MwQgVpt7FB4Xc3GWTUJaQmOQwvW6kHU02fb4RspSQ/kSMbaTvwhV/vQI4WmP22Lr+IYaaOrL1ZS1HLhubVuMOUcxVJbCvZDlX22O70RpJNJuhytZGAV8s9GXouV7IigboBZ0DGYEgjRHb5Va6KVGUQoaf1HmMbhhANwxIMw35Pq9pu2kCl+JOto5FYJ8uYIGDaUiCVVaqTXHZCbBCB/mhrfk3v+ODheW643ihrzrhTPdBv8cHTeNYTmhpur4iSzFfEolpIZ4EEJ0yHgPHD4wpitjJw2IHEE8HxDTfMyoCPSzWwbBYwlzBwP29IqUlk5xdFGX4uuQdTLBcNzfrADqIq5YVy6FhJtSdL4plK1HyFVWTUPPn4MSEjVxIuMXMSsS1mVmmGwEb+K99y17LH87+2utOl2GMqVenlR+tWwMpZptO5FPbjLIjn4SwSMrZmuMmF3vf5DI7ZXax+/xLGkuVZxrTOVsu6O3mgt57Rj23Jw1xTXG/hmFIUVHARYpKMPpZoKqhHcygq3smvLRLByJH/PXRAbf8ybjTkle/cQCybRWgHnwGyPqxEPI0hG4JCcXGGG7+OA9uZnsfjJz5b6cCZl8uDP3AsT1/tm8GPBOq18W3h+u9a9CreeUh+L6OUTWkB6YUwXCj3+DUomDFp2BJh+zDx6Y3g8ZlTIEjR9qbx8jB7gjmc2nM1HLV5Gx3DLAiWPKpdsGsHqjuSKKoQEyz0+3k32C6et3tfPbozkfLA6ZeERknEezceZ1nIugCfPJEMNuBbwXHd8RgCBksSReD3FnQvsAdxI93blY+IGH2HAXRrYO26ziu04Pp8UDV9zQaLD2bkm9wgdI7UmCGidD1gjBGsj5UUtFzDoNfcYluG4ryNwhcBVB9GgT6IDMM37gxqghdHfoohQOjJ3ZjUnhTHJC8mJZGOo6nqo663fPsOLNwjladcz8L8TlWkQfIivHuKs9pzxC8zwJUq4ynVPHMHVW8stU9Yjoa6mBP1L2Tgq7zpEzUjmI0UOl+FYmqSIix307s0K322xJ1ly5D5EdBb0oOfLSIxonRM2+qJl/L7Em+4F3DtH6s9fKRx/EN4VCqfQ6GYFtWLTmdKcw+SyJQYjfKw0vx9c9ku4Pikhss1j76NwH+G2N2fxnDNVXuZs0cdYiBMsZYJQqgV0/3PwL9Px4cJTH0na6PFA14s/kUrbKh5ns+6/o/ -------------------------------------------------------------------------------- /docs/concepts-ideas-architecture/architecture.xml: -------------------------------------------------------------------------------- 1 | 7Vtbc9o4FP41zGwfwliWbeAxEJrsTJLNDGnTPgpbgDfGorIIkF+/ki3ji8AYAtgkm8401tGRL+f7zkWXNGBvurylaDZ5IA72GrrmLBvwpqHrQGu3+C8hWUWSdgtEgjF1HamUCAbuO45HSuncdXCQUWSEeMydZYU28X1ss4wMUUoWWbUR8bJPnaExVgQDG3mq9MV12ERKgaYlHXfYHU/ko9um7JiiWFkKgglyyCIlgv0G7FFCWHQ1XfawJ4wX2yUa931L7/rFKPZZmQHdu7nzetXx++DP84MxJFf9+/crGN3lDXlz+cE/EUUBI+KFMX3DVL48W8UWoWTuO1jcVGvA7mLiMjyYIVv0LjgHuGzCph5vAX7Jb8Bcbs1rzx37XMaIUECyZfNX50+A3ZHreT3iERo+Ao5M8Y/LA0bJK071WOGPGEF8lpJHP1yuWkUaSrwJXqZE0kq3mEwxoyuuEvd2JGKSsrplRu1FQgAz5uckhT2IFZEk3Xh97wQXfiGh2QMmoMDU9cjwSBjljO8g3B7ZG41vt/FwdBwjw3bWyBBqipGBvsHIlnYiG+uKjR8wQw5i6JLtvA6k9bGzqZgROzzmyiahbELGxEdeP5F2s4ZOdO6JCCihef/FjK1kAkFzHsEyxsdLl/2Sw8X1b3HdNGXrZpnqulnFDZ9/76/4BqKRGiWaybCwFY/bCltA5tTGBbYxZIZDdIzZ7ngg7FZIAoo9xNy3bC47PqJWpYg2dTMNajlItWbLzKBaOaZ6rTA1lGj4gkXC+fH3x4LgrtR/nqhomiWiIjxnVASqwe+en59q5FigrGOdya/MI/uVpMYV/+yOHFPa1cK7XVOKVimFGXF9FqQe9iQEqdTcgVkSarkCPq/f0or0+UX0BrnR8euQ0SjghsoTdW2Bw7kbO0qtuHsQdfWacbdsnk+4CywAz0FemCOjbhSTN6+/i+xGvm6tC9lNhevJjNn2XBy+hucOKZJZ4cJTZX42DNpqqjQ2zYZPlio7/weWrbaxSgYWs1bFplXGp3r3vPS0PP5V3SGfiVtjcfXXvevPlw2dP1d7QHZDzGQtNA0XmcL/ueTF9R2yCL6poz+jf3bUxSpdO2sle4n+2YJGdo6vhauJRU4atp4wdbnZBPwf9NzWRXpuq8Bzv/8Y9MV44R0j4URbvXeD03Jn/vYZ3DNfx2xwT2Cd1T3bn6VYP1NKBWVzqrFfsc6LAhCz4bS1umWYe9XeFijUr672Bmqh0BVbfGJlXokt176w0YwbdkTo9JNmf3N39gdnLc51dS0gtrzjviWmj0TBDPmx7J7YKNw7doPXIFbg75DWSYkzdzsqkPk9yyFhjExVhKvdpcxPqA2jo0APNy1hrvnwEezBYw9hvX33OJ39nMzpO/39T3/DDlrPI3MnxIZQsdG+wUtfBrx/AKMS/paQMQdR126oiIY8mDK7+cE6YCucl7ULbRjtcpXD6fBVDwtkXDuFkPVnLo41dPm3sitp52sxmYtMve4vigc3PA7wIaDKUFDNli20mtkEbLTUNRerpUIfU+ToyHeqQF7/csjDvM9XjTvYkc1PA3yz2fxy0Jta3aBXzxydAfrHLwe8BeoGvFrGVb5AkFoeiA9u1GmFoKEsEBTVT8dbujts5y538i5eoSq77ZzTz64GHDDT30xCWCnf6n3yoSzfOrXgW34F1MifU87r60aR/on4ZlTLt89Atxjnqvlm7Mc3E1bBt2oPwSY59KIJd/RjsIcRLp8gdy2vG0aR/r6E483kj0gi9eRPcWD/Pw== -------------------------------------------------------------------------------- /pkg/stomediascanner/stateandchangefeed.go: -------------------------------------------------------------------------------- 1 | package stomediascanner 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/function61/gokit/ezhttp" 7 | "github.com/function61/varasto/pkg/stoclient" 8 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 9 | ) 10 | 11 | func discoverChanges( 12 | ctx context.Context, 13 | after string, 14 | conf *stoclient.ClientConfig, 15 | ) ([]stoservertypes.CollectionChangefeedItem, error) { 16 | changefeedItems := []stoservertypes.CollectionChangefeedItem{} 17 | if _, err := ezhttp.Get( 18 | ctx, 19 | conf.UrlBuilder().CollectionChangefeed(after), 20 | ezhttp.RespondsJson(&changefeedItems, false), 21 | ezhttp.AuthBearer(conf.AuthToken), 22 | ezhttp.Client(conf.HttpClient()), 23 | ); err != nil { 24 | return nil, err 25 | } 26 | 27 | return changefeedItems, nil 28 | } 29 | 30 | // for reset: 31 | // $ curl -k -H 'Content-Type: application/json' -d '{"State":""}' https://localhost/command/config.SetMediascannerState 32 | 33 | func discoverState(ctx context.Context, conf *stoclient.ClientConfig) (string, error) { 34 | return fetchServerConfig(ctx, stoservertypes.CfgMediascannerState, conf) 35 | } 36 | 37 | func fetchServerConfig(ctx context.Context, confKey string, conf *stoclient.ClientConfig) (string, error) { 38 | ctx, cancel := context.WithTimeout(ctx, ezhttp.DefaultTimeout10s) 39 | defer cancel() 40 | 41 | configValue := stoservertypes.ConfigValue{} 42 | if _, err := ezhttp.Get( 43 | ctx, 44 | conf.UrlBuilder().GetConfig(confKey), 45 | ezhttp.RespondsJson(&configValue, false), 46 | ezhttp.AuthBearer(conf.AuthToken), 47 | ezhttp.Client(conf.HttpClient()), 48 | ); err != nil { 49 | return "", err 50 | } 51 | 52 | // can return empty string, but that conveniently works for us, because the changefeed 53 | // accepts empty string to mean "start from beginning" 54 | return configValue.Value, nil 55 | } 56 | 57 | func setState(ctx context.Context, lastProcessed string, conf *stoclient.ClientConfig) error { 58 | return conf.CommandClient().Exec(ctx, &stoservertypes.ConfigSetMediascannerState{ 59 | State: lastProcessed, 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /pkg/stoclient/uploadprogessui_test.go: -------------------------------------------------------------------------------- 1 | package stoclient 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/function61/gokit/assert" 8 | ) 9 | 10 | func TestSpeedMbps(t *testing.T) { 11 | t0 := time.Date(2019, 11, 4, 15, 32, 0, 0, time.UTC) 12 | 13 | megabytes := func(n int) int64 { 14 | return int64(1024 * 1024 * n) 15 | } 16 | 17 | tcs := []struct { 18 | input []fileProgressEvent 19 | expectedOutput string 20 | }{ 21 | { 22 | []fileProgressEvent{ 23 | { 24 | bytesUploadedInBlob: megabytes(4), 25 | started: t0, 26 | completed: t0, 27 | }, 28 | }, 29 | "+Inf Mbps", 30 | }, 31 | { 32 | []fileProgressEvent{ 33 | { 34 | bytesUploadedInBlob: megabytes(4), 35 | started: t0, 36 | completed: t0.Add(1 * time.Second), 37 | }, 38 | }, 39 | "32.00 Mbps", 40 | }, 41 | { 42 | []fileProgressEvent{ 43 | { 44 | bytesUploadedInBlob: megabytes(4), 45 | started: t0, 46 | completed: t0.Add(2 * time.Second), 47 | }, 48 | }, 49 | "16.00 Mbps", 50 | }, 51 | { 52 | []fileProgressEvent{ 53 | { 54 | bytesUploadedInBlob: megabytes(8), 55 | started: t0, 56 | completed: t0.Add(1 * time.Second), 57 | }, 58 | { 59 | bytesUploadedInBlob: megabytes(8), 60 | started: t0, 61 | completed: t0.Add(1 * time.Second), 62 | }, 63 | }, 64 | "128.00 Mbps", 65 | }, 66 | { 67 | []fileProgressEvent{ 68 | { 69 | bytesUploadedInBlob: megabytes(8), 70 | started: t0, 71 | completed: t0.Add(1 * time.Second), 72 | }, 73 | { 74 | bytesUploadedInBlob: megabytes(8), 75 | started: t0, 76 | completed: t0.Add(2 * time.Second), 77 | }, 78 | }, 79 | "64.00 Mbps", 80 | }, 81 | } 82 | 83 | for _, tc := range tcs { 84 | tc := tc // pin 85 | t.Run(tc.expectedOutput, func(t *testing.T) { 86 | assert.EqualString(t, speedMbps(tc.input), tc.expectedOutput) 87 | }) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /pkg/stoserver/stohealth/ivchecker.go: -------------------------------------------------------------------------------- 1 | package stohealth 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/function61/varasto/pkg/duration" 8 | "github.com/function61/varasto/pkg/stoserver/stodb" 9 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 10 | "github.com/function61/varasto/pkg/stotypes" 11 | "go.etcd.io/bbolt" 12 | ) 13 | 14 | func NewLastIntegrityVerificationJob(db *bbolt.DB) HealthChecker { 15 | return &lastIvJob{db} 16 | } 17 | 18 | type lastIvJob struct { 19 | db *bbolt.DB 20 | } 21 | 22 | // TODO: this check only checks the latest completed check, and trusts the user having 23 | // ran it for each applicable volume 24 | func (h *lastIvJob) CheckHealth() (*stoservertypes.Health, error) { 25 | tx, err := h.db.Begin(false) 26 | if err != nil { 27 | return nil, err 28 | } 29 | defer func() { ignoreError(tx.Rollback()) }() 30 | 31 | newest := time.Time{} 32 | 33 | if err := stodb.IntegrityVerificationJobRepository.Each(func(record interface{}) error { 34 | job := record.(*stotypes.IntegrityVerificationJob) 35 | 36 | if job.Completed.After(newest) { 37 | newest = job.Completed 38 | } 39 | 40 | return nil 41 | }, tx); err != nil { 42 | return nil, err 43 | } 44 | 45 | since := time.Since(newest) 46 | 47 | status := func() stoservertypes.HealthStatus { 48 | day := 24 * time.Hour // naive 49 | 50 | switch { 51 | case since > 30*day: 52 | return stoservertypes.HealthStatusFail 53 | case since > 14*day: 54 | return stoservertypes.HealthStatusWarn 55 | default: 56 | return stoservertypes.HealthStatusPass 57 | } 58 | }() 59 | 60 | return NewStaticHealthNode( 61 | "File integrity verification", 62 | status, 63 | sinceHumanReadable(since), 64 | stoservertypes.HealthKindVolumeIntegrity.Ptr(), 65 | ).CheckHealth() 66 | } 67 | 68 | func sinceHumanReadable(since time.Duration) string { 69 | year := float64(24 * 365) // [h], naive 70 | 71 | // reference was zero 72 | if since.Hours() > 100*year { 73 | return "Never checked" 74 | } 75 | 76 | return fmt.Sprintf("%s since last check", duration.Humanize(since)) 77 | } 78 | 79 | func ignoreError(err error) { 80 | // no-op 81 | } 82 | -------------------------------------------------------------------------------- /frontend/component/filetypes.ts: -------------------------------------------------------------------------------- 1 | import { unrecognizedValue } from 'f61ui/utils'; 2 | import { File } from 'generated/stoserver/stoservertypes_types'; 3 | 4 | export enum Filetype { 5 | Word, 6 | Excel, 7 | Powerpoint, 8 | Spreadsheet, 9 | Picture, 10 | Video, 11 | Pdf, 12 | Audio, 13 | Archive, 14 | Other, 15 | } 16 | 17 | export function iconForFiletype(cl: Filetype) { 18 | switch (cl) { 19 | case Filetype.Word: 20 | return 'word.png'; 21 | case Filetype.Excel: 22 | return 'excel.png'; 23 | case Filetype.Powerpoint: 24 | return 'powerpoint.png'; 25 | case Filetype.Spreadsheet: 26 | return 'spreadsheet.png'; 27 | case Filetype.Picture: 28 | return 'picture.png'; 29 | case Filetype.Video: 30 | return 'video.png'; 31 | case Filetype.Pdf: 32 | return 'pdf.png'; 33 | case Filetype.Audio: 34 | return 'audio.png'; 35 | case Filetype.Archive: 36 | return 'archive.png'; 37 | case Filetype.Other: 38 | return 'generic.png'; 39 | default: 40 | throw unrecognizedValue(cl); 41 | } 42 | } 43 | 44 | export function filetypeForFile(file: File): Filetype { 45 | const ext = /\.([^.]+)$/.exec(file.Path); 46 | if (!ext) { 47 | return Filetype.Other; 48 | } 49 | 50 | switch (ext[1].toLowerCase()) { 51 | case 'doc': 52 | case 'docx': 53 | return Filetype.Word; 54 | case 'xls': 55 | case 'xlsx': 56 | return Filetype.Excel; 57 | case 'ppt': 58 | case 'pptx': 59 | return Filetype.Powerpoint; 60 | case 'ods': 61 | return Filetype.Spreadsheet; 62 | case 'jpg': 63 | case 'jpeg': 64 | case 'gif': 65 | case 'png': 66 | case 'bmp': 67 | return Filetype.Picture; 68 | case 'avi': 69 | case 'divx': 70 | case 'mp4': 71 | case 'mkv': 72 | case 'mov': 73 | case 'flv': 74 | case 'wmv': 75 | case '3gp': 76 | case 'webm': 77 | return Filetype.Video; 78 | case 'pdf': 79 | return Filetype.Pdf; 80 | case 'mp3': 81 | case 'wav': 82 | case 'mid': 83 | case 'flac': 84 | case 'ogg': 85 | return Filetype.Audio; 86 | case 'zip': 87 | case 'gz': 88 | case 'tar': 89 | case '7z': 90 | return Filetype.Archive; 91 | default: 92 | return Filetype.Other; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/mutexmap/mutexmap.go: -------------------------------------------------------------------------------- 1 | package mutexmap 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // Think of this as an infinite number of named bathroom stalls. Each named stall can only 8 | // be occupied by one person. 9 | // When you TryLock(): 10 | // a) it won't open if it's already occupied. (because it's locked inside) decide to do something else 11 | // b) it opens and you get in and the stall gets reserved/locked for you. When you get out 12 | // 13 | // you call the unlock callback you obtained from TryLock() to return the stall for use. 14 | type M struct { 15 | // value is chan that Lock() can use to listen for unlock event (close of channel) 16 | locks map[string]chan bool 17 | masterMu sync.Mutex 18 | } 19 | 20 | func New() *M { 21 | return &M{ 22 | locks: map[string]chan bool{}, 23 | } 24 | } 25 | 26 | func (n *M) Lock(key string) func() { 27 | for { 28 | unlock, tryAgain := n.tryLockInternal(key) 29 | if tryAgain != nil { 30 | // wait for someone to unlock (signalled by close of the chan), so we can try 31 | // locking again (not guaranteed - someone else might try locking same gate) 32 | <-tryAgain 33 | continue 34 | } else { 35 | return unlock 36 | } 37 | } 38 | } 39 | 40 | // returns false if gate already open/reserved 41 | // returns true if gate was opened for you. you have to use the returned func to release it 42 | func (n *M) TryLock(key string) (func(), bool) { 43 | unlock, tryAgain := n.tryLockInternal(key) 44 | if tryAgain != nil { 45 | return unlock, false 46 | } else { 47 | return unlock, true 48 | } 49 | } 50 | 51 | // first return is "unlock" function, which will be nil if tryAgain is non-nil 52 | // second return is "tryAgain" whose close you can wait on to to try locking again 53 | func (n *M) tryLockInternal(key string) (func(), chan bool) { 54 | n.masterMu.Lock() 55 | defer n.masterMu.Unlock() 56 | 57 | if tryAgain, open := n.locks[key]; open { 58 | return nil, tryAgain 59 | } 60 | 61 | unlocked := make(chan bool) 62 | n.locks[key] = unlocked 63 | 64 | return func() { 65 | n.masterMu.Lock() 66 | defer n.masterMu.Unlock() 67 | 68 | delete(n.locks, key) 69 | close(unlocked) 70 | }, nil 71 | } 72 | -------------------------------------------------------------------------------- /pkg/fssnapshot/windows_test.go: -------------------------------------------------------------------------------- 1 | package fssnapshot 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | ) 8 | 9 | func TestFindSnapshotDeviceFromDetailsOutput(t *testing.T) { 10 | exampleOutput := `vssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool 11 | (C) Copyright 2001-2013 Microsoft Corp. 12 | 13 | Contents of shadow copy set ID: {2caa3819-940a-42ef-a39f-f01f4c75260d} 14 | Contained 1 shadow copies at creation time: 28/11/2018 15.08.49 15 | Shadow Copy ID: {984628b9-4972-4af3-8748-e9ec2c810dec} 16 | Original Volume: (D:)\\?\Volume{10eaffff-0000-0000-0000-602219000000}\ 17 | Shadow Copy Volume: \\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy2 18 | Originating Machine: joonas3 19 | Service Machine: joonas3 20 | Provider: 'Microsoft Software Shadow Copy provider 1.0' 21 | Type: ClientAccessible 22 | Attributes: Persistent, Client-accessible, No auto release, No writers, Differential 23 | 24 | ` 25 | 26 | assert.EqualString( 27 | t, 28 | findSnapshotDeviceFromDetailsOutput(exampleOutput), 29 | `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy2`) 30 | 31 | assert.EqualString( 32 | t, 33 | findSnapshotDeviceFromDetailsOutput("foo"), 34 | "") 35 | } 36 | 37 | func TestFindSnapshotIdFromCreateOutput(t *testing.T) { 38 | exampleOutput := `Executing (Win32_ShadowCopy)->create() 39 | Method execution successful. 40 | Out Parameters: 41 | instance of __PARAMETERS 42 | { 43 | ReturnValue = 0; 44 | ShadowID = "{984628B9-4972-4AF3-8748-E9EC2C810DEC}"; 45 | }; 46 | ` 47 | 48 | assert.EqualString(t, 49 | findSnapshotIdFromCreateOutput(exampleOutput), 50 | "{984628B9-4972-4AF3-8748-E9EC2C810DEC}") 51 | } 52 | 53 | func TestDriveLetterFromPath(t *testing.T) { 54 | assert.EqualString(t, driveLetterFromPath("C:/windows"), "C") 55 | assert.EqualString(t, driveLetterFromPath("D:/games"), "D") 56 | } 57 | 58 | func TestOriginPathInSnapshotForWindows(t *testing.T) { 59 | assert.EqualString( 60 | t, 61 | originPathInSnapshot("D:/data/my-cool-origin", "D:/", "D:/snapshots/mysnapshot"), 62 | "D:/snapshots/mysnapshot/data/my-cool-origin") 63 | } 64 | -------------------------------------------------------------------------------- /frontend/layout/appdefaultlayout.tsx: -------------------------------------------------------------------------------- 1 | import { getCurrentLocation } from 'f61ui/browserutils'; 2 | import { Breadcrumb } from 'f61ui/component/breadcrumbtrail'; 3 | import { NavLink } from 'f61ui/component/navigation'; 4 | import { GlyphiconIcon } from 'f61ui/component/bootstrap'; 5 | import { globalConfig } from 'f61ui/globalconfig'; 6 | import { DefaultLayout } from 'f61ui/layout/defaultlayout'; 7 | import { RootFolderId } from 'generated/stoserver/stoservertypes_types'; 8 | import { version } from 'generated/version'; 9 | import * as React from 'react'; 10 | import { browseUrl, gettingStartedUrl, serverInfoUrl } from 'generated/frontend_uiroutes'; 11 | 12 | interface AppDefaultLayoutProps { 13 | title: string; 14 | titleElem?: React.ReactNode; 15 | breadcrumbs: Breadcrumb[]; 16 | children: React.ReactNode; 17 | } 18 | 19 | // app's default layout uses the default layout with props that are common to the whole app 20 | export class AppDefaultLayout extends React.Component { 21 | render() { 22 | const currLoc = getCurrentLocation(); 23 | 24 | function mkLink(title: string, icon: GlyphiconIcon, url: string): NavLink { 25 | return { 26 | title, 27 | glyphicon: icon, 28 | url, 29 | active: url === currLoc, 30 | }; 31 | } 32 | 33 | const navLinks: NavLink[] = [ 34 | mkLink('Browse', 'folder-open', browseUrl({ dir: RootFolderId })), 35 | mkLink('Help', 'book', gettingStartedUrl({ section: 'welcome' })), 36 | mkLink('Admin', 'cog', serverInfoUrl()), 37 | ]; 38 | 39 | const appName = 'Varasto'; 40 | 41 | return ( 42 | 52 | } 53 | logoClickUrl={browseUrl({ dir: RootFolderId })} 54 | breadcrumbs={this.props.breadcrumbs.concat({ 55 | title: this.props.titleElem || this.props.title, 56 | })} 57 | content={this.props.children} 58 | version={version} 59 | pageTitle={this.props.title} 60 | /> 61 | ); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /docs/security/privacy/index.md: -------------------------------------------------------------------------------- 1 | Who can access my files? 2 | ------------------------ 3 | 4 | Only you (and the people you explicitly choose to share with) have access to your files. 5 | 6 | 7 | Does Varasto phone home? 8 | ------------------------ 9 | 10 | No, except for update checking. 11 | 12 | 13 | ### Update checking 14 | 15 | This is done to tell you if there is a better version available (displayed in server UI), 16 | and possibly to alert you if there are critical security updates available. 17 | 18 | This version check request doesn't contain any additional data other than what is required 19 | to do the check: 20 | 21 | | Data | Example | Used for analytics[^1] | Why we send this | 22 | |------|---------|--------------------|------------------| 23 | | OS | Linux | ☑️ | So we can tell you the latest version for your OS | 24 | | Architecture | x86-64 | ☑️ | So we can tell you the latest version for your architecture | 25 | | Current version | 20200418_1637_fa31fb5e | ☐ | So we can tell you if your version contains critical vulnerabilities | 26 | | IP address | 84.15.186.115 | ☐ | One can't check for updates (or use the internet) without revealing one's IP address | 27 | 28 | !!! tip 29 | You can audit the 30 | [version checking code](https://github.com/function61/varasto/blob/6eb3f4d6f18ce61be453291ab644fe8ef64aad62/pkg/stoserver/updatechecker.go#L63) 31 | yourself. 32 | 33 | 34 | The data we record about Varasto users 35 | -------------------------------------- 36 | 37 | ### Self-hosted Varasto 38 | 39 | Nothing. 40 | 41 | 42 | ### Cloud-hosted Varasto 43 | 44 | Since this is a paid offering, we have to keep your billing details on file, and of course 45 | an email to reach out to you for important updates. 46 | 47 | 48 | ### Varasto website visitors 49 | 50 | We don't use any analytics except for aggregate request counts. 51 | 52 | 53 | ### Varasto newsletter 54 | 55 | If you sign up for [Varasto newsletter](https://buttondown.email/varasto), we can see your 56 | email address. We won't sell your email address ever, and only use it for purposes stated 57 | in the newsletter's description. 58 | 59 | 60 | [^1]: To gather metrics like "Active Varasto users on Linux/x86-64" so we know which 61 | platforms to focus our development efforts on. 62 | -------------------------------------------------------------------------------- /pkg/stoserver/stohealth/healthinterface.go: -------------------------------------------------------------------------------- 1 | // Health checks for Varasto server 2 | package stohealth 3 | 4 | import ( 5 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 6 | ) 7 | 8 | type HealthChecker interface { 9 | CheckHealth() (*stoservertypes.Health, error) 10 | } 11 | 12 | type healthFolder struct { 13 | title string 14 | children []HealthChecker 15 | kind *stoservertypes.HealthKind 16 | } 17 | 18 | func NewHealthFolder(title string, kind *stoservertypes.HealthKind, children ...HealthChecker) HealthChecker { 19 | return &healthFolder{title, children, kind} 20 | } 21 | 22 | func (h *healthFolder) CheckHealth() (*stoservertypes.Health, error) { 23 | return mkHealthWithChildren(h.title, stoservertypes.HealthStatusPass, "", h.children, h.kind) 24 | } 25 | 26 | func mkHealthWithChildren( 27 | title string, 28 | health stoservertypes.HealthStatus, 29 | details string, 30 | children []HealthChecker, 31 | kind *stoservertypes.HealthKind, 32 | ) (*stoservertypes.Health, error) { 33 | childDtos := []stoservertypes.Health{} 34 | 35 | for _, child := range children { 36 | childHealth, err := child.CheckHealth() 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | childDtos = append(childDtos, *childHealth) 42 | } 43 | 44 | return &stoservertypes.Health{ 45 | Title: title, 46 | Health: worstOf(childDtos, health), 47 | Details: details, 48 | Children: childDtos, 49 | Kind: kind, 50 | }, nil 51 | } 52 | 53 | func worstOf(list []stoservertypes.Health, initial stoservertypes.HealthStatus) stoservertypes.HealthStatus { 54 | worst := initial 55 | 56 | for _, item := range list { 57 | if statusWorse(item.Health, worst) { 58 | worst = item.Health 59 | } 60 | } 61 | 62 | return worst 63 | } 64 | 65 | func statusWorse(a stoservertypes.HealthStatus, b stoservertypes.HealthStatus) bool { 66 | return statusToInt(a) < statusToInt(b) 67 | } 68 | 69 | func statusToInt(status stoservertypes.HealthStatus) int { 70 | switch stoservertypes.HealthStatusExhaustive97fd15(status) { 71 | case stoservertypes.HealthStatusPass: 72 | return 3 73 | case stoservertypes.HealthStatusWarn: 74 | return 2 75 | case stoservertypes.HealthStatusFail: 76 | return 1 77 | default: 78 | panic("unknown") 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /docs/using/replication-policies/replication-policies.drawio: -------------------------------------------------------------------------------- 1 | 7Vpbc5s6EP41fowHxMX4Mc6l5yE5zTSnSfrUkY1sq5GRR8i3/PqzAmHAwjFtwE4yJTMxWl1A334r7WrpOBez9ReB59NbHhLWQVa47jiXHYT6vgf/lWCTCnpBkAomgoapyM4F9/SFaKGlpQsakrjUUHLOJJ2XhSMeRWQkSzIsBF+Vm405Kz91jifEENyPMDOljzSUUy21/X5e8Q+hk6l+dIB6acUMZ431TOIpDvmqIHKuOs6F4Fymd7P1BWEKuwyXtN/1ntrtiwkSyTodaDR9eLlF/07sx8HX2bdltLx5OkPpKEvMFnrCD1jgWHI1AyJxiCVW2oOfIY6JnoncZPAIvohCop5gdZzBakoluZ/jkapdAR9ANpUzBiUbbpdESArQnjM6iUAmuWowpoxdcMZFMqIz9tQfyGMp+DMp1PjJpXrwSBbk6QVyPRd4ClnvBcneQg+UJRwmKTbQJOvQ09rSdPV1cZXr3nW1bFpQu+dpIdZ8m2yHzlUCN1orv6Eh21TRNzJngKOkPFKT41CgYCON6mbIpeSzZkB1ba8EqoNMVLfAF1Hdwt88qgaoBnwV4NzgIWF3PKYJ9AWQsEZtBIgQoOWAqZYDPHqeJFoo8ju59lpDyvpsqUFVdmBZgXWtVLpdUVQhxPF0q2yomatpzNYTtSp3KY97XQprZNxNLLoRtfpW2VY8U6vIrVBqv2v1i1fQko6dwzo+qNiDq5eh+bK6Ih4Rc42zrMHVNarQE35ZCNKF1uSnqp5zqka9WsLgsX6NBhSHynqzUd9c5Hpdr2KVa8scPUNVt3yZrGlWSAXs61w/pOG9pxEwy2iifsXi5lUtblZbaGYuSAErEoJXo4tcyCmf8Aizq1w6KKOZt7nhCqkEw19Eyo120fACfIQSwmRN5ZPuru5/qPuup0uX60LV5SYrRDDfp2wAVUh79bysnPdLSlnHdIJqVq+rDUDgCzEir6Dlp+0kFhMiX2nXr6aBIAx24mX5PRpXqW8YyCUZ4wWTyRsY7gA80Ic6ZzAUcDeRCWK7knuwKqJ8YlhulKWple+8TsczaPzA2WKmeqseeKaMKRrG88PtB8fwIcdjNBpV7Z2hP/Q9vyG3xi1bvuN4puWjCsu37bYs37EMdN+75ReN/UdxJWjf8ns1Ld87peX39m+NpzX8z2vIbkXUV23IrW3hgaH1/wAfZH1ZRG+DfQfgEJNgXAmwPwrIcNyQj7QbALo1V8rW4r++ge8jF89qUoL/An/zI3qefatrFS+nnh+6jbybD7ItA2UIjEcK5stNhGcA2ycg8/Z042Rkts3TjLuUxsnuRtmKRuEnQNpG7kGGHxd4M279Pmcch+bWuUqXF73lns5Hy+5r+WhQuCOCAljqaKNp98up6X7Ze1hyHP/LNkOvYuzTvgt00vNw3y/vMaf3jGzTIR4wPnzj8ffu+uaRIHSrQA/Q0PEb8jv9YOcgzjLRDSrAzbyn5rE13c7juvvviuvIcrvmyeix6W56qp+F7hB3VeB7VMIj00M1gK06hN+feSmAXCcT4Fnqr6MTRoUMxDafcDBFUW1ZfCEZjeDpWVpcPcRMVuzkH/h4TEekG5Il/MTdKRbhz5DGz80QoLeTXLUD11C/U6X+1g7KbUPb79QV23IsTwE14oNlocNhJ6x3SicMmbm87zEweFd7GZ8XM3auQvc69vPHqdqCSiutrQGLQYcPpqoMprWAB9UPeGb6lPJj2FdVqJMfUXdKyakj5qay/PlB8wxOap3u4V30b6Zd2WrFJxJHzrQj08X/Gin/PtnmVS6Ameed79Vo/9BA29pMg7p76VtTyUnXcyHwptAgoXBcGPlOCXIuOjvut9Pzimw62B75vR32pW+Qc3E7lTfQ0wx3bhfqa0+WcfTjbCl72IlORc/6vt4efmpinMEcUFA+mj1D+pjs9zhskC4bZlMuZgNAfBLDq+8ugg0Qr8YW9jcQbCsQdHY/CWwtLoRi/sV1yp38s3Xn6n8= -------------------------------------------------------------------------------- /pkg/stoclient/localstate.go: -------------------------------------------------------------------------------- 1 | package stoclient 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/function61/gokit/fileexists" 9 | "github.com/function61/gokit/jsonfile" 10 | "github.com/function61/varasto/pkg/stotypes" 11 | ) 12 | 13 | const ( 14 | LocalStatefile = ".varasto" 15 | ) 16 | 17 | type BupManifest struct { 18 | ChangesetId string `json:"changeset_id"` 19 | Collection stotypes.Collection `json:"collection"` // snapshot at time of server fetch 20 | } 21 | 22 | // TODO: rename to some kind of context 23 | type workdirLocation struct { 24 | path string 25 | clientConfig ClientConfig 26 | manifest *BupManifest 27 | } 28 | 29 | func (w *workdirLocation) Join(comp string) string { 30 | return filepath.Join(w.path, comp) 31 | } 32 | 33 | func (w *workdirLocation) SaveToDisk() error { 34 | return jsonfile.Write(w.Join(LocalStatefile), w.manifest) 35 | } 36 | 37 | func NewWorkdirLocation(path string) (*workdirLocation, error) { 38 | clientConfig, err := ReadConfig() 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | loc := &workdirLocation{ 44 | path: path, 45 | clientConfig: *clientConfig, 46 | } 47 | 48 | statefile, err := os.Open(loc.Join(LocalStatefile)) 49 | if err != nil { 50 | if os.IsNotExist(err) { 51 | return nil, fmt.Errorf("not a Varasto workdir: %s", loc.path) 52 | } 53 | 54 | return nil, err // some other error 55 | } 56 | defer statefile.Close() 57 | 58 | loc.manifest = &BupManifest{} 59 | return loc, jsonfile.Unmarshal(statefile, loc.manifest, true) 60 | } 61 | 62 | func statefileExists(path string) (bool, error) { 63 | // init this in "hack mode" (i.e. statefile not being read to memory). as soon as we 64 | // manage to write the statefile to disk, use normal procedure to init wd 65 | halfBakedWd := &workdirLocation{ 66 | path: path, 67 | } 68 | 69 | return fileexists.Exists(halfBakedWd.Join(LocalStatefile)) 70 | } 71 | 72 | func assertStatefileNotExists(path string) error { 73 | if exists, err := statefileExists(path); err != nil || exists { 74 | if err != nil { // error doing the check 75 | return err 76 | } 77 | 78 | return fmt.Errorf("%s already exists in %s - adopting would be dangerous", LocalStatefile, path) 79 | } 80 | 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /pkg/smart/smart.go: -------------------------------------------------------------------------------- 1 | // Access SMART data of disks 2 | package smart 3 | 4 | import ( 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "os/exec" 9 | ) 10 | 11 | type Backend func(device string) ([]byte, error) 12 | 13 | func Scan(device string, back Backend) (*SmartCtlJsonReport, error) { 14 | smartCtlOutput, err := back(device) 15 | if err != nil { 16 | return nil, fmt.Errorf("%v, output: %s", err, smartCtlOutput) 17 | } 18 | 19 | return parseSmartCtlJsonReport(smartCtlOutput) 20 | } 21 | 22 | func SmartCtlBackend(device string) ([]byte, error) { 23 | stdout, err := exec.Command("smartctl", "--json", "--all", device).Output() 24 | 25 | return stdout, silenceSmartCtlAutomationHostileErrors(err) 26 | } 27 | 28 | /* 29 | joonas/smartmontools built from simple Dockerfile: 30 | 31 | FROM alpine:edge 32 | RUN apk add --update smartmontools 33 | */ 34 | func SmartCtlViaDockerBackend(device string) ([]byte, error) { 35 | // disks in /dev are visible without --privileged (by mapping /dev:/dev) but 36 | // /dev/disk/by-uuid et al. are not 37 | // maybe related: https://github.com/moby/moby/issues/16160 38 | stdout, err := exec.Command( 39 | "docker", "run", 40 | "--rm", 41 | "-t", 42 | "--privileged", 43 | "-v", "/dev:/dev:ro", 44 | "joonas/smartmontools:20191015", 45 | "smartctl", 46 | "--json", 47 | "--all", 48 | device, 49 | ).Output() 50 | 51 | return stdout, silenceSmartCtlAutomationHostileErrors(err) 52 | } 53 | 54 | func parseSmartCtlJsonReport(reportJson []byte) (*SmartCtlJsonReport, error) { 55 | rep := &SmartCtlJsonReport{} 56 | 57 | if err := json.Unmarshal(reportJson, rep); err != nil { 58 | return nil, err 59 | } 60 | 61 | if len(rep.JsonFormatVersion) < 2 || rep.JsonFormatVersion[0] != 1 { 62 | return nil, errors.New("invalid json_format_version") 63 | } 64 | 65 | return rep, nil 66 | } 67 | 68 | func silenceSmartCtlAutomationHostileErrors(err error) error { 69 | if err != nil { 70 | if exitError, is := err.(*exec.ExitError); is { 71 | // unset bits 4-8 because they're not errors in getting the report itself 72 | // https://sourceforge.net/p/smartmontools/mailman/message/7330895/ 73 | masked := exitError.ExitCode() &^ 0xf8 74 | 75 | if masked == 0 { // not error anymore 76 | return nil 77 | } 78 | } 79 | } 80 | 81 | return err 82 | } 83 | -------------------------------------------------------------------------------- /frontend/layout/AdminLayout.tsx: -------------------------------------------------------------------------------- 1 | import { getCurrentLocation } from 'f61ui/browserutils'; 2 | import { Panel, GlyphiconIcon } from 'f61ui/component/bootstrap'; 3 | import { Breadcrumb } from 'f61ui/component/breadcrumbtrail'; 4 | import { NavLink, renderNavLink } from 'f61ui/component/navigation'; 5 | import { AppDefaultLayout } from 'layout/appdefaultlayout'; 6 | import * as React from 'react'; 7 | import * as r from 'generated/frontend_uiroutes'; 8 | 9 | interface AdminLayoutProps { 10 | title: string; 11 | breadcrumbs: Breadcrumb[]; 12 | children: React.ReactNode; 13 | } 14 | 15 | export class AdminLayout extends React.Component { 16 | render() { 17 | const currLoc = getCurrentLocation(); 18 | 19 | function mkLink(title: string, icon: GlyphiconIcon, url: string): NavLink { 20 | return { 21 | title, 22 | glyphicon: icon, 23 | url, 24 | active: url === currLoc, 25 | }; 26 | } 27 | 28 | const settingsLinks: NavLink[] = [ 29 | mkLink('Health & server info', 'dashboard', r.serverInfoUrl()), 30 | mkLink('Volumes & mounts', 'hdd', r.volumesUrl()), 31 | mkLink('Subsystems', 'tasks', r.subsystemsUrl()), 32 | mkLink('Scheduled jobs', 'time', r.scheduledJobsUrl()), 33 | mkLink('Metadata backup', 'cloud-upload', r.metadataBackupUrl({ view: '' })), 34 | mkLink('Users', 'user', r.usersUrl()), 35 | mkLink('Metrics', 'stats', r.metricsUrl()), 36 | mkLink('Logs', 'list-alt', r.logsUrl()), 37 | mkLink('Servers', 'th-large', r.nodesUrl()), 38 | mkLink('Replication policies', 'retweet', r.replicationPoliciesUrl()), 39 | mkLink('Content metadata', 'book', r.contentMetadataUrl()), 40 | mkLink('FUSE server & network folders', 'folder-open', r.fuseServerUrl()), 41 | ]; 42 | 43 | return ( 44 | 52 |
53 | 54 |
    55 | {settingsLinks.map(renderNavLink)} 56 |
57 | 58 |
59 |
{this.props.children}
60 | 61 | } 62 | /> 63 | ); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /docs/security/encryption/encrypted-integrity-verification.drawio: -------------------------------------------------------------------------------- 1 | 5VvbcuI4EP0aqjIPoWz5AjwGyG6maqYqNcxmJo/CFuCNsChZBDxfv7It3yRjDJhLsnlIrLYsW92nu09LSscYLbd/U7hafCcuwh2guduOMe4A0Nd7/HckCBNBLxXMqecmIj0XTLw/SAg1IV17LgpKHRkhmHmrstAhvo8cVpJBSsmm3G1GcPmtKzhHimDiQKxKf3kuWwipbg/yG0/Imy/Eq/tAzG8J085iJsECumRTEBmPHWNECWHJ1XI7QjjSXaqX5Lm/dtzNPowinzV6YLP98xSaLvC//nzx2BR7P4N73UiGeYd4LWYsvpaFqQqQ7z5EmuQtn/hcOHRhsEDRsDpvLNgSi8vkUeQqKs2/Uc9mzhGDyBIxGvIum1y3ltDXoqDVVEYRhsx7Lw8PhYnn2XDZG56Jx18MNIFGK7WFAKPZ18pDBGRNHSSeKupyz0BAGodBOkdMGYdfFGadi2JLHWI1TbHaDwRdLpliMo3+RDqdPD0Ay1bNyc0zEU1C2YLMiQ/xYy4dUrL23di+Gm/lfb4RshKW/hcxFgpvhWtGyjhAW4/9Fo9H16/RddcSrfG2cGscigaGU4SH0Hmbx68fEUxo/MHGLP45AF58urEdazQIqmEozHuvdQ1zYLaCOVMrQ0WXMbcDKyrm+lb9QHvBm3Yks1mAzgNM8BGB2atHJgcXDX8Xekbt1+LN/Lm4lT7YNl556ouxUmeAHfH11MApYQ+Yva7VCH1tIQv0FWSNfowMcId8h4Yrhty7DrAx181wSvnVnMUGsOGSI2PoT4NV3NYqREsYTtGILFcUBQEfZ4UhnwTasi9fvtwOSEFDkOolhBYQuwOkpVwOsTf3+TVGM3YdAAOrNjJr3b5mGscEZk5hYFjosIqQGtRBXorbAoG7w7PUX+tLfpB8wcVjshqSHxOX8YivwLsM3s3CY2iygrFlN5zfl4H6jijzOFV+EKiZEsbIMsNNdBttD2WC4gFbyyJMWjmIZoEq6ralcsWsdKiCQkm5h2pS5cpp1IhVKUefOxLrGGI1ityOmg29CyRekWu+qGo9kxaVrZvWmZRtK8p+gRQGPFhyTSLKZ61oNYrZZdUFjJI3lHJJUb3MPIwlkRT3ZI0vPdeNY3uVpcq2bMEmhsQYDU2xxqCiRpJrkNYs0VMsoWbFT1Mi8sBTUv7RJaI80IVLRJUu5e7jYA+Jt35C99GN23IfoKaNMdn4mBTqolshmcdxzH0Us6nPN6CRYtVtfx1k12KE02lD8LOjA8b5mdtAAc4wqaLT/KcwjtHDJLt9IusounkHGC5E/ZmjxAR+x3b6aDprKfVZEom2KpiIVsFDrHN5r64rRpg4PLOtMYrcNyoU59RjYTJzb+ZV8JLb1zsYAIltG0ANmxfWvKmmMILXS8RlKdAZoXCOVKR/klRmmXtTmW5eMpfpKq14TldLcg9w1cgUF5xkHm0RRerE4VVzXpbnXgt3qnNei7lLb5y8erWouOfZ69ZTl15TvinYEHRUw96UQqpC4/bjp65LNfT185Zatf0TVOSmYAFX0eV6iR8cFukmi3nfoh2ZZxJ48UpRYd2hwVaNHDcZkcxE1gx7PrdHum2r7Vp+PC2CyqtIRl+NoUZFCLXPZhiV1Y1Rvh4nOwe/V7fG9FXhHw48fWHvOj4kU5DKdAdqSv/2Szd1n/WqeeuAWq3NvGU3zVuDWgNrXcM+LW/tirbHrtH0pNggO/3erdkLrN5fF3ElpqTVIu4GVgeyE0LHIrUxME8LK2pNWdySkWP8GwqDVuP5bIZspzKeu73BVGtrHU4uXq5eUQKVEyVHEQpbvh/d28BB+71t+p7Z0Pf27vCemCQuEJPVlYlCFdyqo87ADke1p7Zlt+OotrLoBio2APsVxMs8m6eqJPklYrfRV8fHPq7rpqD5+kHh8BA46PDQ2fzUapojTz1KdNpeu0q9bwkBvf8DAq5KkgyVJGUAuLUDhI3PD34wAOwt6MCg3yvljvQ4yIkV3r1pVQ57gdxuKbD7jhh0IYNcOh5+QCKubu70zsfEeTP/l4bEKPn/hRiP/wE= -------------------------------------------------------------------------------- /pkg/igdbapi/externalidextractor.go: -------------------------------------------------------------------------------- 1 | package igdbapi 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | ) 7 | 8 | type ExternalIds struct { 9 | Official *string // official website, "homepage" 10 | SteamId *string 11 | GogSlug *string 12 | EnglishWikipediaSlug *string 13 | RedditSlug *string 14 | GooglePlayAppId *string 15 | AppleAppStoreAppId *string 16 | } 17 | 18 | type idExtractor func(url string, ids *ExternalIds) error 19 | 20 | // keyed by IGDB's website "category" (= specific website) 21 | var extractorByCategory = map[int]idExtractor{ 22 | // 23 | WebsiteOfficial: func(url string, ids *ExternalIds) error { 24 | ids.Official = &url 25 | return nil 26 | }, 27 | // https://store.steampowered.com/app/270910 28 | WebsiteSteam: regex(`steampowered.com/app/(\d+)`, func(key string, ids *ExternalIds) { ids.SteamId = &key }), 29 | // https://www.gog.com/game/worms_world_party_remastered 30 | WebsiteGog: regex(`gog.com/game/([^/]+)`, func(key string, ids *ExternalIds) { ids.GogSlug = &key }), 31 | // https://en.wikipedia.org/wiki/Battle_City_(video_game) 32 | // https://wikipedia.org/wiki/Sonic_Colors 33 | WebsiteWikipedia: regex(`(?:en.)?wikipedia.org/wiki/(.+)`, func(key string, ids *ExternalIds) { ids.EnglishWikipediaSlug = &key }), 34 | // https://www.reddit.com/r/dukenukem/ 35 | WebsiteReddit: regex(`reddit.com/r/([^/]+)`, func(key string, ids *ExternalIds) { ids.RedditSlug = &key }), 36 | // https://play.google.com/store/apps/details?id=com.frogmind.badland&hl=en 37 | WebsiteAndroid: regex(`\?id=([^&]+)`, func(key string, ids *ExternalIds) { ids.GooglePlayAppId = &key }), 38 | // https://itunes.apple.com/us/app/badland/id535176909?mt=8&uo=4 39 | WebsiteIphone: regex(`/id(\d+)`, func(key string, ids *ExternalIds) { ids.AppleAppStoreAppId = &key }), 40 | WebsiteIpad: regex(`/id(\d+)`, func(key string, ids *ExternalIds) { ids.AppleAppStoreAppId = &key }), 41 | } 42 | 43 | func regex(expression string, assign func(string, *ExternalIds)) idExtractor { 44 | idRegex := regexp.MustCompile(expression) 45 | 46 | return func(url string, ids *ExternalIds) error { 47 | matches := idRegex.FindStringSubmatch(url) 48 | if matches == nil { 49 | return fmt.Errorf("unable to parse ID from URL: %s", url) 50 | } 51 | 52 | assign(matches[1], ids) 53 | 54 | return nil 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pkg/blobstore/s3blobstore/s3_test.go: -------------------------------------------------------------------------------- 1 | package s3blobstore 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/function61/gokit/assert" 7 | "github.com/function61/varasto/pkg/stotypes" 8 | ) 9 | 10 | func TestDeserializeConfig(t *testing.T) { 11 | // serialize + deserialize to cover both directions 12 | conf, err := deserializeConfig((&Config{ 13 | Bucket: "varasto-test", 14 | Prefix: "/", 15 | AccessKeyId: "AKIAUZHTE3U35WCD5EHB", 16 | AccessKeySecret: "wXQJhB...", 17 | RegionId: "eu-central-1", 18 | }).Serialize()) 19 | assert.Assert(t, err == nil) 20 | 21 | assert.EqualString(t, conf.Bucket, "varasto-test") 22 | assert.EqualString(t, conf.Prefix, "/") 23 | assert.EqualString(t, conf.AccessKeyId, "AKIAUZHTE3U35WCD5EHB") 24 | assert.EqualString(t, conf.AccessKeySecret, "wXQJhB...") 25 | assert.EqualString(t, conf.RegionId, "eu-central-1") 26 | assert.EqualString(t, conf.Endpoint, "") 27 | } 28 | 29 | func TestDeserializeConfigWithEndpoint(t *testing.T) { 30 | // serialize + deserialize to cover both directions 31 | conf, err := deserializeConfig((&Config{ 32 | Bucket: "varasto-test", 33 | Prefix: "/", 34 | AccessKeyId: "AKIAUZHTE3U35WCD5EHB", 35 | AccessKeySecret: "wXQJhB...", 36 | RegionId: "eu-central-1", 37 | Endpoint: "s3.us-east-1.amazonaws.com", 38 | }).Serialize()) 39 | assert.Assert(t, err == nil) 40 | 41 | assert.EqualString(t, conf.Bucket, "varasto-test") 42 | assert.EqualString(t, conf.Prefix, "/") 43 | assert.EqualString(t, conf.AccessKeyId, "AKIAUZHTE3U35WCD5EHB") 44 | assert.EqualString(t, conf.AccessKeySecret, "wXQJhB...") 45 | assert.EqualString(t, conf.RegionId, "eu-central-1") 46 | assert.EqualString(t, conf.Endpoint, "s3.us-east-1.amazonaws.com") 47 | } 48 | 49 | func TestDeserializeConfigInvalid(t *testing.T) { 50 | _, err := deserializeConfig("varasto-test:/:AKIAUZHTE3U35WCD5EHB.missingSecret:eu-central-1") 51 | assert.EqualString(t, err.Error(), "s3 options not in format bucket:prefix:accessKeyId:secret:region[:endpoint]") 52 | } 53 | 54 | func TestBlobNamer(t *testing.T) { 55 | namer := s3BlobNamer{"/mypath/"} 56 | 57 | ref, _ := stotypes.BlobRefFromHex("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592") 58 | 59 | name := namer.Ref(*ref) 60 | 61 | assert.EqualString(t, *name, "/mypath/16j7swfXgJRpypq8sAguT41WUeRtPNt2LQLQvzfJ5ZI") 62 | } 63 | -------------------------------------------------------------------------------- /docs/data-interfaces/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Overview on ways to access your data 3 | --- 4 | 5 | Overview on ways to access your data 6 | ==================================== 7 | 8 | Varasto's web UI is good for quickly: 9 | 10 | - fetching the occasional PDF/DOCX/etc. document 11 | - looking at photos 12 | - watching movies/videos 13 | - listening to music. 14 | 15 | But there are times when you just need to interact with raw files on the OS level. This is 16 | why we have many interfaces to access your data - each with different pros and cons on use 17 | cases and OS support. 18 | 19 | 20 | Comparison 21 | ---------- 22 | 23 | | Feature/interface | [Web UI](web-ui/index.md) | [Network folders](network-folders/index.md) | [Cloning](client/index.md#how-does-the-cloning-interface-look-like) | [FUSE filesystem](fuse/index.md) | 24 | |------------------------|--------|-----------------|---------|-----------------| 25 | | Fast-changing data[^1] | ☐ | ☐ | ☑️ | ☐ | 26 | | Streaming[^2] | ☑️ | ☑️ | ☐ | ☑️ | 27 | | Open&edit in native apps[^3] | ☐ | On most cases | ☑️ | ☑️ | 28 | | (OS) Linux | ☑️ | ☑️ | ☑️ | ☑️ | 29 | | (OS) Windows | ☑️ | ☑️ | ☑️ | ☐ | 30 | | (OS) macOS | ☑️ | ☑️ | ☑️ | ☐ | 31 | | (OS) Android | ☑️ | ☑️ | ☐ | ☐ | 32 | | (OS) iOS | ☑️ | ☑️ | ☐ | ☐ | 33 | 34 | !!! tip 35 | We also have [API for programmatic access](../developers/api-overview.md), but the above summary is 36 | focused on end-users. 37 | 38 | 39 | [^1]: Data that has very frequent changes, cannot be stored in Varasto. The only option is 40 | to have fast changes happen on another device and have Varasto take periodic snapshots. 41 | 42 | [^2]: You don't need to download the entire file collection to your device before using it. 43 | No storage space is used, because the files are streamed on-demand from Varasto server. 44 | 45 | [^3]: How well an interface lets you open&edit a file in your native app - e.g. open a 46 | photo in Photoshop for editing. Most (but not all apps) work well with network folders - 47 | but all apps work well with local files (= cloning or FUSE). 48 | -------------------------------------------------------------------------------- /turbobob.json: -------------------------------------------------------------------------------- 1 | { 2 | "for_description_of_this_file_see": "https://github.com/function61/turbobob", 3 | "version_major": 1, 4 | "project_name": "varasto", 5 | "project_emoji_icon": "📦", 6 | "subrepos": [ 7 | { 8 | "source": "https://github.com/function61/f61ui.git", 9 | "kind": "git", 10 | "destination": "frontend/f61ui", 11 | "revision": "dab10d8" 12 | } 13 | ], 14 | "builders": [ 15 | { 16 | "name": "default", 17 | "uses": "docker://fn61/buildkit-golang:20240405_0714_856c11bd", 18 | "mount_destination": "/workspace", 19 | "workdir": "/workspace", 20 | "dev_pro_tips": [ 21 | "to mess with FUSE, 'apt install fuse', add to Docker: --privileged --cap-add SYS_ADMIN --cap-add MKNOD --device /dev/fuse" 22 | ], 23 | "dev_http_ingress": "443", 24 | "commands": { 25 | "prepare": ["bash","-c","exec go generate ./bin"], 26 | "build": ["bin/build.sh"], 27 | "dev": ["bash"] 28 | } 29 | }, 30 | { 31 | "name": "frontend", 32 | "uses": "docker://fn61/buildkit-js:20200323_0913_131f6b10", 33 | "mount_destination": "/workspace", 34 | "commands": { 35 | "prepare": ["bin/build-frontend.sh"], 36 | "dev": ["bash"] 37 | } 38 | }, 39 | { 40 | "name": "docs", 41 | "uses": "docker://fn61/buildkit-mkdocs:20200406_1529_aac78367", 42 | "mount_destination": "/workspace", 43 | "workdir": "/workspace", 44 | "dev_http_ingress": "8000", 45 | "dev_pro_tips": [ 46 | "for preview: $ preview.sh" 47 | ], 48 | "commands": { 49 | "build": ["run-mkdocs.sh", "docs/", "rel/docs-website.tar.gz"], 50 | "dev": ["sh"] 51 | } 52 | }, 53 | { 54 | "name": "publisher", 55 | "uses": "docker://fn61/buildkit-publisher:20200228_1755_83c203ff", 56 | "mount_destination": "/workspace", 57 | "commands": { 58 | "publish": ["publish-gh.sh", "function61/varasto", "rel/"], 59 | "build": ["true"], 60 | "dev": ["bash"] 61 | }, 62 | "pass_envs": [ 63 | "GITHUB_TOKEN", 64 | "EVENTHORIZON" 65 | ] 66 | } 67 | ], 68 | "os_arches": { 69 | "linux-amd64": true, 70 | "linux-arm": true, 71 | "darwin-amd64": true, 72 | "windows-amd64": true 73 | }, 74 | "docker_images": [ 75 | { 76 | "image": "fn61/varasto", 77 | "dockerfile_path": "Dockerfile", 78 | "platforms": ["linux/amd64", "linux/arm/v7"] 79 | } 80 | ], 81 | "experiments_i_consent_to_breakage": { 82 | "prepare_step": true 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /pkg/stateresolver/dirpeek_test.go: -------------------------------------------------------------------------------- 1 | package stateresolver 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/function61/gokit/assert" 8 | "github.com/function61/varasto/pkg/stotypes" 9 | ) 10 | 11 | func TestDirPeek(t *testing.T) { 12 | dumpStringSlice := func(sl []string) string { 13 | return strings.Join(sl, ",") 14 | } 15 | 16 | dirStructure := []stotypes.File{ 17 | mkFile("foo.txt"), 18 | mkFile("bar.txt"), 19 | mkFile("sub/baz.txt"), 20 | mkFile("sub/subsub1/loooool.png"), 21 | mkFile("sub/subsub2/hahah.png"), 22 | mkFile("sub/subsub2/README.md"), 23 | mkFile("sub/subsub2/inception/going-deeper.mp4"), 24 | mkFile("not/content/in/a/few/levels.doc"), 25 | } 26 | 27 | oneCase := func(path string, fileCount int, subDirs string, parentDirs string) { 28 | peekResult := DirPeek(dirStructure, path) 29 | 30 | assert.Assert(t, len(peekResult.Files) == fileCount) 31 | assert.EqualString(t, dumpStringSlice(peekResult.SubDirs), subDirs) 32 | assert.EqualString(t, dumpStringSlice(peekResult.ParentDirs), parentDirs) 33 | } 34 | 35 | oneCase(".", 2, "sub,not", "") 36 | oneCase("sub", 1, "sub/subsub1,sub/subsub2", "") 37 | oneCase("sub/subsub1", 1, "", "sub") 38 | oneCase("sub/subsub2", 2, "sub/subsub2/inception", "sub") 39 | oneCase("sub/subsub2/inception", 1, "", "sub/subsub2,sub") 40 | } 41 | 42 | func TestParents(t *testing.T) { 43 | assert.EqualString( 44 | t, 45 | strings.Join(parents("sub/subsub2/inception/going-deeper.mp4"), ","), 46 | "sub/subsub2/inception,sub/subsub2,sub") 47 | } 48 | 49 | func TestDirsWithSamePrefix(t *testing.T) { 50 | // testing bugfix where peeking at "foo" panic'd because it has same prefix as "foobar" 51 | dirStructure := []stotypes.File{ 52 | mkFile("README.md"), 53 | mkFile("foo/foo.txt"), 54 | mkFile("foobar/bar.txt"), 55 | // also test with above being in a subdir 56 | mkFile("subdir/foo/foo2.txt"), 57 | mkFile("subdir/foobar/bar2.txt"), 58 | } 59 | 60 | assert.EqualString(t, DirPeek(dirStructure, "foo").Files[0].Path, "foo/foo.txt") 61 | assert.EqualString(t, DirPeek(dirStructure, "foobar").Files[0].Path, "foobar/bar.txt") 62 | 63 | assert.EqualString(t, DirPeek(dirStructure, "subdir/foo").Files[0].Path, "subdir/foo/foo2.txt") 64 | assert.EqualString(t, DirPeek(dirStructure, "subdir/foobar").Files[0].Path, "subdir/foobar/bar2.txt") 65 | } 66 | 67 | func mkFile(path string) stotypes.File { 68 | return stotypes.File{ 69 | Path: path, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/stoclient/adopt.go: -------------------------------------------------------------------------------- 1 | package stoclient 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/function61/gokit/osutil" 11 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | func adopt(ctx context.Context, fullPath string, parentDirectoryId string) error { 16 | // cloneCollectionExistingDir() also checks for this, but we must do this before asking 17 | // server to create a collection because we don't want it to be created and error when 18 | // we begin cloning 19 | if err := assertStatefileNotExists(fullPath); err != nil { 20 | return err 21 | } 22 | 23 | clientConfig, err := ReadConfig() 24 | if err != nil { 25 | return err 26 | } 27 | 28 | // TODO: maybe the struct ctor should be codegen'd? 29 | collectionId, err := clientConfig.CommandClient().ExecExpectingCreatedRecordId(ctx, &stoservertypes.CollectionCreate{ 30 | ParentDir: parentDirectoryId, 31 | Name: filepath.Base(fullPath), 32 | }) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | collection, err := clientConfig.Client().FetchCollectionMetadata(ctx, collectionId) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | log.Printf("Collection %s created with id %s", collection.Name, collection.ID) 43 | 44 | // since we created an empty collection, there's actually nothing to download, 45 | // but this does other important housekeeping 46 | return cloneCollectionExistingDir(ctx, fullPath, "", collection) 47 | } 48 | 49 | func adoptEntrypoint() *cobra.Command { 50 | push := false 51 | 52 | cmd := &cobra.Command{ 53 | Use: "adopt [parentDirectoryId]", 54 | Short: "Adopts current directory as Varasto collection", 55 | Args: cobra.ExactArgs(1), 56 | Run: func(cmd *cobra.Command, args []string) { 57 | osutil.ExitIfError(func(parentDirectoryId string) error { 58 | ctx := osutil.CancelOnInterruptOrTerminate(nil) 59 | 60 | fullPath, err := os.Getwd() 61 | if err != nil { 62 | return err 63 | } 64 | 65 | if err := adopt(ctx, fullPath, parentDirectoryId); err != nil { 66 | return err 67 | } 68 | 69 | if push { 70 | if err := pushCurrentWorkdir(ctx); err != nil { 71 | return fmt.Errorf("push: %w", err) 72 | } 73 | } 74 | 75 | return nil 76 | }(args[0])) 77 | }, 78 | } 79 | 80 | cmd.Flags().BoolVarP(&push, "push", "", push, "Push after adopting") 81 | 82 | return cmd 83 | } 84 | -------------------------------------------------------------------------------- /pkg/stofuse/dirbyidquery.go: -------------------------------------------------------------------------------- 1 | package stofuse 2 | 3 | // /sto/dir/ => queries directory dynamically from Varasto and projects it as directory adapter 4 | // which itself projects subdirs and collections as symlinks to further directory or collection queries. 5 | 6 | import ( 7 | "context" 8 | "os" 9 | "sync" 10 | 11 | "bazil.org/fuse" 12 | "bazil.org/fuse/fs" 13 | "github.com/function61/varasto/pkg/mutexmap" 14 | ) 15 | 16 | type dirByIDQuery struct { 17 | srv *FsServer 18 | inode uint64 19 | fetchByDirID *mutexmap.M 20 | cache map[string]fs.Node 21 | cacheDents []fuse.Dirent 22 | cacheMu sync.Mutex 23 | } 24 | 25 | var _ interface { 26 | fs.Node 27 | fs.NodeStringLookuper 28 | } = (*dirByIDQuery)(nil) 29 | 30 | func NewDirByIDQuery(srv *FsServer) *dirByIDQuery { 31 | return &dirByIDQuery{ 32 | srv: srv, 33 | inode: nextInode(), 34 | fetchByDirID: mutexmap.New(), 35 | cache: map[string]fs.Node{}, 36 | cacheDents: []fuse.Dirent{}, 37 | } 38 | } 39 | 40 | func (b *dirByIDQuery) Attr(ctx context.Context, a *fuse.Attr) error { 41 | a.Inode = b.inode 42 | a.Mode = os.ModeDir | 0555 43 | return nil 44 | } 45 | 46 | func (b *dirByIDQuery) Lookup(ctx context.Context, name string) (fs.Node, error) { 47 | dirID := name 48 | 49 | unlock := b.fetchByDirID.Lock(dirID) 50 | defer unlock() 51 | 52 | // cache check needs to be inside lock to prevent unnecessary double fetch 53 | node := b.getCached(dirID) 54 | if node != nil { 55 | return node, nil 56 | } 57 | 58 | dir := NewDirAdapter(dirID, b.srv) 59 | if _, err := dir.ReadDirAll(ctx); err != nil { // FIXME: not sure if all errors are ENOENT 60 | return nil, fuse.ENOENT 61 | } 62 | 63 | b.setCached(dirID, dir) 64 | 65 | return dir, nil 66 | } 67 | 68 | func (b *dirByIDQuery) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { 69 | // FIXME: does this lock work properly? 70 | b.cacheMu.Lock() 71 | defer b.cacheMu.Unlock() 72 | 73 | return b.cacheDents, nil 74 | } 75 | 76 | func (b *dirByIDQuery) getCached(dirID string) fs.Node { 77 | b.cacheMu.Lock() 78 | defer b.cacheMu.Unlock() 79 | 80 | return b.cache[dirID] 81 | } 82 | 83 | func (b *dirByIDQuery) setCached(dirID string, dir *dirAdapter) { 84 | b.cacheMu.Lock() 85 | defer b.cacheMu.Unlock() 86 | 87 | b.cache[dirID] = dir 88 | b.cacheDents = append(b.cacheDents, fuse.Dirent{ 89 | Inode: dir.inode, 90 | Name: dirID, 91 | Type: fuse.DT_Dir, 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /pkg/sslca/sslca.go: -------------------------------------------------------------------------------- 1 | // Managing a CA and signing server certs. 2 | // Used in a setting where we control both the servers and clients. 3 | // Some code borrowed from https://golang.org/src/crypto/tls/generate_cert.go 4 | package sslca 5 | 6 | import ( 7 | "crypto/ecdsa" 8 | "crypto/elliptic" 9 | "crypto/rand" 10 | "crypto/x509" 11 | "crypto/x509/pkix" 12 | "log" 13 | "math/big" 14 | "time" 15 | 16 | "github.com/function61/gokit/cryptoutil" 17 | ) 18 | 19 | func SelfSignedServerCert(hostname string, organisationName string, privateKeyPem []byte) ([]byte, error) { 20 | privateKey, err := cryptoutil.ParsePemEncodedPrivateKey(privateKeyPem) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | publicKey, err := cryptoutil.PublicKeyFromPrivateKey(privateKey) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | notBefore := time.Now().Add(time.Hour * -1) // account for clock drift 31 | notAfter := notBefore.AddDate(10, 0, 0) // years 32 | 33 | certTemplate := &x509.Certificate{ 34 | SerialNumber: generateSerialNumber(), 35 | Subject: pkix.Name{ 36 | Organization: []string{organisationName}, 37 | }, 38 | 39 | NotBefore: notBefore, 40 | NotAfter: notAfter, 41 | 42 | KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, 43 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, 44 | 45 | DNSNames: []string{hostname}, 46 | } 47 | 48 | certDer, err := x509.CreateCertificate(rand.Reader, certTemplate, certTemplate, publicKey, privateKey) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | return cryptoutil.MarshalPemBytes(certDer, cryptoutil.PemTypeCertificate), nil 54 | } 55 | 56 | func GenEcP256PrivateKeyPem() ([]byte, error) { 57 | // why EC: https://blog.cloudflare.com/ecdsa-the-digital-signature-algorithm-of-a-better-internet/ 58 | privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | privateKeyX509, err := x509.MarshalECPrivateKey(privateKey) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | return cryptoutil.MarshalPemBytes(privateKeyX509, cryptoutil.PemTypeEcPrivateKey), nil 69 | } 70 | 71 | func generateSerialNumber() *big.Int { 72 | serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) 73 | serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) 74 | if err != nil { 75 | log.Fatalf("failed to generate serial number: %s", err) 76 | } 77 | 78 | return serialNumber 79 | } 80 | -------------------------------------------------------------------------------- /pkg/stoserver/commandhandlersscheduledjobs.go: -------------------------------------------------------------------------------- 1 | package stoserver 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/function61/eventkit/command" 7 | "github.com/function61/varasto/pkg/scheduler" 8 | "github.com/function61/varasto/pkg/stoserver/stodb" 9 | "github.com/function61/varasto/pkg/stoserver/stoservertypes" 10 | "github.com/function61/varasto/pkg/stotypes" 11 | "go.etcd.io/bbolt" 12 | ) 13 | 14 | func (c *cHandlers) ScheduledjobEnable(cmd *stoservertypes.ScheduledjobEnable, ctx *command.Ctx) error { 15 | return c.db.Update(func(tx *bbolt.Tx) error { 16 | job, err := openScheduledJobNotUpdateChecker(cmd.Id, tx) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | if !job.Enabled { 22 | job.Enabled = true 23 | } else { 24 | return errors.New("job already enabled") 25 | } 26 | 27 | return stodb.ScheduledJobRepository.Update(job, tx) 28 | }) 29 | } 30 | 31 | func (c *cHandlers) ScheduledjobDisable(cmd *stoservertypes.ScheduledjobDisable, ctx *command.Ctx) error { 32 | return c.db.Update(func(tx *bbolt.Tx) error { 33 | job, err := openScheduledJobNotUpdateChecker(cmd.Id, tx) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if job.Enabled { 39 | job.Enabled = false 40 | } else { 41 | return errors.New("job already disabled") 42 | } 43 | 44 | return stodb.ScheduledJobRepository.Update(job, tx) 45 | }) 46 | } 47 | 48 | func (c *cHandlers) ScheduledjobChangeSchedule(cmd *stoservertypes.ScheduledjobChangeSchedule, ctx *command.Ctx) error { 49 | return c.db.Update(func(tx *bbolt.Tx) error { 50 | job, err := openScheduledJobNotUpdateChecker(cmd.Id, tx) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | job.Schedule = cmd.Schedule 56 | 57 | if _, err := scheduler.ValidateSpec(dbJobToJobSpec(*job)); err != nil { 58 | return err 59 | } 60 | 61 | return stodb.ScheduledJobRepository.Update(job, tx) 62 | }) 63 | } 64 | 65 | func (c *cHandlers) ScheduledjobStart(cmd *stoservertypes.ScheduledjobStart, ctx *command.Ctx) error { 66 | c.conf.Scheduler.Trigger(cmd.Id) 67 | 68 | return nil 69 | } 70 | 71 | // this would mess up our analytics 72 | func openScheduledJobNotUpdateChecker(id string, tx *bbolt.Tx) (*stotypes.ScheduledJob, error) { 73 | job, err := stodb.Read(tx).ScheduledJob(id) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | if job.Kind == stoservertypes.ScheduledJobKindVersionupdatecheck { 79 | return nil, errors.New("editing update checker is disabled") 80 | } 81 | 82 | return job, nil 83 | } 84 | -------------------------------------------------------------------------------- /frontend/pages/MetricsPage.tsx: -------------------------------------------------------------------------------- 1 | import { Result } from 'f61ui/component/result'; 2 | import { WarningAlert } from 'f61ui/component/alerts'; 3 | import { CommandInlineForm } from 'f61ui/component/CommandButton'; 4 | import { CollapsePanel } from 'f61ui/component/bootstrap'; 5 | import { ConfigSetGrafanaUrl } from 'generated/stoserver/stoservertypes_commands'; 6 | import { getConfig } from 'generated/stoserver/stoservertypes_endpoints'; 7 | import { CfgGrafanaUrl, ConfigValue } from 'generated/stoserver/stoservertypes_types'; 8 | import { AppDefaultLayout } from 'layout/appdefaultlayout'; 9 | import * as React from 'react'; 10 | import { serverInfoUrl } from 'generated/frontend_uiroutes'; 11 | 12 | interface MetricsPageState { 13 | grafanaUrl: Result; 14 | } 15 | 16 | export default class MetricsPage extends React.Component<{}, MetricsPageState> { 17 | state: MetricsPageState = { 18 | grafanaUrl: new Result((_) => { 19 | this.setState({ grafanaUrl: _ }); 20 | }), 21 | }; 22 | 23 | componentDidMount() { 24 | this.fetchData(); 25 | } 26 | 27 | componentWillReceiveProps() { 28 | this.fetchData(); 29 | } 30 | 31 | render() { 32 | const [grafanaUrl, loadingOrError] = this.state.grafanaUrl.unwrap(); 33 | if (!grafanaUrl || loadingOrError) { 34 | return loadingOrError; 35 | } 36 | return ( 37 | 45 |
46 |
47 | {this.state.grafanaUrl.draw((_) => this.renderGrafanaEmbed(_.Value))} 48 | {this.state.grafanaUrl.draw((_) => this.renderConfig(_))} 49 |
50 |
51 | 52 | ); 53 | } 54 | 55 | private renderGrafanaEmbed(grafanaUrl: string) { 56 | if (!grafanaUrl) { 57 | return Grafana integration not configured.; 58 | } 59 | 60 | return