├── .gitignore ├── .travis.yml ├── .vscode ├── .gitignore └── settings.json.in ├── AUTHORS ├── CONTRIBUTING ├── CONTRIBUTORS ├── LICENSE ├── Makefile.in ├── README.md ├── admin ├── lint.sh ├── pre-commit └── travis-install.sh ├── cmd └── sourcachefs │ ├── integration_test.go │ └── main.go ├── configure └── internal ├── cache ├── cache.go ├── cache_test.go ├── content.go ├── content_test.go ├── metadata.go ├── metadata_test.go ├── sqlite.go ├── sqlite_test.go ├── types.go └── types_test.go ├── fs ├── dir.go ├── file.go ├── root.go └── symlink.go ├── real └── syscalls.go ├── stats ├── stats.go └── status.go └── test └── test.go /.gitignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | .*.swp 3 | /bin 4 | /deps 5 | /pkg 6 | /src 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy 5 | # of the License at: 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | os: 16 | - linux 17 | - osx 18 | 19 | # Pin to macOS 10.12 (indirectly by using the Xcode 8.3 image). We must do 20 | # this to get the OSXFUSE kernel extension to work because there currently 21 | # is no know way to programmatically grant permissions to load a kernel 22 | # extension in macOS 10.13. 23 | # 24 | # See https://github.com/travis-ci/travis-ci/issues/10017 for details. 25 | osx_image: xcode8.3 26 | 27 | language: go 28 | go_import_path: github.com/jmmv/sourcachefs 29 | 30 | go: 31 | - 1.11.x 32 | - 1.12.x 33 | - 1.13.x 34 | 35 | install: ./admin/travis-install.sh 36 | 37 | script: 38 | - ./configure 39 | - make 40 | - make check 41 | -------------------------------------------------------------------------------- /.vscode/.gitignore: -------------------------------------------------------------------------------- 1 | launch.json 2 | settings.json 3 | -------------------------------------------------------------------------------- /.vscode/settings.json.in: -------------------------------------------------------------------------------- 1 | { 2 | "editor.renderWhitespace": true, 3 | "editor.rulers": [100], 4 | "editor.wordWrapColumn": 100, 5 | "editor.wrappingIndent": "none", 6 | "files.autoSave": "afterDelay", 7 | "files.autoSaveDelay": 500, 8 | "files.exclude": { 9 | "**/.git": true, 10 | "**/.DS_Store": true, 11 | "deps/**": true, 12 | "src/**": true 13 | }, 14 | "files.trimTrailingWhitespace": true, 15 | "go.formatOnSave": false, 16 | "go.gopath": "__GOPATH__", 17 | "go.goroot": "__GOROOT__" 18 | } 19 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of sourcachefs authors for copyright purposes. 2 | # 3 | # This file is distinct from the CONTRIBUTORS files; see the latter for 4 | # an explanation. 5 | # 6 | # Names are sorted alphabetically and should be added to this file as: 7 | # 8 | # * Name 9 | # * Organization 10 | 11 | * Google Inc. 12 | -------------------------------------------------------------------------------- /CONTRIBUTING: -------------------------------------------------------------------------------- 1 | Before we can use your code, you must sign the Google Individual Contributor 2 | License Agreement, also known as the CLA, which you can easily do online. The 3 | CLA is necessary mainly because you own the copyright to your changes, even 4 | after your contribution becomes part of our codebase, so we need your 5 | permission to use and distribute your code. We also need to be sure of various 6 | other things--for instance that you will tell us if you know that your code 7 | infringes on other people's patents. You do not have to sign the CLA until 8 | after you have submitted your code for review and a member has approved it, but 9 | you must do it before we can put your code into our codebase. 10 | 11 | https://developers.google.com/open-source/cla/individual 12 | 13 | Contributions made by corporations are covered by a different agreement than 14 | the one above: the Google Software Grant and Corporate Contributor License 15 | Agreement. Please get your company to sign this agreement instead if your 16 | contribution is on their behalf. 17 | 18 | https://developers.google.com/open-source/cla/corporate 19 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the list of people who have agreed to one of the CLAs and can 2 | # contribute patches to the sourcachefs project. 3 | # 4 | # The AUTHORS file lists the copyright holders; this file lists people. 5 | # For example: Google employees are listed here but not in AUTHORS 6 | # because Google holds the copyright. 7 | # 8 | # See the CONTRIBUTING file for more details on the CLA. 9 | # 10 | # Names are sorted by last name and should be added as: 11 | # 12 | # * Name 13 | 14 | * Julio Merino 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile.in: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy 5 | # of the License at: 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | GOPATH := $(SRCDIR):$(DEPSDIR) 16 | 17 | GO = GOPATH="$(GOPATH)" GOROOT="$(GOROOT)" "$(GOROOT)/bin/go" 18 | GOFMT = $(GOROOT)/bin/gofmt 19 | 20 | CLEANFILES += bin pkg 21 | 22 | .PHONY: default 23 | default: Makefile all # Keep this target definition as the very first one. 24 | 25 | # TODO(jmmv): Should automatically reinvoke configure... but this is difficult 26 | # because we need to remember the flags originally passed by the user, and 27 | # we need to tell make to reload the Makefile somehow. 28 | Makefile: configure Makefile.in 29 | @echo "Makefile out of date; rerun ./configure with desired args" 30 | @false 31 | 32 | .PHONY: all 33 | all: bin/sourcachefs 34 | 35 | STATS = github.com/jmmv/sourcachefs/stats 36 | 37 | LDFLAGS = -X $(STATS).buildTimestamp=$$(date "+%Y-%m-%dT%H:%M:%S") \ 38 | -X $(STATS).buildWhere=$$(id -un)@$$(hostname) \ 39 | -X $(STATS).gitRevision=$$(git rev-parse HEAD) 40 | 41 | # TODO(jmmv): This should NOT be a phony target. However, I do not know yet 42 | # how to track dependencies for .go source files, nor whether this should be 43 | # done. Force a rebuild for now. 44 | .PHONY: bin/sourcachefs 45 | CLEANFILES += bin/sourcachefs 46 | CLEANALLFILES += bin 47 | bin/sourcachefs: 48 | @mkdir -p bin 49 | $(GO) build -ldflags "$(LDFLAGS)" -o bin/sourcachefs \ 50 | ./cmd/sourcachefs/main.go 51 | 52 | .PHONY: check 53 | check: bin/sourcachefs 54 | SOURCACHEFS="$$(pwd)/bin/sourcachefs" $(GO) test \ 55 | ./cmd/sourcachefs ./internal/cache 56 | 57 | .PHONY: fmt 58 | fmt: 59 | find . -name .git -o -name deps -prune -o -name "*.go" -print \ 60 | | xargs "$(GOFMT)" -w 61 | 62 | .PHONY: lint 63 | lint: 64 | @GOFMT="$(GOFMT)" GOLINT="$(DEPSDIR)/bin/golint" \ 65 | ./admin/lint.sh $(LINT_FILES) 66 | 67 | .PHONY: clean 68 | clean: 69 | rm -rf $(CLEANFILES) 70 | 71 | .PHONY: cleanall 72 | cleanall: clean 73 | rm -rf $(CLEANALLFILES) 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sourcachefs 2 | 3 | sourcachefs is a **persistent, read-only, FUSE-based caching file system**. 4 | The goal of sourcachefs is to provide a caching layer over immutable slow 5 | file systems, possibly networked ones such as SSHFS. 6 | 7 | sourcachefs was initially designed to cache source files exposed via a 8 | slow networked file system, where this file system maintains immutable views 9 | of each commit from a version control system. In this scheme, each commit is 10 | a separate directory and can be cached indefinitely because the contents 11 | are assumed to not change. 12 | 13 | sourcachefs should support offline operation: once the contents of the 14 | remote file system have been cached, operations on those same files and 15 | directories should continue to work locally. 16 | 17 | This is not an official Google product. 18 | 19 | ## Installation 20 | 21 | Run: 22 | 23 | ./configure && make 24 | 25 | This command will download all required Go dependencies into a temporary 26 | directory, apply necessary patches to them, and build sourcachefs. (This is 27 | a bit of a non-standard Go workflow for your sanity, but if you don't care 28 | about fetching dependencies correctly, you can still use `go get` and 29 | `go build` as usual.) 30 | 31 | The results of the build will be left as the self-contained binary 32 | `bin/sourcachefs`. You can copy it anywhere you want it to live. 33 | 34 | ## Usage 35 | 36 | The basic usage is: 37 | 38 | sourcachefs 39 | 40 | where `target_dir` is the (slow) file system to be reexposed at `mount_point` 41 | and `cache_dir` is the directory that will hold the persistent cached 42 | contents. 43 | 44 | The following non-standard flags are recognized: 45 | 46 | * `--cached_path_regex=string`: Specifies a regular expression to match 47 | relative paths within the mount point. If there is a match, the file is 48 | cached by sourcachefs; if there is not, accesses go directly to the 49 | target directory. 50 | 51 | * `--listen_address=string`: Enables an HTTP server on the given address 52 | for pprof support and for statistics tracking. Statistics are useful to 53 | see how much the cache is helping, if at all, to avoid falling through 54 | the remote file system. 55 | 56 | ## Contributing 57 | 58 | Want to contribute? Great! But please first read the guidelines provided 59 | in [CONTRIBUTING](CONTRIBUTING). 60 | 61 | If you are curious about who made this project possible, you can check out 62 | the [list of copyright holders](AUTHORS) and the [list of 63 | individuals](CONTRIBUTORS). 64 | -------------------------------------------------------------------------------- /admin/lint.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # Copyright 2016 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy 6 | # of the License at: 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | ProgName="${0##*/}" 17 | 18 | : ${GOFMT:=gofmt} 19 | : ${GOLINT:=golint} 20 | 21 | warn() { 22 | echo "${ProgName}: W: ${@}" 1>&2 23 | } 24 | 25 | lint_header() { 26 | local file="${1}"; shift 27 | 28 | local failed=no 29 | if ! grep 'Copyright.*Google' "${file}" >/dev/null; then 30 | warn "${file} does not have a copyright heading" 31 | failed=yes 32 | fi 33 | if ! grep 'Apache License.*2.0' "${file}" >/dev/null; then 34 | warn "${file} does not have a license notice" 35 | failed=yes 36 | fi 37 | [ "${failed}" = no ] 38 | } 39 | 40 | lint_gofmt() { 41 | local gofile="${1}"; shift 42 | local tmpdir="${1}"; shift 43 | 44 | "${GOFMT}" -e -s -d "${gofile}" >"${tmpdir}/gofmt.out" 2>&1 45 | if [ ${?} -ne 0 -o -s "${tmpdir}/gofmt.out" ]; then 46 | warn "${gofile} failed gofmt validation:" 47 | cat "${tmpfile}/gofmt.out" 1>&2 48 | return 1 49 | fi 50 | return 0 51 | } 52 | 53 | lint_golint() { 54 | local gofile="${1}"; shift 55 | local tmpdir="${1}"; shift 56 | 57 | # Lower confidence levels raise a per-file warning to remind about having 58 | # a package-level docstring... but the warning is issued blindly, without 59 | # checking for the existing of this docstring in other packages. 60 | local min_confidence=0.3 61 | 62 | "${GOLINT}" -min_confidence="${min_confidence}" "${gofile}" \ 63 | >"${tmpdir}/golint.out" 64 | if [ ${?} -ne 0 -o -s "${tmpdir}/golint.out" ]; then 65 | warn "${gofile} failed golint validation:" 66 | cat "${tmpdir}/golint.out" 1>&2 67 | return 1 68 | fi 69 | return 0 70 | } 71 | 72 | main() { 73 | # Lint all source files by default if none are provided. 74 | [ ${#} -gt 1 ] || set -- $(find . -name .git -o -name deps -prune \ 75 | -o \( -name "*.go" -o -name Makefile.in -o -name "*.sh" \) -print) 76 | 77 | local ok=yes 78 | 79 | local tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/lint.XXXXXX")" 80 | trap "rm -rf '${tmpdir}'" EXIT 81 | 82 | for file in "${@}"; do 83 | case "${file}" in 84 | AUTHORS|CONTRIBUTING|CONTRIBUTORS|LICENSE|README.md|.gitignore) 85 | continue 86 | ;; 87 | esac 88 | 89 | lint_header "${file}" || ok=no 90 | 91 | case "${file}" in 92 | *.go) 93 | lint_gofmt "${file}" "${tmpdir}" || ok=no 94 | lint_golint "${file}" "${tmpdir}" || ok=no 95 | ;; 96 | esac 97 | done 98 | 99 | [ "${ok}" = yes ] 100 | } 101 | 102 | main "${@}" 103 | -------------------------------------------------------------------------------- /admin/pre-commit: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # Copyright 2016 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy 6 | # of the License at: 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | exec 1>&2 17 | 18 | # Obtain the list of files from the index (i.e. the files to be committed). 19 | # From the list, resolve renames to their target. 20 | index_files="$(git status --porcelain --untracked-files=no \ 21 | | grep '^[MARC]' | cut -c 4- | sed -e 's,^.* -> ,,')" 22 | 23 | if [ -n "${index_files}" ]; then 24 | make lint LINT_FILES="$(echo ${index_files})" || exit 1 25 | fi 26 | -------------------------------------------------------------------------------- /admin/travis-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy 6 | # of the License at: 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | set -e -u 17 | 18 | # Default to no features to avoid cluttering .travis.yml. 19 | : "${FEATURES:=}" 20 | 21 | install_fuse() { 22 | case "${TRAVIS_OS_NAME}" in 23 | linux) 24 | sudo apt-get update 25 | sudo apt-get install -qq fuse libfuse-dev pkg-config 26 | 27 | sudo /bin/sh -c 'echo user_allow_other >>/etc/fuse.conf' 28 | sudo chmod 644 /etc/fuse.conf 29 | ;; 30 | 31 | osx) 32 | brew update 33 | brew cask install osxfuse 34 | 35 | sudo /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse 36 | sudo sysctl -w vfs.generic.osxfuse.tunables.allow_other=1 37 | ;; 38 | 39 | *) 40 | echo "Don't know how to install FUSE for OS ${TRAVIS_OS_NAME}" 1>&2 41 | exit 1 42 | ;; 43 | esac 44 | } 45 | 46 | install_fuse 47 | -------------------------------------------------------------------------------- /cmd/sourcachefs/integration_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bytes" 19 | "os" 20 | "os/exec" 21 | "path/filepath" 22 | "syscall" 23 | "testing" 24 | "time" 25 | 26 | "bazil.org/fuse" 27 | 28 | "github.com/jmmv/sourcachefs/internal/test" 29 | "github.com/stretchr/testify/suite" 30 | ) 31 | 32 | var ( 33 | sourcachefs = mustGetenv("SOURCACHEFS") 34 | ) 35 | 36 | // mustGetenv gets an environment variable and panics when not defined. To be used from a static 37 | // context. 38 | func mustGetenv(name string) string { 39 | value := os.Getenv(name) 40 | if value == "" { 41 | panic(name + " not defined in environment") 42 | } 43 | return value 44 | } 45 | 46 | // runData holds runtime information for a sourcachefs execution. 47 | type runData struct { 48 | cmd *exec.Cmd 49 | out bytes.Buffer 50 | err bytes.Buffer 51 | } 52 | 53 | // run starts a background process to run sourcachefs and passes it the given arguments. 54 | func run(s *suite.Suite, arg ...string) *runData { 55 | var data runData 56 | data.cmd = exec.Command(sourcachefs, arg...) 57 | data.cmd.Stdout = &data.out 58 | data.cmd.Stderr = &data.err 59 | s.NoError(data.cmd.Start()) 60 | return &data 61 | } 62 | 63 | // wait awaits for completion of the process started by run and checks its exit status. 64 | func wait(s *suite.Suite, data *runData, expectedExitStatus int) { 65 | err := data.cmd.Wait() 66 | if expectedExitStatus == 0 { 67 | s.NoError(err) 68 | } else { 69 | status := err.(*exec.ExitError).ProcessState.Sys().(syscall.WaitStatus) 70 | s.Equal(expectedExitStatus, status.ExitStatus()) 71 | } 72 | } 73 | 74 | type FunctionalSuite struct { 75 | test.SuiteWithTempDir 76 | 77 | targetDir string 78 | cacheDir string 79 | mountPoint string 80 | 81 | cmd *exec.Cmd 82 | } 83 | 84 | func TestFunctional(t *testing.T) { 85 | suite.Run(t, new(FunctionalSuite)) 86 | } 87 | 88 | func (s *FunctionalSuite) SetupTest() { 89 | s.SuiteWithTempDir.SetupTest() 90 | 91 | targetDir := filepath.Join(s.TempDir(), "target") 92 | cacheDir := filepath.Join(s.TempDir(), "cache") 93 | mountPoint := filepath.Join(s.TempDir(), "mnt") 94 | for _, dir := range []string{targetDir, cacheDir, mountPoint} { 95 | s.Require().NoError(os.Mkdir(dir, 0755)) 96 | } 97 | 98 | s.Require().NoError(test.WriteFile(filepath.Join(targetDir, "cookie"), "")) 99 | 100 | cmd := exec.Command(sourcachefs, targetDir, cacheDir, mountPoint) 101 | cmd.Stdout = os.Stdout 102 | cmd.Stderr = os.Stderr 103 | s.Require().NoError(cmd.Start()) 104 | defer func() { 105 | // Only clean up the child process if we haven't completed cleanup. 106 | if s.cmd == nil { 107 | cmd.Process.Kill() 108 | s.Error(cmd.Wait()) 109 | fuse.Unmount(mountPoint) // Best effort; errors don't matter. 110 | } 111 | }() 112 | 113 | for tries := 0; tries < 10; tries++ { 114 | _, err := os.Stat(filepath.Join(mountPoint, "cookie")) 115 | if err == nil { 116 | break 117 | } 118 | if err != nil && tries == 9 { 119 | s.Require().NoErrorf(err, "File system failed to come up") 120 | } 121 | time.Sleep(100 * 1000 * 1000) // Duration is in nanoseconds. 122 | } 123 | 124 | // Now that setup went well, initialize all fields. No operation that can fail should happen 125 | // after this point, or the cleanup routines scheduled with defer may do the wrong thing. 126 | s.targetDir = targetDir 127 | s.cacheDir = cacheDir 128 | s.mountPoint = mountPoint 129 | s.cmd = cmd 130 | } 131 | 132 | func (s *FunctionalSuite) TearDownTest() { 133 | if s.cmd != nil { 134 | s.NoError(fuse.Unmount(s.mountPoint)) // Causes server to exit cleanly. 135 | s.NoError(s.cmd.Wait()) 136 | } 137 | 138 | s.SuiteWithTempDir.TearDownTest() 139 | } 140 | 141 | func (s *FunctionalSuite) TestBasicCaching() { 142 | // Create a file and check that it appears in the mount point. 143 | s.NoError(test.WriteFile(filepath.Join(s.targetDir, "a"), "foo")) 144 | s.NoError(test.CheckFileContents(filepath.Join(s.mountPoint, "a"), "foo")) 145 | 146 | // Overwrite the previous file and check that we still see the old contents in the mount point. 147 | s.NoError(test.WriteFile(filepath.Join(s.targetDir, "a"), "bar")) 148 | s.NoError(test.CheckFileContents(filepath.Join(s.mountPoint, "a"), "foo")) 149 | 150 | // Delete the previous file and check that we still see the old contents in the mount point. 151 | s.NoError(os.Remove(filepath.Join(s.targetDir, "a"))) 152 | s.NoError(test.CheckFileExists(filepath.Join(s.mountPoint, "a"))) 153 | 154 | // Check that a non-existent file also does not appear in the mount point. 155 | s.NoError(test.CheckFileNotExists(filepath.Join(s.mountPoint, "b"))) 156 | 157 | // Create the checked file and check that it still does not appear in the mount point. 158 | s.NoError(test.WriteFile(filepath.Join(s.targetDir, "b"), "baz")) 159 | s.NoError(test.CheckFileNotExists(filepath.Join(s.mountPoint, "b"))) 160 | } 161 | 162 | func (s *FunctionalSuite) TestSignalHandling() { 163 | s.NoError(test.WriteFile(filepath.Join(s.targetDir, "a"), "")) 164 | s.NoError(test.CheckFileExists(filepath.Join(s.mountPoint, "a"))) 165 | 166 | s.NoError(s.cmd.Process.Signal(os.Interrupt)) 167 | s.Error(s.cmd.Wait()) 168 | s.False(s.cmd.ProcessState.Success()) 169 | 170 | s.Error(fuse.Unmount(s.mountPoint)) 171 | s.NoError(test.CheckFileNotExists(filepath.Join(s.mountPoint, "a"))) 172 | 173 | s.cmd = nil // Tell tearDown that we did the cleanup ourselves. 174 | } 175 | 176 | type FlagParsingSuite struct { 177 | suite.Suite 178 | } 179 | 180 | func TestFlagParsing(t *testing.T) { 181 | suite.Run(t, new(FlagParsingSuite)) 182 | } 183 | 184 | func (s *FlagParsingSuite) TestInvalidSyntax() { 185 | data := []struct { 186 | args []string 187 | expectedStderr string 188 | }{ 189 | {[]string{"--foo"}, "not defined.*-foo"}, 190 | {[]string{"foo"}, "number of arguments.* expected 3"}, 191 | {[]string{"foo", "bar"}, "number of arguments.* expected 3"}, 192 | {[]string{"foo", "bar", "ok", "baz"}, "number of arguments.* expected 3"}, 193 | } 194 | for _, d := range data { 195 | cmd := run(&s.Suite, d.args...) 196 | wait(&s.Suite, cmd, 2) 197 | s.Empty(cmd.out.String()) 198 | s.Regexp(d.expectedStderr, cmd.err.String(), "No error details found") 199 | s.Regexp("Type 'sourcachefs -help'", cmd.err.String(), "No instructions found") 200 | } 201 | 202 | cmd := run(&s.Suite, "--foo") 203 | wait(&s.Suite, cmd, 2) 204 | s.Empty(cmd.out.String()) 205 | s.Regexp("not defined.*-foo", cmd.err.String(), "No error details found") 206 | s.Regexp("Type 'sourcachefs -help'", cmd.err.String(), "No instructions found") 207 | } 208 | 209 | func (s *FlagParsingSuite) TestHelpOk() { 210 | data := []struct { 211 | args []string 212 | }{ 213 | {[]string{"--help"}}, 214 | {[]string{"--stderrthreshold", "error", "--help"}}, 215 | {[]string{"--help", "--stderrthreshold", "error"}}, 216 | } 217 | for _, d := range data { 218 | cmd := run(&s.Suite, d.args...) 219 | wait(&s.Suite, cmd, 0) 220 | s.Empty(cmd.out.String()) 221 | s.Regexp("Usage: sourcachefs .*mount_point", cmd.err.String(), "No usage line found") 222 | s.Regexp("-logtostderr", cmd.err.String(), "No flag information found") 223 | } 224 | } 225 | 226 | func (s *FlagParsingSuite) TestHelpWithInvalidSyntax() { 227 | data := []struct { 228 | args []string 229 | expectedStderr string 230 | }{ 231 | {[]string{"--invalid_flag", "--help"}, "not defined.*-invalid_flag"}, 232 | {[]string{"--help", "foo"}, "number of arguments.* expected 0"}, 233 | } 234 | for _, d := range data { 235 | cmd := run(&s.Suite, d.args...) 236 | wait(&s.Suite, cmd, 2) 237 | s.Empty(cmd.out.String()) 238 | s.Regexp(d.expectedStderr, cmd.err.String(), "No error details found") 239 | s.Regexp("Type 'sourcachefs -help'", cmd.err.String(), "No instructions found") 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /cmd/sourcachefs/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | "net/http" 21 | _ "net/http/pprof" // Automatically exports pprof endpoints. 22 | "os" 23 | "path/filepath" 24 | "regexp" 25 | 26 | "github.com/golang/glog" 27 | 28 | "github.com/jmmv/sourcachefs/internal/cache" 29 | "github.com/jmmv/sourcachefs/internal/fs" 30 | "github.com/jmmv/sourcachefs/internal/real" 31 | "github.com/jmmv/sourcachefs/internal/stats" 32 | ) 33 | 34 | // progname computes and returns the name of the current program. 35 | func progname() string { 36 | return filepath.Base(os.Args[0]) 37 | } 38 | 39 | // flagUsage redirects the user to the -help flag. This is the handler to pass to the flag module 40 | // for usage errors. 41 | func flagUsage() { 42 | fmt.Fprintf(os.Stderr, "Type '%s -help' for usage details\n", progname()) 43 | } 44 | 45 | // usageError prints an error triggered by the user. 46 | // 47 | // This function receives an error instead of just a string so that the linter can catch malformed 48 | // errors when formatted with fmt.Errorf. 49 | func usageError(err error) { 50 | fmt.Fprintf(os.Stderr, "%s: %v\n", progname(), err) 51 | flagUsage() 52 | } 53 | 54 | // runHTTPServer starts and runs the loop of the HTTP server. We use this to serve stats and the 55 | // pprof endpoints. 56 | func runHTTPServer(address string, root string, s *stats.Stats) { 57 | glog.Infof("starting HTTP server listening on %s", address) 58 | stats.SetupHTTPHandlers(s, "/stats", root) 59 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 60 | http.Redirect(w, r, "/stats/summary", http.StatusFound) 61 | }) 62 | err := http.ListenAndServe(address, nil) 63 | if err != nil { 64 | glog.Fatalf("cannot start HTTP server: %s", err) 65 | } 66 | } 67 | 68 | // showHelp prints the interactive help usage message. 69 | func showHelp() { 70 | fmt.Fprintf(os.Stderr, "Usage: %s \n\n", progname()) 71 | flag.PrintDefaults() 72 | } 73 | 74 | // main is the program entry point. 75 | func main() { 76 | // TODO(jmmv): Add daemon mode and enable as default. 77 | var help = flag.Bool("help", false, "show help and exit") 78 | var listenAddress = flag.String("listen_address", "", "enable HTTP server on the given address for pprof support and statistics tracking") 79 | var cachedPathRegex = flag.String("cached_path_regex", ".*", "only cache files whose path relative to the root match this regex") 80 | flag.Usage = flagUsage 81 | flag.Parse() 82 | 83 | if *help { 84 | if flag.NArg() != 0 { 85 | usageError(fmt.Errorf("invalid number of arguments; expected 0")) 86 | os.Exit(2) 87 | } 88 | showHelp() 89 | os.Exit(0) 90 | } 91 | 92 | if flag.NArg() != 3 { 93 | usageError(fmt.Errorf("invalid number of arguments; expected 3")) 94 | os.Exit(2) 95 | } 96 | targetDir := flag.Arg(0) 97 | cacheDir := flag.Arg(1) 98 | mountPoint := flag.Arg(2) 99 | 100 | compiledCachedPathRegex, err := regexp.Compile(*cachedPathRegex) 101 | if err != nil { 102 | glog.Fatalf("failed to compile regexp %s: %s", *cachedPathRegex, err) 103 | } 104 | 105 | stats := stats.NewStats() 106 | syscalls := real.NewSyscalls(stats) 107 | 108 | cache, err := cache.NewCache(cacheDir, syscalls) 109 | if err != nil { 110 | glog.Fatalf("failed to initialize cache: %s", err) 111 | } 112 | defer cache.Close() 113 | 114 | if len(*listenAddress) > 0 { 115 | go runHTTPServer(*listenAddress, mountPoint, stats) 116 | } 117 | 118 | err = fs.Loop(targetDir, syscalls, cache, stats, mountPoint, compiledCachedPathRegex) 119 | if err != nil { 120 | glog.Fatalf("failed to serve file system: %s", err) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /configure: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # Copyright 2016 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy 6 | # of the License at: 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | ProgName="${0##*/}" 17 | 18 | # Static package settings. 19 | DEPSDIR="$(cd "$(dirname "${0}")" && pwd -P)/deps" 20 | PKGNAME=sourcachefs 21 | SRCDIR="$(cd "$(dirname "${0}")" && pwd -P)" 22 | 23 | # Dynamically-configured settings. 24 | APP_ID= 25 | CLEANFILES= 26 | CLEANALLFILES= 27 | GO_APPENGINE= 28 | USE_LOCAL_MYSQL= 29 | 30 | # List of variables exposed to the Makefile. 31 | MK_VARS="CLEANFILES CLEANALLFILES DEPSDIR GOROOT PKGNAME SRCDIR" 32 | 33 | info() { 34 | echo "${ProgName}: I: ${@}" 1>&2 35 | } 36 | 37 | err() { 38 | echo "${ProgName}: E: ${@}" 1>&2 39 | exit 1 40 | } 41 | 42 | setup_git() { 43 | local git_dir="${1}"; shift 44 | 45 | cd "${git_dir}/hooks" 46 | for hook in ../../admin/pre-commit; do 47 | info "Installing git hook ${hook##*/}" 48 | ln -s -f "${hook}" . 49 | done 50 | cd - >/dev/null 2>&1 51 | 52 | # We don't add the hooks to CLEAN*FILES on purpose. Once we have run 53 | # configure once, we want the git checkout to remain configured with the 54 | # hooks because it's perfectly reasonable to attempt to commit a change 55 | # after running a "make clean". 56 | } 57 | 58 | find_in_path() { 59 | local prog="${1}"; shift 60 | 61 | local oldifs="${IFS}" 62 | IFS=: 63 | set -- ${PATH} 64 | IFS="${oldifs}" 65 | 66 | while [ ${#} -gt 0 ]; do 67 | if [ -x "${1}/${prog}" ]; then 68 | echo "${1}/${prog}" 69 | break 70 | else 71 | shift 72 | fi 73 | done 74 | } 75 | 76 | # TODO(jmmv): This is ugly. Fetching dependencies from their HEAD is bad 77 | # because we are pulling "untested" code and therefore can introduce 78 | # regressions without knowing about them. We should either be able to rely 79 | # on dependencies provided by the OS via its packaging system, or we should 80 | # be able to specify the specific version to retrieve from the remote site. 81 | get_go_dep() { 82 | local dependency="${1}"; shift 83 | 84 | mkdir -p "${DEPSDIR}" 85 | info "Getting ${dependency}" 86 | GOPATH="${DEPSDIR}" "${GOROOT}/bin/go" get -t "${dependency}" \ 87 | || err "Failed to get ${dependency}" 88 | } 89 | 90 | set_goroot() { 91 | local user_override="${1}"; shift 92 | 93 | if [ -n "${user_override}" ]; then 94 | [ -e "${user_override}/bin/go" ] || err "go not found in" \ 95 | "${user_override}; bogus argument to --goroot?" 96 | GOROOT="${user_override}" 97 | else 98 | local go="$(find_in_path go)" 99 | [ -n "${go}" ] || err "Cannot find go in path; pass" \ 100 | "--goroot=/path/to/goroot to configure" 101 | local dir 102 | if [ -h "${go}" ]; then 103 | local target="$(readlink "${go}")" 104 | case "${target}" in 105 | /*) dir="$(dirname "${target}")/.." ;; 106 | *) dir="$(dirname "${go}")/$(dirname "${target}")/.." ;; 107 | esac 108 | else 109 | dir="$(dirname "${go}")/.." 110 | fi 111 | GOROOT="$(cd "${dir}" && pwd -P)" 112 | fi 113 | 114 | info "Using Go from: ${GOROOT}" 115 | } 116 | 117 | setup_vscode() { 118 | # These dependencies come from the documentation of the Go plugin for the 119 | # Visual Studio Code editor. 120 | get_go_dep github.com/derekparker/delve/cmd/dlv 121 | get_go_dep github.com/golang/lint/golint 122 | get_go_dep github.com/lukehoban/go-outline 123 | get_go_dep github.com/newhook/go-symbols 124 | get_go_dep github.com/nsf/gocode 125 | get_go_dep github.com/ramya-rao-a/go-outline 126 | get_go_dep github.com/rogpeppe/godef 127 | get_go_dep github.com/tpng/gopkgs 128 | get_go_dep golang.org/x/tools/cmd/godoc 129 | get_go_dep golang.org/x/tools/cmd/goimports 130 | get_go_dep golang.org/x/tools/cmd/gorename 131 | get_go_dep golang.org/x/tools/cmd/guru 132 | get_go_dep sourcegraph.com/sqs/goreturns 133 | 134 | # TODO(jmmv): Should be a global, but I do not want to give the impression 135 | # in the script that GOPATH should be available all the time because, 136 | # depending on what we are doing (e.g. fetching dependencies) we do not 137 | # want the default value. 138 | local gopath="${SRCDIR}:${DEPSDIR}" 139 | 140 | CLEANALLFILES="${CLEANALLFILES} .vscode/settings.json" 141 | { 142 | echo '// AUTOMATICALLY GENERATED!!!' 143 | echo '// EDIT settings.json.in INSTEAD' 144 | sed -e "s,__GOPATH__,${gopath},g" -e "s,__GOROOT__,${GOROOT},g" \ 145 | .vscode/settings.json.in 146 | } >.vscode/settings.json 147 | } 148 | 149 | generate_makefile() { 150 | local src="${1}"; shift 151 | local dest="${1}"; shift 152 | 153 | info "Generating ${dest}" 154 | echo "# AUTOMATICALLY GENERATED; DO NOT EDIT!" >"${dest}.tmp" 155 | for var in ${MK_VARS}; do 156 | local value 157 | eval "value=\"\$${var}\"" 158 | echo "${var} = ${value}" >>"${dest}.tmp" 159 | done 160 | cat "${src}" >>"${dest}.tmp" 161 | mv "${dest}.tmp" "${dest}" 162 | } 163 | 164 | main() { 165 | cd "${SRCDIR}" 166 | 167 | local enable_vscode=no 168 | local goroot= 169 | for arg in "${@}"; do 170 | case "${arg}" in 171 | --enable-vscode) enable_vscode=yes ;; 172 | --goroot=*) goroot="${arg#*=}" ;; 173 | *) err "Unknown argument ${arg}" ;; 174 | esac 175 | done 176 | 177 | set_goroot "${goroot}" 178 | 179 | CLEANALLFILES="${DEPSDIR}" 180 | 181 | [ -d .git ] && setup_git .git 182 | 183 | get_go_dep bazil.org/fuse 184 | get_go_dep github.com/dustin/go-humanize 185 | get_go_dep github.com/golang/glog 186 | get_go_dep github.com/mattn/go-sqlite3 187 | get_go_dep github.com/stretchr/testify/suite 188 | get_go_dep golang.org/x/lint/golint 189 | 190 | [ "${enable_vscode}" = no ] || setup_vscode 191 | 192 | CLEANALLFILES="${CLEANALLFILES} src" 193 | mkdir -p src/github.com/jmmv 194 | [ -h src/github.com/jmmv/sourcachefs ] \ 195 | || ln -fs ../../.. src/github.com/jmmv/sourcachefs \ 196 | || err "Failed to create src directory to mimic GOPATH" 197 | 198 | CLEANALLFILES="${CLEANALLFILES} Makefile" 199 | generate_makefile Makefile.in Makefile 200 | } 201 | 202 | main "${@}" 203 | -------------------------------------------------------------------------------- /internal/cache/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "path/filepath" 19 | 20 | "github.com/golang/glog" 21 | 22 | "github.com/jmmv/sourcachefs/internal/real" 23 | "github.com/jmmv/sourcachefs/internal/stats" 24 | ) 25 | 26 | // Cache implements the unified cache. 27 | type Cache struct { 28 | Metadata *MetadataCache 29 | Content *ContentCache 30 | } 31 | 32 | // NewCache instantantiates a new cache. 33 | func NewCache(root string, syscalls real.Syscalls) (*Cache, error) { 34 | contentCache := NewContentCache(root, syscalls) 35 | 36 | metadataFile := filepath.Join(root, "metadata.db") 37 | metadataCache, err := NewMetadataCache(metadataFile) 38 | if err != nil { 39 | if err2 := syscalls.Remove(stats.LocalDomain, metadataFile); err2 != nil { 40 | glog.Warning("failed to delete metadata cache on error") 41 | } 42 | if err2 := syscalls.Remove(stats.LocalDomain, root); err2 != nil { 43 | glog.Warning("failed to delete cache directory") 44 | } 45 | return nil, err 46 | } 47 | 48 | return &Cache{ 49 | Metadata: metadataCache, 50 | Content: contentCache, 51 | }, nil 52 | } 53 | 54 | // Close closes the cache in a controller manner. 55 | func (c *Cache) Close() error { 56 | return c.Metadata.Close() 57 | } 58 | -------------------------------------------------------------------------------- /internal/cache/cache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "os" 19 | "path/filepath" 20 | "testing" 21 | 22 | "github.com/jmmv/sourcachefs/internal/real" 23 | "github.com/jmmv/sourcachefs/internal/stats" 24 | "github.com/jmmv/sourcachefs/internal/test" 25 | "github.com/stretchr/testify/suite" 26 | ) 27 | 28 | type NewCacheSuite struct { 29 | test.SuiteWithTempDir 30 | } 31 | 32 | func TestNewCache(t *testing.T) { 33 | suite.Run(t, new(NewCacheSuite)) 34 | } 35 | 36 | func (s *NewCacheSuite) TestOk() { 37 | stats := stats.NewStats() 38 | cache, err := NewCache(s.TempDir(), real.NewSyscalls(stats)) 39 | s.NoError(err) 40 | defer cache.Close() 41 | 42 | s.NotNil(cache.Metadata) 43 | s.NotNil(cache.Content) 44 | } 45 | 46 | func (s *NewCacheSuite) TestFailMissingDirectory() { 47 | missingDir := filepath.Join(s.TempDir(), "subdir") 48 | 49 | stats := stats.NewStats() 50 | _, err := NewCache(missingDir, real.NewSyscalls(stats)) 51 | s.Error(err) 52 | 53 | _, err = os.Stat(missingDir) 54 | s.True(os.IsNotExist(err), "cache directory was created but should not have been") 55 | } 56 | -------------------------------------------------------------------------------- /internal/cache/content.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "crypto/md5" 19 | "encoding/hex" 20 | "fmt" 21 | "io" 22 | "io/ioutil" 23 | "os" 24 | "path/filepath" 25 | 26 | "github.com/jmmv/sourcachefs/internal/real" 27 | "github.com/jmmv/sourcachefs/internal/stats" 28 | ) 29 | 30 | // ContentCache implements the content cache. 31 | type ContentCache struct { 32 | root string 33 | syscalls real.Syscalls 34 | } 35 | 36 | // Key represents a key in the cache. Keys are file digests. 37 | type Key string 38 | 39 | // NewContentCache instantiates a new content cache. 40 | func NewContentCache(root string, syscalls real.Syscalls) *ContentCache { 41 | return &ContentCache{ 42 | root: root, 43 | syscalls: syscalls, 44 | } 45 | } 46 | 47 | // Root returns the path to the root of the cache. 48 | func (cache *ContentCache) Root() string { 49 | return cache.root 50 | } 51 | 52 | func (cache *ContentCache) bucketForKey(key Key) string { 53 | return string(key)[0:2] 54 | } 55 | 56 | func (cache *ContentCache) pathForKey(key Key) string { 57 | bucket := cache.bucketForKey(key) 58 | return filepath.Join(cache.root, bucket, string(key)) 59 | } 60 | 61 | // GetFileForContents returns the path to file containing the contents for the key. 62 | func (cache *ContentCache) GetFileForContents(key Key) (string, error) { 63 | // TODO(jmmv): Have an internal hasKey() that takes the joined path so that we don't do it 64 | // twice: once inside HasKey and once below. 65 | found, err := cache.HasKey(key) 66 | if err != nil { 67 | return "", err 68 | } 69 | if !found { 70 | return "", fmt.Errorf("attempted to get non-existent key %s", key) 71 | } 72 | return cache.pathForKey(key), nil 73 | } 74 | 75 | // HasKey checks if the given key is in the cache. 76 | func (cache *ContentCache) HasKey(key Key) (bool, error) { 77 | path := cache.pathForKey(key) 78 | _, err := cache.syscalls.Lstat(stats.LocalDomain, path) 79 | if err == nil { 80 | return true, nil 81 | } else if os.IsNotExist(err) { 82 | return false, nil 83 | } else { 84 | return false, err 85 | } 86 | } 87 | 88 | func copy2(dst1 io.Writer, dst2 io.Writer, src io.Reader) (written int64, err error) { 89 | buf := make([]byte, 32*1024) 90 | for { 91 | nr, er := src.Read(buf) 92 | if nr > 0 { 93 | nw, ew := dst1.Write(buf[0:nr]) 94 | if nw > 0 { 95 | written += int64(nw) 96 | } 97 | if ew != nil { 98 | err = ew 99 | break 100 | } 101 | if nr != nw { 102 | err = io.ErrShortWrite 103 | break 104 | } 105 | 106 | dst2.Write(buf[0:nr]) 107 | // TODO 108 | } 109 | if er == io.EOF { 110 | break 111 | } 112 | if er != nil { 113 | err = er 114 | break 115 | } 116 | } 117 | return written, err 118 | } 119 | 120 | // PutContents stores the given file in the cache and returns the key. 121 | func (cache *ContentCache) PutContents(path string) (Key, error) { 122 | input, err := cache.syscalls.Open(stats.RemoteDomain, path) 123 | if err != nil { 124 | return "", fmt.Errorf("failed to open %s: %s", path, err) 125 | } 126 | defer input.Close() 127 | 128 | temp, err := ioutil.TempFile(cache.root, "partial") 129 | if err != nil { 130 | return "", fmt.Errorf("cannot create temporary file: %s", err) 131 | } 132 | defer temp.Close() 133 | defer cache.syscalls.Remove(stats.LocalDomain, temp.Name()) 134 | 135 | h := md5.New() 136 | _, err = copy2(temp, h, input) 137 | if err != nil { 138 | return "", fmt.Errorf("failed to fetch %s: %s", path, err) 139 | } 140 | input.Close() 141 | temp.Close() 142 | 143 | key := Key(hex.EncodeToString(h.Sum(nil))) 144 | 145 | found, err := cache.HasKey(key) 146 | if err != nil { 147 | return "", fmt.Errorf("failed to check for existence: %s", err) 148 | } 149 | if found { 150 | // TODO: Is this nice? 151 | return key, nil 152 | } 153 | 154 | bucket := cache.bucketForKey(key) 155 | if err = cache.syscalls.Mkdir(stats.LocalDomain, filepath.Join(cache.root, bucket), 0755); err != nil && !os.IsExist(err) { 156 | return "", fmt.Errorf("failed to create bucket %s: %s", bucket, err) 157 | } 158 | 159 | err = cache.syscalls.Rename(stats.LocalDomain, temp.Name(), cache.pathForKey(key)) 160 | if err != nil { 161 | return "", err 162 | } 163 | 164 | return key, nil 165 | } 166 | -------------------------------------------------------------------------------- /internal/cache/content_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "io/ioutil" 19 | "os" 20 | "path/filepath" 21 | "testing" 22 | 23 | "github.com/jmmv/sourcachefs/internal/real" 24 | "github.com/jmmv/sourcachefs/internal/stats" 25 | "github.com/jmmv/sourcachefs/internal/test" 26 | "github.com/stretchr/testify/suite" 27 | ) 28 | 29 | func newTemporaryContentCache(s *suite.Suite, dir string) *ContentCache { 30 | stats := stats.NewStats() 31 | cache := NewContentCache(dir, real.NewSyscalls(stats)) 32 | s.T().Logf("Initialized content cache: %s", dir) 33 | return cache 34 | } 35 | 36 | type PutAndGetSuite struct { 37 | test.SuiteWithTempDir 38 | } 39 | 40 | func TestContentCache(t *testing.T) { 41 | suite.Run(t, new(PutAndGetSuite)) 42 | } 43 | 44 | func (s *PutAndGetSuite) TestPutAndGet() { 45 | cache := newTemporaryContentCache(&s.Suite, filepath.Join(s.TempDir(), "cache")) 46 | s.NoError(os.Mkdir(cache.Root(), 0755)) 47 | 48 | path1 := filepath.Join(s.TempDir(), "file1") 49 | s.NoError(ioutil.WriteFile(path1, []byte("Some text\n"), 0666)) 50 | 51 | key, err := cache.PutContents(path1) 52 | s.NoError(err) 53 | 54 | expKey := Key("3c825ca59d58209eae5924221497780c") 55 | s.Equal(expKey, key) 56 | s.NoError(test.CheckFileExists(filepath.Join(cache.Root(), "3c", string(key)))) 57 | 58 | found, err := cache.HasKey(key) 59 | s.NoError(err) 60 | s.True(found) 61 | 62 | cached, err := cache.GetFileForContents(key) 63 | s.NoError(err) 64 | s.Equal(string(cached), filepath.Join(cache.Root(), "3c", string(key))) 65 | } 66 | -------------------------------------------------------------------------------- /internal/cache/metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "database/sql" 19 | "errors" 20 | "fmt" 21 | "os" 22 | "path/filepath" 23 | "strconv" 24 | "strings" 25 | "time" 26 | 27 | "github.com/golang/glog" 28 | ) 29 | 30 | const ( 31 | dirEntrySeparator = "/" 32 | dirEntryFieldSeparator = "\x01" 33 | ) 34 | 35 | var schemaDDL = []string{ 36 | "PRAGMA cache_size = 10000", 37 | "PRAGMA journal_mode = OFF", 38 | "PRAGMA read_uncommitted = ON", 39 | "PRAGMA synchronous = OFF", 40 | 41 | `CREATE TABLE IF NOT EXISTS paths ( 42 | path STRING PRIMARY KEY, 43 | type STRING NOT NULL, 44 | 45 | mode INTEGER, 46 | size INTEGER, 47 | mtime_nsec INTEGER, 48 | 49 | dirEntries STRING, 50 | contentHash STRING, 51 | 52 | target STRING 53 | )`, 54 | } 55 | 56 | // MetadataCache is a persistent database of the file system metadata for all known entries. 57 | type MetadataCache struct { 58 | db *sqliteDB 59 | 60 | getDirEntriesStmt *RetriableStmt 61 | getInodeStmt *RetriableStmt 62 | getFullEntryStmt *RetriableStmt 63 | updateEntryStmt *RetriableStmt 64 | putNewEntryStmt *RetriableStmt 65 | putFileInfoStmt *RetriableStmt 66 | putContentHashStmt *RetriableStmt 67 | putSymlinkTargetStmt *RetriableStmt 68 | putDirEntriesStmt *RetriableStmt 69 | } 70 | 71 | // NewMetadataCache opens a new connection to the given database. 72 | func NewMetadataCache(path string) (*MetadataCache, error) { 73 | db, err := newSqliteDb(path, schemaDDL) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | var stmtError error 79 | prepareStmt := func(query string) *RetriableStmt { 80 | if stmtError != nil { 81 | return nil 82 | } 83 | stmt, err := db.RetriablePrepare(query) 84 | if err != nil { 85 | stmtError = err 86 | return nil 87 | } 88 | return stmt 89 | } 90 | 91 | cache := &MetadataCache{ 92 | db: db, 93 | getDirEntriesStmt: prepareStmt("SELECT dirEntries FROM paths WHERE ROWID = ?"), 94 | getInodeStmt: prepareStmt("SELECT ROWID FROM paths WHERE path = ?"), 95 | getFullEntryStmt: prepareStmt("SELECT ROWID, type, mode, size, mtime_nsec, dirEntries, contentHash, target FROM paths WHERE path = ?"), 96 | updateEntryStmt: prepareStmt("UPDATE paths SET type = ? WHERE path = ?"), 97 | putNewEntryStmt: prepareStmt("INSERT INTO paths (path, type) VALUES (?, ?)"), 98 | putFileInfoStmt: prepareStmt("UPDATE paths SET mode = ?, size = ?, mtime_nsec = ? WHERE ROWID = ?"), 99 | putContentHashStmt: prepareStmt("UPDATE paths SET contentHash = ? WHERE ROWID = ?"), 100 | putSymlinkTargetStmt: prepareStmt("UPDATE paths SET target = ? WHERE ROWID = ?"), 101 | putDirEntriesStmt: prepareStmt("UPDATE paths SET dirEntries = ? WHERE ROWID = ?"), 102 | } 103 | 104 | if stmtError != nil { 105 | db.Close() 106 | return nil, stmtError 107 | } 108 | 109 | return cache, nil 110 | } 111 | 112 | func (cache *MetadataCache) closeStmts() error { 113 | var firstError error 114 | closeStmt := func(stmt **RetriableStmt) { 115 | if *stmt == nil || firstError != nil { 116 | return 117 | } 118 | if err := (*stmt).Close(); err != nil { 119 | firstError = err 120 | return 121 | } 122 | *stmt = nil 123 | } 124 | closeStmt(&cache.putDirEntriesStmt) 125 | closeStmt(&cache.putSymlinkTargetStmt) 126 | closeStmt(&cache.putContentHashStmt) 127 | closeStmt(&cache.putFileInfoStmt) 128 | closeStmt(&cache.putNewEntryStmt) 129 | closeStmt(&cache.updateEntryStmt) 130 | closeStmt(&cache.getFullEntryStmt) 131 | closeStmt(&cache.getInodeStmt) 132 | closeStmt(&cache.getDirEntriesStmt) 133 | return firstError 134 | } 135 | 136 | // Close terminates the connection to the database. 137 | func (cache *MetadataCache) Close() error { 138 | if err := cache.closeStmts(); err != nil { 139 | return err 140 | } 141 | return cache.db.Close() 142 | } 143 | 144 | func rowIDToInode(rowID int64) uint64 { 145 | if rowID <= 0 { 146 | panic("invalid ROWID") 147 | } 148 | return uint64(rowID) 149 | } 150 | 151 | func addDirEntry(dirEntries *map[string]OneDirEntry, inode uint64, basename string, typeName string) error { 152 | _, has := (*dirEntries)[basename] 153 | if has { 154 | return nil 155 | } 156 | 157 | dirEntry := OneDirEntry{ 158 | Inode: inode, 159 | } 160 | switch typeName { 161 | case "directory": 162 | dirEntry.Valid = true 163 | dirEntry.ModeType = os.ModeDir 164 | case "file": 165 | dirEntry.Valid = true 166 | dirEntry.ModeType = 0 167 | case "noentry": 168 | dirEntry.Valid = false 169 | case "symlink": 170 | dirEntry.Valid = true 171 | dirEntry.ModeType = os.ModeSymlink 172 | default: 173 | return errors.New("Invalid type " + typeName) 174 | } 175 | (*dirEntries)[basename] = dirEntry 176 | return nil 177 | } 178 | 179 | // GetDirEntries reads the directory entries stored in an inode. 180 | func (cache *MetadataCache) GetDirEntries(inode uint64, outEntries *map[string]OneDirEntry) error { 181 | var dirEntries sql.NullString 182 | err := cache.getDirEntriesStmt.RetryQueryRow("NO PATH", inode).Scan(&dirEntries) 183 | if err != nil { 184 | return err 185 | } 186 | if !dirEntries.Valid { 187 | return nil 188 | } 189 | return parseDirEntries(dirEntries.String, outEntries) 190 | } 191 | 192 | func parseDirEntries(all string, outEntries *map[string]OneDirEntry) error { 193 | if len(all) == 0 { 194 | return nil 195 | } 196 | for _, one := range strings.Split(all, dirEntrySeparator) { 197 | parts := strings.Split(one, dirEntryFieldSeparator) 198 | basename := parts[0] 199 | inode, err := strconv.ParseUint(parts[1], 10, 64) 200 | if err != nil { 201 | return err 202 | } 203 | typeName := parts[2] 204 | 205 | err = addDirEntry(outEntries, inode, basename, typeName) 206 | if err != nil { 207 | return err 208 | } 209 | } 210 | return nil 211 | } 212 | 213 | // GetFullEntry reads the full metadata for a file. 214 | func (cache *MetadataCache) GetFullEntry(path string) (*CachedRow, error) { 215 | var rowID int64 216 | var typeName string 217 | var mode sql.NullInt64 218 | var size sql.NullInt64 219 | var mtimeNsec sql.NullInt64 220 | var dirEntries sql.NullString 221 | var contentHash sql.NullString 222 | var target sql.NullString 223 | err := cache.getFullEntryStmt.RetryQueryRow(path, path).Scan(&rowID, &typeName, &mode, &size, &mtimeNsec, &dirEntries, &contentHash, &target) 224 | switch { 225 | case err == sql.ErrNoRows: 226 | return nil, nil 227 | case err != nil: 228 | return nil, err 229 | default: 230 | row := &CachedRow{ 231 | Inode: uint64(rowID), 232 | TypeName: typeName, 233 | } 234 | if mode.Valid || size.Valid || mtimeNsec.Valid { 235 | if !(mode.Valid && size.Valid && mtimeNsec.Valid) { 236 | glog.Fatal("incomplete fileInfo data in cache row") 237 | } 238 | mtime := time.Unix(0, mtimeNsec.Int64) 239 | fileInfo := newFileInfo(path, size.Int64, os.FileMode(mode.Int64), mtime) 240 | row.FileInfo = &fileInfo 241 | } 242 | if dirEntries.Valid { 243 | row.DirEntries = make(map[string]OneDirEntry) 244 | err := parseDirEntries(dirEntries.String, &row.DirEntries) 245 | if err != nil { 246 | return nil, err 247 | } 248 | } 249 | if contentHash.Valid { 250 | row.ContentHash = (*Key)(&contentHash.String) 251 | } 252 | if target.Valid { 253 | row.Target = &target.String 254 | } 255 | return row, nil 256 | } 257 | } 258 | 259 | func (cache *MetadataCache) resultToInode(result sql.Result) (uint64, error) { 260 | rowID, err := result.LastInsertId() 261 | if err != nil { 262 | return 0, err 263 | } 264 | if rowID < 0 { 265 | glog.Fatal("got negative row ID; cannot use as inode number") 266 | } 267 | return uint64(rowID), nil 268 | } 269 | 270 | func (cache *MetadataCache) putNew(path string, typeName string) (uint64, error) { 271 | glog.Infof("putNew: path %s, type %s", path, typeName) 272 | 273 | result, err := cache.putNewEntryStmt.RetryExec(path, path, typeName) 274 | if err != nil { 275 | glog.Errorf("failed to put %s: %s", path, err) 276 | return 0, err 277 | } 278 | return cache.resultToInode(result) 279 | } 280 | 281 | func (cache *MetadataCache) putNewOrUpdate(path string, typeName string) error { 282 | glog.Infof("putNewOrUpdate: path %s, type %s", path, typeName) 283 | 284 | result, err := cache.updateEntryStmt.RetryExec(path, typeName, path) 285 | if err != nil { 286 | glog.Errorf("failed to update %s: %s", path, err) 287 | return err 288 | } 289 | count, err := result.RowsAffected() 290 | if err != nil { 291 | glog.Errorf("failed to update %s: %s", path, err) 292 | return err 293 | } 294 | if count == 0 { 295 | _, err = cache.putNewEntryStmt.RetryExec(path, path, typeName) 296 | if err != nil { 297 | glog.Errorf("failed to put %s: %s", path, err) 298 | return err 299 | } 300 | } else if count == 1 { 301 | glog.Infof("putNewOrUpdate updated %s to %s", path, typeName) 302 | } else { 303 | panic("Updated more than one row") 304 | } 305 | return err 306 | } 307 | 308 | func sqlTypeName(modeType os.FileMode) string { 309 | if modeType & ^os.ModeType != 0 { 310 | panic("Input modeType contains permissions") 311 | } 312 | 313 | switch modeType { 314 | case 0: 315 | return "file" 316 | case os.ModeDir: 317 | return "directory" 318 | case os.ModeSymlink: 319 | return "symlink" 320 | default: 321 | panic("Unknown entry type") 322 | } 323 | } 324 | 325 | // PutNewWithType stores a new entry for a path given its file type. 326 | func (cache *MetadataCache) PutNewWithType(path string, fileMode os.FileMode) (uint64, error) { 327 | if fileMode & ^os.ModeType != 0 { 328 | panic("fileMode contains more than just the file type") 329 | } 330 | typeName := sqlTypeName(fileMode & os.ModeType) 331 | return cache.putNew(path, typeName) 332 | } 333 | 334 | // PutNewNoEntry stores a new whiteout entry for the given path. 335 | func (cache *MetadataCache) PutNewNoEntry(path string) (uint64, error) { 336 | return cache.putNew(path, "noentry") 337 | } 338 | 339 | // PutNewOrUpdateWithType stores a new entry or updates an existing one to match the given type. 340 | func (cache *MetadataCache) PutNewOrUpdateWithType(path string, fileMode os.FileMode) error { 341 | if fileMode & ^os.ModeType != 0 { 342 | panic("fileMode contains more than just the file type") 343 | } 344 | typeName := sqlTypeName(fileMode & os.ModeType) 345 | return cache.putNewOrUpdate(path, typeName) 346 | } 347 | 348 | // PutNewOrUpdateNoEntry stores a new whiteout entry or updates an existing one. 349 | func (cache *MetadataCache) PutNewOrUpdateNoEntry(path string) error { 350 | return cache.putNewOrUpdate(path, "noentry") 351 | } 352 | 353 | func checkUpdateAffectedOneRow(result sql.Result) error { 354 | count, err := result.RowsAffected() 355 | if err != nil { 356 | return err 357 | } 358 | if count != 1 { 359 | glog.WarningDepth(1, fmt.Sprintf("update affected %d rows; expected only 1", count)) 360 | return nil 361 | } 362 | return nil 363 | } 364 | 365 | // PutFileInfo stores new file details for an existing inode. 366 | func (cache *MetadataCache) PutFileInfo(inode uint64, fileInfo os.FileInfo) error { 367 | result, err := cache.putFileInfoStmt.RetryExec("NO PATH", fileInfo.Mode(), fileInfo.Size(), fileInfo.ModTime().UnixNano(), inode) 368 | if err != nil { 369 | return err 370 | } 371 | return checkUpdateAffectedOneRow(result) 372 | } 373 | 374 | // PutContentHash stores a new content hash for an existing inode. 375 | func (cache *MetadataCache) PutContentHash(inode uint64, contentHash Key) error { 376 | result, err := cache.putContentHashStmt.RetryExec("NO PATH", string(contentHash), inode) 377 | if err != nil { 378 | return err 379 | } 380 | return checkUpdateAffectedOneRow(result) 381 | } 382 | 383 | // PutSymlinkTarget stores a new symlink target for an existing inode. 384 | func (cache *MetadataCache) PutSymlinkTarget(inode uint64, target string) error { 385 | result, err := cache.putSymlinkTargetStmt.RetryExec("NO PATH", target, inode) 386 | if err != nil { 387 | return err 388 | } 389 | return checkUpdateAffectedOneRow(result) 390 | } 391 | 392 | // PutDirEntries stores a new set of directory entries for an existing inode. 393 | func (cache *MetadataCache) PutDirEntries(inode uint64, path string, entries []os.FileInfo) error { 394 | formatted := make([]string, 0, len(entries)) 395 | for _, dirEntry := range entries { 396 | entryPath := filepath.Join(path, dirEntry.Name()) 397 | 398 | typeName := sqlTypeName(dirEntry.Mode() & os.ModeType) 399 | 400 | var rowID int64 401 | err := cache.getInodeStmt.RetryQueryRow(path, entryPath).Scan(&rowID) 402 | switch { 403 | case err == sql.ErrNoRows: 404 | result, err := cache.putNewEntryStmt.RetryExec(path, entryPath, typeName) 405 | if err != nil { 406 | glog.Errorf("failed to put %s: %s", path, err) 407 | return err 408 | } 409 | 410 | rowID, err = result.LastInsertId() 411 | if err != nil { 412 | glog.Errorf("failed to put %s: %s", path, err) 413 | return err 414 | } 415 | case err != nil: 416 | return err 417 | default: 418 | } 419 | inode := uint64(rowID) 420 | 421 | formatted = append(formatted, fmt.Sprintf("%s%s%d%s%s", dirEntry.Name(), dirEntryFieldSeparator, inode, dirEntryFieldSeparator, typeName)) 422 | } 423 | 424 | result, err := cache.putDirEntriesStmt.RetryExec("NO PATH", strings.Join(formatted, dirEntrySeparator), inode) 425 | if err != nil { 426 | return err 427 | } 428 | return checkUpdateAffectedOneRow(result) 429 | } 430 | -------------------------------------------------------------------------------- /internal/cache/metadata_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "os" 19 | "path/filepath" 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/suite" 24 | 25 | "github.com/jmmv/sourcachefs/internal/test" 26 | ) 27 | 28 | type NewMetadataCacheSuite struct { 29 | test.SuiteWithTempDir 30 | } 31 | 32 | func TestNewMetadataCache(t *testing.T) { 33 | suite.Run(t, new(NewMetadataCacheSuite)) 34 | } 35 | 36 | func (s *NewMetadataCacheSuite) TestOk() { 37 | path := filepath.Join(s.TempDir(), "metadata.db") 38 | cache, err := NewMetadataCache(path) 39 | s.Require().NoError(err) 40 | defer cache.Close() 41 | defer os.Remove(path) 42 | } 43 | 44 | func (s *NewMetadataCacheSuite) TestFailMissingDirectory() { 45 | missingDir := filepath.Join(s.TempDir(), "subdir/metadata.db") 46 | 47 | _, err := NewMetadataCache(missingDir) 48 | s.Require().Error(err) 49 | 50 | _, err = os.Stat(missingDir) 51 | s.True(os.IsNotExist(err), "cache directory was created but should not have been") 52 | } 53 | 54 | type MetadataCacheSuite struct { 55 | test.SuiteWithTempDir 56 | path string 57 | cache *MetadataCache 58 | } 59 | 60 | func TestMetadataCache(t *testing.T) { 61 | suite.Run(t, new(MetadataCacheSuite)) 62 | } 63 | 64 | func (s *MetadataCacheSuite) SetupTest() { 65 | s.path = filepath.Join(s.TempDir(), "metadata.db") 66 | cache, err := NewMetadataCache(s.path) 67 | s.Require().NoError(err) 68 | s.cache = cache 69 | } 70 | 71 | func (s *MetadataCacheSuite) TearDownTest() { 72 | defer os.Remove(s.path) 73 | err := s.cache.Close() 74 | s.Require().NoError(err) 75 | } 76 | 77 | func (s *MetadataCacheSuite) checkedGetFullEntry(path string, expectedType string) *CachedRow { 78 | row, err := s.cache.GetFullEntry(path) 79 | s.Require().NoError(err) 80 | s.Equal(expectedType, row.TypeName) 81 | return row 82 | } 83 | 84 | func (s *MetadataCacheSuite) TestPutNewDirectory() { 85 | inode, err := s.cache.PutNewWithType("/some/path", os.ModeDir) 86 | s.Require().NoError(err) 87 | s.Equal(inode, s.checkedGetFullEntry("/some/path", "directory").Inode) 88 | 89 | _, err = s.cache.PutNewWithType("/some/path", 0) 90 | s.Require().Error(err) 91 | s.checkedGetFullEntry("/some/path", "directory") 92 | } 93 | 94 | func (s *MetadataCacheSuite) TestPutNewFile() { 95 | inode, err := s.cache.PutNewWithType("/some/path", 0) 96 | s.Require().NoError(err) 97 | s.Equal(inode, s.checkedGetFullEntry("/some/path", "file").Inode) 98 | 99 | _, err = s.cache.PutNewWithType("/some/path", os.ModeDir) 100 | s.Require().Error(err) 101 | s.checkedGetFullEntry("/some/path", "file") 102 | } 103 | 104 | func (s *MetadataCacheSuite) TestPutNewSymlink() { 105 | inode, err := s.cache.PutNewWithType("/some/path", os.ModeSymlink) 106 | s.Require().NoError(err) 107 | s.Equal(inode, s.checkedGetFullEntry("/some/path", "symlink").Inode) 108 | 109 | _, err = s.cache.PutNewWithType("/some/path", 0) 110 | s.Require().Error(err) 111 | s.checkedGetFullEntry("/some/path", "symlink") 112 | } 113 | 114 | func (s *MetadataCacheSuite) TestPutNewNoEntry() { 115 | inode, err := s.cache.PutNewNoEntry("/some/path") 116 | s.Require().NoError(err) 117 | s.Equal(inode, s.checkedGetFullEntry("/some/path", "noentry").Inode) 118 | 119 | _, err = s.cache.PutNewNoEntry("/some/path") 120 | s.Require().Error(err) 121 | s.checkedGetFullEntry("/some/path", "noentry") 122 | } 123 | 124 | func (s *MetadataCacheSuite) TestPutNewOrUpdate() { 125 | s.Require().NoError(s.cache.PutNewOrUpdateNoEntry("/path1")) 126 | s.Require().NoError(s.cache.PutNewOrUpdateWithType("/path2", os.ModeDir)) 127 | 128 | inode1 := s.checkedGetFullEntry("/path1", "noentry").Inode 129 | inode2 := s.checkedGetFullEntry("/path2", "directory").Inode 130 | 131 | s.Require().NoError(s.cache.PutNewOrUpdateWithType("/path1", os.ModeDir)) 132 | s.Require().NoError(s.cache.PutNewOrUpdateNoEntry("/path2")) 133 | 134 | s.Equal(inode1, s.checkedGetFullEntry("/path1", "directory").Inode) 135 | s.Equal(inode2, s.checkedGetFullEntry("/path2", "noentry").Inode) 136 | 137 | s.Require().NoError(s.cache.PutNewOrUpdateNoEntry("/path1")) 138 | s.Require().NoError(s.cache.PutNewOrUpdateWithType("/path2", os.ModeSymlink)) 139 | 140 | s.Equal(inode1, s.checkedGetFullEntry("/path1", "noentry").Inode) 141 | s.Equal(inode2, s.checkedGetFullEntry("/path2", "symlink").Inode) 142 | } 143 | 144 | func (s *MetadataCacheSuite) TestPutFileInfoAndGet() { 145 | inode, err := s.cache.PutNewWithType("/path", 0) 146 | s.Require().NoError(err) 147 | // Strip monotonic clock reading for comparisons with require.Equal. 148 | // See https://github.com/stretchr/testify/issues/502 for details. 149 | fileInfo := newFileInfo("/path", 123, 0, time.Now().Round(0)) 150 | s.Require().NoError(s.cache.PutFileInfo(inode, fileInfo)) 151 | 152 | row := s.checkedGetFullEntry("/path", "file") 153 | s.Equal(fileInfo, *row.FileInfo) 154 | } 155 | 156 | func (s *MetadataCacheSuite) TestPutContentHashAndGet() { 157 | inode, err := s.cache.PutNewWithType("/path", 0) 158 | s.Require().NoError(err) 159 | 160 | s.Require().NoError(s.cache.PutContentHash(inode, "123456")) 161 | 162 | row := s.checkedGetFullEntry("/path", "file") 163 | s.Equal(Key("123456"), *row.ContentHash) 164 | } 165 | 166 | func (s *MetadataCacheSuite) TestPutSymlinkTargetAndGet() { 167 | inode, err := s.cache.PutNewWithType("/path", os.ModeSymlink) 168 | s.Require().NoError(err) 169 | 170 | s.Require().NoError(s.cache.PutSymlinkTarget(inode, "/other/path")) 171 | 172 | row := s.checkedGetFullEntry("/path", "symlink") 173 | s.Equal("/other/path", *row.Target) 174 | } 175 | 176 | func (s *MetadataCacheSuite) TestPutDirEntriesAndGet() { 177 | inode, err := s.cache.PutNewWithType("/path", os.ModeDir) 178 | s.Require().NoError(err) 179 | 180 | entries := []os.FileInfo{ 181 | newFileInfo("1", 123, 0, time.Now()), 182 | newFileInfo("2", 0, os.ModeDir, time.Now()), 183 | newFileInfo("3", 0, os.ModeSymlink, time.Now()), 184 | } 185 | 186 | s.Require().NoError(s.cache.PutDirEntries(inode, "/path", entries)) 187 | 188 | inode1 := s.checkedGetFullEntry("/path/1", "file").Inode 189 | inode2 := s.checkedGetFullEntry("/path/2", "directory").Inode 190 | inode3 := s.checkedGetFullEntry("/path/3", "symlink").Inode 191 | 192 | expEntries := map[string]OneDirEntry{ 193 | "1": {Valid: true, Inode: inode1, ModeType: 0}, 194 | "2": {Valid: true, Inode: inode2, ModeType: os.ModeDir}, 195 | "3": {Valid: true, Inode: inode3, ModeType: os.ModeSymlink}, 196 | } 197 | 198 | row := s.checkedGetFullEntry("/path", "directory") 199 | s.Equal(len(entries), len(row.DirEntries)) 200 | s.Equal(expEntries["1"], row.DirEntries["1"]) 201 | s.Equal(expEntries["2"], row.DirEntries["2"]) 202 | s.Equal(expEntries["3"], row.DirEntries["3"]) 203 | 204 | readEntries := make(map[string]OneDirEntry) 205 | s.Require().NoError(s.cache.GetDirEntries(inode, &readEntries)) 206 | s.Equal(expEntries, readEntries) 207 | } 208 | -------------------------------------------------------------------------------- /internal/cache/sqlite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "database/sql" 19 | "fmt" 20 | "math/rand" 21 | "time" 22 | 23 | "github.com/golang/glog" 24 | "github.com/mattn/go-sqlite3" 25 | ) 26 | 27 | const ( 28 | // Baseline delay to back off on contention. 29 | baseDelay = 100 * time.Millisecond 30 | 31 | // Maximum increment in the delay each time an operation is retried. 32 | backoffMaxDelay = 500 * time.Millisecond 33 | ) 34 | 35 | // sqliteDB extends an SQLite connection to wrap all SQL methods with versions they retry 36 | // automatically when there is contention. 37 | type sqliteDB struct { 38 | *sql.DB 39 | } 40 | 41 | // initDatabase sets up the database using the provided schema DDL. The schema is provided as a 42 | // list of DDL statements. 43 | func initDatabase(db *sql.DB, schemaDDL []string) error { 44 | for _, statement := range schemaDDL { 45 | if _, err := db.Exec(statement); err != nil { 46 | return err 47 | } 48 | } 49 | return nil 50 | } 51 | 52 | // newSqliteDb instantiates a new SQLite database. If the database given in path does not exist, it 53 | // is created. No matter what, the list of statements given in schemaDDL is executed so they should 54 | // be idempotent. It is OK to include PRAGMAs in the schemaDDL because they will be run each time 55 | // the database is open. 56 | func newSqliteDb(path string, schemaDDL []string) (*sqliteDB, error) { 57 | db, err := sql.Open("sqlite3", path) 58 | if err != nil { 59 | return nil, err 60 | } 61 | if err = db.Ping(); err != nil { 62 | return nil, err 63 | } 64 | 65 | if err = initDatabase(db, schemaDDL); err != nil { 66 | db.Close() 67 | return nil, err 68 | } 69 | 70 | return &sqliteDB{db}, nil 71 | } 72 | 73 | // waitOrErr sleeps for a bit if the given error indicates contention in the database, or else 74 | // returns the real error. The id parameter is used for logging purposes and should identify the 75 | // key of the database that was affected by the contention. The delayAccumulator parameter is 76 | // incremented by a random amount if the call decides to sleep; subsequent calls to waitOrErr should 77 | // use the same delayAccumulator parameter. 78 | func waitOrErr(id string, err error, delayAccumulator *time.Duration) error { 79 | switch typedErr := err.(type) { 80 | case sqlite3.Error: 81 | switch typedErr.Code { 82 | case sqlite3.ErrBusy: 83 | increment := rand.Int63n(int64(backoffMaxDelay)) 84 | *delayAccumulator = *delayAccumulator + time.Duration(increment) 85 | 86 | glog.ErrorDepth(1, fmt.Sprintf("retrying in %s for %s: %s\n", delayAccumulator.String(), id, err)) 87 | time.Sleep(*delayAccumulator) 88 | return nil 89 | } 90 | return err 91 | default: 92 | return err 93 | } 94 | } 95 | 96 | // RetriableStmt extends an sql.Stmt with functions to retry queries. 97 | type RetriableStmt struct { 98 | *sql.Stmt 99 | } 100 | 101 | // RetriablePrepare prepares a new statement that supports automatic retry of queries during 102 | // contention. 103 | func (db *sqliteDB) RetriablePrepare(query string) (*RetriableStmt, error) { 104 | stmt, err := db.Prepare(query) 105 | if err != nil { 106 | return nil, err 107 | } 108 | return &RetriableStmt{stmt}, nil 109 | } 110 | 111 | // RetryQuery wraps sql.DB.Query to automatically retry on database contention. 112 | func (stmt *RetriableStmt) RetryQuery(id string, args ...interface{}) ( 113 | *sql.Rows, error) { 114 | delay := baseDelay 115 | retry: 116 | rows, err := stmt.Query(args...) 117 | if err != nil { 118 | err = waitOrErr(id, err, &delay) 119 | if err != nil { 120 | return nil, err 121 | } 122 | goto retry 123 | } 124 | return rows, nil 125 | } 126 | 127 | // RetriableRow wraps an sql.Row to be used as a helper for RetryQueryRow. 128 | type RetriableRow struct { 129 | id string 130 | row *sql.Row 131 | } 132 | 133 | // Scan wraps sql.DB.Row.Scan to automatically retry on database contention. 134 | func (row *RetriableRow) Scan(dest ...interface{}) error { 135 | delay := baseDelay 136 | retry: 137 | err := row.row.Scan(dest...) 138 | if err != nil { 139 | err = waitOrErr(row.id, err, &delay) 140 | if err != nil { 141 | return err 142 | } 143 | goto retry 144 | } 145 | return nil 146 | } 147 | 148 | // RetryQueryRow wraps sql.DB.QueryRow to automatically retry on database contention. 149 | func (stmt *RetriableStmt) RetryQueryRow(id string, args ...interface{}) *RetriableRow { 150 | return &RetriableRow{ 151 | id: id, 152 | row: stmt.QueryRow(args...), 153 | } 154 | } 155 | 156 | // RetryExec wraps sql.DB.Exec to automatically retry on database contention. 157 | func (stmt *RetriableStmt) RetryExec(id string, args ...interface{}) ( 158 | sql.Result, error) { 159 | delay := baseDelay 160 | retry: 161 | result, err := stmt.Exec(args...) 162 | if err != nil { 163 | err = waitOrErr(id, err, &delay) 164 | if err != nil { 165 | return nil, err 166 | } 167 | goto retry 168 | } 169 | return result, nil 170 | } 171 | -------------------------------------------------------------------------------- /internal/cache/sqlite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "path/filepath" 19 | "testing" 20 | 21 | "github.com/stretchr/testify/suite" 22 | 23 | "github.com/jmmv/sourcachefs/internal/test" 24 | ) 25 | 26 | // newTestSqliteDb creates or opens an SQLite database and fails the test if 27 | // the connection cannot be established. 28 | func newTestSqliteDb(s *suite.Suite, path string, schemaDDL []string) *sqliteDB { 29 | db, err := newSqliteDb(path, schemaDDL) 30 | s.NoError(err) 31 | return db 32 | } 33 | 34 | type NewSqliteDbSuite struct { 35 | test.SuiteWithTempDir 36 | } 37 | 38 | func TestNewSqliteDbSuite(t *testing.T) { 39 | suite.Run(t, new(NewSqliteDbSuite)) 40 | } 41 | 42 | func (s *NewSqliteDbSuite) TestSchemaDDLIsExecuted() { 43 | schemaDDL := []string{ 44 | "CREATE TABLE first (a STRING)", 45 | "INSERT INTO first VALUES ('foo')", 46 | } 47 | db := newTestSqliteDb(&s.Suite, ":memory:", schemaDDL) 48 | defer db.Close() 49 | 50 | var value string 51 | s.NoError(db.QueryRow("SELECT a FROM first").Scan(&value)) 52 | s.Equal("foo", value) 53 | } 54 | 55 | func (s *NewSqliteDbSuite) TestSchemaDDLIsExecutedOnEachOpen() { 56 | dbPath := filepath.Join(s.TempDir(), "test.db") 57 | 58 | schemaDDL := []string{ 59 | "CREATE TABLE IF NOT EXISTS first (a STRING)", 60 | "INSERT INTO first VALUES ('foo')", 61 | } 62 | db := newTestSqliteDb(&s.Suite, dbPath, schemaDDL) 63 | db.Close() 64 | db = newTestSqliteDb(&s.Suite, dbPath, schemaDDL) 65 | defer db.Close() 66 | 67 | var count int 68 | query := "SELECT COUNT(a) FROM first" 69 | s.NoError(db.QueryRow(query).Scan(&count)) 70 | s.Equal(2, count) 71 | } 72 | 73 | type RetriablePrepareSuite struct { 74 | test.SuiteWithTempDir 75 | } 76 | 77 | func TestRetriablePrepareSuite(t *testing.T) { 78 | suite.Run(t, new(RetriablePrepareSuite)) 79 | } 80 | 81 | func (s *RetriablePrepareSuite) TestRetryExec() { 82 | // We cannot use an in-memory database in this test: SQLite3 doesn't 83 | // seem to like concurrent accesses to such a database even when the 84 | // database is opened in "full mutex" mode. 85 | dbPath := filepath.Join(s.TempDir(), "test.db") 86 | 87 | schemaDDL := []string{ 88 | "CREATE TABLE foo (a INTEGER)", 89 | "INSERT INTO foo VALUES (500)", 90 | } 91 | db := newTestSqliteDb(&s.Suite, dbPath, schemaDDL) 92 | defer db.Close() 93 | 94 | stmt, err := db.RetriablePrepare("UPDATE foo SET a = a + 1") 95 | s.NoError(err) 96 | defer stmt.Close() 97 | 98 | tries := 100 99 | done := make(chan bool, tries) 100 | for i := 0; i < tries; i++ { 101 | go func() { 102 | _, err := stmt.RetryExec("test") 103 | s.NoError(err) 104 | done <- true 105 | }() 106 | } 107 | for i := 0; i < tries; i++ { 108 | <-done 109 | } 110 | 111 | var value int 112 | s.NoError(db.QueryRow("SELECT a FROM foo").Scan(&value)) 113 | s.Equal(500+tries, value) 114 | } 115 | -------------------------------------------------------------------------------- /internal/cache/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "os" 19 | "path/filepath" 20 | "time" 21 | ) 22 | 23 | // OneDirEntry represents a single directory entry. 24 | type OneDirEntry struct { 25 | Valid bool 26 | Inode uint64 27 | ModeType os.FileMode 28 | } 29 | 30 | // CachedRow contains all the data for an inode as known by the cache. 31 | type CachedRow struct { 32 | Inode uint64 33 | TypeName string 34 | FileInfo *os.FileInfo 35 | DidReadDir bool 36 | DirEntries map[string]OneDirEntry 37 | ContentHash *Key 38 | Target *string 39 | } 40 | 41 | // cachedFileInfo exposes an all in-memory os.FileInfo implementation with 42 | // data retrieved from the database. 43 | type cachedFileInfo struct { 44 | basename string 45 | size int64 46 | mode os.FileMode 47 | mtime time.Time 48 | } 49 | 50 | var _ os.FileInfo = (*cachedFileInfo)(nil) 51 | 52 | // newFileInfo instantiates a new os.FileInfo with explicit values for each 53 | // of the publicly available fields. 54 | func newFileInfo(path string, size int64, mode os.FileMode, mtime time.Time) os.FileInfo { 55 | return &cachedFileInfo{ 56 | basename: filepath.Base(path), 57 | size: size, 58 | mode: mode, 59 | mtime: mtime, 60 | } 61 | } 62 | 63 | func (fi *cachedFileInfo) Name() string { 64 | return fi.basename 65 | } 66 | 67 | func (fi *cachedFileInfo) Size() int64 { 68 | return fi.size 69 | } 70 | 71 | func (fi *cachedFileInfo) Mode() os.FileMode { 72 | return fi.mode 73 | } 74 | 75 | func (fi *cachedFileInfo) ModTime() time.Time { 76 | return fi.mtime 77 | } 78 | 79 | func (fi *cachedFileInfo) IsDir() bool { 80 | return fi.mode.IsDir() 81 | } 82 | 83 | func (fi *cachedFileInfo) Sys() interface{} { 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /internal/cache/types_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "os" 19 | "testing" 20 | "time" 21 | 22 | "github.com/stretchr/testify/suite" 23 | ) 24 | 25 | type CachedFileInfoSuite struct { 26 | suite.Suite 27 | } 28 | 29 | func TestCachedFileInfo(t *testing.T) { 30 | suite.Run(t, new(CachedFileInfoSuite)) 31 | } 32 | 33 | func (s *CachedFileInfoSuite) TestFile() { 34 | mtime := time.Date(2017, 6, 25, 17, 26, 30, 1234, time.Local) 35 | fi := newFileInfo("/some/long/name", 12345, 0644, mtime) 36 | s.Equal("name", fi.Name()) 37 | s.Equal(int64(12345), fi.Size()) 38 | s.Equal(os.FileMode(0644), fi.Mode()) 39 | s.Equal(mtime, fi.ModTime()) 40 | s.False(fi.IsDir()) 41 | s.Nil(fi.Sys()) 42 | } 43 | 44 | func (s *CachedFileInfoSuite) TestDir() { 45 | mtime := time.Date(2017, 6, 25, 17, 27, 45, 0, time.Local) 46 | fi := newFileInfo("../this/is/a/dir", 0, 0755|os.ModeDir, mtime) 47 | s.Equal("dir", fi.Name()) 48 | s.Equal(int64(0), fi.Size()) 49 | s.Equal(0755|os.ModeDir, fi.Mode()) 50 | s.Equal(mtime, fi.ModTime()) 51 | s.True(fi.IsDir()) 52 | s.Nil(fi.Sys()) 53 | } 54 | -------------------------------------------------------------------------------- /internal/fs/dir.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package fs 16 | 17 | import ( 18 | "context" 19 | "os" 20 | "path/filepath" 21 | "sync" 22 | 23 | "bazil.org/fuse" 24 | "bazil.org/fuse/fs" 25 | "github.com/golang/glog" 26 | 27 | "github.com/jmmv/sourcachefs/internal/cache" 28 | "github.com/jmmv/sourcachefs/internal/stats" 29 | ) 30 | 31 | type lazyDirData struct { 32 | fileInfo *os.FileInfo 33 | didReadDir bool 34 | dirEntries map[string]cache.OneDirEntry 35 | rows map[string]*cache.CachedRow 36 | } 37 | 38 | // Dir implements both Node and Handle for the root directory. 39 | type Dir struct { 40 | globals *globalState 41 | path string 42 | belowPath string 43 | lock sync.Mutex 44 | inode uint64 45 | direct bool 46 | data *lazyDirData 47 | } 48 | 49 | var _ fs.Node = (*Dir)(nil) 50 | 51 | // Attr queries the properties of the directory. 52 | func (dir *Dir) Attr(ctx context.Context, a *fuse.Attr) error { 53 | dir.lock.Lock() 54 | defer dir.lock.Unlock() 55 | 56 | return commonAttr(dir.globals, dir.inode, dir.direct, dir.belowPath, &dir.data.fileInfo, a) 57 | } 58 | 59 | var _ fs.NodeStringLookuper = (*Dir)(nil) 60 | 61 | // Lookup searches for a file entry. 62 | func (dir *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) { 63 | dir.lock.Lock() 64 | defer dir.lock.Unlock() 65 | 66 | path := filepath.Join(dir.path, name) 67 | 68 | if !dir.direct { // Fast path in case we already know about the entry. 69 | cachedRow, ok := dir.data.rows[name] 70 | if ok { 71 | return newNodeForEntry(dir.globals, dir.path, dir.belowPath, cachedRow, name) 72 | } 73 | } 74 | 75 | cachedRow, err := dir.globals.cache.Metadata.GetFullEntry(path) 76 | if err != nil { 77 | glog.Errorf("%s: %s", dir.path, err) 78 | return nil, err 79 | } 80 | cached := cachedRow != nil 81 | 82 | if dir.direct { 83 | dir.globals.stats.AccountDirect(stats.LookupFuseOp) 84 | 85 | belowPath := filepath.Join(dir.belowPath, name) 86 | fileInfo, err := dir.globals.syscalls.Lstat(stats.RemoteDomain, belowPath) 87 | switch { 88 | case err == nil: 89 | err = dir.globals.cache.Metadata.PutNewOrUpdateWithType(path, fileInfo.Mode()&os.ModeType) 90 | if err != nil { 91 | glog.Errorf("%s: %s", path, err) 92 | return nil, err 93 | } 94 | 95 | case err != nil && cached: 96 | // Reuse cached row, queried later. 97 | 98 | case err != nil && !cached && os.IsNotExist(err): 99 | err := dir.globals.cache.Metadata.PutNewOrUpdateNoEntry(path) 100 | if err != nil { 101 | glog.Errorf("%s: %s", path, err) 102 | return nil, err 103 | } 104 | return nil, fuse.ENOENT 105 | 106 | case err != nil && !cached && !os.IsNotExist(err): 107 | glog.Errorf("%s: %s", path, err) 108 | return nil, err 109 | } 110 | 111 | cachedRow, err = dir.globals.cache.Metadata.GetFullEntry(path) 112 | if err != nil { 113 | glog.Errorf("%s: %s", dir.path, err) 114 | return nil, err 115 | } 116 | if cachedRow == nil { 117 | glog.Errorf("couldn't reload entry after put for %s", dir.path) 118 | } 119 | } else if !cached { 120 | dir.globals.stats.AccountCacheMiss(stats.LookupFuseOp) 121 | belowPath := filepath.Join(dir.belowPath, name) 122 | 123 | fileInfo, err := dir.globals.syscalls.Lstat(stats.RemoteDomain, belowPath) 124 | switch { 125 | case err != nil && !os.IsNotExist(err): 126 | glog.Errorf("%s: %s", dir.path, err) 127 | return nil, err 128 | case err != nil: 129 | _, err := dir.globals.cache.Metadata.PutNewNoEntry(path) 130 | if err != nil { 131 | glog.Errorf("%s: %s", path, err) 132 | return nil, err 133 | } 134 | return nil, fuse.ENOENT 135 | default: 136 | // Fallthrough. 137 | } 138 | 139 | _, err = dir.globals.cache.Metadata.PutNewWithType(path, fileInfo.Mode()&os.ModeType) 140 | if err != nil { 141 | glog.Errorf("%s: %s", path, err) 142 | return nil, err 143 | } 144 | 145 | cachedRow, err = dir.globals.cache.Metadata.GetFullEntry(path) 146 | if err != nil { 147 | glog.Errorf("%s: %s", dir.path, err) 148 | return nil, err 149 | } 150 | if cachedRow == nil { 151 | glog.Errorf("couldn't reload entry after put for %s", dir.path) 152 | } 153 | } else { 154 | dir.globals.stats.AccountCacheHit(stats.LookupFuseOp) 155 | } 156 | dir.data.rows[name] = cachedRow 157 | 158 | return newNodeForEntry(dir.globals, dir.path, dir.belowPath, cachedRow, name) 159 | } 160 | 161 | var _ fs.HandleReadDirAller = (*Dir)(nil) 162 | 163 | // ReadDirAll returns all entries in a directory. 164 | // TODO(jmmv): Decouple handle implementation from the Dir node. 165 | func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { 166 | dir.lock.Lock() 167 | defer dir.lock.Unlock() 168 | 169 | cached := dir.data.dirEntries != nil 170 | 171 | if dir.direct { 172 | dir.globals.stats.AccountDirect(stats.ReaddirFuseOp) 173 | 174 | f, err := dir.globals.syscalls.Open(stats.RemoteDomain, dir.belowPath) 175 | switch { 176 | case err == nil: 177 | defer f.Close() 178 | 179 | belowEntries, err := f.Readdir(0) 180 | if err != nil { 181 | glog.Errorf("%s: %s", dir.path, err) 182 | return nil, err 183 | } 184 | 185 | err = dir.globals.cache.Metadata.PutDirEntries(dir.inode, dir.path, belowEntries) 186 | if err != nil { 187 | glog.Errorf("%s: %s", dir.path, err) 188 | return nil, err 189 | } 190 | 191 | case err != nil && cached: 192 | // Reuse dir.data.dirEntries 193 | 194 | case err != nil && !cached: 195 | glog.Errorf("%s: %s", dir.path, err) 196 | return nil, err 197 | } 198 | 199 | // TODO: Suboptimal probably, but we need the inode numbers. Maybe 200 | // GetDirEntries should return a set of cache rows instead, but then 201 | // we need to merge those with any partially-known results created by 202 | // lookup. 203 | entries := make(map[string]cache.OneDirEntry) 204 | err = dir.globals.cache.Metadata.GetDirEntries(dir.inode, &entries) 205 | if err != nil { 206 | glog.Errorf("%s: %s", dir.path, err) 207 | return nil, err 208 | } 209 | dir.data.dirEntries = entries 210 | } else if !cached { 211 | dir.globals.stats.AccountCacheMiss(stats.ReaddirFuseOp) 212 | 213 | f, err := dir.globals.syscalls.Open(stats.RemoteDomain, dir.belowPath) 214 | if err != nil { 215 | glog.Errorf("%s: %s", dir.path, err) 216 | return nil, err 217 | } 218 | defer f.Close() 219 | 220 | belowEntries, err := f.Readdir(0) 221 | if err != nil { 222 | glog.Errorf("%s: %s", dir.path, err) 223 | return nil, err 224 | } 225 | 226 | err = dir.globals.cache.Metadata.PutDirEntries(dir.inode, dir.path, belowEntries) 227 | if err != nil { 228 | glog.Errorf("%s: %s", dir.path, err) 229 | return nil, err 230 | } 231 | 232 | // TODO: Suboptimal probably, but we need the inode numbers. Maybe 233 | // GetDirEntries should return a set of cache rows instead, but then 234 | // we need to merge those with any partially-known results created by 235 | // lookup. 236 | entries := make(map[string]cache.OneDirEntry) 237 | err = dir.globals.cache.Metadata.GetDirEntries(dir.inode, &entries) 238 | if err != nil { 239 | glog.Errorf("%s: %s", dir.path, err) 240 | return nil, err 241 | } 242 | dir.data.dirEntries = entries 243 | } else { 244 | dir.globals.stats.AccountCacheHit(stats.ReaddirFuseOp) 245 | } 246 | 247 | dirents := make([]fuse.Dirent, 0, len(dir.data.dirEntries)) 248 | for basename, dirEntry := range dir.data.dirEntries { 249 | if !dirEntry.Valid { 250 | continue 251 | } 252 | 253 | var de fuse.Dirent 254 | de.Inode = dirEntry.Inode 255 | de.Name = basename 256 | switch dirEntry.ModeType & os.ModeType { 257 | case 0: 258 | de.Type = fuse.DT_File 259 | case os.ModeDir: 260 | de.Type = fuse.DT_Dir 261 | case os.ModeSymlink: 262 | de.Type = fuse.DT_Link 263 | default: 264 | panic("unknown type") 265 | } 266 | dirents = append(dirents, de) 267 | } 268 | return dirents, nil 269 | } 270 | -------------------------------------------------------------------------------- /internal/fs/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package fs 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "os" 21 | "sync" 22 | 23 | "bazil.org/fuse" 24 | "bazil.org/fuse/fs" 25 | "github.com/golang/glog" 26 | 27 | "github.com/jmmv/sourcachefs/internal/cache" 28 | "github.com/jmmv/sourcachefs/internal/real" 29 | "github.com/jmmv/sourcachefs/internal/stats" 30 | ) 31 | 32 | type openFile struct { 33 | globals *globalState 34 | path string 35 | reader real.FileReader 36 | } 37 | 38 | var _ fs.Handle = (*openFile)(nil) 39 | 40 | var _ fs.HandleReader = (*openFile)(nil) 41 | 42 | func (file *openFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { 43 | file.globals.stats.AccountCacheHit(stats.ReadFuseOp) 44 | resp.Data = resp.Data[:req.Size] 45 | n, err := file.reader.ReadAt(resp.Data, req.Offset) 46 | if err == io.EOF { 47 | err = nil 48 | } 49 | if err != nil { 50 | glog.Errorf("read error: %s", err) 51 | } 52 | resp.Data = resp.Data[:n] 53 | return err 54 | } 55 | 56 | var _ fs.HandleReleaser = (*openFile)(nil) 57 | 58 | func (file *openFile) Release(ctx context.Context, req *fuse.ReleaseRequest) error { 59 | if file.reader == nil { 60 | glog.Warningf("double-close of already-closed file handle") 61 | return nil 62 | } 63 | err := file.reader.Close() 64 | if err != nil { 65 | glog.Errorf("close error: %s", err) 66 | return err 67 | } 68 | file.reader = nil 69 | return nil 70 | } 71 | 72 | type lazyFileData struct { 73 | fileInfo *os.FileInfo 74 | contentHash *cache.Key 75 | } 76 | 77 | // File implements both Node and Handle for the hello file. 78 | type File struct { 79 | globals *globalState 80 | path string 81 | belowPath string 82 | lock sync.Mutex 83 | inode uint64 84 | direct bool 85 | data *lazyFileData 86 | } 87 | 88 | var _ fs.Node = (*File)(nil) 89 | 90 | // Attr queries the file properties of the file. 91 | func (file *File) Attr(ctx context.Context, a *fuse.Attr) error { 92 | file.lock.Lock() 93 | defer file.lock.Unlock() 94 | 95 | return commonAttr(file.globals, file.inode, file.direct, file.belowPath, &file.data.fileInfo, a) 96 | } 97 | 98 | var _ fs.NodeOpener = (*File)(nil) 99 | 100 | // Open obtains a file handle for the open request if the file exists. 101 | func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { 102 | file.lock.Lock() 103 | defer file.lock.Unlock() 104 | 105 | cached := file.data.contentHash != nil 106 | 107 | if file.direct { 108 | file.globals.stats.AccountDirect(stats.OpenFuseOp) 109 | 110 | glog.Infof("fetching remote file %s", file.belowPath) 111 | key, err := file.globals.cache.Content.PutContents(file.belowPath) 112 | switch { 113 | case err == nil: 114 | err = file.globals.cache.Metadata.PutContentHash(file.inode, key) 115 | if err != nil { 116 | glog.Errorf("%s: %s", file.path, err) 117 | return nil, err 118 | } 119 | 120 | file.data.contentHash = &key 121 | 122 | case err != nil && cached: 123 | // Reuse file.data.contentHash 124 | 125 | case err != nil && !cached: 126 | glog.Errorf("%s: %s", file.path, err) 127 | return nil, err 128 | } 129 | } else if !cached { 130 | file.globals.stats.AccountCacheMiss(stats.OpenFuseOp) 131 | 132 | glog.Infof("fetching remote file %s", file.belowPath) 133 | key, err := file.globals.cache.Content.PutContents(file.belowPath) 134 | if err != nil { 135 | glog.Errorf("%s: %s", file.path, err) 136 | return nil, err 137 | } 138 | 139 | err = file.globals.cache.Metadata.PutContentHash(file.inode, key) 140 | if err != nil { 141 | glog.Errorf("%s: %s", file.path, err) 142 | return nil, err 143 | } 144 | 145 | file.data.contentHash = &key 146 | } else { 147 | file.globals.stats.AccountCacheHit(stats.OpenFuseOp) 148 | } 149 | 150 | path, err := file.globals.cache.Content.GetFileForContents(cache.Key(*file.data.contentHash)) 151 | if err != nil { 152 | glog.Errorf("%s: %s", file.path, err) 153 | return nil, err 154 | } 155 | 156 | input, err := file.globals.syscalls.Open(stats.LocalDomain, path) 157 | if err != nil { 158 | glog.Errorf("%s: %s", file.path, err) 159 | return nil, err 160 | } 161 | return &openFile{ 162 | globals: file.globals, 163 | path: file.path, 164 | reader: input, 165 | }, nil 166 | } 167 | -------------------------------------------------------------------------------- /internal/fs/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package fs 16 | 17 | import ( 18 | "os" 19 | "os/signal" 20 | "path/filepath" 21 | "regexp" 22 | "syscall" 23 | "time" 24 | 25 | "bazil.org/fuse" 26 | "bazil.org/fuse/fs" 27 | "github.com/golang/glog" 28 | 29 | "github.com/jmmv/sourcachefs/internal/cache" 30 | "github.com/jmmv/sourcachefs/internal/real" 31 | "github.com/jmmv/sourcachefs/internal/stats" 32 | ) 33 | 34 | type globalState struct { 35 | syscalls real.Syscalls 36 | cache *cache.Cache 37 | stats stats.Updater 38 | cachedPathRegex *regexp.Regexp 39 | } 40 | 41 | func newNodeForEntry(globals *globalState, parentPath string, parentBelowPath string, row *cache.CachedRow, name string) (fs.Node, error) { 42 | path := filepath.Join(parentPath, name) 43 | belowPath := filepath.Join(parentBelowPath, name) 44 | 45 | direct := !globals.cachedPathRegex.MatchString(path) 46 | 47 | switch row.TypeName { 48 | case "directory": 49 | return &Dir{ 50 | globals: globals, 51 | path: path, 52 | belowPath: belowPath, 53 | inode: row.Inode, 54 | direct: direct, 55 | data: &lazyDirData{ 56 | fileInfo: row.FileInfo, 57 | dirEntries: row.DirEntries, 58 | rows: make(map[string]*cache.CachedRow), 59 | }, 60 | }, nil 61 | case "file": 62 | return &File{ 63 | globals: globals, 64 | path: path, 65 | belowPath: belowPath, 66 | inode: row.Inode, 67 | direct: direct, 68 | data: &lazyFileData{ 69 | fileInfo: row.FileInfo, 70 | contentHash: row.ContentHash, 71 | }, 72 | }, nil 73 | case "noentry": 74 | return nil, fuse.ENOENT 75 | case "symlink": 76 | return &Symlink{ 77 | globals: globals, 78 | path: path, 79 | belowPath: belowPath, 80 | inode: row.Inode, 81 | direct: direct, 82 | data: &lazySymlinkData{ 83 | fileInfo: row.FileInfo, 84 | target: row.Target, 85 | }, 86 | }, nil 87 | default: 88 | glog.Fatalf("unknown entry type") 89 | return nil, nil // golang/go #10037 90 | } 91 | } 92 | 93 | func newNodeForOneDirEntry(globals *globalState, parentPath string, parentBelowPath string, dirEntry *cache.OneDirEntry, name string) (fs.Node, error) { 94 | path := filepath.Join(parentPath, name) 95 | belowPath := filepath.Join(parentBelowPath, name) 96 | 97 | direct := !globals.cachedPathRegex.MatchString(path) 98 | 99 | switch dirEntry.ModeType { 100 | case os.ModeDir: 101 | return &Dir{ 102 | globals: globals, 103 | path: path, 104 | belowPath: belowPath, 105 | inode: dirEntry.Inode, 106 | direct: direct, 107 | data: &lazyDirData{ 108 | fileInfo: nil, 109 | dirEntries: nil, 110 | rows: make(map[string]*cache.CachedRow), 111 | }, 112 | }, nil 113 | case 0: 114 | return &File{ 115 | globals: globals, 116 | path: path, 117 | belowPath: belowPath, 118 | inode: dirEntry.Inode, 119 | direct: direct, 120 | data: &lazyFileData{ 121 | fileInfo: nil, 122 | contentHash: nil, 123 | }, 124 | }, nil 125 | case os.ModeSymlink: 126 | return &Symlink{ 127 | globals: globals, 128 | path: path, 129 | belowPath: belowPath, 130 | inode: dirEntry.Inode, 131 | direct: direct, 132 | data: &lazySymlinkData{ 133 | fileInfo: nil, 134 | target: nil, 135 | }, 136 | }, nil 137 | default: 138 | glog.Fatalf("unknown entry type") 139 | return nil, nil // golang/go #10037 140 | } 141 | } 142 | 143 | func commonAttr(globals *globalState, inode uint64, direct bool, belowPath string, fileInfo **os.FileInfo, a *fuse.Attr) error { 144 | cached := *fileInfo != nil 145 | 146 | if direct { 147 | globals.stats.AccountDirect(stats.AttrFuseOp) 148 | 149 | rawFileInfo, err := globals.syscalls.Lstat(stats.RemoteDomain, belowPath) 150 | switch { 151 | case err != nil && cached: 152 | // Reuse cached row. 153 | 154 | case err != nil && !cached: 155 | glog.Errorf("%s: %s", belowPath, err) 156 | return err 157 | 158 | case err == nil: 159 | err = globals.cache.Metadata.PutFileInfo(inode, rawFileInfo) 160 | if err != nil { 161 | glog.Errorf("%s: %s", belowPath, err) 162 | return err 163 | } 164 | 165 | *fileInfo = &rawFileInfo 166 | } 167 | } else if !cached { 168 | globals.stats.AccountCacheMiss(stats.AttrFuseOp) 169 | 170 | rawFileInfo, err := globals.syscalls.Lstat(stats.RemoteDomain, belowPath) 171 | if err != nil { 172 | glog.Errorf("%s: %s", belowPath, err) 173 | return err 174 | } 175 | 176 | err = globals.cache.Metadata.PutFileInfo(inode, rawFileInfo) 177 | if err != nil { 178 | glog.Errorf("%s: %s", belowPath, err) 179 | return err 180 | } 181 | 182 | *fileInfo = &rawFileInfo 183 | } else { 184 | globals.stats.AccountCacheHit(stats.AttrFuseOp) 185 | } 186 | 187 | a.Inode = inode 188 | a.Mode = (**fileInfo).Mode() 189 | a.Size = uint64((**fileInfo).Size()) 190 | a.Mtime = (**fileInfo).ModTime() 191 | return nil 192 | } 193 | 194 | func putAndGetRow(globals *globalState, path string, modeType os.FileMode) ( 195 | *cache.CachedRow, error) { 196 | direct := !globals.cachedPathRegex.MatchString(path) 197 | 198 | cacheRow, err := globals.cache.Metadata.GetFullEntry(path) 199 | if err != nil { 200 | glog.Errorf("%s: %s", path, err) 201 | return nil, err 202 | } 203 | cached := cacheRow != nil 204 | 205 | if direct { 206 | err := globals.cache.Metadata.PutNewOrUpdateWithType(path, modeType) 207 | if err != nil { 208 | glog.Errorf("%s: %s", path, err) 209 | return nil, err 210 | } 211 | } else if !cached { 212 | _, err := globals.cache.Metadata.PutNewWithType(path, modeType) 213 | if err != nil { 214 | glog.Errorf("%s: %s", path, err) 215 | return nil, err 216 | } 217 | } 218 | 219 | cacheRow, err = globals.cache.Metadata.GetFullEntry(path) 220 | if err != nil { 221 | glog.Errorf("%s: %s", path, err) 222 | return nil, err 223 | } 224 | if cacheRow == nil { 225 | glog.Errorf("couldn't reload entry after put for %s", path) 226 | return nil, err 227 | } 228 | 229 | return cacheRow, nil 230 | } 231 | 232 | // unmountOnSignal captures termination signals and unmounts the filesystem. 233 | // 234 | // This allows the main program to exit the FUSE serving loop cleanly and avoids 235 | // leaking a mount point without the backing FUSE server. 236 | func unmountOnSignal(mountPoint string, caught chan<- os.Signal) { 237 | wait := make(chan os.Signal, 1) 238 | signal.Notify(wait, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) 239 | caught <- <-wait 240 | signal.Reset() 241 | err := fuse.Unmount(mountPoint) 242 | if err != nil { 243 | glog.Warningf("unmounting filesystem failed with error: %v", err) 244 | } 245 | } 246 | 247 | // Loop mounts the file system and starts serving. 248 | func Loop(remotedir string, sc real.Syscalls, c *cache.Cache, s stats.Updater, mountpoint string, cachedPathRegex *regexp.Regexp) error { 249 | connection, err := fuse.Mount( 250 | mountpoint, 251 | fuse.FSName("sourcachefs"), 252 | fuse.Subtype("sourcachefs"), 253 | //fuse.LocalVolume(), 254 | fuse.VolumeName("sourcachefs"), 255 | fuse.MaxReadahead(8*1024*1024), 256 | //fuse.Async(), 257 | fuse.AsyncRead(), 258 | fuse.ReadOnly(), 259 | // Avoid Access() calls. Replace with an Access handler that returns ENOSYS. 260 | fuse.DefaultPermissions(), 261 | // Because we prefetch files on Open, such operations can be very slow. Bump the timeout, 262 | // though maybe this is too much and we should be doing something different instead of 263 | // prefetching. 264 | fuse.DaemonTimeout("600"), 265 | ) 266 | if err != nil { 267 | return err 268 | } 269 | defer connection.Close() 270 | 271 | caughtSignal := make(chan os.Signal, 1) 272 | go unmountOnSignal(mountpoint, caughtSignal) 273 | 274 | globals := &globalState{ 275 | syscalls: sc, 276 | cache: c, 277 | stats: s, 278 | cachedPathRegex: cachedPathRegex, 279 | } 280 | 281 | cacheRow, err := putAndGetRow(globals, "/", os.ModeDir) 282 | if err != nil { 283 | return err 284 | } 285 | 286 | rootDir, err := newNodeForEntry(globals, "/", remotedir, cacheRow, "/") 287 | if err != nil { 288 | return err 289 | } 290 | root := FS{ 291 | rootNode: rootDir, 292 | } 293 | 294 | err = fs.Serve(connection, root) 295 | if err != nil { 296 | return err 297 | } 298 | 299 | <-connection.Ready 300 | if connection.MountError != nil { 301 | return connection.MountError 302 | } 303 | 304 | // If we reach this point, the FUSE serve loop has terminated either because the user unmounted 305 | // the file system or we have received a signal. The signal handler also unmounted the file 306 | // system, so either way the file system is unmounted at this point. 307 | select { 308 | case signal := <-caughtSignal: 309 | // Redeliver the signal to ourselves (the handler was reset to the default) to exit with 310 | // the correct exit code. 311 | proc, err := os.FindProcess(os.Getpid()) 312 | if err != nil { 313 | return err 314 | } 315 | proc.Signal(signal) 316 | 317 | // Block indefinitely until the signal is delivered. 318 | for { 319 | time.Sleep(1 * time.Second) 320 | } 321 | default: 322 | return nil 323 | } 324 | } 325 | 326 | // FS represents the file system. 327 | type FS struct { 328 | rootNode fs.Node 329 | } 330 | 331 | // Root returns the root node for the file system. 332 | func (fs FS) Root() (fs.Node, error) { 333 | return fs.rootNode, nil 334 | } 335 | -------------------------------------------------------------------------------- /internal/fs/symlink.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package fs 16 | 17 | import ( 18 | "context" 19 | "os" 20 | "sync" 21 | 22 | "bazil.org/fuse" 23 | "bazil.org/fuse/fs" 24 | "github.com/golang/glog" 25 | 26 | "github.com/jmmv/sourcachefs/internal/stats" 27 | ) 28 | 29 | type lazySymlinkData struct { 30 | fileInfo *os.FileInfo 31 | target *string 32 | } 33 | 34 | // Symlink implements both Node and Handle for the hello file. 35 | type Symlink struct { 36 | globals *globalState 37 | path string 38 | belowPath string 39 | lock sync.Mutex 40 | inode uint64 41 | direct bool 42 | data *lazySymlinkData 43 | } 44 | 45 | var _ fs.Node = (*Symlink)(nil) 46 | 47 | // Attr queries the file properties of the file. 48 | func (symlink *Symlink) Attr(ctx context.Context, a *fuse.Attr) error { 49 | symlink.lock.Lock() 50 | defer symlink.lock.Unlock() 51 | 52 | return commonAttr(symlink.globals, symlink.inode, symlink.direct, symlink.belowPath, &symlink.data.fileInfo, a) 53 | } 54 | 55 | var _ fs.NodeReadlinker = (*Symlink)(nil) 56 | 57 | // Readlink retrieves the target of the symlink. 58 | func (symlink *Symlink) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { 59 | symlink.lock.Lock() 60 | defer symlink.lock.Unlock() 61 | 62 | cached := symlink.data.target != nil 63 | 64 | if symlink.direct { 65 | symlink.globals.stats.AccountDirect(stats.ReadlinkFuseOp) 66 | 67 | target, err := symlink.globals.syscalls.Readlink(stats.RemoteDomain, symlink.belowPath) 68 | switch { 69 | case err == nil: 70 | err = symlink.globals.cache.Metadata.PutSymlinkTarget(symlink.inode, target) 71 | if err != nil { 72 | glog.Errorf("%s: %s", symlink.path, err) 73 | return "", err 74 | } 75 | 76 | symlink.data.target = &target 77 | 78 | case err != nil && cached: 79 | // Reuse symlink.data.target 80 | 81 | case err != nil && !cached: 82 | glog.Errorf("%s: %s", symlink.path, err) 83 | return "", err 84 | } 85 | } else if !cached { 86 | symlink.globals.stats.AccountCacheMiss(stats.ReadlinkFuseOp) 87 | 88 | target, err := symlink.globals.syscalls.Readlink(stats.RemoteDomain, symlink.belowPath) 89 | if err != nil { 90 | glog.Errorf("%s: %s", symlink.path, err) 91 | return "", err 92 | } 93 | 94 | err = symlink.globals.cache.Metadata.PutSymlinkTarget(symlink.inode, target) 95 | if err != nil { 96 | glog.Errorf("%s: %s", symlink.path, err) 97 | return "", err 98 | } 99 | 100 | symlink.data.target = &target 101 | } else { 102 | symlink.globals.stats.AccountCacheHit(stats.ReadlinkFuseOp) 103 | } 104 | 105 | return *symlink.data.target, nil 106 | } 107 | -------------------------------------------------------------------------------- /internal/real/syscalls.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package real 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | "os" 21 | 22 | "github.com/golang/glog" 23 | 24 | "github.com/jmmv/sourcachefs/internal/stats" 25 | ) 26 | 27 | // FileReader is the interface to invoke read-only operations with statistics tracking. 28 | type FileReader interface { 29 | io.Closer 30 | io.Reader 31 | io.ReaderAt 32 | Readdir(int) ([]os.FileInfo, error) 33 | } 34 | 35 | type tracedReader struct { 36 | stats stats.OsStatsUpdater 37 | domain stats.OpDomain 38 | file *os.File 39 | } 40 | 41 | var _ io.Closer = (*tracedReader)(nil) 42 | 43 | func (tr *tracedReader) Close() error { 44 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Close for %s", tr.file.Name())) 45 | tr.stats.AccountOsOp(tr.domain, stats.CloseOsOp) 46 | return tr.file.Close() 47 | } 48 | 49 | var _ io.Reader = (*tracedReader)(nil) 50 | 51 | func (tr *tracedReader) Read(p []byte) (int, error) { 52 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Read for %s", tr.file.Name())) 53 | tr.stats.AccountOsOp(tr.domain, stats.ReadOsOp) 54 | n, err := tr.file.Read(p) 55 | if err == nil && n > 0 { 56 | tr.stats.AccountReadBytes(tr.domain, n) 57 | } 58 | return n, err 59 | } 60 | 61 | var _ io.ReaderAt = (*tracedReader)(nil) 62 | 63 | func (tr *tracedReader) ReadAt(p []byte, off int64) (int, error) { 64 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Read for %s", tr.file.Name())) 65 | tr.stats.AccountOsOp(tr.domain, stats.ReadOsOp) 66 | n, err := tr.file.ReadAt(p, off) 67 | if err == nil && n > 0 { 68 | tr.stats.AccountReadBytes(tr.domain, n) 69 | } 70 | return n, err 71 | } 72 | 73 | func (tr *tracedReader) Readdir(n int) ([]os.FileInfo, error) { 74 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Readdir for %s", tr.file.Name())) 75 | tr.stats.AccountOsOp(tr.domain, stats.ReaddirOsOp) 76 | return tr.file.Readdir(n) 77 | } 78 | 79 | // Syscalls is the interface to invoke kernel-level operations with statistics tracking. 80 | type Syscalls interface { 81 | Lstat(stats.OpDomain, string) (os.FileInfo, error) 82 | Mkdir(stats.OpDomain, string, os.FileMode) error 83 | Open(stats.OpDomain, string) (FileReader, error) 84 | Readlink(stats.OpDomain, string) (string, error) 85 | Remove(stats.OpDomain, string) error 86 | Rename(stats.OpDomain, string, string) error 87 | } 88 | 89 | type syscalls struct { 90 | stats stats.OsStatsUpdater 91 | } 92 | 93 | // NewSyscalls creates a new Syscalls object that accounts statistics towards s. 94 | func NewSyscalls(s stats.OsStatsUpdater) Syscalls { 95 | return &syscalls{ 96 | stats: s, 97 | } 98 | } 99 | 100 | var _ Syscalls = (*syscalls)(nil) 101 | 102 | func (sc *syscalls) Lstat(domain stats.OpDomain, path string) (os.FileInfo, error) { 103 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Lstat for %s", path)) 104 | sc.stats.AccountOsOp(domain, stats.LstatOsOp) 105 | return os.Lstat(path) 106 | } 107 | 108 | func (sc *syscalls) Mkdir(domain stats.OpDomain, path string, mode os.FileMode) error { 109 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Mkdir for %s", path)) 110 | sc.stats.AccountOsOp(domain, stats.MkdirOsOp) 111 | return os.Mkdir(path, mode) 112 | } 113 | 114 | func (sc *syscalls) Open(domain stats.OpDomain, path string) (FileReader, error) { 115 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Open for %s", path)) 116 | sc.stats.AccountOsOp(domain, stats.OpenOsOp) 117 | file, err := os.Open(path) 118 | if err != nil { 119 | return nil, err 120 | } 121 | return &tracedReader{ 122 | stats: sc.stats, 123 | domain: domain, 124 | file: file, 125 | }, nil 126 | } 127 | 128 | func (sc *syscalls) Readlink(domain stats.OpDomain, path string) (string, error) { 129 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Readlink for %s", path)) 130 | sc.stats.AccountOsOp(domain, stats.ReadlinkOsOp) 131 | return os.Readlink(path) 132 | } 133 | 134 | func (sc *syscalls) Rename(domain stats.OpDomain, from string, to string) error { 135 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Rename for %s to %s", from, to)) 136 | sc.stats.AccountOsOp(domain, stats.RenameOsOp) 137 | return os.Rename(from, to) 138 | } 139 | 140 | func (sc *syscalls) Remove(domain stats.OpDomain, path string) error { 141 | glog.InfoDepth(1, fmt.Sprintf("issuing syscall Remove for %s", path)) 142 | sc.stats.AccountOsOp(domain, stats.RemoveOsOp) 143 | return os.Remove(path) 144 | } 145 | -------------------------------------------------------------------------------- /internal/stats/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package stats 16 | 17 | import ( 18 | "sync/atomic" 19 | ) 20 | 21 | // OsOp represents a kernel-level file operation initiated by this file system. 22 | type OsOp int 23 | 24 | // Kernel-level file operations initiated by this file system. 25 | const ( 26 | CloseOsOp OsOp = iota 27 | LstatOsOp OsOp = iota 28 | MkdirOsOp OsOp = iota 29 | OpenOsOp OsOp = iota 30 | ReadOsOp OsOp = iota 31 | ReaddirOsOp OsOp = iota 32 | ReadlinkOsOp OsOp = iota 33 | RemoveOsOp OsOp = iota 34 | RenameOsOp OsOp = iota 35 | ) 36 | 37 | // allOsOps contains the list of all possible kernel-level system calls tracked in statistics. 38 | var allOsOps = []OsOp{CloseOsOp, LstatOsOp, MkdirOsOp, OpenOsOp, ReadOsOp, ReaddirOsOp, ReadlinkOsOp, RemoveOsOp, RenameOsOp} 39 | 40 | // OpDomain indicates where the file operation happened. 41 | type OpDomain int 42 | 43 | // Identifier for the source of a file operation. 44 | const ( 45 | // LocalDomain are operations issued against the local file system. 46 | LocalDomain OpDomain = iota 47 | 48 | // RemoteDomain are operations issued against the target of the sourcachefs file system. 49 | RemoteDomain OpDomain = iota 50 | ) 51 | 52 | // allDomains contains the list of all possible operation domains tracked in statistics. 53 | var allDomains = []OpDomain{LocalDomain, RemoteDomain} 54 | 55 | // PerOsOpStats contains the statistics for a specific kernel-level system call. 56 | type PerOsOpStats struct { 57 | LocalCount uint64 58 | RemoteCount uint64 59 | } 60 | 61 | // FuseOp represents a FUSE-level file operation received by this file system. 62 | type FuseOp int 63 | 64 | // FUSE-level file operations received by this file system. 65 | const ( 66 | AttrFuseOp FuseOp = iota 67 | LookupFuseOp FuseOp = iota 68 | OpenFuseOp FuseOp = iota 69 | ReadFuseOp FuseOp = iota 70 | ReaddirFuseOp FuseOp = iota 71 | ReadlinkFuseOp FuseOp = iota 72 | ) 73 | 74 | // allFuseOps contains the list of all possible FUSE-level operations tracked in statistics. 75 | var allFuseOps = []FuseOp{AttrFuseOp, LookupFuseOp, OpenFuseOp, ReadFuseOp, ReaddirFuseOp, ReadlinkFuseOp} 76 | 77 | // PerFuseOpStats contains the statistics for a specific FUSE-level served call. 78 | type PerFuseOpStats struct { 79 | CacheHits uint64 80 | CacheMisses uint64 81 | Direct uint64 82 | } 83 | 84 | // OsStatsUpdater is the interface that groups the operations to update statistics about the 85 | // kernel-level operations initiated by this file system. 86 | type OsStatsUpdater interface { 87 | AccountOsOp(OpDomain, OsOp) 88 | AccountReadBytes(OpDomain, int) 89 | } 90 | 91 | // FuseStatsUpdater is the interface that groups the operations to update statistics about the 92 | // FUSE-level operations received by this file system. 93 | type FuseStatsUpdater interface { 94 | AccountCacheHit(FuseOp) 95 | AccountCacheMiss(FuseOp) 96 | AccountDirect(FuseOp) 97 | } 98 | 99 | // Clearer is the interface that groups the operations to reset the statistics. 100 | type Clearer interface { 101 | Clear() 102 | } 103 | 104 | // Updater is the interface that groups the operations to modify statistics in any way. 105 | type Updater interface { 106 | OsStatsUpdater 107 | FuseStatsUpdater 108 | Clearer 109 | } 110 | 111 | // Retriever is the interface that groups the operations to fetch statistics. 112 | type Retriever interface { 113 | GetOsOpStats() map[OsOp]PerOsOpStats 114 | GetReadStats() map[OpDomain]Bytes 115 | GetFuseOpStats() map[FuseOp]PerFuseOpStats 116 | } 117 | 118 | // Bytes represents a bytes quantity. 119 | type Bytes uint64 120 | 121 | // Stats contains the statistics data for the file system. 122 | type Stats struct { 123 | osOpStats map[OsOp]*PerOsOpStats 124 | readStats map[OpDomain]*Bytes 125 | fuseOpStats map[FuseOp]*PerFuseOpStats 126 | } 127 | 128 | var _ FuseStatsUpdater = (*Stats)(nil) 129 | var _ OsStatsUpdater = (*Stats)(nil) 130 | var _ Clearer = (*Stats)(nil) 131 | var _ Retriever = (*Stats)(nil) 132 | var _ Updater = (*Stats)(nil) 133 | 134 | // NewStats creates a new Stats instance with all stats set to their nil value. 135 | func NewStats() *Stats { 136 | osOpStats := make(map[OsOp]*PerOsOpStats) 137 | for _, op := range allOsOps { 138 | osOpStats[op] = &PerOsOpStats{} 139 | } 140 | 141 | readStats := make(map[OpDomain]*Bytes) 142 | for _, domain := range allDomains { 143 | readStats[domain] = new(Bytes) 144 | } 145 | 146 | fuseOpStats := make(map[FuseOp]*PerFuseOpStats) 147 | for _, op := range allFuseOps { 148 | fuseOpStats[op] = &PerFuseOpStats{} 149 | } 150 | 151 | return &Stats{ 152 | osOpStats: osOpStats, 153 | readStats: readStats, 154 | fuseOpStats: fuseOpStats, 155 | } 156 | } 157 | 158 | // Clear resets all statistics to their nil value. 159 | func (stats *Stats) Clear() { 160 | for _, op := range allOsOps { 161 | opStats := stats.osOpStats[op] 162 | atomic.StoreUint64(&opStats.LocalCount, 0) 163 | atomic.StoreUint64(&opStats.RemoteCount, 0) 164 | } 165 | 166 | for _, domain := range allDomains { 167 | atomic.StoreUint64((*uint64)(stats.readStats[domain]), 0) 168 | } 169 | 170 | for _, op := range allFuseOps { 171 | opStats := stats.fuseOpStats[op] 172 | atomic.StoreUint64(&opStats.CacheHits, 0) 173 | atomic.StoreUint64(&opStats.CacheMisses, 0) 174 | atomic.StoreUint64(&opStats.Direct, 0) 175 | } 176 | } 177 | 178 | func (stats *Stats) getPerOsOpStats(op OsOp) *PerOsOpStats { 179 | opStats, ok := stats.osOpStats[op] 180 | if !ok { 181 | panic("OS operation not previously registered in map") 182 | } 183 | return opStats 184 | } 185 | 186 | func (stats *Stats) getPerFuseOpStats(op FuseOp) *PerFuseOpStats { 187 | opStats, ok := stats.fuseOpStats[op] 188 | if !ok { 189 | panic("Fuse operation not previously registered in map") 190 | } 191 | return opStats 192 | } 193 | 194 | // AccountOsOp increases the counter for the given OsOp by 1 on the given OpDomain. 195 | func (stats *Stats) AccountOsOp(domain OpDomain, op OsOp) { 196 | opStats := stats.getPerOsOpStats(op) 197 | switch domain { 198 | case LocalDomain: 199 | atomic.AddUint64(&opStats.LocalCount, 1) 200 | case RemoteDomain: 201 | atomic.AddUint64(&opStats.RemoteCount, 1) 202 | default: 203 | panic("Unknown domain") 204 | } 205 | } 206 | 207 | // AccountReadBytes increases the read bytes quantity on the given OpDomain by the given amount. 208 | func (stats *Stats) AccountReadBytes(domain OpDomain, bytes int) { 209 | count, ok := stats.readStats[domain] 210 | if !ok { 211 | panic("Unknown domain") 212 | } 213 | atomic.AddUint64((*uint64)(count), uint64(bytes)) 214 | } 215 | 216 | // AccountCacheHit increases the cache hit counter for the given FuseOp by 1. 217 | // 218 | // Operations that count towards cache hits are those FUSE operations received by sourcachefs that 219 | // did not have to spill into the remote file system. These do not account for direct operations. 220 | func (stats *Stats) AccountCacheHit(op FuseOp) { 221 | opStats := stats.getPerFuseOpStats(op) 222 | atomic.AddUint64(&opStats.CacheHits, 1) 223 | } 224 | 225 | // AccountCacheMiss increases the cache miss counter for the given FuseOp by 1. 226 | // 227 | // Operations that count towards cache misses are those FUSE operations received by sourcachefs 228 | // that incurred a request into the remote file system. These do not account for direct operations. 229 | func (stats *Stats) AccountCacheMiss(op FuseOp) { 230 | opStats := stats.getPerFuseOpStats(op) 231 | atomic.AddUint64(&opStats.CacheMisses, 1) 232 | } 233 | 234 | // AccountDirect increases the direct counter for the given FuseOp by 1. 235 | // 236 | // Operations that count towards direct are those FUSE operations received by sourcachefs that 237 | // were served locally (because the file system was explicitly configured as such). 238 | func (stats *Stats) AccountDirect(op FuseOp) { 239 | opStats := stats.getPerFuseOpStats(op) 240 | atomic.AddUint64(&opStats.Direct, 1) 241 | } 242 | 243 | // GetOsOpStats retrieves details about all kernel-level operations initiated by this file system. 244 | func (stats *Stats) GetOsOpStats() map[OsOp]PerOsOpStats { 245 | opStats := make(map[OsOp]PerOsOpStats) 246 | for op, data := range stats.osOpStats { 247 | opStats[op] = PerOsOpStats{ 248 | LocalCount: atomic.LoadUint64(&data.LocalCount), 249 | RemoteCount: atomic.LoadUint64(&data.RemoteCount), 250 | } 251 | } 252 | return opStats 253 | } 254 | 255 | // GetReadStats retrieves the total amount of data read, broken down by domain. 256 | func (stats *Stats) GetReadStats() map[OpDomain]Bytes { 257 | readStats := make(map[OpDomain]Bytes) 258 | for domain, count := range stats.readStats { 259 | readStats[domain] = Bytes(atomic.LoadUint64((*uint64)(count))) 260 | } 261 | return readStats 262 | } 263 | 264 | // GetFuseOpStats retrieves details about all FUSE-level operations recieved by this file system. 265 | func (stats *Stats) GetFuseOpStats() map[FuseOp]PerFuseOpStats { 266 | opStats := make(map[FuseOp]PerFuseOpStats) 267 | for op, data := range stats.fuseOpStats { 268 | opStats[op] = PerFuseOpStats{ 269 | CacheHits: atomic.LoadUint64(&data.CacheHits), 270 | CacheMisses: atomic.LoadUint64(&data.CacheMisses), 271 | Direct: atomic.LoadUint64(&data.Direct), 272 | } 273 | } 274 | return opStats 275 | } 276 | -------------------------------------------------------------------------------- /internal/stats/status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package stats 16 | 17 | import ( 18 | "html/template" 19 | "io" 20 | "net/http" 21 | "sort" 22 | "strings" 23 | 24 | "github.com/dustin/go-humanize" 25 | ) 26 | 27 | const ( 28 | statusHTML = ` 29 | 30 | 31 | sourcachefs on {{ .StatsRoot }} 32 | 41 | 42 | 43 | 44 |

sourcachefs on {{ .MountPoint }}

45 | 46 |

Built on: {{ .BuildTimestamp }} by {{ .BuildWhere }}
47 | Git revision: {{ .GitRevision }}

48 | 49 |

pprof

50 | 51 |

Clear stats

52 | 53 |

Fuse layer

54 | 55 |

Operation counts:

56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | {{ range .FuseOpStats }} 66 | 67 | 68 | 69 | 70 | 71 | 72 | {{ end }} 73 |
OperationCache hitsCache missesDirect
{{ .Name }}{{ .CacheHits }}{{ .CacheMisses }}{{ .Direct }}
74 | 75 |

OS syscall layer

76 | 77 |

Operation counts:

78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | {{ range .OsOpStats }} 87 | 88 | 89 | 90 | 91 | 92 | {{ end }} 93 |
OperationLocalRemote
{{ .Name }}{{ .LocalCount }}{{ .RemoteCount }}
94 | 95 |

Data processed:

96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 |
Rx/TxLocalRemote
Read{{ .LocalBytes }}{{ .RemoteBytes }}
110 | 111 |

Note: read counters do not account for file content served from the kernel 112 | buffer cache.

113 | 114 | 115 | 116 | ` 117 | ) 118 | 119 | var ( 120 | buildTimestamp string 121 | buildWhere string 122 | gitRevision string 123 | 124 | osOpNames = map[OsOp]string{ 125 | CloseOsOp: "Close", 126 | LstatOsOp: "Lstat", 127 | MkdirOsOp: "Mkdir", 128 | OpenOsOp: "Open", 129 | ReadOsOp: "Read", 130 | ReaddirOsOp: "Readdir", 131 | ReadlinkOsOp: "Readlink", 132 | RemoveOsOp: "Remove", 133 | RenameOsOp: "Rename", 134 | } 135 | 136 | fuseOpNames = map[FuseOp]string{ 137 | AttrFuseOp: "Attr", 138 | LookupFuseOp: "Lookup", 139 | OpenFuseOp: "Open", 140 | ReadFuseOp: "Read", 141 | ReaddirFuseOp: "Readdir", 142 | ReadlinkFuseOp: "Readlink", 143 | } 144 | ) 145 | 146 | type osOpStatsRow struct { 147 | Name string 148 | LocalCount uint64 149 | RemoteCount uint64 150 | } 151 | 152 | type osOpStatsSlice []osOpStatsRow 153 | 154 | var _ (sort.Interface) = (*osOpStatsSlice)(nil) 155 | 156 | func (slice osOpStatsSlice) Len() int { 157 | return len(slice) 158 | } 159 | 160 | func (slice osOpStatsSlice) Less(i, j int) bool { 161 | return strings.Compare(slice[i].Name, slice[j].Name) == -1 162 | } 163 | 164 | func (slice osOpStatsSlice) Swap(i, j int) { 165 | slice[i], slice[j] = slice[j], slice[i] 166 | } 167 | 168 | type fuseOpStatsRow struct { 169 | Name string 170 | CacheHits uint64 171 | CacheMisses uint64 172 | Direct uint64 173 | } 174 | 175 | type fuseOpStatsSlice []fuseOpStatsRow 176 | 177 | var _ (sort.Interface) = (*fuseOpStatsSlice)(nil) 178 | 179 | func (slice fuseOpStatsSlice) Len() int { 180 | return len(slice) 181 | } 182 | 183 | func (slice fuseOpStatsSlice) Less(i, j int) bool { 184 | return strings.Compare(slice[i].Name, slice[j].Name) == -1 185 | } 186 | 187 | func (slice fuseOpStatsSlice) Swap(i, j int) { 188 | slice[i], slice[j] = slice[j], slice[i] 189 | } 190 | 191 | func generateStatusPage(root string, mountPoint string, stats Retriever, writer io.Writer) error { 192 | osOpStats := stats.GetOsOpStats() 193 | osOpRows := make([]osOpStatsRow, 0, len(osOpStats)) 194 | for op, data := range osOpStats { 195 | osOpRows = append(osOpRows, osOpStatsRow{ 196 | Name: osOpNames[op], 197 | LocalCount: data.LocalCount, 198 | RemoteCount: data.RemoteCount, 199 | }) 200 | } 201 | 202 | readStats := stats.GetReadStats() 203 | 204 | fuseOpStats := stats.GetFuseOpStats() 205 | fuseOpRows := make([]fuseOpStatsRow, 0, len(fuseOpStats)) 206 | for op, data := range fuseOpStats { 207 | fuseOpRows = append(fuseOpRows, fuseOpStatsRow{ 208 | Name: fuseOpNames[op], 209 | CacheHits: data.CacheHits, 210 | CacheMisses: data.CacheMisses, 211 | Direct: data.Direct, 212 | }) 213 | } 214 | 215 | sort.Sort(osOpStatsSlice(osOpRows)) 216 | sort.Sort(fuseOpStatsSlice(fuseOpRows)) 217 | 218 | statusTemplate := template.Must( 219 | template.New("status").Parse(statusHTML)) 220 | data := struct { 221 | BuildTimestamp string 222 | BuildWhere string 223 | GitRevision string 224 | StatsRoot string 225 | MountPoint string 226 | OsOpStats osOpStatsSlice 227 | FuseOpStats fuseOpStatsSlice 228 | LocalBytes string 229 | RemoteBytes string 230 | }{ 231 | BuildTimestamp: buildTimestamp, 232 | BuildWhere: buildWhere, 233 | GitRevision: gitRevision, 234 | StatsRoot: root, 235 | MountPoint: mountPoint, 236 | OsOpStats: osOpRows, 237 | FuseOpStats: fuseOpRows, 238 | LocalBytes: humanize.Bytes((uint64)(readStats[LocalDomain])), 239 | RemoteBytes: humanize.Bytes((uint64)(readStats[RemoteDomain])), 240 | } 241 | 242 | return statusTemplate.Execute(writer, data) 243 | } 244 | 245 | // SetupHTTPHandlers configures the HTTP server with the endpoints to access statistics. 246 | // 247 | // The configured handlers operate on the given Stats object, and the endpoints are installed 248 | // relative to the given root path. 249 | func SetupHTTPHandlers(stats *Stats, root string, mountPoint string) { 250 | if len(buildTimestamp) == 0 { 251 | panic("Missing build timestamp") 252 | } 253 | if len(buildWhere) == 0 { 254 | panic("Missing build where") 255 | } 256 | if len(gitRevision) == 0 { 257 | panic("Missing git revision") 258 | } 259 | 260 | http.HandleFunc(root+"/summary", func(w http.ResponseWriter, r *http.Request) { 261 | err := generateStatusPage(root, mountPoint, stats, w) 262 | if err != nil { 263 | http.Error(w, err.Error(), http.StatusInternalServerError) 264 | } 265 | }) 266 | http.HandleFunc(root+"/clear", func(w http.ResponseWriter, r *http.Request) { 267 | stats.Clear() 268 | http.Redirect(w, r, root+"/summary", http.StatusFound) 269 | }) 270 | } 271 | -------------------------------------------------------------------------------- /internal/test/test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy 5 | // of the License at: 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package test 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | "os" 21 | 22 | "github.com/stretchr/testify/suite" 23 | ) 24 | 25 | // SuiteWithTempDir is an extension to suite.Suite that manages a temporary directory for all 26 | // test cases to use. 27 | // 28 | // Suites implementing this type must be careful to invoke SetupTest() and TearDownTest() on their 29 | // own if they redefine those methods. 30 | type SuiteWithTempDir struct { 31 | suite.Suite 32 | 33 | tempDir string 34 | } 35 | 36 | var _ suite.SetupTestSuite = (*SuiteWithTempDir)(nil) 37 | var _ suite.TearDownTestSuite = (*SuiteWithTempDir)(nil) 38 | 39 | // SetupTest creates a temporary directory. Fails the test case if the directory creation fails. 40 | func (s *SuiteWithTempDir) SetupTest() { 41 | tempDir, err := ioutil.TempDir("", "test") 42 | s.Require().NoError(err) 43 | defer func() { 44 | // Only clean up the temporary directory if we haven't completed setup. 45 | if s.tempDir == "" { 46 | os.RemoveAll(tempDir) 47 | } 48 | }() 49 | 50 | // Now that setup went well, initialize all fields. No operation that can fail should happen 51 | // after this point, or the cleanup routines scheduled with defer may do the wrong thing. 52 | s.tempDir = tempDir 53 | } 54 | 55 | // TearDownTest recursively destroys the temporary directory created for the test case. Fails the 56 | // test case if the destruction fails. 57 | func (s *SuiteWithTempDir) TearDownTest() { 58 | s.NoError(os.RemoveAll(s.tempDir)) 59 | } 60 | 61 | // TempDir retrieves the path to the temporary directory for the test case. 62 | func (s *SuiteWithTempDir) TempDir() string { 63 | return s.tempDir 64 | } 65 | 66 | // CheckFileExists checks if a file exists and returns an error otherwise. 67 | func CheckFileExists(path string) error { 68 | if _, err := os.Stat(path); os.IsNotExist(err) { 69 | return fmt.Errorf("expected file %s does not exist", path) 70 | } 71 | return nil 72 | } 73 | 74 | // CheckFileNotExists checks if a file does not exist and returns an error otherwise. 75 | func CheckFileNotExists(path string) error { 76 | if _, err := os.Stat(path); os.IsExist(err) { 77 | return fmt.Errorf("unexpected file %s exists", path) 78 | } 79 | return nil 80 | } 81 | 82 | // CheckFileContents checks if a file matches the given contents. 83 | func CheckFileContents(path string, expectedContents string) error { 84 | contents, err := ioutil.ReadFile(path) 85 | if err != nil { 86 | return err 87 | } 88 | if string(contents) != expectedContents { 89 | return fmt.Errorf("file %s doesn't match expected contents: got '%s', want '%s'", path, contents, expectedContents) 90 | } 91 | return nil 92 | } 93 | 94 | // WriteFile creates or overwrites a file with the given contents. 95 | func WriteFile(path string, contents string) error { 96 | file, err := os.Create(path) 97 | if err != nil { 98 | return err 99 | } 100 | n, err := file.WriteString(contents) 101 | if err != nil { 102 | return err 103 | } else if n != len(contents) { 104 | return fmt.Errorf("failed to write contents in file %s: got %d length, want %d", path, n, len(contents)) 105 | } 106 | return file.Close() 107 | } 108 | --------------------------------------------------------------------------------