├── .appveyor.yml ├── .gitignore ├── .gitmodules ├── License.txt ├── Makefile ├── README.md ├── _tools ├── authtool.go ├── listtool.go ├── run-tests ├── run-tests.cmd └── vendor.sh ├── assets └── assets.go ├── auth ├── auth.go └── auth_test.go ├── cache ├── cache.go ├── cache_test.go ├── link.go ├── node.go ├── node_test.go ├── openfile_unix.go └── openfile_windows.go ├── cache_commands.go ├── commands.go ├── debug.go ├── errno ├── errno.go ├── errno_error.go └── errno_string.go ├── fs ├── errc.go ├── mount.go ├── objfs │ └── objfs.go └── tracefs.go ├── httputil └── httputil.go ├── make.cmd ├── objfs.1 ├── objfs.1.asciidoc ├── objfs.go ├── objio ├── objio.go └── tracestg.go ├── objreg ├── objreg.go └── objreg_test.go ├── registry.go └── registry.go.in /.appveyor.yml: -------------------------------------------------------------------------------- 1 | version: '{build}' 2 | 3 | clone_folder: C:\projects\go\src\github.com\billziss-gh\objfs 4 | 5 | environment: 6 | GOPATH: C:\projects\go 7 | PATH: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%PATH% 8 | CRED_ONEDRIVE_CLIENT_ID: 9 | secure: UT11dYGRv3c3lhoGnCy5ovV8wv8yG4O4Q4SWds188IzXoTPAfBXUIqvlfnkYKRU48yiB804PJNJcozwov7xiJQ== 10 | CRED_ONEDRIVE_CLIENT_SECRET: 11 | secure: 3XPF6jI1pL85NB/1KLZL8ts+4AXBVbL3ToqRij2ZGDkdhxoeNrcFvcbrRKVHjIU1 12 | CRED_ONEDRIVE_REFRESH_TOKEN: 13 | secure: WNdc7OGBuyPM3GM5HIyCkzQhZpUHoryEBdb/c3al9yaWYxOq56Hk8F3rgfiDxXMQysFyicidjLxVhIAiLP20ab+rbipmuj/Umg75lQ+P8s8yQAzXSLcalEe7JwzuO3rNNG3kLs/DfMIxtQY+Idz2E2+FVSeJKu5ANhd4n9Dq/59PmofTzgPLkSNQXQAz+WMNpppZBdO4mcDBUcDyrzV+3ZCrCTOmNFwihbLfKiK8szfCr710aLEbDaheECktC73x3/rPPUmeYgKYD98FujYr6bwahgWoZEeGsMh0qFUeFCqtst4aTKpqSSR8+VmiB9j4XqSzFY0m5vOKAdSrd542j5QVddFLryAh+90awBWPLDftazgsLrTjXTAdixbNaRVmXBteSXWCoCc4dWUcGtAi6k49k49o4Ry9S0waPBBfUWC0teE+sCPwFATcdx6GpdpFWDj/RjNPAHcDvf6gnVbNOhvoEznD3iRcc6oyNtPA47bw1/xmiE7gl0eI5GzqJtuf6UGhvvsVFupeWfCMeMu5/139TIZa4IVOg5pS9sI+0Mw= 14 | CRED_DROPBOX_CLIENT_ID: 15 | secure: VOtCMn3h+Nc6e73ltKHDwfSPwyi5+YGD5JAaNK/91lA= 16 | CRED_DROPBOX_CLIENT_SECRET: 17 | secure: TCUqhc9RPVu08+GiuQ54gVSaQmKbUgAPwIzTuJ/PqSM= 18 | CRED_DROPBOX_ACCESS_TOKEN: 19 | secure: kC1OC3iN+t5NebzM6rQrWYm0CFcIYDrmd+S3lfJ1sEEc+EXTzg3ip4GF9FOlX+eaFUaRAgXNGH2heNVEsz5ATiH+WtrX62AJG+Qb3NSZN0w= 20 | 21 | install: 22 | - git submodule update --init --recursive 23 | - git clone -q https://github.com/billziss-gh/winfsp.git C:\projects\winfsp 24 | - git -C C:\projects\winfsp checkout -q release/1.2 25 | - git clone -q https://github.com/billziss-gh/secfs.test.git C:\projects\secfs.test 26 | - git -C C:\projects\secfs.test checkout -q a00c9165646f78ad53d3ec052860384a029683e5 27 | - choco install winfsp --version=1.2.17346 -y 28 | 29 | build_script: 30 | - make racy 31 | 32 | test_script: 33 | - call "%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat" x64 34 | - devenv C:\projects\winfsp\build\VStudio\winfsp.sln /build "Release|x64" 35 | - pushd C:\projects\secfs.test && nmake /f Nmakefile && popd 36 | 37 | - echo access_token=0 >C:\projects\onedrive.txt 38 | - echo %CRED_ONEDRIVE_CLIENT_ID% >>C:\projects\onedrive.txt 39 | - echo %CRED_ONEDRIVE_CLIENT_SECRET% >>C:\projects\onedrive.txt 40 | - echo %CRED_ONEDRIVE_REFRESH_TOKEN% >>C:\projects\onedrive.txt 41 | 42 | # hard code fsreg parameters because appveyor does not like percents ("--VolumePrefix=%1 %2") 43 | - C:\projects\winfsp\tools\fsreg.bat objfs.onedrive "C:\projects\go\src\github.com\billziss-gh\objfs\objfs.exe" "-storage=onedrive -credentials=C:\projects\onedrive.txt mount -o uid=11,gid=65792,umask=0077,VolumePrefix=/objfs.onedrive/share M:" "D:P(A;;RPWPLC;;;WD)" 44 | 45 | - 'net use M: \\objfs.onedrive\share' 46 | - 'M: & cd' 47 | - C:\projects\winfsp\build\VStudio\build\Release\winfsp-tests-x64.exe --case-insensitive-cmp --external --resilient --share-prefix=\objfs.onedrive\share -reparse* -stream* -create_allocation_test -create_backup_test -create_restore_test -create_fileattr_test -create_notraverse_test -getfileinfo_name_test -setfileinfo_test -delete_access_test -setsecurity_test -exec_rename_test 48 | - C:\projects\secfs.test\fstools\src\fsx\fsx.exe -N 10000 test xxxxxx 49 | - 'C: & cd' 50 | - 'net use M: /delete' 51 | 52 | - echo token_type="Bearer" >C:\projects\dropbox.txt 53 | - echo %CRED_DROPBOX_CLIENT_ID% >>C:\projects\dropbox.txt 54 | - echo %CRED_DROPBOX_CLIENT_SECRET% >>C:\projects\dropbox.txt 55 | - echo %CRED_DROPBOX_ACCESS_TOKEN% >>C:\projects\dropbox.txt 56 | 57 | # hard code fsreg parameters because appveyor does not like percents ("--VolumePrefix=%1 %2") 58 | - C:\projects\winfsp\tools\fsreg.bat objfs.dropbox "C:\projects\go\src\github.com\billziss-gh\objfs\objfs.exe" "-storage=dropbox -credentials=C:\projects\dropbox.txt mount -o uid=11,gid=65792,umask=0077,VolumePrefix=/objfs.dropbox/share M:" "D:P(A;;RPWPLC;;;WD)" 59 | 60 | - 'net use M: \\objfs.dropbox\share' 61 | - 'M: & cd' 62 | - C:\projects\winfsp\build\VStudio\build\Release\winfsp-tests-x64.exe --case-insensitive-cmp --external --resilient --share-prefix=\objfs.dropbox\share -reparse* -stream* -create_allocation_test -create_backup_test -create_restore_test -create_fileattr_test -create_notraverse_test -getfileinfo_test -getfileinfo_name_test -setfileinfo_test -delete_access_test -rename_caseins_test -setsecurity_test -exec_rename_test 63 | - C:\projects\secfs.test\fstools\src\fsx\fsx.exe -N 10000 test xxxxxx 64 | - 'C: & cd' 65 | - 'net use M: /delete' 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | objfs 3 | objfs.exe 4 | debug 5 | debug.exe 6 | debug.test 7 | debug.test.exe 8 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/github.com/billziss-gh/cgofuse"] 2 | path = vendor/github.com/billziss-gh/cgofuse 3 | url = https://github.com/billziss-gh/cgofuse.git 4 | [submodule "vendor/github.com/billziss-gh/golib"] 5 | path = vendor/github.com/billziss-gh/golib 6 | url = https://github.com/billziss-gh/golib.git 7 | [submodule "vendor/github.com/boltdb/bolt"] 8 | path = vendor/github.com/boltdb/bolt 9 | url = https://github.com/boltdb/bolt.git 10 | [submodule "vendor/github.com/billziss-gh/objfs.pkg"] 11 | path = vendor/github.com/billziss-gh/objfs.pkg 12 | url = https://github.com/billziss-gh/objfs.pkg.git 13 | -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile 2 | 3 | MyBuildNumber=$(shell date +%y%j) 4 | MyVersion=0.9.$(MyBuildNumber) 5 | 6 | Packages=\ 7 | ./vendor/github.com/billziss-gh/objfs.pkg/objio/onedrive\ 8 | ./vendor/github.com/billziss-gh/objfs.pkg/objio/dropbox 9 | 10 | ifeq ($(OS),Windows_NT) 11 | PathSep=\$(strip) 12 | else 13 | PathSep=/ 14 | endif 15 | 16 | .PHONY: default 17 | default: build 18 | 19 | .PHONY: generate 20 | generate: 21 | go generate ./... 22 | 23 | .PHONY: build 24 | build: registry.go 25 | go build -ldflags "-s -w -X \"main.MyVersion=$(MyVersion)\"" 26 | 27 | .PHONY: racy 28 | racy: registry.go 29 | go build -race -ldflags "-s -w -X \"main.MyVersion=$(MyVersion)-racy\"" 30 | 31 | .PHONY: debug 32 | debug: registry.go 33 | go build -race -tags debug -gcflags all="-N -l" 34 | 35 | registry.go: registry.go.in Makefile 36 | go run _tools/listtool.go registry.go.in $(Packages) > registry.go 37 | 38 | .PHONY: manpage 39 | manpage: $(patsubst %.1.asciidoc,%.1,$(wildcard *.1.asciidoc)) 40 | %.1: %.1.asciidoc 41 | asciidoctor -b manpage *.1.asciidoc 42 | 43 | .PHONY: test 44 | test: 45 | _tools$(PathSep)run-tests -count=1 ./... ./vendor/github.com/billziss-gh/objfs.pkg/... 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # objfs - Object Storage File System 2 | 3 | The [objfs](https://github.com/billziss-gh/objfs) repository and its companion repository [objfs.pkg](https://github.com/billziss-gh/objfs.pkg) contain the implementation of objfs, the "object storage file system". 4 | 5 | Objfs exposes objects from an object storage, such as a cloud drive, etc. as files in a file system that is fully integrated with the operating system. Programs that run on the operating system are able to access these files as if they are stored in a local "drive" (perhaps with some delay due to network operations). 6 | 7 | - Supported operating systems: Windows, macOS, and Linux. 8 | - Supported object storages: OneDrive 9 | 10 | ## How to use 11 | 12 | Objfs is implemented as a command-line program that accepts commands such as `auth` and `mount`, but also shell-like commands, such as `ls`, `stat`, etc. 13 | 14 | ``` 15 | $ ./objfs help 16 | usage: objfs [-options] command args... 17 | 18 | commands: 19 | version 20 | get current version information 21 | config 22 | get or set configuration options 23 | keyring 24 | get or set keys 25 | auth 26 | perform authentication/authorization 27 | mount 28 | mount file system 29 | statfs 30 | get storage information 31 | ls 32 | list files 33 | stat 34 | display file information 35 | mkdir 36 | make directories 37 | rmdir 38 | remove directories 39 | rm 40 | remove files 41 | mv 42 | move (rename) files 43 | get 44 | get (download) files 45 | put 46 | put (upload) files 47 | cache-pending 48 | list pending cache files 49 | cache-reset 50 | reset cache (upload and evict files) 51 | 52 | options: 53 | -accept-tls-cert 54 | accept any TLS certificate presented by the server (insecure) 55 | -auth name 56 | auth name to use 57 | -config path 58 | path to configuration file 59 | -credentials path 60 | auth credentials path (keyring:service/user or /file/path) 61 | -datadir path 62 | path to supporting data and caches 63 | -keyring string 64 | keyring type to use: system, private (default "private") 65 | -storage name 66 | storage name to access (default "onedrive") 67 | -storage-uri uri 68 | storage uri to access 69 | -v verbose 70 | ``` 71 | 72 | ### Default Storage 73 | 74 | Objfs uses defaults to simplify command line invocation. In the default build of objfs, the default storage is `onedrive`. 75 | 76 | ### Auth 77 | 78 | Objfs supports multiple "auth" (authentication or authorization) mechanisms through the `-credentials path` option and the `auth` command. 79 | 80 | In general before an object storage service can be used it requires auth. The specific auth mechanism used depends on the service and it ranges from no auth, to username/password, to Oauth2, etc. Auth mechanisms require credentials, which can be supplied using the `-credentials path` option. 81 | 82 | In some cases the object storage service cannot readily accept the supplied credentials, they must be converted to other credentials first. As an authentication example, a particular service may require username/password credentials to be converted to some form of service-level token before they can be used. As an authorization example Oauth2 requires application-level credentials together with user consent to form a service-level token that can be used to access the service. 83 | 84 | The `auth` command can be used for this purpose. It takes user-level or application-level credentials and converts them to service-level credentials. 85 | 86 | Credentials can be stored in the local file system or the system keyring. The syntax `/file/path` is used to name credentials stored in the file system. The syntax `keyring:service/user` is used to name credentials stored in the system keyring. 87 | 88 | #### Example - Oauth2 Flow 89 | 90 | - Prepare the Oauth2 `client_secret` credentials in a file or the system keyring: 91 | ``` 92 | client_id="XXXXXXXX" 93 | client_secret="XXXXXXXX" 94 | redirect_uri="http://localhost:xxxxx" 95 | scope="files.readwrite.all offline_access" 96 | ``` 97 | - Issue the command: 98 | ``` 99 | $ ./objfs -credentials=CLIENT_SECRET_PATH auth TOKEN_PATH 100 | ``` 101 | - This will launch your browser and ask for authorization. If the access is authorized the Oauth2 `access_token` and `refresh_token` will be stored in the specified path. 102 | - The object storage can now be mounted using the command: 103 | ``` 104 | $ ./objfs -credentials=TOKEN_PATH mount MOUNTPOINT 105 | ``` 106 | 107 | ### Mount 108 | 109 | The objfs `mount` command is used to mount an object storage as a file system on a mountpoint. On Windows the mount point must be a non-existing drive or directory; it is recommended that an object storage is only mounted as a drive when the object storage is case-sensitive. On macOS and Linux the mount point must be an existing directory. 110 | 111 | To mount on Windows: 112 | 113 | ``` 114 | > objfs -credentials=TOKEN_PATH mount -o uid=-1,gid=-1 mount X: 115 | ``` 116 | 117 | To mount on macOS and Linux: 118 | 119 | ``` 120 | $ ./objfs -credentials=TOKEN_PATH mount MOUNTPOINT 121 | ``` 122 | 123 | Objfs uses a local file cache to speed up file system operations. This caches files locally when they are first opened; subsequent I/O operations will be performed against the local file and are therefore fast. Modified files will be uploaded to the object storage when they are closed. File system operations such as creating and deleting files and listing directories are sent directly to the object storage and are therefore slow (although some of their results are cached). 124 | 125 | The Objfs cache was inspired by an early version of the Andrew File System (AFS). For more information see this [paper](http://pages.cs.wisc.edu/~remzi/OSTEP/dist-afs.pdf). 126 | 127 | ### Diagnostics 128 | 129 | Objfs includes a tracing facility that can be used to troubleshoot problems, to gain insights into its internal workings, etc. This facility is enabled when the `-v` option is used. 130 | 131 | The environment variable `GOLIB_TRACE` controls which traces are enabled. This variable accepts a comma separated list of file-style patterns containing wildcards such as `*` and `?`. 132 | 133 | ``` 134 | $ export GOLIB_TRACE=pattern1,...,patternN 135 | ``` 136 | 137 | Examples: 138 | 139 | ``` 140 | $ export GOLIB_TRACE=github.com/billziss-gh/objfs/fs.* # file system traces 141 | $ export GOLIB_TRACE=github.com/billziss-gh/objfs/objio.* # object storage traces 142 | $ export GOLIB_TRACE=github.com/billziss-gh/objfs/fs.*,github.com/billziss-gh/objfs/objio.* 143 | $ ./objfs -v -credentials=TOKEN_PATH mount MOUNTPOINT 144 | ``` 145 | 146 | ## How to build 147 | 148 | Objfs is written in Go and uses [cgofuse](https://github.com/billziss-gh/cgofuse) to interface with the operating system. It requires the relevant FUSE drivers/libraries for each operating system. 149 | 150 | Prerequisites: 151 | - Windows: [WinFsp](https://github.com/billziss-gh/winfsp), gcc (e.g. from [Mingw-builds](http://mingw-w64.org/doku.php/download)) 152 | - macOS: [FUSE for macOS](https://osxfuse.github.io), [command line tools](https://developer.apple.com/library/content/technotes/tn2339/_index.html) 153 | - Linux: libfuse-dev, gcc 154 | 155 | To build the following is usually sufficient: 156 | 157 | ``` 158 | $ go get -d github.com/billziss-gh/objfs 159 | $ make # on windows you may have to update the PATH in make.cmd 160 | ``` 161 | 162 | This will include all supported storages. Objfs storage and auth mechanisms are maintained in the separate repository objfs.pkg. You can customize the supported storages for licensing or other reasons by modifying the [`Makefile Packages`](Makefile) variable. 163 | 164 | ## License 165 | 166 | The objfs and objfs.pkg repositories are available under the [AGPLv3](License.txt) license. 167 | 168 | The project has the following dependencies and their licensing: 169 | 170 | - [cgofuse](https://github.com/billziss-gh/cgofuse) - Cross-platform FUSE library for Go. 171 | - License: MIT 172 | - [boltdb](https://github.com/boltdb/bolt) - An embedded key/value database for Go. 173 | - License: MIT 174 | - [golib](https://github.com/billziss-gh/golib) - Collection of Go libraries. 175 | - License: MIT 176 | - [oauth2-helper](https://github.com/billziss-gh/oauth2-helper) - OAuth 2.0 for Native Apps 177 | - License: MIT 178 | - [WinFsp](https://github.com/billziss-gh/winfsp) - Windows File System Proxy - FUSE for Windows. 179 | - License: GPLv3 w/ FLOSS exception 180 | - [FUSE for macOS](https://osxfuse.github.io) - File system integration made easy. 181 | - License: BSD-style 182 | - [libfuse](https://github.com/libfuse/libfuse) - The reference implementation of the Linux FUSE (Filesystem in Userspace) interface. 183 | - License: LGPLv2.1 184 | -------------------------------------------------------------------------------- /_tools/authtool.go: -------------------------------------------------------------------------------- 1 | ///usr/bin/env go run "$0" "$@"; exit 2 | // +build tool 3 | 4 | /* 5 | * authtool.go 6 | * 7 | * Copyright 2018 Bill Zissimopoulos 8 | */ 9 | /* 10 | * This file is part of Objfs. 11 | * 12 | * You can redistribute it and/or modify it under the terms of the GNU 13 | * Affero General Public License version 3 as published by the Free 14 | * Software Foundation. 15 | * 16 | * Licensees holding a valid commercial license may use this file in 17 | * accordance with the commercial license agreement provided with the 18 | * software. 19 | */ 20 | 21 | package main 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | "github.com/billziss-gh/objfs.pkg/auth/oauth2" 28 | "github.com/billziss-gh/objfs/auth" 29 | ) 30 | 31 | func fail(err error) { 32 | fmt.Fprintln(os.Stderr, "error:", err) 33 | os.Exit(1) 34 | } 35 | 36 | func usage() { 37 | fmt.Fprintln(os.Stderr, `usage: authtool kind args... ipath opath`) 38 | os.Exit(2) 39 | } 40 | 41 | func main() { 42 | if 5 > len(os.Args) { 43 | usage() 44 | } 45 | 46 | kind := os.Args[1] 47 | args := []interface{}{} 48 | for _, a := range os.Args[2 : len(os.Args)-2] { 49 | args = append(args, a) 50 | } 51 | ipath := os.Args[len(os.Args)-2] 52 | opath := os.Args[len(os.Args)-1] 53 | 54 | cmap, err := auth.ReadCredentials(ipath) 55 | if nil != err { 56 | fail(err) 57 | } 58 | 59 | a, err := auth.Registry.NewObject(kind, args...) 60 | if nil != err { 61 | fail(err) 62 | } 63 | 64 | sess, err := a.(auth.Auth).Session(cmap) 65 | if nil != err { 66 | fail(err) 67 | } 68 | 69 | cmap = sess.Credentials() 70 | err = auth.WriteCredentials(opath, cmap) 71 | if nil != err { 72 | fail(err) 73 | } 74 | } 75 | 76 | func init() { 77 | oauth2.Load() 78 | } 79 | -------------------------------------------------------------------------------- /_tools/listtool.go: -------------------------------------------------------------------------------- 1 | ///usr/bin/env go run "$0" "$@"; exit 2 | // +build tool 3 | 4 | /* 5 | * listtool.go 6 | * 7 | * Copyright 2018 Bill Zissimopoulos 8 | */ 9 | /* 10 | * This file is part of Objfs. 11 | * 12 | * You can redistribute it and/or modify it under the terms of the GNU 13 | * Affero General Public License version 3 as published by the Free 14 | * Software Foundation. 15 | * 16 | * Licensees holding a valid commercial license may use this file in 17 | * accordance with the commercial license agreement provided with the 18 | * software. 19 | */ 20 | 21 | package main 22 | 23 | import ( 24 | "bytes" 25 | "encoding/json" 26 | "fmt" 27 | "go/build" 28 | "os" 29 | "os/exec" 30 | "path/filepath" 31 | "strings" 32 | "text/template" 33 | ) 34 | 35 | func fail(err error) { 36 | fmt.Fprintln(os.Stderr, "error:", err) 37 | os.Exit(1) 38 | } 39 | 40 | func usage() { 41 | fmt.Fprintln(os.Stderr, `usage: listtool template-file packages`) 42 | os.Exit(2) 43 | } 44 | 45 | func stripVendor(path string) string { 46 | if i := strings.LastIndex(path, "/vendor/"); -1 != i { 47 | return path[i+len("/vendor/"):] 48 | } 49 | return path 50 | } 51 | 52 | func main() { 53 | if 2 > len(os.Args) { 54 | usage() 55 | } 56 | 57 | tname := os.Args[1] 58 | funcs := template.FuncMap{ 59 | "stripVendor": stripVendor, 60 | } 61 | templ, err := template.New(filepath.Base(tname)).Funcs(funcs).ParseFiles(tname) 62 | if nil != err { 63 | fail(err) 64 | } 65 | 66 | args := append([]string{"list", "-json"}, os.Args[2:]...) 67 | out, err := exec.Command("go", args...).Output() 68 | if nil != err { 69 | if e, ok := err.(*exec.ExitError); ok { 70 | fmt.Fprintln(os.Stderr, string(e.Stderr)) 71 | os.Exit(1) 72 | } else { 73 | fail(err) 74 | } 75 | } 76 | 77 | var packages []build.Package 78 | for dec := json.NewDecoder(bytes.NewReader(out)); dec.More(); { 79 | var p build.Package 80 | err := dec.Decode(&p) 81 | if nil != err { 82 | fail(err) 83 | } 84 | 85 | packages = append(packages, p) 86 | } 87 | 88 | err = templ.Execute(os.Stdout, packages) 89 | if nil != err { 90 | fail(err) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /_tools/run-tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname "$0")/.. 4 | 5 | trap "rm -f ./_test.onedrive_token" EXIT 6 | go run _tools/authtool.go oauth2 https://login.microsoftonline.com/common/oauth2/v2.0/authorize https://login.microsoftonline.com/common/oauth2/v2.0/token keyring:objfs/onedrive_client_secret ./_test.onedrive_token && \ 7 | go test "$@" 8 | -------------------------------------------------------------------------------- /_tools/run-tests.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | pushd %~dp0.. 4 | 5 | go run _tools/authtool.go oauth2 https://login.microsoftonline.com/common/oauth2/v2.0/authorize https://login.microsoftonline.com/common/oauth2/v2.0/token keyring:objfs/onedrive_client_secret ./_test.onedrive_token && ^ 6 | go test %* 7 | del _test.onedrive_token 8 | 9 | popd 10 | -------------------------------------------------------------------------------- /_tools/vendor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | Hosts="(github\.com|bitbucket\.org|gitlab\.com)" 4 | Root=$(cd "$0/../.." | pwd) 5 | 6 | usage() { 7 | echo "usage: vendor.sh {list|add} dir..." 1>&2 8 | exit 2 9 | } 10 | 11 | subadd() { 12 | (cd "$Root" && git submodule add "https://$1.git" "vendor/$1") 13 | } 14 | 15 | case "$1" in 16 | add) Command="subadd"; shift;; 17 | list) Command="echo"; shift;; 18 | *) usage;; 19 | esac 20 | [[ $# -gt 0 ]] || usage 21 | 22 | for a in "$@"; do 23 | ( 24 | cd "$a" 25 | Package=$(go list | sed -E -n -e 's@^([^/]*/[^/]*/[^/]*).*$@\1@p') 26 | go list -f '{{join .Deps "\n"}}' | 27 | sed -E -n \ 28 | -e '\'"@^$Package@d" \ 29 | -e '\'"@^$Hosts@"'s@^([^/]*/[^/]*/[^/]*).*$@\1@p' 30 | ) 31 | done | sort | uniq | 32 | while read Package; do 33 | if [[ ! -e "$Root/vendor/$Package" ]]; then 34 | $Command $Package 35 | fi 36 | done 37 | -------------------------------------------------------------------------------- /assets/assets.go: -------------------------------------------------------------------------------- 1 | /* 2 | * assets.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package assets 19 | 20 | import ( 21 | "os" 22 | "path/filepath" 23 | "runtime" 24 | "strings" 25 | ) 26 | 27 | var basedir string 28 | 29 | func init() { 30 | exe, err := os.Executable() 31 | if nil == err && !strings.HasSuffix(exe, ".test") && !strings.HasSuffix(exe, ".test.exe") { 32 | basedir = filepath.Join(filepath.Dir(exe), "assets") 33 | if info, err := os.Stat(filepath.Join(basedir, "sys")); nil == err && info.IsDir() { 34 | return 35 | } 36 | } 37 | 38 | basedir = "\000" 39 | } 40 | 41 | // GetPath returns the full path for an asset. 42 | func GetPath(subdir string, name string) string { 43 | dir := basedir 44 | if "\000" == dir { 45 | _, file, _, ok := runtime.Caller(1) 46 | if ok { 47 | testdir := filepath.Join(filepath.Dir(file), "assets") 48 | if info, err := os.Stat(filepath.Join(testdir, "sys")); nil == err && info.IsDir() { 49 | dir = testdir 50 | } 51 | } 52 | } 53 | 54 | if "sys" == subdir { 55 | return filepath.Join(dir, "sys", runtime.GOOS+"_"+runtime.GOARCH, name) 56 | } else { 57 | return filepath.Join(dir, subdir, name) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /auth/auth.go: -------------------------------------------------------------------------------- 1 | /* 2 | * auth.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package auth 19 | 20 | import ( 21 | "bytes" 22 | "fmt" 23 | "net/url" 24 | "os" 25 | "strings" 26 | 27 | "github.com/billziss-gh/golib/config" 28 | "github.com/billziss-gh/golib/keyring" 29 | "github.com/billziss-gh/golib/util" 30 | "github.com/billziss-gh/objfs/objreg" 31 | ) 32 | 33 | // CredentialMap maps names to credentials. 34 | type CredentialMap map[string]interface{} 35 | 36 | // Get gets a credential by name. It will convert the credential 37 | // to a string using fmt.Sprint if required. 38 | func (self CredentialMap) Get(name string) string { 39 | v := self[name] 40 | if nil == v { 41 | return "" 42 | } 43 | if s, ok := v.(string); ok { 44 | return s 45 | } 46 | return fmt.Sprint(v) 47 | } 48 | 49 | // String implements flag.Value.String. 50 | func (self *CredentialMap) String() string { 51 | return "" 52 | } 53 | 54 | // Set implements flag.Value.Set. 55 | func (self *CredentialMap) Set(s string) (err error) { 56 | *self, err = ReadCredentials(s) 57 | return 58 | } 59 | 60 | // ReadCredentials reads credentials from a file or the system keyring. 61 | // The path can be a file path, a "file:/path" URI or a 62 | // "keyring:service/user" URI. 63 | func ReadCredentials(path string) (cmap CredentialMap, err error) { 64 | uri, err := url.Parse(path) 65 | if nil != err || ("file" != uri.Scheme && "keyring" != uri.Scheme) { 66 | uri = &url.URL{Scheme: "file", Path: path} 67 | err = nil 68 | } 69 | 70 | var conf config.TypedConfig 71 | if "file" == uri.Scheme { 72 | var iconf interface{} 73 | iconf, err = util.ReadFunc(uri.Path, func(file *os.File) (interface{}, error) { 74 | return config.ReadTyped(file) 75 | }) 76 | if nil == err { 77 | conf = iconf.(config.TypedConfig) 78 | } 79 | } else if "keyring" == uri.Scheme { 80 | service, user := "", "" 81 | parts := strings.SplitN(uri.Opaque, "/", 2) 82 | if 1 <= len(parts) { 83 | service = parts[0] 84 | } 85 | if 2 <= len(parts) { 86 | user = parts[1] 87 | } 88 | var pass string 89 | pass, err = keyring.Get(service, user) 90 | if nil != err { 91 | return nil, err 92 | } 93 | conf, err = config.ReadTyped(strings.NewReader(pass)) 94 | } 95 | 96 | if nil != err { 97 | return nil, err 98 | } 99 | 100 | cmap = CredentialMap(conf[""]) 101 | if nil == cmap { 102 | cmap = CredentialMap{} 103 | } 104 | return cmap, nil 105 | } 106 | 107 | // WriteCredentials writes credentials to a file or the system keyring. 108 | // The path can be a file path, a "file:/path" URI or a 109 | // "keyring:service/user" URI. 110 | func WriteCredentials(path string, cmap CredentialMap) (err error) { 111 | uri, err := url.Parse(path) 112 | if nil != err || ("file" != uri.Scheme && "keyring" != uri.Scheme) { 113 | uri = &url.URL{Scheme: "file", Path: path} 114 | err = nil 115 | } 116 | 117 | conf := config.TypedConfig{} 118 | conf[""] = config.TypedSection(cmap) 119 | if "file" == uri.Scheme { 120 | err = util.WriteFunc(path, 0600, func(file *os.File) error { 121 | return config.WriteTyped(file, conf) 122 | }) 123 | } else if "keyring" == uri.Scheme { 124 | var buf bytes.Buffer 125 | err = config.WriteTyped(&buf, conf) 126 | if nil != err { 127 | return 128 | } 129 | service, user := "", "" 130 | parts := strings.SplitN(uri.Opaque, "/", 2) 131 | if 1 <= len(parts) { 132 | service = parts[0] 133 | } 134 | if 2 <= len(parts) { 135 | user = parts[1] 136 | } 137 | err = keyring.Set(service, user, buf.String()) 138 | } 139 | 140 | return 141 | } 142 | 143 | // DeleteCredentials deletes credentials to a file or the system keyring. 144 | // The path can be a file path, a "file:/path" URI or a 145 | // "keyring:service/user" URI. 146 | func DeleteCredentials(path string) (err error) { 147 | uri, err := url.Parse(path) 148 | if nil != err || ("file" != uri.Scheme && "keyring" != uri.Scheme) { 149 | uri = &url.URL{Scheme: "file", Path: path} 150 | err = nil 151 | } 152 | 153 | if "file" == uri.Scheme { 154 | err = os.Remove(path) 155 | } else if "keyring" == uri.Scheme { 156 | service, user := "", "" 157 | parts := strings.SplitN(uri.Opaque, "/", 2) 158 | if 1 <= len(parts) { 159 | service = parts[0] 160 | } 161 | if 2 <= len(parts) { 162 | user = parts[1] 163 | } 164 | err = keyring.Delete(service, user) 165 | } 166 | 167 | return 168 | } 169 | 170 | // Session represents an authentication/authorization session. 171 | type Session interface { 172 | Credentials() CredentialMap 173 | } 174 | 175 | // SessionRefresher refreshes a session. 176 | type SessionRefresher interface { 177 | Refresh(force bool) error 178 | } 179 | 180 | // SessionDestroyer destroys a session. 181 | type SessionDestroyer interface { 182 | Destroy() 183 | } 184 | 185 | // Auth is the primary interface implemented by an authenticator/authorizer. 186 | type Auth interface { 187 | Session(credentials CredentialMap) (Session, error) 188 | } 189 | 190 | // Registry is the default authenticator/authorizer factory registry. 191 | var Registry = objreg.NewObjectFactoryRegistry() 192 | -------------------------------------------------------------------------------- /auth/auth_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * auth_test.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package auth 19 | 20 | import ( 21 | "os" 22 | "path/filepath" 23 | "reflect" 24 | "testing" 25 | ) 26 | 27 | func testReadWriteCredentials(t *testing.T, path string) { 28 | cmap0 := CredentialMap{} 29 | cmap0["username"] = "emanresu" 30 | cmap0["password"] = "drowssap" 31 | 32 | err := WriteCredentials(path, cmap0) 33 | if nil != err { 34 | t.Error(err) 35 | } 36 | 37 | cmap, err := ReadCredentials(path) 38 | if nil != err { 39 | t.Error(err) 40 | } 41 | 42 | if !reflect.DeepEqual(cmap0, cmap) { 43 | t.Error() 44 | } 45 | 46 | err = DeleteCredentials(path) 47 | if nil != err { 48 | t.Error(err) 49 | } 50 | } 51 | 52 | func TestReadWriteCredentials(t *testing.T) { 53 | testReadWriteCredentials(t, "keyring:objfs/auth_test") 54 | 55 | path := filepath.Join(os.TempDir(), "auth_test") 56 | os.Remove(path) 57 | testReadWriteCredentials(t, path) 58 | } 59 | -------------------------------------------------------------------------------- /cache/cache_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * cache_test.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package cache 19 | 20 | import ( 21 | "testing" 22 | ) 23 | 24 | func TestPartialPaths(t *testing.T) { 25 | var paths []string 26 | 27 | paths = partialPaths("/") 28 | if 1 != len(paths) || "/" != paths[0] { 29 | t.Error() 30 | } 31 | 32 | paths = partialPaths("//foo") 33 | if 2 != len(paths) || "/" != paths[0] || "/foo" != paths[1] { 34 | t.Error() 35 | } 36 | 37 | paths = partialPaths("/foo/") 38 | if 2 != len(paths) || "/" != paths[0] || "/foo" != paths[1] { 39 | t.Error() 40 | } 41 | 42 | paths = partialPaths("/foo/bar") 43 | if 3 != len(paths) || "/" != paths[0] || "/foo" != paths[1] || "/foo/bar" != paths[2] { 44 | t.Error() 45 | } 46 | 47 | paths = partialPaths("/foo/bar/Δοκιμή") 48 | if 4 != len(paths) || "/" != paths[0] || "/foo" != paths[1] || "/foo/bar" != paths[2] || 49 | "/foo/bar/Δοκιμή" != paths[3] { 50 | t.Error() 51 | } 52 | 53 | paths = partialPaths("/foo/bar/Δοκιμή/baz") 54 | if 5 != len(paths) || "/" != paths[0] || "/foo" != paths[1] || "/foo/bar" != paths[2] || 55 | "/foo/bar/Δοκιμή" != paths[3] || "/foo/bar/Δοκιμή/baz" != paths[4] { 56 | t.Error() 57 | } 58 | 59 | paths = partialPaths("/foo/bar///Δοκιμή/////baz") 60 | if 5 != len(paths) || "/" != paths[0] || "/foo" != paths[1] || "/foo/bar" != paths[2] || 61 | "/foo/bar/Δοκιμή" != paths[3] || "/foo/bar/Δοκιμή/baz" != paths[4] { 62 | t.Error() 63 | } 64 | } 65 | 66 | func TestNormalizeCase(t *testing.T) { 67 | s := "" 68 | 69 | s = normalizeCase("TEST test") 70 | if "TEST TEST" != s { 71 | t.Error(s) 72 | } 73 | 74 | s = normalizeCase("ΔΟΚΙΜΉ Δοκιμή") 75 | if "ΔΟΚͅµΉ ΔΟΚͅµΉ" != s { 76 | t.Error(s) 77 | } 78 | 79 | s = normalizeCase("ΣΊΣΥΦΟΣ Σίσυφος") 80 | if "ΣΊΣΥΦΟΣ ΣΊΣΥΦΟΣ" != s { 81 | t.Error(s) 82 | } 83 | 84 | s = normalizeCase("TSCHÜSS, TSCHÜẞ, tschüß, tschüss") 85 | if "TSCHÜSS, TSCHÜß, TSCHÜß, TSCHÜSS" != s { 86 | t.Error(s) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /cache/link.go: -------------------------------------------------------------------------------- 1 | /* 2 | * link.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package cache 19 | 20 | import "unsafe" 21 | 22 | type link_t struct { 23 | prev, next *link_t 24 | } 25 | 26 | func (link *link_t) Init() { 27 | link.prev = link 28 | link.next = link 29 | } 30 | 31 | func (link *link_t) InsertTail(list *link_t) { 32 | prev := list.prev 33 | link.next = list 34 | link.prev = prev 35 | prev.next = link 36 | list.prev = link 37 | } 38 | 39 | func (link *link_t) Remove() { 40 | next := link.next 41 | prev := link.prev 42 | prev.next = next 43 | next.prev = prev 44 | } 45 | 46 | func containerOf(p unsafe.Pointer, o uintptr) unsafe.Pointer { 47 | return unsafe.Pointer(uintptr(p) - o) 48 | } 49 | -------------------------------------------------------------------------------- /cache/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | * node.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package cache 19 | 20 | import ( 21 | "bytes" 22 | "os" 23 | "path" 24 | "time" 25 | 26 | "github.com/billziss-gh/objfs/errno" 27 | "github.com/billziss-gh/objfs/objio" 28 | "github.com/boltdb/bolt" 29 | ) 30 | 31 | type nodetx_t struct { 32 | Tx *bolt.Tx 33 | idx *bolt.Bucket 34 | cat *bolt.Bucket 35 | } 36 | 37 | func (tx *nodetx_t) Idx() *bolt.Bucket { 38 | if nil == tx.idx { 39 | tx.idx = tx.Tx.Bucket(idxname) 40 | } 41 | 42 | return tx.idx 43 | } 44 | 45 | func (tx *nodetx_t) Cat() *bolt.Bucket { 46 | if nil == tx.cat { 47 | tx.cat = tx.Tx.Bucket(catname) 48 | } 49 | 50 | return tx.cat 51 | } 52 | 53 | type node_t struct { 54 | // persistent 55 | Ino uint64 // read-only after init 56 | Path string // guarded by lockPath/unlockPath 57 | Size int64 // -ditto- 58 | Btime time.Time // -ditto- 59 | Mtime time.Time // -ditto- 60 | IsDir bool // -ditto- 61 | Sig string // -ditto- 62 | Hash []byte // -ditto- 63 | 64 | // transient 65 | Valid bool 66 | Deleted bool 67 | File *os.File 68 | refcnt int 69 | } 70 | 71 | func (node *node_t) Get(tx *nodetx_t, k []byte) (err error) { 72 | v := tx.Cat().Get(k) 73 | if nil == v || nil != node.Decode(v) { 74 | err = errno.ENOENT 75 | } 76 | 77 | return 78 | } 79 | 80 | func (node *node_t) GetWithIno(tx *nodetx_t, ino uint64) (err error) { 81 | var vbuf [8]byte 82 | v := vbuf[:] 83 | 84 | putUint64(v, 0, ino) 85 | k := tx.Idx().Get(v) 86 | if nil == k { 87 | err = errno.ENOENT 88 | return 89 | } 90 | 91 | err = node.Get(tx, k) 92 | 93 | return 94 | } 95 | 96 | func (node *node_t) NextIno(tx *nodetx_t) (ino uint64, err error) { 97 | if 0 == node.Ino { 98 | node.Ino, err = tx.Idx().NextSequence() 99 | if nil != err { 100 | return 101 | } 102 | } 103 | 104 | ino = node.Ino 105 | 106 | return 107 | } 108 | 109 | func (node *node_t) Put(tx *nodetx_t, k []byte) (err error) { 110 | if nil != node { 111 | v := make([]byte, node.EncodeLen()) 112 | putUint64(v, 0, node.Ino) 113 | 114 | if k0 := tx.Idx().Get(v[:8]); !bytes.Equal(k0, k) { 115 | err = tx.Idx().Put(v[:8], k) 116 | } 117 | if nil == err { 118 | v = node.Encode(v) 119 | err = tx.Cat().Put(k, v) 120 | } 121 | } else { 122 | n := node_t{} 123 | err = n.Get(tx, k) 124 | if nil == err { 125 | var vbuf [8]byte 126 | v := vbuf[:] 127 | 128 | putUint64(v, 0, n.Ino) 129 | err = tx.Idx().Delete(v) 130 | if nil == err { 131 | err = tx.Cat().Delete(k) 132 | } 133 | } 134 | if errno.ENOENT == err { 135 | err = nil 136 | } 137 | } 138 | 139 | return 140 | } 141 | 142 | func (node *node_t) CopyStat(info objio.ObjectInfo) { 143 | node.Size = info.Size() 144 | node.Btime = info.Btime() 145 | node.Mtime = info.Mtime() 146 | node.IsDir = info.IsDir() 147 | node.Sig = info.Sig() 148 | node.Valid = true 149 | } 150 | 151 | func (node *node_t) Stat() (info objio.ObjectInfo, err error) { 152 | if 0 == node.Ino || "" == node.Path || !node.Valid || node.Deleted { 153 | panic(errno.EINVAL) 154 | } 155 | 156 | nodeinfo := nodeinfo_t{ 157 | name: path.Base(node.Path), 158 | size: node.Size, 159 | btime: node.Btime, 160 | mtime: node.Mtime, 161 | isdir: node.IsDir, 162 | sig: node.Sig, 163 | } 164 | 165 | if nil != node.File { 166 | var fileinfo os.FileInfo 167 | fileinfo, err = node.File.Stat() 168 | if nil != err { 169 | return 170 | } 171 | 172 | nodeinfo.size = fileinfo.Size() 173 | nodeinfo.mtime = fileinfo.ModTime() 174 | } 175 | 176 | info = &nodeinfo 177 | 178 | return 179 | } 180 | 181 | func (node *node_t) Reference() { 182 | node.refcnt++ 183 | } 184 | 185 | func (node *node_t) Dereference() int { 186 | node.refcnt-- 187 | return node.refcnt 188 | } 189 | 190 | func (node *node_t) EncodeLen() int { 191 | lp, ls, lh := len(node.Path), len(node.Sig), len(node.Hash) 192 | return 8 + 8 + 8 + 8 + 2 + 2 + 1 + 1 + lp + ls + lh 193 | } 194 | 195 | func (node *node_t) Encode(b []byte) []byte { 196 | // encode order: uint64*, uint32*, uint16*, uint8* 197 | // Ino, Size, Btime, Mtime, len(Path), len(Sig), IsDir, len(Hash), Path, Sig, Hash 198 | 199 | if 0 == node.Ino || "" == node.Path || !node.Valid || node.Deleted { 200 | panic(errno.EINVAL) 201 | } 202 | 203 | isdir := uint8(0) 204 | if node.IsDir { 205 | isdir = uint8(1) 206 | } 207 | lp, ls, lh := len(node.Path), len(node.Sig), len(node.Hash) 208 | 209 | i := 0 210 | i = putUint64(b, i, node.Ino) 211 | i = putUint64(b, i, uint64(node.Size)) 212 | i = putTime(b, i, node.Btime) 213 | i = putTime(b, i, node.Mtime) 214 | i = putUint16(b, i, uint16(lp)) 215 | i = putUint16(b, i, uint16(ls)) 216 | i = putUint8(b, i, isdir) 217 | i = putUint8(b, i, uint8(lh)) 218 | i = putString(b, i, node.Path, 1<<16-1) 219 | i = putString(b, i, node.Sig, 1<<16-1) 220 | i = putBytes(b, i, node.Hash, 1<<8-1) 221 | return b[:i] 222 | } 223 | 224 | func (node *node_t) Decode(b []byte) (err error) { 225 | // encode order: uint64*, uint32*, uint16*, uint8* 226 | // Ino, Size, Btime, Mtime, len(Path), len(Sig), IsDir, len(Hash), Path, Sig, Hash 227 | 228 | defer func() { 229 | if r := recover(); nil != r { 230 | err = errno.EIO 231 | } 232 | }() 233 | 234 | i := 0 235 | i, ino := getUint64(b, i) 236 | i, size := getUint64(b, i) 237 | i, btime := getTime(b, i) 238 | i, mtime := getTime(b, i) 239 | i, lp := getUint16(b, i) 240 | i, ls := getUint16(b, i) 241 | i, isdir := getUint8(b, i) 242 | i, lh := getUint8(b, i) 243 | i, path := getString(b, i, int(lp)) 244 | i, sig := getString(b, i, int(ls)) 245 | i, hash := getBytes(b, i, int(lh)) 246 | 247 | node.Ino = ino 248 | node.Size = int64(size) 249 | node.Btime = btime 250 | node.Mtime = mtime 251 | node.IsDir = 0 != isdir 252 | node.Path = path 253 | node.Sig = sig 254 | node.Hash = hash 255 | node.Valid = true 256 | 257 | return nil 258 | } 259 | 260 | type nodeinfo_t struct { 261 | name string 262 | size int64 263 | btime time.Time 264 | mtime time.Time 265 | isdir bool 266 | sig string 267 | } 268 | 269 | func (info *nodeinfo_t) Name() string { 270 | return info.name 271 | } 272 | 273 | func (info *nodeinfo_t) Size() int64 { 274 | return info.size 275 | } 276 | 277 | func (info *nodeinfo_t) Btime() time.Time { 278 | return info.btime 279 | } 280 | 281 | func (info *nodeinfo_t) Mtime() time.Time { 282 | return info.mtime 283 | } 284 | 285 | func (info *nodeinfo_t) IsDir() bool { 286 | return info.isdir 287 | } 288 | 289 | func (info *nodeinfo_t) Sig() string { 290 | return info.sig 291 | } 292 | 293 | func putUint8(b []byte, i int, v uint8) int { 294 | b[i] = byte(v) 295 | return i + 1 296 | } 297 | 298 | func putUint16(b []byte, i int, v uint16) int { 299 | b[i+0] = byte(v >> 8) 300 | b[i+1] = byte(v) 301 | return i + 2 302 | } 303 | 304 | func putUint32(b []byte, i int, v uint32) int { 305 | b[i+0] = byte(v >> 24) 306 | b[i+1] = byte(v >> 16) 307 | b[i+2] = byte(v >> 8) 308 | b[i+3] = byte(v) 309 | return i + 4 310 | } 311 | 312 | func putUint64(b []byte, i int, v uint64) int { 313 | b[i+0] = byte(v >> 56) 314 | b[i+1] = byte(v >> 48) 315 | b[i+2] = byte(v >> 40) 316 | b[i+3] = byte(v >> 32) 317 | b[i+4] = byte(v >> 24) 318 | b[i+5] = byte(v >> 16) 319 | b[i+6] = byte(v >> 8) 320 | b[i+7] = byte(v) 321 | return i + 8 322 | } 323 | 324 | func putTime(b []byte, i int, v time.Time) int { 325 | return putUint64(b, i, uint64(v.UnixNano())) 326 | } 327 | 328 | func putString(b []byte, i int, v string, maxlen int) int { 329 | if maxlen > len(v) { 330 | maxlen = len(v) 331 | } 332 | return i + copy(b[i:], ([]byte)(v[:maxlen])) 333 | } 334 | 335 | func putBytes(b []byte, i int, v []byte, maxlen int) int { 336 | if maxlen > len(v) { 337 | maxlen = len(v) 338 | } 339 | return i + copy(b[i:], v[:maxlen]) 340 | } 341 | 342 | func getUint8(b []byte, i int) (int, uint8) { 343 | return i + 1, uint8(b[i]) 344 | } 345 | 346 | func getUint16(b []byte, i int) (int, uint16) { 347 | return i + 2, uint16(b[i])<<8 | uint16(b[i+1]) 348 | } 349 | 350 | func getUint32(b []byte, i int) (int, uint32) { 351 | return i + 4, uint32(b[i])<<24 | uint32(b[i+1])<<16 | uint32(b[i+2])<<8 | uint32(b[i+3]) 352 | } 353 | 354 | func getUint64(b []byte, i int) (int, uint64) { 355 | return i + 8, 356 | uint64(b[i])<<56 | uint64(b[i+1])<<48 | uint64(b[i+2])<<40 | uint64(b[i+3])<<32 | 357 | uint64(b[i+4])<<24 | uint64(b[i+5])<<16 | uint64(b[i+6])<<8 | uint64(b[i+7]) 358 | } 359 | 360 | func getTime(b []byte, i int) (int, time.Time) { 361 | i, v := getUint64(b, i) 362 | return i, time.Unix(0, int64(v)).UTC() 363 | } 364 | 365 | func getString(b []byte, i int, l int) (int, string) { 366 | return i + l, string(b[i : i+l]) 367 | } 368 | 369 | func getBytes(b []byte, i int, l int) (int, []byte) { 370 | return i + l, b[i : i+l] 371 | } 372 | 373 | var ( 374 | idxname = []byte("i") 375 | catname = []byte("c") 376 | ) 377 | -------------------------------------------------------------------------------- /cache/node_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * node_test.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package cache 19 | 20 | import ( 21 | "bytes" 22 | "os" 23 | "path" 24 | "path/filepath" 25 | "testing" 26 | "time" 27 | 28 | "github.com/billziss-gh/objfs/errno" 29 | "github.com/boltdb/bolt" 30 | ) 31 | 32 | func TestStat(t *testing.T) { 33 | now := time.Now().UTC() 34 | i := nodeinfo_t{ 35 | size: 0x5152535455565758, 36 | btime: now, 37 | mtime: now, 38 | isdir: true, 39 | sig: "fortytwo", 40 | } 41 | 42 | n := node_t{ 43 | Ino: 0x4142434445464748, 44 | Path: "/foo/Δοκιμή/bar", 45 | } 46 | 47 | n.CopyStat(&i) 48 | 49 | info, err := n.Stat() 50 | if nil != err { 51 | t.Error(err) 52 | } 53 | 54 | if path.Base(n.Path) != info.Name() { 55 | t.Error() 56 | } 57 | 58 | if n.Size != info.Size() { 59 | t.Error() 60 | } 61 | 62 | if !n.Btime.Equal(info.Btime()) || n.Btime.String() != info.Btime().String() { 63 | t.Error() 64 | } 65 | 66 | if !n.Mtime.Equal(info.Mtime()) || n.Mtime.String() != info.Mtime().String() { 67 | t.Error() 68 | } 69 | 70 | if n.IsDir != info.IsDir() { 71 | t.Error() 72 | } 73 | 74 | if n.Sig != info.Sig() { 75 | t.Error() 76 | } 77 | } 78 | 79 | func TestEncodeDecode(t *testing.T) { 80 | now := time.Now().UTC() 81 | n := node_t{ 82 | Ino: 0x4142434445464748, 83 | Path: "/foo/Δοκιμή/bar", 84 | Size: 0x5152535455565758, 85 | Btime: now, 86 | Mtime: now, 87 | IsDir: true, 88 | Sig: "fortytwo", 89 | Hash: []byte{41, 42, 43, 44}, 90 | Valid: true, 91 | } 92 | 93 | b := make([]byte, n.EncodeLen()) 94 | b = n.Encode(b) 95 | if len(b) != n.EncodeLen() { 96 | t.Error() 97 | } 98 | 99 | n2 := node_t{} 100 | err := n2.Decode(b) 101 | if nil != err { 102 | t.Error(err) 103 | } 104 | 105 | if n.Ino != n2.Ino { 106 | t.Error() 107 | } 108 | 109 | if n.Path != n2.Path { 110 | t.Error() 111 | } 112 | 113 | if n.Size != n2.Size { 114 | t.Error() 115 | } 116 | 117 | if !n.Btime.Equal(n2.Btime) || n.Btime.String() != n2.Btime.String() { 118 | t.Error() 119 | } 120 | 121 | if !n.Mtime.Equal(n2.Mtime) || n.Mtime.String() != n2.Mtime.String() { 122 | t.Error() 123 | } 124 | 125 | if n.IsDir != n2.IsDir { 126 | t.Error() 127 | } 128 | 129 | if n.Sig != n2.Sig { 130 | t.Error() 131 | } 132 | 133 | if !bytes.Equal(n.Hash, n2.Hash) { 134 | t.Error() 135 | } 136 | 137 | if true != n2.Valid { 138 | t.Error() 139 | } 140 | } 141 | 142 | func TestPutGetDelete(t *testing.T) { 143 | path := filepath.Join(os.TempDir(), "cache_node_test") 144 | os.Remove(path) 145 | defer os.Remove(path) 146 | 147 | now := time.Now().UTC() 148 | n := node_t{ 149 | Ino: 0x4142434445464748, 150 | Path: "/foo/Δοκιμή/bar", 151 | Size: 0x5152535455565758, 152 | Btime: now, 153 | Mtime: now, 154 | IsDir: true, 155 | Sig: "fortytwo", 156 | Hash: []byte{41, 42, 43, 44}, 157 | Valid: true, 158 | } 159 | 160 | db, err := bolt.Open(path, 0600, nil) 161 | if nil != err { 162 | t.Fatal(err) 163 | } 164 | defer db.Close() 165 | 166 | err = db.Update(func(tx *bolt.Tx) (err error) { 167 | _, err = tx.CreateBucketIfNotExists(idxname) 168 | if nil == err { 169 | _, err = tx.CreateBucketIfNotExists(catname) 170 | } 171 | return 172 | }) 173 | if nil != err { 174 | t.Fatal(err) 175 | } 176 | 177 | key := []byte("KEY") 178 | 179 | err = db.Update(func(tx *bolt.Tx) (err error) { 180 | ntx := nodetx_t{Tx: tx} 181 | err = n.Put(&ntx, key) 182 | return 183 | }) 184 | if nil != err { 185 | t.Error(err) 186 | } 187 | 188 | n2 := node_t{} 189 | err = db.View(func(tx *bolt.Tx) (err error) { 190 | ntx := nodetx_t{Tx: tx} 191 | err = n2.Get(&ntx, key) 192 | return 193 | }) 194 | if nil != err { 195 | t.Error(err) 196 | } 197 | 198 | if n.Ino != n2.Ino { 199 | t.Error() 200 | } 201 | 202 | if n.Path != n2.Path { 203 | t.Error() 204 | } 205 | 206 | if n.Size != n2.Size { 207 | t.Error() 208 | } 209 | 210 | if !n.Btime.Equal(n2.Btime) || n.Btime.String() != n2.Btime.String() { 211 | t.Error() 212 | } 213 | 214 | if !n.Mtime.Equal(n2.Mtime) || n.Mtime.String() != n2.Mtime.String() { 215 | t.Error() 216 | } 217 | 218 | if n.IsDir != n2.IsDir { 219 | t.Error() 220 | } 221 | 222 | if n.Sig != n2.Sig { 223 | t.Error() 224 | } 225 | 226 | if !bytes.Equal(n.Hash, n2.Hash) { 227 | t.Error() 228 | } 229 | 230 | if true != n2.Valid { 231 | t.Error() 232 | } 233 | 234 | n2 = node_t{} 235 | err = db.View(func(tx *bolt.Tx) (err error) { 236 | ntx := nodetx_t{Tx: tx} 237 | err = n2.GetWithIno(&ntx, 0x4142434445464748) 238 | return 239 | }) 240 | if nil != err { 241 | t.Error(err) 242 | } 243 | 244 | if n.Ino != n2.Ino { 245 | t.Error() 246 | } 247 | 248 | if n.Path != n2.Path { 249 | t.Error() 250 | } 251 | 252 | if n.Size != n2.Size { 253 | t.Error() 254 | } 255 | 256 | if !n.Btime.Equal(n2.Btime) || n.Btime.String() != n2.Btime.String() { 257 | t.Error() 258 | } 259 | 260 | if !n.Mtime.Equal(n2.Mtime) || n.Mtime.String() != n2.Mtime.String() { 261 | t.Error() 262 | } 263 | 264 | if n.IsDir != n2.IsDir { 265 | t.Error() 266 | } 267 | 268 | if n.Sig != n2.Sig { 269 | t.Error() 270 | } 271 | 272 | if !bytes.Equal(n.Hash, n2.Hash) { 273 | t.Error() 274 | } 275 | 276 | if true != n2.Valid { 277 | t.Error() 278 | } 279 | 280 | err = db.Update(func(tx *bolt.Tx) (err error) { 281 | ntx := nodetx_t{Tx: tx} 282 | err = (*node_t)(nil).Put(&ntx, key) 283 | return 284 | }) 285 | if nil != err { 286 | t.Error(err) 287 | } 288 | 289 | n2 = node_t{} 290 | err = db.View(func(tx *bolt.Tx) (err error) { 291 | ntx := nodetx_t{Tx: tx} 292 | err = n2.Get(&ntx, key) 293 | return 294 | }) 295 | if errno.ENOENT != err { 296 | t.Error(err) 297 | } 298 | 299 | n2 = node_t{} 300 | err = db.View(func(tx *bolt.Tx) (err error) { 301 | ntx := nodetx_t{Tx: tx} 302 | err = n2.GetWithIno(&ntx, 0x4142434445464748) 303 | return 304 | }) 305 | if errno.ENOENT != err { 306 | t.Error(err) 307 | } 308 | } 309 | -------------------------------------------------------------------------------- /cache/openfile_unix.go: -------------------------------------------------------------------------------- 1 | // +build darwin linux 2 | 3 | /* 4 | * openfile_unix.go 5 | * 6 | * Copyright 2018 Bill Zissimopoulos 7 | */ 8 | /* 9 | * This file is part of Objfs. 10 | * 11 | * You can redistribute it and/or modify it under the terms of the GNU 12 | * Affero General Public License version 3 as published by the Free 13 | * Software Foundation. 14 | * 15 | * Licensees holding a valid commercial license may use this file in 16 | * accordance with the commercial license agreement provided with the 17 | * software. 18 | */ 19 | 20 | package cache 21 | 22 | import ( 23 | "os" 24 | ) 25 | 26 | func openFile(path string, flag int, perm os.FileMode) (*os.File, error) { 27 | return os.OpenFile(path, flag, perm) 28 | } 29 | -------------------------------------------------------------------------------- /cache/openfile_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | /* 4 | * openfile_windows.go 5 | * 6 | * Copyright 2018 Bill Zissimopoulos 7 | */ 8 | /* 9 | * This file is part of Objfs. 10 | * 11 | * You can redistribute it and/or modify it under the terms of the GNU 12 | * Affero General Public License version 3 as published by the Free 13 | * Software Foundation. 14 | * 15 | * Licensees holding a valid commercial license may use this file in 16 | * accordance with the commercial license agreement provided with the 17 | * software. 18 | */ 19 | 20 | package cache 21 | 22 | import ( 23 | "unsafe" 24 | 25 | "os" 26 | "syscall" 27 | ) 28 | 29 | // openFile opens files with FILE_SHARE_READ, FILE_SHARE_WRITE *and* FILE_SHARE_DELETE. 30 | func openFile(path string, flag int, perm os.FileMode) (*os.File, error) { 31 | fd, err := windowsOpenFile(path, flag|syscall.O_CLOEXEC, uint32(perm)) 32 | if err != nil { 33 | return nil, &os.PathError{"open", path, err} 34 | } 35 | return os.NewFile(uintptr(fd), path), nil 36 | } 37 | 38 | // lifted from ./go/src/syscall/syscall_windows.go. Modified to add FILE_SHARE_DELETE 39 | func windowsOpenFile(path string, mode int, perm uint32) (fd syscall.Handle, err error) { 40 | if len(path) == 0 { 41 | return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND 42 | } 43 | pathp, err := syscall.UTF16PtrFromString(path) 44 | if err != nil { 45 | return syscall.InvalidHandle, err 46 | } 47 | var access uint32 48 | switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { 49 | case syscall.O_RDONLY: 50 | access = syscall.GENERIC_READ 51 | case syscall.O_WRONLY: 52 | access = syscall.GENERIC_WRITE 53 | case syscall.O_RDWR: 54 | access = syscall.GENERIC_READ | syscall.GENERIC_WRITE 55 | } 56 | if mode&syscall.O_CREAT != 0 { 57 | access |= syscall.GENERIC_WRITE 58 | } 59 | if mode&syscall.O_APPEND != 0 { 60 | access &^= syscall.GENERIC_WRITE 61 | access |= syscall.FILE_APPEND_DATA 62 | } 63 | sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | 64 | syscall.FILE_SHARE_DELETE) 65 | var sa *syscall.SecurityAttributes 66 | if mode&syscall.O_CLOEXEC == 0 { 67 | sa = makeInheritSa() 68 | } 69 | var createmode uint32 70 | switch { 71 | case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): 72 | createmode = syscall.CREATE_NEW 73 | case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): 74 | createmode = syscall.CREATE_ALWAYS 75 | case mode&syscall.O_CREAT == syscall.O_CREAT: 76 | createmode = syscall.OPEN_ALWAYS 77 | case mode&syscall.O_TRUNC == syscall.O_TRUNC: 78 | createmode = syscall.TRUNCATE_EXISTING 79 | default: 80 | createmode = syscall.OPEN_EXISTING 81 | } 82 | h, e := syscall.CreateFile( 83 | pathp, access, sharemode, sa, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0) 84 | return h, e 85 | } 86 | 87 | func makeInheritSa() *syscall.SecurityAttributes { 88 | var sa syscall.SecurityAttributes 89 | sa.Length = uint32(unsafe.Sizeof(sa)) 90 | sa.InheritHandle = 1 91 | return &sa 92 | } 93 | -------------------------------------------------------------------------------- /cache_commands.go: -------------------------------------------------------------------------------- 1 | // +build debug 2 | 3 | /* 4 | * cache_commands.go 5 | * Additional objfs commands used for testing. 6 | * 7 | * Copyright 2018 Bill Zissimopoulos 8 | */ 9 | /* 10 | * This file is part of Objfs. 11 | * 12 | * You can redistribute it and/or modify it under the terms of the GNU 13 | * Affero General Public License version 3 as published by the Free 14 | * Software Foundation. 15 | * 16 | * Licensees holding a valid commercial license may use this file in 17 | * accordance with the commercial license agreement provided with the 18 | * software. 19 | */ 20 | 21 | package main 22 | 23 | import ( 24 | "io" 25 | "os" 26 | "path" 27 | 28 | "github.com/billziss-gh/golib/cmd" 29 | "github.com/billziss-gh/golib/errors" 30 | "github.com/billziss-gh/objfs/cache" 31 | "github.com/billziss-gh/objfs/objio" 32 | ) 33 | 34 | func init() { 35 | initCacheCommands(cmd.DefaultCmdMap) 36 | shellCommands = append(shellCommands, initCacheCommands) 37 | } 38 | 39 | func initCacheCommands(cmdmap *cmd.CmdMap) { 40 | var c *cmd.Cmd 41 | addcmd(cmdmap, "cache-statfs\nget storage information", 42 | CacheStatfs) 43 | c = addcmd(cmdmap, "cache-ls [-l][-n count] path...\nlist files", 44 | CacheLs) 45 | c.Flag.Bool("l", false, "long format") 46 | c = addcmd(cmdmap, "cache-stat [-l] path...\ndisplay file information", 47 | CacheStat) 48 | c.Flag.Bool("l", false, "long format") 49 | addcmd(cmdmap, "cache-mkdir path...\nmake directories", 50 | CacheMkdir) 51 | addcmd(cmdmap, "cache-rmdir path...\nremove directories", 52 | CacheRmdir) 53 | addcmd(cmdmap, "cache-rm path...\nremove files", 54 | CacheRm) 55 | addcmd(cmdmap, "cache-mv oldpath newpath\nmove (rename) files", 56 | CacheMv) 57 | addcmd(cmdmap, "cache-get path [local-path]\nget (download) files", 58 | CacheGet) 59 | addcmd(cmdmap, "cache-put [local-path] path\nput (upload) files", 60 | CachePut) 61 | } 62 | 63 | func CacheStatfs(cmd *cmd.Cmd, args []string) { 64 | needvar(&storage, &cachePath) 65 | 66 | cmd.Flag.Parse(args) 67 | 68 | if 0 != cmd.Flag.NArg() { 69 | usage(cmd) 70 | } 71 | 72 | c, err := openCache(cache.Open) 73 | if nil != err { 74 | fail(errors.New("cache-statfs", err)) 75 | } 76 | defer c.CloseCache() 77 | 78 | info, err := c.Statfs() 79 | if nil != err { 80 | fail(errors.New("cache-statfs", err)) 81 | } 82 | 83 | printStorageInfo(info) 84 | } 85 | 86 | func CacheLs(cmd *cmd.Cmd, args []string) { 87 | needvar(&storage, &cachePath) 88 | 89 | cmd.Flag.Parse(args) 90 | long := cmd.GetFlag("l").(bool) 91 | 92 | c, err := openCache(cache.Open) 93 | if nil != err { 94 | fail(errors.New("cache-ls", err)) 95 | } 96 | defer c.CloseCache() 97 | 98 | failed := false 99 | 100 | for _, path := range cmd.Flag.Args() { 101 | ino, err := c.Open(path) 102 | if nil == err { 103 | var infos []objio.ObjectInfo 104 | infos, err = c.Readdir(ino, 0) 105 | if nil == err { 106 | for _, info := range infos { 107 | printObjectInfo(info, long) 108 | } 109 | } 110 | 111 | c.Close(ino) 112 | } 113 | 114 | if nil != err { 115 | failed = true 116 | warn(errors.New("cache-ls "+path, err)) 117 | continue 118 | } 119 | } 120 | 121 | if failed { 122 | exit(1) 123 | } 124 | } 125 | 126 | func CacheStat(cmd *cmd.Cmd, args []string) { 127 | needvar(&storage, &cachePath) 128 | 129 | cmd.Flag.Parse(args) 130 | long := cmd.GetFlag("l").(bool) 131 | 132 | c, err := openCache(cache.Open) 133 | if nil != err { 134 | fail(errors.New("cache-stat", err)) 135 | } 136 | defer c.CloseCache() 137 | 138 | failed := false 139 | 140 | for _, path := range cmd.Flag.Args() { 141 | ino, err := c.Open(path) 142 | if nil == err { 143 | var info objio.ObjectInfo 144 | info, err = c.Stat(ino) 145 | if nil == err { 146 | printObjectInfo(info, long) 147 | } 148 | 149 | c.Close(ino) 150 | } 151 | 152 | if nil != err { 153 | failed = true 154 | warn(errors.New("cache-stat "+path, err)) 155 | continue 156 | } 157 | } 158 | 159 | if failed { 160 | exit(1) 161 | } 162 | } 163 | 164 | func CacheMkdir(cmd *cmd.Cmd, args []string) { 165 | needvar(&storage, &cachePath) 166 | 167 | cmd.Flag.Parse(args) 168 | 169 | c, err := openCache(cache.Open) 170 | if nil != err { 171 | fail(errors.New("cache-mkdir", err)) 172 | } 173 | defer c.CloseCache() 174 | 175 | failed := false 176 | 177 | for _, path := range cmd.Flag.Args() { 178 | ino, err := c.Open(path) 179 | if nil == err { 180 | err = c.Make(ino, true) 181 | c.Close(ino) 182 | } 183 | 184 | if nil != err { 185 | failed = true 186 | warn(errors.New("cache-mkdir "+path, err)) 187 | continue 188 | } 189 | } 190 | 191 | if failed { 192 | exit(1) 193 | } 194 | } 195 | 196 | func CacheRmdir(cmd *cmd.Cmd, args []string) { 197 | needvar(&storage, &cachePath) 198 | 199 | cmd.Flag.Parse(args) 200 | 201 | c, err := openCache(cache.Open) 202 | if nil != err { 203 | fail(errors.New("cache-rmdir", err)) 204 | } 205 | defer c.CloseCache() 206 | 207 | failed := false 208 | 209 | for _, path := range cmd.Flag.Args() { 210 | ino, err := c.Open(path) 211 | if nil == err { 212 | err = c.Remove(ino, true) 213 | c.Close(ino) 214 | } 215 | 216 | if nil != err { 217 | failed = true 218 | warn(errors.New("cache-rmdir "+path, err)) 219 | continue 220 | } 221 | } 222 | 223 | if failed { 224 | exit(1) 225 | } 226 | } 227 | 228 | func CacheRm(cmd *cmd.Cmd, args []string) { 229 | needvar(&storage, &cachePath) 230 | 231 | cmd.Flag.Parse(args) 232 | 233 | c, err := openCache(cache.Open) 234 | if nil != err { 235 | fail(errors.New("cache-rm", err)) 236 | } 237 | defer c.CloseCache() 238 | 239 | failed := false 240 | 241 | for _, path := range cmd.Flag.Args() { 242 | ino, err := c.Open(path) 243 | if nil == err { 244 | err = c.Remove(ino, false) 245 | c.Close(ino) 246 | } 247 | 248 | if nil != err { 249 | failed = true 250 | warn(errors.New("cache-rm "+path, err)) 251 | continue 252 | } 253 | } 254 | 255 | if failed { 256 | exit(1) 257 | } 258 | } 259 | 260 | func CacheMv(cmd *cmd.Cmd, args []string) { 261 | needvar(&storage, &cachePath) 262 | 263 | cmd.Flag.Parse(args) 264 | 265 | if 1 > cmd.Flag.NArg() || 2 < cmd.Flag.NArg() { 266 | usage(cmd) 267 | } 268 | 269 | oldpath := cmd.Flag.Arg(0) 270 | newpath := cmd.Flag.Arg(1) 271 | 272 | c, err := openCache(cache.Open) 273 | if nil != err { 274 | fail(errors.New("cache-mv", err)) 275 | } 276 | defer c.CloseCache() 277 | 278 | ino, err := c.Open(oldpath) 279 | if nil == err { 280 | err = c.Rename(ino, newpath) 281 | c.Close(ino) 282 | } 283 | if nil != err { 284 | fail(errors.New("cache-mv "+oldpath, err)) 285 | } 286 | } 287 | 288 | func CacheGet(cmd *cmd.Cmd, args []string) { 289 | needvar(&storage, &cachePath) 290 | 291 | cmd.Flag.Parse(args) 292 | 293 | if 1 > cmd.Flag.NArg() || 2 < cmd.Flag.NArg() { 294 | usage(cmd) 295 | } 296 | 297 | ipath := cmd.Flag.Arg(0) 298 | opath := cmd.Flag.Arg(1) 299 | if "" == opath { 300 | opath = path.Base(ipath) 301 | } 302 | 303 | c, err := openCache(cache.Open) 304 | if nil != err { 305 | fail(errors.New("cache-get", err)) 306 | } 307 | defer c.CloseCache() 308 | 309 | ino, err := c.Open(ipath) 310 | if nil != err { 311 | fail(errors.New("cache-get "+ipath, err)) 312 | } 313 | defer c.Close(ino) 314 | 315 | writer, err := os.OpenFile(opath, os.O_CREATE|os.O_WRONLY, 0666) 316 | if nil != err { 317 | fail(errors.New("cache-get "+ipath, err)) 318 | } 319 | defer writer.Close() 320 | 321 | reader := &cacheReader{cache: c, ino: ino} 322 | _, err = io.Copy(writer, reader) 323 | if nil != err { 324 | fail(errors.New("cache-get "+ipath, err)) 325 | } 326 | } 327 | 328 | func CachePut(cmd *cmd.Cmd, args []string) { 329 | needvar(&storage, &cachePath) 330 | 331 | cmd.Flag.Parse(args) 332 | 333 | if 1 > cmd.Flag.NArg() || 2 < cmd.Flag.NArg() { 334 | usage(cmd) 335 | } 336 | 337 | ipath := cmd.Flag.Arg(0) 338 | opath := cmd.Flag.Arg(1) 339 | if "" == opath { 340 | opath = ipath 341 | ipath = path.Base(ipath) 342 | } 343 | 344 | c, err := openCache(cache.Open) 345 | if nil != err { 346 | fail(errors.New("cache-put", err)) 347 | } 348 | defer c.CloseCache() 349 | 350 | reader, err := os.OpenFile(ipath, os.O_RDONLY, 0) 351 | if nil != err { 352 | fail(errors.New("cache-put "+opath, err)) 353 | } 354 | defer reader.Close() 355 | 356 | ino, err := c.Open(opath) 357 | if nil != err { 358 | fail(errors.New("cache-put "+opath, err)) 359 | } 360 | defer c.Close(ino) 361 | 362 | err = c.Make(ino, false) 363 | if nil != err { 364 | fail(errors.New("cache-put "+opath, err)) 365 | } 366 | 367 | writer := &cacheWriter{cache: c, ino: ino} 368 | _, err = io.Copy(writer, reader) 369 | if nil != err { 370 | fail(errors.New("cache-put "+opath, err)) 371 | } 372 | } 373 | 374 | type cacheReader struct { 375 | cache *cache.Cache 376 | ino uint64 377 | off int64 378 | } 379 | 380 | func (self *cacheReader) Read(p []byte) (n int, err error) { 381 | n, err = self.cache.ReadAt(self.ino, p, self.off) 382 | self.off += int64(n) 383 | return 384 | } 385 | 386 | type cacheWriter struct { 387 | cache *cache.Cache 388 | ino uint64 389 | off int64 390 | } 391 | 392 | func (self *cacheWriter) Write(p []byte) (n int, err error) { 393 | n, err = self.cache.WriteAt(self.ino, p, self.off) 394 | self.off += int64(n) 395 | return 396 | } 397 | -------------------------------------------------------------------------------- /commands.go: -------------------------------------------------------------------------------- 1 | /* 2 | * commands.go 3 | * Main objfs commands. 4 | * 5 | * Copyright 2018 Bill Zissimopoulos 6 | */ 7 | /* 8 | * This file is part of Objfs. 9 | * 10 | * You can redistribute it and/or modify it under the terms of the GNU 11 | * Affero General Public License version 3 as published by the Free 12 | * Software Foundation. 13 | * 14 | * Licensees holding a valid commercial license may use this file in 15 | * accordance with the commercial license agreement provided with the 16 | * software. 17 | */ 18 | 19 | package main 20 | 21 | import ( 22 | "flag" 23 | "fmt" 24 | "io" 25 | "io/ioutil" 26 | "os" 27 | "path" 28 | "runtime" 29 | "sort" 30 | "strings" 31 | "time" 32 | 33 | "github.com/billziss-gh/golib/cmd" 34 | "github.com/billziss-gh/golib/config" 35 | "github.com/billziss-gh/golib/errors" 36 | "github.com/billziss-gh/golib/keyring" 37 | "github.com/billziss-gh/golib/shlex" 38 | "github.com/billziss-gh/golib/terminal" 39 | "github.com/billziss-gh/golib/terminal/editor" 40 | "github.com/billziss-gh/golib/util" 41 | "github.com/billziss-gh/objfs/auth" 42 | "github.com/billziss-gh/objfs/cache" 43 | "github.com/billziss-gh/objfs/fs" 44 | "github.com/billziss-gh/objfs/objio" 45 | ) 46 | 47 | type mntopts []string 48 | 49 | // String implements flag.Value.String. 50 | func (opts *mntopts) String() string { 51 | return "" 52 | } 53 | 54 | // Set implements flag.Value.Set. 55 | func (opts *mntopts) Set(s string) error { 56 | *opts = append(*opts, s) 57 | return nil 58 | } 59 | 60 | // Get implements flag.Getter.Get. 61 | func (opts *mntopts) Get() interface{} { 62 | return *opts 63 | } 64 | 65 | func init() { 66 | initCommands(cmd.DefaultCmdMap) 67 | addcmd(cmd.DefaultCmdMap, "shell\ninteractive shell", Shell) 68 | } 69 | 70 | func initCommands(cmdmap *cmd.CmdMap) { 71 | var c *cmd.Cmd 72 | addcmd(cmdmap, "version\nget current version information", 73 | Version) 74 | addcmd(cmdmap, 75 | "config {get|set|delete} [section.]name [value]\nget or set configuration options", 76 | Config) 77 | addcmd(cmdmap, "keyring {get|set|delete} service/user\nget or set keys", 78 | Keyring) 79 | addcmd(cmdmap, "auth output-credentials\nperform authentication/authorization", 80 | Auth) 81 | c = addcmd(cmdmap, "mount [-o option...] mountpoint\nmount file system", 82 | Mount) 83 | c.Flag.Var(new(mntopts), "o", "FUSE mount `option`") 84 | addcmd(cmdmap, "statfs\nget storage information", 85 | Statfs) 86 | c = addcmd(cmdmap, "ls [-l][-n count] path...\nlist files", 87 | Ls) 88 | c.Flag.Bool("l", false, "long format") 89 | c.Flag.Int("n", 0, "max `count` of list entries") 90 | c = addcmd(cmdmap, "stat [-l] path...\ndisplay file information", 91 | Stat) 92 | c.Flag.Bool("l", false, "long format") 93 | addcmd(cmdmap, "mkdir path...\nmake directories", 94 | Mkdir) 95 | addcmd(cmdmap, "rmdir path...\nremove directories", 96 | Rmdir) 97 | addcmd(cmdmap, "rm path...\nremove files", 98 | Rm) 99 | addcmd(cmdmap, "mv oldpath newpath\nmove (rename) files", 100 | Mv) 101 | c = addcmd(cmdmap, "get [-r range][-s signature] path [local-path]\nget (download) files", 102 | Get) 103 | c.Flag.String("r", "", "`range` to request (startpos-endpos)") 104 | c.Flag.String("s", "", "only get file if it does not match `signature`") 105 | addcmd(cmdmap, "put [local-path] path\nput (upload) files", 106 | Put) 107 | addcmd(cmdmap, "cache-pending\nlist pending cache files", 108 | CachePending) 109 | addcmd(cmdmap, "cache-reset\nreset cache (upload and evict files)", 110 | CacheReset) 111 | } 112 | 113 | func Version(cmd *cmd.Cmd, args []string) { 114 | cmd.Flag.Parse(args) 115 | 116 | if 0 != cmd.Flag.NArg() { 117 | usage(cmd) 118 | } 119 | 120 | curryear := time.Now().Year() 121 | scanyear := 0 122 | fmt.Sscan(MyCopyright, &scanyear) 123 | copyright := MyCopyright 124 | if curryear != scanyear { 125 | copyright = strings.Replace(MyCopyright, 126 | fmt.Sprint(scanyear), fmt.Sprintf("%d-%d", scanyear, curryear), 1) 127 | } 128 | 129 | fmt.Printf("%s - %s; version %s\ncopyright %s\n", 130 | MyProductName, MyDescription, MyVersion, copyright) 131 | if "" != MyRepository { 132 | fmt.Printf("%s\n", MyRepository) 133 | } 134 | 135 | fmt.Printf("\nsupported storages:\n") 136 | names := objio.Registry.GetNames() 137 | sort.Sort(sort.StringSlice(names)) 138 | for _, n := range names { 139 | if n == defaultStorageName { 140 | fmt.Printf(" %s (default)\n", n) 141 | } else { 142 | fmt.Printf(" %s\n", n) 143 | } 144 | } 145 | } 146 | 147 | func Config(c *cmd.Cmd, args []string) { 148 | cmdmap := cmd.NewCmdMap() 149 | c.Flag.Usage = cmd.UsageFunc(c, cmdmap) 150 | 151 | addcmd(cmdmap, "config.get [section.]name", ConfigGet) 152 | addcmd(cmdmap, "config.set [section.]name value", ConfigSet) 153 | addcmd(cmdmap, "config.delete [section.]name", ConfigDelete) 154 | 155 | run(cmdmap, c.Flag, args) 156 | } 157 | 158 | func ConfigGet(cmd *cmd.Cmd, args []string) { 159 | needvar() 160 | 161 | cmd.Flag.Parse(args) 162 | k := cmd.Flag.Arg(0) 163 | if "" == k { 164 | usage(cmd) 165 | } 166 | 167 | v := programConfig.Get(k) 168 | if nil != v { 169 | fmt.Printf("%v\n", v) 170 | } 171 | } 172 | 173 | func ConfigSet(cmd *cmd.Cmd, args []string) { 174 | needvar() 175 | 176 | cmd.Flag.Parse(args) 177 | k := cmd.Flag.Arg(0) 178 | v := cmd.Flag.Arg(1) 179 | if "" == v { 180 | if i := strings.IndexByte(k, '='); -1 != i { 181 | v = k[i+1:] 182 | k = k[:i] 183 | } 184 | } 185 | if "" == k || "" == v { 186 | usage(cmd) 187 | } 188 | 189 | programConfig.Set(k, v) 190 | 191 | util.WriteFunc(configPath, 0600, func(file *os.File) error { 192 | return config.WriteTyped(file, programConfig) 193 | }) 194 | } 195 | 196 | func ConfigDelete(cmd *cmd.Cmd, args []string) { 197 | needvar() 198 | 199 | cmd.Flag.Parse(args) 200 | k := cmd.Flag.Arg(0) 201 | if "" == k { 202 | usage(cmd) 203 | } 204 | 205 | programConfig.Delete(k) 206 | 207 | util.WriteFunc(configPath, 0600, func(file *os.File) error { 208 | return config.WriteTyped(file, programConfig) 209 | }) 210 | } 211 | 212 | func Keyring(c *cmd.Cmd, args []string) { 213 | cmdmap := cmd.NewCmdMap() 214 | c.Flag.Usage = cmd.UsageFunc(c, cmdmap) 215 | 216 | var c1 *cmd.Cmd 217 | addcmd(cmdmap, "keyring.get service/user", KeyringGet) 218 | c1 = addcmd(cmdmap, "keyring.set [-k] service/user", KeyringSet) 219 | c1.Flag.Bool("k", false, "keep terminating newline when on a terminal") 220 | addcmd(cmdmap, "keyring.delete service/user", KeyringDelete) 221 | 222 | run(cmdmap, c.Flag, args) 223 | } 224 | 225 | func KeyringGet(cmd *cmd.Cmd, args []string) { 226 | needvar() 227 | 228 | cmd.Flag.Parse(args) 229 | 230 | service := cmd.Flag.Arg(0) 231 | user := cmd.Flag.Arg(1) 232 | if "" == user { 233 | if i := strings.IndexByte(service, '/'); -1 != i { 234 | user = service[i+1:] 235 | service = service[:i] 236 | } 237 | } 238 | if "" == service || "" == user { 239 | usage(cmd) 240 | } 241 | 242 | pass, err := keyring.Get(service, user) 243 | if nil != err { 244 | fail(err) 245 | } 246 | 247 | os.Stdout.WriteString(pass) 248 | if !strings.HasSuffix(pass, "\n") && terminal.IsTerminal(os.Stdout.Fd()) { 249 | os.Stdout.WriteString("\n") 250 | } 251 | } 252 | 253 | func KeyringSet(cmd *cmd.Cmd, args []string) { 254 | needvar() 255 | 256 | cmd.Flag.Parse(args) 257 | keep := cmd.GetFlag("k").(bool) 258 | 259 | service := cmd.Flag.Arg(0) 260 | user := cmd.Flag.Arg(1) 261 | if "" == user { 262 | if i := strings.IndexByte(service, '/'); -1 != i { 263 | user = service[i+1:] 264 | service = service[:i] 265 | } 266 | } 267 | if "" == service || "" == user { 268 | usage(cmd) 269 | } 270 | 271 | pass, err := ioutil.ReadAll(os.Stdin) 272 | if nil != err { 273 | fail(err) 274 | } 275 | 276 | p := string(pass) 277 | if !keep && terminal.IsTerminal(os.Stdin.Fd()) { 278 | p = strings.TrimSuffix(p, "\n") 279 | } 280 | 281 | err = keyring.Set(service, user, p) 282 | if nil != err { 283 | fail(err) 284 | } 285 | } 286 | 287 | func KeyringDelete(cmd *cmd.Cmd, args []string) { 288 | needvar() 289 | 290 | cmd.Flag.Parse(args) 291 | 292 | service := cmd.Flag.Arg(0) 293 | user := cmd.Flag.Arg(1) 294 | if "" == user { 295 | if i := strings.IndexByte(service, '/'); -1 != i { 296 | user = service[i+1:] 297 | service = service[:i] 298 | } 299 | } 300 | if "" == service || "" == user { 301 | usage(cmd) 302 | } 303 | 304 | err := keyring.Delete(service, user) 305 | if nil != err { 306 | fail(err) 307 | } 308 | } 309 | 310 | func Auth(cmd *cmd.Cmd, args []string) { 311 | needvar(&authSession) 312 | 313 | cmd.Flag.Parse(args) 314 | opath := cmd.Flag.Arg(0) 315 | if "" == opath { 316 | usage(cmd) 317 | } 318 | 319 | err := auth.WriteCredentials(opath, authSession.Credentials()) 320 | if nil != err { 321 | fail(errors.New("auth", err)) 322 | } 323 | } 324 | 325 | func Mount(cmd *cmd.Cmd, args []string) { 326 | needvar(&storage, &cachePath) 327 | 328 | cmd.Flag.Parse(args) 329 | opts := cmd.GetFlag("o").(mntopts) 330 | mountpoint := cmd.Flag.Arg(0) 331 | if "" == mountpoint { 332 | usage(cmd) 333 | } 334 | 335 | for i := range opts { 336 | opts[i] = "-o" + opts[i] 337 | } 338 | 339 | c, err := openCache(cache.Activate) 340 | if nil != err { 341 | fail(errors.New("mount", err)) 342 | } 343 | defer c.CloseCache() 344 | 345 | fsys, err := fs.Registry.NewObject("objfs", c) 346 | if nil != err { 347 | fail(errors.New("mount", err)) 348 | } 349 | 350 | err = fs.Mount(fsys, cmd.Flag.Arg(0), opts) 351 | if nil != err { 352 | fail(errors.New("mount", err)) 353 | } 354 | } 355 | 356 | func Statfs(cmd *cmd.Cmd, args []string) { 357 | needvar(&storage) 358 | 359 | cmd.Flag.Parse(args) 360 | 361 | if 0 != cmd.Flag.NArg() { 362 | usage(cmd) 363 | } 364 | 365 | info, err := storage.Info(true) 366 | if nil != err { 367 | fail(errors.New("statfs", err)) 368 | } 369 | 370 | printStorageInfo(info) 371 | } 372 | 373 | func Ls(cmd *cmd.Cmd, args []string) { 374 | needvar(&storage) 375 | 376 | cmd.Flag.Parse(args) 377 | long := cmd.GetFlag("l").(bool) 378 | maxcount := cmd.GetFlag("n").(int) 379 | count := maxcount 380 | 381 | failed := false 382 | 383 | for _, path := range cmd.Flag.Args() { 384 | marker := "" 385 | infos := ([]objio.ObjectInfo)(nil) 386 | for { 387 | var err error 388 | marker, infos, err = storage.List(path, marker, count) 389 | if nil != err { 390 | failed = true 391 | warn(errors.New("ls "+path, err)) 392 | break 393 | } 394 | 395 | for _, info := range infos { 396 | printObjectInfo(info, long) 397 | } 398 | 399 | if "" == marker { 400 | break 401 | } 402 | if 0 < maxcount { 403 | count -= len(infos) 404 | if 0 >= count { 405 | break 406 | } 407 | } 408 | } 409 | } 410 | 411 | if failed { 412 | exit(1) 413 | } 414 | } 415 | 416 | func Stat(cmd *cmd.Cmd, args []string) { 417 | needvar(&storage) 418 | 419 | cmd.Flag.Parse(args) 420 | long := cmd.GetFlag("l").(bool) 421 | 422 | failed := false 423 | 424 | for _, path := range cmd.Flag.Args() { 425 | info, err := storage.Stat(path) 426 | if nil != err { 427 | failed = true 428 | warn(errors.New("stat "+path, err)) 429 | continue 430 | } 431 | 432 | printObjectInfo(info, long) 433 | } 434 | 435 | if failed { 436 | exit(1) 437 | } 438 | } 439 | 440 | func Mkdir(cmd *cmd.Cmd, args []string) { 441 | needvar(&storage) 442 | 443 | cmd.Flag.Parse(args) 444 | 445 | failed := false 446 | 447 | for _, path := range cmd.Flag.Args() { 448 | info, err := storage.Mkdir(path) 449 | if nil != err { 450 | failed = true 451 | warn(errors.New("mkdir "+path, err)) 452 | continue 453 | } 454 | 455 | printObjectInfo(info, false) 456 | } 457 | 458 | if failed { 459 | exit(1) 460 | } 461 | } 462 | 463 | func Rmdir(cmd *cmd.Cmd, args []string) { 464 | needvar(&storage) 465 | 466 | cmd.Flag.Parse(args) 467 | 468 | failed := false 469 | 470 | for _, path := range cmd.Flag.Args() { 471 | err := storage.Rmdir(path) 472 | if nil != err { 473 | failed = true 474 | warn(errors.New("rmdir "+path, err)) 475 | continue 476 | } 477 | } 478 | 479 | if failed { 480 | exit(1) 481 | } 482 | } 483 | 484 | func Rm(cmd *cmd.Cmd, args []string) { 485 | needvar(&storage) 486 | 487 | cmd.Flag.Parse(args) 488 | 489 | failed := false 490 | 491 | for _, path := range cmd.Flag.Args() { 492 | err := storage.Remove(path) 493 | if nil != err { 494 | failed = true 495 | warn(errors.New("rm "+path, err)) 496 | continue 497 | } 498 | } 499 | 500 | if failed { 501 | exit(1) 502 | } 503 | } 504 | 505 | func Mv(cmd *cmd.Cmd, args []string) { 506 | needvar(&storage) 507 | 508 | cmd.Flag.Parse(args) 509 | 510 | if 1 > cmd.Flag.NArg() || 2 < cmd.Flag.NArg() { 511 | usage(cmd) 512 | } 513 | 514 | oldpath := cmd.Flag.Arg(0) 515 | newpath := cmd.Flag.Arg(1) 516 | 517 | err := storage.Rename(oldpath, newpath) 518 | if nil != err { 519 | fail(errors.New("mv "+oldpath, err)) 520 | } 521 | } 522 | 523 | func Get(cmd *cmd.Cmd, args []string) { 524 | needvar(&storage) 525 | 526 | cmd.Flag.Parse(args) 527 | rng := cmd.GetFlag("r").(string) 528 | sig := cmd.GetFlag("s").(string) 529 | 530 | var off, n int64 531 | if "" != rng { 532 | _, err := fmt.Sscanf(rng, "%d-%d", &off, &n) 533 | n -= off 534 | if nil != err || 0 >= n { 535 | usage(cmd) 536 | } 537 | } 538 | 539 | if 1 > cmd.Flag.NArg() || 2 < cmd.Flag.NArg() { 540 | usage(cmd) 541 | } 542 | 543 | ipath := cmd.Flag.Arg(0) 544 | opath := cmd.Flag.Arg(1) 545 | if "" == opath { 546 | opath = path.Base(ipath) 547 | } 548 | 549 | info, reader, err := storage.OpenRead(ipath, sig) 550 | if nil != err { 551 | fail(errors.New("get "+ipath, err)) 552 | } 553 | if nil == reader { 554 | fmt.Printf("%s matches sig %s; not downloaded\n", ipath, sig) 555 | return 556 | } 557 | defer reader.Close() 558 | 559 | if "" != rng { 560 | readat, ok := reader.(io.ReaderAt) 561 | if !ok { 562 | fail(errors.New("get " + ipath + "; storage does not implement GET with Range")) 563 | } 564 | reader = ioutil.NopCloser(io.NewSectionReader(readat, off, n)) 565 | } 566 | 567 | writer, err := os.OpenFile(opath, os.O_CREATE|os.O_WRONLY, 0666) 568 | if nil != err { 569 | fail(errors.New("get "+ipath, err)) 570 | } 571 | defer writer.Close() 572 | 573 | if "" != rng { 574 | _, err = writer.Seek(off, io.SeekStart) 575 | if nil != err { 576 | fail(errors.New("get "+ipath, err)) 577 | } 578 | } 579 | 580 | _, err = io.Copy(writer, reader) 581 | if nil != err { 582 | fail(errors.New("get "+ipath, err)) 583 | } 584 | 585 | printObjectInfo(info, false) 586 | } 587 | 588 | func Put(cmd *cmd.Cmd, args []string) { 589 | needvar(&storage) 590 | 591 | cmd.Flag.Parse(args) 592 | 593 | if 1 > cmd.Flag.NArg() || 2 < cmd.Flag.NArg() { 594 | usage(cmd) 595 | } 596 | 597 | ipath := cmd.Flag.Arg(0) 598 | opath := cmd.Flag.Arg(1) 599 | if "" == opath { 600 | opath = ipath 601 | ipath = path.Base(ipath) 602 | } 603 | 604 | reader, err := os.OpenFile(ipath, os.O_RDONLY, 0) 605 | if nil != err { 606 | fail(errors.New("put "+opath, err)) 607 | } 608 | defer reader.Close() 609 | 610 | stat, err := reader.Stat() 611 | if nil != err { 612 | fail(errors.New("put "+opath, err)) 613 | } 614 | 615 | writer, err := storage.OpenWrite(opath, stat.Size()) 616 | if nil != err { 617 | fail(errors.New("put "+opath, err)) 618 | } 619 | defer writer.Close() 620 | 621 | _, err = io.Copy(writer, reader) 622 | if nil != err { 623 | fail(errors.New("put "+opath, err)) 624 | } 625 | 626 | info, err := writer.Wait() 627 | if nil != err { 628 | fail(errors.New("put "+opath, err)) 629 | } 630 | 631 | printObjectInfo(info, false) 632 | } 633 | 634 | func CachePending(cmd *cmd.Cmd, args []string) { 635 | needvar(&storage, &cachePath) 636 | 637 | cmd.Flag.Parse(args) 638 | 639 | if 0 != cmd.Flag.NArg() { 640 | usage(cmd) 641 | } 642 | 643 | fmt.Printf("%s:\n", cachePath) 644 | 645 | c, err := openCache(cache.OpenIfExists) 646 | if nil != err { 647 | fail(errors.New("cache-pending", err)) 648 | } 649 | defer c.CloseCache() 650 | 651 | for _, p := range c.ListCache() { 652 | fmt.Printf("\t%s\n", p) 653 | } 654 | } 655 | 656 | func CacheReset(cmd *cmd.Cmd, args []string) { 657 | needvar(&storage, &cachePath) 658 | 659 | cmd.Flag.Parse(args) 660 | 661 | if 0 != cmd.Flag.NArg() { 662 | usage(cmd) 663 | } 664 | 665 | fmt.Printf("%s:\n", cachePath) 666 | 667 | c, err := openCache(cache.Open) 668 | if nil != err { 669 | fail(errors.New("cache-reset", err)) 670 | } 671 | defer c.CloseCache() 672 | 673 | err = c.ResetCache(func(path string) { 674 | fmt.Printf("\t%s\n", path) 675 | }) 676 | if nil != err { 677 | fail(errors.New("cache-reset", err)) 678 | } 679 | } 680 | 681 | var shellCommands = []func(cmdmap *cmd.CmdMap){ 682 | initCommands, 683 | } 684 | 685 | func Shell(c *cmd.Cmd, args []string) { 686 | editor.DefaultEditor.History().SetCap(100) 687 | 688 | if "windows" == runtime.GOOS { 689 | fmt.Println("Type \"help\" for help. Type ^Z to quit.") 690 | } else { 691 | fmt.Println("Type \"help\" for help. Type ^D to quit.") 692 | } 693 | 694 | for { 695 | line, err := editor.DefaultEditor.GetLine("> ") 696 | if nil != err { 697 | if io.EOF == err { 698 | fmt.Println("QUIT") 699 | return 700 | } 701 | fail(err) 702 | } 703 | 704 | if "windows" == runtime.GOOS { 705 | args = shlex.Windows.Split(line) 706 | } else { 707 | args = shlex.Posix.Split(line) 708 | } 709 | if 0 == len(args) { 710 | continue 711 | } 712 | 713 | cmdmap := cmd.NewCmdMap() 714 | for _, fn := range shellCommands { 715 | fn(cmdmap) 716 | } 717 | 718 | flagSet := flag.NewFlagSet("shell", flag.PanicOnError) 719 | flagSet.Usage = func() { 720 | fmt.Fprintln(os.Stderr, "commands:") 721 | cmdmap.PrintCmds() 722 | } 723 | 724 | ec := run(cmdmap, flagSet, args) 725 | if 2 != ec { 726 | editor.DefaultEditor.History().Add(line) 727 | } 728 | } 729 | } 730 | 731 | func printStorageInfo(info objio.StorageInfo) { 732 | fmt.Printf(`IsCaseInsensitive = %v 733 | IsReadOnly = %v 734 | MaxComponentLength = %v 735 | TotalSize = %v 736 | FreeSize = %v 737 | `, 738 | info.IsCaseInsensitive(), 739 | info.IsReadOnly(), 740 | info.MaxComponentLength(), 741 | info.TotalSize(), 742 | info.FreeSize()) 743 | } 744 | 745 | var dtype = map[bool]string{ 746 | true: "d", 747 | false: "-", 748 | } 749 | 750 | func printObjectInfo(info objio.ObjectInfo, long bool) { 751 | if long { 752 | fmt.Printf("%s %10d %s %s %s %s\n", 753 | dtype[info.IsDir()], 754 | info.Size(), 755 | info.Mtime().Format(time.RFC3339), 756 | info.Btime().Format(time.RFC3339), 757 | info.Sig(), 758 | info.Name()) 759 | } else { 760 | fmt.Printf("%s %10d %s %s\n", 761 | dtype[info.IsDir()], 762 | info.Size(), 763 | info.Mtime().Format(time.RFC3339), 764 | info.Name()) 765 | } 766 | } 767 | 768 | func openCache(flag int) (*cache.Cache, error) { 769 | return cache.OpenCache(cachePath, storage, nil, flag) 770 | } 771 | -------------------------------------------------------------------------------- /debug.go: -------------------------------------------------------------------------------- 1 | // +build debug 2 | 3 | /* 4 | * debug.go 5 | * 6 | * Copyright 2018 Bill Zissimopoulos 7 | */ 8 | /* 9 | * This file is part of Objfs. 10 | * 11 | * You can redistribute it and/or modify it under the terms of the GNU 12 | * Affero General Public License version 3 as published by the Free 13 | * Software Foundation. 14 | * 15 | * Licensees holding a valid commercial license may use this file in 16 | * accordance with the commercial license agreement provided with the 17 | * software. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "fmt" 24 | "net" 25 | "net/http" 26 | "os" 27 | 28 | _ "net/http/pprof" 29 | ) 30 | 31 | func init() { 32 | listen, err := net.Listen("tcp", "localhost:0") 33 | if nil != err { 34 | fmt.Fprintf(os.Stderr, "debug: cannot listen: %v\n", err) 35 | return 36 | } 37 | 38 | fmt.Fprintf(os.Stderr, "debug: listening on %v\n", listen.Addr()) 39 | 40 | go http.Serve(listen, nil) 41 | } 42 | -------------------------------------------------------------------------------- /errno/errno.go: -------------------------------------------------------------------------------- 1 | /* 2 | * errno.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package errno 19 | 20 | import ( 21 | "github.com/billziss-gh/golib/errors" 22 | ) 23 | 24 | //go:generate $GOPATH/bin/stringer -type=Errno $GOFILE 25 | 26 | // Errno contains an error number (code) and implements error. 27 | type Errno int 28 | 29 | // Generic errno constants. 30 | const ( 31 | _ Errno = iota 32 | E2BIG 33 | EACCES 34 | EADDRINUSE 35 | EADDRNOTAVAIL 36 | EAFNOSUPPORT 37 | EAGAIN 38 | EALREADY 39 | EBADF 40 | EBADMSG 41 | EBUSY 42 | ECANCELED 43 | ECHILD 44 | ECONNABORTED 45 | ECONNREFUSED 46 | ECONNRESET 47 | EDEADLK 48 | EDESTADDRREQ 49 | EDOM 50 | EEXIST 51 | EFAULT 52 | EFBIG 53 | EHOSTUNREACH 54 | EIDRM 55 | EILSEQ 56 | EINPROGRESS 57 | EINTR 58 | EINVAL 59 | EIO 60 | EISCONN 61 | EISDIR 62 | ELOOP 63 | EMFILE 64 | EMLINK 65 | EMSGSIZE 66 | ENAMETOOLONG 67 | ENETDOWN 68 | ENETRESET 69 | ENETUNREACH 70 | ENFILE 71 | ENOATTR 72 | ENOBUFS 73 | ENODATA 74 | ENODEV 75 | ENOENT 76 | ENOEXEC 77 | ENOLCK 78 | ENOLINK 79 | ENOMEM 80 | ENOMSG 81 | ENOPROTOOPT 82 | ENOSPC 83 | ENOSR 84 | ENOSTR 85 | ENOSYS 86 | ENOTCONN 87 | ENOTDIR 88 | ENOTEMPTY 89 | ENOTRECOVERABLE 90 | ENOTSOCK 91 | ENOTSUP 92 | ENOTTY 93 | ENXIO 94 | EOPNOTSUPP 95 | EOVERFLOW 96 | EOWNERDEAD 97 | EPERM 98 | EPIPE 99 | EPROTO 100 | EPROTONOSUPPORT 101 | EPROTOTYPE 102 | ERANGE 103 | EROFS 104 | ESPIPE 105 | ESRCH 106 | ETIME 107 | ETIMEDOUT 108 | ETXTBSY 109 | EWOULDBLOCK 110 | EXDEV 111 | ) 112 | 113 | // ErrnoFromErr converts a Go error to an Errno. 114 | // 115 | // If err is nil then ErrnoFromErr is 0. Otherwise the ErrnoFromErr will be EIO, 116 | // unless a more specific Errno is found in the causal chain of err. 117 | func ErrnoFromErr(err error) (errno Errno) { 118 | if nil == err { 119 | return 120 | } 121 | 122 | errno = EIO 123 | for e := err; nil != e; e = errors.Cause(e) { 124 | a := errors.Attachment(e) 125 | if nil == a { 126 | a = e 127 | } 128 | 129 | if i, ok := a.(Errno); ok && EIO != i { 130 | errno = i 131 | return 132 | } 133 | } 134 | 135 | return 136 | } 137 | -------------------------------------------------------------------------------- /errno/errno_error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * errno_error.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package errno 19 | 20 | func (self Errno) Error() string { 21 | return "errno." + self.String() 22 | } 23 | -------------------------------------------------------------------------------- /errno/errno_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Errno errno.go"; DO NOT EDIT. 2 | 3 | package errno 4 | 5 | import "strconv" 6 | 7 | const _Errno_name = "E2BIGEACCESEADDRINUSEEADDRNOTAVAILEAFNOSUPPORTEAGAINEALREADYEBADFEBADMSGEBUSYECANCELEDECHILDECONNABORTEDECONNREFUSEDECONNRESETEDEADLKEDESTADDRREQEDOMEEXISTEFAULTEFBIGEHOSTUNREACHEIDRMEILSEQEINPROGRESSEINTREINVALEIOEISCONNEISDIRELOOPEMFILEEMLINKEMSGSIZEENAMETOOLONGENETDOWNENETRESETENETUNREACHENFILEENOATTRENOBUFSENODATAENODEVENOENTENOEXECENOLCKENOLINKENOMEMENOMSGENOPROTOOPTENOSPCENOSRENOSTRENOSYSENOTCONNENOTDIRENOTEMPTYENOTRECOVERABLEENOTSOCKENOTSUPENOTTYENXIOEOPNOTSUPPEOVERFLOWEOWNERDEADEPERMEPIPEEPROTOEPROTONOSUPPORTEPROTOTYPEERANGEEROFSESPIPEESRCHETIMEETIMEDOUTETXTBSYEWOULDBLOCKEXDEV" 8 | 9 | var _Errno_index = [...]uint16{0, 5, 11, 21, 34, 46, 52, 60, 65, 72, 77, 86, 92, 104, 116, 126, 133, 145, 149, 155, 161, 166, 178, 183, 189, 200, 205, 211, 214, 221, 227, 232, 238, 244, 252, 264, 272, 281, 292, 298, 305, 312, 319, 325, 331, 338, 344, 351, 357, 363, 374, 380, 385, 391, 397, 405, 412, 421, 436, 444, 451, 457, 462, 472, 481, 491, 496, 501, 507, 522, 532, 538, 543, 549, 554, 559, 568, 575, 586, 591} 10 | 11 | func (i Errno) String() string { 12 | i -= 1 13 | if i < 0 || i >= Errno(len(_Errno_index)-1) { 14 | return "Errno(" + strconv.FormatInt(int64(i+1), 10) + ")" 15 | } 16 | return _Errno_name[_Errno_index[i]:_Errno_index[i+1]] 17 | } 18 | -------------------------------------------------------------------------------- /fs/errc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * errc.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package fs 19 | 20 | import ( 21 | "github.com/billziss-gh/cgofuse/fuse" 22 | "github.com/billziss-gh/objfs/errno" 23 | ) 24 | 25 | var errnomap = map[errno.Errno]int{ 26 | errno.E2BIG: -fuse.E2BIG, 27 | errno.EACCES: -fuse.EACCES, 28 | errno.EADDRINUSE: -fuse.EADDRINUSE, 29 | errno.EADDRNOTAVAIL: -fuse.EADDRNOTAVAIL, 30 | errno.EAFNOSUPPORT: -fuse.EAFNOSUPPORT, 31 | errno.EAGAIN: -fuse.EAGAIN, 32 | errno.EALREADY: -fuse.EALREADY, 33 | errno.EBADF: -fuse.EBADF, 34 | errno.EBADMSG: -fuse.EBADMSG, 35 | errno.EBUSY: -fuse.EBUSY, 36 | errno.ECANCELED: -fuse.ECANCELED, 37 | errno.ECHILD: -fuse.ECHILD, 38 | errno.ECONNABORTED: -fuse.ECONNABORTED, 39 | errno.ECONNREFUSED: -fuse.ECONNREFUSED, 40 | errno.ECONNRESET: -fuse.ECONNRESET, 41 | errno.EDEADLK: -fuse.EDEADLK, 42 | errno.EDESTADDRREQ: -fuse.EDESTADDRREQ, 43 | errno.EDOM: -fuse.EDOM, 44 | errno.EEXIST: -fuse.EEXIST, 45 | errno.EFAULT: -fuse.EFAULT, 46 | errno.EFBIG: -fuse.EFBIG, 47 | errno.EHOSTUNREACH: -fuse.EHOSTUNREACH, 48 | errno.EIDRM: -fuse.EIDRM, 49 | errno.EILSEQ: -fuse.EILSEQ, 50 | errno.EINPROGRESS: -fuse.EINPROGRESS, 51 | errno.EINTR: -fuse.EINTR, 52 | errno.EINVAL: -fuse.EINVAL, 53 | errno.EISCONN: -fuse.EISCONN, 54 | errno.EISDIR: -fuse.EISDIR, 55 | errno.ELOOP: -fuse.ELOOP, 56 | errno.EMFILE: -fuse.EMFILE, 57 | errno.EMLINK: -fuse.EMLINK, 58 | errno.EMSGSIZE: -fuse.EMSGSIZE, 59 | errno.ENAMETOOLONG: -fuse.ENAMETOOLONG, 60 | errno.ENETDOWN: -fuse.ENETDOWN, 61 | errno.ENETRESET: -fuse.ENETRESET, 62 | errno.ENETUNREACH: -fuse.ENETUNREACH, 63 | errno.ENFILE: -fuse.ENFILE, 64 | errno.ENOATTR: -fuse.ENOATTR, 65 | errno.ENOBUFS: -fuse.ENOBUFS, 66 | errno.ENODATA: -fuse.ENODATA, 67 | errno.ENODEV: -fuse.ENODEV, 68 | errno.ENOENT: -fuse.ENOENT, 69 | errno.ENOEXEC: -fuse.ENOEXEC, 70 | errno.ENOLCK: -fuse.ENOLCK, 71 | errno.ENOLINK: -fuse.ENOLINK, 72 | errno.ENOMEM: -fuse.ENOMEM, 73 | errno.ENOMSG: -fuse.ENOMSG, 74 | errno.ENOPROTOOPT: -fuse.ENOPROTOOPT, 75 | errno.ENOSPC: -fuse.ENOSPC, 76 | errno.ENOSR: -fuse.ENOSR, 77 | errno.ENOSTR: -fuse.ENOSTR, 78 | errno.ENOSYS: -fuse.ENOSYS, 79 | errno.ENOTCONN: -fuse.ENOTCONN, 80 | errno.ENOTDIR: -fuse.ENOTDIR, 81 | errno.ENOTEMPTY: -fuse.ENOTEMPTY, 82 | errno.ENOTRECOVERABLE: -fuse.ENOTRECOVERABLE, 83 | errno.ENOTSOCK: -fuse.ENOTSOCK, 84 | errno.ENOTSUP: -fuse.ENOTSUP, 85 | errno.ENOTTY: -fuse.ENOTTY, 86 | errno.ENXIO: -fuse.ENXIO, 87 | errno.EOPNOTSUPP: -fuse.EOPNOTSUPP, 88 | errno.EOVERFLOW: -fuse.EOVERFLOW, 89 | errno.EOWNERDEAD: -fuse.EOWNERDEAD, 90 | errno.EPERM: -fuse.EPERM, 91 | errno.EPIPE: -fuse.EPIPE, 92 | errno.EPROTO: -fuse.EPROTO, 93 | errno.EPROTONOSUPPORT: -fuse.EPROTONOSUPPORT, 94 | errno.EPROTOTYPE: -fuse.EPROTOTYPE, 95 | errno.ERANGE: -fuse.ERANGE, 96 | errno.EROFS: -fuse.EROFS, 97 | errno.ESPIPE: -fuse.ESPIPE, 98 | errno.ESRCH: -fuse.ESRCH, 99 | errno.ETIME: -fuse.ETIME, 100 | errno.ETIMEDOUT: -fuse.ETIMEDOUT, 101 | errno.ETXTBSY: -fuse.ETXTBSY, 102 | errno.EWOULDBLOCK: -fuse.EWOULDBLOCK, 103 | errno.EXDEV: -fuse.EXDEV, 104 | 105 | //errno.EIO: -fuse.EIO, 106 | } 107 | 108 | // FuseErrc converts a Go error to a FUSE error code. 109 | // If err is nil then FuseErrc is 0. Otherwise the FuseErrc will be -EIO, 110 | // unless a more specific FuseErrc is found in the causal chain of err. 111 | func FuseErrc(err error) (errc int) { 112 | errc = -fuse.EIO 113 | if rc, ok := errnomap[errno.ErrnoFromErr(err)]; ok { 114 | errc = rc 115 | } 116 | return 117 | } 118 | -------------------------------------------------------------------------------- /fs/mount.go: -------------------------------------------------------------------------------- 1 | /* 2 | * mount.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package fs 19 | 20 | import ( 21 | "os" 22 | 23 | "github.com/billziss-gh/cgofuse/fuse" 24 | "github.com/billziss-gh/golib/errors" 25 | "github.com/billziss-gh/golib/trace" 26 | "github.com/billziss-gh/objfs/errno" 27 | "github.com/billziss-gh/objfs/objio" 28 | "github.com/billziss-gh/objfs/objreg" 29 | ) 30 | 31 | type FileSystemCaseInsensitive interface { 32 | IsCaseInsensitive() bool 33 | } 34 | 35 | // Mount mounts a file system to a mountpoint. 36 | func Mount(fsys interface{}, mountpoint string, opts []string) error { 37 | fsif, ok := fsys.(fuse.FileSystemInterface) 38 | if !ok { 39 | return errors.New(": invalid argument", nil, errno.EINVAL) 40 | } 41 | 42 | cins := false 43 | if i, ok := fsys.(FileSystemCaseInsensitive); ok { 44 | cins = i.IsCaseInsensitive() 45 | } 46 | 47 | if trace.Verbose { 48 | fsif = &TraceFs{FileSystemInterface: fsif} 49 | } 50 | 51 | host := fuse.NewFileSystemHost(fsif) 52 | host.SetCapCaseInsensitive(cins) 53 | host.SetCapReaddirPlus(true) 54 | 55 | ok = host.Mount(mountpoint, opts) 56 | if !ok { 57 | return errors.New(": mount failed", nil, errno.EINVAL) 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // Registry is the default file system factory registry. 64 | var Registry = objreg.NewObjectFactoryRegistry() 65 | 66 | func CopyFusestatfsFromStorageInfo(dst *fuse.Statfs_t, src objio.StorageInfo) { 67 | *dst = fuse.Statfs_t{} 68 | dst.Frsize = 4096 69 | dst.Bsize = dst.Frsize 70 | dst.Blocks = uint64(src.TotalSize()) / dst.Frsize 71 | dst.Bfree = uint64(src.FreeSize()) / dst.Frsize 72 | dst.Bavail = dst.Bfree 73 | dst.Namemax = uint64(src.MaxComponentLength()) 74 | } 75 | 76 | var startUid = uint32(os.Geteuid()) 77 | var startGid = uint32(os.Getegid()) 78 | 79 | func CopyFusestatFromObjectInfo(dst *fuse.Stat_t, src objio.ObjectInfo) { 80 | *dst = fuse.Stat_t{} 81 | dst.Mode = fuse.S_IFREG | 0600 82 | if src.IsDir() { 83 | dst.Mode = fuse.S_IFDIR | 0700 84 | } 85 | dst.Nlink = 1 86 | dst.Uid = startUid 87 | dst.Gid = startGid 88 | dst.Size = src.Size() 89 | dst.Mtim = fuse.NewTimespec(src.Mtime()) 90 | dst.Atim = dst.Mtim 91 | dst.Ctim = dst.Mtim 92 | dst.Birthtim = fuse.NewTimespec(src.Btime()) 93 | } 94 | -------------------------------------------------------------------------------- /fs/objfs/objfs.go: -------------------------------------------------------------------------------- 1 | /* 2 | * objfs.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package objfs 19 | 20 | import ( 21 | "io" 22 | "runtime" 23 | 24 | "github.com/billziss-gh/cgofuse/fuse" 25 | "github.com/billziss-gh/golib/errors" 26 | "github.com/billziss-gh/objfs/cache" 27 | "github.com/billziss-gh/objfs/errno" 28 | "github.com/billziss-gh/objfs/fs" 29 | ) 30 | 31 | type objfs struct { 32 | fuse.FileSystemBase 33 | cache *cache.Cache 34 | } 35 | 36 | func (self *objfs) Statfs(path string, stat *fuse.Statfs_t) (errc int) { 37 | info, err := self.cache.Statfs() 38 | if nil != err { 39 | return fs.FuseErrc(err) 40 | } 41 | 42 | fs.CopyFusestatfsFromStorageInfo(stat, info) 43 | 44 | return 0 45 | } 46 | 47 | func (self *objfs) Mkdir(path string, mode uint32) (errc int) { 48 | ino, err := self.cache.Open(path) 49 | if nil != err { 50 | return fs.FuseErrc(err) 51 | } 52 | defer self.cache.Close(ino) 53 | 54 | err = self.cache.Make(ino, true) 55 | if nil != err { 56 | return fs.FuseErrc(err) 57 | } 58 | 59 | return 0 60 | } 61 | 62 | func (self *objfs) Unlink(path string) (errc int) { 63 | ino, err := self.cache.Open(path) 64 | if nil != err { 65 | return fs.FuseErrc(err) 66 | } 67 | defer self.cache.Close(ino) 68 | 69 | err = self.cache.Remove(ino, false) 70 | if nil != err { 71 | return fs.FuseErrc(err) 72 | } 73 | 74 | return 0 75 | } 76 | 77 | func (self *objfs) Rmdir(path string) (errc int) { 78 | ino, err := self.cache.Open(path) 79 | if nil != err { 80 | return fs.FuseErrc(err) 81 | } 82 | defer self.cache.Close(ino) 83 | 84 | err = self.cache.Remove(ino, true) 85 | if nil != err { 86 | return fs.FuseErrc(err) 87 | } 88 | 89 | return 0 90 | } 91 | 92 | func (self *objfs) Rename(oldpath string, newpath string) (errc int) { 93 | ino, err := self.cache.Open(oldpath) 94 | if nil != err { 95 | return fs.FuseErrc(err) 96 | } 97 | defer self.cache.Close(ino) 98 | 99 | err = self.cache.Rename(ino, newpath) 100 | if nil != err { 101 | return fs.FuseErrc(err) 102 | } 103 | 104 | return 0 105 | } 106 | 107 | func (self *objfs) Utimens(path string, tmsp []fuse.Timespec) (errc int) { 108 | return -fuse.ENOSYS 109 | } 110 | 111 | func (self *objfs) Create(path string, flags int, mode uint32) (errc int, ino uint64) { 112 | ino, err := self.cache.Open(path) 113 | if nil != err { 114 | return fs.FuseErrc(err), ^uint64(0) 115 | } 116 | 117 | err = self.cache.Make(ino, false) 118 | if nil != err { 119 | self.cache.Close(ino) 120 | return fs.FuseErrc(err), ^uint64(0) 121 | } 122 | 123 | return 0, ino 124 | } 125 | 126 | func (self *objfs) Open(path string, flags int) (errc int, ino uint64) { 127 | ino, err := self.cache.Open(path) 128 | if nil != err { 129 | return fs.FuseErrc(err), ^uint64(0) 130 | } 131 | 132 | _, err = self.cache.Stat(ino) 133 | if nil != err { 134 | self.cache.Close(ino) 135 | return fs.FuseErrc(err), ^uint64(0) 136 | } 137 | 138 | return 0, ino 139 | } 140 | 141 | func (self *objfs) Getattr(path string, stat *fuse.Stat_t, ino uint64) (errc int) { 142 | if ^uint64(0) == ino { 143 | var err error 144 | ino, err = self.cache.Open(path) 145 | if nil != err { 146 | return fs.FuseErrc(err) 147 | } 148 | defer self.cache.Close(ino) 149 | } 150 | 151 | info, err := self.cache.Stat(ino) 152 | if nil != err { 153 | return fs.FuseErrc(err) 154 | } 155 | 156 | fs.CopyFusestatFromObjectInfo(stat, info) 157 | 158 | return 0 159 | } 160 | 161 | func (self *objfs) Truncate(path string, size int64, ino uint64) (errc int) { 162 | if ^uint64(0) == ino { 163 | var err error 164 | ino, err = self.cache.Open(path) 165 | if nil != err { 166 | return fs.FuseErrc(err) 167 | } 168 | defer self.cache.Close(ino) 169 | } 170 | 171 | err := self.cache.Truncate(ino, size) 172 | if nil != err { 173 | return fs.FuseErrc(err) 174 | } 175 | 176 | return 0 177 | } 178 | 179 | func (self *objfs) Read(path string, buff []byte, ofst int64, ino uint64) (n int) { 180 | n, err := self.cache.ReadAt(ino, buff, ofst) 181 | if nil != err && io.EOF != err { 182 | return fs.FuseErrc(err) 183 | } 184 | 185 | return n 186 | } 187 | 188 | func (self *objfs) Write(path string, buff []byte, ofst int64, ino uint64) (n int) { 189 | n, err := self.cache.WriteAt(ino, buff, ofst) 190 | if nil != err { 191 | return fs.FuseErrc(err) 192 | } 193 | 194 | return n 195 | } 196 | 197 | func (self *objfs) Release(path string, ino uint64) (errc int) { 198 | err := self.cache.Close(ino) 199 | if nil != err { 200 | return fs.FuseErrc(err) 201 | } 202 | 203 | return 0 204 | } 205 | 206 | func (self *objfs) Fsync(path string, datasync bool, ino uint64) (errc int) { 207 | err := self.cache.Sync(ino) 208 | if nil != err { 209 | return fs.FuseErrc(err) 210 | } 211 | 212 | return 0 213 | } 214 | 215 | func (self *objfs) Opendir(path string) (errc int, ino uint64) { 216 | ino, err := self.cache.Open(path) 217 | if nil != err { 218 | return fs.FuseErrc(err), ^uint64(0) 219 | } 220 | 221 | _, err = self.cache.Stat(ino) 222 | if nil != err { 223 | self.cache.Close(ino) 224 | return fs.FuseErrc(err), ^uint64(0) 225 | } 226 | 227 | return 0, ino 228 | } 229 | 230 | func (self *objfs) Readdir(path string, 231 | fill func(name string, stat *fuse.Stat_t, ofst int64) bool, 232 | ofst int64, 233 | ino uint64) (errc int) { 234 | infos, err := self.cache.Readdir(ino, 0) 235 | if nil != err { 236 | return fs.FuseErrc(err) 237 | } 238 | 239 | // on Windows only add dot entries for non-root 240 | if !onWindows || "/" != path { 241 | fill(".", nil, 0) 242 | fill("..", nil, 0) 243 | } 244 | 245 | for _, info := range infos { 246 | stat := fuse.Stat_t{} 247 | 248 | fs.CopyFusestatFromObjectInfo(&stat, info) 249 | 250 | if !fill(info.Name(), &stat, 0) { 251 | break 252 | } 253 | } 254 | 255 | return 0 256 | } 257 | 258 | func (self *objfs) Releasedir(path string, ino uint64) (errc int) { 259 | err := self.cache.Close(ino) 260 | if nil != err { 261 | return fs.FuseErrc(err) 262 | } 263 | 264 | return 0 265 | } 266 | 267 | func (self *objfs) Setxattr(path string, name string, value []byte, flags int) int { 268 | // return EPERM to completely disable the xattr mechanism on Darwin 269 | return -fuse.EPERM 270 | } 271 | 272 | func (self *objfs) Getxattr(path string, name string) (int, []byte) { 273 | // return EPERM to completely disable the xattr mechanism on Darwin 274 | return -fuse.EPERM, nil 275 | } 276 | 277 | func (self *objfs) Removexattr(path string, name string) int { 278 | // return EPERM to completely disable the xattr mechanism on Darwin 279 | return -fuse.EPERM 280 | } 281 | 282 | func (self *objfs) Listxattr(path string, fill func(name string) bool) int { 283 | // return EPERM to completely disable the xattr mechanism on Darwin 284 | return -fuse.EPERM 285 | } 286 | 287 | func (self *objfs) IsCaseInsensitive() (res bool) { 288 | res = false 289 | info, err := self.cache.Storage().Info(false) 290 | if nil == err { 291 | res = info.IsCaseInsensitive() 292 | } 293 | return 294 | } 295 | 296 | func New(args ...interface{}) (interface{}, error) { 297 | var c *cache.Cache 298 | for _, arg := range args { 299 | switch a := arg.(type) { 300 | case *cache.Cache: 301 | c = a 302 | } 303 | } 304 | 305 | if nil == c { 306 | return nil, errors.New(": missing cache", nil, errno.EINVAL) 307 | } 308 | 309 | self := &objfs{ 310 | cache: c, 311 | } 312 | 313 | return self, nil 314 | } 315 | 316 | var _ fuse.FileSystemInterface = (*objfs)(nil) 317 | var _ fs.FileSystemCaseInsensitive = (*objfs)(nil) 318 | 319 | var onWindows = "windows" == runtime.GOOS 320 | 321 | // Load is used to ensure that this package is linked. 322 | func Load() { 323 | } 324 | 325 | func init() { 326 | fs.Registry.RegisterFactory("objfs", New) 327 | } 328 | -------------------------------------------------------------------------------- /fs/tracefs.go: -------------------------------------------------------------------------------- 1 | /* 2 | * tracefs.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package fs 19 | 20 | import ( 21 | "fmt" 22 | 23 | "github.com/billziss-gh/cgofuse/fuse" 24 | "github.com/billziss-gh/golib/trace" 25 | ) 26 | 27 | type TraceFs struct { 28 | fuse.FileSystemInterface 29 | } 30 | 31 | func traceFuse(vals ...interface{}) func(vals ...interface{}) { 32 | uid, gid, _ := fuse.Getcontext() 33 | return trace.Trace(1, fmt.Sprintf("{{yellow}}[uid=%v,gid=%v]{{off}}", uid, gid), vals...) 34 | } 35 | 36 | func (self *TraceFs) Init() { 37 | defer traceFuse()() 38 | self.FileSystemInterface.Init() 39 | } 40 | 41 | func (self *TraceFs) Destroy() { 42 | defer traceFuse()() 43 | self.FileSystemInterface.Destroy() 44 | } 45 | 46 | func (self *TraceFs) Statfs(path string, stat *fuse.Statfs_t) (errc int) { 47 | defer traceFuse(path)(traceErrc{&errc}, stat) 48 | return self.FileSystemInterface.Statfs(path, stat) 49 | } 50 | 51 | func (self *TraceFs) Mknod(path string, mode uint32, dev uint64) (errc int) { 52 | defer traceFuse(path, mode, dev)(traceErrc{&errc}) 53 | return self.FileSystemInterface.Mknod(path, mode, dev) 54 | } 55 | 56 | func (self *TraceFs) Mkdir(path string, mode uint32) (errc int) { 57 | defer traceFuse(path, mode)(traceErrc{&errc}) 58 | return self.FileSystemInterface.Mkdir(path, mode) 59 | } 60 | 61 | func (self *TraceFs) Unlink(path string) (errc int) { 62 | defer traceFuse(path)(traceErrc{&errc}) 63 | return self.FileSystemInterface.Unlink(path) 64 | } 65 | 66 | func (self *TraceFs) Rmdir(path string) (errc int) { 67 | defer traceFuse(path)(traceErrc{&errc}) 68 | return self.FileSystemInterface.Rmdir(path) 69 | } 70 | 71 | func (self *TraceFs) Link(oldpath string, newpath string) (errc int) { 72 | defer traceFuse(oldpath, newpath)(traceErrc{&errc}) 73 | return self.FileSystemInterface.Link(oldpath, newpath) 74 | } 75 | 76 | func (self *TraceFs) Symlink(target string, newpath string) (errc int) { 77 | defer traceFuse(target, newpath)(traceErrc{&errc}) 78 | return self.FileSystemInterface.Symlink(target, newpath) 79 | } 80 | 81 | func (self *TraceFs) Readlink(path string) (errc int, target string) { 82 | defer traceFuse(path)(traceErrc{&errc}, &target) 83 | return self.FileSystemInterface.Readlink(path) 84 | } 85 | 86 | func (self *TraceFs) Rename(oldpath string, newpath string) (errc int) { 87 | defer traceFuse(oldpath, newpath)(traceErrc{&errc}) 88 | return self.FileSystemInterface.Rename(oldpath, newpath) 89 | } 90 | 91 | func (self *TraceFs) Chmod(path string, mode uint32) (errc int) { 92 | defer traceFuse(path, mode)(traceErrc{&errc}) 93 | return self.FileSystemInterface.Chmod(path, mode) 94 | } 95 | 96 | func (self *TraceFs) Chown(path string, uid uint32, gid uint32) (errc int) { 97 | defer traceFuse(path, uid, gid)(traceErrc{&errc}) 98 | return self.FileSystemInterface.Chown(path, uid, gid) 99 | } 100 | 101 | func (self *TraceFs) Utimens(path string, tmsp []fuse.Timespec) (errc int) { 102 | defer traceFuse(path, tmsp)(traceErrc{&errc}) 103 | return self.FileSystemInterface.Utimens(path, tmsp) 104 | } 105 | 106 | func (self *TraceFs) Access(path string, mask uint32) (errc int) { 107 | defer traceFuse(path, mask)(traceErrc{&errc}) 108 | return self.FileSystemInterface.Access(path, mask) 109 | } 110 | 111 | func (self *TraceFs) Create(path string, flags int, mode uint32) (errc int, fh uint64) { 112 | defer traceFuse(path, flags, mode)(traceErrc{&errc}, &fh) 113 | return self.FileSystemInterface.Create(path, flags, mode) 114 | } 115 | 116 | func (self *TraceFs) Open(path string, flags int) (errc int, fh uint64) { 117 | defer traceFuse(path, flags)(traceErrc{&errc}, &fh) 118 | return self.FileSystemInterface.Open(path, flags) 119 | } 120 | 121 | func (self *TraceFs) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) { 122 | defer traceFuse(path, fh)(traceErrc{&errc}, stat) 123 | return self.FileSystemInterface.Getattr(path, stat, fh) 124 | } 125 | 126 | func (self *TraceFs) Truncate(path string, size int64, fh uint64) (errc int) { 127 | defer traceFuse(path, size, fh)(traceErrc{&errc}) 128 | return self.FileSystemInterface.Truncate(path, size, fh) 129 | } 130 | 131 | func (self *TraceFs) Read(path string, buff []byte, ofst int64, fh uint64) (n int) { 132 | defer traceFuse(path, ofst, fh)(&n) 133 | return self.FileSystemInterface.Read(path, buff, ofst, fh) 134 | } 135 | 136 | func (self *TraceFs) Write(path string, buff []byte, ofst int64, fh uint64) (n int) { 137 | defer traceFuse(path, ofst, fh)(&n) 138 | return self.FileSystemInterface.Write(path, buff, ofst, fh) 139 | } 140 | 141 | func (self *TraceFs) Flush(path string, fh uint64) (errc int) { 142 | defer traceFuse(path, fh)(traceErrc{&errc}) 143 | return self.FileSystemInterface.Flush(path, fh) 144 | } 145 | 146 | func (self *TraceFs) Release(path string, fh uint64) (errc int) { 147 | defer traceFuse(path, fh)(traceErrc{&errc}) 148 | return self.FileSystemInterface.Release(path, fh) 149 | } 150 | 151 | func (self *TraceFs) Fsync(path string, datasync bool, fh uint64) (errc int) { 152 | defer traceFuse(path, datasync, fh)(traceErrc{&errc}) 153 | return self.FileSystemInterface.Fsync(path, datasync, fh) 154 | } 155 | 156 | func (self *TraceFs) Opendir(path string) (errc int, fh uint64) { 157 | defer traceFuse(path)(traceErrc{&errc}, &fh) 158 | return self.FileSystemInterface.Opendir(path) 159 | } 160 | 161 | func (self *TraceFs) Readdir(path string, 162 | fill func(name string, stat *fuse.Stat_t, ofst int64) bool, 163 | ofst int64, 164 | fh uint64) (errc int) { 165 | defer traceFuse(path, ofst, fh)(traceErrc{&errc}) 166 | return self.FileSystemInterface.Readdir(path, fill, ofst, fh) 167 | } 168 | 169 | func (self *TraceFs) Releasedir(path string, fh uint64) (errc int) { 170 | defer traceFuse(path, fh)(traceErrc{&errc}) 171 | return self.FileSystemInterface.Releasedir(path, fh) 172 | } 173 | 174 | func (self *TraceFs) Fsyncdir(path string, datasync bool, fh uint64) (errc int) { 175 | defer traceFuse(path, datasync, fh)(traceErrc{&errc}) 176 | return self.FileSystemInterface.Fsyncdir(path, datasync, fh) 177 | } 178 | 179 | func (self *TraceFs) Setxattr(path string, name string, value []byte, flags int) (errc int) { 180 | defer traceFuse(path, name, value, flags)(traceErrc{&errc}) 181 | return self.FileSystemInterface.Setxattr(path, name, value, flags) 182 | } 183 | 184 | func (self *TraceFs) Getxattr(path string, name string) (errc int, xatr []byte) { 185 | defer traceFuse(path, name)(traceErrc{&errc}, &xatr) 186 | return self.FileSystemInterface.Getxattr(path, name) 187 | } 188 | 189 | func (self *TraceFs) Removexattr(path string, name string) (errc int) { 190 | defer traceFuse(path, name)(traceErrc{&errc}) 191 | return self.FileSystemInterface.Removexattr(path, name) 192 | } 193 | 194 | func (self *TraceFs) Listxattr(path string, fill func(name string) bool) (errc int) { 195 | defer traceFuse(path)(traceErrc{&errc}) 196 | return self.FileSystemInterface.Listxattr(path, fill) 197 | } 198 | 199 | type traceErrc struct { 200 | v *int 201 | } 202 | 203 | func (t traceErrc) GoString() string { 204 | if 0 == *t.v { 205 | return "{{bold green}}OK{{off}}" 206 | } 207 | return "{{bold red}}" + fuse.Error(*t.v).Error() + "{{off}}" 208 | } 209 | -------------------------------------------------------------------------------- /httputil/httputil.go: -------------------------------------------------------------------------------- 1 | /* 2 | * httputil.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package httputil 19 | 20 | import ( 21 | "errors" // remain compatible with package http 22 | "io" 23 | "net" 24 | "net/http" 25 | "net/url" 26 | "sync" 27 | "time" 28 | 29 | "github.com/billziss-gh/golib/retry" 30 | ) 31 | 32 | var DefaultTransport = NewTransport() 33 | var DefaultClient = NewClient(DefaultTransport) 34 | var DefaultRetryCount = 10 35 | 36 | func NewTransport() *http.Transport { 37 | transport := http.DefaultTransport.(*http.Transport) 38 | return &http.Transport{ 39 | Proxy: transport.Proxy, 40 | DialContext: transport.DialContext, 41 | MaxIdleConns: transport.MaxIdleConns, 42 | MaxIdleConnsPerHost: 4, 43 | IdleConnTimeout: transport.IdleConnTimeout, 44 | TLSHandshakeTimeout: transport.TLSHandshakeTimeout, 45 | ExpectContinueTimeout: transport.ExpectContinueTimeout, 46 | } 47 | } 48 | 49 | func NewClient(transport *http.Transport) *http.Client { 50 | return &http.Client{ 51 | CheckRedirect: CheckRedirect, 52 | Transport: transport, 53 | } 54 | } 55 | 56 | func Retry(body io.Seeker, do func() (*http.Response, error)) (rsp *http.Response, err error) { 57 | retry.Retry( 58 | retry.Count(DefaultRetryCount), 59 | retry.Backoff(time.Second, time.Second*30), 60 | func(i int) bool { 61 | 62 | // rewind body if there is one 63 | if nil != body { 64 | _, err := body.Seek(0, 0) 65 | if nil != err { 66 | return false 67 | } 68 | } 69 | 70 | rsp, err = do() 71 | 72 | if nil != err { 73 | // retry on connection errors without body 74 | if nil == body { 75 | return true 76 | } 77 | 78 | // retry on Dial and DNS errors 79 | for e := err; nil != e; { 80 | switch t := e.(type) { 81 | case *url.Error: 82 | e = t.Err 83 | case *net.OpError: 84 | e = t.Err 85 | if "dial" == t.Op { 86 | return true 87 | } 88 | case *net.DNSError: 89 | e = nil 90 | if t.Temporary() { 91 | return true 92 | } 93 | } 94 | } 95 | 96 | return false 97 | } 98 | 99 | // retry on HTTP 429, 503, 509 100 | if 429 == rsp.StatusCode || 503 == rsp.StatusCode || 509 == rsp.StatusCode { 101 | rsp.Body.Close() 102 | return true 103 | } 104 | 105 | return false 106 | }) 107 | 108 | return 109 | } 110 | 111 | func AllowRedirect(req *http.Request, allow bool) { 112 | redirMux.Lock() 113 | defer redirMux.Unlock() 114 | if allow { 115 | delete(redirMap, req) 116 | } else { 117 | redirMap[req] = http.ErrUseLastResponse 118 | } 119 | } 120 | 121 | func CheckRedirect(req *http.Request, via []*http.Request) error { 122 | // remain compatible with package http 123 | if len(via) >= 10 { 124 | return errors.New("stopped after 10 redirects") 125 | } 126 | redirMux.Lock() 127 | defer redirMux.Unlock() 128 | return redirMap[via[0]] 129 | } 130 | 131 | var redirMap = map[*http.Request]error{} 132 | var redirMux = sync.Mutex{} 133 | -------------------------------------------------------------------------------- /make.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set PATH=C:\Program Files\mingw-w64\x86_64-6.3.0-win32-seh-rt_v5-rev2\mingw64\bin;%PATH% 4 | set CPATH=C:\Program Files (x86)\WinFsp\inc\fuse 5 | 6 | for /f %%d in ('powershell -NoProfile -NonInteractive -ExecutionPolicy Unrestricted "Get-Date -UFormat %%y%%j"') do set MyBuildNumber=%%d 7 | 8 | mingw32-make MyBuildNumber=%MyBuildNumber% %* 9 | -------------------------------------------------------------------------------- /objfs.1: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: objfs 3 | .\" Author: [see the "AUTHOR(S)" section] 4 | .\" Generator: Asciidoctor 1.5.7 5 | .\" Date: 2018-05-25 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "OBJFS" "1" "2018-05-25" "\ \&" "\ \&" 11 | .ie \n(.g .ds Aq \(aq 12 | .el .ds Aq ' 13 | .ss \n[.ss] 0 14 | .nh 15 | .ad l 16 | .de URL 17 | \fI\\$2\fP <\\$1>\\$3 18 | .. 19 | .als MTO URL 20 | .if \n[.g] \{\ 21 | . mso www.tmac 22 | . am URL 23 | . ad l 24 | . . 25 | . am MTO 26 | . ad l 27 | . . 28 | . LINKSTYLE blue R < > 29 | .\} 30 | .SH "NAME" 31 | objfs \- object storage file system 32 | .SH "SYNOPSIS" 33 | .sp 34 | \f(CRobjfs [\-options] command args...\fP 35 | 36 | .br 37 | .SH "DESCRIPTION" 38 | .sp 39 | The objfs program implements the \(lqobject storage file system\(rq. 40 | .sp 41 | Objfs exposes objects from an object storage, such as a cloud drive, etc. as files in a file system that is fully integrated with the operating system. Programs that run on the operating system are able to access these files as if they are stored in a local "drive" (perhaps with some delay due to network operations). 42 | .sp 43 | Objfs accepts commands such as \f(CRauth\fP and \f(CRmount\fP, but also shell\-like commands, such as \f(CRls\fP, \f(CRstat\fP, etc. 44 | .sp 45 | Objfs configures itself on every run according to these sources of configuration in order of precedence: 46 | .sp 47 | .RS 4 48 | .ie n \{\ 49 | \h'-04'\(bu\h'+03'\c 50 | .\} 51 | .el \{\ 52 | . sp -1 53 | . IP \(bu 2.3 54 | .\} 55 | Command line options. 56 | .RE 57 | .sp 58 | .RS 4 59 | .ie n \{\ 60 | \h'-04'\(bu\h'+03'\c 61 | .\} 62 | .el \{\ 63 | . sp -1 64 | . IP \(bu 2.3 65 | .\} 66 | A configuration file, which is found in a platform\-specific location (unless overriden by the \f(CR\-config\fP option). 67 | .RE 68 | .sp 69 | .RS 4 70 | .ie n \{\ 71 | \h'-04'\(bu\h'+03'\c 72 | .\} 73 | .el \{\ 74 | . sp -1 75 | . IP \(bu 2.3 76 | .\} 77 | Sensible defaults. 78 | 79 | .br 80 | .RE 81 | .SS "Default Storage" 82 | .sp 83 | Objfs uses defaults to simplify command line invocation. In the default build of objfs, the default storage is \f(CRonedrive\fP. 84 | .SS "Auth" 85 | .sp 86 | Objfs supports multiple "auth" (authentication or authorization) mechanisms through the \f(CR\-credentials path\fP option and the \f(CRauth\fP command. 87 | .sp 88 | In general before an object storage service can be used it requires auth. The specific auth mechanism used depends on the service and it ranges from no auth, to username/password, to Oauth2, etc. Auth mechanisms require credentials, which can be supplied using the \f(CR\-credentials path\fP option. 89 | .sp 90 | In some cases the object storage service cannot readily accept the supplied credentials, they must be converted to other credentials first. As an authentication example, a particular service may require username/password credentials to be converted to some form of service\-level token before they can be used. As an authorization example Oauth2 requires application\-level credentials together with user consent to form a service\-level token that can be used to access the service. 91 | .sp 92 | The \f(CRauth\fP command can be used for this purpose. It takes user\-level or application\-level credentials and converts them to service\-level credentials. 93 | .sp 94 | Credentials can be stored in the local file system or the system keyring. The syntax \f(CR/file/path\fP is used to name credentials stored in the file system. The syntax \f(CRkeyring:service/user\fP is used to name credentials stored in the system keyring. 95 | .SS "Example \- Oauth2 Flow" 96 | .sp 97 | .RS 4 98 | .ie n \{\ 99 | \h'-04'\(bu\h'+03'\c 100 | .\} 101 | .el \{\ 102 | . sp -1 103 | . IP \(bu 2.3 104 | .\} 105 | Prepare the Oauth2 \f(CRclient_secret\fP credentials in a file or the system keyring: 106 | .sp 107 | .if n .RS 4 108 | .nf 109 | client_id="XXXXXXXX" 110 | client_secret="XXXXXXXX" 111 | redirect_uri="http://localhost:xxxxx" 112 | scope="files.readwrite.all offline_access" 113 | .fi 114 | .if n .RE 115 | .RE 116 | .sp 117 | .RS 4 118 | .ie n \{\ 119 | \h'-04'\(bu\h'+03'\c 120 | .\} 121 | .el \{\ 122 | . sp -1 123 | . IP \(bu 2.3 124 | .\} 125 | Issue the command: 126 | .sp 127 | .if n .RS 4 128 | .nf 129 | $ ./objfs \-credentials=CLIENT_SECRET_PATH auth TOKEN_PATH 130 | .fi 131 | .if n .RE 132 | .RE 133 | .sp 134 | .RS 4 135 | .ie n \{\ 136 | \h'-04'\(bu\h'+03'\c 137 | .\} 138 | .el \{\ 139 | . sp -1 140 | . IP \(bu 2.3 141 | .\} 142 | This will launch your browser and ask for authorization. If the access is authorized the Oauth2 \f(CRaccess_token\fP and \f(CRrefresh_token\fP will be stored in the specified path. 143 | .RE 144 | .sp 145 | .RS 4 146 | .ie n \{\ 147 | \h'-04'\(bu\h'+03'\c 148 | .\} 149 | .el \{\ 150 | . sp -1 151 | . IP \(bu 2.3 152 | .\} 153 | The object storage can now be mounted using the command: 154 | .sp 155 | .if n .RS 4 156 | .nf 157 | $ ./objfs \-credentials=TOKEN_PATH mount MOUNTPOINT 158 | .fi 159 | .if n .RE 160 | .RE 161 | .SS "Mount" 162 | .sp 163 | The objfs \f(CRmount\fP command is used to mount an object storage as a file system on a mountpoint. On Windows the mount point must be a non\-existing drive or directory; it is recommended that an object storage is only mounted as a drive when the object storage is case\-sensitive. On macOS and Linux the mount point must be an existing directory. 164 | .sp 165 | To mount on Windows: 166 | .sp 167 | .if n .RS 4 168 | .nf 169 | > objfs \-credentials=TOKEN_PATH mount \-o uid=\-1,gid=\-1 mount X: 170 | .fi 171 | .if n .RE 172 | .sp 173 | To mount on macOS and Linux: 174 | .sp 175 | .if n .RS 4 176 | .nf 177 | $ ./objfs \-credentials=TOKEN_PATH mount MOUNTPOINT 178 | .fi 179 | .if n .RE 180 | .sp 181 | Objfs uses a local file cache to speed up file system operations. This caches files locally when they are first opened; subsequent I/O operations will be performed against the local file and are therefore fast. Modified files will be uploaded to the object storage when they are closed. File system operations such as creating and deleting files and listing directories are sent directly to the object storage and are therefore slow (although some of their results are cached). 182 | .sp 183 | The Objfs cache was inspired by an early version of the Andrew File System (AFS). For more information see the paper \c 184 | .URL "http://pages.cs.wisc.edu/~remzi/OSTEP/dist\-afs.pdf" "" "." 185 | 186 | .br 187 | .SS "Diagnostics" 188 | .sp 189 | Objfs includes a tracing facility that can be used to troubleshoot problems, to gain insights into its internal workings, etc. This facility is enabled when the \f(CR\-v\fP option is used. 190 | .sp 191 | The environment variable \f(CRGOLIB_TRACE\fP controls which traces are enabled. This variable accepts a comma separated list of file\-style patterns containing wildcards such as \f(CR*\fP and \f(CR?\fP. 192 | .sp 193 | .if n .RS 4 194 | .nf 195 | $ export GOLIB_TRACE=pattern1,...,patternN 196 | .fi 197 | .if n .RE 198 | .sp 199 | Examples: 200 | .sp 201 | .if n .RS 4 202 | .nf 203 | $ export GOLIB_TRACE=github.com/billziss\-gh/objfs/fs.* # file system traces 204 | $ export GOLIB_TRACE=github.com/billziss\-gh/objfs/objio.* # object storage traces 205 | $ export GOLIB_TRACE=github.com/billziss\-gh/objfs/fs.*,github.com/billziss\-gh/objfs/objio.* 206 | $ ./objfs \-v \-credentials=TOKEN_PATH mount MOUNTPOINT 207 | .fi 208 | .if n .RE 209 | .sp 210 | 211 | .br 212 | .SH "COMMANDS" 213 | .sp 214 | The following commands may be used: 215 | .sp 216 | \f(CRversion\fP 217 | .RS 4 218 | get current version information 219 | .RE 220 | .sp 221 | \f(CRconfig\fP 222 | .RS 4 223 | get or set configuration options 224 | .RE 225 | .sp 226 | \f(CRkeyring\fP 227 | .RS 4 228 | get or set keys 229 | .RE 230 | .sp 231 | \f(CRauth output\-credentials\fP 232 | .RS 4 233 | perform authentication/authorization 234 | .RE 235 | .sp 236 | \f(CRmount [\-o option...] mountpoint\fP 237 | .RS 4 238 | mount file system 239 | .RE 240 | .sp 241 | \f(CRstatfs\fP 242 | .RS 4 243 | get storage information 244 | .RE 245 | .sp 246 | \f(CRls [\-l][\-n count] path...\fP 247 | .RS 4 248 | list files 249 | .RE 250 | .sp 251 | \f(CRstat [\-l] path...\fP 252 | .RS 4 253 | display file information 254 | .RE 255 | .sp 256 | \f(CRmkdir path...\fP 257 | .RS 4 258 | make directories 259 | .RE 260 | .sp 261 | \f(CRrmdir path...\fP 262 | .RS 4 263 | remove directories 264 | .RE 265 | .sp 266 | \f(CRrm path...\fP 267 | .RS 4 268 | remove files 269 | .RE 270 | .sp 271 | \f(CRmv oldpath newpath\fP 272 | .RS 4 273 | move (rename) files 274 | .RE 275 | .sp 276 | \f(CRget [\-r range][\-s signature] path [local\-path]\fP 277 | .RS 4 278 | get (download) files 279 | .RE 280 | .sp 281 | \f(CRput [local\-path] path\fP 282 | .RS 4 283 | put (upload) files 284 | .RE 285 | .sp 286 | \f(CRcache\-pending\fP 287 | .RS 4 288 | list pending cache files 289 | .RE 290 | .sp 291 | \f(CRcache\-reset\fP 292 | .RS 4 293 | reset cache (upload and evict files) 294 | 295 | .br 296 | .RE 297 | .SH "GENERAL OPTIONS" 298 | .sp 299 | The following options apply to all commands: 300 | .sp 301 | \f(CR\-accept\-tls\-cert\fP 302 | .RS 4 303 | accept any TLS certificate presented by the server (insecure) 304 | .RE 305 | .sp 306 | \f(CR\-auth name\fP 307 | .RS 4 308 | auth name to use 309 | .RE 310 | .sp 311 | \f(CR\-config path\fP 312 | .RS 4 313 | path to configuration file 314 | .RE 315 | .sp 316 | \f(CR\-credentials path\fP 317 | .RS 4 318 | auth credentials path (keyring:service/user or /file/path) 319 | .RE 320 | .sp 321 | \f(CR\-datadir path\fP 322 | .RS 4 323 | path to supporting data and caches 324 | .RE 325 | .sp 326 | \f(CR\-keyring string\fP 327 | .RS 4 328 | keyring type to use: system, private (default "private") 329 | .RE 330 | .sp 331 | \f(CR\-storage name\fP 332 | .RS 4 333 | storage name to access (default "onedrive") 334 | .RE 335 | .sp 336 | \f(CR\-storage\-uri uri\fP 337 | .RS 4 338 | storage uri to access 339 | .RE 340 | .sp 341 | \f(CR\-v\fP 342 | .RS 4 343 | verbose 344 | 345 | .br 346 | .RE 347 | .SH "CONFIGURATION FILE" 348 | .sp 349 | During startup objfs consults a congifuration file from a platform\-specific location (see the \fBFILES\fP section); this location can be overriden with the \f(CR\-config\fP option. 350 | .sp 351 | The configuration file stores a list of properties (key/value) pairs, that may also be grouped into sections. The basic syntax of the configuration file is as follows: 352 | .sp 353 | .if n .RS 4 354 | .nf 355 | name1=value1 356 | name2=value2 357 | \&... 358 | [section] 359 | name3=value3 360 | name4=value4 361 | \&... 362 | .fi 363 | .if n .RE 364 | .sp 365 | The valid property names are a subset of the command\-line options: \f(CRauth\fP, \f(CRcredentials\fP, \f(CRstorage\fP, \f(CRstorage\-uri\fP. They specify the same value as the equivalent command\-line option. 366 | .sp 367 | The command line option or property \f(CRstorage\fP may specify the name of a storage service (e.g. \f(CRonedrive\fP), but it may also specify a section within the configuration file, which should be used to retrieve additional configuration options. For example, given the configuration file below and a command line option \f(CR\-storage=onedrive2\fP, it will instruct objfs to act on the OneDrive storage identified by the credentials \f(CRkeyring:objfs/onedrive2\fP: 368 | .sp 369 | .if n .RS 4 370 | .nf 371 | [onedrive1] 372 | storage=onedrive 373 | credentials=keyring:objfs/onedrive1 374 | 375 | [onedrive2] 376 | storage=onedrive 377 | credentials=keyring:objfs/onedrive2 378 | .fi 379 | .if n .RE 380 | .sp 381 | 382 | .br 383 | .SH "FILES" 384 | .sp 385 | Windows 386 | .RS 4 387 | .sp 388 | .RS 4 389 | .ie n \{\ 390 | \h'-04'\(bu\h'+03'\c 391 | .\} 392 | .el \{\ 393 | . sp -1 394 | . IP \(bu 2.3 395 | .\} 396 | \fBconfig\fP: \f(CR%USERPROFILE%\(rsAppData\(rsRoaming\(rsobjfs.conf\fP 397 | .RE 398 | .sp 399 | .RS 4 400 | .ie n \{\ 401 | \h'-04'\(bu\h'+03'\c 402 | .\} 403 | .el \{\ 404 | . sp -1 405 | . IP \(bu 2.3 406 | .\} 407 | \fBdatadir\fP: \f(CR%USERPROFILE%\(rsAppData\(rsRoaming\(rsobjfs\fP 408 | .RE 409 | .RE 410 | .sp 411 | macOS 412 | .RS 4 413 | .sp 414 | .RS 4 415 | .ie n \{\ 416 | \h'-04'\(bu\h'+03'\c 417 | .\} 418 | .el \{\ 419 | . sp -1 420 | . IP \(bu 2.3 421 | .\} 422 | \fBconfig\fP: \f(CR~/Library/Preferences/objfs.conf\fP 423 | .RE 424 | .sp 425 | .RS 4 426 | .ie n \{\ 427 | \h'-04'\(bu\h'+03'\c 428 | .\} 429 | .el \{\ 430 | . sp -1 431 | . IP \(bu 2.3 432 | .\} 433 | \fBdatadir\fP: \f(CR~/Library/Application Support/objfs\fP 434 | .RE 435 | .RE 436 | .sp 437 | Linux 438 | .RS 4 439 | .sp 440 | .RS 4 441 | .ie n \{\ 442 | \h'-04'\(bu\h'+03'\c 443 | .\} 444 | .el \{\ 445 | . sp -1 446 | . IP \(bu 2.3 447 | .\} 448 | \fBconfig\fP: \f(CR~/.config/objfs.conf\fP 449 | .RE 450 | .sp 451 | .RS 4 452 | .ie n \{\ 453 | \h'-04'\(bu\h'+03'\c 454 | .\} 455 | .el \{\ 456 | . sp -1 457 | . IP \(bu 2.3 458 | .\} 459 | \fBdatadir\fP: \f(CR~/.local/share/objfs\fP 460 | .RE 461 | .RE 462 | .sp 463 | 464 | .br 465 | .SH "COPYRIGHT" 466 | .sp 467 | \(co 2018 Bill Zissimopoulos -------------------------------------------------------------------------------- /objfs.1.asciidoc: -------------------------------------------------------------------------------- 1 | objfs(1) 2 | ======== 3 | :blank: pass:[ +] 4 | 5 | NAME 6 | ---- 7 | objfs - object storage file system 8 | 9 | SYNOPSIS 10 | -------- 11 | `objfs [-options] command args...` 12 | {blank} 13 | 14 | DESCRIPTION 15 | ----------- 16 | The objfs program implements the ``object storage file system''. 17 | 18 | Objfs exposes objects from an object storage, such as a cloud drive, etc. as files in a file system that is fully integrated with the operating system. Programs that run on the operating system are able to access these files as if they are stored in a local "drive" (perhaps with some delay due to network operations). 19 | 20 | Objfs accepts commands such as `auth` and `mount`, but also shell-like commands, such as `ls`, `stat`, etc. 21 | 22 | Objfs configures itself on every run according to these sources of configuration in order of precedence: 23 | 24 | - Command line options. 25 | - A configuration file, which is found in a platform-specific location (unless overriden by the `-config` option). 26 | - Sensible defaults. 27 | {blank} 28 | 29 | Default Storage 30 | ~~~~~~~~~~~~~~~ 31 | 32 | Objfs uses defaults to simplify command line invocation. In the default build of objfs, the default storage is `onedrive`. 33 | 34 | Auth 35 | ~~~~ 36 | 37 | Objfs supports multiple "auth" (authentication or authorization) mechanisms through the `-credentials path` option and the `auth` command. 38 | 39 | In general before an object storage service can be used it requires auth. The specific auth mechanism used depends on the service and it ranges from no auth, to username/password, to Oauth2, etc. Auth mechanisms require credentials, which can be supplied using the `-credentials path` option. 40 | 41 | In some cases the object storage service cannot readily accept the supplied credentials, they must be converted to other credentials first. As an authentication example, a particular service may require username/password credentials to be converted to some form of service-level token before they can be used. As an authorization example Oauth2 requires application-level credentials together with user consent to form a service-level token that can be used to access the service. 42 | 43 | The `auth` command can be used for this purpose. It takes user-level or application-level credentials and converts them to service-level credentials. 44 | 45 | Credentials can be stored in the local file system or the system keyring. The syntax `/file/path` is used to name credentials stored in the file system. The syntax `keyring:service/user` is used to name credentials stored in the system keyring. 46 | 47 | Example - Oauth2 Flow 48 | ^^^^^^^^^^^^^^^^^^^^^ 49 | 50 | - Prepare the Oauth2 `client_secret` credentials in a file or the system keyring: 51 | + 52 | ---- 53 | client_id="XXXXXXXX" 54 | client_secret="XXXXXXXX" 55 | redirect_uri="http://localhost:xxxxx" 56 | scope="files.readwrite.all offline_access" 57 | ---- 58 | 59 | - Issue the command: 60 | + 61 | ---- 62 | $ ./objfs -credentials=CLIENT_SECRET_PATH auth TOKEN_PATH 63 | ---- 64 | 65 | - This will launch your browser and ask for authorization. If the access is authorized the Oauth2 `access_token` and `refresh_token` will be stored in the specified path. 66 | 67 | - The object storage can now be mounted using the command: 68 | + 69 | ---- 70 | $ ./objfs -credentials=TOKEN_PATH mount MOUNTPOINT 71 | ---- 72 | 73 | Mount 74 | ~~~~~ 75 | 76 | The objfs `mount` command is used to mount an object storage as a file system on a mountpoint. On Windows the mount point must be a non-existing drive or directory; it is recommended that an object storage is only mounted as a drive when the object storage is case-sensitive. On macOS and Linux the mount point must be an existing directory. 77 | 78 | To mount on Windows: 79 | 80 | ---- 81 | > objfs -credentials=TOKEN_PATH mount -o uid=-1,gid=-1 mount X: 82 | ---- 83 | 84 | To mount on macOS and Linux: 85 | 86 | ---- 87 | $ ./objfs -credentials=TOKEN_PATH mount MOUNTPOINT 88 | ---- 89 | 90 | Objfs uses a local file cache to speed up file system operations. This caches files locally when they are first opened; subsequent I/O operations will be performed against the local file and are therefore fast. Modified files will be uploaded to the object storage when they are closed. File system operations such as creating and deleting files and listing directories are sent directly to the object storage and are therefore slow (although some of their results are cached). 91 | 92 | The Objfs cache was inspired by an early version of the Andrew File System (AFS). For more information see the paper http://pages.cs.wisc.edu/~remzi/OSTEP/dist-afs.pdf. 93 | {blank} 94 | 95 | Diagnostics 96 | ~~~~~~~~~~~ 97 | 98 | Objfs includes a tracing facility that can be used to troubleshoot problems, to gain insights into its internal workings, etc. This facility is enabled when the `-v` option is used. 99 | 100 | The environment variable `GOLIB_TRACE` controls which traces are enabled. This variable accepts a comma separated list of file-style patterns containing wildcards such as `*` and `?`. 101 | 102 | ---- 103 | $ export GOLIB_TRACE=pattern1,...,patternN 104 | ---- 105 | 106 | Examples: 107 | 108 | ---- 109 | $ export GOLIB_TRACE=github.com/billziss-gh/objfs/fs.* # file system traces 110 | $ export GOLIB_TRACE=github.com/billziss-gh/objfs/objio.* # object storage traces 111 | $ export GOLIB_TRACE=github.com/billziss-gh/objfs/fs.*,github.com/billziss-gh/objfs/objio.* 112 | $ ./objfs -v -credentials=TOKEN_PATH mount MOUNTPOINT 113 | ---- 114 | {blank} 115 | 116 | COMMANDS 117 | -------- 118 | The following commands may be used: 119 | 120 | `version`:: 121 | get current version information 122 | 123 | `config`:: 124 | get or set configuration options 125 | 126 | `keyring`:: 127 | get or set keys 128 | 129 | `auth output-credentials`:: 130 | perform authentication/authorization 131 | 132 | `mount [-o option...] mountpoint`:: 133 | mount file system 134 | 135 | `statfs`:: 136 | get storage information 137 | 138 | `ls [-l][-n count] path...`:: 139 | list files 140 | 141 | `stat [-l] path...`:: 142 | display file information 143 | 144 | `mkdir path...`:: 145 | make directories 146 | 147 | `rmdir path...`:: 148 | remove directories 149 | 150 | `rm path...`:: 151 | remove files 152 | 153 | `mv oldpath newpath`:: 154 | move (rename) files 155 | 156 | `get [-r range][-s signature] path [local-path]`:: 157 | get (download) files 158 | 159 | `put [local-path] path`:: 160 | put (upload) files 161 | 162 | `cache-pending`:: 163 | list pending cache files 164 | 165 | `cache-reset`:: 166 | reset cache (upload and evict files) 167 | {blank} 168 | 169 | GENERAL OPTIONS 170 | --------------- 171 | The following options apply to all commands: 172 | 173 | `-accept-tls-cert`:: 174 | accept any TLS certificate presented by the server (insecure) 175 | 176 | `-auth name`:: 177 | auth name to use 178 | 179 | `-config path`:: 180 | path to configuration file 181 | 182 | `-credentials path`:: 183 | auth credentials path (keyring:service/user or /file/path) 184 | 185 | `-datadir path`:: 186 | path to supporting data and caches 187 | 188 | `-keyring string`:: 189 | keyring type to use: system, private (default "private") 190 | 191 | `-storage name`:: 192 | storage name to access (default "onedrive") 193 | 194 | `-storage-uri uri`:: 195 | storage uri to access 196 | 197 | `-v`:: 198 | verbose 199 | {blank} 200 | 201 | CONFIGURATION FILE 202 | ------------------ 203 | During startup objfs consults a congifuration file from a platform-specific location (see the *FILES* section); this location can be overriden with the `-config` option. 204 | 205 | The configuration file stores a list of properties (key/value) pairs, that may also be grouped into sections. The basic syntax of the configuration file is as follows: 206 | 207 | ---- 208 | name1=value1 209 | name2=value2 210 | ... 211 | [section] 212 | name3=value3 213 | name4=value4 214 | ... 215 | ---- 216 | 217 | The valid property names are a subset of the command-line options: `auth`, `credentials`, `storage`, `storage-uri`. They specify the same value as the equivalent command-line option. 218 | 219 | The command line option or property `storage` may specify the name of a storage service (e.g. `onedrive`), but it may also specify a section within the configuration file, which should be used to retrieve additional configuration options. For example, given the configuration file below and a command line option `-storage=onedrive2`, it will instruct objfs to act on the OneDrive storage identified by the credentials `keyring:objfs/onedrive2`: 220 | 221 | ---- 222 | [onedrive1] 223 | storage=onedrive 224 | credentials=keyring:objfs/onedrive1 225 | 226 | [onedrive2] 227 | storage=onedrive 228 | credentials=keyring:objfs/onedrive2 229 | ---- 230 | {blank} 231 | 232 | FILES 233 | ----- 234 | Windows:: 235 | - *config*: `%USERPROFILE%\AppData\Roaming\objfs.conf` 236 | - *datadir*: `%USERPROFILE%\AppData\Roaming\objfs` 237 | 238 | macOS:: 239 | - *config*: `~/Library/Preferences/objfs.conf` 240 | - *datadir*: `~/Library/Application Support/objfs` 241 | 242 | Linux:: 243 | - *config*: `~/.config/objfs.conf` 244 | - *datadir*: `~/.local/share/objfs` 245 | 246 | {blank} 247 | 248 | COPYRIGHT 249 | --------- 250 | (C) 2018 Bill Zissimopoulos 251 | -------------------------------------------------------------------------------- /objfs.go: -------------------------------------------------------------------------------- 1 | ///usr/bin/env go run objfs.go registry.go commands.go "$@"; exit 2 | 3 | /* 4 | * objfs.go 5 | * 6 | * Copyright 2018 Bill Zissimopoulos 7 | */ 8 | /* 9 | * This file is part of Objfs. 10 | * 11 | * You can redistribute it and/or modify it under the terms of the GNU 12 | * Affero General Public License version 3 as published by the Free 13 | * Software Foundation. 14 | * 15 | * Licensees holding a valid commercial license may use this file in 16 | * accordance with the commercial license agreement provided with the 17 | * software. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "crypto/rand" 24 | "crypto/tls" 25 | "flag" 26 | "fmt" 27 | "os" 28 | "path/filepath" 29 | "sync" 30 | 31 | "github.com/billziss-gh/golib/appdata" 32 | "github.com/billziss-gh/golib/cmd" 33 | "github.com/billziss-gh/golib/config" 34 | cflag "github.com/billziss-gh/golib/config/flag" 35 | "github.com/billziss-gh/golib/errors" 36 | "github.com/billziss-gh/golib/keyring" 37 | "github.com/billziss-gh/golib/trace" 38 | "github.com/billziss-gh/golib/util" 39 | "github.com/billziss-gh/objfs/auth" 40 | "github.com/billziss-gh/objfs/errno" 41 | "github.com/billziss-gh/objfs/httputil" 42 | "github.com/billziss-gh/objfs/objio" 43 | ) 44 | 45 | // Product variables. These variables can be overriden using the go build 46 | // -ldflags switch. For example: 47 | // 48 | // go build -ldflags "-X main.MyVersion=0.9" 49 | var ( 50 | MyProductName = "objfs" 51 | MyDescription = "Object Storage File System" 52 | MyCopyright = "2018 Bill Zissimopoulos" 53 | MyRepository = "https://github.com/billziss-gh/objfs" 54 | MyVersion = "DEVEL" 55 | ) 56 | 57 | // Configuration variables. These variables control the overall operation of objfs. 58 | // 59 | // The logic of initializing these variables is rather complicated: 60 | // 61 | // - The configuration is determined by a combination of command-line parameters 62 | // and a configuration file. When there is a conflict between the two, the 63 | // command-line parameters take precendence. 64 | // 65 | // - The configuration file is named objfs.conf and placed in the appropriate 66 | // directory for the underlying system, unless the -config command-line parameter 67 | // is specified. The configuration file (if it exists) stores key/value pairs and 68 | // may also have [sections]. 69 | // 70 | // - The process starts by creating an empty "flag map" and proceeds by merging 71 | // key/value pairs from the different sources. 72 | // 73 | // - If the configuration file exists it is read and the unnamed empty section ("") 74 | // is merged into the flag map. Then any "-storage" command line parameter 75 | // is merged into the flag map. Then if there is a configuration section with the 76 | // name specified by "storage" that section is merged into the flag map. 77 | // 78 | // - The remaining command-line options (other than -storage) are merged 79 | // into the flag map. 80 | // 81 | // - Finally the flag map is used to initialize the configuration variables. 82 | // 83 | // For the full logic see needvar. 84 | var ( 85 | configPath string 86 | dataDir string 87 | programConfig config.TypedConfig 88 | 89 | acceptTlsCert bool 90 | authName string 91 | authSession auth.Session 92 | cachePath string 93 | credentialPath string 94 | credentials auth.CredentialMap 95 | keyringKind string 96 | storage objio.ObjectStorage 97 | storageName string 98 | storageUri string 99 | ) 100 | 101 | func init() { 102 | flag.CommandLine.Init(flag.CommandLine.Name(), flag.PanicOnError) 103 | flag.Usage = cmd.UsageFunc() 104 | 105 | flag.StringVar(&configPath, "config", "", 106 | "`path` to configuration file") 107 | flag.String("datadir", "", 108 | "`path` to supporting data and caches") 109 | flag.BoolVar(&trace.Verbose, "v", false, 110 | "verbose") 111 | 112 | flag.Bool("accept-tls-cert", false, 113 | "accept any TLS certificate presented by the server (insecure)") 114 | flag.String("auth", "", 115 | "auth `name` to use") 116 | flag.String("keyring", "user", 117 | "keyring type to use: system, user, userplain") 118 | flag.String("credentials", "", 119 | "auth credentials `path` (keyring:service/user or /file/path)") 120 | flag.String("storage", defaultStorageName, 121 | "storage `name` to access") 122 | flag.String("storage-uri", "", 123 | "storage `uri` to access") 124 | } 125 | 126 | func usage(cmd *cmd.Cmd) { 127 | if nil == cmd { 128 | flag.Usage() 129 | } else { 130 | cmd.Flag.Usage() 131 | } 132 | exit(2) 133 | } 134 | 135 | func usageWithError(err error) { 136 | flag.Usage() 137 | warn(err) 138 | exit(2) 139 | } 140 | 141 | func initKeyring(kind string, path string) { 142 | var key []byte 143 | 144 | switch kind { 145 | case "system": 146 | case "user": 147 | pass, err := keyring.Get("objfs", "keyring") 148 | if nil != err { 149 | key = make([]byte, 16) 150 | _, err = rand.Read(key) 151 | if nil != err { 152 | fail(err) 153 | } 154 | err = keyring.Set("objfs", "keyring", string(key)) 155 | if nil != err { 156 | fail(err) 157 | } 158 | } else { 159 | key = []byte(pass) 160 | } 161 | fallthrough 162 | case "userplain": 163 | keyring.DefaultKeyring = &keyring.OverlayKeyring{ 164 | Keyrings: []keyring.Keyring{ 165 | &keyring.FileKeyring{ 166 | Path: filepath.Join(path, "keyring"), 167 | Key: key, 168 | }, 169 | keyring.DefaultKeyring, 170 | }, 171 | } 172 | default: 173 | usageWithError(errors.New("unknown keyring type; specify -keyring in the command line")) 174 | } 175 | } 176 | 177 | var needvarOnce sync.Once 178 | 179 | func needvar(args ...interface{}) { 180 | needvarOnce.Do(func() { 181 | if "" == configPath { 182 | dir, err := appdata.ConfigDir() 183 | if nil != err { 184 | fail(err) 185 | } 186 | 187 | configPath = filepath.Join(dir, "objfs.conf") 188 | } 189 | 190 | flagMap := config.TypedSection{} 191 | cflag.VisitAll(nil, flagMap, 192 | "accept-tls-cert", 193 | "auth", 194 | "credentials", 195 | "datadir", 196 | "keyring", 197 | "storage", 198 | "storage-uri") 199 | 200 | c, err := util.ReadFunc(configPath, func(file *os.File) (interface{}, error) { 201 | return config.ReadTyped(file) 202 | }) 203 | if nil == err { 204 | programConfig = c.(config.TypedConfig) 205 | 206 | for k, v := range programConfig[""] { 207 | flagMap[k] = v 208 | } 209 | 210 | cflag.Visit(nil, flagMap, "storage") 211 | 212 | for k, v := range programConfig[flagMap["storage"].(string)] { 213 | flagMap[k] = v 214 | } 215 | 216 | cflag.Visit(nil, flagMap, 217 | "accept-tls-cert", 218 | "auth", 219 | "credentials", 220 | "datadir", 221 | "keyring", 222 | "storage-uri") 223 | } else { 224 | programConfig = config.TypedConfig{} 225 | } 226 | 227 | acceptTlsCert = flagMap["accept-tls-cert"].(bool) 228 | authName = flagMap["auth"].(string) 229 | credentialPath = flagMap["credentials"].(string) 230 | dataDir = flagMap["datadir"].(string) 231 | keyringKind = flagMap["keyring"].(string) 232 | storageName = flagMap["storage"].(string) 233 | storageUri = flagMap["storage-uri"].(string) 234 | 235 | if "" == dataDir { 236 | dir, err := appdata.DataDir() 237 | if nil != err { 238 | fail(err) 239 | } 240 | 241 | dataDir = filepath.Join(dir, "objfs") 242 | } 243 | 244 | initKeyring(keyringKind, dataDir) 245 | 246 | if false { 247 | fmt.Printf("configPath=%#v\n", configPath) 248 | fmt.Printf("dataDir=%#v\n", dataDir) 249 | fmt.Println() 250 | fmt.Printf("acceptTlsCert=%#v\n", acceptTlsCert) 251 | fmt.Printf("authName=%#v\n", authName) 252 | fmt.Printf("credentialPath=%#v\n", credentialPath) 253 | fmt.Printf("keyringKind=%#v\n", keyringKind) 254 | fmt.Printf("storageName=%#v\n", storageName) 255 | fmt.Printf("storageUri=%#v\n", storageUri) 256 | } 257 | 258 | if acceptTlsCert { 259 | httputil.DefaultTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 260 | } 261 | }) 262 | 263 | for _, a := range args { 264 | switch a { 265 | case &authName: 266 | if "" != authName { 267 | continue 268 | } 269 | needvar(&storageName) 270 | authName = storageName 271 | 272 | case &authSession: 273 | if nil != authSession { 274 | continue 275 | } 276 | needvar(&authName, &credentials) 277 | a, err := auth.Registry.NewObject(authName) 278 | if nil != err { 279 | usageWithError(errors.New("unknown auth; specify -auth in the command line")) 280 | } 281 | s, err := a.(auth.Auth).Session(credentials) 282 | if nil != err { 283 | fail(err) 284 | } 285 | authSession = s 286 | 287 | case &cachePath: 288 | if "" != cachePath { 289 | continue 290 | } 291 | needvar(&storageName) 292 | cachePath = filepath.Join(dataDir, storageName) 293 | 294 | case &credentialPath: 295 | if "" != credentialPath { 296 | continue 297 | } 298 | needvar(&storageName) 299 | credentialPath = "keyring:objfs/" + storageName 300 | 301 | case &credentials: 302 | if nil != credentials { 303 | continue 304 | } 305 | needvar(&credentialPath) 306 | credentials, _ = auth.ReadCredentials(credentialPath) 307 | if nil == credentials { 308 | usageWithError( 309 | errors.New("unknown credentials; specify -credentials in the command line")) 310 | } 311 | 312 | case &storageName: 313 | if "" != storageName { 314 | continue 315 | } 316 | usageWithError(errors.New("unknown storage; specify -storage in the command line")) 317 | 318 | case &storage: 319 | if nil != storage { 320 | continue 321 | } 322 | var creds interface{} 323 | if "" != authName { 324 | needvar(&authSession, &storageName) 325 | creds = authSession 326 | } else { 327 | needvar(&credentials, &storageName) 328 | creds = credentials 329 | } 330 | s, err := objio.Registry.NewObject(storageName, storageUri, creds) 331 | if nil != err { 332 | fail(err) 333 | } 334 | storage = s.(objio.ObjectStorage) 335 | if trace.Verbose { 336 | storage = &objio.TraceObjectStorage{ObjectStorage: storage} 337 | } 338 | } 339 | } 340 | } 341 | 342 | func warn(err error) { 343 | fmt.Fprintf(os.Stderr, "error: %v (%v)\n", err, errno.ErrnoFromErr(err)) 344 | } 345 | 346 | func fail(err error) { 347 | warn(err) 348 | exit(1) 349 | } 350 | 351 | type exitcode int 352 | 353 | func exit(c int) { 354 | panic(exitcode(c)) 355 | } 356 | 357 | func run(self *cmd.CmdMap, flagSet *flag.FlagSet, args []string) (ec int) { 358 | defer func() { 359 | if r := recover(); nil != r { 360 | if c, ok := r.(exitcode); ok { 361 | ec = int(c) 362 | } else if _, ok := r.(error); ok { 363 | ec = 2 364 | } else { 365 | panic(r) 366 | } 367 | } 368 | }() 369 | 370 | flagSet.Parse(args) 371 | arg := flagSet.Arg(0) 372 | cmd := self.Get(arg) 373 | 374 | if nil == cmd { 375 | if "help" == arg { 376 | args = flagSet.Args()[1:] 377 | if 0 == len(args) { 378 | flagSet.Usage() 379 | } else { 380 | for _, name := range args { 381 | cmd := self.Get(name) 382 | if nil == cmd { 383 | continue 384 | } 385 | cmd.Flag.Usage() 386 | } 387 | } 388 | } else { 389 | flagSet.Usage() 390 | } 391 | exit(2) 392 | } 393 | 394 | cmd.Main(cmd, flagSet.Args()[1:]) 395 | return 396 | } 397 | 398 | func addcmd(self *cmd.CmdMap, name string, main func(*cmd.Cmd, []string)) (cmd *cmd.Cmd) { 399 | c := self.Add(name, main) 400 | c.Flag.Init(c.Flag.Name(), flag.PanicOnError) 401 | return c 402 | } 403 | 404 | func main() { 405 | ec := run(cmd.DefaultCmdMap, flag.CommandLine, os.Args[1:]) 406 | if 0 != ec { 407 | os.Exit(ec) 408 | } 409 | } 410 | -------------------------------------------------------------------------------- /objio/objio.go: -------------------------------------------------------------------------------- 1 | /* 2 | * objio.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package objio 19 | 20 | import ( 21 | "io" 22 | "time" 23 | 24 | "github.com/billziss-gh/objfs/objreg" 25 | ) 26 | 27 | // StorageInfo contains information about the storage. 28 | type StorageInfo interface { 29 | // determines if the storage is case-insensitive 30 | IsCaseInsensitive() bool 31 | 32 | // determines if the storage is read-only 33 | IsReadOnly() bool 34 | 35 | // maximum object name component length 36 | MaxComponentLength() int 37 | 38 | // total storage size 39 | TotalSize() int64 40 | 41 | // free storage size 42 | FreeSize() int64 43 | } 44 | 45 | // ObjectInfo contains information about an object. 46 | type ObjectInfo interface { 47 | // object name (no path) 48 | Name() string 49 | 50 | // object size 51 | Size() int64 52 | 53 | // object birth time 54 | Btime() time.Time 55 | 56 | // object modification time 57 | Mtime() time.Time 58 | 59 | // isdir flag 60 | IsDir() bool 61 | 62 | // object signature 63 | Sig() string 64 | } 65 | 66 | // WriteWaiter wraps a WriteCloser and a Wait method that waits until 67 | // all transfers are complete. After Wait has been called no further 68 | // Write's are possible and Close must be called. Calling Close without 69 | // Wait cancels any pending tranfers. 70 | type WriteWaiter interface { 71 | io.WriteCloser 72 | Wait() (ObjectInfo, error) 73 | } 74 | 75 | // ObjectStorage is the interface that an object storage must implement. 76 | // It provides methods to list, open and manipulate objects. 77 | type ObjectStorage interface { 78 | // Info gets storage information. The getsize parameter instructs the 79 | // implementation to also contact the object storage for size information. 80 | Info(getsize bool) (StorageInfo, error) 81 | 82 | // List lists all objects that have names with the specified prefix. 83 | // A marker can be used to continue a paginated listing. The listing 84 | // will contain up to maxcount items; a 0 specifies no limit (but the 85 | // underlying storage may still limit the number of items returned). 86 | // List returns an (optionally empty) marker and a slice of ObjectInfo. 87 | List(prefix string, marker string, maxcount int) (string, []ObjectInfo, error) 88 | 89 | // Stat gets object information. 90 | Stat(name string) (ObjectInfo, error) 91 | 92 | // Mkdir makes an object directory if the storage supports it. 93 | Mkdir(prefix string) (ObjectInfo, error) 94 | 95 | // Rmdir removes an object directory if the storage supports it. 96 | Rmdir(prefix string) error 97 | 98 | // Remove deletes an object from storage. 99 | Remove(name string) error 100 | 101 | // Rename renames an object. 102 | Rename(oldname string, newname string) error 103 | 104 | // OpenRead opens an object for reading. If sig is not empty, OpenRead 105 | // opens the object only if its current signature is different from sig. 106 | // It returns the current object info and an io.ReadCloser or any error; 107 | // if the object is not opened because of a matching non-empty sig, a nil 108 | // io.ReadCloser and nil error are returned. 109 | // 110 | // The returned io.ReadCloser may also support the io.ReaderAt interface. 111 | OpenRead(name string, sig string) (ObjectInfo, io.ReadCloser, error) 112 | 113 | // OpenWrite opens an object for writing. The parameter size specifies 114 | // the size that the written object will have. 115 | OpenWrite(name string, size int64) (WriteWaiter, error) 116 | } 117 | 118 | // Registry is the default object storage factory registry. 119 | var Registry = objreg.NewObjectFactoryRegistry() 120 | -------------------------------------------------------------------------------- /objio/tracestg.go: -------------------------------------------------------------------------------- 1 | /* 2 | * tracestg.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package objio 19 | 20 | import ( 21 | "fmt" 22 | "io" 23 | 24 | "github.com/billziss-gh/golib/trace" 25 | ) 26 | 27 | // TraceObjectStorage wraps a storage and traces calls to it. 28 | type TraceObjectStorage struct { 29 | ObjectStorage 30 | } 31 | 32 | func traceStg(val0 interface{}, vals ...interface{}) func(vals ...interface{}) { 33 | return trace.Trace(1, fmt.Sprintf("{{yellow}}%T{{off}}", val0), vals...) 34 | } 35 | 36 | func (self *TraceObjectStorage) Info(getsize bool) (info StorageInfo, err error) { 37 | defer traceStg(self.ObjectStorage, getsize)(traceWrap{&info}, traceWrap{&err}) 38 | return self.ObjectStorage.Info(getsize) 39 | } 40 | 41 | func (self *TraceObjectStorage) List( 42 | prefix string, imarker string, maxcount int) ( 43 | omarker string, infos []ObjectInfo, err error) { 44 | defer traceStg( 45 | self.ObjectStorage, prefix, imarker, maxcount)( 46 | &omarker, traceWrap{&infos}, traceWrap{&err}) 47 | return self.ObjectStorage.List(prefix, imarker, maxcount) 48 | } 49 | 50 | func (self *TraceObjectStorage) Stat(name string) (info ObjectInfo, err error) { 51 | defer traceStg(self.ObjectStorage, name)(traceWrap{&info}, traceWrap{&err}) 52 | return self.ObjectStorage.Stat(name) 53 | } 54 | 55 | func (self *TraceObjectStorage) Mkdir(prefix string) (info ObjectInfo, err error) { 56 | defer traceStg(self.ObjectStorage, prefix)(traceWrap{&info}, traceWrap{&err}) 57 | return self.ObjectStorage.Mkdir(prefix) 58 | } 59 | 60 | func (self *TraceObjectStorage) Rmdir(prefix string) (err error) { 61 | defer traceStg(self.ObjectStorage, prefix)(traceWrap{&err}) 62 | return self.ObjectStorage.Rmdir(prefix) 63 | } 64 | 65 | func (self *TraceObjectStorage) Remove(name string) (err error) { 66 | defer traceStg(self.ObjectStorage, name)(traceWrap{&err}) 67 | return self.ObjectStorage.Remove(name) 68 | } 69 | 70 | func (self *TraceObjectStorage) Rename(oldname string, newname string) (err error) { 71 | defer traceStg(self.ObjectStorage, oldname, newname)(traceWrap{&err}) 72 | return self.ObjectStorage.Rename(oldname, newname) 73 | } 74 | 75 | func (self *TraceObjectStorage) OpenRead( 76 | name string, sig string) ( 77 | info ObjectInfo, reader io.ReadCloser, err error) { 78 | defer traceStg(self.ObjectStorage, name, sig)(traceWrap{&info}, traceWrap{&err}) 79 | return self.ObjectStorage.OpenRead(name, sig) 80 | } 81 | 82 | func (self *TraceObjectStorage) OpenWrite( 83 | name string, size int64) ( 84 | writer WriteWaiter, err error) { 85 | defer traceStg(self.ObjectStorage, name, size)(traceWrap{&err}) 86 | writer, err = self.ObjectStorage.OpenWrite(name, size) 87 | if nil == err { 88 | writer = &traceWriteWaiter{writer} 89 | } 90 | return 91 | } 92 | 93 | type traceWriteWaiter struct { 94 | WriteWaiter 95 | } 96 | 97 | func (self *traceWriteWaiter) Wait() (info ObjectInfo, err error) { 98 | defer traceStg(self.WriteWaiter)(traceWrap{&info}, traceWrap{&err}) 99 | return self.WriteWaiter.Wait() 100 | } 101 | 102 | type traceWrap struct { 103 | v interface{} 104 | } 105 | 106 | func (t traceWrap) GoString() string { 107 | switch i := t.v.(type) { 108 | case *error: 109 | if nil == *i { 110 | return "{{bold green}}OK{{off}}" 111 | } 112 | return fmt.Sprintf("{{bold red}}error(\"%v\"){{off}}", *i) 113 | case *StorageInfo: 114 | return fmt.Sprintf("%#v", *i) 115 | case *ObjectInfo: 116 | return fmt.Sprintf("%#v", *i) 117 | case *[]ObjectInfo: 118 | return fmt.Sprintf("%T (len=%d)", i, len(*i)) 119 | default: 120 | return fmt.Sprintf("%#v", t.v) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /objreg/objreg.go: -------------------------------------------------------------------------------- 1 | /* 2 | * objreg.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package objreg 19 | 20 | import ( 21 | "sync" 22 | 23 | "github.com/billziss-gh/golib/errors" 24 | "github.com/billziss-gh/objfs/errno" 25 | ) 26 | 27 | // ObjectFactory acts as a constructor for a class of objects. 28 | type ObjectFactory func(args ...interface{}) (interface{}, error) 29 | 30 | // ObjectFactoryRegistry maintains a mapping of names to object factories. 31 | type ObjectFactoryRegistry struct { 32 | reg map[string]ObjectFactory 33 | mux sync.Mutex 34 | } 35 | 36 | // RegisterFactory registers an object factory. 37 | func (self *ObjectFactoryRegistry) RegisterFactory(name string, factory ObjectFactory) { 38 | self.mux.Lock() 39 | defer self.mux.Unlock() 40 | self.reg[name] = factory 41 | } 42 | 43 | // UnregisterFactory unregisters an object factory. 44 | func (self *ObjectFactoryRegistry) UnregisterFactory(name string) { 45 | self.mux.Lock() 46 | defer self.mux.Unlock() 47 | delete(self.reg, name) 48 | } 49 | 50 | // GetNames gets the factory names. 51 | func (self *ObjectFactoryRegistry) GetNames() []string { 52 | self.mux.Lock() 53 | defer self.mux.Unlock() 54 | names := make([]string, len(self.reg)) 55 | i := 0 56 | for n := range self.reg { 57 | names[i] = n 58 | i++ 59 | } 60 | return names 61 | } 62 | 63 | // GetFactory gets an object factory by name. 64 | func (self *ObjectFactoryRegistry) GetFactory(name string) ObjectFactory { 65 | self.mux.Lock() 66 | defer self.mux.Unlock() 67 | return self.reg[name] 68 | } 69 | 70 | // NewObject creates an object using its registered object factory. 71 | func (self *ObjectFactoryRegistry) NewObject( 72 | name string, args ...interface{}) (interface{}, error) { 73 | factory := self.GetFactory(name) 74 | if nil == factory { 75 | return nil, errors.New(": unknown object factory "+name, nil, errno.EINVAL) 76 | } 77 | return factory(args...) 78 | } 79 | 80 | // NewObjectFactoryRegistry creates a new object factory registry. 81 | func NewObjectFactoryRegistry() *ObjectFactoryRegistry { 82 | return &ObjectFactoryRegistry{ 83 | reg: map[string]ObjectFactory{}, 84 | mux: sync.Mutex{}, 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /objreg/objreg_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * objreg_test.go 3 | * 4 | * Copyright 2018 Bill Zissimopoulos 5 | */ 6 | /* 7 | * This file is part of Objfs. 8 | * 9 | * You can redistribute it and/or modify it under the terms of the GNU 10 | * Affero General Public License version 3 as published by the Free 11 | * Software Foundation. 12 | * 13 | * Licensees holding a valid commercial license may use this file in 14 | * accordance with the commercial license agreement provided with the 15 | * software. 16 | */ 17 | 18 | package objreg 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/billziss-gh/golib/errors" 24 | "github.com/billziss-gh/objfs/errno" 25 | ) 26 | 27 | func fooCtor(args ...interface{}) (interface{}, error) { 28 | return "foo", nil 29 | } 30 | 31 | func errCtor(args ...interface{}) (interface{}, error) { 32 | return nil, errors.New("err") 33 | } 34 | 35 | func TestObjreg(t *testing.T) { 36 | registry := NewObjectFactoryRegistry() 37 | registry.RegisterFactory("foo", fooCtor) 38 | registry.RegisterFactory("err", errCtor) 39 | 40 | if nil == registry.GetFactory("foo") { 41 | t.Error() 42 | } 43 | if nil != registry.GetFactory("bar") { 44 | t.Error() 45 | } 46 | if nil == registry.GetFactory("err") { 47 | t.Error() 48 | } 49 | 50 | obj, err := registry.NewObject("foo") 51 | if "foo" != obj || nil != err { 52 | t.Error() 53 | } 54 | obj, err = registry.NewObject("bar") 55 | if nil != obj || nil == err || errno.EINVAL != errors.Attachment(err) { 56 | t.Error() 57 | } 58 | obj, err = registry.NewObject("err") 59 | if nil != obj || nil == err || nil != errors.Attachment(err) || "err" != err.Error() { 60 | t.Error() 61 | } 62 | 63 | registry.UnregisterFactory("foo") 64 | if nil != registry.GetFactory("foo") { 65 | t.Error() 66 | } 67 | obj, err = registry.NewObject("foo") 68 | if nil != obj || nil == err || errno.EINVAL != errors.Attachment(err) { 69 | t.Error() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /registry.go: -------------------------------------------------------------------------------- 1 | // registry.go is autogenerated from registry.go.in 2 | 3 | /* 4 | * registry.go 5 | * 6 | * Copyright 2018 Bill Zissimopoulos 7 | */ 8 | /* 9 | * This file is part of Objfs. 10 | * 11 | * You can redistribute it and/or modify it under the terms of the GNU 12 | * Affero General Public License version 3 as published by the Free 13 | * Software Foundation. 14 | * 15 | * Licensees holding a valid commercial license may use this file in 16 | * accordance with the commercial license agreement provided with the 17 | * software. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "github.com/billziss-gh/objfs/fs/objfs" 24 | 25 | "github.com/billziss-gh/objfs.pkg/objio/onedrive" 26 | "github.com/billziss-gh/objfs.pkg/objio/dropbox" 27 | ) 28 | 29 | const defaultStorageName = "onedrive" 30 | 31 | func init() { 32 | objfs.Load() 33 | 34 | onedrive.Load() 35 | dropbox.Load() 36 | } 37 | -------------------------------------------------------------------------------- /registry.go.in: -------------------------------------------------------------------------------- 1 | // registry.go is autogenerated from registry.go.in 2 | 3 | /* 4 | * registry.go 5 | * 6 | * Copyright 2018 Bill Zissimopoulos 7 | */ 8 | /* 9 | * This file is part of Objfs. 10 | * 11 | * You can redistribute it and/or modify it under the terms of the GNU 12 | * Affero General Public License version 3 as published by the Free 13 | * Software Foundation. 14 | * 15 | * Licensees holding a valid commercial license may use this file in 16 | * accordance with the commercial license agreement provided with the 17 | * software. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "github.com/billziss-gh/objfs/fs/objfs" 24 | {{range .}} 25 | "{{stripVendor .ImportPath}}" 26 | {{- end}} 27 | ) 28 | 29 | const defaultStorageName = "{{with index . 0}}{{.Name}}{{end}}" 30 | 31 | func init() { 32 | objfs.Load() 33 | {{range .}} 34 | {{.Name}}.Load() 35 | {{- end}} 36 | } 37 | --------------------------------------------------------------------------------