├── .coveragerc ├── .flake8 ├── .gitignore ├── .gitmodules ├── CHANGES.txt ├── README.rst ├── acidfs └── __init__.py ├── docs ├── Makefile ├── _static │ └── placeholder ├── conf.py ├── index.rst └── make.bat ├── noxfile.py ├── setup.cfg ├── setup.py └── tests.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | 4 | [report] 5 | fail_under = 100 6 | show_missing = True 7 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | [flake8] 18 | ignore = E203, E266, E501, W503 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | .*.swp 4 | .cache 5 | .coverage 6 | .ropeproject 7 | .tox 8 | dist 9 | venv 10 | docs/_build 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "docs/_themes"] 2 | path = docs/_themes 3 | url = git://github.com/Pylons/pylons_sphinx_theme.git 4 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | Change Log 2 | ========== 3 | 4 | 2.0 (2022-01-18) 5 | ---------------- 6 | 7 | - Fix bug where commits without changes could produce extraneous git commits in 8 | repository. 9 | 10 | 1.1 (2022-01-08) 11 | ---------------- 12 | 13 | - Fix bug with spaces in directory names. 14 | 15 | - Fix bug where calling setUser with '' as path (as pyramid_tm does) would 16 | cause an exception when committing the transaction. 17 | 18 | 1.0 (2013-01-03) 19 | ---------------- 20 | 21 | Initial release. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | AcidFS 3 | ====== 4 | 5 | AcidFS allows interaction with the filesystem using transactions with ACID 6 | semantics. `Git` is used as a back end, and `AcidFS` integrates with the 7 | `transaction `_ package allowing use of 8 | multiple databases in a single transaction. AcidFS makes concurrent persistent 9 | to the filesystem safe and reliable. 10 | 11 | Full documentation is available at `Read the Docs 12 | `_. 13 | -------------------------------------------------------------------------------- /acidfs/__init__.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import fcntl 3 | import io 4 | import logging 5 | import os 6 | import shutil 7 | import subprocess 8 | import tempfile 9 | import traceback 10 | import transaction 11 | import weakref 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | class AcidFS(object): 17 | """ 18 | An instance of `AcidFS` exposes a transactional filesystem view of a `Git` 19 | repository. Instances of `AcidFS` are not threadsafe and should not be 20 | shared across threads, greenlets, etc. 21 | 22 | **Paths** 23 | 24 | Many methods take a `path` as an argument. All paths use forward slash `/` 25 | as a separator, regardless of the path separator of the 26 | underlying operating system. The path `/` represents the root folder of 27 | the repository. Paths may be relative or absolute: paths beginning with a 28 | `/` are absolute with respect to the repository root, paths not beginning 29 | with a `/` are relative to the current working directory. The current 30 | working directory always starts at the root of the repository. The current 31 | working directory can be changed using the :meth:`chdir` and 32 | :meth:`cd` methods. 33 | 34 | **Constructor Arguments** 35 | 36 | ``repo`` 37 | 38 | The path to the repository in the real, local filesystem. 39 | 40 | ``head`` 41 | 42 | The name of a branch to use as the head for this transaction. Changes 43 | made using this instance will be merged to the given head. The default, 44 | if omitted, is to use the repository's current head. 45 | 46 | ``create`` 47 | 48 | If there is not a Git repository in the indicated directory, should one 49 | be created? The default is `True`. 50 | 51 | ``bare`` 52 | 53 | If the Git repository is to be created, create it as a bare repository. 54 | If the repository is already created or `create` is False, this argument 55 | has no effect. 56 | 57 | ``user_name`` 58 | 59 | If the Git repository is to be created, set the user name for the 60 | repository to this value. This is the same as creating the repository 61 | and running `git config user.name ""`. 62 | 63 | ``user_email`` 64 | 65 | If the Git repository is to be created, set the user email for the 66 | repository to this value. This is the same as creating the repository 67 | and running `git config user.email ""`. 68 | 69 | ``name`` 70 | 71 | Name to be used as a sort key when ordering the various databases 72 | (datamanagers in the parlance of the transaction package) during a 73 | commit. It is exceedingly rare that you would need to use anything other 74 | than the default, here. 75 | 76 | ``path_encoding`` 77 | 78 | Encode paths with this encoding. The default is `ascii`. 79 | """ 80 | 81 | session = None 82 | _cwd = () 83 | 84 | def __init__( 85 | self, 86 | repo, 87 | head="HEAD", 88 | create=True, 89 | bare=False, 90 | user_name=None, 91 | user_email=None, 92 | name="AcidFS", 93 | path_encoding="ascii", 94 | ): 95 | wdpath = repo 96 | dbpath = os.path.join(repo, ".git") 97 | if not os.path.exists(dbpath): 98 | wdpath = None 99 | dbpath = repo 100 | if create: 101 | args = ["git", "init", repo] 102 | if bare: 103 | args.append("--bare") 104 | else: 105 | wdpath = repo 106 | dbpath = os.path.join(repo, ".git") 107 | _check_output(args) 108 | if user_name: 109 | args = ["git", "config", "user.name", user_name] 110 | _check_output(args, cwd=dbpath) 111 | if user_email: 112 | args = ["git", "config", "user.email", user_email] 113 | _check_output(args, cwd=dbpath) 114 | args = ["git", "config", "core.quotepath", "false"] 115 | _check_output(args, cwd=dbpath) 116 | else: 117 | raise ValueError("No database found in %s" % dbpath) 118 | 119 | self.wd = wdpath 120 | self.db = dbpath 121 | self.head = head 122 | self.name = name 123 | self.path_encoding = path_encoding 124 | 125 | def _session(self): 126 | """ 127 | Make sure we're in a session. 128 | """ 129 | if not self.session or self.session.closed: 130 | self.session = _Session( 131 | self.wd, self.db, self.head, self.name, self.path_encoding 132 | ) 133 | return self.session 134 | 135 | def _mkpath(self, path): 136 | if path == ".": 137 | parsed = [] 138 | else: 139 | parsed = list(filter(None, path.split("/"))) 140 | if not path.startswith("/"): 141 | parsed = list(self._cwd) + parsed 142 | return parsed 143 | 144 | def get_base(self): 145 | """ 146 | Returns the id of the commit that is the current base for the 147 | transaction. 148 | """ 149 | session = self._session() 150 | return session.prev_commit 151 | 152 | def set_base(self, commit): 153 | """ 154 | Sets the base commit for the current transaction. The `commit` 155 | argument may be the SHA1 of a commit or the name of a reference (eg. 156 | branch or tag). The current transaction must be clean. If any changes 157 | have been made in the transaction, a ConflictError will be raised. 158 | """ 159 | session = self._session() 160 | session.set_base(commit) 161 | 162 | def cwd(self): 163 | """ 164 | Returns the path to the current working directory in the repository. 165 | """ 166 | return "/" + "/".join(self._cwd) 167 | 168 | def chdir(self, path): 169 | """ 170 | Change the current working directory in repository. 171 | """ 172 | session = self._session() 173 | parsed = self._mkpath(path) 174 | obj = session.find(parsed) 175 | if not obj: 176 | raise _NoSuchFileOrDirectory(path) 177 | if not isinstance(obj, _TreeNode): 178 | raise _NotADirectory(path) 179 | self._cwd = parsed 180 | 181 | @contextlib.contextmanager 182 | def cd(self, path): 183 | """ 184 | A context manager that changes the current working directory only in 185 | the scope of the 'with' context. Eg:: 186 | 187 | import acidfs 188 | 189 | fs = acidfs.AcidFS('myrepo') 190 | with fs.cd('some/folder'): 191 | fs.open('a/file') # relative to /some/folder 192 | fs.open('another/file') # relative to / 193 | """ 194 | prev = self._cwd 195 | self.chdir(path) 196 | yield 197 | self._cwd = prev 198 | 199 | def open( 200 | self, path, mode="r", buffering=-1, encoding=None, errors=None, newline=None 201 | ): 202 | """ 203 | Open a file for reading or writing. 204 | 205 | Implements the semantics of the `open` function in Python's `io module 206 | `_, which is the 207 | default implementation `in Python 3 208 | `_. Opening a 209 | file in text mode will return a file-like object which reads or writes 210 | unicode strings, while opening a file in binary mode will return a 211 | file-like object which reads or writes raw bytes. 212 | 213 | Because the underlying implementation uses a pipe to a `Git` plumbing 214 | command, opening for update (read and write) is not supported, nor is 215 | seeking. 216 | """ 217 | session = self._session() 218 | parsed = self._mkpath(path) 219 | 220 | if "b" in mode: 221 | text = False 222 | if "t" in mode: 223 | raise ValueError("can't have text and binary mode at once") 224 | else: 225 | if not buffering: 226 | raise ValueError("can't have unbuffered text I/O") 227 | text = True 228 | 229 | if "+" in mode: 230 | raise ValueError("Read/write mode is not supported") 231 | 232 | mode = mode.replace("b", "") 233 | mode = mode.replace("t", "") 234 | if mode == "a": 235 | mode = "w" 236 | append = True 237 | else: 238 | append = False 239 | if mode == "x": 240 | mode = "w" 241 | exclusive = True 242 | else: 243 | exclusive = False 244 | 245 | if buffering < 0: 246 | buffer_size = io.DEFAULT_BUFFER_SIZE 247 | line_buffering = False 248 | elif buffering == 1: 249 | buffer_size = io.DEFAULT_BUFFER_SIZE 250 | line_buffering = True 251 | else: 252 | buffer_size = buffering 253 | line_buffering = False 254 | 255 | if mode == "r": 256 | obj = session.find(parsed) 257 | if not obj: 258 | raise _NoSuchFileOrDirectory(path) 259 | if isinstance(obj, _TreeNode): 260 | raise _IsADirectory(path) 261 | stream = obj.open() 262 | if buffering: 263 | stream = io.BufferedReader(stream, buffer_size) 264 | if text: 265 | stream = io.TextIOWrapper( 266 | stream, encoding, errors, newline, line_buffering 267 | ) 268 | return stream 269 | 270 | elif mode == "w": 271 | if not parsed: 272 | raise _IsADirectory(path) 273 | name = parsed[-1] 274 | dirpath = parsed[:-1] 275 | obj = session.find(dirpath) 276 | if not obj: 277 | raise _NoSuchFileOrDirectory(path) 278 | if not isinstance(obj, _TreeNode): 279 | raise _NotADirectory(path) 280 | prev = obj.get(name) 281 | if isinstance(prev, _TreeNode): 282 | raise _IsADirectory(path) 283 | if prev and exclusive: 284 | raise _FileExists(path) 285 | blob = obj.new_blob(name, prev) 286 | if append and prev: 287 | shutil.copyfileobj(prev.open(), blob) 288 | if buffering: 289 | blob = io.BufferedWriter(blob, buffer_size) 290 | if text: 291 | blob = io.TextIOWrapper(blob, encoding, errors, newline, line_buffering) 292 | return blob 293 | 294 | raise ValueError("Bad mode: %s" % mode) 295 | 296 | def hash(self, path=""): 297 | """ 298 | Returns the sha1 hash of the object referred to by `path`. If `path` is 299 | omitted the current working directory is used. 300 | """ 301 | session = self._session() 302 | obj = session.find(self._mkpath(path)) 303 | if not obj: 304 | raise _NoSuchFileOrDirectory(path) 305 | return obj.hash() 306 | 307 | def listdir(self, path=""): 308 | """ 309 | Return list of files in indicated directory. If `path` is omitted, the 310 | current working directory is used. 311 | """ 312 | session = self._session() 313 | obj = session.find(self._mkpath(path)) 314 | if not obj: 315 | raise _NoSuchFileOrDirectory(path) 316 | if not isinstance(obj, _TreeNode): 317 | raise _NotADirectory(path) 318 | return list(obj.contents.keys()) 319 | 320 | def mkdir(self, path): 321 | """ 322 | Create a new directory. The parent of the new directory must already 323 | exist. 324 | """ 325 | session = self._session() 326 | parsed = self._mkpath(path) 327 | name = parsed[-1] 328 | 329 | parent = session.find(parsed[:-1]) 330 | if not parent: 331 | raise _NoSuchFileOrDirectory(path) 332 | if not isinstance(parent, _TreeNode): 333 | raise _NotADirectory(path) 334 | if name in parent.contents: 335 | raise _FileExists(path) 336 | 337 | parent.new_tree(name) 338 | 339 | def mkdirs(self, path): 340 | """ 341 | Create a new directory, including any ancestors which need to be created 342 | in order to create the directory with the given `path`. 343 | """ 344 | session = self._session() 345 | parsed = self._mkpath(path) 346 | node = session.tree 347 | for name in parsed: 348 | next_node = node.get(name) 349 | if not next_node: 350 | next_node = node.new_tree(name) 351 | elif not isinstance(next_node, _TreeNode): 352 | raise _NotADirectory(path) 353 | node = next_node 354 | 355 | def rm(self, path): 356 | """ 357 | Remove a single file. 358 | """ 359 | session = self._session() 360 | parsed = self._mkpath(path) 361 | 362 | obj = session.find(parsed) 363 | if not obj: 364 | raise _NoSuchFileOrDirectory(path) 365 | if isinstance(obj, _TreeNode): 366 | raise _IsADirectory(path) 367 | obj.parent.remove(obj.name) 368 | 369 | def rmdir(self, path): 370 | """ 371 | Remove a single directory. The directory must be empty. 372 | """ 373 | session = self._session() 374 | parsed = self._mkpath(path) 375 | 376 | if not parsed: 377 | raise ValueError("Can't remove root directory.") 378 | 379 | obj = session.find(parsed) 380 | if not obj: 381 | raise _NoSuchFileOrDirectory(path) 382 | if not isinstance(obj, _TreeNode): 383 | raise _NotADirectory(path) 384 | if not obj.empty(): 385 | raise _DirectoryNotEmpty(path) 386 | 387 | obj.parent.remove(obj.name) 388 | 389 | def rmtree(self, path): 390 | """ 391 | Remove a directory and any of its contents. 392 | """ 393 | session = self._session() 394 | parsed = self._mkpath(path) 395 | 396 | if not parsed: 397 | raise ValueError("Can't remove root directory.") 398 | 399 | obj = session.find(parsed) 400 | if not obj: 401 | raise _NoSuchFileOrDirectory(path) 402 | if not isinstance(obj, _TreeNode): 403 | raise _NotADirectory(path) 404 | 405 | obj.parent.remove(obj.name) 406 | 407 | def mv(self, src, dst): 408 | """ 409 | Move a file or directory from `src` path to `dst` path. 410 | """ 411 | session = self._session() 412 | spath = self._mkpath(src) 413 | if not spath: 414 | raise _NoSuchFileOrDirectory(src) 415 | sname = spath[-1] 416 | sfolder = session.find(spath[:-1]) 417 | if not sfolder or sname not in sfolder: 418 | raise _NoSuchFileOrDirectory(src) 419 | 420 | dpath = self._mkpath(dst) 421 | dobj = session.find(dpath) 422 | if not dobj: 423 | dname = dpath[-1] 424 | dfolder = session.find(dpath[:-1]) 425 | if dfolder: 426 | dfolder.set(dname, sfolder.remove(sname)) 427 | return 428 | raise _NoSuchFileOrDirectory(dst) 429 | if isinstance(dobj, _TreeNode): 430 | dobj.set(sname, sfolder.remove(sname)) 431 | else: 432 | dobj.parent.set(dobj.name, sfolder.remove(sname)) 433 | 434 | def exists(self, path): 435 | """ 436 | Returns boolean indicating whether a file or directory exists at the 437 | given `path`. 438 | """ 439 | session = self._session() 440 | return bool(session.find(self._mkpath(path))) 441 | 442 | def isdir(self, path): 443 | """ 444 | Returns boolean indicating whether the given `path` is a directory. 445 | """ 446 | session = self._session() 447 | return isinstance(session.find(self._mkpath(path)), _TreeNode) 448 | 449 | def empty(self, path): 450 | """ 451 | Returns boolean indicating whether the directory indicated by `path` is 452 | empty. 453 | """ 454 | session = self._session() 455 | obj = session.find(self._mkpath(path)) 456 | if not obj: 457 | raise _NoSuchFileOrDirectory(path) 458 | if not isinstance(obj, _TreeNode): 459 | raise _NotADirectory(path) 460 | return obj.empty() 461 | 462 | 463 | class ConflictError(Exception): 464 | def __init__(self, msg="Unable to merge changes to repository."): 465 | super(ConflictError, self).__init__(msg) 466 | 467 | 468 | class _Session(object): 469 | closed = False 470 | lockfd = None 471 | 472 | def __init__(self, wd, db, head, name, path_encoding): 473 | self.wd = wd 474 | self.db = db 475 | self.name = name 476 | self.path_encoding = path_encoding 477 | self.lock_file = os.path.join(db, "acidfs.lock") 478 | transaction.get().join(self) 479 | 480 | curhead = open(os.path.join(db, "HEAD")).read().strip()[16:] 481 | if head == curhead: 482 | head = "HEAD" 483 | if head == "HEAD": 484 | self.headref = os.path.join(db, "refs", "heads", curhead) 485 | else: 486 | self.headref = os.path.join(db, "refs", "heads", head) 487 | self.head = head 488 | 489 | if os.path.exists(self.headref): 490 | # Existing head, get head revision 491 | self.prev_commit = _check_output( 492 | ["git", "rev-list", "--max-count=1", head], cwd=db 493 | ).strip() 494 | tree = _check_output( 495 | [ 496 | "git", 497 | "rev-parse", 498 | f"{self.prev_commit.decode('ascii')}^{{tree}}", 499 | ], 500 | cwd=self.db, 501 | ).strip() 502 | self.tree = _TreeNode.read(db, tree, path_encoding) 503 | else: 504 | # New head, no commits yet 505 | self.tree = _TreeNode(db, path_encoding) # empty tree 506 | self.prev_commit = None 507 | 508 | def set_base(self, ref): 509 | if self.tree.dirty: 510 | raise ConflictError( 511 | "Cannot set base when changes already made in transaction." 512 | ) 513 | self.prev_commit = _check_output( 514 | ["git", "rev-list", "--max-count=1", ref], cwd=self.db 515 | ).strip() 516 | self.tree = _TreeNode.read(self.db, self.prev_commit, self.path_encoding) 517 | 518 | def find(self, path): 519 | assert isinstance(path, (list, tuple)) 520 | return self.tree.find(path) 521 | 522 | def abort(self, tx): 523 | """ 524 | Part of datamanager API. 525 | """ 526 | self.close() 527 | 528 | def tpc_begin(self, tx): 529 | """ 530 | Part of datamanager API. 531 | """ 532 | 533 | def commit(self, tx): 534 | """ 535 | Part of datamanager API. 536 | """ 537 | 538 | def tpc_vote(self, tx): 539 | """ 540 | Part of datamanager API. 541 | """ 542 | if not self.tree.dirty: 543 | # Nothing to do 544 | return 545 | 546 | # Write tree to db 547 | tree_oid = self.tree.save() 548 | if self.tree.committed_oid == tree_oid: 549 | # Nothing actually changed 550 | self.tree.dirty = False 551 | return 552 | 553 | if self.prev_commit: 554 | parents = [self.prev_commit] 555 | else: 556 | parents = [] 557 | commit_oid = self.mkcommit(tx, tree_oid, parents) 558 | 559 | # Acquire an exclusive (aka write) lock for merge. 560 | self.acquire_lock() 561 | 562 | # If this is initial commit, there's not really anything to merge 563 | if not self.prev_commit: 564 | # Make sure there haven't been other commits 565 | if os.path.exists(self.headref): 566 | # This was to be the initial commit, but somebody got to it 567 | # first No idea how to try to resolve that one. Luckily it 568 | # will be very rare. 569 | raise ConflictError() 570 | 571 | # New commit is new head 572 | self.next_commit = commit_oid 573 | return 574 | 575 | # Find the merge base 576 | current = _check_output( 577 | ["git", "rev-list", "--max-count=1", "HEAD"], cwd=self.db 578 | ).strip() 579 | merge_base = _check_output( 580 | ["git", "merge-base", current, commit_oid], cwd=self.db 581 | ).strip() 582 | 583 | # If the merge base is the current commit, it means there have been no 584 | # intervening changes and we can just fast forward to the new commit. 585 | # This is the most common case. 586 | if merge_base == current: 587 | self.next_commit = commit_oid 588 | return 589 | 590 | # Darn it, now we have to actually try to merge 591 | self.merge(merge_base, current, tree_oid) 592 | self.next_commit = self.mkcommit( 593 | tx, self.tree.save(), [current, commit_oid], "Merge" 594 | ) 595 | 596 | def tpc_finish(self, tx): 597 | """ 598 | Part of datamanager API. 599 | """ 600 | if not self.tree.dirty: 601 | # Nothing to do 602 | return 603 | 604 | # Make our commit the new head 605 | if self.head == "HEAD": 606 | # Use git reset to update current head 607 | args = ["git", "reset", self.next_commit] 608 | if self.wd: 609 | args.append("--hard") 610 | cwd = self.wd 611 | else: 612 | args.append("--soft") 613 | cwd = self.db 614 | _check_output(args, cwd=cwd) 615 | 616 | else: 617 | # If not updating current head, just write the commit to the ref 618 | # file directly. 619 | reffile = os.path.join(self.db, "refs", "heads", self.head) 620 | with open(reffile, "wb") as f: 621 | f.write(self.next_commit) 622 | f.write(b"\n") 623 | 624 | self.close() 625 | 626 | def tpc_abort(self, tx): 627 | """ 628 | Part of datamanager API. 629 | """ 630 | self.close() 631 | 632 | def sortKey(self): 633 | return self.name 634 | 635 | def close(self): 636 | self.closed = True 637 | self.release_lock() 638 | 639 | def acquire_lock(self): 640 | assert not self.lockfd 641 | self.lockfd = fd = os.open(self.lock_file, os.O_WRONLY | os.O_CREAT) 642 | fcntl.lockf(fd, fcntl.LOCK_EX) 643 | 644 | def release_lock(self): 645 | fd = self.lockfd 646 | if fd is not None: 647 | fcntl.lockf(fd, fcntl.LOCK_UN) 648 | os.close(fd) 649 | self.lockfd = None 650 | 651 | def mkcommit(self, tx, tree_oid, parents, message=None): 652 | # Prepare metadata for commit 653 | if not message: 654 | message = tx.description 655 | if not message: 656 | message = "AcidFS transaction" 657 | gitenv = os.environ.copy() 658 | extension = tx._extension # "Official" API despite underscore 659 | user = extension.get("acidfs_user") 660 | if not user: 661 | user = extension.get("user") 662 | if not user: 663 | user = tx.user 664 | if user: 665 | if user.startswith(" "): 666 | user = user[1:] 667 | else: 668 | # strip Zope's "path" 669 | user = user.split(None, 1) 670 | if len(user) == 2: 671 | user = user[1] 672 | else: 673 | user = user[0] 674 | if user: 675 | gitenv["GIT_AUTHOR_NAME"] = gitenv["GIT_COMMITER_NAME"] = user 676 | 677 | email = extension.get("acidfs_email") 678 | if not email: 679 | email = extension.get("email") 680 | if email: 681 | gitenv["GIT_AUTHOR_EMAIL"] = gitenv["GIT_COMMITTER_EMAIL"] = gitenv[ 682 | "EMAIL" 683 | ] = email 684 | 685 | # Write commit to db 686 | args = ["git", "commit-tree", tree_oid, "-m", message] 687 | for parent in parents: 688 | args.append("-p") 689 | args.append(parent) 690 | return _check_output(args, cwd=self.db, env=gitenv).strip() 691 | 692 | def merge(self, base_oid, current, tree_oid): 693 | """ 694 | This attempts to interpret the output of 'git merge-tree', given the 695 | current head, the tree we're currently working on, and the nearest 696 | common ancestor commit (base_oid). 697 | 698 | I haven't found any documentation on the format of the output of 699 | 'git merge-tree' so this is basically reverse engineered from studying 700 | its output in different situations. I try to be as conservative as 701 | possible here and bail as soon as I hit anything I'm not 100% sure 702 | about. It is far preferable to raise a ConflictError than incorrectly 703 | merge. As such, the code below is peppered with assertions using the 704 | 'expect' function, which will raise a ConflictError if any of our 705 | expectations aren't met. I also attempt to log as much useful debug 706 | information as possible in the case of an unmet expectation, so I can go 707 | back and take into account more cases as they are encountered. 708 | 709 | The basic algorithm here is a finite state machine operating on the 710 | output of 'git merge-tree' one line at a time. This should be fairly 711 | memory efficient for even large changesets, with the caveat there may 712 | have been added a large binary file which contains few or no line break 713 | characters, which could cause a buffer to get large while scanning 714 | through the merge data. 715 | 716 | One might ask, why not use the porcelain 'git merge' command? One 717 | reason is, in the context of the two phase commit protocol, we'd rather 718 | do pretty much everything we possibly can in the voting stage, leaving 719 | ourselves with nothing to do in the finish phase except updating the 720 | head to the commit we just created, and possibly updating the working 721 | directory--operations that are guaranteed to work. Since 'git merge' 722 | will update the head, we'd prefer to do it during the final phase of the 723 | commit, but we can't guarantee it will work. There is not a convenient 724 | way to do a merge dry run during the voting phase. Although I can 725 | conceive of ways to do the merge during the voting phase and roll back 726 | to the previous head if we need to, that feels a little riskier. Doing 727 | the merge ourselves, here, also frees us from having to work with a 728 | working directory, required by the porcelain 'git merge' command. This 729 | means we can use bare repositories and/or have transactions that use 730 | a head other than the repositories 'current' head. 731 | 732 | In general, tranactions will be short and will not have much a of a 733 | chance to get very far behind the head, so merges will tend not to be 734 | terribly complicated. We should be able to handle the vast majority of 735 | cases here, even if there are some rare corner cases the porcelain 736 | command might be able to handle that we can't. I think that's a 737 | reasonable trade off for the flexibility this approach provides. 738 | 739 | Some dead/unreachable branches are left in here, just in case we haven't 740 | entirely characterized the behavior of 'git merge-tree'. These are marked with 741 | 'pragma NO COVER' and are easily recognized. 742 | """ 743 | with _popen( 744 | ["git", "merge-tree", base_oid, tree_oid, current], 745 | cwd=self.db, 746 | stdout=subprocess.PIPE, 747 | ) as proc: 748 | # Messy finite state machine 749 | state = None 750 | extra_state = None 751 | stream = proc.stdout 752 | line = stream.readline() 753 | 754 | def expect(expectation, *msg): 755 | if not expectation: # pragma no cover 756 | log.debug("Unmet expectation during merge.") 757 | log.debug("".join(traceback.format_stack())) 758 | if msg: 759 | log.debug(msg[0], *msg[1:]) 760 | if extra_state: 761 | log.debug("Extra state: %s", extra_state) 762 | raise ConflictError() 763 | 764 | while line: 765 | if state is None: # default, scanning for start of a change 766 | if line[0:1].isalpha(): 767 | # If first column is a letter, then we have the first 768 | # line of a change, which describes the change. 769 | line = line.strip() 770 | if line in ( 771 | b"added in local", 772 | b"removed in local", 773 | b"removed in both", 774 | ): # pragma NO COVER 775 | # This doesn't seem to come up in practice 776 | # We don't care about changes to our current tree. 777 | # We already know about those. 778 | pass 779 | 780 | elif line == b"added in remote": 781 | # The head got a new file, we should grab it 782 | state = _MERGE_ADDED_IN_REMOTE 783 | extra_state = [] 784 | 785 | elif line == b"removed in remote": 786 | # File got deleted from head, remove it 787 | state = _MERGE_REMOVED_IN_REMOTE 788 | extra_state = [] 789 | 790 | elif line == b"changed in both": 791 | # File was edited in both branches, see if we can 792 | # patch 793 | state = _MERGE_CHANGED_IN_BOTH 794 | extra_state = [] 795 | 796 | elif line == b"added in both": 797 | state = _MERGE_ADDED_IN_BOTH 798 | extra_state = [] 799 | 800 | else: # pragma NO COVER 801 | log.debug("Don't know how to merge: %s", line) 802 | raise ConflictError() 803 | 804 | elif state is _MERGE_ADDED_IN_REMOTE: 805 | if line[0:1].isalpha() or line.startswith(b"@"): 806 | # Done collecting tree lines, only expecting one 807 | expect(len(extra_state) == 1, "Wrong number of lines") 808 | whose, mode, oid, path = _parsetree(extra_state[0]) 809 | expect(whose == b"their", "Unexpected whose: %s", whose) 810 | expect(mode == b"100644", "Unexpected mode: %s", mode) 811 | parsed = path.decode("ascii").split("/") 812 | folder = self.find(parsed[:-1]) 813 | expect(isinstance(folder, _TreeNode), "Not a folder: %s", path) 814 | folder.set(parsed[-1], (b"blob", oid, None)) 815 | state = extra_state = None 816 | continue 817 | 818 | else: 819 | extra_state.append(line) 820 | 821 | elif state is _MERGE_REMOVED_IN_REMOTE: 822 | if line[0:1].isalpha() or line.startswith(b"@"): 823 | # Done collecting tree lines, expect two, one for base, 824 | # one for our copy, whose sha1s should match 825 | expect(len(extra_state) == 2, "Wrong number of lines") 826 | whose, mode, oid, path = _parsetree(extra_state[0]) 827 | expect( 828 | whose in (b"our", b"base"), "Unexpected whose: %s", whose 829 | ) 830 | expect(mode == b"100644", "Unexpected mode: %s", mode) 831 | whose, mode, oid2, path2 = _parsetree(extra_state[1]) 832 | expect( 833 | whose in (b"our", b"base"), "Unexpected whose: %s", whose 834 | ) 835 | expect(mode == b"100644", "Unexpected mode: %s", mode) 836 | expect(oid == oid2, "SHA1s don't match") 837 | expect(path == path2, "Paths don't match") 838 | path = path.decode("ascii").split("/") 839 | folder = self.find(path[:-1]) 840 | expect(isinstance(folder, _TreeNode), "Not a folder") 841 | folder.remove(path[-1]) 842 | state = extra_state = None 843 | continue 844 | 845 | else: 846 | extra_state.append(line) 847 | 848 | elif state is _MERGE_CHANGED_IN_BOTH: 849 | if line.startswith(b"@"): 850 | # Done collecting tree lines, expect three, one for base 851 | # and one for each copy 852 | expect(len(extra_state) == 3, "Wrong number of lines") 853 | whose, mode, oid, path = _parsetree(extra_state[0]) 854 | expect( 855 | whose in (b"base", b"our", b"their"), 856 | "Unexpected whose: %s", 857 | whose, 858 | ) 859 | expect(mode == b"100644", "Unexpected mode: %s", mode) 860 | for extra_line in extra_state[1:]: 861 | whose, mode, oid2, path2 = _parsetree(extra_line) 862 | expect( 863 | whose in (b"base", b"our", b"their"), 864 | "Unexpected whose: %s", 865 | whose, 866 | ) 867 | expect(mode == b"100644", "Unexpected mode: %s", mode) 868 | expect(path == path2, "Paths don't match") 869 | parsed = path.decode("ascii").split("/") 870 | folder = self.find(parsed[:-1]) 871 | expect(isinstance(folder, _TreeNode), "Not a folder") 872 | name = parsed[-1] 873 | blob = folder.get(name) 874 | expect(isinstance(blob, _Blob), "Not a blob") 875 | with _tempfile() as tmp: 876 | shutil.copyfileobj(blob.open(), open(tmp, "wb")) 877 | with _popen( 878 | ["patch", "-s", tmp, "-"], 879 | stdin=subprocess.PIPE, 880 | stdout=subprocess.PIPE, 881 | stderr=subprocess.PIPE, 882 | ) as p: 883 | f = p.stdin 884 | while line and not line[0:1].isalpha(): 885 | if line[1:9] == b"<<<<<<< ": 886 | raise ConflictError() 887 | f.write(line) 888 | line = stream.readline() 889 | newblob = folder.new_blob(name, blob) 890 | shutil.copyfileobj(open(tmp, "rb"), newblob) 891 | 892 | state = extra_state = None 893 | continue 894 | 895 | else: 896 | extra_state.append(line) 897 | 898 | elif state is _MERGE_ADDED_IN_BOTH: # pragma NO BRANCH 899 | # NO BRANCH pragma added to workaround what seems to be a bug in 900 | # coverage. This if..elif structure handles every case that's thrown 901 | # at it, but coverage seems concerned that there isn't a case that 902 | # doesn't get handled. 903 | if line[0:1].isalpha() or line.startswith(b"@"): 904 | # Done collecting tree lines, expect two, one for base, 905 | # one for our copy, whose sha1s should match 906 | expect(len(extra_state) == 2, "Wrong number of lines") 907 | whose, mode, oid, path = _parsetree(extra_state[0]) 908 | expect( 909 | whose in (b"our", b"their"), "Unexpected whose: %s", whose 910 | ) 911 | expect(mode == b"100644", "Unexpected mode: %s", mode) 912 | whose, mode, oid2, path2 = _parsetree(extra_state[1]) 913 | expect( 914 | whose in (b"our", b"their"), "Unexpected whose: %s", whose 915 | ) 916 | expect(mode == b"100644", "Unexpected mode: %s", mode) 917 | expect(path == path2, "Paths don't match") 918 | # Either it's the same file or a different file. 919 | if oid != oid2: 920 | # Different files, can't merge 921 | raise ConflictError() 922 | 923 | else: # pragma NO COVER 924 | # Seems to not come up. Probably merge-tree detects this and 925 | # doesn't bother us about it. 926 | # Same file, nothing to do 927 | state = extra_state = None 928 | continue 929 | 930 | else: 931 | extra_state.append(line) 932 | 933 | line = stream.readline() 934 | 935 | 936 | class _TreeNode(object): 937 | parent = None 938 | name = None 939 | dirty = False 940 | oid = None 941 | committed_oid = None 942 | 943 | @classmethod 944 | def read(cls, db, oid, path_encoding): 945 | node = cls(db, path_encoding) 946 | node.committed_oid = node.oid = oid 947 | contents = node.contents 948 | with _popen(["git", "ls-tree", oid], stdout=subprocess.PIPE, cwd=db) as lstree: 949 | for line in lstree.stdout.readlines(): 950 | mode, type, oid, name = _parsetree(line) 951 | name = name.decode(path_encoding) 952 | contents[name] = (type, oid, None) 953 | 954 | return node 955 | 956 | def __init__(self, db, path_encoding): 957 | self.db = db 958 | self.path_encoding = path_encoding 959 | self.contents = {} 960 | 961 | def get(self, name): 962 | contents = self.contents 963 | obj = contents.get(name) 964 | if not obj: 965 | return None 966 | type, oid, obj = obj 967 | assert type in (b"tree", b"blob") 968 | if not obj: 969 | if type == b"tree": 970 | obj = _TreeNode.read(self.db, oid, self.path_encoding) 971 | else: 972 | obj = _Blob(self.db, oid) 973 | obj.parent = self 974 | obj.name = name 975 | contents[name] = (type, oid, obj) 976 | return obj 977 | 978 | def find(self, path): 979 | if not path: 980 | return self 981 | obj = self.get(path[0]) 982 | if obj: 983 | return obj.find(path[1:]) 984 | 985 | def new_blob(self, name, prev): 986 | obj = _NewBlob(self.db, prev) 987 | obj.parent = self 988 | obj.name = name 989 | self.contents[name] = (b"blob", None, weakref.proxy(obj)) 990 | self.set_dirty() 991 | return obj 992 | 993 | def new_tree(self, name): 994 | node = _TreeNode(self.db, self.path_encoding) 995 | node.parent = self 996 | node.name = name 997 | self.contents[name] = (b"tree", None, node) 998 | self.set_dirty() 999 | return node 1000 | 1001 | def remove(self, name): 1002 | entry = self.contents.pop(name) 1003 | self.set_dirty() 1004 | return entry 1005 | 1006 | def set(self, name, entry): 1007 | self.contents[name] = entry 1008 | self.set_dirty() 1009 | 1010 | def set_dirty(self): 1011 | node = self 1012 | while node and not node.dirty: 1013 | node.oid = None 1014 | node.dirty = True 1015 | node = node.parent 1016 | 1017 | def save(self): 1018 | # Recursively save children, first 1019 | for name, (type, oid, obj) in list(self.contents.items()): 1020 | if not obj: 1021 | continue # Nothing to do 1022 | if isinstance(obj, _NewBlob): 1023 | raise ValueError("Cannot commit transaction with open files.") 1024 | elif type == b"tree" and (obj.dirty or not oid): 1025 | new_oid = obj.save() 1026 | self.contents[name] = (b"tree", new_oid, None) 1027 | 1028 | # Save tree object out to database 1029 | with _popen( 1030 | ["git", "mktree"], 1031 | cwd=self.db, 1032 | stdin=subprocess.PIPE, 1033 | stdout=subprocess.PIPE, 1034 | ) as proc: 1035 | for name, (type, oid, obj) in self.contents.items(): 1036 | proc.stdin.write(b"100644" if type == b"blob" else b"040000") 1037 | proc.stdin.write(b" ") 1038 | proc.stdin.write(type) 1039 | proc.stdin.write(b" ") 1040 | proc.stdin.write(oid) 1041 | proc.stdin.write(b"\t") 1042 | proc.stdin.write(name.encode(self.path_encoding)) 1043 | proc.stdin.write(b"\n") 1044 | proc.stdin.close() 1045 | oid = proc.stdout.read().strip() 1046 | self.oid = oid 1047 | return oid 1048 | 1049 | def empty(self): 1050 | return not self.contents 1051 | 1052 | def __contains__(self, name): 1053 | return name in self.contents 1054 | 1055 | def hash(self): 1056 | if not self.oid: 1057 | self.save() 1058 | return self.oid 1059 | 1060 | 1061 | def _parsetree(line): 1062 | return line.strip().split(None, 3) 1063 | 1064 | 1065 | class _Blob(object): 1066 | def __init__(self, db, oid): 1067 | self.db = db 1068 | self.oid = oid 1069 | 1070 | def open(self): 1071 | return _BlobStream(self.db, self.oid) 1072 | 1073 | def find(self, path): 1074 | if not path: 1075 | return self 1076 | 1077 | def hash(self): 1078 | return self.oid 1079 | 1080 | 1081 | class _NewBlob(io.RawIOBase): 1082 | def __init__(self, db, prev): 1083 | self.db = db 1084 | self.prev = prev 1085 | 1086 | self.proc = subprocess.Popen( 1087 | ["git", "hash-object", "-w", "--stdin"], 1088 | stdin=subprocess.PIPE, 1089 | stdout=subprocess.PIPE, 1090 | stderr=subprocess.STDOUT, 1091 | cwd=db, 1092 | ) 1093 | 1094 | def write(self, b): 1095 | self.proc.stdin.write(b) 1096 | return len(b) 1097 | 1098 | def close(self): 1099 | super(_NewBlob, self).close() 1100 | self.proc.stdin.close() 1101 | oid = self.proc.stdout.read().strip() 1102 | self.proc.stdout.close() 1103 | retcode = self.proc.wait() 1104 | if retcode != 0: 1105 | raise subprocess.CalledProcessError(retcode, "git hash-object -w --stdin") 1106 | self.parent.contents[self.name] = (b"blob", oid, None) 1107 | 1108 | def writable(self): 1109 | return True 1110 | 1111 | def open(self): 1112 | if self.prev: 1113 | return self.prev.open() 1114 | raise _NoSuchFileOrDirectory(_object_path(self)) 1115 | 1116 | def hash(self): 1117 | if self.prev: 1118 | return self.prev.hash() 1119 | raise _NoSuchFileOrDirectory(_object_path(self)) 1120 | 1121 | def find(self, path): 1122 | if not path: 1123 | return self 1124 | 1125 | 1126 | class _BlobStream(io.RawIOBase): 1127 | def __init__(self, db, oid): 1128 | self.proc = subprocess.Popen( 1129 | ["git", "cat-file", "blob", oid], 1130 | stdout=subprocess.PIPE, 1131 | stderr=subprocess.PIPE, 1132 | cwd=db, 1133 | ) 1134 | self.oid = oid 1135 | 1136 | def readable(self): 1137 | return True 1138 | 1139 | def read(self, n=-1): 1140 | return self.proc.stdout.read(n) 1141 | 1142 | def readinto(self, b): 1143 | """ 1144 | Although the documentation asserts a default implementation of this 1145 | method can be found in io.RawIOBase, there actually isn't one:: 1146 | 1147 | http://docs.python.org/py3k/library/io.html#io.RawIOBase 1148 | 1149 | See:: 1150 | 1151 | http://bugs.python.org/issue9858 1152 | """ 1153 | return self.proc.stdout.readinto(b) 1154 | 1155 | def close(self): 1156 | super(_BlobStream, self).close() 1157 | self.proc.stdout.close() 1158 | self.proc.stderr.close() 1159 | retcode = self.proc.wait() 1160 | if retcode != 0: 1161 | raise subprocess.CalledProcessError( 1162 | retcode, "git cat-file blob %s" % self.oid 1163 | ) 1164 | 1165 | 1166 | def _object_path(obj): 1167 | path = [] 1168 | node = obj 1169 | while node.parent: 1170 | path.insert(0, node.name) 1171 | node = node.parent 1172 | return "/".join(path) 1173 | 1174 | 1175 | @contextlib.contextmanager 1176 | def _popen(args, **kw): 1177 | proc = subprocess.Popen(args, **kw) 1178 | yield proc 1179 | for stream in (proc.stdin, proc.stdout, proc.stderr): 1180 | if stream is not None: 1181 | stream.close() 1182 | retcode = proc.wait() 1183 | if retcode != 0: 1184 | raise subprocess.CalledProcessError(retcode, repr(args)) 1185 | 1186 | 1187 | @contextlib.contextmanager 1188 | def _tempfile(): 1189 | fd, tmp = tempfile.mkstemp(".acidfs-merge") 1190 | os.close(fd) 1191 | yield tmp 1192 | os.remove(tmp) 1193 | 1194 | 1195 | def _NoSuchFileOrDirectory(path): 1196 | return IOError(2, "No such file or directory", path) 1197 | 1198 | 1199 | def _IsADirectory(path): 1200 | return IOError(21, "Is a directory", path) 1201 | 1202 | 1203 | def _NotADirectory(path): 1204 | return IOError(20, "Not a directory", path) 1205 | 1206 | 1207 | def _FileExists(path): 1208 | return IOError(17, "File exists", path) 1209 | 1210 | 1211 | def _DirectoryNotEmpty(path): 1212 | return IOError(39, "Directory not empty", path) 1213 | 1214 | 1215 | _MERGE_ADDED_IN_REMOTE = object() 1216 | _MERGE_REMOVED_IN_REMOTE = object() 1217 | _MERGE_CHANGED_IN_BOTH = object() 1218 | _MERGE_ADDED_IN_BOTH = object() 1219 | 1220 | _check_output = subprocess.check_output 1221 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: themes 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: themes 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/AcidFS.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/AcidFS.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/AcidFS" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/AcidFS" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | 155 | themes: 156 | @echo "Updating _themes git submodule" 157 | cd ..; git submodule update --init --recursive; cd docs; 158 | 159 | -------------------------------------------------------------------------------- /docs/_static/placeholder: -------------------------------------------------------------------------------- 1 | Sphinx won't build without this directory, even though we don't have anything to put in 2 | it. Git won't let you check in empty directories. So now this directory isn't empty. 3 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # AcidFS documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Sep 11 21:26:36 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | import pylons_sphinx_themes 16 | import datetime 17 | 18 | year = datetime.date.today().year 19 | 20 | 21 | # If extensions (or modules to document with autodoc) are in another directory, 22 | # add these directories to sys.path here. If the directory is relative to the 23 | # documentation root, use os.path.abspath to make it absolute, like shown here. 24 | #sys.path.insert(0, os.path.abspath('.')) 25 | 26 | # -- General configuration ----------------------------------------------------- 27 | 28 | # If your documentation needs a minimal Sphinx version, state it here. 29 | #needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be extensions 32 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 33 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode'] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ['_templates'] 37 | 38 | # The suffix of source filenames. 39 | source_suffix = '.rst' 40 | 41 | # The encoding of source files. 42 | #source_encoding = 'utf-8-sig' 43 | 44 | # The master toctree document. 45 | master_doc = 'index' 46 | 47 | # General information about the project. 48 | project = u'AcidFS' 49 | copyright = f'2012-{year}, Chris Rossi' 50 | 51 | # The version info for the project you're documenting, acts as replacement for 52 | # |version| and |release|, also used in various other places throughout the 53 | # built documents. 54 | # 55 | # The short X.Y version. 56 | version = '2.0' 57 | # The full version, including alpha/beta/rc tags. 58 | release = '2.0' 59 | 60 | # The language for content autogenerated by Sphinx. Refer to documentation 61 | # for a list of supported languages. 62 | #language = None 63 | 64 | # There are two options for replacing |today|: either, you set today to some 65 | # non-false value, then it is used: 66 | #today = '' 67 | # Else, today_fmt is used as the format for a strftime call. 68 | #today_fmt = '%B %d, %Y' 69 | 70 | # List of patterns, relative to source directory, that match files and 71 | # directories to ignore when looking for source files. 72 | exclude_patterns = ['_build'] 73 | 74 | # The reST default role (used for this markup: `text`) to use for all documents. 75 | #default_role = None 76 | 77 | # If true, '()' will be appended to :func: etc. cross-reference text. 78 | #add_function_parentheses = True 79 | 80 | # If true, the current module name will be prepended to all description 81 | # unit titles (such as .. function::). 82 | #add_module_names = True 83 | 84 | # If true, sectionauthor and moduleauthor directives will be shown in the 85 | # output. They are ignored by default. 86 | #show_authors = False 87 | 88 | # The name of the Pygments (syntax highlighting) style to use. 89 | pygments_style = 'sphinx' 90 | 91 | # A list of ignored prefixes for module index sorting. 92 | #modindex_common_prefix = [] 93 | 94 | 95 | # -- Options for HTML output --------------------------------------------------- 96 | 97 | # The theme to use for HTML and HTML Help pages. See the documentation for 98 | # a list of builtin themes. 99 | 100 | # Add and use Pylons theme 101 | if 'sphinx-build' in ' '.join(sys.argv): # protect against dumb importers 102 | from subprocess import call, Popen, PIPE 103 | 104 | p = Popen('which git', shell=True, stdout=PIPE) 105 | git = p.stdout.read().strip() 106 | cwd = os.getcwd() 107 | _themes = os.path.join(cwd, '_themes') 108 | 109 | if not os.path.isdir(_themes): 110 | call([git, 'clone', 'git://github.com/Pylons/pylons_sphinx_theme.git', 111 | '_themes']) 112 | else: 113 | os.chdir(_themes) 114 | call([git, 'checkout', 'master']) 115 | call([git, 'pull']) 116 | os.chdir(cwd) 117 | 118 | sys.path.append(os.path.abspath('_themes')) 119 | 120 | html_theme_path = pylons_sphinx_themes.get_html_themes_path() 121 | html_theme = 'pylons' 122 | html_theme_options = dict(github_url='https://github.com/Pylons/acidfs') 123 | 124 | # Theme options are theme-specific and customize the look and feel of a theme 125 | # further. For a list of options available for each theme, see the 126 | # documentation. 127 | #html_theme_options = {} 128 | 129 | # Add any paths that contain custom themes here, relative to this directory. 130 | #html_theme_path = [] 131 | 132 | # The name for this set of Sphinx documents. If None, it defaults to 133 | # " v documentation". 134 | #html_title = None 135 | 136 | # A shorter title for the navigation bar. Default is the same as html_title. 137 | #html_short_title = None 138 | 139 | # The name of an image file (relative to this directory) to place at the top 140 | # of the sidebar. 141 | #html_logo = None 142 | 143 | # The name of an image file (within the static path) to use as favicon of the 144 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 145 | # pixels large. 146 | #html_favicon = None 147 | 148 | # Add any paths that contain custom static files (such as style sheets) here, 149 | # relative to this directory. They are copied after the builtin static files, 150 | # so a file named "default.css" will overwrite the builtin "default.css". 151 | html_static_path = ['_static'] 152 | 153 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 154 | # using the given strftime format. 155 | #html_last_updated_fmt = '%b %d, %Y' 156 | 157 | # If true, SmartyPants will be used to convert quotes and dashes to 158 | # typographically correct entities. 159 | #html_use_smartypants = True 160 | 161 | # Custom sidebar templates, maps document names to template names. 162 | #html_sidebars = {} 163 | 164 | # Additional templates that should be rendered to pages, maps page names to 165 | # template names. 166 | #html_additional_pages = {} 167 | 168 | # If false, no module index is generated. 169 | #html_domain_indices = True 170 | 171 | # If false, no index is generated. 172 | #html_use_index = True 173 | 174 | # If true, the index is split into individual pages for each letter. 175 | #html_split_index = False 176 | 177 | # If true, links to the reST sources are added to the pages. 178 | #html_show_sourcelink = True 179 | 180 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 181 | #html_show_sphinx = True 182 | 183 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 184 | #html_show_copyright = True 185 | 186 | # If true, an OpenSearch description file will be output, and all pages will 187 | # contain a tag referring to it. The value of this option must be the 188 | # base URL from which the finished HTML is served. 189 | #html_use_opensearch = '' 190 | 191 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 192 | #html_file_suffix = None 193 | 194 | # Output file base name for HTML help builder. 195 | htmlhelp_basename = 'AcidFSdoc' 196 | 197 | 198 | # -- Options for LaTeX output -------------------------------------------------- 199 | 200 | latex_elements = { 201 | # The paper size ('letterpaper' or 'a4paper'). 202 | #'papersize': 'letterpaper', 203 | 204 | # The font size ('10pt', '11pt' or '12pt'). 205 | #'pointsize': '10pt', 206 | 207 | # Additional stuff for the LaTeX preamble. 208 | #'preamble': '', 209 | } 210 | 211 | # Grouping the document tree into LaTeX files. List of tuples 212 | # (source start file, target name, title, author, documentclass [howto/manual]). 213 | latex_documents = [ 214 | ('index', 'AcidFS.tex', u'AcidFS Documentation', 215 | u'Chris Rossi', 'manual'), 216 | ] 217 | 218 | # The name of an image file (relative to this directory) to place at the top of 219 | # the title page. 220 | #latex_logo = None 221 | 222 | # For "manual" documents, if this is true, then toplevel headings are parts, 223 | # not chapters. 224 | #latex_use_parts = False 225 | 226 | # If true, show page references after internal links. 227 | #latex_show_pagerefs = False 228 | 229 | # If true, show URL addresses after external links. 230 | #latex_show_urls = False 231 | 232 | # Documents to append as an appendix to all manuals. 233 | #latex_appendices = [] 234 | 235 | # If false, no module index is generated. 236 | #latex_domain_indices = True 237 | 238 | 239 | # -- Options for manual page output -------------------------------------------- 240 | 241 | # One entry per manual page. List of tuples 242 | # (source start file, name, description, authors, manual section). 243 | man_pages = [ 244 | ('index', 'acidfs', u'AcidFS Documentation', 245 | [u'Chris Rossi'], 1) 246 | ] 247 | 248 | # If true, show URL addresses after external links. 249 | #man_show_urls = False 250 | 251 | 252 | # -- Options for Texinfo output ------------------------------------------------ 253 | 254 | # Grouping the document tree into Texinfo files. List of tuples 255 | # (source start file, target name, title, author, 256 | # dir menu entry, description, category) 257 | texinfo_documents = [ 258 | ('index', 'AcidFS', u'AcidFS Documentation', 259 | u'Chris Rossi', 'AcidFS', 'One line description of project.', 260 | 'Miscellaneous'), 261 | ] 262 | 263 | # Documents to append as an appendix to all manuals. 264 | #texinfo_appendices = [] 265 | 266 | # If false, no module index is generated. 267 | #texinfo_domain_indices = True 268 | 269 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 270 | #texinfo_show_urls = 'footnote' 271 | 272 | 273 | # -- Options for Epub output --------------------------------------------------- 274 | 275 | # Bibliographic Dublin Core info. 276 | epub_title = u'AcidFS' 277 | epub_author = u'Chris Rossi' 278 | epub_publisher = u'Chris Rossi' 279 | epub_copyright = f'2012-{year}, Chris Rossi' 280 | 281 | # The language of the text. It defaults to the language option 282 | # or en if the language is not set. 283 | #epub_language = '' 284 | 285 | # The scheme of the identifier. Typical schemes are ISBN or URL. 286 | #epub_scheme = '' 287 | 288 | # The unique identifier of the text. This can be a ISBN number 289 | # or the project homepage. 290 | #epub_identifier = '' 291 | 292 | # A unique identification for the text. 293 | #epub_uid = '' 294 | 295 | # A tuple containing the cover image and cover page html template filenames. 296 | #epub_cover = () 297 | 298 | # HTML files that should be inserted before the pages created by sphinx. 299 | # The format is a list of tuples containing the path and title. 300 | #epub_pre_files = [] 301 | 302 | # HTML files shat should be inserted after the pages created by sphinx. 303 | # The format is a list of tuples containing the path and title. 304 | #epub_post_files = [] 305 | 306 | # A list of files that should not be packed into the epub file. 307 | #epub_exclude_files = [] 308 | 309 | # The depth of the table of contents in toc.ncx. 310 | #epub_tocdepth = 3 311 | 312 | # Allow duplicate toc entries. 313 | #epub_tocdup = True 314 | 315 | 316 | # Example configuration for intersphinx: refer to the Python standard library. 317 | intersphinx_mapping = {'http://docs.python.org/': None} 318 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. AcidFS documentation master file, created by 2 | sphinx-quickstart on Tue Sep 11 21:26:36 2012. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ====== 7 | AcidFS 8 | ====== 9 | 10 | **The filesystem on ACID** 11 | 12 | `AcidFS` allows interaction with the filesystem using transactions with ACID 13 | semantics. `Git` is used as a back end, and `AcidFS` integrates with the 14 | `transaction `_ package allowing use of 15 | multiple databases in a single transaction. 16 | 17 | Features 18 | ======== 19 | 20 | + Changes to the filesystem will only be persisted when a transaction is 21 | committed and if the transaction succeeds. 22 | 23 | + Within the scope of a transaction, your application will only see a view of 24 | the filesystem consistent with that filesystem's state at the beginning of the 25 | transaction. Concurrent writes do not affect the current context. 26 | 27 | + A full history of all changes is available, since files are stored in a 28 | backing `Git` repository. The standard `Git` toolchain can be used to recall 29 | past states, roll back particular changes, replicate the repository remotely, 30 | etc. 31 | 32 | + Changes to a `AcidFS` filesystem are synced automatically with any other 33 | database making use of the `transaction` package and its two phase commit 34 | protocol, eg. `ZODB` or `SQLAlchemy`. 35 | 36 | + Most common concurrent changes can be merged. There's even a decent chance 37 | concurrent modifications to the same text file can be merged. 38 | 39 | + Transactions can be started from an arbitrary commit point, allowing, for 40 | example, a web application to apply the results of a form submission to the 41 | state of your data at the time the form was rendered, making concurrent edits 42 | to the same resource less risky and effectively giving you transactions that 43 | can span request boundaries. 44 | 45 | Motivation 46 | ========== 47 | 48 | The motivation for this package is the fact that it often is convenient for 49 | certain very simple problems to simply write and read data from a fileystem, 50 | but often a database of some sort winds up being used simply because of the 51 | power and safety available with a system which uses transactions and ACID 52 | semantics. For example, you wouldn't want a web application with any amount of 53 | concurrency at all to be writing directly to the filesystem, since it would be 54 | easy for two threads or processes to both attempt to write to the same file at 55 | the same time, with the result that one change is clobbered by another, or even 56 | worse, the application is left in an inconsistent, corrupted state. After 57 | thinking about various ways to attack this problem and looking at `Git's` 58 | datastore and plumbing commands, it was determined that `Git` was a very good fit, 59 | allowing a graceful solution to this problem. 60 | 61 | Limitations 62 | =========== 63 | 64 | In a nutshell: 65 | 66 | + Only platforms where `fcntl` is available are supported. This excludes 67 | Microsoft Windows and probably the JVM as well. 68 | 69 | + Kernel level locking is used to manage concurrency. This means `AcidFS` 70 | cannot handle multiple application servers writing to a shared network drive. 71 | 72 | + The type of locking used only synchronizes other instances of `AcidFS`. 73 | Other processes manipulating the `Git` repository without using `AcidFS` 74 | could cause a race condition. A repository used by `AcidFS` should only be 75 | written to by `AcidFS` in order to avoid unpleasant race conditions. 76 | 77 | All of the above limitations are a result of the locking used to synchronize 78 | commits. For the most part, during a transaction, nothing special needs to 79 | be done to manage concurrency since `Git's` storage model makes management of 80 | multiple, parallel trees trivially easy. At commit time, however, any new data 81 | has to be merged with the current head which may have changed since the 82 | transaction began. This last step should be synchronized such that only one 83 | instance of `AcidFS` is attempting this at a time. The mechanism, currently, 84 | for doing this is use of the `fcntl` module which takes advantage of an 85 | advisory locking mechanism available in Unix kernels. 86 | 87 | Usage 88 | ===== 89 | 90 | `AcidFS` is easy to use. Just create an instance of `acidfs.AcidFS` and start 91 | using the filesystem:: 92 | 93 | import acidfs 94 | 95 | fs = acidfs.AcidFS('path/to/my/repo') 96 | fs.mkdir('foo') 97 | with fs.open('/foo/bar', 'w') as f: 98 | print >> f, 'Hello!' 99 | 100 | If there is not already a `Git` repository at the path specified, one is created. 101 | An instance of `AcidFS` is not thread safe. The same `AcidFS` instance should 102 | not be shared across threads or greenlets, etc. 103 | 104 | The `transaction `_ package is used to 105 | commit and abort transactions:: 106 | 107 | import transaction 108 | 109 | transaction.commit() 110 | # If no exception has been thrown, then changes are saved! Yeah! 111 | 112 | .. note:: 113 | 114 | If you're using `Pyramid `_, you should use 115 | `pyramid_tm `_. For other WSGI 116 | frameworks there is also `repoze.tm2 117 | `_. 118 | 119 | Commit Metadata 120 | =============== 121 | 122 | The `transaction `_ package has built 123 | in support for providing metadata about a particular transaction. This 124 | metadata is used to set the commit data for the underlying git commit for a 125 | transaction. Use of these hooks is optional but recommended to provide 126 | meaningful audit information in the history of your repository. An example is 127 | the best illustration:: 128 | 129 | import transaction 130 | 131 | current = transaction.get() 132 | current.note('Added blog entry: "Bedrock Bro Culture: Yabba Dabba Dude!"') 133 | current.setUser('Fred Flintstone') 134 | current.setExtendedInfo('email', 'fred@bed.rock') 135 | 136 | A users's name may also be set by using the ``setExtendedInfo`` method:: 137 | 138 | current.setExtendedInfo('user', 'Fred Flintstone') 139 | 140 | The keys ``acidfs_user`` and ``acidfs_email`` are available in extended info in 141 | case you are sharing a transaction with a system that has a different notion of 142 | what user and email should be set to. Substance D, for examples, sets the user 143 | to an integer OID that represents the user in its system, but that might not be 144 | what you want to see in the Git log for your repository:: 145 | 146 | 147 | current.setExtendedInfo('acidfs_user', 'Fred Flintstone') 148 | current.setExtendedInfo('acidfs_email', 'fred@bed.rock') 149 | 150 | The transaction might look something like this in the git log:: 151 | 152 | commit 3aa61073ea755f2c642ef7e258abe77215fe54a2 153 | Author: Fred Flintstone 154 | Date: Sun Sep 16 22:08:08 2012 -0400 155 | 156 | Added blog entry: "Bedrock Bro Culture: Yabba Dabba Dude!" 157 | 158 | API 159 | === 160 | 161 | .. automodule:: acidfs 162 | 163 | .. autoclass:: AcidFS 164 | 165 | .. automethod:: open 166 | .. automethod:: cwd 167 | .. automethod:: chdir 168 | .. automethod:: cd(path) 169 | .. automethod:: listdir 170 | .. automethod:: mkdir 171 | .. automethod:: mkdirs 172 | .. automethod:: rm 173 | .. automethod:: rmdir 174 | .. automethod:: rmtree 175 | .. automethod:: mv 176 | .. automethod:: exists 177 | .. automethod:: isdir 178 | .. automethod:: empty 179 | .. automethod:: get_base 180 | .. automethod:: set_base 181 | .. automethod:: hash 182 | 183 | .. toctree:: 184 | :maxdepth: 2 185 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\AcidFS.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\AcidFS.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | """Build and test configuration file. """ 2 | import os 3 | import shutil 4 | 5 | import nox 6 | 7 | NOX_DIR = os.path.abspath(os.path.dirname(__file__)) 8 | DEFAULT_INTERPRETER = "3.9" 9 | ALL_INTERPRETERS = ("3.6", "3.7", "3.8", "3.9") 10 | 11 | 12 | def get_path(*names): 13 | return os.path.join(NOX_DIR, *names) 14 | 15 | 16 | @nox.session(py=ALL_INTERPRETERS) 17 | def unit(session): 18 | # Install all dependencies. 19 | session.install("-e", ".[testing]") 20 | 21 | # Run py.test against the unit tests. 22 | run_args = ["pytest"] 23 | if session.posargs: 24 | run_args.extend(session.posargs) 25 | else: 26 | run_args.extend( 27 | [ 28 | "--cov=acidfs", 29 | "--cov=tests", 30 | "--cov-report=term-missing", 31 | ] 32 | ) 33 | run_args.append(get_path("tests.py")) 34 | session.run(*run_args) 35 | 36 | 37 | def run_black(session, use_check=False): 38 | args = ["black"] 39 | if use_check: 40 | args.append("--check") 41 | 42 | args.extend( 43 | [ 44 | get_path("noxfile.py"), 45 | get_path("setup.py"), 46 | get_path("acidfs"), 47 | get_path("tests.py"), 48 | ] 49 | ) 50 | 51 | session.run(*args) 52 | 53 | 54 | @nox.session(py=DEFAULT_INTERPRETER) 55 | def lint(session): 56 | """Run linters. 57 | 58 | Returns a failure if the linters find linting errors or sufficiently 59 | serious code quality issues. 60 | """ 61 | session.install("flake8", "black") 62 | run_black(session, use_check=True) 63 | session.run("flake8", "acidfs", "tests.py", "setup.py") 64 | 65 | 66 | @nox.session(py=DEFAULT_INTERPRETER) 67 | def blacken(session): 68 | # Install all dependencies. 69 | session.install("black") 70 | # Run ``black``. 71 | run_black(session) 72 | 73 | 74 | @nox.session(py=DEFAULT_INTERPRETER) 75 | def docs(session): 76 | """Build the docs for this library.""" 77 | 78 | session.install("-e", ".[docs]") 79 | 80 | shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) 81 | session.run( 82 | "sphinx-build", 83 | "-W", # warnings as errors 84 | "-T", # show full traceback on exception 85 | "-b", 86 | "html", 87 | "-d", 88 | os.path.join("docs", "_build", "doctrees", ""), 89 | os.path.join("docs", ""), 90 | os.path.join("docs", "_build", "html", ""), 91 | ) 92 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | addopts = --cov acidfs --cov-report=term-missing 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | from setuptools import find_packages 4 | 5 | VERSION = "2.0" 6 | 7 | requires = [ 8 | "transaction", 9 | ] 10 | 11 | tests_require = ["pytest", "pytest-cov", "nox"] 12 | docs_require = ["Sphinx", "pylons-sphinx-themes"] 13 | 14 | here = os.path.abspath(os.path.dirname(__file__)) 15 | README = open(os.path.join(here, "README.rst")).read() 16 | 17 | setup( 18 | name="acidfs", 19 | version=VERSION, 20 | description="ACID semantics for the filesystem.", 21 | long_description=README, 22 | classifiers=[ 23 | "Development Status :: 5 - Production/Stable", 24 | "Intended Audience :: Developers", 25 | "Topic :: Database", 26 | "License :: Repoze Public License", 27 | ], 28 | keywords="git acid filesystem transaction", 29 | author="Chris Rossi", 30 | author_email="pylons-discuss@googlegroups.com", 31 | url="http://pylonsproject.org", 32 | license="BSD-derived (http://www.repoze.org/LICENSE.txt)", 33 | packages=find_packages(), 34 | include_package_data=True, 35 | zip_safe=False, 36 | install_requires=requires, 37 | tests_require=tests_require, 38 | extras_require={ 39 | "testing": tests_require, 40 | "docs": docs_require, 41 | }, 42 | ) 43 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import io 3 | import os 4 | import pytest 5 | import shutil 6 | import subprocess 7 | import tempfile 8 | import transaction 9 | 10 | from unittest import mock 11 | 12 | from acidfs import AcidFS, _check_output 13 | 14 | 15 | @pytest.fixture 16 | def tmp(request): 17 | tmp = tempfile.mkdtemp() 18 | return tmp 19 | 20 | 21 | @pytest.fixture 22 | def factory(request, tmp): 23 | cwd = os.getcwd() 24 | 25 | def mkstore(*args, **kw): 26 | store = AcidFS(tmp, *args, **kw) 27 | 28 | if "user_name" not in kw: 29 | tx = transaction.get() 30 | tx.setUser("Test User") 31 | tx.setExtendedInfo("email", "test@example.com") 32 | 33 | os.chdir(tmp) 34 | subprocess.check_call(["git", "config", "user.name", "Test User"]) 35 | subprocess.check_call(["git", "config", "user.email", "test@example.com"]) 36 | os.chdir(cwd) 37 | 38 | return store 39 | 40 | def cleanup(): 41 | transaction.abort() 42 | shutil.rmtree(tmp) 43 | 44 | request.addfinalizer(cleanup) 45 | return mkstore 46 | 47 | 48 | @contextlib.contextmanager 49 | def assert_no_such_file_or_directory(path): 50 | try: 51 | yield 52 | raise AssertionError("IOError not raised") # pragma no cover 53 | except IOError as e: 54 | assert e.errno == 2 55 | assert e.strerror == "No such file or directory" 56 | assert e.filename == path 57 | 58 | 59 | @contextlib.contextmanager 60 | def assert_is_a_directory(path): 61 | try: 62 | yield 63 | raise AssertionError("IOError not raised") # pragma no cover 64 | except IOError as e: 65 | assert e.errno == 21 66 | assert e.strerror == "Is a directory" 67 | assert e.filename == path 68 | 69 | 70 | @contextlib.contextmanager 71 | def assert_not_a_directory(path): 72 | try: 73 | yield 74 | raise AssertionError("IOError not raised") # pragma no cover 75 | except IOError as e: 76 | assert e.errno == 20 77 | assert e.strerror == "Not a directory" 78 | assert e.filename == path 79 | 80 | 81 | @contextlib.contextmanager 82 | def assert_file_exists(path): 83 | try: 84 | yield 85 | raise AssertionError("IOError not raised") # pragma no cover 86 | except IOError as e: 87 | assert e.errno == 17 88 | assert e.strerror == "File exists" 89 | assert e.filename == path 90 | 91 | 92 | @contextlib.contextmanager 93 | def assert_directory_not_empty(path): 94 | try: 95 | yield 96 | raise AssertionError("IOError not raised") # pragma no cover 97 | except IOError as e: 98 | assert e.errno == 39 99 | assert e.strerror == "Directory not empty" 100 | assert e.filename == path 101 | 102 | 103 | def test_new_repo_w_working_directory(factory, tmp): 104 | factory() 105 | assert os.path.exists(os.path.join(tmp, ".git")) 106 | 107 | 108 | def test_no_repo_dont_create(factory): 109 | with pytest.raises(ValueError) as cm: 110 | factory(create=False) 111 | assert str(cm.exconly()).startswith("ValueError: No database found") 112 | 113 | 114 | def test_read_write_file(factory, tmp): 115 | fs = factory() 116 | assert fs.hash() == b"4b825dc642cb6eb9a060e54bf8d69288fbee4904" 117 | # twice for branch coverage/excercising lazy computation and caching of the hash 118 | assert fs.hash() == b"4b825dc642cb6eb9a060e54bf8d69288fbee4904" 119 | with assert_no_such_file_or_directory("foo"): 120 | fs.hash("foo") 121 | with fs.open("foo", "wb") as f: 122 | assert f.writable() 123 | fprint(f, b"Hello") 124 | with assert_no_such_file_or_directory("foo"): 125 | fs.open("foo", "rb") 126 | with assert_no_such_file_or_directory("foo"): 127 | fs.hash("foo") 128 | assert fs.open("foo", "rb").read() == b"Hello\n" 129 | assert fs.hash("foo") == b"e965047ad7c57865823c7d992b1d046ea66edf78" 130 | actual_file = os.path.join(tmp, "foo") 131 | assert not os.path.exists(actual_file) 132 | transaction.commit() 133 | with fs.open("foo", "rb", buffering=80) as f: 134 | assert f.readable() 135 | assert f.read() == b"Hello\n" 136 | with fs.open("foo", "wb", buffering=0) as f: 137 | assert f.writable() 138 | fprint(f, b"Hello Dad") 139 | with fs.open("foo", "rb", buffering=0) as f: 140 | assert f.readable() 141 | assert f.read() == b"Hello Dad\n" 142 | with open(actual_file, "rb") as f: 143 | assert f.read() == b"Hello\n" 144 | transaction.commit() # Nothing to commit 145 | 146 | 147 | def test_read_write_nonascii_name(factory): 148 | fs = factory(path_encoding="utf-8") 149 | filename = b"Hell\xc3\xb2".decode("utf-8") 150 | with fs.open(filename, "wb") as f: 151 | fprint(f, b"Hello") 152 | transaction.commit() 153 | assert fs.listdir(), [filename] 154 | 155 | 156 | def test_read_write_text_file(factory, tmp): 157 | fs = factory() 158 | with fs.open("foo", "wt") as f: 159 | assert f.writable() 160 | fprint(f, u"Hell\xf2") 161 | with assert_no_such_file_or_directory("foo"): 162 | fs.open("foo", "r") 163 | assert fs.open("foo", "r").read() == u"Hell\xf2\n" 164 | actual_file = os.path.join(tmp, "foo") 165 | assert not os.path.exists(actual_file) 166 | transaction.commit() 167 | with fs.open("foo", "r", buffering=1) as f: 168 | assert f.readable() 169 | assert f.read() == u"Hell\xf2\n" 170 | with open(actual_file, "rb") as f: 171 | assert f.read() == b"Hell\xc3\xb2\n" 172 | transaction.commit() # Nothing to commit 173 | 174 | 175 | def test_read_write_file_in_subfolder(factory, tmp): 176 | fs = factory() 177 | assert not fs.isdir("foo") 178 | fs.mkdir("foo") 179 | assert fs.isdir("foo") 180 | assert fs.hash("foo") == b"4b825dc642cb6eb9a060e54bf8d69288fbee4904" 181 | with fs.open("foo/bar", "wb") as f: 182 | fprint(f, b"Hello") 183 | with fs.open("foo/bar", "rb") as f: 184 | assert f.read() == b"Hello\n" 185 | assert fs.hash("foo") == b"c57c02051dae8e2e4803530c217ac38121f393d3" 186 | actual_file = os.path.join(tmp, "foo", "bar") 187 | assert not os.path.exists(actual_file) 188 | transaction.commit() 189 | assert fs.isdir("foo") 190 | assert not fs.isdir("foo/bar") 191 | with fs.open("foo/bar", "rb") as f: 192 | assert f.read() == b"Hello\n" 193 | with open(actual_file, "rb") as f: 194 | assert f.read() == b"Hello\n" 195 | 196 | 197 | def test_read_write_file_in_subfolder_bare_repo(factory): 198 | fs = factory(bare=True) 199 | assert not fs.isdir("foo") 200 | fs.mkdir("foo") 201 | assert fs.isdir("foo") 202 | with fs.open("foo/bar", "wb") as f: 203 | fprint(f, b"Hello") 204 | with fs.open("foo/bar", "rb") as f: 205 | assert f.read() == b"Hello\n" 206 | transaction.commit() 207 | assert fs.isdir("foo") 208 | assert not fs.isdir("foo/bar") 209 | with fs.open("foo/bar", "rb") as f: 210 | assert f.read() == b"Hello\n" 211 | 212 | 213 | def test_append_twice_to_same_file(factory): 214 | fs = factory() 215 | with fs.open("foo", "a") as f: 216 | fprint(f, u"One") 217 | with fs.open("foo", "a") as f: 218 | fprint(f, u"Two") 219 | with fs.open("foo") as f: 220 | assert f.read() == ("One\n" "Two\n") 221 | transaction.commit() 222 | with fs.open("foo") as f: 223 | assert f.read() == ("One\n" "Two\n") 224 | 225 | 226 | def test_open_edge_cases(factory): 227 | fs = factory() 228 | 229 | with assert_no_such_file_or_directory("foo"): 230 | fs.open("foo", "rb") 231 | 232 | with assert_no_such_file_or_directory("foo/bar"): 233 | fs.open("foo/bar", "wb") 234 | 235 | with assert_is_a_directory("."): 236 | fs.open(".", "rb") 237 | 238 | with assert_is_a_directory("."): 239 | fs.open(".", "wb") 240 | 241 | fs.mkdir("foo") 242 | 243 | with assert_is_a_directory("foo"): 244 | fs.open("foo", "wb") 245 | 246 | fs.open("bar", "wb").write(b"Howdy") 247 | 248 | with assert_not_a_directory("bar/foo"): 249 | fs.open("bar/foo", "wb") 250 | 251 | with pytest.raises(ValueError): 252 | fs.open("foo", "wtf") 253 | 254 | with pytest.raises(ValueError): 255 | fs.open("foo", "wbt") 256 | 257 | with pytest.raises(ValueError): 258 | fs.open("foo", "w+") 259 | 260 | with pytest.raises(ValueError): 261 | fs.open("foo", "r", buffering=0) 262 | 263 | with fs.open("bar", "wb") as f: 264 | fprint(f, b"Howdy!") 265 | with pytest.raises(ValueError) as cm: 266 | transaction.commit() 267 | assert ( 268 | str(cm.exconly()) 269 | == "ValueError: Cannot commit transaction with open files." 270 | ) 271 | transaction.abort() 272 | 273 | fs.open("bar", "xb").write(b"Hello!") 274 | with assert_file_exists("bar"): 275 | fs.open("bar", "xb") 276 | 277 | with fs.open("o", "w") as f: 278 | with assert_no_such_file_or_directory("o/m/g"): 279 | fs.open("o/m/g", "w") 280 | f.write("hi mom!") 281 | 282 | 283 | def test_mkdir_edge_cases(factory): 284 | fs = factory() 285 | 286 | with assert_no_such_file_or_directory("foo/bar"): 287 | fs.mkdir("foo/bar") 288 | 289 | fs.open("foo", "wb").write(b"Howdy!") 290 | 291 | with assert_not_a_directory("foo/bar"): 292 | fs.mkdir("foo/bar") 293 | 294 | fs.mkdir("bar") 295 | with assert_file_exists("bar"): 296 | fs.mkdir("bar") 297 | 298 | 299 | def test_commit_name_and_email_from_factory(factory, tmp): 300 | fs = factory(user_name="Fred Flintstone", user_email="fred@bed.rock") 301 | fs.open("foo", "wb").write(b"Howdy!") 302 | transaction.commit() 303 | 304 | output = _check_output(["git", "log"], cwd=tmp) 305 | assert b"Author: Fred Flintstone " in output 306 | 307 | 308 | def test_commit_metadata(factory, tmp): 309 | fs = factory() 310 | tx = transaction.get() 311 | tx.note("A test commit.") 312 | tx.setUser("Fred Flintstone") 313 | tx.setExtendedInfo("email", "fred@bed.rock") 314 | fs.open("foo", "wb").write(b"Howdy!") 315 | transaction.commit() 316 | 317 | output = _check_output(["git", "log"], cwd=tmp) 318 | assert b"Author: Fred Flintstone " in output 319 | assert b"A test commit." in output 320 | 321 | 322 | def test_commit_metadata_blank_name(factory, tmp): 323 | fs = factory() 324 | tx = transaction.get() 325 | tx.note("A test commit.") 326 | tx.setUser("") 327 | tx.setExtendedInfo("email", "fred@bed.rock") 328 | fs.open("foo", "wb").write(b"Howdy!") 329 | transaction.commit() 330 | 331 | output = _check_output(["git", "log"], cwd=tmp) 332 | assert b"Author: / " in output 333 | assert b"A test commit." in output 334 | 335 | 336 | def test_commit_metadata_for_acidfs(factory, tmp): 337 | fs = factory() 338 | tx = transaction.get() 339 | tx.note("A test commit.") 340 | tx.extension["acidfs_user"] = "Fred Flintstone" 341 | tx.extension["acidfs_email"] = "fred@bed.rock" 342 | fs.open("foo", "wb").write(b"Howdy!") 343 | transaction.commit() 344 | 345 | output = _check_output(["git", "log"], cwd=tmp) 346 | assert b"Author: Fred Flintstone " in output 347 | assert b"A test commit." in output 348 | 349 | 350 | def test_commit_metadata_user_path_is_blank(factory, tmp): 351 | # pyramid_tm calls setUser with '' for path 352 | fs = factory() 353 | tx = transaction.get() 354 | tx.note("A test commit.") 355 | tx.setUser("Fred", "") 356 | tx.setExtendedInfo("email", "fred@bed.rock") 357 | fs.open("foo", "wb").write(b"Howdy!") 358 | transaction.commit() 359 | 360 | output = _check_output(["git", "log"], cwd=tmp) 361 | assert b"Author: Fred " in output 362 | assert b"A test commit." in output 363 | 364 | 365 | def test_commit_metadata_extended_info_for_user(factory, tmp): 366 | fs = factory() 367 | tx = transaction.get() 368 | tx.note("A test commit.") 369 | tx.setExtendedInfo("user", "Fred Flintstone") 370 | tx.setExtendedInfo("email", "fred@bed.rock") 371 | fs.open("foo", "wb").write(b"Howdy!") 372 | transaction.commit() 373 | 374 | output = _check_output(["git", "log"], cwd=tmp) 375 | assert b"Author: Fred Flintstone " in output 376 | assert b"A test commit." in output 377 | 378 | 379 | def test_modify_file(factory, tmp): 380 | fs = factory() 381 | with fs.open("foo", "wb") as f: 382 | fprint(f, b"Howdy!") 383 | transaction.commit() 384 | 385 | path = os.path.join(tmp, "foo") 386 | with fs.open("foo", "wb") as f: 387 | fprint(f, b"Hello!") 388 | assert fs.open("foo", "rb").read() == b"Howdy!\n" 389 | assert fs.hash("foo") == b"c564dac563c1974addaa0ac0ae028fc92b2370f1" 390 | assert fs.open("foo", "rb").read() == b"Hello!\n" 391 | assert fs.hash("foo") == b"10ddd6d257e01349d514541981aeecea6b2e741d" 392 | assert open(path, "rb").read() == b"Howdy!\n" 393 | transaction.commit() 394 | 395 | assert open(path, "rb").read() == b"Hello!\n" 396 | 397 | 398 | def test_error_writing_blob(factory): 399 | fs = factory() 400 | with pytest.raises((IOError, subprocess.CalledProcessError)): 401 | with fs.open("foo", "wb") as f: 402 | wait = f.raw.proc.wait 403 | 404 | def dummy_wait(): 405 | wait() 406 | return 1 407 | 408 | f.raw.proc.wait = dummy_wait 409 | fprint(f, b"Howdy!") 410 | 411 | 412 | def test_error_reading_blob(factory): 413 | fs = factory() 414 | fs.open("foo", "wb").write(b"a" * 10000) 415 | with pytest.raises(subprocess.CalledProcessError): 416 | with fs.open("foo", "rb") as f: 417 | wait = f.raw.proc.wait 418 | 419 | def dummy_wait(): 420 | wait() 421 | return 1 422 | 423 | f.raw.proc.wait = dummy_wait 424 | f.read() 425 | 426 | 427 | def test_append(factory, tmp): 428 | fs = factory() 429 | fs.open("foo", "wb").write(b"Hello!\n") 430 | transaction.commit() 431 | 432 | path = os.path.join(tmp, "foo") 433 | with fs.open("foo", "ab") as f: 434 | fprint(f, b"Daddy!") 435 | assert fs.open("foo", "rb").read() == b"Hello!\n" 436 | assert open(path, "rb").read() == b"Hello!\n" 437 | assert fs.open("foo", "rb").read() == b"Hello!\nDaddy!\n" 438 | assert open(path, "rb").read() == b"Hello!\n" 439 | 440 | transaction.commit() 441 | assert fs.open("foo", "rb").read() == b"Hello!\nDaddy!\n" 442 | assert open(path, "rb").read() == b"Hello!\nDaddy!\n" 443 | 444 | 445 | def test_rm(factory, tmp): 446 | fs = factory() 447 | fs.open("foo", "wb").write(b"Hello\n") 448 | transaction.commit() 449 | 450 | path = os.path.join(tmp, "foo") 451 | assert fs.exists("foo") 452 | fs.rm("foo") 453 | assert not fs.exists("foo") 454 | assert os.path.exists(path) 455 | 456 | transaction.commit() 457 | assert not fs.exists("foo") 458 | assert not os.path.exists(path) 459 | 460 | with assert_no_such_file_or_directory("foo"): 461 | fs.rm("foo") 462 | with assert_is_a_directory("."): 463 | fs.rm(".") 464 | 465 | 466 | def test_rmdir(factory, tmp): 467 | fs = factory() 468 | fs.mkdir("foo") 469 | fs.open("foo/bar", "wb").write(b"Hello\n") 470 | transaction.commit() 471 | 472 | path = os.path.join(tmp, "foo") 473 | with assert_not_a_directory("foo/bar"): 474 | fs.rmdir("foo/bar") 475 | with assert_directory_not_empty("foo"): 476 | fs.rmdir("foo") 477 | with assert_no_such_file_or_directory("bar"): 478 | fs.rmdir("bar") 479 | fs.rm("foo/bar") 480 | fs.rmdir("foo") 481 | assert not fs.exists("foo") 482 | assert os.path.exists(path) 483 | 484 | transaction.commit() 485 | assert not fs.exists("foo") 486 | assert not os.path.exists(path) 487 | 488 | 489 | def test_rmtree(factory, tmp): 490 | fs = factory() 491 | fs.mkdirs("foo/bar") 492 | fs.open("foo/bar/baz", "wb").write(b"Hello\n") 493 | with assert_not_a_directory("foo/bar/baz/boz"): 494 | fs.mkdirs("foo/bar/baz/boz") 495 | transaction.commit() 496 | 497 | path = os.path.join(tmp, "foo", "bar", "baz") 498 | with assert_not_a_directory("foo/bar/baz"): 499 | fs.rmtree("foo/bar/baz") 500 | with assert_no_such_file_or_directory("bar"): 501 | fs.rmtree("bar") 502 | assert fs.exists("foo/bar") 503 | assert fs.exists("foo") 504 | assert not fs.empty("/") 505 | fs.rmtree("foo") 506 | assert not fs.exists("foo/bar") 507 | assert not fs.exists("foo") 508 | assert fs.empty("/") 509 | assert os.path.exists(path) 510 | 511 | transaction.commit() 512 | assert not os.path.exists(path) 513 | 514 | 515 | def test_cant_remove_root_dir(factory): 516 | fs = factory() 517 | with pytest.raises(ValueError): 518 | fs.rmdir("/") 519 | with pytest.raises(ValueError): 520 | fs.rmtree("/") 521 | 522 | 523 | def test_empty(factory): 524 | fs = factory() 525 | assert fs.empty("/") 526 | fs.open("foo", "wb").write(b"Hello!") 527 | assert not fs.empty("/") 528 | with assert_not_a_directory("foo"): 529 | fs.empty("foo") 530 | with assert_no_such_file_or_directory("foo/bar"): 531 | fs.empty("foo/bar") 532 | 533 | 534 | def test_mv(factory, tmp): 535 | fs = factory() 536 | fs.mkdirs("one/a") 537 | fs.mkdirs("one/b") 538 | fs.open("one/a/foo", "wb").write(b"Hello!") 539 | fs.open("one/b/foo", "wb").write(b"Howdy!") 540 | transaction.commit() 541 | 542 | with assert_no_such_file_or_directory("/"): 543 | fs.mv("/", "one") 544 | with assert_no_such_file_or_directory("bar"): 545 | fs.mv("bar", "one") 546 | with assert_no_such_file_or_directory("bar/baz"): 547 | fs.mv("one", "bar/baz") 548 | 549 | pexists = os.path.exists 550 | j = os.path.join 551 | fs.mv("one/a/foo", "one/a/bar") 552 | assert not fs.exists("one/a/foo") 553 | assert fs.exists("one/a/bar") 554 | assert pexists(j(tmp, "one", "a", "foo")) 555 | assert not pexists(j(tmp, "one", "a", "bar")) 556 | 557 | transaction.commit() 558 | assert not fs.exists("one/a/foo") 559 | assert fs.exists("one/a/bar") 560 | assert not pexists(j(tmp, "one", "a", "foo")) 561 | assert pexists(j(tmp, "one", "a", "bar")) 562 | 563 | fs.mv("one/b/foo", "one/a/bar") 564 | assert not fs.exists("one/b/foo") 565 | assert fs.open("one/a/bar", "rb").read() == b"Howdy!" 566 | assert pexists(j(tmp, "one", "b", "foo")) 567 | assert open(j(tmp, "one", "a", "bar"), "rb").read() == b"Hello!" 568 | 569 | transaction.commit() 570 | assert not fs.exists("one/b/foo") 571 | assert fs.open("one/a/bar", "rb").read() == b"Howdy!" 572 | assert not pexists(j(tmp, "one", "b", "foo")) 573 | assert open(j(tmp, "one", "a", "bar"), "rb").read() == b"Howdy!" 574 | 575 | fs.mv("one/a", "one/b") 576 | assert not fs.exists("one/a") 577 | assert fs.exists("one/b/a") 578 | assert pexists(j(tmp, "one", "a")) 579 | assert not pexists(j(tmp, "one", "b", "a")) 580 | 581 | transaction.commit() 582 | assert not fs.exists("one/a") 583 | assert fs.exists("one/b/a") 584 | assert not pexists(j(tmp, "one", "a")) 585 | assert pexists(j(tmp, "one", "b", "a")) 586 | 587 | 588 | def test_mv_noop(factory): 589 | """ 590 | Tests an error report from Hasan Karahan. 591 | """ 592 | fs = factory() 593 | fs.open("foo", "wb").write(b"Hello!") 594 | fs.mv("foo", "foo") 595 | assert fs.open("foo", "rb").read() == b"Hello!" 596 | transaction.commit() 597 | assert fs.open("foo", "rb").read() == b"Hello!" 598 | 599 | fs.mv("foo", "foo") 600 | assert fs.open("foo", "rb").read() == b"Hello!" 601 | transaction.commit() 602 | assert fs.open("foo", "rb").read() == b"Hello!" 603 | 604 | 605 | def test_listdir(factory): 606 | fs = factory() 607 | fs.mkdirs("one/a") 608 | fs.mkdir("two") 609 | fs.open("three", "wb").write(b"Hello!") 610 | 611 | with assert_no_such_file_or_directory("bar"): 612 | fs.listdir("bar") 613 | with assert_not_a_directory("three"): 614 | fs.listdir("three") 615 | assert sorted(fs.listdir()) == ["one", "three", "two"] 616 | assert fs.listdir("/one") == ["a"] 617 | 618 | transaction.commit() 619 | assert sorted(fs.listdir()) == ["one", "three", "two"] 620 | assert fs.listdir("/one") == ["a"] 621 | 622 | 623 | def test_chdir(factory): 624 | fs = factory() 625 | 626 | fs.mkdirs("one/a") 627 | fs.mkdir("two") 628 | fs.open("three", "wb").write(b"Hello!") 629 | fs.open("two/three", "wb").write(b"Haha!") 630 | 631 | assert fs.cwd() == "/" 632 | assert sorted(fs.listdir()) == ["one", "three", "two"] 633 | 634 | with assert_no_such_file_or_directory("foo"): 635 | fs.chdir("foo") 636 | 637 | fs.chdir("one") 638 | assert fs.cwd() == "/one" 639 | assert fs.listdir() == ["a"] 640 | 641 | with assert_not_a_directory("/three"): 642 | with fs.cd("/three"): 643 | pass # pragma no cover 644 | 645 | with fs.cd("/two"): 646 | assert fs.cwd() == "/two" 647 | assert fs.listdir() == ["three"] 648 | assert fs.open("three", "rb").read() == b"Haha!" 649 | assert fs.open("/three", "rb").read() == b"Hello!" 650 | 651 | assert fs.cwd() == "/one" 652 | assert fs.listdir() == ["a"] 653 | 654 | 655 | def test_nochange_commit(factory, tmp): 656 | fs = factory() 657 | with fs.open("foo", "w") as f: 658 | f.write("hi mom!") 659 | 660 | transaction.commit() 661 | 662 | assert count_commits(tmp) == 1 663 | 664 | with open(os.path.join(tmp, "foo"), "r") as f: 665 | assert f.read() == "hi mom!" 666 | 667 | with fs.open("foo", "w") as f: 668 | f.write("hi mom!") 669 | 670 | transaction.commit() 671 | 672 | assert count_commits(tmp) == 1 673 | 674 | 675 | def test_conflict_error_on_first_commit(factory, tmp): 676 | from acidfs import ConflictError 677 | 678 | fs = factory() 679 | fs.open("foo", "wb").write(b"Hello!") 680 | open(os.path.join(tmp, "foo"), "wb").write(b"Howdy!") 681 | _check_output(["git", "add", "."], cwd=tmp) 682 | _check_output(["git", "commit", "-m", "Haha! First!"], cwd=tmp) 683 | with pytest.raises(ConflictError): 684 | transaction.commit() 685 | 686 | 687 | def test_unable_to_merge_file(factory, tmp): 688 | from acidfs import ConflictError 689 | 690 | fs = factory() 691 | fs.open("foo", "wb").write(b"Hello!") 692 | transaction.commit() 693 | fs.open("foo", "wb").write(b"Party!") 694 | open(os.path.join(tmp, "foo"), "wb").write(b"Howdy!") 695 | _check_output(["git", "add", "."], cwd=tmp) 696 | _check_output(["git", "commit", "-m", "Haha! First!"], cwd=tmp) 697 | with pytest.raises(ConflictError): 698 | transaction.commit() 699 | 700 | 701 | def test_merge_add_file(factory, tmp): 702 | fs = factory() 703 | fs.open("foo", "wb").write(b"Hello!\n") 704 | transaction.commit() 705 | 706 | fs.open("bar", "wb").write(b"Howdy!\n") 707 | open(os.path.join(tmp, "baz"), "wb").write(b"Ciao!\n") 708 | _check_output(["git", "add", "baz"], cwd=tmp) 709 | _check_output(["git", "commit", "-m", "haha"], cwd=tmp) 710 | transaction.commit() 711 | 712 | assert fs.exists("foo") 713 | assert fs.exists("bar") 714 | assert fs.exists("baz") 715 | 716 | 717 | def test_merge_rm_file(factory, tmp): 718 | fs = factory(head="master") 719 | fs.open("foo", "wb").write(b"Hello\n") 720 | fs.open("bar", "wb").write(b"Grazie\n") 721 | fs.open("baz", "wb").write(b"Prego\n") 722 | transaction.commit() 723 | 724 | fs.rm("foo") 725 | _check_output(["git", "rm", "baz"], cwd=tmp) 726 | _check_output(["git", "commit", "-m", "gotcha"], cwd=tmp) 727 | transaction.commit() 728 | 729 | assert not fs.exists("foo") 730 | assert fs.exists("bar") 731 | assert not fs.exists("baz") 732 | 733 | 734 | def test_merge_rm_same_file(factory, tmp): 735 | fs = factory(head="master") 736 | fs.open("foo", "wb").write(b"Hello\n") 737 | fs.open("bar", "wb").write(b"Grazie\n") 738 | transaction.commit() 739 | 740 | base = fs.get_base() 741 | fs.rm("foo") 742 | transaction.commit() 743 | 744 | fs.set_base(base) 745 | fs.rm("foo") 746 | # Do something else besides, so commit has different sha1 747 | fs.open("baz", "wb").write(b"Prego\n") 748 | transaction.commit() 749 | 750 | assert not fs.exists("foo") 751 | assert fs.exists("bar") 752 | 753 | 754 | def test_merge_add_same_file(factory): 755 | fs = factory(head="master") 756 | fs.open("foo", "wb").write(b"Hello\n") 757 | transaction.commit() 758 | 759 | base = fs.get_base() 760 | fs.open("bar", "wb").write(b"Grazie\n") 761 | transaction.commit() 762 | 763 | fs.set_base(base) 764 | fs.open("bar", "wb").write(b"Grazie\n") 765 | # Do something else besides, so commit has different sha1 766 | fs.open("baz", "wb").write(b"Prego\n") 767 | transaction.commit() 768 | 769 | assert fs.open("bar", "rb").read() == b"Grazie\n" 770 | assert fs.open("baz", "rb").read() == b"Prego\n" 771 | 772 | 773 | def test_merge_add_different_file_same_path(factory): 774 | from acidfs import ConflictError 775 | 776 | fs = factory(head="master") 777 | fs.open("foo", "wb").write(b"Hello\n") 778 | transaction.commit() 779 | 780 | base = fs.get_base() 781 | fs.open("bar", "wb").write(b"Grazie\n") 782 | transaction.commit() 783 | 784 | fs.set_base(base) 785 | fs.open("bar", "wb").write(b"Prego\n") 786 | with pytest.raises(ConflictError): 787 | transaction.commit() 788 | 789 | 790 | def test_merge_file(factory): 791 | fs = factory() 792 | with fs.open("foo", "wb") as f: 793 | fprint(f, b"One") 794 | fprint(f, b"Two") 795 | fprint(f, b"Three") 796 | fprint(f, b"Four") 797 | fprint(f, b"Five") 798 | transaction.commit() 799 | 800 | base = fs.get_base() 801 | with fs.open("foo", "wb") as f: 802 | fprint(f, b"One") 803 | fprint(f, b"Dos") 804 | fprint(f, b"Three") 805 | fprint(f, b"Four") 806 | fprint(f, b"Five") 807 | transaction.commit() 808 | 809 | fs.set_base(base) 810 | with fs.open("foo", "ab") as f: 811 | fprint(f, b"Sei") 812 | transaction.commit() 813 | 814 | assert list(fs.open("foo", "rb").readlines()) == [ 815 | b"One\n", 816 | b"Dos\n", 817 | b"Three\n", 818 | b"Four\n", 819 | b"Five\n", 820 | b"Sei\n", 821 | ] 822 | 823 | 824 | def test_set_base(factory): 825 | from acidfs import ConflictError 826 | 827 | fs = factory() 828 | fs.open("foo", "wb").write(b"Hello\n") 829 | transaction.commit() 830 | 831 | base = fs.get_base() 832 | fs.open("bar", "wb").write(b"Grazie\n") 833 | with pytest.raises(ConflictError): 834 | fs.set_base("whatever") 835 | transaction.commit() 836 | 837 | fs.set_base(base) 838 | assert fs.exists("foo") 839 | assert not fs.exists("bar") 840 | fs.open("baz", "wb").write(b"Prego\n") 841 | transaction.commit() 842 | 843 | assert fs.exists("foo") 844 | assert fs.exists("bar") 845 | assert fs.exists("baz") 846 | 847 | 848 | def test_use_other_branch(factory): 849 | fs = factory(head="foo") 850 | fs.open("foo", "wb").write(b"Hello\n") 851 | transaction.commit() 852 | 853 | fs2 = factory() 854 | fs2.open("foo", "wb").write(b"Howdy!\n") 855 | transaction.commit() 856 | 857 | assert fs.open("foo", "rb").read() == b"Hello\n" 858 | assert fs2.open("foo", "rb").read() == b"Howdy!\n" 859 | 860 | 861 | def test_branch_and_then_merge(factory, tmp): 862 | fs = factory() 863 | fs.open("foo", "wb").write(b"Hello") 864 | transaction.commit() 865 | 866 | fs2 = factory(head="abranch") 867 | fs2.set_base(fs.get_base()) 868 | fs2.open("bar", "wb").write(b"Ciao") 869 | fs.open("baz", "wb").write(b"Hola") 870 | transaction.commit() 871 | 872 | fs.set_base("abranch") 873 | fs.open("beez", "wb").write(b"buzz") 874 | transaction.commit() 875 | 876 | assert fs.exists("foo") 877 | assert fs.exists("bar") 878 | assert fs.exists("baz") 879 | assert fs.exists("beez") 880 | assert fs2.exists("foo") 881 | assert fs2.exists("bar") 882 | assert not fs2.exists("baz") 883 | assert not fs2.exists("beez") 884 | 885 | # Expecting two parents for commit since it's a merge 886 | commit = ( 887 | _check_output(["git", "cat-file", "-p", "HEAD^{commit}"], cwd=tmp) 888 | .decode("ascii") 889 | .split("\n") 890 | ) 891 | assert commit[1].startswith("parent") 892 | assert commit[2].startswith("parent") 893 | 894 | 895 | def test_directory_name_with_spaces(factory): 896 | fs = factory() 897 | fs.mkdir("foo bar") 898 | with fs.cd("foo bar"): 899 | fs.open("foo", "wb").write(b"bar") 900 | transaction.commit() 901 | 902 | with fs.cd("foo bar"): 903 | assert fs.open("foo", "rb").read() == b"bar" 904 | 905 | 906 | @mock.patch("acidfs.subprocess.Popen") 907 | def test_called_process_error(Popen): 908 | from acidfs import _popen 909 | 910 | Popen.return_value.return_value.wait.return_value = 1 911 | with pytest.raises(subprocess.CalledProcessError): 912 | with _popen(["what", "ever"]): 913 | pass 914 | 915 | 916 | def fprint(f, s): 917 | f.write(s) 918 | if isinstance(f, io.TextIOWrapper): 919 | f.write(u"\n") 920 | else: 921 | f.write(b"\n") 922 | 923 | 924 | def count_commits(tmp): 925 | output = _check_output(["git", "log"], cwd=tmp) 926 | commits = 0 927 | for line in output.split(b"\n"): 928 | if line.startswith(b"commit "): 929 | commits += 1 930 | 931 | return commits 932 | --------------------------------------------------------------------------------