├── core ├── fs │ ├── local │ │ ├── windows │ │ │ ├── __init__.py │ │ │ ├── network.py │ │ │ └── drives.py │ │ └── __init__.py │ ├── __init__.py │ └── zip.py ├── tests │ ├── commands │ │ ├── __init__.py │ │ ├── test_goto.py │ │ └── test___init__.py │ ├── fs │ │ ├── ZipFileSystemTest.zip │ │ ├── __init__.py │ │ ├── test_columns.py │ │ ├── test_local.py │ │ └── test_zip.py │ ├── test_util.py │ ├── test_quicksearch_matchers.py │ ├── __init__.py │ └── test_fileoperations.py ├── commands │ ├── util.py │ ├── explorer_properties.py │ └── goto.py ├── trash.py ├── quicksearch_matchers.py ├── util.py ├── github.py ├── os_.py ├── __init__.py └── fileoperations.py ├── .gitignore ├── bin ├── mac │ └── 7za ├── linux │ └── 7za └── windows │ └── 7za.exe ├── Theme (Linux).css ├── Theme (Windows).css ├── .extrafiles ├── Core Settings.json ├── Core Settings (Mac).json ├── Folder Context Menu (Linux).json ├── Core Settings (Windows).json ├── Folder Context Menu (Mac).json ├── Theme (Mac).css ├── .editorconfig ├── Folder Context Menu (Windows).json ├── File Context Menu (Mac).json ├── File Context Menu (Linux).json ├── Context Menu (Linux).json ├── File Context Menu (Windows).json ├── Theme.css ├── Context Menu (Windows).json ├── Key Bindings (Linux).json ├── Key Bindings (Windows).json ├── README.md ├── Key Bindings (Mac).json └── Key Bindings.json /core/fs/local/windows/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /core/tests/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.py[cod] 3 | *.ttf -------------------------------------------------------------------------------- /core/fs/__init__.py: -------------------------------------------------------------------------------- 1 | from .local import * 2 | from .zip import * -------------------------------------------------------------------------------- /bin/mac/7za: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fman-users/Core/HEAD/bin/mac/7za -------------------------------------------------------------------------------- /bin/linux/7za: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fman-users/Core/HEAD/bin/linux/7za -------------------------------------------------------------------------------- /bin/windows/7za.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fman-users/Core/HEAD/bin/windows/7za.exe -------------------------------------------------------------------------------- /Theme (Linux).css: -------------------------------------------------------------------------------- 1 | * { 2 | font-family: "Open Sans"; 3 | } 4 | 5 | .locationbar { 6 | padding: .5ex; 7 | } -------------------------------------------------------------------------------- /Theme (Windows).css: -------------------------------------------------------------------------------- 1 | * { 2 | font-family: "Roboto Bold"; 3 | } 4 | 5 | .locationbar { 6 | padding: 1ex; 7 | } -------------------------------------------------------------------------------- /core/tests/fs/ZipFileSystemTest.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fman-users/Core/HEAD/core/tests/fs/ZipFileSystemTest.zip -------------------------------------------------------------------------------- /.extrafiles: -------------------------------------------------------------------------------- 1 | # List the files in this repo that do not come from the Core plugin: 2 | .extrafiles 3 | .git 4 | .editorconfig 5 | .gitignore 6 | README.md -------------------------------------------------------------------------------- /Core Settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "archive_handlers": { 3 | ".zip": "zip://", 4 | ".zipx": "zip://", 5 | ".jar": "zip://", 6 | ".xpi": "zip://", 7 | ".7z": "7z://", 8 | ".tar": "tar://" 9 | } 10 | } -------------------------------------------------------------------------------- /Core Settings (Mac).json: -------------------------------------------------------------------------------- 1 | { 2 | "terminal": { 3 | "args": ["/usr/bin/open", "-a", "Terminal", "{curr_dir}"] 4 | }, 5 | "native_file_manager": { 6 | "args": ["/usr/bin/open", "-a", "Finder", "{curr_dir}"] 7 | } 8 | } -------------------------------------------------------------------------------- /Folder Context Menu (Linux).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "-", "id": "new" }, 3 | { "command": "create_directory", "caption": "New &Folder" }, 4 | { "caption": "-", "id": "clipboard" }, 5 | { "command": "paste", "caption": "&Paste" } 6 | ] -------------------------------------------------------------------------------- /Core Settings (Windows).json: -------------------------------------------------------------------------------- 1 | { 2 | "terminal": { 3 | "args": "start C:\\Windows\\System32\\cmd.exe", "shell": true, "cwd": "{curr_dir}" 4 | }, 5 | "native_file_manager": { 6 | "args": ["start", "explorer", "{curr_dir}"], "shell": true 7 | } 8 | } -------------------------------------------------------------------------------- /Folder Context Menu (Mac).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "-", "id": "new" }, 3 | { "command": "create_directory" }, 4 | { "caption": "-", "id": "properties" }, 5 | { "command": "get_info" }, 6 | { "caption": "-", "id": "clipboard" }, 7 | { "command": "paste" } 8 | ] -------------------------------------------------------------------------------- /Theme (Mac).css: -------------------------------------------------------------------------------- 1 | * { 2 | font-size: 13pt; 3 | } 4 | 5 | th { 6 | font-size: 12pt; 7 | } 8 | 9 | .statusbar { 10 | font-size: 11pt; 11 | } 12 | 13 | .quicksearch-query, .quicksearch-item-title { 14 | font-size: 15pt; 15 | } 16 | 17 | .quicksearch-item-hint { 18 | font-size: 12pt; 19 | } -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | 7 | # Matches multiple files with brace expansion notation 8 | [*.{py,css,json}] 9 | charset = utf-8 10 | indent_style = tab 11 | indent_size = 4 12 | trim_trailing_whitespace = true 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false -------------------------------------------------------------------------------- /Folder Context Menu (Windows).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "-", "id": "clipboard" }, 3 | { "command": "paste", "caption": "&Paste" }, 4 | { "caption": "-", "id": "new" }, 5 | { "command": "create_directory", "caption": "New &Folder" }, 6 | { "caption": "-", "id": "properties" }, 7 | { "command": "show_explorer_properties", "caption": "P&roperties" } 8 | ] -------------------------------------------------------------------------------- /File Context Menu (Mac).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "-", "id": "open" }, 3 | { "command": "open_selected_files", "caption": "Open" }, 4 | { "command": "open_with" }, 5 | { "caption": "-", "id": "file_operations" }, 6 | { "command": "move_to_trash", "caption": "Move to Trash" }, 7 | { "command": "rename" }, 8 | { "caption": "-" }, 9 | { "command": "get_info", "id": "properties" }, 10 | { "command": "pack", "caption": "Compress...", "id": "archive" }, 11 | { "caption": "-", "id": "clipboard" }, 12 | { "command": "copy_to_clipboard", "caption": "Copy" } 13 | ] -------------------------------------------------------------------------------- /core/commands/util.py: -------------------------------------------------------------------------------- 1 | from getpass import getuser 2 | from os.path import expanduser 3 | from PyQt5.QtCore import QFileInfo 4 | 5 | import os 6 | 7 | def get_program_files(): 8 | return os.environ.get('PROGRAMW6432', r'C:\Program Files') 9 | 10 | def get_program_files_x86(): 11 | return os.environ.get('PROGRAMFILES', r'C:\Program Files (x86)') 12 | 13 | def get_user(): 14 | try: 15 | return getuser() 16 | except Exception: 17 | return os.path.basename(expanduser('~')) 18 | 19 | def is_hidden(file_path): 20 | return QFileInfo(file_path).isHidden() -------------------------------------------------------------------------------- /File Context Menu (Linux).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "-", "id": "open" }, 3 | { "command": "open_selected_files", "caption": "&Open" }, 4 | { "command": "open_with", "caption": "Open wit&h..." }, 5 | { "caption": "-", "id": "clipboard" }, 6 | { "command": "cut", "caption": "Cu&t" }, 7 | { "command": "copy_to_clipboard", "caption": "&Copy" }, 8 | { "caption": "-", "id": "file_operations" }, 9 | { "command": "rename", "caption": "Rena&me" }, 10 | { "command": "move_to_trash", "caption": "Mo&ve to Trash" }, 11 | { "caption": "-", "id": "archive" }, 12 | { "command": "pack", "caption": "Com&press..." } 13 | ] -------------------------------------------------------------------------------- /Context Menu (Linux).json: -------------------------------------------------------------------------------- 1 | [{ 2 | "on_file": [ 3 | { "command": "open_selected_files", "caption": "&Open" }, 4 | { "command": "open_with", "caption": "Open wit&h..." }, 5 | { "caption": "-" }, 6 | { "command": "cut", "caption": "Cu&t" }, 7 | { "command": "copy_to_clipboard", "caption": "&Copy" }, 8 | { "caption": "-" }, 9 | { "command": "rename", "caption": "Rena&me" }, 10 | { "command": "move_to_trash", "caption": "Mo&ve to Trash" }, 11 | { "caption": "-" }, 12 | { "command": "pack", "caption": "Com&press..." } 13 | ], 14 | "in_directory": [ 15 | { "command": "create_directory", "caption": "New &Folder" }, 16 | { "caption": "-" }, 17 | { "command": "paste", "caption": "&Paste" } 18 | ] 19 | }] -------------------------------------------------------------------------------- /File Context Menu (Windows).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "-", "id": "open" }, 3 | { "command": "open_selected_files", "caption": "&Open" }, 4 | { "command": "open_with", "caption": "Open wit&h..." }, 5 | { "caption": "-", "id": "clipboard" }, 6 | { "command": "cut", "caption": "Cu&t" }, 7 | { "command": "copy_to_clipboard", "caption": "&Copy" }, 8 | { "caption": "-", "id": "file_operations" }, 9 | { "command": "move_to_trash", "caption": "&Delete" }, 10 | { "command": "rename", "caption": "Rena&me" }, 11 | { "caption": "-", "id": "archive" }, 12 | { "command": "pack", "caption": "Add to archive..." }, 13 | { "caption": "-", "id": "properties" }, 14 | { "command": "show_explorer_properties", "caption": "P&roperties" } 15 | ] -------------------------------------------------------------------------------- /core/tests/test_util.py: -------------------------------------------------------------------------------- 1 | from core import strformat_dict_values 2 | from unittest import TestCase 3 | 4 | class TestStrformatDictValues(TestCase): 5 | def test_empty(self): 6 | self.assertEqual({}, strformat_dict_values({}, {'a': 'b'})) 7 | def test_simple_replacement(self): 8 | self.assertEqual( 9 | {'a': 'a was replaced', 'c': 'd'}, 10 | strformat_dict_values( 11 | {'a': '{a}', 'c': 'd'}, 12 | {'a': 'a was replaced'} 13 | ) 14 | ) 15 | def test_list(self): 16 | self.assertEqual( 17 | {'list': ['replaced']}, 18 | strformat_dict_values( 19 | {'list': ['{replaceme}']}, 20 | {'replaceme': 'replaced'} 21 | ) 22 | ) 23 | def test_list_ints(self): 24 | dict_ = {'list': [1]} 25 | self.assertEqual(dict_, strformat_dict_values(dict_, {'a': 'b'})) -------------------------------------------------------------------------------- /Theme.css: -------------------------------------------------------------------------------- 1 | * { 2 | font-size: 9pt; 3 | } 4 | 5 | th { 6 | font-size: 8pt; 7 | } 8 | 9 | .locationbar { 10 | padding: 0.25ex; 11 | border-bottom: 1px solid #262626; 12 | } 13 | 14 | .statusbar { 15 | font-size: 8pt; 16 | } 17 | 18 | .quicksearch-query, .quicksearch-item-title { 19 | font-size: 10pt; 20 | } 21 | 22 | .quicksearch-item { 23 | padding-top: 1px; 24 | padding-left: 4px; 25 | padding-right: 4px; 26 | border-top: 1px solid #4d4d4d; 27 | border-bottom: 1px solid #363636; 28 | } 29 | 30 | .quicksearch-item-title { 31 | color: #c8c8c8; 32 | } 33 | 34 | .quicksearch-item-title-highlight { 35 | color: white; 36 | } 37 | 38 | .quicksearch-item-hint { 39 | font-size: 8pt; 40 | color: white; 41 | } 42 | 43 | .quicksearch-item-description { 44 | color: #8f908a; 45 | } -------------------------------------------------------------------------------- /Context Menu (Windows).json: -------------------------------------------------------------------------------- 1 | [{ 2 | "on_file": [ 3 | { "command": "open_selected_files", "caption": "&Open" }, 4 | { "command": "open_with", "caption": "Open wit&h..." }, 5 | { "caption": "-" }, 6 | { "command": "cut", "caption": "Cu&t" }, 7 | { "command": "copy_to_clipboard", "caption": "&Copy" }, 8 | { "caption": "-" }, 9 | { "command": "move_to_trash", "caption": "&Delete" }, 10 | { "command": "rename", "caption": "Rena&me" }, 11 | { "caption": "-" }, 12 | { "command": "pack", "caption": "Add to archive..." }, 13 | { "caption": "-" }, 14 | { "command": "show_explorer_properties", "caption": "P&roperties" } 15 | ], 16 | "in_directory": [ 17 | { "command": "paste", "caption": "&Paste" }, 18 | { "caption": "-" }, 19 | { "command": "create_directory", "caption": "New &Folder" }, 20 | { "caption": "-" }, 21 | { "command": "show_explorer_properties", "caption": "P&roperties" } 22 | ] 23 | }] -------------------------------------------------------------------------------- /core/tests/test_quicksearch_matchers.py: -------------------------------------------------------------------------------- 1 | from core.quicksearch_matchers import contains_chars_after_separator 2 | from unittest import TestCase 3 | 4 | class ContainsCharsAfterSeparatorTest(TestCase): 5 | def test_simple(self): 6 | self.assertEqual( 7 | [0, 5], self.find_chars_after_space('copy paths', 'cp') 8 | ) 9 | def test_chars_in_first_and_second_part(self): 10 | self.assertEqual( 11 | [0, 1, 2], self.find_chars_after_space('copy paths', 'cop') 12 | ) 13 | def test_no_match(self): 14 | self.assertIsNone(self.find_chars_after_space('copy paths', 'cd')) 15 | def test_full_word_match(self): 16 | self.assertEqual( 17 | [0, 1, 2, 3, 5], 18 | self.find_chars_after_space('copy paths', 'copyp') 19 | ) 20 | def test_prefix_match(self): 21 | self.assertEqual( 22 | [0, 1], 23 | self.find_chars_after_space('column count', 'co') 24 | ) 25 | def setUp(self): 26 | super().setUp() 27 | self.find_chars_after_space = contains_chars_after_separator(' ') -------------------------------------------------------------------------------- /core/tests/fs/__init__.py: -------------------------------------------------------------------------------- 1 | from fman.fs import FileSystem, cached 2 | from fman.impl.util import filenotfounderror 3 | from fman.impl.util.path import normalize 4 | 5 | class StubFileSystem(FileSystem): 6 | 7 | scheme = 'stub://' 8 | 9 | def __init__(self, items): 10 | super().__init__() 11 | self._items = items 12 | def exists(self, path): 13 | return normalize(path) in self._items 14 | @cached # Mirror a typical implementation 15 | def is_dir(self, existing_path): 16 | path_resolved = normalize(existing_path) 17 | try: 18 | item = self._items[path_resolved] 19 | except KeyError: 20 | raise filenotfounderror(existing_path) 21 | return item.get('is_dir', False) 22 | @cached # Mirror a typical implementation 23 | def size_bytes(self, path): 24 | return self._items[normalize(path)].get('size', 1) 25 | @cached # Mirror a typical implementation 26 | def modified_datetime(self, path): 27 | return self._items[normalize(path)].get('mtime', 1473339041.0) 28 | def touch(self, path): 29 | file_existed = self.exists(path) 30 | self._items[normalize(path)] = {} 31 | if not file_existed: 32 | self.notify_file_added(path) -------------------------------------------------------------------------------- /Key Bindings (Linux).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "keys": ["Ctrl+Right"], "command": "open_in_right_pane" }, 3 | { "keys": ["Ctrl+Left"], "command": "open_in_left_pane" }, 4 | { "keys": ["Ins"], "command": "move_cursor_down", "args": {"toggle_selection": true} }, 5 | { "keys": ["Ctrl+C"], "command": "copy_to_clipboard" }, 6 | { "keys": ["Ctrl+X"], "command": "cut" }, 7 | { "keys": ["Ctrl+V"], "command": "paste" }, 8 | { "keys": ["Ctrl+A"], "command": "select_all" }, 9 | { "keys": ["Ctrl+D"], "command": "deselect" }, 10 | { "keys": ["Ctrl+."], "command": "toggle_hidden_files" }, 11 | { "keys": ["Ctrl+P"], "command": "go_to" }, 12 | { "keys": ["Ctrl+Shift+P"], "command": "command_palette" }, 13 | { "keys": ["Ctrl+Q"], "command": "quit" }, 14 | { "keys": ["Alt+Left"], "command": "go_back" }, 15 | { "keys": ["Alt+Right"], "command": "go_forward" }, 16 | { "keys": ["Alt+F5"], "command": "pack" }, 17 | { "keys": ["Ctrl+R"], "command": "reload" }, 18 | { "keys": ["Ctrl+F1"], "command": "sort_by_column", "args": {"column_index": 0}}, 19 | { "keys": ["Ctrl+F2"], "command": "sort_by_column", "args": {"column_index": 1}}, 20 | { "keys": ["Ctrl+F3"], "command": "sort_by_column", "args": {"column_index": 2}}, 21 | { "keys": ["Alt+Up"], "command": "go_up" } 22 | ] -------------------------------------------------------------------------------- /core/trash.py: -------------------------------------------------------------------------------- 1 | from errno import EINVAL 2 | from fman import PLATFORM 3 | from os import strerror 4 | 5 | if PLATFORM == 'Mac': 6 | from osxtrash import move_to_trash 7 | else: 8 | def move_to_trash(*files): 9 | send2trash = _import_send2trash() 10 | for file in files: 11 | try: 12 | send2trash(file) 13 | except OSError as e: 14 | if PLATFORM == 'Windows': 15 | if e.errno == EINVAL and e.winerror == _DE_INVALIDFILES: 16 | message = strerror(EINVAL) + ' (file may be in use)' 17 | raise OSError(EINVAL, message) 18 | raise 19 | 20 | def _import_send2trash(): 21 | # We import send2trash as late as possible. Here's why: On Ubuntu 22 | # (/Gnome), send2trash uses GIO - if it is available and initialized. 23 | # Whether that happens is determined at *import time*. Importing 24 | # send2trash at the last possible moment, ensures that it picks up GIO. 25 | from send2trash import send2trash as result 26 | if PLATFORM == 'Linux': 27 | try: 28 | from gi.repository import GObject 29 | except ImportError: 30 | pass 31 | else: 32 | # Fix for elementary OS / Pantheon: 33 | if not hasattr(GObject, 'GError'): 34 | from send2trash.plat_other import send2trash as result 35 | return result 36 | 37 | # Windows constant: 38 | _DE_INVALIDFILES = 0x7C -------------------------------------------------------------------------------- /Key Bindings (Windows).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "keys": ["Ctrl+Right"], "command": "open_in_right_pane" }, 3 | { "keys": ["Ctrl+Left"], "command": "open_in_left_pane" }, 4 | { "keys": ["Ins"], "command": "move_cursor_down", "args": {"toggle_selection": true} }, 5 | { "keys": ["Ctrl+C"], "command": "copy_to_clipboard" }, 6 | { "keys": ["Ctrl+X"], "command": "cut" }, 7 | { "keys": ["Ctrl+V"], "command": "paste" }, 8 | { "keys": ["Ctrl+A"], "command": "select_all" }, 9 | { "keys": ["Ctrl+D"], "command": "deselect" }, 10 | { "keys": ["Ctrl+."], "command": "toggle_hidden_files" }, 11 | { "keys": ["Ctrl+P"], "command": "go_to" }, 12 | { "keys": ["Ctrl+Shift+P"], "command": "command_palette" }, 13 | { "keys": ["Alt+Left"], "command": "go_back" }, 14 | { "keys": ["Alt+Right"], "command": "go_forward" }, 15 | { "keys": ["Alt+F5"], "command": "pack" }, 16 | { "keys": ["Ctrl+R"], "command": "reload"}, 17 | { "keys": ["Ctrl+F1"], "command": "sort_by_column", "args": {"column_index": 0}}, 18 | { "keys": ["Ctrl+F2"], "command": "sort_by_column", "args": {"column_index": 1}}, 19 | { "keys": ["Ctrl+F3"], "command": "sort_by_column", "args": {"column_index": 2}}, 20 | { "keys": ["Alt+Up"], "command": "go_up" }, 21 | { "keys": ["Alt+Enter"], "command": "show_explorer_properties" }, 22 | { "keys": ["Ctrl+\\"], "command": "go_to_root_of_current_drive" } 23 | ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Core 2 | The Core plugin implements most of [fman](https://fman.io)'s features, such as copying files, navigating to a folder etc. It relies heavily on the [plugin API](https://fman.io/docs/api). fman uses its own plugin system to ensure that the API is stable and powerful enough for real world use. 3 | 4 | ## Examples 5 | 6 | * [Key Bindings.json](Key%20Bindings.json) defines the default key bindings 7 | * [Theme.css](Theme.css) defines fman's visual appearance (to [some extent](https://github.com/fman-users/fman/issues/45)) 8 | * [commands/](core/commands/__init__.py) implements virtually all commands 9 | * [local/](core/fs/local/__init__.py) lets fman work with the files on your local hard drive 10 | * [zip.py](core/fs/zip.py) adds support for ZIP files 11 | 12 | ## License 13 | This repository is made public for educational purposes only. You may only use it to implement plugins for [fman](https://fman.io). You are not allowed to use any parts of this code for other projects, in particular to implement other file managers. 14 | 15 | ## Location in your installation directory 16 | You can also find these source files in your fman installation directory. Their exact path depends on your operating system: 17 | 18 | * **Windows:** `C:/​Users/​/​AppData/​Local/​fman/​Versions/​/​Plugins/​Core` 19 | * **Mac:** `/​Applications/​fman.app/​Contents/​Resources/​Plugins/​Core` 20 | * **Linux:** `/opt/​fman/​Plugins/​Core` 21 | -------------------------------------------------------------------------------- /core/quicksearch_matchers.py: -------------------------------------------------------------------------------- 1 | from os.path import basename 2 | 3 | import os 4 | 5 | def path_starts_with(path, query): 6 | # We do want to return ~/Downloads when query is ~/Downloads/: 7 | query = query.rstrip(os.sep) 8 | if path.lower().startswith(query.lower()): 9 | return list(range(len(query))) 10 | 11 | def basename_starts_with(path, query): 12 | name = basename(path.lower()) 13 | if name.startswith(query.lower()): 14 | offset = len(path) - len(name) 15 | return [i + offset for i in range(len(query))] 16 | 17 | def contains_chars(text, query): 18 | indices = [] 19 | i = 0 20 | for char in query: 21 | try: 22 | i += text[i:].index(char) 23 | except ValueError: 24 | return None 25 | indices.append(i) 26 | i += 1 27 | return indices 28 | 29 | def contains_substring(text, query): 30 | try: 31 | start = text.index(query) 32 | except ValueError: 33 | return None 34 | return list(range(start, start + len(query))) 35 | 36 | def contains_chars_after_separator(separator): 37 | def result(text, query): 38 | result_ = [] 39 | skip_to_next_part = False 40 | for i, char in enumerate(text): 41 | if skip_to_next_part: 42 | if char == separator: 43 | skip_to_next_part = False 44 | continue 45 | if not query: 46 | break 47 | if char == query[0]: 48 | result_.append(i) 49 | query = query[1:] 50 | else: 51 | skip_to_next_part = char != separator 52 | if query: 53 | return None 54 | return result_ 55 | return result -------------------------------------------------------------------------------- /core/util.py: -------------------------------------------------------------------------------- 1 | from fman.url import dirname 2 | from os import listdir, strerror 3 | from os.path import join 4 | from pathlib import PurePosixPath 5 | 6 | import errno 7 | import fman.fs 8 | 9 | def strformat_dict_values(dict_, replacements): 10 | result = {} 11 | def replace(value): 12 | if isinstance(value, str): 13 | return value.format(**replacements) 14 | return value 15 | for key, value in dict_.items(): 16 | if isinstance(value, list): 17 | value = list(map(replace, value)) 18 | else: 19 | value = replace(value) 20 | result[key] = value 21 | return result 22 | 23 | def listdir_absolute(dir_path): 24 | return [join(dir_path, file_name) for file_name in listdir(dir_path)] 25 | 26 | def filenotfounderror(path): 27 | # The correct way of instantiating FileNotFoundError in a way that respects 28 | # the parent class (OSError)'s arguments: 29 | return FileNotFoundError(errno.ENOENT, strerror(errno.ENOENT), path) 30 | 31 | def parent(path): 32 | if path == '/': 33 | return '' 34 | result = str(PurePosixPath(path).parent) if path else '' 35 | return '' if result == '.' else result 36 | 37 | def is_parent(dir_url, file_url, fs=fman.fs): 38 | for parent_url in _iter_parents(file_url): 39 | try: 40 | if fs.samefile(parent_url, dir_url): 41 | return True 42 | except FileNotFoundError: 43 | continue 44 | return False 45 | 46 | def _iter_parents(url): 47 | while True: 48 | yield url 49 | new_url = dirname(url) 50 | if new_url == url: 51 | break 52 | url = new_url -------------------------------------------------------------------------------- /Key Bindings (Mac).json: -------------------------------------------------------------------------------- 1 | [ 2 | { "keys": ["Alt+Right"], "command": "open_in_right_pane" }, 3 | { "keys": ["Alt+Left"], "command": "open_in_left_pane" }, 4 | { "keys": ["Cmd+C"], "command": "copy_to_clipboard" }, 5 | { "keys": ["Cmd+X"], "command": "cut" }, 6 | { "keys": ["Cmd+V"], "command": "paste" }, 7 | { "keys": ["Cmd+Alt+V"], "command": "paste_cut" }, 8 | { "keys": ["Cmd+A"], "command": "select_all" }, 9 | { "keys": ["Cmd+D"], "command": "deselect" }, 10 | { "keys": ["Space"], "command": "move_cursor_down", "args": {"toggle_selection": true} }, 11 | { "keys": ["Cmd+Backspace"], "command": "move_to_trash" }, 12 | { "keys": ["Cmd+."], "command": "toggle_hidden_files" }, 13 | { "keys": ["Cmd+P"], "command": "go_to" }, 14 | { "keys": ["Cmd+Shift+P"], "command": "command_palette" }, 15 | { "keys": ["Cmd+Left"], "command": "go_back" }, 16 | { "keys": ["Cmd+Right"], "command": "go_forward" }, 17 | { "keys": ["Cmd+I"], "command": "get_info" }, 18 | { "keys": ["Cmd+F5"], "command": "pack" }, 19 | { "keys": ["Cmd+R"], "command": "reload" }, 20 | { "keys": ["Cmd+F1"], "command": "sort_by_column", "args": {"column_index": 0}}, 21 | { "keys": ["Cmd+F2"], "command": "sort_by_column", "args": {"column_index": 1}}, 22 | { "keys": ["Cmd+F3"], "command": "sort_by_column", "args": {"column_index": 2}}, 23 | { "keys": ["Cmd+M"], "command": "minimize" }, 24 | { "keys": ["Cmd+Q"], "command": "quit" }, 25 | { "keys": ["Cmd+Up"], "command": "go_up" }, 26 | { "keys": ["Shift+Space"], "command": "quick_look" }, 27 | { "keys": ["Cmd+Enter"], "command": "open_selected_files" }, 28 | { "keys": ["Cmd+Ctrl+F"], "command": "toggle_fullscreen" } 29 | ] -------------------------------------------------------------------------------- /core/fs/local/windows/network.py: -------------------------------------------------------------------------------- 1 | from core.util import filenotfounderror 2 | from fman.fs import FileSystem 3 | from fman.url import as_url 4 | from win32wnet import WNetOpenEnum, WNetEnumResource, error as WNetError, \ 5 | NETRESOURCE, WNetGetResourceInformation 6 | 7 | RESOURCE_GLOBALNET = 0x00000002 8 | RESOURCETYPE_DISK = 0x00000001 9 | 10 | class NetworkFileSystem(FileSystem): 11 | 12 | scheme = 'network://' 13 | 14 | def resolve(self, path): 15 | if '/' in path: 16 | return as_url(r'\\' + path.replace('/', '\\')) 17 | return super().resolve(path) 18 | def exists(self, path): 19 | if not path: 20 | return True 21 | nr = NETRESOURCE() 22 | nr.lpRemoteName = r'\\' + path.replace('/', '\\') 23 | try: 24 | WNetGetResourceInformation(nr) 25 | except WNetError: 26 | return False 27 | return True 28 | def is_dir(self, existing_path): 29 | if self.exists(existing_path): 30 | return True 31 | raise filenotfounderror(existing_path) 32 | def iterdir(self, path): 33 | if path: 34 | handle = NETRESOURCE() 35 | handle.lpRemoteName = r'\\' + path.replace('/', '\\') 36 | try: 37 | WNetGetResourceInformation(handle) 38 | except WNetError: 39 | raise filenotfounderror(path) 40 | else: 41 | handle = None 42 | for subpath in self._iter_handle(handle): 43 | if '/' in subpath: 44 | head, tail = subpath.rsplit('/', 1) 45 | if head == path: 46 | yield tail 47 | elif not path: 48 | yield subpath 49 | def _iter_handle(self, handle, already_visited=None): 50 | if already_visited is None: 51 | already_visited = set() 52 | try: 53 | enum = \ 54 | WNetOpenEnum(RESOURCE_GLOBALNET, RESOURCETYPE_DISK, 0, handle) 55 | except WNetError: 56 | pass 57 | else: 58 | try: 59 | items = iter(WNetEnumResource(enum, 0)) 60 | while True: 61 | try: 62 | item = next(items) 63 | except (StopIteration, WNetError): 64 | break 65 | else: 66 | path = item.lpRemoteName 67 | if path.startswith(r'\\'): 68 | yield path[2:].replace('\\', '/').rstrip('/') 69 | if path not in already_visited: 70 | already_visited.add(path) 71 | yield from self._iter_handle(item, already_visited) 72 | finally: 73 | enum.Close() -------------------------------------------------------------------------------- /core/fs/local/windows/drives.py: -------------------------------------------------------------------------------- 1 | from core.fs.local.windows.network import NetworkFileSystem 2 | from core.util import filenotfounderror 3 | from ctypes import windll 4 | from fman.fs import FileSystem, Column 5 | from fman.url import as_url, splitscheme 6 | 7 | import ctypes 8 | import string 9 | 10 | class DrivesFileSystem(FileSystem): 11 | 12 | scheme = 'drives://' 13 | 14 | NETWORK = 'Network...' 15 | 16 | def get_default_columns(self, path): 17 | return DriveName.__module__ + '.' + DriveName.__name__, 18 | def resolve(self, path): 19 | if not path: 20 | # Showing the list of all drives: 21 | return self.scheme 22 | if path in self._get_drives(): 23 | return as_url(path + '\\') 24 | if path == self.NETWORK: 25 | return NetworkFileSystem.scheme 26 | raise filenotfounderror(path) 27 | def iterdir(self, path): 28 | if path: 29 | raise filenotfounderror(path) 30 | return self._get_drives() + [self.NETWORK] 31 | def is_dir(self, existing_path): 32 | if self.exists(existing_path): 33 | return True 34 | raise filenotfounderror(existing_path) 35 | def exists(self, path): 36 | return not path or path in self._get_drives() or path == self.NETWORK 37 | def _get_drives(self): 38 | result = [] 39 | bitmask = windll.kernel32.GetLogicalDrives() 40 | for letter in string.ascii_uppercase: 41 | if bitmask & 1: 42 | result.append(letter + ':') 43 | bitmask >>= 1 44 | return result 45 | 46 | class DriveName(Column): 47 | 48 | display_name = 'Name' 49 | 50 | def get_str(self, url): 51 | scheme, path = splitscheme(url) 52 | if scheme != 'drives://': 53 | raise ValueError('Unsupported URL: %r' % url) 54 | if path == DrivesFileSystem.NETWORK: 55 | return path 56 | result = path 57 | try: 58 | vol_name = self._get_volume_name(path + '\\') 59 | except WindowsError: 60 | pass 61 | else: 62 | if vol_name: 63 | result += ' ' + vol_name 64 | return result 65 | def get_sort_value(self, url, is_ascending): 66 | path = splitscheme(url)[1] 67 | # Always show "Network..." at the bottom/top: 68 | return path == DrivesFileSystem.NETWORK, self.get_str(url).lower() 69 | def _get_volume_name(self, volume_path): 70 | kernel32 = windll.kernel32 71 | buffer = ctypes.create_unicode_buffer(1024) 72 | if not kernel32.GetVolumeInformationW( 73 | ctypes.c_wchar_p(volume_path), buffer, ctypes.sizeof(buffer), 74 | None, None, None, None, 0 75 | ): 76 | raise ctypes.WinError() 77 | return buffer.value -------------------------------------------------------------------------------- /Key Bindings.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "keys": ["Down"], "command": "move_cursor_down" }, 3 | { "keys": ["Num+Down"], "command": "move_cursor_down" }, 4 | { "keys": ["Shift+Down"], "command": "move_cursor_down", "args": {"toggle_selection": true} }, 5 | { "keys": ["Up"], "command": "move_cursor_up" }, 6 | { "keys": ["Num+Up"], "command": "move_cursor_up" }, 7 | { "keys": ["Shift+Up"], "command": "move_cursor_up", "args": {"toggle_selection": true} }, 8 | { "keys": ["Home"], "command": "move_cursor_home" }, 9 | { "keys": ["Num+Home"], "command": "move_cursor_home" }, 10 | { "keys": ["Shift+Home"], "command": "move_cursor_home", "args": {"toggle_selection": true} }, 11 | { "keys": ["End"], "command": "move_cursor_end" }, 12 | { "keys": ["Num+End"], "command": "move_cursor_end" }, 13 | { "keys": ["Shift+End"], "command": "move_cursor_end", "args": {"toggle_selection": true} }, 14 | { "keys": ["PgDown"], "command": "move_cursor_page_down" }, 15 | { "keys": ["Num+PgDown"], "command": "move_cursor_page_down" }, 16 | { "keys": ["Shift+PgDown"], "command": "move_cursor_page_down", "args": {"toggle_selection": true} }, 17 | { "keys": ["PgUp"], "command": "move_cursor_page_up" }, 18 | { "keys": ["Num+PgUp"], "command": "move_cursor_page_up" }, 19 | { "keys": ["Shift+PgUp"], "command": "move_cursor_page_up", "args": {"toggle_selection": true} }, 20 | { "keys": ["Space"], "command": "toggle_selection" }, 21 | { "keys": ["Tab"], "command": "switch_panes" }, 22 | { "keys": ["Backspace"], "command": "go_up" }, 23 | { "keys": ["Enter"], "command": "open" }, 24 | { "keys": ["F1"], "command": "help" }, 25 | { "keys": ["F4"], "command": "open_with_editor" }, 26 | { "keys": ["Shift+F4"], "command": "create_and_edit_file" }, 27 | { "keys": ["F5"], "command": "copy" }, 28 | { "keys": ["Shift+F5"], "command": "symlink" }, 29 | { "keys": ["Shift+F6"], "command": "rename" }, 30 | { "keys": ["F6"], "command": "move" }, 31 | { "keys": ["F7"], "command": "create_directory" }, 32 | { "keys": ["F8"], "command": "move_to_trash" }, 33 | { "keys": ["Delete"], "command": "move_to_trash" }, 34 | { "keys": ["Num+Delete"], "command": "move_to_trash" }, 35 | { "keys": ["F9"], "command": "open_terminal" }, 36 | { "keys": ["F10"], "command": "open_native_file_manager" }, 37 | { "keys": ["F11"], "command": "copy_paths_to_clipboard" }, 38 | { "keys": ["Shift+Delete"], "command": "delete_permanently" }, 39 | { "keys": ["Alt+F1"], "command": "show_volumes", "args": {"pane_index": 0} }, 40 | { "keys": ["Alt+F2"], "command": "show_volumes", "args": {"pane_index": 1} }, 41 | { "keys": ["Num+Insert"], "command": "move_cursor_down", "args": {"toggle_selection": true} } 42 | ] -------------------------------------------------------------------------------- /core/github.py: -------------------------------------------------------------------------------- 1 | from requests import RequestException 2 | from urllib.error import HTTPError, URLError 3 | from urllib.request import urlopen 4 | 5 | import json 6 | import re 7 | import requests 8 | import sys 9 | 10 | def find_repos(topics): 11 | query = '+'.join('topic:' + topic for topic in topics) 12 | url = "https://api.github.com/search/repositories?q=" + query 13 | return list(map(GitHubRepo, _fetch_all_pages(url))) 14 | 15 | def _fetch_all_pages(json_url, page_size=100): 16 | for page in range(1, sys.maxsize): 17 | data = _get_json(json_url + '&per_page=%d&page=%d' % (page_size, page)) 18 | yield from data['items'] 19 | has_more = page * page_size < data['total_count'] 20 | if not has_more: 21 | break 22 | 23 | class GitHubRepo: 24 | @classmethod 25 | def fetch(cls, repo): 26 | url = 'https://api.github.com/repos/' + repo 27 | return cls(_get_json(url)) 28 | def __init__(self, data): 29 | self._data = data 30 | def __str__(self): 31 | return self._data['full_name'] 32 | def __repr__(self): 33 | return '<%s: %s>' % (self.__class__.__name__, self) 34 | @property 35 | def num_stars(self): 36 | return self._data['stargazers_count'] 37 | @property 38 | def name(self): 39 | return self._data['name'] 40 | @property 41 | def description(self): 42 | return self._data['description'] 43 | @property 44 | def url(self): 45 | return self._data['url'] 46 | def get_latest_release(self): 47 | try: 48 | data = _get_json(self._url('releases', id='latest')) 49 | except HTTPError as e: 50 | if e.code == 404: 51 | raise LookupError() 52 | raise 53 | return data['tag_name'] 54 | def get_latest_commit(self): 55 | return _get_json(self._url('commits'))[0]['sha'] 56 | def download_zipball(self, ref): 57 | zipball_url = self._url('archive', archive_format='zipball', ref=ref) 58 | return _get(zipball_url) 59 | def _url(self, name, **kwargs): 60 | url = self._data[name + '_url'] 61 | required_url_params = re.finditer(r'{([^/][^}]+)}', url) 62 | for match in required_url_params: 63 | url = url.replace(match.group(0), kwargs[match.group(1)]) 64 | optional_url_params = re.finditer(r'{/([^}]+)}', url) 65 | for match in optional_url_params: 66 | param = match.group(1) 67 | value = '/' + kwargs[param] if param in kwargs else '' 68 | url = url.replace(match.group(0), value) 69 | return url 70 | 71 | def _get_json(url): 72 | return json.loads(_get(url).decode('utf-8')) 73 | 74 | def _get(url): 75 | try: 76 | return urlopen(url).read() 77 | except HTTPError: 78 | raise 79 | except URLError: 80 | # Fallback: Some users get "SSL: CERTIFICATE_VERIFY_FAILED" for urlopen. 81 | try: 82 | response = requests.get(url) 83 | except RequestException as e: 84 | raise URLError(e.__class__.__name__) 85 | if response.status_code != 200: 86 | raise HTTPError( 87 | url, response.status_code, response.reason, response.headers, 88 | None 89 | ) 90 | return response.content -------------------------------------------------------------------------------- /core/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from core import LocalFileSystem 2 | from fman import Task 3 | from fman.fs import FileSystem 4 | from fman.url import splitscheme, basename 5 | 6 | class StubUI: 7 | def __init__(self, test_case): 8 | self._expected_alerts = [] 9 | self._expected_prompts = [] 10 | self._test_case = test_case 11 | def expect_alert(self, args, answer): 12 | self._expected_alerts.append((args, answer)) 13 | def expect_prompt(self, args, answer): 14 | self._expected_prompts.append((args, answer)) 15 | def verify_expected_dialogs_were_shown(self): 16 | self._test_case.assertEqual( 17 | [], self._expected_alerts, 'Did not receive all expected alerts.' 18 | ) 19 | self._test_case.assertEqual( 20 | [], self._expected_prompts, 'Did not receive all expected prompts.' 21 | ) 22 | def show_alert(self, *args, **_): 23 | if not self._expected_alerts: 24 | self._test_case.fail('Unexpected alert: %r' % args[0]) 25 | return 26 | expected_args, answer = self._expected_alerts.pop(0) 27 | self._test_case.assertEqual(expected_args, args, "Wrong alert") 28 | return answer 29 | def show_prompt(self, *args, **_): 30 | if not self._expected_prompts: 31 | self._test_case.fail('Unexpected prompt: %r' % args[0]) 32 | return 33 | expected_args, answer = self._expected_prompts.pop(0) 34 | self._test_case.assertEqual(expected_args, args, "Wrong prompt") 35 | return answer 36 | def show_status_message(self, _): 37 | pass 38 | def clear_status_message(self): 39 | pass 40 | 41 | class StubFS(FileSystem): 42 | def __init__(self, backend=None): 43 | if backend is None: 44 | backend = LocalFileSystem() 45 | super().__init__() 46 | self._backends = {backend.scheme: backend} 47 | def add_child(self, backend): 48 | self._backends[backend.scheme] = backend 49 | def is_dir(self, url): 50 | scheme, path = splitscheme(url) 51 | return self._backends[scheme].is_dir(path) 52 | def exists(self, url): 53 | scheme, path = splitscheme(url) 54 | return self._backends[scheme].exists(path) 55 | def samefile(self, url1, url2): 56 | scheme1, path1 = splitscheme(url1) 57 | scheme2, path2 = splitscheme(url2) 58 | if scheme1 != scheme2: 59 | return False 60 | return self._backends[scheme1].samefile(path1, path2) 61 | def iterdir(self, url): 62 | scheme, path = splitscheme(url) 63 | return self._backends[scheme].iterdir(path) 64 | def makedirs(self, url, exist_ok=False): 65 | scheme, path = splitscheme(url) 66 | self._backends[scheme].makedirs(path, exist_ok=exist_ok) 67 | def copy(self, src_url, dst_url): 68 | scheme = splitscheme(src_url)[0] 69 | self._backends[scheme].copy(src_url, dst_url) 70 | def delete(self, url): 71 | scheme, path = splitscheme(url) 72 | self._backends[scheme].delete(path) 73 | def move(self, src_url, dst_url): 74 | scheme = splitscheme(src_url)[0] 75 | self._backends[scheme].move(src_url, dst_url) 76 | def touch(self, url): 77 | scheme, path = splitscheme(url) 78 | self._backends[scheme].touch(path) 79 | def mkdir(self, url): 80 | scheme, path = splitscheme(url) 81 | self._backends[scheme].mkdir(path) 82 | def query(self, url, fs_method_name): 83 | scheme, path = splitscheme(url) 84 | return getattr(self._backends[scheme], fs_method_name)(path) -------------------------------------------------------------------------------- /core/os_.py: -------------------------------------------------------------------------------- 1 | from core.util import strformat_dict_values 2 | from fman import load_json, show_alert, show_status_message, PLATFORM 3 | from shutil import which 4 | from subprocess import Popen, check_output 5 | 6 | import os 7 | import sys 8 | 9 | def is_arch(): 10 | try: 11 | return _get_os_release_name().startswith('Arch Linux') 12 | except FileNotFoundError: 13 | return False 14 | 15 | def is_mac(): 16 | return sys.platform == 'darwin' 17 | 18 | def get_popen_kwargs_for_opening(files, with_): 19 | args = [with_] + files 20 | if PLATFORM == 'Mac': 21 | args = ['/usr/bin/open', '-a'] + args 22 | return {'args': args} 23 | 24 | def open_terminal_in_directory(dir_path): 25 | settings = load_json('Core Settings.json', default={}) 26 | app = settings.get('terminal', {}) 27 | if app: 28 | _run_app_from_setting(app, dir_path) 29 | else: 30 | alternatives = [ 31 | 'x-terminal-emulator', # Debian-based 32 | 'konsole', # KDE 33 | 'gnome-terminal' # Gnome-based / Fedora 34 | ] 35 | for alternative in alternatives: 36 | binary = which(alternative) 37 | if binary: 38 | app = {'args': [binary], 'cwd': '{curr_dir}'} 39 | _run_app_from_setting(app, dir_path) 40 | break 41 | else: 42 | show_alert( 43 | 'Could not determine the Popen(...) arguments for opening the ' 44 | 'terminal. Please configure the "terminal" dictionary in ' 45 | '"Core Settings.json" as explained ' 46 | 'here.' 47 | ) 48 | 49 | def open_native_file_manager(dir_path): 50 | settings = load_json('Core Settings.json', default={}) 51 | app = settings.get('native_file_manager', {}) 52 | if app: 53 | _run_app_from_setting(app, dir_path) 54 | else: 55 | xdg_open = which('xdg-open') 56 | if xdg_open: 57 | app = {'args': [xdg_open, '{curr_dir}']} 58 | _run_app_from_setting(app, dir_path) 59 | if _is_ubuntu(): 60 | try: 61 | fpl = \ 62 | check_output(['dconf', 'read', _FOCUS_PREVENTION_LEVEL]) 63 | except FileNotFoundError as dconf_not_installed: 64 | pass 65 | else: 66 | if fpl in (b'', b'1\n'): 67 | show_status_message( 68 | 'Hint: If your OS\'s file manager opened in the ' 69 | 'background, click ' 70 | 'here.', 71 | timeout_secs=10 72 | ) 73 | else: 74 | show_alert( 75 | 'Could not determine the Popen(...) arguments for opening the ' 76 | 'native file manager. Please configure the ' 77 | '"native_file_manager" dictionary in "Core Settings.json" ' 78 | 'similarly to what\'s explained ' 79 | 'here.' 80 | ) 81 | 82 | def _is_ubuntu(): 83 | try: 84 | return _get_os_release_name().startswith('Ubuntu') 85 | except FileNotFoundError: 86 | return False 87 | 88 | def _run_app_from_setting(app, curr_dir): 89 | popen_kwargs = strformat_dict_values(app, {'curr_dir': curr_dir}) 90 | Popen(**popen_kwargs) 91 | 92 | def _is_gnome_based(): 93 | curr_desktop = os.environ.get('XDG_CURRENT_DESKTOP', '').lower() 94 | return curr_desktop in ('unity', 'gnome', 'x-cinnamon') 95 | 96 | def _get_os_release_name(): 97 | with open('/etc/os-release', 'r') as f: 98 | for line in f: 99 | line = line.rstrip() 100 | if line.startswith('NAME='): 101 | name = line[len('NAME='):] 102 | return name.strip('"') 103 | 104 | _FOCUS_PREVENTION_LEVEL = \ 105 | '/org/compiz/profiles/unity/plugins/core/focus-prevention-level' -------------------------------------------------------------------------------- /core/__init__.py: -------------------------------------------------------------------------------- 1 | from core.commands import * 2 | from core.fs import * 3 | from datetime import datetime 4 | from fman.fs import Column 5 | from fman.url import basename 6 | from math import log 7 | from PyQt5.QtCore import QLocale, QDateTime 8 | 9 | import fman.fs 10 | import re 11 | 12 | # Define here so get_default_columns(...) can reference it as core.Name: 13 | class Name(Column): 14 | def __init__(self, fs=fman.fs): 15 | super().__init__() 16 | self._fs = fs 17 | def get_str(self, url): 18 | return self._fs.query(url, 'name') 19 | def get_sort_value(self, url, is_ascending): 20 | try: 21 | is_dir = self._fs.is_dir(url) 22 | except FileNotFoundError: 23 | raise 24 | except OSError: 25 | is_dir = False 26 | major = is_dir ^ is_ascending 27 | str_ = self.get_str(url).lower() 28 | minor = '' 29 | while str_: 30 | match = re.search(r'\d+', str_) 31 | if match: 32 | minor += str_[:match.start()] 33 | minor += '%06d' % int(match.group(0)) 34 | str_ = str_[match.end():] 35 | else: 36 | minor += str_ 37 | break 38 | return major, minor 39 | 40 | # Define here so get_default_columns(...) can reference it as core.Size: 41 | class Size(Column): 42 | def __init__(self, fs=fman.fs): 43 | super().__init__() 44 | self._fs = fs 45 | def get_str(self, url): 46 | try: 47 | is_dir = self._fs.is_dir(url) 48 | except FileNotFoundError: 49 | raise 50 | except OSError: 51 | return '' 52 | if is_dir: 53 | return '' 54 | try: 55 | size_bytes = self._get_size(url) 56 | except OSError: 57 | return '' 58 | if size_bytes is None: 59 | return '' 60 | units = ('%d B', '%d KB', '%.1f MB', '%.1f GB') 61 | if size_bytes <= 0: 62 | unit_index = 0 63 | else: 64 | unit_index = min(int(log(size_bytes, 1000)), len(units) - 1) 65 | unit = units[unit_index] 66 | base = 1024 ** unit_index 67 | return unit % (size_bytes / base) 68 | def get_sort_value(self, url, is_ascending): 69 | try: 70 | is_dir = self._fs.is_dir(url) 71 | except FileNotFoundError: 72 | raise 73 | except OSError: 74 | is_dir = False 75 | if is_dir: 76 | ord_ = ord if is_ascending else lambda c: -ord(c) 77 | minor = tuple(ord_(c) for c in basename(url).lower()) 78 | else: 79 | try: 80 | minor = self._get_size(url) 81 | except OSError: 82 | minor = 0 83 | return is_dir ^ is_ascending, minor 84 | def _get_size(self, url): 85 | return self._fs.query(url, 'size_bytes') 86 | 87 | # Define here so get_default_columns(...) can reference it as core.Modified: 88 | class Modified(Column): 89 | def __init__(self, fs=fman.fs): 90 | super().__init__() 91 | self._fs = fs 92 | def get_str(self, url): 93 | try: 94 | mtime = self._get_mtime(url) 95 | except OSError: 96 | return '' 97 | if mtime is None: 98 | return '' 99 | try: 100 | timestamp = mtime.timestamp() 101 | except OSError: 102 | # This can occur in at least Python 3.6 on Windows. To reproduce: 103 | # datetime.min.timestamp() 104 | # This raises `OSError: [Errno 22] Invalid argument`. 105 | return '' 106 | mtime_qt = QDateTime.fromMSecsSinceEpoch(int(timestamp * 1000)) 107 | time_format = QLocale().dateTimeFormat(QLocale.ShortFormat) 108 | # Always show two-digit years, not four digits: 109 | time_format = time_format.replace('yyyy', 'yy') 110 | return mtime_qt.toString(time_format) 111 | def get_sort_value(self, url, is_ascending): 112 | try: 113 | is_dir = self._fs.is_dir(url) 114 | except FileNotFoundError: 115 | raise 116 | except OSError: 117 | is_dir = False 118 | try: 119 | mtime = self._get_mtime(url) 120 | except OSError: 121 | mtime = None 122 | return is_dir ^ is_ascending, mtime or datetime.min 123 | def _get_mtime(self, url): 124 | return self._fs.query(url, 'modified_datetime') -------------------------------------------------------------------------------- /core/tests/fs/test_columns.py: -------------------------------------------------------------------------------- 1 | from core import Name, Size, Modified 2 | from core.tests import StubFS 3 | from core.tests.fs import StubFileSystem 4 | from fman.url import as_url 5 | from unittest import TestCase 6 | 7 | class ColumnTest: 8 | def setUp(self): 9 | self._fs = StubFileSystem({ 10 | 'a': { 11 | 'is_dir': False, 'size': 1, 'mtime': 1473339042.0 12 | }, 13 | 'b': { 14 | 'is_dir': False, 'size': 0, 'mtime': 1473339043.0 15 | }, 16 | 'B': { 17 | 'is_dir': False, 'size': 2, 'mtime': 1473339042.0 18 | }, 19 | 'a_dir': { 20 | 'is_dir': True, 'size': 3, 'mtime': 1473339045.0 21 | }, 22 | 'b_dir': { 23 | 'is_dir': True, 'size': 4, 'mtime': 1473339046.0 24 | } 25 | 26 | }) 27 | self._column = self.column_class(StubFS(self._fs)) 28 | def assert_is_less(self, left, right, is_ascending=True): 29 | left_val = self._get_sort_value(left, is_ascending) 30 | right_val = self._get_sort_value(right, is_ascending) 31 | self.assertLess(left_val, right_val) 32 | def assert_is_greater(self, left, right, is_ascending=True): 33 | self.assertGreater( 34 | self._get_sort_value(left, is_ascending), 35 | self._get_sort_value(right, is_ascending), 36 | "%s is not > %s" % (left, right) 37 | ) 38 | def check_less_than_chain(self, *chain, is_ascending=True): 39 | for i, left in enumerate(chain[:-1]): 40 | right = chain[i + 1] 41 | self.assert_is_less(left, right, is_ascending) 42 | def _get_sort_value(self, path, is_ascending): 43 | url = as_url(path, StubFileSystem.scheme) 44 | return self._column.get_sort_value(url, is_ascending) 45 | 46 | class NameTest(ColumnTest, TestCase): 47 | 48 | column_class = Name 49 | 50 | def test_less(self): 51 | self.assert_is_less('a', 'b') 52 | def test_less_numbers(self): 53 | self.assert_is_less('foo 2.txt', 'foo 10.txt') 54 | self.assert_is_less('2 foo.txt', '10 foo.txt') 55 | self.assert_is_less('2', '10') 56 | self.assert_is_less('2', 'a1.txt') 57 | self.assert_is_less('file.txt', 'file1.txt') 58 | self.assert_is_less('02 Google Apps.pdf', '15 Tarsnap.pdf') 59 | def test_greater(self): 60 | self.assert_is_greater('b', 'a') 61 | def test_upper_case(self): 62 | self.assert_is_less('a', 'B') 63 | def test_directories_before_files(self): 64 | self.check_less_than_chain('a_dir', 'b_dir', 'a') 65 | def test_descending(self): 66 | self.check_less_than_chain( 67 | 'a', 'b', 'a_dir', 'b_dir', 68 | is_ascending=False 69 | ) 70 | def assert_is_less(self, left, right, is_ascending=True): 71 | if not self._fs.exists(left): 72 | self._fs.touch(left) 73 | if not self._fs.exists(right): 74 | self._fs.touch(right) 75 | super().assert_is_less(left, right, is_ascending) 76 | 77 | class SizeTest(ColumnTest, TestCase): 78 | 79 | column_class = Size 80 | 81 | def test_less(self): 82 | self.assert_is_less('b', 'a') 83 | def test_greater(self): 84 | self.assert_is_greater('a', 'b') 85 | def test_descending(self): 86 | # Qt expects the implementation of less_than to generally be independent 87 | # of the sort order: 88 | self.assert_is_less('b', 'a', False) 89 | def test_directories_by_name_before_files(self): 90 | self.check_less_than_chain('a_dir', 'b_dir', 'b') 91 | def test_directories_by_name_before_files_descending(self): 92 | self.check_less_than_chain( 93 | 'b', 'a', 'b_dir', 'a_dir', is_ascending=False 94 | ) 95 | 96 | class ModifiedTest(ColumnTest, TestCase): 97 | 98 | column_class = Modified 99 | 100 | def test_less(self): 101 | self.assert_is_less('a', 'b') 102 | def test_greater(self): 103 | self.assert_is_greater('b', 'a') 104 | def test_descending(self): 105 | # Qt expects the implementation of less_than to generally be independent 106 | # of the sort order: 107 | self.assert_is_less('a', 'b', False) 108 | def test_directories_before_files(self): 109 | self.check_less_than_chain('a_dir', 'b_dir', 'a') -------------------------------------------------------------------------------- /core/commands/explorer_properties.py: -------------------------------------------------------------------------------- 1 | from fman import DirectoryPaneCommand, show_alert 2 | from fman.fs import resolve 3 | from fman.url import splitscheme, as_human_readable, basename, dirname 4 | from pywintypes import com_error 5 | from win32com.shell.shell import SHILCreateFromPath, SHGetDesktopFolder, \ 6 | IID_IShellFolder, IID_IContextMenu 7 | from win32com.shell.shellcon import CMF_EXPLORE 8 | 9 | import ctypes.wintypes 10 | import re 11 | import win32gui 12 | 13 | class ShowExplorerProperties(DirectoryPaneCommand): 14 | 15 | aliases = 'Properties', 16 | 17 | def __call__(self): 18 | scheme = splitscheme(self.pane.get_path())[0] 19 | if scheme not in ('file://', 'drives://', 'network://'): 20 | show_alert( 21 | 'Sorry, showing the properties of %s files is not ' 22 | 'yet supported.' % scheme 23 | ) 24 | else: 25 | action = self._get_action() 26 | if action: 27 | action() 28 | def _get_action(self): 29 | file_under_cursor = self.pane.get_file_under_cursor() 30 | selected_files = self.pane.get_selected_files() 31 | chosen_files = selected_files or \ 32 | ([file_under_cursor] if file_under_cursor else []) 33 | location = self.pane.get_path() 34 | scheme, path = splitscheme(location) 35 | if scheme == 'file://': 36 | if file_under_cursor is None: 37 | # Either we're in an empty folder, or the user 38 | # right-clicked inside a directory. 39 | if self._is_drive(path): 40 | return lambda: _show_drive_properties(path) 41 | else: 42 | dir_ = as_human_readable(dirname(location)) 43 | filenames = [basename(location)] 44 | else: 45 | dir_ = as_human_readable(location) 46 | filenames = [basename(f) for f in chosen_files] 47 | return lambda: _show_file_properties(dir_, filenames) 48 | elif scheme == 'drives://': 49 | if file_under_cursor is None: 50 | # This usually happens when the user right-clicked in the drive 51 | # overview (but not _on_ a drive). 52 | return None 53 | drive = splitscheme(file_under_cursor)[1] 54 | if self._is_drive(drive): 55 | return lambda: _show_drive_properties(drive) 56 | elif scheme == 'network://': 57 | # We check `path` because when it's empty, we're at the 58 | # overview of network locations. Servers don't have a Properties 59 | # dialog. So we can't do anything there. 60 | if path: 61 | for f in chosen_files: 62 | try: 63 | f_fileurl = resolve(f) 64 | except OSError: 65 | continue 66 | if splitscheme(f_fileurl)[0] != 'file://': 67 | # Sanity check. We don't actually expect this. 68 | continue 69 | dir_ = as_human_readable(dirname(f_fileurl)) 70 | break 71 | else: 72 | return 73 | filenames = [basename(f) for f in chosen_files] 74 | return lambda: _show_file_properties(dir_, filenames) 75 | def is_visible(self): 76 | return bool(self._get_action()) 77 | def _is_drive(self, path): 78 | return re.match('^[A-Z]:$', path) 79 | 80 | def _show_file_properties(dir_, filenames): 81 | # Note: If you ever want to extend this method so it can handle files in 82 | # multiple directories, take a look at SHMultiFileProperties and [1]. 83 | # [1]: https://stackoverflow.com/a/34551988/1839209. 84 | folder = SHILCreateFromPath(dir_, 0)[0] 85 | desktop = SHGetDesktopFolder() 86 | shell_folder = desktop.BindToObject(folder, None, IID_IShellFolder) 87 | children = [] 88 | for filename in filenames: 89 | try: 90 | pidl = shell_folder.ParseDisplayName(None, None, filename)[1] 91 | except com_error: 92 | pass 93 | else: 94 | children.append(pidl) 95 | cm = shell_folder.GetUIObjectOf(None, children, IID_IContextMenu, 0)[1] 96 | if not cm: 97 | return 98 | hMenu = win32gui.CreatePopupMenu() 99 | cm.QueryContextMenu(hMenu, 0, 1, 0x7FFF, CMF_EXPLORE) 100 | cm.InvokeCommand((0, None, 'properties', '', '', 1, 0, None)) 101 | cm.QueryContextMenu(hMenu, 0, 1, 0x7FFF, CMF_EXPLORE) 102 | 103 | def _show_drive_properties(drive_nobackslash): 104 | sei = SHELLEXECUTEINFO() 105 | sei.cbSize = ctypes.sizeof(sei) 106 | sei.fMask = _SEE_MASK_NOCLOSEPROCESS | _SEE_MASK_INVOKEIDLIST 107 | sei.lpVerb = "properties" 108 | sei.lpFile = drive_nobackslash + '\\' 109 | sei.nShow = 1 110 | ShellExecuteEx(ctypes.byref(sei)) 111 | 112 | _SEE_MASK_NOCLOSEPROCESS = 0x00000040 113 | _SEE_MASK_INVOKEIDLIST = 0x0000000C 114 | 115 | class SHELLEXECUTEINFO(ctypes.Structure): 116 | _fields_ = ( 117 | ("cbSize", ctypes.wintypes.DWORD), 118 | ("fMask", ctypes.c_ulong), 119 | ("hwnd", ctypes.wintypes.HANDLE), 120 | ("lpVerb", ctypes.c_wchar_p), 121 | ("lpFile", ctypes.c_wchar_p), 122 | ("lpParameters", ctypes.c_char_p), 123 | ("lpDirectory", ctypes.c_char_p), 124 | ("nShow", ctypes.c_int), 125 | ("hInstApp", ctypes.wintypes.HINSTANCE), 126 | ("lpIDList", ctypes.c_void_p), 127 | ("lpClass", ctypes.c_char_p), 128 | ("hKeyClass", ctypes.wintypes.HKEY), 129 | ("dwHotKey", ctypes.wintypes.DWORD), 130 | ("hIconOrMonitor", ctypes.wintypes.HANDLE), 131 | ("hProcess", ctypes.wintypes.HANDLE), 132 | ) 133 | 134 | ShellExecuteEx = ctypes.windll.shell32.ShellExecuteExW 135 | ShellExecuteEx.restype = ctypes.wintypes.BOOL -------------------------------------------------------------------------------- /core/tests/commands/test_goto.py: -------------------------------------------------------------------------------- 1 | from core.commands.goto import _shrink_visited_paths, SuggestLocations 2 | from core.util import filenotfounderror 3 | from fman import PLATFORM 4 | from os.path import normpath 5 | from unittest import TestCase, skipIf 6 | 7 | import os 8 | 9 | class ShrinkVisiblePathsTest(TestCase): 10 | def test_basic(self): 11 | vps = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5} 12 | _shrink_visited_paths(vps, 3) 13 | self.assertEqual({'c': 1, 'd': 2, 'e': 3}, vps) 14 | def test_multiple_similar(self): 15 | vps = {'a': 1, 'b': 1, 'c': 3, 'd': 3, 'e': 5} 16 | _shrink_visited_paths(vps, 3) 17 | self.assertEqual({'c': 1, 'd': 1, 'e': 2}, vps) 18 | 19 | class SuggestLocationsTest(TestCase): 20 | 21 | class StubLocalFileSystem: 22 | def __init__(self, files, home_dir): 23 | self.files = files 24 | self.home_dir = home_dir 25 | def isdir(self, path): 26 | if PLATFORM == 'Windows' and path.endswith(' '): 27 | # Strange behaviour on Windows: isdir('X ') returns True if X 28 | # (without space) exists. 29 | path = path.rstrip(' ') 30 | try: 31 | self._get_dir(path) 32 | except KeyError: 33 | return False 34 | return True 35 | def _get_dir(self, path): 36 | if not path: 37 | raise KeyError(path) 38 | path = normpath(path) 39 | parts = path.split(os.sep) if path != os.sep else [''] 40 | if len(parts) > 1 and parts[-1] == '': 41 | parts = parts[:-1] 42 | curr = self.files 43 | for part in parts: 44 | for file_name, items in curr.items(): 45 | if self._normcase(file_name) == self._normcase(part): 46 | curr = items 47 | break 48 | else: 49 | raise KeyError(part) 50 | return curr 51 | def expanduser(self, path): 52 | return path.replace('~', self.home_dir) 53 | def listdir(self, path): 54 | try: 55 | return sorted(list(self._get_dir(path))) 56 | except KeyError as e: 57 | raise filenotfounderror(path) from e 58 | def resolve(self, path): 59 | is_case_sensitive = PLATFORM == 'Linux' 60 | if is_case_sensitive: 61 | return path 62 | dir_ = os.path.dirname(path) 63 | if dir_ == path: 64 | # We're at the root of the file system. 65 | return path 66 | dir_ = self.resolve(dir_) 67 | try: 68 | dir_contents = self.listdir(dir_) 69 | except OSError: 70 | matching_names = [] 71 | else: 72 | matching_names = [ 73 | f for f in dir_contents 74 | if f.lower() == os.path.basename(path).lower() 75 | ] 76 | if not matching_names: 77 | return path 78 | return os.path.join(dir_, matching_names[0]) 79 | def samefile(self, f1, f2): 80 | return self._get_dir(f1) == self._get_dir(f2) 81 | def find_folders_starting_with(self, prefix): 82 | return list( 83 | self._find_folders_recursive(self.files, prefix.lower())) 84 | def _find_folders_recursive(self, files, prefix): 85 | for f, subfiles in files.items(): 86 | if f.lower().startswith(prefix): 87 | yield f 88 | for sub_f in self._find_folders_recursive(subfiles, prefix): 89 | # We don't use join(...) here because of the case f=''. We 90 | # want '/sub_f' but join(f, sub_f) would give just 'sub_f'. 91 | yield f + os.sep + sub_f 92 | def _normcase(self, path): 93 | return path if PLATFORM == 'Linux' else path.lower() 94 | 95 | def test_empty_suggests_recent_locations(self): 96 | expected_paths = [ 97 | '~/Dropbox/Work', '~/Dropbox', '~/Downloads', '~/Dropbox/Private', 98 | '~/s-u-b-s-t-r', '~/My-substr', '~' 99 | ] 100 | self._check_query_returns( 101 | '', expected_paths, [[]] * len(expected_paths) 102 | ) 103 | def test_basename_matches(self): 104 | self._check_query_returns( 105 | 'dow', ['~/Downloads', '~/Dropbox/Work'], [[2, 3, 4], [2, 4, 10]] 106 | ) 107 | def test_exact_match_takes_precedence(self): 108 | expected_paths = [ 109 | '~', '~/Dropbox', '~/Downloads', '~/s-u-b-s-t-r', '~/My-substr', 110 | '~/.hidden', '~/Unvisited' 111 | ] 112 | self._check_query_returns( 113 | '~', expected_paths, [[0]] * len(expected_paths) 114 | ) 115 | def test_prefix_match(self): 116 | self._check_query_returns('~/dow', ['~/Downloads'], [[0, 1, 2, 3, 4]]) 117 | def test_existing_path(self): 118 | self._check_query_returns( 119 | '~/Unvisited', ['~/Unvisited', '~/Unvisited/Dir'] 120 | ) 121 | @skipIf(PLATFORM == 'Linux', 'Case-insensitive file systems only') 122 | def test_existing_path_wrong_case(self): 123 | self._check_query_returns( 124 | '~/unvisited', ['~/Unvisited', '~/Unvisited/Dir'] 125 | ) 126 | def test_enter_path_slash(self): 127 | highlight = list(range(len('~/Unvisited'))) 128 | self._check_query_returns( 129 | '~/Unvisited/', ['~/Unvisited', '~/Unvisited/Dir'], 130 | [highlight, highlight] 131 | ) 132 | def test_trailing_space(self): 133 | self._check_query_returns('~/Downloads ', []) 134 | def test_hidden(self): 135 | self._check_query_returns('~/.', ['~/.hidden']) 136 | def test_filesystem_search(self): 137 | # No visited paths: 138 | self.instance = SuggestLocations({}, self.fs) 139 | # Should still find Downloads by prefix: 140 | self._check_query_returns('dow', ['~/Downloads'], [[2, 3, 4]]) 141 | def test_home_dir_expanded(self): 142 | self._check_query_returns( 143 | os.path.dirname(self.home_dir), 144 | [os.path.dirname(self.home_dir), self.home_dir] 145 | ) 146 | def test_substring(self): 147 | # Should return My-substr before ~/s-u-b-s-t-r even though the latter 148 | # has a higher count: 149 | self._check_query_returns( 150 | 'sub', ['~/My-substr', '~/s-u-b-s-t-r'], [[5, 6, 7], [2, 4, 6]] 151 | ) 152 | def setUp(self): 153 | root = 'C:' if PLATFORM == 'Windows' else '' 154 | files = { 155 | root: { 156 | 'Users': { 157 | 'michael': { 158 | '.hidden': {}, 159 | 'Downloads': {}, 160 | 'Dropbox': { 161 | 'Work': {}, 'Private': {} 162 | }, 163 | 'Unvisited': { 164 | 'Dir': {} 165 | }, 166 | 'My-substr': {}, 167 | 's-u-b-s-t-r': {}, 168 | } 169 | } 170 | }, 171 | '.': {} 172 | } 173 | if PLATFORM == 'Windows': 174 | self.home_dir = r'C:\Users\michael' 175 | else: 176 | self.home_dir = '/Users/michael' 177 | self.fs = self.StubLocalFileSystem(files, home_dir=self.home_dir) 178 | visited_paths = { 179 | self._replace_pathsep(self.fs.expanduser(k)): v 180 | for k, v in [ 181 | ('~', 1), 182 | ('~/Downloads', 5), 183 | ('~/Dropbox', 6), 184 | ('~/Dropbox/Work', 7), 185 | ('~/Dropbox/Private', 4), 186 | ('~/My-substr', 2), 187 | ('~/s-u-b-s-t-r', 3) # Note: Higher count than My-substr 188 | ] 189 | } 190 | self.instance = SuggestLocations(visited_paths, self.fs) 191 | def _check_query_returns(self, query, paths, highlights=None): 192 | query = self._replace_pathsep(query) 193 | paths = list(map(self._replace_pathsep, paths)) 194 | if highlights is None: 195 | highlights = [self._full_range(query)] * len(paths) 196 | result = list(self.instance(query)) 197 | self.assertEqual(paths, [item.title for item in result]) 198 | self.assertEqual(highlights, [item.highlight for item in result]) 199 | def _replace_pathsep(self, path): 200 | return path.replace('/', os.sep) 201 | def _full_range(self, string): 202 | return list(range(len(string))) -------------------------------------------------------------------------------- /core/fileoperations.py: -------------------------------------------------------------------------------- 1 | from core.util import is_parent 2 | from fman import Task, YES, NO, YES_TO_ALL, NO_TO_ALL, ABORT, OK 3 | from fman.url import basename, join, dirname, splitscheme, relpath, \ 4 | as_human_readable 5 | from os.path import pardir 6 | 7 | import fman.fs 8 | 9 | class FileTreeOperation(Task): 10 | def __init__( 11 | self, descr_verb, files, dest_dir, dest_name=None, fs=fman.fs 12 | ): 13 | if dest_name and len(files) > 1: 14 | raise ValueError( 15 | 'Destination name can only be given when there is one file.' 16 | ) 17 | super().__init__(self._get_title(descr_verb, files)) 18 | self._files = files 19 | self._dest_dir = dest_dir 20 | self._descr_verb = descr_verb 21 | self._dest_name = dest_name 22 | self._fs = fs 23 | self._src_dir = dirname(files[0]) 24 | self._tasks = [] 25 | self._num_files = 0 26 | self._cannot_move_to_self_shown = False 27 | self._override_all = None 28 | self._ignore_exceptions = False 29 | def _transfer(self, src, dest): 30 | raise NotImplementedError() 31 | def _prepare_transfer(self, src, dest): 32 | raise NotImplementedError() 33 | def _can_transfer_samefile(self): 34 | raise NotImplementedError() 35 | def _does_postprocess_directory(self): 36 | return False 37 | def _postprocess_directory(self, src_dir_path): 38 | return None 39 | def __call__(self): 40 | self.set_text('Gathering files...') 41 | if not self._gather_files(): 42 | return 43 | self.set_size(sum(task.get_size() for task in self._tasks)) 44 | for i, task in enumerate(self._iter(self._tasks)): 45 | is_last = i == len(self._tasks) - 1 46 | progress_before = self.get_progress() 47 | try: 48 | self.run(task) 49 | except (OSError, IOError) as e: 50 | title = task.get_title() 51 | message = 'Error ' + (title[0].lower() + title[1:]) 52 | if not self._handle_exception(message, is_last, e): 53 | break 54 | self.set_progress(progress_before + task.get_size()) 55 | def _gather_files(self): 56 | dest_dir_url = self._get_dest_dir_url() 57 | self._enqueue([Task( 58 | 'Preparing ' + basename(dest_dir_url), fn=self._fs.makedirs, 59 | args=(dest_dir_url,), kwargs={'exist_ok': True} 60 | )]) 61 | for i, src in enumerate(self._iter(self._files)): 62 | is_last = i == len(self._files) - 1 63 | dest = self._get_dest_url(src) 64 | if is_parent(src, dest, self._fs): 65 | if src != dest: 66 | try: 67 | is_samefile = self._fs.samefile(src, dest) 68 | except OSError: 69 | is_samefile = False 70 | if is_samefile: 71 | if self._can_transfer_samefile(): 72 | self._enqueue(self._prepare_transfer(src, dest)) 73 | continue 74 | self.show_alert( 75 | "You cannot %s a file to itself." % self._descr_verb 76 | ) 77 | return False 78 | try: 79 | is_dir = self._fs.is_dir(src) 80 | except OSError as e: 81 | error_message = 'Could not %s %s' % \ 82 | (self._descr_verb, as_human_readable(src)) 83 | if self._handle_exception(error_message, is_last, e): 84 | continue 85 | return False 86 | if is_dir: 87 | if self._fs.exists(dest): 88 | if not self._merge_directory(src): 89 | return False 90 | else: 91 | self._enqueue(self._prepare_transfer(src, dest)) 92 | else: 93 | if self._fs.exists(dest): 94 | should_overwrite = self._should_overwrite(dest) 95 | if should_overwrite == NO: 96 | continue 97 | elif should_overwrite == ABORT: 98 | return False 99 | else: 100 | assert should_overwrite == YES, should_overwrite 101 | self._enqueue(self._prepare_transfer(src, dest)) 102 | return True 103 | def _merge_directory(self, src): 104 | for file_name in self._fs.iterdir(src): 105 | file_url = join(src, file_name) 106 | try: 107 | src_is_dir = self._fs.is_dir(file_url) 108 | except OSError: 109 | src_is_dir = False 110 | dst = self._get_dest_url(file_url) 111 | if src_is_dir: 112 | try: 113 | dst_is_dir = self._fs.is_dir(dst) 114 | except OSError: 115 | dst_is_dir = False 116 | if dst_is_dir: 117 | if not self._merge_directory(file_url): 118 | return False 119 | else: 120 | self._enqueue(self._prepare_transfer(file_url, dst)) 121 | else: 122 | if self._fs.exists(dst): 123 | should_overwrite = self._should_overwrite(dst) 124 | if should_overwrite == NO: 125 | continue 126 | elif should_overwrite == ABORT: 127 | return False 128 | else: 129 | assert should_overwrite == YES, \ 130 | should_overwrite 131 | self._enqueue(self._prepare_transfer(file_url, dst)) 132 | if self._does_postprocess_directory(): 133 | # Post-process the parent directories bottom-up. For Move, this 134 | # ensures that each directory is empty when post-processing. 135 | self._enqueue([self._postprocess_directory(src)]) 136 | return True 137 | def _should_overwrite(self, file_url): 138 | if self._override_all is None: 139 | choice = self.show_alert( 140 | "%s exists. Do you want to overwrite it?" % basename(file_url), 141 | YES | NO | YES_TO_ALL | NO_TO_ALL | ABORT, YES 142 | ) 143 | if choice & YES: 144 | return YES 145 | elif choice & NO: 146 | return NO 147 | elif choice & YES_TO_ALL: 148 | self._override_all = True 149 | elif choice & NO_TO_ALL: 150 | self._override_all = False 151 | else: 152 | assert choice & ABORT, choice 153 | return ABORT 154 | return YES if self._override_all else NO 155 | def _enqueue(self, tasks): 156 | for task in self._iter(tasks): 157 | if task.get_size() > 0: 158 | self._num_files += 1 159 | self.set_text( 160 | 'Preparing to {} {:,} files.' 161 | .format(self._descr_verb, self._num_files) 162 | ) 163 | self._tasks.append(task) 164 | def _handle_exception(self, message, is_last, exc): 165 | if self._ignore_exceptions: 166 | return True 167 | if exc.strerror: 168 | cause = exc.strerror[0].lower() + exc.strerror[1:] 169 | else: 170 | cause = exc.__class__.__name__ 171 | message = '%s (%s).' % (message, cause) 172 | if is_last: 173 | buttons = OK 174 | default_button = OK 175 | else: 176 | buttons = YES | YES_TO_ALL | ABORT 177 | default_button = YES 178 | message += ' Do you want to continue?' 179 | choice = self.show_alert(message, buttons, default_button) 180 | if is_last: 181 | return choice & OK 182 | else: 183 | if choice & YES_TO_ALL: 184 | self._ignore_exceptions = True 185 | return choice & YES or choice & YES_TO_ALL 186 | def _get_dest_dir_url(self): 187 | try: 188 | splitscheme(self._dest_dir) 189 | except ValueError as not_a_url: 190 | is_url = False 191 | else: 192 | is_url = True 193 | return self._dest_dir if is_url else join(self._src_dir, self._dest_dir) 194 | def _get_dest_url(self, src_file): 195 | dest_name = self._dest_name or basename(src_file) 196 | if self._src_dir: 197 | try: 198 | rel_path = \ 199 | relpath(join(dirname(src_file), dest_name), self._src_dir) 200 | except ValueError as e: 201 | raise ValueError( 202 | 'Could not construct path. ' 203 | 'src_file: %r, dest_name: %r, src_dir: %r' % 204 | (src_file, dest_name, self._src_dir) 205 | ) from e 206 | is_in_src_dir = not rel_path.startswith(pardir) 207 | if is_in_src_dir: 208 | try: 209 | splitscheme(self._dest_dir) 210 | except ValueError as no_scheme: 211 | return join(self._src_dir, self._dest_dir, rel_path) 212 | else: 213 | return join(self._dest_dir, rel_path) 214 | return join(self._dest_dir, dest_name) 215 | def _iter(self, iterable): 216 | for item in iterable: 217 | self.check_canceled() 218 | yield item 219 | def _get_title(self, descr_verb, files): 220 | verb = descr_verb.capitalize() 221 | result = (verb[:-1] if verb.endswith('e') else verb) + 'ing ' 222 | if len(files) == 1: 223 | result += basename(files[0]) 224 | else: 225 | result += '%d files' % len(files) 226 | return result 227 | 228 | class CopyFiles(FileTreeOperation): 229 | def __init__(self, *super_args, **super_kwargs): 230 | super().__init__('copy', *super_args, **super_kwargs) 231 | def _transfer(self, src, dest): 232 | self._fs.copy(src, dest) 233 | def _can_transfer_samefile(self): 234 | # Can never copy to the same file. 235 | return False 236 | def _prepare_transfer(self, src, dest): 237 | return self._fs.prepare_copy(src, dest) 238 | 239 | class MoveFiles(FileTreeOperation): 240 | def __init__(self, *super_args, **super_kwargs): 241 | super().__init__('move', *super_args, **super_kwargs) 242 | def _transfer(self, src, dest): 243 | self._fs.move(src, dest) 244 | def _can_transfer_samefile(self): 245 | # May be able to move to the same file on case insensitive file systems. 246 | # Consider a/ and A/: They are the "same" file yet it does make sense to 247 | # rename one to the other. 248 | return True 249 | def _prepare_transfer(self, src, dest): 250 | return self._fs.prepare_move(src, dest) 251 | def _does_postprocess_directory(self): 252 | return True 253 | def _postprocess_directory(self, src_dir_path): 254 | return Task( 255 | 'Postprocessing ' + basename(src_dir_path), 256 | fn=self._do_postprocess_directory, args=(src_dir_path,) 257 | ) 258 | def _do_postprocess_directory(self, src_dir_path): 259 | if self._is_empty(src_dir_path): 260 | try: 261 | self._fs.delete(src_dir_path) 262 | except OSError: 263 | pass 264 | def _is_empty(self, dir_url): 265 | try: 266 | next(iter(self._fs.iterdir(dir_url))) 267 | except StopIteration: 268 | return True 269 | return False -------------------------------------------------------------------------------- /core/tests/commands/test___init__.py: -------------------------------------------------------------------------------- 1 | from core.commands import History, Move, _from_human_readable, \ 2 | get_dest_suggestion, _find_extension_start, _get_shortcuts_for_command 3 | from core.tests import StubUI 4 | from core.util import filenotfounderror 5 | from fman import OK, YES, NO, PLATFORM 6 | from fman.url import join, as_human_readable, as_url, dirname 7 | from unittest import TestCase 8 | 9 | import os 10 | import os.path 11 | 12 | class FindExtensionStartTest(TestCase): 13 | def test_no_extension(self): 14 | self.assertIsNone(_find_extension_start('File')) 15 | def test_normal_extension(self): 16 | self.assertEqual(4, _find_extension_start('test.zip')) 17 | def test_tar_xz(self): 18 | self.assertEqual(7, _find_extension_start('archive.tar.xz')) 19 | def test_tar_gz(self): 20 | self.assertEqual(7, _find_extension_start('archive.tar.gz')) 21 | 22 | class ConfirmTreeOperationTest(TestCase): 23 | 24 | class FileSystem: 25 | def __init__(self, files, case_sensitive=PLATFORM == 'Linux'): 26 | self._files = files 27 | self._case_sensitive = case_sensitive 28 | 29 | def exists(self, url): 30 | try: 31 | self._get(url) 32 | except KeyError: 33 | return False 34 | return True 35 | 36 | def is_dir(self, url): 37 | try: 38 | file_info = self._get(url) 39 | except KeyError: 40 | raise filenotfounderror(url) from None 41 | return file_info['is_dir'] 42 | 43 | def samefile(self, url1, url2): 44 | if not self._case_sensitive: 45 | url1 = url1.lower() 46 | url2 = url2.lower() 47 | return url1 == url2 48 | 49 | def _get(self, url): 50 | dict_ = self._files 51 | if not self._case_sensitive: 52 | dict_ = {k.lower(): v for k, v in self._files.items()} 53 | url = url.lower() 54 | return dict_[url] 55 | 56 | def test_no_files(self): 57 | self._expect_alert(('No file is selected!',), answer=OK) 58 | self._check([], None) 59 | def test_one_file(self): 60 | dest_path = as_human_readable(join(self._dest, 'a.txt')) 61 | sel_start = dest_path.rindex(os.sep) + 1 62 | self._expect_prompt( 63 | ('Move "a.txt" to', dest_path, sel_start, sel_start + 1), 64 | (dest_path, True) 65 | ) 66 | self._check([self._a_txt], (self._dest, 'a.txt')) 67 | def test_one_dir(self): 68 | dest_path = as_human_readable(self._dest) 69 | self._expect_prompt( 70 | ('Move "a" to', dest_path, 0, None), (dest_path, True) 71 | ) 72 | self._check([self._a], (self._dest, None)) 73 | def test_rename_dir_to_uppercase(self): 74 | dest_path = as_human_readable(self._src) 75 | self._expect_prompt( 76 | ('Move "a" to', dest_path, 0, None), ('A', True) 77 | ) 78 | self._check([self._a], (self._src, 'A'), dest_dir=self._src) 79 | def test_two_files(self): 80 | dest_path = as_human_readable(self._dest) 81 | self._expect_prompt( 82 | ('Move 2 files to', dest_path, 0, None), (dest_path, True) 83 | ) 84 | self._check([self._a_txt, self._b_txt], (self._dest, None)) 85 | def test_into_subfolder(self): 86 | dest_path = as_human_readable(join(self._dest, 'a.txt')) 87 | sel_start = dest_path.rindex(os.sep) + 1 88 | self._expect_prompt( 89 | ('Move "a.txt" to', dest_path, sel_start, sel_start + 1), 90 | ('a', True) 91 | ) 92 | self._check([self._a_txt], (self._a, None)) 93 | def test_overwrite_single_file(self): 94 | dest_url = join(self._dest, 'a.txt') 95 | self._fs._files[dest_url] = {'is_dir': False} 96 | dest_path = as_human_readable(dest_url) 97 | sel_start = dest_path.rindex(os.sep) + 1 98 | self._expect_prompt( 99 | ('Move "a.txt" to', dest_path, sel_start, sel_start + 1), 100 | (dest_path, True) 101 | ) 102 | self._check([self._a_txt], (self._dest, 'a.txt')) 103 | def test_multiple_files_over_one(self): 104 | dest_url = join(self._dest, 'a.txt') 105 | self._fs._files[dest_url] = {'is_dir': False} 106 | dest_path = as_human_readable(dest_url) 107 | self._expect_prompt( 108 | ('Move 2 files to', as_human_readable(self._dest), 0, None), 109 | (dest_path, True) 110 | ) 111 | self._expect_alert( 112 | ('You cannot move multiple files to a single file!',), answer=OK 113 | ) 114 | self._check([self._a_txt, self._b_txt], None) 115 | def test_multiple_into_self(self): 116 | dest_path = as_human_readable(self._a) 117 | self._expect_prompt( 118 | ('Move 2 files to', dest_path, 0, None), (dest_path, True) 119 | ) 120 | self._expect_alert(('You cannot move a file to itself!',), answer=OK) 121 | self._check([self._a_txt, self._a], None, dest_dir=self._a) 122 | def test_renamed_destination(self): 123 | dest_path = as_human_readable(join(self._dest, 'a.txt')) 124 | sel_start = dest_path.rindex(os.sep) + 1 125 | self._expect_prompt( 126 | ('Move "a.txt" to', dest_path, sel_start, sel_start + 1), 127 | (as_human_readable(join(self._dest, 'z.txt')), True) 128 | ) 129 | self._check([self._a_txt], (self._dest, 'z.txt')) 130 | def test_multiple_files_nonexistent_dest(self): 131 | dest_url = join(self._dest, 'dir') 132 | dest_path = as_human_readable(dest_url) 133 | self._expect_prompt( 134 | ('Move 2 files to', as_human_readable(self._dest), 0, None), 135 | (dest_path, True) 136 | ) 137 | self._expect_alert( 138 | ('%s does not exist. Do you want to create it as a directory and ' 139 | 'move the files there?' % dest_path, YES | NO, YES), 140 | answer=YES 141 | ) 142 | self._check([self._a_txt, self._b_txt], (dest_url, None)) 143 | def test_file_system_root(self): 144 | dest_path = as_human_readable(join(self._root, 'a.txt')) 145 | sel_start = dest_path.rindex(os.sep) + 1 146 | self._expect_prompt( 147 | ('Move "a.txt" to', dest_path, sel_start, sel_start + 1), 148 | (dest_path, True) 149 | ) 150 | self._check([self._a_txt], (self._root, 'a.txt'), dest_dir=self._root) 151 | def test_different_scheme(self): 152 | dest_path = as_human_readable(join(self._dest, 'a.txt')) 153 | sel_start = dest_path.rindex(os.sep) + 1 154 | self._expect_prompt( 155 | ('Move "a.txt" to', dest_path, sel_start, sel_start + 1), 156 | (dest_path, True) 157 | ) 158 | src_url = 'zip:///dest.zip/a.txt' 159 | src_dir = dirname(src_url) 160 | self._check([src_url], (self._dest, 'a.txt'), src_dir=src_dir) 161 | def _expect_alert(self, args, answer): 162 | self._ui.expect_alert(args, answer) 163 | def _expect_prompt(self, args, answer): 164 | self._ui.expect_prompt(args, answer) 165 | def _check(self, files, expected_result, src_dir=None, dest_dir=None): 166 | if src_dir is None: 167 | src_dir = self._src 168 | if dest_dir is None: 169 | dest_dir = self._dest 170 | actual_result = Move._confirm_tree_operation( 171 | files, dest_dir, src_dir, self._ui, self._fs 172 | ) 173 | self._ui.verify_expected_dialogs_were_shown() 174 | self.assertEqual(expected_result, actual_result) 175 | def setUp(self): 176 | super().setUp() 177 | self._ui = StubUI(self) 178 | self._root = as_url('C:\\' if PLATFORM == 'Windows' else '/') 179 | self._src = join(self._root, 'src') 180 | self._dest = join(self._root, 'dest') 181 | self._a = join(self._root, 'src/a') 182 | self._a_txt = join(self._root, 'src/a.txt') 183 | self._b_txt = join(self._root, 'src/b.txt') 184 | self._fs = self.FileSystem({ 185 | self._src: {'is_dir': True}, 186 | self._dest: {'is_dir': True}, 187 | self._a: {'is_dir': True}, 188 | self._a_txt: {'is_dir': False}, 189 | self._b_txt: {'is_dir': False}, 190 | }) 191 | 192 | class GetDestSuggestionTest(TestCase): 193 | def test_file(self): 194 | file_path = os.path.join(self._root, 'file.txt') 195 | selection_start = file_path.rindex(os.sep) + 1 196 | selection_end = selection_start + len('file') 197 | self.assertEqual( 198 | (file_path, selection_start, selection_end), 199 | get_dest_suggestion(as_url(file_path)) 200 | ) 201 | def test_dir(self): 202 | dir_path = os.path.join(self._root, 'dir') 203 | selection_start = dir_path.rindex(os.sep) + 1 204 | selection_end = None 205 | self.assertEqual( 206 | (dir_path, selection_start, selection_end), 207 | get_dest_suggestion(as_url(dir_path)) 208 | ) 209 | def setUp(self): 210 | super().setUp() 211 | self._root = 'C:\\' if PLATFORM == 'Windows' else '/' 212 | 213 | class FromHumanReadableTest(TestCase): 214 | def test_no_src_dir(self): 215 | path = __file__ 216 | dir_url = as_url(os.path.dirname(path)) 217 | self.assertEqual( 218 | as_url(path), 219 | _from_human_readable(path, dir_url, None) 220 | ) 221 | 222 | class GetShortcutsForCommandTest(TestCase): 223 | def test_no_shortcut(self): 224 | self._check([{'keys': ['Enter'], 'command': 'open'}], 'copy', []) 225 | def test_simple(self): 226 | self._check([{'keys': ['Enter'], 'command': 'open'}], 'open', ['Enter']) 227 | def test_two_shortcuts(self): 228 | self._check( 229 | [{'keys': ['Enter'], 'command': 'open'}, 230 | {'keys': ['Down'], 'command': 'open'}], 231 | 'open', ['Enter', 'Down'] 232 | ) 233 | def test_shortcut_only_displayed_for_one_command(self): 234 | bindings = [ 235 | {'keys': ['Enter'], 'command': 'open'}, 236 | {'keys': ['Enter'], 'command': 'alternative'} 237 | ] 238 | self._check(bindings, 'open', ['Enter']) 239 | self._check(bindings, 'alternative', []) 240 | def _check(self, key_bindings, command, expected_shortcuts): 241 | actual = list(_get_shortcuts_for_command(key_bindings, command)) 242 | self.assertEqual(expected_shortcuts, actual) 243 | 244 | class HistoryTest(TestCase): 245 | def test_empty_back(self): 246 | with self.assertRaises(ValueError): 247 | self._go_back() 248 | def test_empty_forward(self): 249 | with self.assertRaises(ValueError): 250 | self._go_forward() 251 | def test_single_back(self): 252 | self._go_to('single item') 253 | with self.assertRaises(ValueError): 254 | self._go_back() 255 | def test_single_forward(self): 256 | self._go_to('single item') 257 | with self.assertRaises(ValueError): 258 | self._go_forward() 259 | def test_go_back_forward(self): 260 | self._go_to('a', 'b', 'c') 261 | self.assertEqual('b', self._go_back()) 262 | self.assertEqual('a', self._go_back()) 263 | self.assertEqual('b', self._go_forward()) 264 | self.assertEqual('c', self._go_forward()) 265 | def test_go_to_after_back(self): 266 | self._go_to('a', 'b') 267 | self.assertEqual('a', self._go_back()) 268 | self._go_to('c') 269 | self.assertEqual(['a', 'c'], self._history._paths) 270 | def setUp(self): 271 | super().setUp() 272 | self._history = History() 273 | def _go_back(self): 274 | path = self._history.go_back() 275 | self._history.path_changed(path) 276 | return path 277 | def _go_forward(self): 278 | path = self._history.go_forward() 279 | self._history.path_changed(path) 280 | return path 281 | def _go_to(self, *paths): 282 | for path in paths: 283 | self._history.path_changed(path) -------------------------------------------------------------------------------- /core/tests/fs/test_local.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from fman import PLATFORM 3 | from fman.url import join, as_url, splitscheme 4 | from core import LocalFileSystem 5 | from pathlib import Path 6 | from stat import S_IWRITE 7 | from tempfile import TemporaryDirectory 8 | from unittest import TestCase, skipIf 9 | 10 | import os 11 | 12 | class LocalFileSystemTest(TestCase): 13 | def test_mkdir_root(self): 14 | with self.assertRaises(FileExistsError): 15 | self._fs.mkdir('C:' if PLATFORM == 'Windows' else '/') 16 | def test_iterdir_nonexistent(self): 17 | root = 'C:/' if PLATFORM == 'Windows' else '/' 18 | path = root + 'nonexistent' 19 | with self.assertRaises(FileNotFoundError): 20 | next(iter(self._fs.iterdir(path))) 21 | def test_empty_path_does_not_exist(self): 22 | self.assertFalse(self._fs.exists('')) 23 | def test_relative_paths(self): 24 | subdir_name = 'subdir' 25 | with TemporaryCwd() as tmp_dir: 26 | Path(tmp_dir, subdir_name).mkdir() 27 | self.assertFalse(self._fs.exists(subdir_name)) 28 | with self.assertRaises(FileNotFoundError): 29 | list(self._fs.iterdir(subdir_name)) 30 | with self.assertRaises(FileNotFoundError): 31 | self._fs.is_dir(subdir_name) 32 | with self.assertRaises(FileNotFoundError): 33 | self._fs.stat(subdir_name) 34 | with self.assertRaises(FileNotFoundError): 35 | self._fs.size_bytes(subdir_name) 36 | with self.assertRaises(FileNotFoundError): 37 | self._fs.modified_datetime(subdir_name) 38 | with self.assertRaises(ValueError): 39 | self._fs.touch('test.txt') 40 | with self.assertRaises(ValueError): 41 | self._fs.mkdir('other_dir') 42 | src_url = join(as_url(tmp_dir), subdir_name) 43 | dst_url = as_url('dir2') 44 | with self.assertRaises(ValueError): 45 | self._fs.move(src_url, dst_url) 46 | with self.assertRaises(ValueError): 47 | self._fs.prepare_move(src_url, dst_url) 48 | with self.assertRaises(ValueError): 49 | self._fs.copy(src_url, dst_url) 50 | with self.assertRaises(ValueError): 51 | self._fs.prepare_copy(src_url, dst_url) 52 | with self.assertRaises(FileNotFoundError): 53 | self._fs.move_to_trash(subdir_name) 54 | with self.assertRaises(FileNotFoundError): 55 | list(self._fs.prepare_trash(subdir_name)) 56 | with self.assertRaises(FileNotFoundError): 57 | self._fs.delete(subdir_name) 58 | file_name = 'test.txt' 59 | Path(tmp_dir, file_name).touch() 60 | with self.assertRaises(FileNotFoundError): 61 | self._fs.delete(file_name) 62 | with self.assertRaises(FileNotFoundError): 63 | self._fs.resolve(subdir_name) 64 | @skipIf(PLATFORM != 'Windows', 'Skip Windows-only test') 65 | def test_isabs_windows(self): 66 | self.assertTrue(self._fs._isabs(r'\\host')) 67 | self.assertTrue(self._fs._isabs(r'\\host\share')) 68 | self.assertTrue(self._fs._isabs(r'\\host\share\subfolder')) 69 | self.assertFalse(self._fs._isabs('dir')) 70 | self.assertFalse(self._fs._isabs(r'dir\subdir')) 71 | def test_stat_nonexistent_symlink(self): 72 | with TemporaryDirectory() as tmp_dir: 73 | path = Path(tmp_dir, 'symlink') 74 | path.symlink_to('nonexistent') 75 | self._fs.stat(_urlpath(path)) 76 | def test_samefile(self): 77 | this = _urlpath(__file__) 78 | pardir = _urlpath(Path(__file__).parent) 79 | init = pardir + '/__init__.py' 80 | self.assertTrue(self._fs.samefile(this, this)) 81 | self.assertTrue(self._fs.samefile(pardir, pardir)) 82 | self.assertTrue(self._fs.samefile(init, init)) 83 | self.assertFalse(self._fs.samefile(this, pardir)) 84 | self.assertFalse(self._fs.samefile(this, init)) 85 | self.assertFalse(self._fs.samefile(pardir, init)) 86 | def test_samefile_gdrive_file_stream(self): 87 | with TemporaryDirectory() as tmp_dir: 88 | a = Path(tmp_dir, 'a') 89 | a.mkdir() 90 | b = Path(tmp_dir, 'b') 91 | b.mkdir() 92 | a_path = _urlpath(a) 93 | b_path = _urlpath(b) 94 | self._fs.cache.put(a_path, 'stat', fake_statresult(0, 0)) 95 | self._fs.cache.put(b_path, 'stat', fake_statresult(0, 0)) 96 | self.assertFalse(self._fs.samefile(a_path, b_path)) 97 | def test_delete_readonly_file(self): 98 | with TemporaryDirectory() as tmp_dir: 99 | path = Path(tmp_dir, 'file') 100 | path.touch() 101 | path.chmod(path.stat().st_mode ^ S_IWRITE) 102 | self._fs.delete(_urlpath(path)) 103 | def test_delete_symlink_to_directory(self): 104 | with TemporaryDirectory() as tmp_dir: 105 | a = Path(tmp_dir, 'a') 106 | a.mkdir() 107 | b = Path(tmp_dir, 'b') 108 | b.symlink_to(a) 109 | self._fs.delete(_urlpath(b)) 110 | self.assertFalse(b.exists(), 'Failed to delete symlink to folder') 111 | def test_copy_file(self): 112 | self._test_transfer_file(self._fs.copy, deletes_src=False) 113 | def test_move_file(self): 114 | self._test_transfer_file(self._fs.move, deletes_src=True) 115 | def _test_transfer_file(self, transfer_fn, deletes_src): 116 | with TemporaryDirectory() as tmp_dir: 117 | src = Path(tmp_dir, 'src') 118 | f_contents = '1234' 119 | src.write_text(f_contents) 120 | dst = Path(tmp_dir, 'dst') 121 | transfer_fn(as_url(src), as_url(dst)) 122 | self.assertTrue(dst.exists()) 123 | self.assertEqual(f_contents, dst.read_text()) 124 | if deletes_src: 125 | self.assertFalse(src.exists()) 126 | def test_copy_directory(self): 127 | with TemporaryDirectory() as tmp_dir: 128 | src = Path(tmp_dir, 'src') 129 | src.mkdir() 130 | self._create_test_directory_structure(src) 131 | dst = Path(tmp_dir, 'dst') 132 | self._fs.copy(as_url(src), as_url(dst)) 133 | self.assertEqual( 134 | self._jsonify_directory(src), self._jsonify_directory(dst) 135 | ) 136 | def test_move_directory(self, use_rename=True): 137 | with TemporaryDirectory() as tmp_dir: 138 | src = Path(tmp_dir, 'src') 139 | src.mkdir() 140 | self._create_test_directory_structure(src) 141 | src_contents = self._jsonify_directory(src) 142 | dst = Path(tmp_dir, 'dst') 143 | for task in self._fs._prepare_move( 144 | as_url(src), as_url(dst), use_rename=use_rename 145 | ): 146 | task() 147 | self.assertFalse(src.exists()) 148 | self.assertEqual(src_contents, self._jsonify_directory(dst)) 149 | def test_move_directory_without_rename(self): 150 | self.test_move_directory(use_rename=False) 151 | def _create_test_directory_structure(self, parent_dir): 152 | file_1 = parent_dir / 'file.txt' 153 | file_txt_contents = '12345' 154 | file_1.write_text(file_txt_contents) 155 | empty_dir = parent_dir / 'empty' 156 | empty_dir.mkdir() 157 | dir_ = parent_dir / 'dir' 158 | dir_.mkdir() 159 | file_2 = dir_ / 'file2.txt' 160 | file_2_contents = '6789' 161 | file_2.write_text(file_2_contents) 162 | subdir = dir_ / 'subdir' 163 | subdir.mkdir() 164 | file_3 = subdir / 'file_3.txt' 165 | file_3_contents = 'Hello!' 166 | file_3.write_text(file_3_contents) 167 | file_4 = subdir / 'file_4.txt' 168 | file_4_contents = 'Hello 2!' 169 | file_4.write_text(file_4_contents) 170 | def _jsonify_directory(self, dir_): 171 | result = {} 172 | for f in dir_.iterdir(): 173 | result[f.name] = \ 174 | self._jsonify_directory(f) if f.is_dir() else f.read_bytes() 175 | return result 176 | def test_prepare_move_fails_cleanly(self): 177 | """ 178 | Consider moving a file from src to dst. When src and dst are on the same 179 | drive (as indicated by stat().st_dev having the same value), then a 180 | simple os.rename(src, dst) suffices to "move" the file. 181 | 182 | On the other hand, if src and dst are not on the same device, then 183 | LocalFileSystem (LFS) needs to 1) copy src to dst and 2) delete src. 184 | 185 | This test checks that 2) is only performed if 1) was successful, and 186 | thus that no data loss occurs. It does this by forcing LFS to use the 187 | copy-move (and not the rename) implementation. Then, it makes 1) fail by 188 | making dst read-only. 189 | """ 190 | with TemporaryDirectory() as tmp_dir: 191 | src = Path(tmp_dir, 'src') 192 | # Need to give src some contents. Otherwise, the write to dst goes 193 | # through without raising a PermissionError. 194 | src.write_text('some_contents') 195 | src_url = as_url(src) 196 | dst_dir = Path(tmp_dir, 'dst_dir') 197 | dst_dir.mkdir() 198 | dst = dst_dir / 'dst' 199 | dst.touch() 200 | # Make dst read-only. 201 | dst.chmod(dst.stat().st_mode ^ S_IWRITE) 202 | try: 203 | permission_error_raised = False 204 | for task in self._fs._prepare_move( 205 | src_url, as_url(dst), use_rename=False 206 | ): 207 | try: 208 | task() 209 | except PermissionError: 210 | permission_error_raised = True 211 | self.assertTrue( 212 | permission_error_raised, 213 | 'PermissionError was not raised upon writing to read-only ' 214 | 'dst. This test may have to be updated to trigger this ' 215 | 'error in a different way.' 216 | ) 217 | self.assertTrue( 218 | src.exists(), 219 | 'LocalFileSystem deleted the source file even though ' 220 | 'copying it to the destination failed. This can lead to ' 221 | 'data loss!' 222 | ) 223 | finally: 224 | # Make file writable again. Otherwise cleaning up the temporary 225 | # directory fails on Windows. 226 | dst.chmod(dst.stat().st_mode | S_IWRITE) 227 | def test_move_across_devices(self): 228 | with TemporaryDirectory() as tmp_dir: 229 | src_parent = Path(tmp_dir, 'src_parent') 230 | src_parent.mkdir() 231 | src = src_parent / 'src' 232 | src.mkdir() 233 | src_file = src / 'file.txt' 234 | src_file_contents = 'contents' 235 | src_file.write_text(src_file_contents) 236 | src_subdir = src / 'subdir' 237 | src_subdir.mkdir() 238 | src_subfile = src_subdir / 'subfile.txt' 239 | src_subfile_contents = '1234' 240 | src_subfile.write_text(src_subfile_contents) 241 | dst_parent = Path(tmp_dir, 'dst_parent') 242 | dst_parent.mkdir() 243 | dst = dst_parent / src.name 244 | # Pretend that src_parent and dst_parent are on different devices: 245 | self._fs.cache.put( 246 | _urlpath(dst_parent), 'stat', fake_statresult(object(), 1) 247 | ) 248 | # We don't want to just call `self._fs.move(...)` here, for the 249 | # following reason: The bug which this test case prevents initially 250 | # occurred because `_prepare_move(...)` returned these tasks: 251 | # 1) Create `dst` 252 | # 2) Move `src_file` into `dst` 253 | # 3) Delete `src` 254 | # When returning 2), the implementation checks whether `src_file` 255 | # and `dst_file` are on the same device. But! At this point, `dst` 256 | # (and thus `dst_file`) do not yet exist. So this raised a 257 | # FileNotFoundError. 258 | # If we did just call `.move(...)`, then 2) would be computed 259 | # *after* `dst` was created in 1), thus not triggering the error. 260 | # Hence we use list(...) to force 2) to be computed before 1) runs: 261 | tasks = list(self._fs.prepare_move(as_url(src), as_url(dst))) 262 | for task in tasks: 263 | task() 264 | self.assertFalse(src.exists()) 265 | self.assertTrue(dst.exists()) 266 | dst_file_contents = (dst / src_file.name).read_text() 267 | self.assertEqual(src_file_contents, dst_file_contents) 268 | dst_subfile = dst / 'subdir' / 'subfile.txt' 269 | dst_subfile_contents = dst_subfile.read_text() 270 | self.assertEqual(src_subfile_contents, dst_subfile_contents) 271 | def setUp(self): 272 | super().setUp() 273 | self._fs = LocalFileSystem() 274 | 275 | class TemporaryCwd: 276 | def __init__(self): 277 | self._cwd_before = None 278 | self._tmp_dir = None 279 | def __enter__(self): 280 | self._cwd_before = os.getcwd() 281 | self._tmp_dir = TemporaryDirectory() 282 | tmp_dir_path = self._tmp_dir.name 283 | os.chdir(tmp_dir_path) 284 | return tmp_dir_path 285 | def __exit__(self, *_): 286 | os.chdir(self._cwd_before) 287 | self._tmp_dir.cleanup() 288 | 289 | fake_statresult = namedtuple('fake_statresult', ('st_dev', 'st_ino')) 290 | 291 | def _urlpath(file_path): 292 | return splitscheme(as_url(file_path))[1] -------------------------------------------------------------------------------- /core/commands/goto.py: -------------------------------------------------------------------------------- 1 | from core.commands.util import get_user, is_hidden 2 | from core.quicksearch_matchers import path_starts_with, basename_starts_with, \ 3 | contains_substring, contains_chars 4 | from fman import DirectoryPaneCommand, show_quicksearch, PLATFORM, load_json, \ 5 | DirectoryPaneListener, QuicksearchItem 6 | from fman.fs import exists, resolve 7 | from fman.url import as_url, splitscheme, as_human_readable 8 | from itertools import islice, chain 9 | from os.path import expanduser, islink, isabs, normpath 10 | from pathlib import Path, PurePath 11 | from random import shuffle 12 | from time import time 13 | 14 | import os 15 | import re 16 | import sys 17 | 18 | __all__ = ['GoTo', 'GoToListener'] 19 | 20 | class GoTo(DirectoryPaneCommand): 21 | def __call__(self, query=''): 22 | visited_paths = self._get_visited_paths() 23 | get_items = SuggestLocations(visited_paths) 24 | result = show_quicksearch(get_items, self._get_tab_completion, query) 25 | if result: 26 | url = self._get_target_location(*result) 27 | if exists(url): 28 | # Use OpenDirectory because it handles errors gracefully: 29 | self.pane.run_command('open_directory', {'url': url}) 30 | else: 31 | path = as_human_readable(url) 32 | _remove_from_visited_paths(visited_paths, path) 33 | def _get_target_location(self, query, suggested_dir): 34 | if suggested_dir: 35 | # The suggested_dir is for instance set when the user clicks on it 36 | # with the mouse. If set, it always takes precedence. So return it: 37 | return as_url(suggested_dir) 38 | url = as_url(expanduser(query.rstrip())) 39 | if PLATFORM == 'Windows' and re.match(r'\\[^\\]', query): 40 | # Resolve '\Some\dir' to 'C:\Some\dir'. 41 | try: 42 | url = resolve(url) 43 | except OSError: 44 | pass 45 | return url 46 | def _get_tab_completion(self, query, curr_item): 47 | if curr_item: 48 | result = curr_item.title 49 | if not result.endswith(os.sep): 50 | result += os.sep 51 | return result 52 | def _get_visited_paths(self): 53 | # TODO: Rename to Visited Locations.json? 54 | result = load_json('Visited Paths.json', default={}) 55 | # Check for length 2 because the directories in which fman opens are 56 | # already in Visited Paths: 57 | if len(result) <= 2: 58 | result.update({ 59 | path: 0 for path in self._get_default_paths() 60 | }) 61 | return result 62 | def _get_default_paths(self, exclude={'/proc', '/run', '/sys'}): 63 | home_dir = expanduser('~') 64 | result = set(self._get_nonhidden_subdirs(home_dir)) 65 | # Add directories in C:\ on Windows, or / on Unix: 66 | try: 67 | children = Path(PurePath(sys.executable).anchor).iterdir() 68 | except OSError: 69 | pass 70 | else: 71 | for child in children: 72 | if str(child) in exclude: 73 | continue 74 | try: 75 | if child.is_dir(): 76 | try: 77 | next(iter(child.iterdir())) 78 | except StopIteration: 79 | pass 80 | else: 81 | result.add(str(child)) 82 | except OSError: 83 | pass 84 | if PLATFORM == 'Linux': 85 | media_user = os.path.join('/media', get_user()) 86 | if os.path.exists(media_user): 87 | result.add(media_user) 88 | # We need to add more suggestions on Linux, because unlike Windows 89 | # and Mac, we (currently) do not integrate with the OS's native 90 | # search functionality: 91 | result.update(islice(self._traverse_by_mtime(home_dir), 500)) 92 | result.update( 93 | islice(self._traverse_by_mtime('/', exclude=exclude), 500) 94 | ) 95 | return result 96 | def _get_nonhidden_subdirs(self, dir_path): 97 | for file_name in os.listdir(dir_path): 98 | file_path = os.path.join(dir_path, file_name) 99 | if os.path.isdir(file_path) and not is_hidden(file_path): 100 | yield os.path.join(dir_path, file_name) 101 | def _traverse_by_mtime(self, dir_path, exclude=None): 102 | if exclude is None: 103 | exclude = set() 104 | to_visit = [(os.stat(dir_path), dir_path)] 105 | already_yielded = set() 106 | while to_visit: 107 | stat, parent = to_visit.pop() 108 | if parent in exclude: 109 | continue 110 | yield parent 111 | try: 112 | parent_contents = os.listdir(parent) 113 | except OSError: 114 | continue 115 | for file_name in parent_contents: 116 | if file_name.startswith('.'): 117 | continue 118 | file_path = os.path.join(parent, file_name) 119 | try: 120 | if not os.path.isdir(file_path) or islink(file_path): 121 | continue 122 | except OSError: 123 | continue 124 | try: 125 | stat = os.stat(file_path) 126 | except OSError: 127 | pass 128 | else: 129 | already_yielded.add(stat.st_ino) 130 | to_visit.append((stat, file_path)) 131 | to_visit.sort(key=lambda tpl: tpl[0].st_mtime) 132 | 133 | class GoToListener(DirectoryPaneListener): 134 | def __init__(self, *args, **kwargs): 135 | super().__init__(*args, **kwargs) 136 | self.is_first_path_change = True 137 | def on_path_changed(self): 138 | if self.is_first_path_change: 139 | # on_path_changed() is also called when fman starts. Since this is 140 | # not a user-initiated path change, we don't want to count it: 141 | self.is_first_path_change = False 142 | return 143 | url = self.pane.get_path() 144 | scheme, path = splitscheme(url) 145 | if scheme != 'file://': 146 | return 147 | visited_paths = \ 148 | load_json('Visited Paths.json', default={}, save_on_quit=True) 149 | # Ensure we're using backslashes \ on Windows: 150 | path = as_human_readable(url) 151 | visited_paths[path] = visited_paths.get(path, 0) + 1 152 | if len(visited_paths) > 500: 153 | _shrink_visited_paths(visited_paths, 250) 154 | else: 155 | # Spend a little time cleaning up outdated paths. This method is 156 | # called asynchronously, so no problem performing some work here. 157 | _remove_nonexistent(visited_paths, timeout_secs=0.01) 158 | 159 | def _shrink_visited_paths(vps, size): 160 | paths_per_count = {} 161 | for p, count in vps.items(): 162 | paths_per_count.setdefault(count, []).append(p) 163 | count_paths = sorted(paths_per_count.items()) 164 | # Remove least frequently visited paths until we reach the desired size: 165 | while len(vps) > size: 166 | count, paths = count_paths[0] 167 | del vps[paths.pop()] 168 | if not paths: 169 | count_paths = count_paths[1:] 170 | # Re-scale those paths that are left. This gives more recent paths a chance 171 | # to rise to the top. Eg. {'a': 1000, 'b': 1} -> {'a': 2, 'b': 1}. 172 | for i, (count, paths) in enumerate(count_paths): 173 | for p in paths: 174 | vps[p] = i + 1 175 | 176 | def _remove_nonexistent(vps, timeout_secs): 177 | # Randomly check visited paths for existence, until the timeout expires. 178 | # Choose all elts. with equal probability. It's tempting to be more clever 179 | # and use the visit counts somehow. But it's not clear this will be better: 180 | # Higher counts may indicate that a directory is "stable" and doesn't change 181 | # often. On the other hand, they also appear in search results more often, 182 | # hence incur a higher "penalty" when we get it wrong. So keep it simple. 183 | end_time = time() + timeout_secs 184 | paths = list(vps) 185 | shuffle(paths) 186 | for path in paths: 187 | if time() >= end_time: 188 | break 189 | try: 190 | is_dir = os.path.isdir(path) 191 | except OSError: 192 | continue 193 | if not is_dir: 194 | _remove_from_visited_paths(vps, path) 195 | 196 | def _remove_from_visited_paths(vps, path): 197 | for p in list(vps): 198 | if p == path or p.startswith(path + os.sep): 199 | try: 200 | del vps[p] 201 | except KeyError: 202 | pass 203 | 204 | def unexpand_user(path, expanduser_=expanduser): 205 | home_dir = expanduser_('~') 206 | if path.startswith(home_dir): 207 | path = '~' + path[len(home_dir):] 208 | return path 209 | 210 | class SuggestLocations: 211 | 212 | _MATCHERS = ( 213 | path_starts_with, basename_starts_with, contains_substring, 214 | contains_chars 215 | ) 216 | 217 | class LocalFileSystem: 218 | def isdir(self, path): 219 | return os.path.isdir(path) 220 | def expanduser(self, path): 221 | return expanduser(path) 222 | def listdir(self, path): 223 | return os.listdir(path) 224 | def resolve(self, path): 225 | return str(Path(path).resolve()) 226 | def samefile(self, f1, f2): 227 | return os.path.samefile(f1, f2) 228 | def find_folders_starting_with(self, pattern, timeout_secs=0.02): 229 | if PLATFORM == 'Mac': 230 | from objc import loadBundle 231 | ns = {} 232 | loadBundle( 233 | 'CoreServices.framework', ns, 234 | bundle_identifier='com.apple.CoreServices' 235 | ) 236 | pred = ns['NSPredicate'].predicateWithFormat_argumentArray_( 237 | "kMDItemContentType == 'public.folder' && " 238 | "kMDItemFSName BEGINSWITH[c] %@", [pattern] 239 | ) 240 | query = ns['NSMetadataQuery'].alloc().init() 241 | query.setPredicate_(pred) 242 | query.setSearchScopes_(ns['NSArray'].arrayWithObject_('/')) 243 | query.startQuery() 244 | ns['NSRunLoop'].currentRunLoop().runUntilDate_( 245 | ns['NSDate'].dateWithTimeIntervalSinceNow_(timeout_secs) 246 | ) 247 | query.stopQuery() 248 | for item in query.results(): 249 | yield item.valueForAttribute_("kMDItemPath") 250 | elif PLATFORM == 'Windows': 251 | import adodbapi 252 | from pythoncom import com_error 253 | try: 254 | conn = adodbapi.connect( 255 | "Provider=Search.CollatorDSO;" 256 | "Extended Properties='Application=Windows';" 257 | ) 258 | cursor = conn.cursor() 259 | 260 | # adodbapi claims to support "paramstyles", which would let us 261 | # pass parameters as an extra arg to .execute(...), without 262 | # having to worry about escaping them. Alas, adodbapi raises an 263 | # error when this feature is used. We thus have to escape the 264 | # param ourselves: 265 | def escape(param): 266 | return re.subn(r'([%_\[\]\^])', r'[\1]', param)[0] 267 | 268 | cursor.execute( 269 | "SELECT TOP 5 System.ItemPathDisplay FROM SYSTEMINDEX " 270 | "WHERE " 271 | "System.ItemType = 'Directory' AND " 272 | "System.ItemNameDisplay LIKE %r " 273 | "ORDER BY System.ItemPathDisplay" 274 | % (escape(pattern) + '%') 275 | ) 276 | for row in iter(cursor.fetchone, None): 277 | value = row['System.ItemPathDisplay'] 278 | # Seems to be None sometimes: 279 | if value: 280 | yield value 281 | except (adodbapi.Error, com_error): 282 | pass 283 | 284 | def __init__(self, visited_paths, file_system=None): 285 | if file_system is None: 286 | # Encapsulating filesystem-related functionality in a separate field 287 | # allows us to use a different implementation for testing. 288 | file_system = self.LocalFileSystem() 289 | self.visited_paths = visited_paths 290 | self.fs = file_system 291 | def __call__(self, query): 292 | possible_dirs = self._gather_dirs(query) 293 | return self._filter_matching(possible_dirs, query) 294 | def _gather_dirs(self, query): 295 | path = self._normalize_query(query) 296 | if isabs(path): 297 | if self._is_existing_dir(path): 298 | try: 299 | dir_ = self.fs.resolve(path) 300 | except OSError: 301 | dir_ = path 302 | return [dir_] + self._gather_subdirs(dir_) 303 | else: 304 | parent_path = os.path.dirname(path) 305 | if self._is_existing_dir(parent_path): 306 | try: 307 | parent = self.fs.resolve(parent_path) 308 | except OSError: 309 | pass 310 | else: 311 | return self._gather_subdirs(parent) 312 | result = set(self.visited_paths) 313 | if len(query) > 2: 314 | """Compensate for directories not yet in self.visited_paths:""" 315 | fs_folders = islice(self.fs.find_folders_starting_with(query), 100) 316 | result.update(self._sorted(fs_folders)[:10]) 317 | return self._sorted(result) 318 | def _normalize_query(self, query): 319 | result = normpath(self.fs.expanduser(query)) 320 | if PLATFORM == 'Windows': 321 | # Windows completely ignores trailing spaces in directory names at 322 | # all times. Make our implementation reflect this: 323 | result = result.rstrip(' ') 324 | # Handle the case where the user has entered a drive such as 'E:' 325 | # without the trailing backslash: 326 | if re.match(r'^[A-Z]:$', result): 327 | result += '\\' 328 | return result 329 | def _sorted(self, dirs): 330 | return sorted(dirs, key=lambda dir_: ( 331 | -self.visited_paths.get(dir_, 0), len(dir_), dir_.lower() 332 | )) 333 | def _is_existing_dir(self, path): 334 | try: 335 | return self.fs.isdir(path) 336 | except OSError: 337 | return False 338 | def _filter_matching(self, dirs, query): 339 | use_tilde = \ 340 | not query.startswith(os.path.dirname(self.fs.expanduser('~'))) 341 | result = [[] for _ in self._MATCHERS] 342 | for dir_ in dirs: 343 | title = self._unexpand_user(dir_) if use_tilde else dir_ 344 | for i, matcher in enumerate(self._MATCHERS): 345 | match = matcher(title.lower(), query.lower()) 346 | if match is not None: 347 | result[i].append(QuicksearchItem( 348 | dir_, title, highlight=match 349 | )) 350 | break 351 | return list(chain.from_iterable(result)) 352 | def _gather_subdirs(self, dir_): 353 | result = [] 354 | try: 355 | dir_contents = self.fs.listdir(dir_) 356 | except OSError: 357 | pass 358 | else: 359 | for name in dir_contents: 360 | file_path = os.path.join(dir_, name) 361 | try: 362 | if self.fs.isdir(file_path): 363 | result.append(file_path) 364 | except OSError: 365 | pass 366 | return self._sorted(result) 367 | def _unexpand_user(self, path): 368 | return unexpand_user(path, self.fs.expanduser) -------------------------------------------------------------------------------- /core/fs/local/__init__.py: -------------------------------------------------------------------------------- 1 | from core.trash import move_to_trash 2 | from core.util import filenotfounderror 3 | from datetime import datetime 4 | from errno import ENOENT 5 | from fman import PLATFORM, Task 6 | from fman.fs import FileSystem, cached 7 | from fman.impl.util.qt.thread import run_in_main_thread 8 | from fman.url import as_url, splitscheme, as_human_readable, join, basename, \ 9 | dirname 10 | from io import UnsupportedOperation 11 | from os import remove, rmdir 12 | from os.path import islink, samestat, isabs, splitdrive 13 | from pathlib import Path 14 | from PyQt5.QtCore import QFileSystemWatcher 15 | from shutil import copystat 16 | from stat import S_ISDIR, S_IWRITE 17 | 18 | import errno 19 | import os 20 | 21 | if PLATFORM == 'Windows': 22 | from core.fs.local.windows.drives import DrivesFileSystem, DriveName 23 | from core.fs.local.windows.network import NetworkFileSystem 24 | 25 | class LocalFileSystem(FileSystem): 26 | 27 | scheme = 'file://' 28 | 29 | def __init__(self): 30 | super().__init__() 31 | self._watcher = None 32 | def get_default_columns(self, path): 33 | return 'core.Name', 'core.Size', 'core.Modified' 34 | def exists(self, path): 35 | os_path = self._url_to_os_path(path) 36 | return self._isabs(os_path) and Path(os_path).exists() 37 | def iterdir(self, path): 38 | os_path = self._url_to_os_path(path) 39 | if not self._isabs(os_path): 40 | raise filenotfounderror(path) 41 | # Use os.listdir(...) instead of Path(...).iterdir() for performance: 42 | return os.listdir(os_path) 43 | def is_dir(self, existing_path): 44 | # Like Python's isdir(...) except raises FileNotFoundError if the file 45 | # does not exist and OSError if there is another error. 46 | return S_ISDIR(self.stat(existing_path).st_mode) 47 | @cached 48 | def stat(self, path): 49 | os_path = self._url_to_os_path(path) 50 | if not self._isabs(os_path): 51 | raise filenotfounderror(path) 52 | try: 53 | return os.stat(os_path) 54 | except FileNotFoundError: 55 | return os.stat(os_path, follow_symlinks=False) 56 | def size_bytes(self, path): 57 | return self.stat(path).st_size 58 | def modified_datetime(self, path): 59 | return datetime.fromtimestamp(self.stat(path).st_mtime) 60 | def touch(self, path): 61 | os_path = Path(self._url_to_os_path(path)) 62 | if not os_path.is_absolute(): 63 | raise ValueError('Path must be absolute') 64 | try: 65 | os_path.touch(exist_ok=False) 66 | except FileExistsError: 67 | os_path.touch(exist_ok=True) 68 | else: 69 | self.notify_file_added(path) 70 | def mkdir(self, path): 71 | os_path = Path(self._url_to_os_path(path)) 72 | if not os_path.is_absolute(): 73 | raise ValueError('Path must be absolute') 74 | try: 75 | os_path.mkdir() 76 | except FileNotFoundError: 77 | raise 78 | except IsADirectoryError: # macOS 79 | raise FileExistsError(path) 80 | except OSError as e: 81 | if e.errno == ENOENT: 82 | raise filenotfounderror(path) from e 83 | elif os_path.exists(): 84 | # On at least Windows, Path('Z:\\').mkdir() raises 85 | # PermissionError instead of FileExistsError. We want the latter 86 | # however because #makedirs(...) relies on it. So handle this 87 | # case generally here: 88 | raise FileExistsError(path) from e 89 | else: 90 | raise 91 | self.notify_file_added(path) 92 | def move(self, src_url, dst_url): 93 | self._check_transfer_precnds(src_url, dst_url) 94 | for task in self._prepare_move(src_url, dst_url): 95 | task() 96 | def prepare_move(self, src_url, dst_url): 97 | self._check_transfer_precnds(src_url, dst_url) 98 | return self._prepare_move(src_url, dst_url, measure_size=True) 99 | def _prepare_move( 100 | self, src_url, dst_url, measure_size=False, use_rename=True, 101 | expected_st_dev=None 102 | ): 103 | if expected_st_dev is None: 104 | expected_st_dev = {} 105 | src_path, dst_path = self._check_transfer_precnds(src_url, dst_url) 106 | if use_rename: 107 | src_stat = self.stat(src_path) 108 | dst_par_path = splitscheme(dirname(dst_url))[1] 109 | try: 110 | dst_par_dev = expected_st_dev[dst_par_path] 111 | except KeyError: 112 | dst_par_dev = self.stat(dst_par_path).st_dev 113 | if src_stat.st_dev == dst_par_dev: 114 | yield Task( 115 | 'Moving ' + basename(src_url), size=1, 116 | fn=self._rename, args=(src_url, dst_url) 117 | ) 118 | return 119 | src_is_dir = self.is_dir(src_path) 120 | if src_is_dir: 121 | yield Task( 122 | 'Creating ' + basename(dst_url), fn=self.mkdir, args=(dst_path,) 123 | ) 124 | dst_par_path = splitscheme(dirname(dst_url))[1] 125 | # Expect `dst_path` to inherit .st_dev from its parent: 126 | try: 127 | expected_st_dev[dst_path] = expected_st_dev[dst_par_path] 128 | except KeyError: 129 | expected_st_dev[dst_path] = self.stat(dst_par_path).st_dev 130 | for name in self.iterdir(src_path): 131 | try: 132 | yield from self._prepare_move( 133 | join(src_url, name), join(dst_url, name), measure_size, 134 | use_rename, expected_st_dev 135 | ) 136 | except FileNotFoundError: 137 | pass 138 | yield DeleteIfEmpty(self, src_url) 139 | else: 140 | size = self.size_bytes(src_path) if measure_size else 0 141 | # It's tempting to "yield copy" then "yield delete" here. But this 142 | # can lead to data loss: fman / the user may choose to ignore errors 143 | # in a particular subtask. (Eg. moving file1.txt failed but the user 144 | # wants to go on to move file2.txt.) If we ignored a failure in 145 | # "yield copy" but then execute the "yield delete", we would delete 146 | # a file that has not been copied! 147 | # To avoid this, we "copy and delete" as a single, atomic task: 148 | yield MoveByCopying(self, src_url, dst_url, size) 149 | def _rename(self, src_url, dst_url): 150 | src_path = splitscheme(src_url)[1] 151 | os_src_path = self._url_to_os_path(src_path) 152 | dst_path = splitscheme(dst_url)[1] 153 | os_dst_path = self._url_to_os_path(dst_path) 154 | Path(os_src_path).replace(os_dst_path) 155 | self.notify_file_removed(src_path) 156 | self.notify_file_added(dst_path) 157 | def move_to_trash(self, path): 158 | for task in self.prepare_trash(path): 159 | task() 160 | def prepare_trash(self, path): 161 | os_path = self._url_to_os_path(path) 162 | if not self._isabs(os_path): 163 | raise filenotfounderror(path) 164 | yield Task( 165 | 'Deleting ' + path.rsplit('/', 1)[-1], size=1, 166 | fn=self._do_trash, args=(path, os_path) 167 | ) 168 | def _do_trash(self, path, os_path): 169 | move_to_trash(os_path) 170 | self.notify_file_removed(path) 171 | def delete(self, path): 172 | for task in self.prepare_delete(path): 173 | task() 174 | def prepare_delete(self, path): 175 | os_path = self._url_to_os_path(path) 176 | if not self._isabs(os_path): 177 | raise filenotfounderror(path) 178 | # is_dir(...) follows symlinks. But if `path` is a symlink, we need to 179 | # use remove(...) instead of rmdir(...) to avoid NotADirectoryError. 180 | # So check if `path` is a symlink: 181 | if self.is_dir(path) and not islink(os_path): 182 | for name in self.iterdir(path): 183 | try: 184 | yield from self.prepare_delete(path + '/' + name) 185 | except FileNotFoundError: 186 | pass 187 | delete_fn = rmdir 188 | else: 189 | delete_fn = remove 190 | yield Task( 191 | 'Deleting ' + path.rsplit('/', 1)[-1], size=1, 192 | fn=self._do_delete, args=(path, delete_fn) 193 | ) 194 | def _do_delete(self, path, delete_fn): 195 | os_path = self._url_to_os_path(path) 196 | try: 197 | delete_fn(os_path) 198 | except OSError as orig_exc: 199 | try: 200 | mode = self.stat(path).st_mode 201 | except OSError: 202 | mode = 0 203 | if not (mode & S_IWRITE): 204 | try: 205 | Path(os_path).chmod(mode | S_IWRITE) 206 | except OSError: 207 | raise orig_exc 208 | # Try again, now the file is writeable: 209 | delete_fn(os_path) 210 | self.notify_file_removed(path) 211 | def resolve(self, path): 212 | path = self._url_to_os_path(path) 213 | if not self._isabs(path): 214 | raise filenotfounderror(path) 215 | if PLATFORM == 'Windows': 216 | is_unc_server = path.startswith(r'\\') and not '\\' in path[2:] 217 | if is_unc_server: 218 | # Python can handle \\server\folder but not \\server. Defer to 219 | # the network:// file system. 220 | return 'network://' + path[2:] 221 | p = Path(path) 222 | try: 223 | try: 224 | path = p.resolve(strict=True) 225 | except TypeError: 226 | # fman's "production Python version" is 3.6 but we want to be 227 | # able to develop using 3.5 as well. So add this workaround for 228 | # Python < 3.6: 229 | path = p.resolve() 230 | except FileNotFoundError: 231 | if not p.exists(): 232 | raise 233 | except OSError as e: 234 | # We for instance get errno.EINVAL ("[WinError 1]") when trying to 235 | # resolve folders on Cryptomator drives on Windows. Ignore it: 236 | if e.errno != errno.EINVAL: 237 | raise 238 | return as_url(path) 239 | def samefile(self, path1, path2): 240 | s1 = self.stat(path1) 241 | s2 = self.stat(path2) 242 | if s1.st_ino and s1.st_dev: 243 | return samestat(s1, s2) 244 | # samestat(...) above simply compares .st_ino and .st_dev. 245 | # However, there are cases where both of these values are 0. 246 | # This for instance happens on Windows Google Drive File Stream folders. 247 | # Fall back to comparing the (resolved) path to give meaningful results: 248 | return self.resolve(path1) == self.resolve(path2) 249 | def copy(self, src_url, dst_url): 250 | self._check_transfer_precnds(src_url, dst_url) 251 | for task in self._prepare_copy(src_url, dst_url): 252 | task() 253 | def prepare_copy(self, src_url, dst_url): 254 | self._check_transfer_precnds(src_url, dst_url) 255 | return self._prepare_copy(src_url, dst_url, measure_size=True) 256 | def _prepare_copy(self, src_url, dst_url, measure_size=False): 257 | src_path, dst_path = self._check_transfer_precnds(src_url, dst_url) 258 | src_is_dir = self.is_dir(src_path) 259 | if src_is_dir: 260 | yield Task( 261 | 'Creating ' + basename(dst_url), fn=self.mkdir, args=(dst_path,) 262 | ) 263 | for name in self.iterdir(src_path): 264 | try: 265 | yield from self._prepare_copy( 266 | join(src_url, name), join(dst_url, name), measure_size 267 | ) 268 | except FileNotFoundError: 269 | pass 270 | else: 271 | size = self.size_bytes(src_path) if measure_size else 0 272 | yield CopyFile(self, src_url, dst_url, size) 273 | @run_in_main_thread 274 | def watch(self, path): 275 | self._get_watcher().addPath(self._url_to_os_path(path)) 276 | @run_in_main_thread 277 | def unwatch(self, path): 278 | self._get_watcher().removePath(self._url_to_os_path(path)) 279 | def _get_watcher(self): 280 | # Instantiate QFileSystemWatcher as late as possible. It requires a 281 | # QApplication which isn't available in some tests. 282 | if self._watcher is None: 283 | if PLATFORM == 'Windows': 284 | # On Windows, QFileSystemWatcher keeps excessive locks on files 285 | # and directories. This sometimes makes it impossible for fman 286 | # or other apps to access them properly. One example of this are 287 | # USB drives that can't be ejected as long as fman is running. 288 | # So don't watch files on Windows for now, perhaps until we have 289 | # a better implementation. 290 | self._watcher = StubFileSystemWatcher() 291 | else: 292 | self._watcher = QFileSystemWatcher() 293 | self._watcher.directoryChanged.connect(self._on_file_changed) 294 | self._watcher.fileChanged.connect(self._on_file_changed) 295 | return self._watcher 296 | def _on_file_changed(self, file_path): 297 | path_forward_slashes = splitscheme(as_url(file_path))[1] 298 | self.notify_file_changed(path_forward_slashes) 299 | def _check_transfer_precnds(self, src_url, dst_url): 300 | src_scheme, src_path = splitscheme(src_url) 301 | dst_scheme, dst_path = splitscheme(dst_url) 302 | if src_scheme != self.scheme or dst_scheme != self.scheme: 303 | raise UnsupportedOperation( 304 | "%s only supports transferring to and from %s." % 305 | (self.__class__.__name__, self.scheme) 306 | ) 307 | if not self._isabs(self._url_to_os_path(dst_path)): 308 | raise ValueError('Destination path must be absolute') 309 | return src_path, dst_path 310 | def _url_to_os_path(self, path): 311 | # Convert a "URL path" a/b to a path understood by the OS, eg. a\b on 312 | # Windows. An important effect of this function is that it turns 313 | # C: -> C:\. This is required for Python functions such as Path#resolve. 314 | return as_human_readable(self.scheme + path) 315 | def _isabs(self, os_path): 316 | # Python's isabs(...) says \\host\share is *not* absolute. But for our 317 | # purposes, it is. So add some extra logic to handle this case: 318 | return isabs(os_path) or splitdrive(os_path)[0] 319 | 320 | class CopyFile(Task): 321 | def __init__(self, fs, src_url, dst_url, size_bytes): 322 | super().__init__('Copying ' + basename(src_url), size=size_bytes) 323 | self._fs = fs 324 | self._src_url = src_url 325 | self._dst_url = dst_url 326 | def __call__(self): 327 | dst_urlpath = splitscheme(self._dst_url)[1] 328 | dst_existed = self._fs.exists(dst_urlpath) 329 | src = as_human_readable(self._src_url) 330 | dst = as_human_readable(self._dst_url) 331 | if islink(src): 332 | os.symlink(os.readlink(src), dst) 333 | else: 334 | with open(src, 'rb') as fsrc: 335 | with open(dst, 'wb') as fdst: 336 | num_written = 0 337 | while True: 338 | self.check_canceled() 339 | buf = fsrc.read(16 * 1024) 340 | if not buf: 341 | break 342 | num_written += fdst.write(buf) 343 | self.set_progress(num_written) 344 | copystat(src, dst, follow_symlinks=False) 345 | if not dst_existed: 346 | self._fs.notify_file_added(dst_urlpath) 347 | 348 | class MoveByCopying(Task): 349 | def __init__(self, fs, src_url, dst_url, size_bytes): 350 | super().__init__('Moving ' + basename(src_url), size=size_bytes) 351 | self._fs = fs 352 | self._src_url = src_url 353 | self._dst_url = dst_url 354 | def __call__(self, *args, **kwargs): 355 | self._fs.copy(self._src_url, self._dst_url) 356 | self._fs.delete(splitscheme(self._src_url)[1]) 357 | 358 | class DeleteIfEmpty(Task): 359 | def __init__(self, fs, dir_url): 360 | super().__init__('Deleting ' + basename(dir_url), size=1) 361 | self._fs = fs 362 | self._dir_url = dir_url 363 | def __call__(self, *args, **kwargs): 364 | try: 365 | rmdir(as_human_readable(self._dir_url)) 366 | except FileNotFoundError: 367 | pass 368 | except OSError as e: 369 | if e.errno != errno.ENOTEMPTY: 370 | raise 371 | else: 372 | self._fs.notify_file_removed(splitscheme(self._dir_url)[1]) 373 | 374 | class StubFileSystemWatcher: 375 | def addPath(self, path): 376 | pass 377 | def removePath(self, path): 378 | pass -------------------------------------------------------------------------------- /core/tests/fs/test_zip.py: -------------------------------------------------------------------------------- 1 | from errno import ENOENT 2 | from core.fs.zip import ZipFileSystem 3 | from core.tests import StubFS 4 | from datetime import date 5 | from fman.url import as_url, join, as_human_readable, splitscheme 6 | from os import listdir 7 | from pathlib import Path 8 | from shutil import copyfile 9 | from tempfile import TemporaryDirectory 10 | from unicodedata import normalize 11 | from unittest import TestCase 12 | from zipfile import ZipFile 13 | 14 | import os 15 | import os.path 16 | 17 | class ZipFileSystemTest(TestCase): 18 | def test_iterdir(self): 19 | self._expect_iterdir_result('', {'ZipFileTest'}) 20 | self._expect_iterdir_result( 21 | 'ZipFileTest', 22 | {'Directory', 'Empty directory', 'file.txt', 'ça va.txt'} 23 | ) 24 | self._expect_iterdir_result( 25 | 'ZipFileTest/Directory', {'Subdirectory', 'file 2.txt'} 26 | ) 27 | self._expect_iterdir_result( 28 | 'ZipFileTest/Directory/Subdirectory', {'file 3.txt'} 29 | ) 30 | self._expect_iterdir_result('ZipFileTest/Empty directory', set()) 31 | def test_iterdir_empty_zip(self): 32 | with TemporaryDirectory() as zip_container: 33 | zip_path = os.path.join(zip_container, 'test.zip') 34 | self._create_empty_zip(zip_path) 35 | self.assertEqual([], self._listdir(self._path('', zip_path))) 36 | def test_iterdir_sparse_zip(self): 37 | with TemporaryDirectory() as tmp_dir: 38 | zip_path = os.path.join(tmp_dir, 'test.zip') 39 | for depth in range(3): 40 | file_relpath = os.path.join(*(['dir'] * depth + ['file.txt'])) 41 | with ZipFile(zip_path, 'w') as zip_file: 42 | zip_file.write(__file__, file_relpath) 43 | for level in range(depth): 44 | dir_path = self._path('/'.join(['dir'] * level), zip_path) 45 | self.assertEqual( 46 | ['dir'], self._listdir(dir_path), 47 | 'Failed at nesting level ' + file_relpath 48 | ) 49 | def test_iterdir_nonexistent_zip(self): 50 | with self.assertRaises(FileNotFoundError): 51 | self._listdir('nonexistent.zip') 52 | def test_iterdir_nonexistent_path_in_zip(self): 53 | with self.assertRaises(FileNotFoundError): 54 | self._listdir(self._path('nonexistent')) 55 | def _listdir(self, zip_urlpath): 56 | return list(self._fs.iterdir(zip_urlpath)) 57 | def test_is_dir(self): 58 | for dir_ in self._dirs_in_zip: 59 | self.assertTrue(self._fs.is_dir(self._path(dir_)), dir_) 60 | for nondir in self._files_in_zip: 61 | self.assertFalse(self._fs.is_dir(self._path(nondir)), nondir) 62 | for nonexistent in ('nonexistent', 'ZipFileTest/nonexistent'): 63 | with self.assertRaises(FileNotFoundError): 64 | self._fs.is_dir(self._path(nonexistent)), nonexistent 65 | def test_exists(self): 66 | for existent in self._dirs_in_zip + self._files_in_zip: 67 | self.assertTrue(self._fs.exists(self._path(existent)), existent) 68 | for nonexistent in ('nonexistent', 'ZipFileTest/nonexistent'): 69 | self.assertFalse( 70 | self._fs.exists(self._path(nonexistent)), nonexistent 71 | ) 72 | def test_extract_entire_zip(self): 73 | self._test_extract_dir('') 74 | def _test_extract_dir(self, path_in_zip): 75 | expected_files = self._get_zip_contents(path_in_zip=path_in_zip) 76 | with TemporaryDirectory() as tmp_dir: 77 | # Create a subdirectory because the destination directory of a copy 78 | # operation must not yet exist: 79 | dst_dir = os.path.join(tmp_dir, 'dest') 80 | self._fs.copy(self._url(path_in_zip), as_url(dst_dir)) 81 | self.assertEqual(expected_files, self._read_directory(dst_dir)) 82 | def test_extract_subdir(self): 83 | self._test_extract_dir('ZipFileTest/Directory') 84 | def test_extract_empty_directory(self): 85 | self._test_extract_dir('ZipFileTest/Empty directory') 86 | def test_extract_file(self): 87 | with TemporaryDirectory() as tmp_dir: 88 | file_path = 'ZipFileTest/file.txt' 89 | dest_path = os.path.join(tmp_dir, 'file.txt') 90 | self._fs.copy(self._url(file_path), as_url(dest_path)) 91 | self.assertEqual(['file.txt'], listdir(tmp_dir)) 92 | expected_contents = self._get_zip_contents(path_in_zip=file_path) 93 | with open(dest_path) as f: 94 | self.assertEqual(expected_contents, f.read()) 95 | def test_extract_nonexistent(self): 96 | with self.assertRaises(FileNotFoundError): 97 | with TemporaryDirectory() as tmp_dir: 98 | self._fs.copy(self._url('nonexistent'), as_url(tmp_dir)) 99 | def test_add_file(self): 100 | with TemporaryDirectory() as tmp_dir: 101 | file_to_add = os.path.join(tmp_dir, 'tmp.txt') 102 | file_contents = 'added!' 103 | with open(file_to_add, 'w') as f: 104 | f.write(file_contents) 105 | dest_url_in_zip = self._url('ZipFileTest/Directory/added.txt') 106 | self._fs.copy(as_url(file_to_add), dest_url_in_zip) 107 | dest_url = join(as_url(tmp_dir), 'extracted.txt') 108 | self._fs.copy(dest_url_in_zip, dest_url) 109 | with open(as_human_readable(dest_url)) as f: 110 | actual_contents = f.read() 111 | self.assertEqual(file_contents, actual_contents) 112 | def test_add_directory(self): 113 | with TemporaryDirectory() as zip_contents: 114 | with ZipFile(self._zip) as zip_file: 115 | zip_file.extractall(zip_contents) 116 | with TemporaryDirectory() as zip_container: 117 | zip_path = os.path.join(zip_container, 'test.zip') 118 | self._create_empty_zip(zip_path) 119 | self._fs.copy( 120 | as_url(os.path.join(zip_contents, 'ZipFileTest')), 121 | join(as_url(zip_path, 'zip://'), 'ZipFileTest') 122 | ) 123 | self._expect_zip_contents(self._get_zip_contents(), zip_path) 124 | def test_replace_file(self): 125 | with TemporaryDirectory() as tmp_dir: 126 | zip_path = os.path.join(tmp_dir, 'test.zip') 127 | some_file = os.path.join(tmp_dir, 'tmp.txt') 128 | with open(some_file, 'w') as f: 129 | f.write('added!') 130 | with ZipFile(zip_path, 'w') as zip_file: 131 | zip_file.write(some_file, 'tmp.txt') 132 | expected_contents = b'replaced!' 133 | with open(some_file, 'wb') as f: 134 | f.write(expected_contents) 135 | dest_url_in_zip = join(as_url(zip_path, 'zip://'), 'tmp.txt') 136 | self._fs.copy(as_url(some_file), dest_url_in_zip) 137 | with ZipFile(zip_path) as zip_file: 138 | # A primitive implementation would have two 'tmp.txt' entries: 139 | self.assertEqual(['tmp.txt'], zip_file.namelist()) 140 | with zip_file.open('tmp.txt') as f_in_zip: 141 | self.assertEqual(expected_contents, f_in_zip.read()) 142 | def test_mkdir(self): 143 | with TemporaryDirectory() as tmp_dir: 144 | zip_path = os.path.join(tmp_dir, 'test.zip') 145 | self._create_empty_zip(zip_path) 146 | self._fs.mkdir(splitscheme(as_url(zip_path, 'zip://'))[1] + '/dir') 147 | self._expect_zip_contents({'dir': {}}, zip_path) 148 | def test_mkdir_raises_fileexistserror(self): 149 | with TemporaryDirectory() as tmp_dir: 150 | zip_path = os.path.join(tmp_dir, 'test.zip') 151 | self._create_empty_zip(zip_path) 152 | dir_url_path = splitscheme(as_url(zip_path, 'zip://'))[1] + '/dir' 153 | self._fs.mkdir(dir_url_path) 154 | with self.assertRaises(FileExistsError): 155 | self._fs.mkdir(dir_url_path) 156 | def test_mkdir_raises_filenotfounderror(self): 157 | with TemporaryDirectory() as tmp_dir: 158 | zip_path = os.path.join(tmp_dir, 'test.zip') 159 | self._create_empty_zip(zip_path) 160 | zip_url_path = splitscheme(as_url(zip_path, 'zip://'))[1] 161 | with self.assertRaises(OSError) as cm: 162 | self._fs.mkdir(zip_url_path + '/nonexistent/dir') 163 | self.assertEqual(ENOENT, cm.exception.errno) 164 | def test_mkdir_empty(self): 165 | with TemporaryDirectory() as tmp_dir: 166 | zip_path = os.path.join(tmp_dir, 'test.zip') 167 | self._fs.mkdir(splitscheme(as_url(zip_path))[1]) 168 | with ZipFile(zip_path) as zip_file: 169 | self.assertEqual([], zip_file.namelist()) 170 | def test_delete_file(self): 171 | self._test_delete('ZipFileTest/Directory/Subdirectory/file 3.txt') 172 | def _test_delete(self, path_in_zip): 173 | expected_contents = self._get_zip_contents() 174 | self._pop_from_dir_dict(expected_contents, path_in_zip) 175 | self._fs.delete(self._path(path_in_zip)) 176 | self.assertEqual(expected_contents, self._get_zip_contents()) 177 | def test_delete_directory(self): 178 | self._test_delete('ZipFileTest/Directory') 179 | def test_delete_empty_directory(self): 180 | self._test_delete('ZipFileTest/Empty directory') 181 | def test_delete_main_directory(self): 182 | self._test_delete('ZipFileTest') 183 | def test_delete_nonexistent(self): 184 | with self.assertRaises(FileNotFoundError): 185 | self._fs.delete(self._path('nonexistent')) 186 | def test_move_file_out_of_archive(self): 187 | file_path = 'ZipFileTest/Directory/Subdirectory/file 3.txt' 188 | expected_zip_contents = self._get_zip_contents() 189 | removed = self._pop_from_dir_dict(expected_zip_contents, file_path) 190 | with TemporaryDirectory() as tmp_dir: 191 | dst = os.path.join(tmp_dir, 'test.tzt') 192 | self._fs.move(self._url(file_path), as_url(dst)) 193 | self.assertEqual(expected_zip_contents, self._get_zip_contents()) 194 | with open(dst) as f: 195 | self.assertEqual(removed, f.read()) 196 | def test_move_dir_out_of_archive(self): 197 | self._test_move_dir_out_of_archive('ZipFileTest/Directory') 198 | def test_move_empty_dir_out_of_archive(self): 199 | self._test_move_dir_out_of_archive('ZipFileTest/Empty directory') 200 | def test_move_main_dir_out_of_archive(self): 201 | self._test_move_dir_out_of_archive('ZipFileTest') 202 | def _test_move_dir_out_of_archive(self, path_in_zip): 203 | expected_zip_contents = self._get_zip_contents() 204 | removed = self._pop_from_dir_dict(expected_zip_contents, path_in_zip) 205 | with TemporaryDirectory() as tmp_dir: 206 | dst_dir = os.path.join(tmp_dir, 'dest') 207 | self._fs.move(self._url(path_in_zip), as_url(dst_dir)) 208 | self.assertEqual(expected_zip_contents, self._get_zip_contents()) 209 | self.assertEqual(removed, self._read_directory(dst_dir)) 210 | def test_move_file_into_archive(self): 211 | expected_zip_contents = self._get_zip_contents() 212 | with TemporaryDirectory() as tmp_dir: 213 | file_path = os.path.join(tmp_dir, 'test.txt') 214 | with open(file_path, 'w') as f: 215 | f.write('success!') 216 | dst_url = self._url('test_dest.txt') 217 | self._fs.move(as_url(file_path), dst_url) 218 | self.assertFalse(Path(file_path).exists()) 219 | expected_zip_contents['test_dest.txt'] = 'success!' 220 | self.assertEqual(expected_zip_contents, self._get_zip_contents()) 221 | def test_rename_directory(self): 222 | expected_contents = self._get_zip_contents() 223 | file_path = 'ZipFileTest/Directory' 224 | expected_contents['Destination'] = \ 225 | self._pop_from_dir_dict(expected_contents, file_path) 226 | self._fs.move(self._url(file_path), self._url('Destination')) 227 | self.assertEqual(expected_contents, self._get_zip_contents()) 228 | def test_rename_file(self): 229 | expected_contents = self._get_zip_contents() 230 | src_path = 'ZipFileTest/Directory/Subdirectory/file 3.txt' 231 | expected_contents['ZipFileTest']['Directory']['destination.txt'] = \ 232 | self._pop_from_dir_dict(expected_contents, src_path) 233 | self._fs.move( 234 | self._url(src_path), 235 | self._url('ZipFileTest/Directory/destination.txt') 236 | ) 237 | self.assertEqual(expected_contents, self._get_zip_contents()) 238 | def test_move_file_between_archives(self, operation=None, get_contents=None): 239 | if operation is None: 240 | operation = self._fs.move 241 | if get_contents is None: 242 | get_contents = self._pop_from_dir_dict 243 | src_path = 'ZipFileTest/Directory/Subdirectory/file 3.txt' 244 | expected_contents = self._get_zip_contents() 245 | src_contents = get_contents(expected_contents, src_path) 246 | with TemporaryDirectory() as dst_dir: 247 | dst_zip = os.path.join(dst_dir, 'dest.zip') 248 | # Give the Zip file some contents: 249 | dummy_txt = os.path.join(dst_dir, 'dummy.txt') 250 | dummy_contents = 'some contents' 251 | with open(dummy_txt, 'w') as f: 252 | f.write(dummy_contents) 253 | with ZipFile(dst_zip, 'w') as zip_file: 254 | zip_file.write(dummy_txt, 'dummy.txt') 255 | dst_url = join(as_url(dst_zip, 'zip://'), 'dest.txt') 256 | operation(self._url(src_path), dst_url) 257 | self.assertEqual(expected_contents, self._get_zip_contents()) 258 | self.assertEqual( 259 | {'dummy.txt': dummy_contents, 'dest.txt': src_contents}, 260 | self._get_zip_contents(dst_zip) 261 | ) 262 | def test_copy_file_between_archives(self): 263 | self.test_move_file_between_archives( 264 | self._fs.copy, self._get_from_dir_dict 265 | ) 266 | def test_size_bytes_file(self): 267 | file_path = 'ZipFileTest/Directory/Subdirectory/file 3.txt' 268 | file_contents = self._get_zip_contents(path_in_zip=file_path) 269 | self.assertEqual( 270 | len(file_contents), self._fs.size_bytes(self._path(file_path)) 271 | ) 272 | def test_size_bytes_dir(self): 273 | dir_path = self._path('ZipFileTest/Directory/Subdirectory') 274 | self.assertIn(self._fs.size_bytes(dir_path), (0, None)) 275 | def test_size_bytes_root(self): 276 | self.assertIsNone(self._fs.size_bytes(self._path(''))) 277 | def test_size_bytes_nonexistent_zip(self): 278 | with self.assertRaises(FileNotFoundError): 279 | self._fs.size_bytes('nonexistent') 280 | def test_size_bytes_nonexistent_path_in_zip(self): 281 | with self.assertRaises(FileNotFoundError): 282 | self._fs.size_bytes(self._path('nonexistent')) 283 | def test_modified_datetime_file(self): 284 | file_path = 'ZipFileTest/Directory/Subdirectory/file 3.txt' 285 | mtime = self._fs.modified_datetime(self._path(file_path)) 286 | # Compare by date only because the time depends on the system time zone: 287 | self.assertEqual(date(2017, 11, 8), mtime.date()) 288 | def test_modified_datetime_dir(self): 289 | dir_path = self._path('ZipFileTest/Empty directory') 290 | mtime = self._fs.modified_datetime(dir_path) 291 | # Compare by date only because the time depends on the system time zone: 292 | self.assertEqual(date(2017, 11, 8), mtime.date()) 293 | def test_modified_datetime_root(self): 294 | self.assertIsNone(self._fs.modified_datetime(self._path(''))) 295 | def test_modified_datetime_nonexistent_zip(self): 296 | with self.assertRaises(FileNotFoundError): 297 | self._fs.modified_datetime('nonexistent') 298 | def test_modified_datetime_nonexistent_path_in_zip(self): 299 | with self.assertRaises(FileNotFoundError): 300 | self._fs.modified_datetime(self._path('nonexistent')) 301 | def test_resolve_nonexistent_zip_raises_filenotfounderror(self): 302 | with self.assertRaises(FileNotFoundError): 303 | tmp_url = as_url(self._tmp_dir.name) 304 | self._fs.resolve(splitscheme(join(tmp_url, 'non-existent.zip'))[1]) 305 | def test_resolve_nonexistent_file(self): 306 | with self.assertRaises(FileNotFoundError): 307 | self._fs.resolve('non-existent') 308 | def _expect_iterdir_result(self, path_in_zip, expected_contents): 309 | full_path = self._path(path_in_zip) 310 | # Consider ç: It can be encoded in Unicode as "latin small letter c 311 | # with cedilla" (U+00E7) but also as a c followed by "combining 312 | # cedilla" (U+0327). This source file uses the former, but on Mac, 313 | # the file system gives us the latter. To accommodate this, we normalize 314 | # Unicode strings first: 315 | norm_unicode = lambda strs: set(normalize('NFC', s) for s in strs) 316 | self.assertEqual( 317 | norm_unicode(expected_contents), 318 | norm_unicode(self._fs.iterdir(full_path)) 319 | ) 320 | def _url(self, path_in_zip): 321 | return as_url(self._path(path_in_zip), 'zip://') 322 | def _path(self, path_in_zip, zip_path=None): 323 | if zip_path is None: 324 | zip_path = self._zip 325 | return zip_path.replace(os.sep, '/') + \ 326 | ('/' if path_in_zip else '') + \ 327 | path_in_zip 328 | def _get_zip_contents(self, zip_path=None, path_in_zip=None): 329 | if zip_path is None: 330 | zip_path = self._zip 331 | with TemporaryDirectory() as tmp_dir: 332 | with ZipFile(zip_path) as zip_file: 333 | zip_file.extractall(tmp_dir) 334 | zip_contents = self._read_directory(tmp_dir) 335 | return self._get_from_dir_dict(zip_contents, path_in_zip) 336 | def _pop_from_dir_dict(self, dir_dict, path): 337 | parts = path.split('/') 338 | for part in parts[:-1]: 339 | dir_dict = dir_dict[part] 340 | return dir_dict.pop(parts[-1]) 341 | def _get_from_dir_dict(self, dir_dict, path): 342 | if not path: 343 | return dir_dict 344 | for part in path.split('/'): 345 | dir_dict = dir_dict[part] 346 | return dir_dict 347 | def _read_directory(self, dir_path): 348 | result = {} 349 | for child in Path(dir_path).iterdir(): 350 | if child.is_dir(): 351 | child_contents = self._read_directory(child) 352 | else: 353 | child_contents = child.read_text() 354 | result[child.name] = child_contents 355 | return result 356 | def _expect_zip_contents(self, contents, zip_file_path): 357 | with TemporaryDirectory() as tmp_dir: 358 | with ZipFile(zip_file_path) as zip_file: 359 | zip_file.extractall(tmp_dir) 360 | self.assertEqual(contents, self._read_directory(tmp_dir)) 361 | def _create_empty_zip(self, path): 362 | ZipFile(path, 'w').close() 363 | def setUp(self): 364 | super().setUp() 365 | fman_fs = StubFS() 366 | self._fs = ZipFileSystem(fman_fs, {'.zip'}) 367 | fman_fs.add_child(self._fs) 368 | self._tmp_dir = TemporaryDirectory() 369 | self._zip = copyfile( 370 | os.path.join(os.path.dirname(__file__), 'ZipFileSystemTest.zip'), 371 | os.path.join(self._tmp_dir.name, 'ZipFileSystemTest.zip') 372 | ) 373 | self._dirs_in_zip = ( 374 | '', 'ZipFileTest', 'ZipFileTest/Directory', 375 | 'ZipFileTest/Directory/Subdirectory', 'ZipFileTest/Empty directory' 376 | ) 377 | self._files_in_zip = ( 378 | 'ZipFileTest/file.txt', 'ZipFileTest/Directory/file 2.txt', 379 | 'ZipFileTest/Directory/Subdirectory/file 3.txt' 380 | ) 381 | self.maxDiff = None 382 | def tearDown(self): 383 | self._tmp_dir.cleanup() 384 | super().tearDown() -------------------------------------------------------------------------------- /core/tests/test_fileoperations.py: -------------------------------------------------------------------------------- 1 | from core.fileoperations import CopyFiles, MoveFiles 2 | from core.tests import StubFS 3 | from fman import YES, NO, OK, YES_TO_ALL, NO_TO_ALL, ABORT, PLATFORM 4 | from fman.url import join, dirname, as_url, as_human_readable 5 | from os.path import exists 6 | from tempfile import TemporaryDirectory 7 | from unittest import TestCase, skipIf 8 | 9 | import os 10 | import os.path 11 | import stat 12 | 13 | class FileTreeOperationAT: 14 | 15 | if PLATFORM == 'Windows': 16 | _NO_SUCH_FILE_MSG = 'the system cannot find the file specified' 17 | else: 18 | _NO_SUCH_FILE_MSG = 'no such file or directory' 19 | 20 | def __init__(self, operation, operation_descr_verb, methodName='runTest'): 21 | super().__init__(methodName=methodName) 22 | self.operation = operation 23 | self.operation_descr_verb = operation_descr_verb 24 | def test_single_file(self, dest_dir=None): 25 | if dest_dir is None: 26 | dest_dir = self.dest 27 | src_file = join(self.src, 'test.txt') 28 | self._touch(src_file, '1234') 29 | self._perform_on(src_file, dest_dir=dest_dir) 30 | self._expect_files({'test.txt'}, dest_dir) 31 | self._assert_file_contents_equal(join(dest_dir, 'test.txt'), '1234') 32 | return src_file 33 | def test_singe_file_dest_dir_does_not_exist(self): 34 | self.test_single_file(dest_dir=join(self.dest, 'subdir')) 35 | def test_empty_directory(self): 36 | empty_dir = join(self.src, 'test') 37 | self._mkdir(empty_dir) 38 | self._perform_on(empty_dir) 39 | self._expect_files({'test'}) 40 | self._expect_files(set(), in_dir=join(self.dest, 'test')) 41 | return empty_dir 42 | def test_directory_several_files(self, dest_dir=None): 43 | if dest_dir is None: 44 | dest_dir = self.dest 45 | file_outside_dir = join(self.src, 'file1.txt') 46 | self._touch(file_outside_dir) 47 | dir_ = join(self.src, 'dir') 48 | self._mkdir(dir_) 49 | file_in_dir = join(dir_, 'file.txt') 50 | self._touch(file_in_dir) 51 | executable_in_dir = join(dir_, 'executable') 52 | self._touch(executable_in_dir, 'abc') 53 | if PLATFORM != 'Windows': 54 | st_mode = self._stat(executable_in_dir).st_mode 55 | self._chmod(executable_in_dir, st_mode | stat.S_IEXEC) 56 | self._perform_on(file_outside_dir, dir_, dest_dir=dest_dir) 57 | self._expect_files({'file1.txt', 'dir'}, dest_dir) 58 | self._expect_files({'executable', 'file.txt'}, join(dest_dir, 'dir')) 59 | executable_dst = join(dest_dir, 'dir', 'executable') 60 | self._assert_file_contents_equal(executable_dst, 'abc') 61 | if PLATFORM != 'Windows': 62 | self.assertTrue(self._stat(executable_dst).st_mode & stat.S_IEXEC) 63 | return [file_outside_dir, dir_] 64 | def test_directory_several_files_dest_dir_does_not_exist(self): 65 | self.test_directory_several_files(dest_dir=join(self.dest, 'subdir')) 66 | def test_overwrite_files( 67 | self, answers=(YES, YES), expect_overrides=(True, True), 68 | files=('a.txt', 'b.txt'), perform_on_files=None 69 | ): 70 | if perform_on_files is None: 71 | perform_on_files = files 72 | src_files = [join(self.src, *relpath.split('/')) for relpath in files] 73 | dest_files = [join(self.dest, *relpath.split('/')) for relpath in files] 74 | file_contents = lambda src_file_path: os.path.basename(src_file_path) 75 | for i, src_file_path in enumerate(src_files): 76 | self._makedirs(dirname(src_file_path), exist_ok=True) 77 | self._touch(src_file_path, file_contents(src_file_path)) 78 | dest_file_path = dest_files[i] 79 | self._makedirs(dirname(dest_file_path), exist_ok=True) 80 | self._touch(dest_file_path) 81 | for i, answer in enumerate(answers): 82 | file_name = os.path.basename(files[i]) 83 | self._expect_alert( 84 | ('%s exists. Do you want to overwrite it?' % file_name, 85 | YES | NO | YES_TO_ALL | NO_TO_ALL | ABORT, YES), 86 | answer=answer 87 | ) 88 | self._perform_on(*[join(self.src, fname) for fname in perform_on_files]) 89 | for i, expect_override in enumerate(expect_overrides): 90 | dest_file = dest_files[i] 91 | with self._open(dest_file, 'r') as f: 92 | contents = f.read() 93 | if expect_override: 94 | self.assertEqual(file_contents(src_files[i]), contents) 95 | else: 96 | self.assertEqual( 97 | '', contents, 98 | 'File %s was overwritten, contrary to expectations.' % 99 | os.path.basename(dest_file) 100 | ) 101 | return src_files 102 | def test_overwrite_files_no_yes(self): 103 | self.test_overwrite_files((NO, YES), (False, True)) 104 | def test_overwrite_files_yes_all(self): 105 | self.test_overwrite_files((YES_TO_ALL,), (True, True)) 106 | def test_overwrite_files_no_all(self): 107 | self.test_overwrite_files((NO_TO_ALL,), (False, False)) 108 | def test_overwrite_files_yes_no_all(self): 109 | self.test_overwrite_files((YES, NO_TO_ALL), (True, False)) 110 | def test_overwrite_files_abort(self): 111 | self.test_overwrite_files((ABORT,), (False, False)) 112 | def test_overwrite_files_in_directory(self): 113 | self.test_overwrite_files( 114 | files=('dir/a.txt', 'b.txt'), perform_on_files=('dir', 'b.txt') 115 | ) 116 | def test_overwrite_directory_abort(self): 117 | self.test_overwrite_files( 118 | (ABORT,), (False, False,), files=('dir/a/a.txt', 'dir/b/b.txt'), 119 | perform_on_files=('dir',) 120 | ) 121 | def test_move_to_self(self): 122 | a, b = join(self.dest, 'a'), join(self.dest, 'b') 123 | c = join(self.external_dir, 'c') 124 | dir_ = join(self.dest, 'dir') 125 | self._makedirs(dir_) 126 | files = [a, b, c] 127 | for file_ in files: 128 | self._touch(file_) 129 | # Expect alert only once: 130 | self._expect_alert( 131 | ('You cannot %s a file to itself.' % self.operation_descr_verb,), 132 | answer=OK 133 | ) 134 | self._perform_on(dir_, *files) 135 | def test_move_dir_to_self(self): 136 | dir_ = join(self.src, 'dir') 137 | self._makedirs(dir_) 138 | self._expect_alert( 139 | ('You cannot %s a file to itself.' % self.operation_descr_verb,), 140 | answer=OK 141 | ) 142 | self._perform_on(dir_, dest_dir=self.src) 143 | def test_move_to_own_subdir(self): 144 | dir_ = join(self.src, 'dir') 145 | subdir = join(dir_, 'subdir') 146 | self._makedirs(subdir) 147 | self._expect_alert( 148 | ('You cannot %s a file to itself.' % self.operation_descr_verb,), 149 | answer=OK 150 | ) 151 | self._perform_on(dir_, dest_dir=subdir) 152 | def test_external_file(self): 153 | external_file = join(self.external_dir, 'test.txt') 154 | self._touch(external_file) 155 | self._perform_on(external_file) 156 | self._expect_files({'test.txt'}) 157 | return external_file 158 | def test_nested_dir(self): 159 | parent_dir = join(self.src, 'parent_dir') 160 | nested_dir = join(parent_dir, 'nested_dir') 161 | text_file = join(nested_dir, 'file.txt') 162 | self._makedirs(nested_dir) 163 | self._touch(text_file) 164 | self._perform_on(parent_dir) 165 | self._expect_files({'parent_dir'}) 166 | self._expect_files({'nested_dir'}, join(self.dest, 'parent_dir')) 167 | self._expect_files( 168 | {'file.txt'}, join(self.dest, 'parent_dir', 'nested_dir') 169 | ) 170 | return parent_dir 171 | def test_symlink(self): 172 | symlink_source = join(self.src, 'symlink_source') 173 | self._touch(symlink_source) 174 | symlink = join(self.src, 'symlink') 175 | self._symlink(symlink_source, symlink) 176 | self._perform_on(symlink) 177 | self._expect_files({'symlink'}) 178 | symlink_dest = join(self.dest, 'symlink') 179 | self.assertTrue(self._islink(symlink_dest)) 180 | symlink_dest_source = self._readlink(symlink_dest) 181 | self.assertTrue(self._fs.samefile(symlink_source, symlink_dest_source)) 182 | return symlink 183 | def test_dest_name(self, src_equals_dest=False, preserves_files=True): 184 | src_dir = self.dest if src_equals_dest else self.src 185 | foo = join(src_dir, 'foo') 186 | self._touch(foo, '1234') 187 | self._perform_on(foo, dest_name='bar') 188 | expected_files = {'bar'} 189 | if preserves_files and src_equals_dest: 190 | expected_files.add('foo') 191 | self._expect_files(expected_files) 192 | self._assert_file_contents_equal(join(self.dest, 'bar'), '1234') 193 | return foo 194 | def test_dest_name_same_dir(self): 195 | self.test_dest_name(src_equals_dest=True) 196 | def test_error(self, answer_1=YES, answer_2=YES): 197 | nonexistent_file_1 = join(self.src, 'foo1.txt') 198 | nonexistent_file_2 = join(self.src, 'foo2.txt') 199 | existent_file = join(self.src, 'bar.txt') 200 | self._touch(existent_file) 201 | self._expect_alert( 202 | ('Could not %s %s (%s). ' 203 | 'Do you want to continue?' % ( 204 | self.operation_descr_verb, 205 | as_human_readable(nonexistent_file_1), self._NO_SUCH_FILE_MSG 206 | ), 207 | YES | YES_TO_ALL | ABORT, YES), 208 | answer=answer_1 209 | ) 210 | if not answer_1 & ABORT and not answer_1 & YES_TO_ALL: 211 | self._expect_alert( 212 | ('Could not %s %s (%s). ' 213 | 'Do you want to continue?' % ( 214 | self.operation_descr_verb, 215 | as_human_readable(nonexistent_file_2), 216 | self._NO_SUCH_FILE_MSG 217 | ), 218 | YES | YES_TO_ALL | ABORT, YES), 219 | answer=answer_2 220 | ) 221 | self._perform_on(nonexistent_file_1, nonexistent_file_2, existent_file) 222 | if not answer_1 & ABORT and not answer_2 & ABORT: 223 | expected_files = {'bar.txt'} 224 | else: 225 | expected_files = set() 226 | self._expect_files(expected_files) 227 | def test_error_yes_to_all(self): 228 | self.test_error(answer_1=YES_TO_ALL) 229 | def test_error_abort(self): 230 | self.test_error(answer_1=ABORT) 231 | def test_error_only_one_file(self): 232 | nonexistent_file = join(self.src, 'foo.txt') 233 | file_path = as_human_readable(nonexistent_file) 234 | message = 'Could not %s %s (%s).' % \ 235 | (self.operation_descr_verb, file_path, self._NO_SUCH_FILE_MSG) 236 | self._expect_alert((message, OK, OK), answer=OK) 237 | self._perform_on(nonexistent_file) 238 | def test_relative_path_parent_dir(self): 239 | src_file = join(self.src, 'test.txt') 240 | self._touch(src_file, '1234') 241 | self._perform_on(src_file, dest_dir='..') 242 | dest_dir_abs = dirname(self.src) 243 | self._expect_files({'src', 'test.txt'}, dest_dir_abs) 244 | self._assert_file_contents_equal(join(dest_dir_abs, 'test.txt'), '1234') 245 | def test_relative_path_subdir(self): 246 | src_file = join(self.src, 'test.txt') 247 | self._touch(src_file, '1234') 248 | subdir = join(self.src, 'subdir') 249 | self._makedirs(subdir, exist_ok=True) 250 | self._perform_on(src_file, dest_dir='subdir') 251 | self._expect_files({'test.txt'}, subdir) 252 | self._assert_file_contents_equal(join(subdir, 'test.txt'), '1234') 253 | def test_drag_and_drop_file(self): 254 | src_file = join(self.src, 'test.txt') 255 | self._touch(src_file, '1234') 256 | self._perform_on(src_file) 257 | self._expect_files({'test.txt'}) 258 | self._assert_file_contents_equal(join(self.dest, 'test.txt'), '1234') 259 | def test_copy_paste_directory(self): 260 | self._touch(join(self.src, 'dir', 'test.txt')) 261 | self._makedirs(join(self.dest, 'dir')) 262 | self._perform_on(join(self.src, 'dir')) 263 | self._expect_files({'test.txt'}, in_dir=join(self.dest, 'dir')) 264 | def test_overwrite_directory_file_in_subdir(self): 265 | self._touch(join(self.src, 'dir1', 'dir2', 'test.txt')) 266 | self._makedirs(join(self.dest, 'dir1', 'dir2')) 267 | self._perform_on(join(self.src, 'dir1')) 268 | self._expect_files({'test.txt'}, in_dir=join(self.dest, 'dir1', 'dir2')) 269 | def setUp(self): 270 | super().setUp() 271 | self._fs = StubFS() 272 | self._progress_dialog = MockProgressDialog(self) 273 | self._tmp_dir = TemporaryDirectory() 274 | self._root = as_url(self._tmp_dir.name) 275 | # We need intermediate 'src-parent' for test_relative_path_parent_dir: 276 | self.src = join(self._root, 'src-parent', 'src') 277 | self._makedirs(self.src) 278 | self.dest = join(self._root, 'dest') 279 | self._makedirs(self.dest) 280 | self.external_dir = join(self._root, 'external-dir') 281 | self._makedirs(self.external_dir) 282 | # Create a dummy file to test that not _all_ files are copied from src: 283 | self._touch(join(self.src, 'dummy')) 284 | def tearDown(self): 285 | self._tmp_dir.cleanup() 286 | super().tearDown() 287 | def _perform_on(self, *files, dest_dir=None, dest_name=None): 288 | if dest_dir is None: 289 | dest_dir = self.dest 290 | op = self.operation(files, dest_dir, dest_name, self._fs) 291 | op._dialog = self._progress_dialog 292 | op() 293 | self._progress_dialog.verify_expected_dialogs_were_shown() 294 | def _assert_file_contents_equal(self, url, expected_contents): 295 | with self._open(url, 'r') as f: 296 | self.assertEqual(expected_contents, f.read()) 297 | def _touch(self, file_url, contents=None): 298 | self._makedirs(dirname(file_url), exist_ok=True) 299 | self._fs.touch(file_url) 300 | if contents is not None: 301 | with self._open(file_url, 'w') as f: 302 | f.write(contents) 303 | def _mkdir(self, dir_url): 304 | self._fs.mkdir(dir_url) 305 | def _makedirs(self, dir_url, exist_ok=False): 306 | self._fs.makedirs(dir_url, exist_ok=exist_ok) 307 | def _open(self, file_url, mode): 308 | return open(as_human_readable(file_url), mode) 309 | def _stat(self, file_url): 310 | return os.stat(as_human_readable(file_url)) 311 | def _chmod(self, file_url, mode): 312 | return os.chmod(as_human_readable(file_url), mode) 313 | def _symlink(self, src_url, dst_url): 314 | os.symlink(as_human_readable(src_url), as_human_readable(dst_url)) 315 | def _islink(self, file_url): 316 | return os.path.islink(as_human_readable(file_url)) 317 | def _readlink(self, link_url): 318 | return as_url(os.readlink(as_human_readable(link_url))) 319 | def _expect_alert(self, args, answer): 320 | self._progress_dialog.expect_alert(args, answer) 321 | def _expect_files(self, files, in_dir=None): 322 | if in_dir is None: 323 | in_dir = self.dest 324 | self.assertEqual(files, set(self._fs.iterdir(in_dir))) 325 | 326 | try: 327 | from os import geteuid 328 | except ImportError: 329 | _is_root = False 330 | else: 331 | _is_root = geteuid() == 0 332 | 333 | class CopyFilesTest(FileTreeOperationAT, TestCase): 334 | def __init__(self, methodName='runTest'): 335 | super().__init__(CopyFiles, 'copy', methodName) 336 | @skipIf(_is_root, 'Skip this test when run by root') 337 | def test_overwrite_locked_file(self): 338 | # Would also like to have this as a test case in MoveFilesTest but the 339 | # call to chmod(0o444) which we use to lock the file doesn't prevent the 340 | # file from being overwritten by a move. Another solution would be to 341 | # chown the file as a different user, but then the test would require 342 | # root privileges. So keep it here only for now. 343 | dir_ = join(self.src, 'dir') 344 | self._fs.makedirs(dir_) 345 | src_file = join(dir_, 'foo.txt') 346 | self._touch(src_file, 'dstn') 347 | dest_dir = join(self.dest, 'dir') 348 | self._fs.makedirs(dest_dir) 349 | locked_dest_file = join(dest_dir, 'foo.txt') 350 | self._touch(locked_dest_file) 351 | self._chmod(locked_dest_file, 0o444) 352 | try: 353 | self._expect_alert( 354 | ('foo.txt exists. Do you want to overwrite it?', 355 | YES | NO | YES_TO_ALL | NO_TO_ALL | ABORT, YES), answer=YES 356 | ) 357 | self._expect_alert( 358 | ('Error copying foo.txt (permission denied).', OK, OK), 359 | answer=OK 360 | ) 361 | self._perform_on(dir_) 362 | finally: 363 | # Make the file writeable again because on Windows, the temp dir 364 | # containing it can't be cleaned up otherwise. 365 | self._chmod(locked_dest_file, 0o777) 366 | 367 | class MoveFilesTest(FileTreeOperationAT, TestCase): 368 | def __init__(self, methodName='runTest'): 369 | super().__init__(MoveFiles, 'move', methodName) 370 | def test_single_file(self, dest_dir=None): 371 | src_file = super().test_single_file(dest_dir) 372 | self.assertFalse(exists(src_file)) 373 | def test_empty_directory(self): 374 | empty_dir_src = super().test_empty_directory() 375 | self.assertFalse(exists(empty_dir_src)) 376 | def test_directory_several_files(self, dest_dir=None): 377 | src_files = super().test_directory_several_files(dest_dir=dest_dir) 378 | for file_ in src_files: 379 | self.assertFalse(exists(file_)) 380 | def test_overwrite_files( 381 | self, answers=(YES, YES), expect_overrides=(True, True), 382 | files=('a.txt', 'b.txt'), perform_on_files=None 383 | ): 384 | src_files = super().test_overwrite_files( 385 | answers, expect_overrides, files, perform_on_files 386 | ) 387 | for i, file_ in enumerate(src_files): 388 | if expect_overrides[i]: 389 | self.assertFalse(exists(file_), file_) 390 | @skipIf(PLATFORM == 'Linux', 'Case-insensitive file systems only') 391 | def test_rename_directory_case(self): 392 | container = join(self.dest, 'container') 393 | directory = join(container, 'a') 394 | self._makedirs(directory) 395 | self._perform_on(directory, dest_dir=container, dest_name='A') 396 | self._expect_files({'A'}, in_dir=container) 397 | def test_external_file(self): 398 | external_file = super().test_external_file() 399 | self.assertFalse(exists(external_file)) 400 | def test_nested_dir(self): 401 | parent_dir = super().test_nested_dir() 402 | self.assertFalse(exists(parent_dir)) 403 | def test_symlink(self): 404 | symlink = super().test_symlink() 405 | self.assertFalse(exists(symlink)) 406 | def test_dest_name(self, src_equals_dest=False): 407 | super().test_dest_name(src_equals_dest, preserves_files=False) 408 | def test_overwrite_dir_skip_file(self): 409 | src_dir = join(self.src, 'dir') 410 | self._makedirs(src_dir) 411 | src_file = join(src_dir, 'test.txt') 412 | self._touch(src_file, 'src contents') 413 | dest_dir = join(self.dest, 'dir') 414 | self._makedirs(dest_dir) 415 | dest_file = join(dest_dir, 'test.txt') 416 | self._touch(dest_file, 'dest contents') 417 | self._expect_alert( 418 | ('test.txt exists. Do you want to overwrite it?', 419 | YES | NO | YES_TO_ALL | NO_TO_ALL | ABORT, YES), 420 | answer=NO 421 | ) 422 | self._perform_on(src_dir) 423 | self.assertTrue( 424 | self._fs.exists(src_file), 425 | "Source file was skipped and should not have been deleted." 426 | ) 427 | self._assert_file_contents_equal(src_file, 'src contents') 428 | self._assert_file_contents_equal(dest_file, 'dest contents') 429 | def test_drag_and_drop_file(self): 430 | super().test_drag_and_drop_file() 431 | self.assertNotIn('test.txt', self._fs.iterdir(self.src)) 432 | def test_overwrite_directory_file_in_subdir(self): 433 | super().test_overwrite_directory_file_in_subdir() 434 | self.assertNotIn('dir1', self._fs.iterdir(self.src)) 435 | 436 | class MockProgressDialog: 437 | def __init__(self, test_case): 438 | self._test_case = test_case 439 | self._progress = 0 440 | self._expected_alerts = [] 441 | def expect_alert(self, args, answer): 442 | self._expected_alerts.append((args, answer)) 443 | def verify_expected_dialogs_were_shown(self): 444 | self._test_case.assertEqual( 445 | [], self._expected_alerts, 'Did not receive all expected alerts.' 446 | ) 447 | def show_alert(self, *args, **_): 448 | if not self._expected_alerts: 449 | self._test_case.fail('Unexpected alert: %r' % args[0]) 450 | return 451 | expected_args, answer = self._expected_alerts.pop(0) 452 | self._test_case.assertEqual(expected_args, args, "Wrong alert") 453 | return answer 454 | def set_text(self, text): 455 | pass 456 | def was_canceled(self): 457 | return False 458 | def set_task_size(self, size): 459 | pass 460 | def get_progress(self): 461 | return self._progress 462 | def set_progress(self, progress): 463 | self._progress = progress -------------------------------------------------------------------------------- /core/fs/zip.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple, deque 2 | from core.os_ import is_arch, is_mac 3 | from core.util import filenotfounderror 4 | from datetime import datetime 5 | from fman import PLATFORM, load_json, Task 6 | from fman.fs import FileSystem 7 | from fman.url import as_url, splitscheme, as_human_readable, basename 8 | from io import UnsupportedOperation, FileIO, BufferedReader, TextIOWrapper 9 | from os.path import join, dirname 10 | from pathlib import PurePosixPath, Path 11 | from subprocess import Popen, PIPE, DEVNULL, CalledProcessError 12 | from tempfile import TemporaryDirectory 13 | 14 | import fman.fs 15 | import os 16 | import os.path 17 | import re 18 | import signal 19 | import sys 20 | 21 | # Prevent 'Rename' below from accidentally overwriting core.Rename: 22 | __all__ = ['ZipFileSystem', 'SevenZipFileSystem', 'TarFileSystem'] 23 | 24 | if is_arch(): 25 | _7ZIP_BINARY = '/usr/bin/7za' 26 | elif is_mac() and getattr(sys, 'frozen', False): 27 | _7ZIP_BINARY = join(dirname(sys.executable), '7za') 28 | else: 29 | _7ZIP_BINARY = join( 30 | dirname(dirname(dirname(__file__))), 'bin', PLATFORM.lower(), '7za' 31 | ) 32 | if PLATFORM == 'Windows': 33 | _7ZIP_BINARY += '.exe' 34 | 35 | class _7ZipFileSystem(FileSystem): 36 | def __init__(self, fs=fman.fs, suffixes=None): 37 | if suffixes is None: 38 | suffixes = self._load_suffixes_from_json() 39 | super().__init__() 40 | self._fs = fs 41 | self._suffixes = suffixes 42 | def _load_suffixes_from_json(self): 43 | settings = load_json('Core Settings.json', default={}) 44 | archive_handlers = settings.get('archive_handlers', {}) 45 | return set( 46 | suffix for suffix, scheme in archive_handlers.items() 47 | if scheme == self.scheme 48 | ) 49 | def get_default_columns(self, path): 50 | return 'core.Name', 'core.Size', 'core.Modified' 51 | def resolve(self, path): 52 | for suffix in self._suffixes: 53 | if suffix in path.lower(): 54 | if not self.exists(path): 55 | raise FileNotFoundError(self.scheme + path) 56 | return super().resolve(path) 57 | return self._fs.resolve(as_url(path)) 58 | def iterdir(self, path): 59 | path_in_zip = self._split(path)[1] 60 | already_yielded = set() 61 | for file_info in self._iter_infos(path): 62 | candidate = file_info.path 63 | while candidate: 64 | candidate_path = PurePosixPath(candidate) 65 | parent = str(candidate_path.parent) 66 | if parent == '.': 67 | parent = '' 68 | if parent == path_in_zip: 69 | name = candidate_path.name 70 | if name not in already_yielded: 71 | yield name 72 | already_yielded.add(name) 73 | candidate = parent 74 | def is_dir(self, existing_path): 75 | zip_path, path_in_zip = self._split(existing_path) 76 | if not path_in_zip: 77 | if Path(zip_path).exists(): 78 | return True 79 | raise filenotfounderror(existing_path) 80 | result = self._query_info_attr(existing_path, 'is_dir', True) 81 | if result is not None: 82 | return result 83 | raise filenotfounderror(existing_path) 84 | def exists(self, path): 85 | try: 86 | zip_path, path_in_zip = self._split(path) 87 | except FileNotFoundError: 88 | return False 89 | if not path_in_zip: 90 | return Path(zip_path).exists() 91 | try: 92 | next(iter(self._iter_infos(path))) 93 | except (StopIteration, FileNotFoundError): 94 | return False 95 | return True 96 | def copy(self, src_url, dst_url): 97 | for task in self.prepare_copy(src_url, dst_url): 98 | task() 99 | def prepare_copy(self, src_url, dst_url): 100 | src_scheme, src_path = splitscheme(src_url) 101 | dst_scheme, dst_path = splitscheme(dst_url) 102 | if src_scheme == self.scheme and dst_scheme == 'file://': 103 | zip_path, path_in_zip = self._split(src_path) 104 | dst_ospath = as_human_readable(dst_url) 105 | return [Extract(self._fs, zip_path, path_in_zip, dst_ospath)] 106 | elif src_scheme == 'file://' and dst_scheme == self.scheme: 107 | zip_path, path_in_zip = self._split(dst_path) 108 | src_ospath = as_human_readable(src_url) 109 | return [ 110 | AddToArchive(self, self._fs, src_ospath, zip_path, path_in_zip) 111 | ] 112 | elif src_scheme == dst_scheme: 113 | # Guaranteed by fman's file system implementation: 114 | assert src_scheme == self.scheme 115 | src_zip_path, path_in_src_zip = self._split(src_path) 116 | dst_zip_path, path_in_dst_zip = self._split(dst_path) 117 | return [CopyBetweenArchives( 118 | self, self._fs, src_zip_path, path_in_src_zip, dst_zip_path, 119 | path_in_dst_zip 120 | )] 121 | else: 122 | raise UnsupportedOperation() 123 | def move(self, src_url, dst_url): 124 | for task in self.prepare_move(src_url, dst_url): 125 | task() 126 | def prepare_move(self, src_url, dst_url): 127 | src_scheme, src_path = splitscheme(src_url) 128 | dst_scheme, dst_path = splitscheme(dst_url) 129 | if src_scheme == dst_scheme: 130 | # Guaranteed by fman's file system implementation: 131 | assert src_scheme == self.scheme 132 | src_zip, src_pth_in_zip = self._split(src_path) 133 | dst_zip, dst_pth_in_zip = self._split(dst_path) 134 | if src_zip == dst_zip: 135 | return [Rename(self, src_zip, src_pth_in_zip, dst_pth_in_zip)] 136 | else: 137 | return [MoveBetweenArchives(self, src_url, dst_url)] 138 | else: 139 | result = list(self.prepare_copy(src_url, dst_url)) 140 | title = 'Cleaning up ' + basename(src_url) 141 | result.append(Task(title, fn=self._fs.delete, args=(src_url,))) 142 | return result 143 | def mkdir(self, path): 144 | if self.exists(path): 145 | raise FileExistsError(path) 146 | zip_path, path_in_zip = self._split(path) 147 | if not path_in_zip: 148 | self._create_empty_archive(zip_path) 149 | elif not self.exists(str(PurePosixPath(path).parent)): 150 | raise filenotfounderror(path) 151 | else: 152 | with TemporaryDirectory() as tmp_dir: 153 | self.copy(as_url(tmp_dir), as_url(path, self.scheme)) 154 | def _create_empty_archive(self, zip_path): 155 | # Run 7-Zip in an empty temporary directory. Create this directory next 156 | # to the Zip file to ensure Path.rename(...) works because it's on the 157 | # same file system. 158 | with _create_temp_dir_next_to(zip_path) as tmp_dir: 159 | name = PurePosixPath(zip_path).name 160 | _run_7zip(['a', name], cwd=tmp_dir) 161 | Path(tmp_dir, name).rename(zip_path) 162 | self.notify_file_added(zip_path) 163 | def delete(self, path): 164 | if not self.exists(path): 165 | raise filenotfounderror(path) 166 | zip_path, path_in_zip = self._split(path) 167 | with self._preserve_empty_parent(zip_path, path_in_zip): 168 | _run_7zip(['d', zip_path, path_in_zip]) 169 | self.notify_file_removed(path) 170 | def prepare_delete(self, path): 171 | return [Task( 172 | 'Deleting ' + path.rsplit('/', 1)[-1], 173 | fn=self.delete, args=(path,), size=1 174 | )] 175 | def size_bytes(self, path): 176 | return self._query_info_attr(path, 'size_bytes', None) 177 | def modified_datetime(self, path): 178 | return self._query_info_attr(path, 'mtime', None) 179 | def _query_info_attr(self, path, attr, folder_default): 180 | def compute_value(): 181 | path_in_zip = self._split(path)[1] 182 | if not path_in_zip: 183 | return folder_default 184 | for info in self._iter_infos(path): 185 | if info.path == path_in_zip: 186 | return getattr(info, attr) 187 | return folder_default 188 | return self.cache.query(path, attr, compute_value) 189 | def _preserve_empty_parent(self, zip_path, path_in_zip): 190 | # 7-Zip deletes empty directories that remain after an operation. For 191 | # instance, when deleting the last file from a directory, or when moving 192 | # it out of the directory. We don't want this to happen. The present 193 | # method allows us to preserve the parent directory, even if empty: 194 | parent = str(PurePosixPath(path_in_zip).parent) 195 | parent_fullpath = zip_path + '/' + parent 196 | class CM: 197 | def __enter__(cm): 198 | if parent != '.': 199 | cm._parent_wasdir_before = self.is_dir(parent_fullpath) 200 | else: 201 | cm._parent_wasdir_before = False 202 | def __exit__(cm, exc_type, exc_val, exc_tb): 203 | if not exc_val: 204 | if cm._parent_wasdir_before: 205 | if not self.exists(parent_fullpath): 206 | self.makedirs(parent_fullpath) 207 | return CM() 208 | def _split(self, path): 209 | for suffix in self._suffixes: 210 | try: 211 | split_point = path.lower().index(suffix) + len(suffix) 212 | except ValueError as suffix_not_found: 213 | continue 214 | else: 215 | return path[:split_point], path[split_point:].lstrip('/') 216 | raise filenotfounderror(self.scheme + path) from None 217 | def _iter_infos(self, path): 218 | zip_path, path_in_zip = self._split(path) 219 | self._raise_filenotfounderror_if_not_exists(zip_path) 220 | args = ['l', '-ba', '-slt', zip_path] 221 | if path_in_zip: 222 | args.append(path_in_zip) 223 | # We can hugely improve performance by making 7-Zip exclude children of 224 | # the given directory. Unfortunately, this has a drawback: If you have 225 | # a/b.txt in an archive but no separate entry for a/, then excluding */* 226 | # filters out a/. We thus exclude */*/*/*. This works for all folders 227 | # that contain at least one subdirectory with a file. 228 | exclude = (path_in_zip + '/' if path_in_zip else '') + '*/*/*/*' 229 | args.append('-x!' + exclude) 230 | with _7zip(args, kill=True) as process: 231 | stdout_lines = process.stdout_lines 232 | file_info = self._read_file_info(stdout_lines) 233 | if path_in_zip and not file_info: 234 | raise filenotfounderror(self.scheme + path) 235 | while file_info: 236 | self._put_in_cache(zip_path, file_info) 237 | yield file_info 238 | file_info = self._read_file_info(stdout_lines) 239 | def _raise_filenotfounderror_if_not_exists(self, zip_path): 240 | os.stat(zip_path) 241 | def _read_file_info(self, stdout): 242 | path = size = mtime = None 243 | is_dir = False 244 | for line in stdout: 245 | line = line.rstrip('\r\n') 246 | if not line: 247 | break 248 | if line.startswith('Path = '): 249 | path = line[len('Path = '):].replace(os.sep, '/') 250 | elif line.startswith('Folder = '): 251 | folder = line[len('Folder = '):] 252 | is_dir = is_dir or folder == '+' 253 | elif line.startswith('Size = '): 254 | size_str = line[len('Size = '):] 255 | if size_str: 256 | size = int(size_str) 257 | elif line.startswith('Modified = '): 258 | mtime_str = line[len('Modified = '):] 259 | if mtime_str: 260 | mtime = datetime.strptime(mtime_str, '%Y-%m-%d %H:%M:%S') 261 | elif line.startswith('Attributes = '): 262 | attributes = line[len('Attributes = '):] 263 | is_dir = is_dir or attributes.startswith('D') 264 | if path: 265 | return _FileInfo(path, is_dir, size, mtime) 266 | def _put_in_cache(self, zip_path, file_info): 267 | for field in file_info._fields: 268 | if field != 'path': 269 | self.cache.put( 270 | zip_path + '/' + file_info.path, field, 271 | getattr(file_info, field) 272 | ) 273 | 274 | class _7zipTaskWithProgress(Task): 275 | def run_7zip_with_progress(self, args, **kwargs): 276 | with _7zip(args, pty=True, **kwargs) as process: 277 | for line in process.stdout_lines: 278 | try: 279 | self.check_canceled() 280 | except Task.Canceled: 281 | process.kill() 282 | raise 283 | # The \r appears on Windows only: 284 | match = re.match('\r? *(\\d\\d?)% ', line) 285 | if match: 286 | percent = int(match.group(1)) 287 | # At least on Linux, 7za shows progress going from 0 to 288 | # 100% twice. The second pass is much faster - maybe 289 | # some kind of verification? Only show the first round: 290 | if percent > self.get_progress(): 291 | self.set_progress(percent) 292 | 293 | class AddToArchive(_7zipTaskWithProgress): 294 | def __init__(self, zip_fs, fman_fs, src_ospath, zip_path, path_in_zip): 295 | if not path_in_zip: 296 | raise ValueError( 297 | 'Must specify the destination path inside the archive' 298 | ) 299 | super().__init__('Packing ' + os.path.basename(src_ospath), size=100) 300 | self._zip_fs = zip_fs 301 | self._fman_fs = fman_fs 302 | self._src_ospath = src_ospath 303 | self._zip_path = zip_path 304 | self._path_in_zip = path_in_zip 305 | def __call__(self): 306 | with TemporaryDirectory() as tmp_dir: 307 | dest = Path(tmp_dir, *self._path_in_zip.split('/')) 308 | dest.parent.mkdir(parents=True, exist_ok=True) 309 | src = Path(self._src_ospath) 310 | try: 311 | dest.symlink_to(src, src.is_dir()) 312 | except OSError: 313 | # This for instance happens on non-NTFS drives on Windows. 314 | # We need to incur the cost of physically copying the file: 315 | self._fman_fs.copy(as_url(src), as_url(dest)) 316 | args = ['a', self._zip_path, self._path_in_zip] 317 | if PLATFORM != 'Windows': 318 | args.insert(1, '-l') 319 | self.run_7zip_with_progress(args, cwd=tmp_dir) 320 | dest_path = self._zip_path + '/' + self._path_in_zip 321 | self._zip_fs.notify_file_added(dest_path) 322 | 323 | class Extract(Task): 324 | def __init__(self, fman_fs, zip_path, path_in_zip, dst_ospath): 325 | super().__init__('Extracting ' + _basename(zip_path, path_in_zip)) 326 | self._fman_fs = fman_fs 327 | self._zip_path = zip_path 328 | self._path_in_zip = path_in_zip 329 | self._dst_ospath = dst_ospath 330 | def __call__(self): 331 | # Create temp dir next to dst_path to ensure Path.replace(...) works 332 | # because it's on the same file system. 333 | tmp_dir = _create_temp_dir_next_to(self._dst_ospath) 334 | try: 335 | args = ['x', self._zip_path, '-o' + tmp_dir.name] 336 | if self._path_in_zip: 337 | args.insert(2, self._path_in_zip) 338 | _run_7zip(args) 339 | # Use fman.fs.move(...) so fman's file:// caches are notified of the 340 | # new file: 341 | self._fman_fs.move( 342 | join(as_url(tmp_dir.name), self._path_in_zip), 343 | as_url(self._dst_ospath) 344 | ) 345 | finally: 346 | try: 347 | tmp_dir.cleanup() 348 | except FileNotFoundError: 349 | # This happens when path_in_zip = '' 350 | pass 351 | 352 | class CopyBetweenArchives(Task): 353 | def __init__( 354 | self, zip_fs, fman_fs, src_zip_path, path_in_src_zip, dst_zip_path, 355 | path_in_dst_zip 356 | ): 357 | title = 'Copying ' + _basename(src_zip_path, path_in_src_zip) 358 | super().__init__(title, size=200) 359 | self._zip_fs = zip_fs 360 | self._fman_fs = fman_fs 361 | self._src_zip_path = src_zip_path 362 | self._path_in_src_zip = path_in_src_zip 363 | self._dst_zip_path = dst_zip_path 364 | self._path_in_dst_zip = path_in_dst_zip 365 | def __call__(self): 366 | with TemporaryDirectory() as tmp_dir: 367 | src_basename = self._path_in_src_zip.rsplit('/', 1)[-1] 368 | # Give temp dir the same name as the source file; This leads to the 369 | # correct name being displayed in the progress dialog: 370 | tmp_dst_ospath = os.path.join(tmp_dir, src_basename) 371 | self.run(Extract( 372 | self._fman_fs, self._src_zip_path, self._path_in_src_zip, 373 | tmp_dst_ospath 374 | )) 375 | self.run(AddToArchive( 376 | self._zip_fs, self._fman_fs, tmp_dst_ospath, self._dst_zip_path, 377 | self._path_in_dst_zip 378 | )) 379 | 380 | class Rename(_7zipTaskWithProgress): 381 | def __init__(self, zip_fs, zip_path, src_in_zip, dst_in_zip): 382 | super().__init__('Renaming ' + src_in_zip.rsplit('/', 1)[-1], size=100) 383 | self._fs = zip_fs 384 | self._zip_path = zip_path 385 | self._src_in_zip = src_in_zip 386 | self._dst_in_zip = dst_in_zip 387 | def __call__(self, *args, **kwargs): 388 | with self._fs._preserve_empty_parent(self._zip_path, self._src_in_zip): 389 | self.run_7zip_with_progress( 390 | ['rn', self._zip_path, self._src_in_zip, self._dst_in_zip] 391 | ) 392 | self._fs.notify_file_removed(self._zip_path + '/' + self._src_in_zip) 393 | self._fs.notify_file_added(self._zip_path + '/' + self._dst_in_zip) 394 | 395 | class MoveBetweenArchives(Task): 396 | def __init__(self, fs, src_url, dst_url): 397 | super().__init__('Moving ' + basename(src_url), size=200) 398 | self._fs = fs 399 | self._src_url = src_url 400 | self._dst_url = dst_url 401 | def __call__(self): 402 | self.set_text('Preparing...') 403 | with TemporaryDirectory() as tmp_dir: 404 | # Give temp dir the same name as the source file; This leads to the 405 | # correct name being displayed in the progress dialog: 406 | tmp_url = as_url(os.path.join(tmp_dir, basename(self._src_url))) 407 | tasks = list(self._fs.prepare_move(self._src_url, tmp_url)) 408 | tasks.extend(self._fs.prepare_move(tmp_url, self._dst_url)) 409 | for task in tasks: 410 | self.run(task) 411 | 412 | def _basename(zip_path, path_in_zip): 413 | sep = ('/' if path_in_zip else '') 414 | return (zip_path + sep + path_in_zip).rsplit('/', 1)[-1] 415 | 416 | def _run_7zip(args, cwd=None, pty=False): 417 | with _7zip(args, cwd=cwd, pty=pty): 418 | pass 419 | 420 | def _create_temp_dir_next_to(path): 421 | return TemporaryDirectory( 422 | dir=str(Path(path).parent), prefix='', suffix='.tmp' 423 | ) 424 | 425 | class _7zip: 426 | 427 | _7ZIP_WARNING = 1 428 | 429 | def __init__(self, args, cwd=None, pty=False, kill=False): 430 | self._args = args 431 | self._cwd = cwd 432 | self._pty = pty 433 | self._kill = kill 434 | self._killed = False 435 | self._process = None 436 | self._stdout_lines = deque(maxlen=100) 437 | def __enter__(self): 438 | if PLATFORM == 'Windows': 439 | cls = Run7ZipViaWinpty if self._pty else Popen7ZipWindows 440 | else: 441 | cls = Run7ZipViaPty if self._pty else Popen7ZipUnix 442 | self._process = cls(self._args, self._cwd) 443 | return self 444 | @property 445 | def stdout_lines(self): 446 | for line in self._process.stdout: 447 | self._stdout_lines.append(line) 448 | yield line 449 | def kill(self): 450 | self._killed = True 451 | self._process.kill() 452 | def wait(self): 453 | return self._process.wait() 454 | def __exit__(self, exc_type, exc_val, exc_tb): 455 | try: 456 | if self._kill: 457 | self._process.kill() 458 | self._process.wait() 459 | else: 460 | exit_code = self._process.wait() 461 | if exit_code and not self._killed and \ 462 | exit_code != self._7ZIP_WARNING: 463 | raise _7zipError( 464 | exit_code, self._args, ''.join(self._stdout_lines) 465 | ) 466 | finally: 467 | self._process.stdout.close() 468 | 469 | class _7zipError(CalledProcessError): 470 | def __str__(self): 471 | result = '7-Zip with args %r returned non-zero exit status %d' % \ 472 | (self.cmd, self.returncode) 473 | if self.output: 474 | result += '. Output: %r' % re.sub('(\r?\n)+', ' ', self.output) 475 | return result 476 | 477 | class Popen7Zip: 478 | def __init__(self, args, cwd, env, encoding=None, **kwargs): 479 | # We need to supply stdin and stderr != None because otherwise on 480 | # Windows, when fman is run as a GUI app, we get: 481 | # OSError: [WinError 6] The handle is invalid 482 | # This is likely caused by https://bugs.python.org/issue3905. 483 | self._process = Popen( 484 | [_7ZIP_BINARY] + args, stdout=PIPE, stderr=DEVNULL, stdin=DEVNULL, 485 | cwd=cwd, env=env, **kwargs 486 | ) 487 | self.stdout = SourceClosingTextIOWrapper(self._process.stdout, encoding) 488 | def kill(self): 489 | self._process.kill() 490 | def wait(self): 491 | return self._process.wait() 492 | 493 | class Popen7ZipWindows(Popen7Zip): 494 | def __init__(self, args, cwd): 495 | args, env = _get_7zip_args_env_windows(args) 496 | super().__init__(args, cwd, env, startupinfo=self._get_startupinfo()) 497 | def _get_startupinfo(self): 498 | from subprocess import STARTF_USESHOWWINDOW, SW_HIDE, STARTUPINFO 499 | result = STARTUPINFO() 500 | result.dwFlags = STARTF_USESHOWWINDOW 501 | result.wShowWindow = SW_HIDE 502 | return result 503 | 504 | def _get_7zip_args_env_windows(args): 505 | # Force an output encoding that works with TextIOWrapper(...): 506 | args = ['-sccWIN'] + args 507 | # Prevent potential interferences with existing env. variables: 508 | env = {} 509 | return args, env 510 | 511 | class Popen7ZipUnix(Popen7Zip): 512 | def __init__(self, args, cwd): 513 | env, encoding = _get_7zip_env_encoding_unix() 514 | super().__init__(args, cwd, env, encoding=encoding) 515 | 516 | def _get_7zip_env_encoding_unix(): 517 | # According to the README in its source code distribution, p7zip can 518 | # only handle unicode file names properly if the environment is UTF-8: 519 | env = {'LANG': 'en_US.UTF-8'} 520 | # Force encoding because TextIOWrapper uses ASCII if 521 | # locale.getpreferredencoding(False) happens to be None: 522 | encoding = 'utf-8' 523 | return env, encoding 524 | 525 | class Run7ZipViaPty: 526 | """ 527 | When run from a terminal, 7-Zip displays progress information for some 528 | operations. This works as follows: It prints a line, say 529 | 41% + Picture.jpg 530 | Then, it outputs ASCII control characters that *delete* the line again. In 531 | the above example on Linux, this would be 18 * '\b', the Backspace 532 | character. Next, it "overwrites" the existing characters with 18 * ' '. 533 | Finally, it outputs the new line: 534 | 59% + Picture.jpg 535 | 536 | Unlike Popen, this class lets us read the progress information as it is 537 | written by 7-Zip. This is achieved by 1) faking a pseudo-terminal (hence the 538 | name "Pty") and 2) by faithfully interpreting '\b' in the subprocess's 539 | output. 540 | """ 541 | 542 | class Stdout: 543 | def __init__(self, fd, encoding): 544 | self._fd = fd 545 | self._encoding = encoding 546 | self._source = BufferedReader(FileIO(self._fd)) 547 | def __iter__(self): 548 | buffer = b'' 549 | prev_len_delta = 0 550 | curr_line = lambda: buffer.decode(self._encoding) 551 | while True: 552 | try: 553 | b = self._source.read(1) 554 | except OSError: 555 | yield curr_line() 556 | break 557 | if b == b'': 558 | yield curr_line() 559 | break 560 | elif b == b'\b': 561 | if prev_len_delta == 1: 562 | l = curr_line() 563 | if l.strip(): 564 | yield l 565 | buffer = buffer[:-1] 566 | prev_len_delta = -1 567 | else: 568 | buffer += b 569 | if b == b'\n': 570 | yield curr_line() 571 | buffer = b'' 572 | prev_len_delta = 0 573 | else: 574 | prev_len_delta = 1 575 | def close(self): 576 | self._source.close() 577 | 578 | def __init__(self, args, cwd): 579 | env, encoding = _get_7zip_env_encoding_unix() 580 | self._pid, fd = self._spawn([_7ZIP_BINARY] + args, cwd, env) 581 | self.stdout = self.Stdout(fd, encoding=encoding) 582 | def kill(self): 583 | os.kill(self._pid, signal.SIGTERM) 584 | def wait(self): 585 | return os.waitpid(self._pid, 0)[1] 586 | def _spawn(self, argv, cwd=None, env=None): 587 | # Copied and adapted from pty.spawn(...). 588 | import pty # <- import late because pty is not available on Windows. 589 | pid, master_fd = pty.fork() 590 | if pid == pty.CHILD: 591 | # In some magical way, this code is executed in the forked child 592 | # process. 593 | if cwd is not None: 594 | os.chdir(cwd) 595 | if env is not None: 596 | os.environ = env 597 | os.execlp(argv[0], *argv) 598 | return pid, master_fd 599 | 600 | class Run7ZipViaWinpty: 601 | 602 | class Stdout: 603 | def __init__(self, process): 604 | self._process = process 605 | self._escape_ansi = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]') 606 | def __iter__(self): 607 | while True: 608 | try: 609 | line = self._process.read() 610 | except EOFError: 611 | break 612 | line = self._escape_ansi.sub('', line) 613 | if line: 614 | yield line 615 | def close(self): 616 | self._process.close() 617 | 618 | def __init__(self, args, cwd): 619 | args, env = _get_7zip_args_env_windows(args) 620 | self._process = self._spawn([_7ZIP_BINARY] + args, cwd, env) 621 | self.stdout = self.Stdout(self._process) 622 | def kill(self): 623 | self._process.sendcontrol('c') 624 | def wait(self): 625 | return self._process.wait() 626 | def _spawn(self, argv, cwd=None, env=None): 627 | from winpty import PtyProcess 628 | return PtyProcess.spawn(argv, cwd, env) 629 | 630 | class ZipFileSystem(_7ZipFileSystem): 631 | scheme = 'zip://' 632 | 633 | class SevenZipFileSystem(_7ZipFileSystem): 634 | scheme = '7z://' 635 | 636 | class TarFileSystem(_7ZipFileSystem): 637 | scheme = 'tar://' 638 | 639 | _FileInfo = namedtuple('_FileInfo', ('path', 'is_dir', 'size_bytes', 'mtime')) 640 | 641 | class SourceClosingTextIOWrapper(TextIOWrapper): 642 | def close(self): 643 | super().close() 644 | self.buffer.close() --------------------------------------------------------------------------------