├── setup.cfg ├── AUTHORS.rst ├── src └── loggedfs │ ├── _core │ ├── __init__.py │ ├── defaults.py │ ├── timing.py │ ├── log.py │ ├── cli.py │ ├── ipc.py │ ├── notify.py │ ├── out.py │ ├── filter.py │ └── fs.py │ └── __init__.py ├── tests ├── __init__.py ├── test_fsx │ └── makefile ├── scripts │ ├── fsx │ └── fsx_analyze.py ├── test_fstest.py ├── lib │ ├── post.py │ ├── __init__.py │ ├── pre.py │ ├── scope.py │ ├── climount.py │ ├── mount.py │ ├── const.py │ ├── install.py │ ├── param.py │ ├── prove.py │ ├── procio.py │ └── base.py ├── test_loggedfs_cfg.xml └── conftest.py ├── docs ├── examples.rst ├── library_example.rst └── library_demo.ipynb ├── .gitignore ├── .travis.yml ├── CHANGES.rst ├── CONTRIBUTING.rst ├── makefile ├── setup.py ├── README.rst └── LICENSE /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | license_file = LICENSE 4 | 5 | [tool:pytest] 6 | testpaths = tests 7 | norecursedirs = test_* *_libtest 8 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | LoggedFS-python contributors 2 | ============================ 3 | 4 | In alphabetical order: 5 | 6 | - Sebastian M. Ernst 7 | 8 | LoggedFS-python is based on the works of 9 | 10 | - Rémi Flament 11 | - Stavros Korokithakis 12 | -------------------------------------------------------------------------------- /src/loggedfs/_core/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/__init__.py: Module core init 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/__init__.py: Test module init 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | from . import lib 33 | -------------------------------------------------------------------------------- /tests/test_fsx/makefile: -------------------------------------------------------------------------------- 1 | # LoggedFS-python 2 | # Filesystem monitoring with Fuse and Python 3 | # https://github.com/pleiszenburg/loggedfs-python 4 | # 5 | # tests/test_fsx/makefile: GNU makefile for fsx-linux 6 | # 7 | # Copyright (C) 2017-2020 Sebastian M. Ernst 8 | # 9 | # 10 | # The contents of this file are subject to the Apache License 11 | # Version 2 ("License"). You may not use this file except in 12 | # compliance with the License. You may obtain a copy of the License at 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | # https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 15 | # 16 | # Software distributed under the License is distributed on an "AS IS" basis, 17 | # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing rights and limitations under the License. 19 | # 20 | 21 | 22 | CC = gcc 23 | CFLAGS = -g -O2 -fno-strict-aliasing -pipe -Wall -W -Wold-style-definition -D_FORTIFY_SOURCE=2 -DNO_XFS -D_LARGEFILE64_SOURCE -D_GNU_SOURCE 24 | LDFLAGS = 25 | 26 | FSX = fsx-linux 27 | 28 | all: $(FSX) 29 | 30 | $(FSX): $(FSX).c 31 | $(CC) $(FSX).c $(CFLAGS) -o $(FSX) $(LDFLAGS) 32 | 33 | install: 34 | cp -av $(FSX) $(VIRTUAL_ENV)/bin/ 35 | 36 | clean: 37 | rm -rf $(FSX) $(FSX).o 38 | -------------------------------------------------------------------------------- /tests/scripts/fsx: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # LoggedFS-python 4 | # Filesystem monitoring with Fuse and Python 5 | # https://github.com/pleiszenburg/loggedfs-python 6 | # 7 | # tests/scripts/fsx: Development script for running fsx-linux 8 | # 9 | # Copyright (C) 2017-2020 Sebastian M. Ernst 10 | # 11 | # 12 | # The contents of this file are subject to the Apache License 13 | # Version 2 ("License"). You may not use this file except in 14 | # compliance with the License. You may obtain a copy of the License at 15 | # https://www.apache.org/licenses/LICENSE-2.0 16 | # https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 17 | # 18 | # Software distributed under the License is distributed on an "AS IS" basis, 19 | # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 20 | # specific language governing rights and limitations under the License. 21 | # 22 | 23 | 24 | FN=iotest 25 | 26 | make init || exit 1 27 | cd tests/test_mount/test_child/ 28 | # change "-d" to "-d -d" for more verbosity 29 | fsx-linux -d -N 1000 $FN -P ../../test_logs || exit 1 30 | cd ../../.. 31 | make destroy_childfs || exit 1 32 | cp -a tests/test_mount/test_child/$FN tests/test_logs/$FN.fsxactual 33 | make destroy_parentfs || exit 1 34 | if grep -q "Traceback (most recent call last)" "tests/test_logs/loggedfs.log"; then 35 | exit 1 36 | fi 37 | -------------------------------------------------------------------------------- /src/loggedfs/_core/defaults.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/defaults.py: Default configurations 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # CONST 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | FUSE_ALLOWOTHER_DEFAULT = False 33 | FUSE_FOREGROUND_DEFAULT = False 34 | 35 | LIB_MODE_DEFAULT = False 36 | 37 | LOG_BUFFERS_DEFAULT = False 38 | LOG_ENABLED_DEFAULT = True 39 | LOG_JSON_DEFAULT = False 40 | LOG_ONLYMODIFYOPERATIONS_DEFAULT = False 41 | LOG_PRINTPROCESSNAME_DEFAULT = True 42 | LOG_SYSLOG_DEFAULT = False 43 | -------------------------------------------------------------------------------- /src/loggedfs/_core/timing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/timing.py: Time related 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import time 33 | 34 | 35 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 36 | # FIX time_ns 37 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 38 | 39 | if not hasattr(time, 'time_ns'): 40 | time.time_ns = lambda: int(time.time() * 1e9) 41 | -------------------------------------------------------------------------------- /src/loggedfs/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/__init__.py: Module init 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | from ._core.cli import cli_entry 33 | from ._core.filter import ( 34 | filter_field_class, 35 | filter_item_class, 36 | filter_pipeline_class 37 | ) 38 | from ._core.fs import ( 39 | _loggedfs, 40 | loggedfs_factory 41 | ) 42 | from ._core.ipc import end_of_transmission 43 | from ._core.notify import notify_class as loggedfs_notify 44 | from ._core.out import decode_buffer 45 | -------------------------------------------------------------------------------- /tests/test_fstest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/test_fstest.py: Runs the fstest-suite 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | from pprint import pprint as pp 33 | 34 | from .lib import fstest_scope 35 | 36 | 37 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 38 | # TESTS 39 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 40 | 41 | def test_fstest(fstest_scope, fstest_group_path): 42 | 43 | fstest_scope.prove(fstest_group_path) 44 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | Launching LoggedFS-python 2 | ========================= 3 | 4 | If you just want to test LoggedFS-python you don't need any configuration file. 5 | 6 | Just use that command: 7 | 8 | .. code:: bash 9 | 10 | sudo loggedfs -s -f -p /var 11 | 12 | You should see logs like these: 13 | 14 | :: 15 | 16 | tail -f /var/log/syslog 17 | 2017-12-09 17:29:34,910 (loggedfs-python) LoggedFS-python running as a public filesystem 18 | 2017-12-09 17:29:34,915 (loggedfs-python) LoggedFS-python not running as a daemon 19 | 2017-12-09 17:29:34,920 (loggedfs-python) LoggedFS-python starting at /var 20 | 2017-12-09 17:29:34,950 (loggedfs-python) chdir to /var 21 | 2017-12-09 17:29:35,246 (loggedfs-python) getattr /var/ {SUCCESS} [ pid = 8700 kded [kdeinit] uid = 1000 ] 22 | 2017-12-09 17:29:41,841 (loggedfs-python) getattr /var/ {SUCCESS} [ pid = 10923 ls uid = 1000 ] 23 | 2017-12-09 17:29:41,858 (loggedfs-python) getattr /var/run {SUCCESS} [ pid = 10923 ls uid = 1000 ] 24 | 2017-12-09 17:29:41,890 (loggedfs-python) getattr /var/run/nscd {FAILURE} [ pid = 10923 ls uid = 1000 ] 25 | 2017-12-09 17:29:41,912 (loggedfs-python) readdir /var/ {SUCCESS} [ pid = 10923 ls uid = 1000 ] 26 | 2017-12-09 17:29:41,987 (loggedfs-python) getattr /var/pouak {SUCCESS} [ pid = 10923 ls uid = 1000 ] 27 | 28 | If you have a configuration file to use you should use this command: 29 | 30 | .. code:: bash 31 | 32 | loggedfs -c loggedfs.xml -p /var 33 | 34 | If you want to log what other users do on your filesystem, you should use the 35 | ``-p`` option to allow them to see your mounted files. For a complete 36 | documentation see the manual page. 37 | -------------------------------------------------------------------------------- /tests/lib/post.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/post.py: Stuff happening after test(s) 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | 34 | 35 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 36 | # CLASS: (3/3) POST 37 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 38 | 39 | class fstest_post_class: 40 | 41 | 42 | def destroy(self): 43 | """Called from project root after tests! 44 | """ 45 | 46 | os.chdir(self.prj_abs_path) 47 | 48 | self.destroy_a_childfs() 49 | self.destroy_b_parentfs() 50 | -------------------------------------------------------------------------------- /tests/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/__init__.py: Test library module init 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | from .climount import ( 33 | quick_cli_cleanup, 34 | quick_cli_init, 35 | quick_cli_init_parentfs, 36 | quick_cli_init_childfs, 37 | quick_cli_destroy, 38 | quick_cli_destroy_parentfs, 39 | quick_cli_destroy_childfs, 40 | ) 41 | from .const import ( 42 | TEST_ROOT_PATH, 43 | TEST_FSTEST_PATH, 44 | TEST_FSTEST_TESTS_SUBPATH, 45 | ) 46 | from .install import ( 47 | install_fstest, 48 | install_fsx, 49 | ) 50 | from .procio import run_command 51 | from .param import fstest_parameters 52 | from .scope import fstest_scope 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | # lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | coverage_html 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *,cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # IPython Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # dotenv 82 | .env 83 | 84 | # virtualenv 85 | venv/ 86 | ENV/ 87 | env*/ 88 | 89 | # Spyder project settings 90 | .spyderproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # Project specific stuff 96 | test_suite 97 | test_mount 98 | test_logs 99 | fsx-linux* 100 | screenshots/ 101 | notebooks/ 102 | notes.md 103 | -------------------------------------------------------------------------------- /tests/lib/pre.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/pre.py: Stuff happening before test(s) 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | 34 | from .const import TEST_FS_LOGGEDFS 35 | 36 | 37 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 38 | # CLASS: (1/3) PRE 39 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 40 | 41 | class fstest_pre_class: 42 | 43 | 44 | def init(self, fs_type = TEST_FS_LOGGEDFS): 45 | 46 | self.init_a_members(fs_type) 47 | self.init_b_cleanup() 48 | self.init_c_parentfs() 49 | self.init_d_childfs() 50 | 51 | os.chdir(self.mount_child_abs_path) 52 | -------------------------------------------------------------------------------- /tests/test_loggedfs_cfg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # LoggedFS-python 2 | # Filesystem monitoring with Fuse and Python 3 | # https://github.com/pleiszenburg/loggedfs-python 4 | # 5 | # .travis.yml: Configuration for Travis CI build test 6 | # 7 | # Copyright (C) 2017-2018 Sebastian M. Ernst 8 | # 9 | # 10 | # The contents of this file are subject to the Apache License 11 | # Version 2 ("License"). You may not use this file except in 12 | # compliance with the License. You may obtain a copy of the License at 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | # https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 15 | # 16 | # Software distributed under the License is distributed on an "AS IS" basis, 17 | # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing rights and limitations under the License. 19 | # 20 | 21 | 22 | # Check this file at: 23 | # http://lint.travis-ci.org/ 24 | 25 | # A virtual machine IS required for loading FUSE kernel module: 26 | # https://github.com/travis-ci/travis-ci/issues/1100#issuecomment-160169121 27 | sudo: required 28 | 29 | # Repository language 30 | language: python 31 | 32 | # Ubuntu & Python versions 33 | matrix: 34 | include: 35 | - dist: xenial 36 | python: "3.5" 37 | - dist: xenial 38 | python: "3.6" 39 | - dist: xenial 40 | python: "3.7" 41 | - dist: xenial 42 | python: "3.8" 43 | 44 | # Install dependencies 45 | addons: 46 | apt: 47 | packages: 48 | - fuse 49 | - libfuse2 50 | - libfuse-dev 51 | 52 | # Get debug output 53 | before_install: 54 | - sudo modprobe fuse 55 | - sudo chmod 666 /dev/fuse 56 | - sudo chown root:$USER /etc/fuse.conf 57 | - sudo echo "user_allow_other" | sudo tee -a /etc/fuse.conf > /dev/null 58 | - python --version 59 | - pip install -U pip 60 | - pip install -U setuptools 61 | - uname -a 62 | 63 | # command to install dependencies and module 64 | install: make install 65 | 66 | # command to run tests 67 | script: make test 68 | 69 | # Notify developers 70 | notifications: 71 | email: 72 | recipients: 73 | - ernst@pleiszenburg.de 74 | on_success: always 75 | on_failure: always 76 | -------------------------------------------------------------------------------- /tests/lib/scope.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/scope.py: Provides test scope 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import pytest 33 | 34 | from .const import TEST_FS_LOGGEDFS 35 | from .base import fstest_base_class 36 | from .pre import fstest_pre_class 37 | from .prove import fstest_prove_class 38 | from .post import fstest_post_class 39 | 40 | 41 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 42 | # ROUTINES 43 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 44 | 45 | @pytest.fixture(scope = 'module') 46 | def fstest_scope(request): 47 | """Runs in project root! 48 | """ 49 | 50 | fs_type = request.config.getoption('M') 51 | 52 | fstest = fstest_class(fs_type = fs_type) 53 | 54 | def __finalizer__(): 55 | fstest.destroy() 56 | 57 | request.addfinalizer(__finalizer__) 58 | 59 | return fstest 60 | 61 | 62 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 63 | # CLASS: CORE 64 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 65 | 66 | class fstest_class( 67 | fstest_base_class, 68 | fstest_pre_class, 69 | fstest_prove_class, 70 | fstest_post_class 71 | ): 72 | 73 | 74 | def __init__(self, fs_type = TEST_FS_LOGGEDFS): 75 | 76 | self.init(fs_type = fs_type) 77 | -------------------------------------------------------------------------------- /tests/lib/climount.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/climount.py: Quick mount from CLI for tests 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | from .base import fstest_base_class 33 | 34 | 35 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 36 | # ROUTINES 37 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 38 | 39 | def quick_cli_cleanup(): 40 | 41 | fs = fstest_base_class() 42 | fs.init_a_members() 43 | fs.init_b_cleanup() 44 | 45 | 46 | def quick_cli_init(): 47 | 48 | fs = fstest_base_class() 49 | fs.init_a_members() 50 | fs.init_b_cleanup() 51 | fs.init_c_parentfs() 52 | fs.init_d_childfs() 53 | 54 | 55 | def quick_cli_init_parentfs(): 56 | 57 | fs = fstest_base_class() 58 | fs.init_a_members() 59 | fs.init_c_parentfs() 60 | 61 | 62 | def quick_cli_init_childfs(): 63 | 64 | fs = fstest_base_class() 65 | fs.init_a_members() 66 | fs.init_d_childfs() 67 | 68 | 69 | def quick_cli_destroy(): 70 | 71 | fs = fstest_base_class() 72 | fs.init_a_members() 73 | fs.destroy_a_childfs() 74 | fs.destroy_b_parentfs() 75 | 76 | 77 | def quick_cli_destroy_parentfs(): 78 | 79 | fs = fstest_base_class() 80 | fs.init_a_members() 81 | fs.assert_parentfs_mountpoint() 82 | fs.destroy_b_parentfs() 83 | 84 | 85 | def quick_cli_destroy_childfs(): 86 | 87 | fs = fstest_base_class() 88 | fs.init_a_members() 89 | fs.assert_childfs_mountpoint() 90 | fs.destroy_a_childfs() 91 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | Changes 2 | ======= 3 | 4 | 0.0.6 (2020-07-11) 5 | ------------------ 6 | 7 | * FEATURE: Added Python 3.8 support 8 | * Dropped Python 3.4 support 9 | * Updated dependency to ``refuse`` 0.0.4 10 | 11 | 0.0.5 (2019-05-06) 12 | ------------------ 13 | 14 | * Switched from `fusepy`_ to `refuse`_ 15 | 16 | .. _fusepy: https://github.com/fusepy/fusepy 17 | .. _refuse: https://github.com/pleiszenburg/refuse 18 | 19 | 0.0.4 (2019-05-01) 20 | ------------------ 21 | 22 | * FEATURE: New flag ``-m``, explicitly excluding all operations from the log that do not have the potential to change the filesystem. Added for convenience. 23 | * FIX: Terminating a notify session started in the background would not join all relevant threads properly. Terminating a notify session started in the foreground would cause an exception due to it attempting to join non-existing threads. 24 | * Added Jupyter Notebook example to documentation. 25 | 26 | 0.0.3 (2019-05-01) 27 | ------------------ 28 | 29 | * FEATURE: LoggedFS-python can be used as a library in other Python software, enabling a user to specify callback functions on filesystem events. The relevant infrastructure is exported as ``loggedfs.loggedfs_notify``. See library example under ``docs``. 30 | * FEATURE: New programmable filter pipeline, see ``loggedfs.filter_field_class``, ``loggedfs.filter_item_class`` and ``loggedfs.filter_pipeline_class`` 31 | * FEATURE: New flag ``-b``, explicitly activating logging of read and write buffers 32 | * FEATURE: In "traditional" logging mode (not JSON), read and write buffers are also logged zlib-compressed and BASE64 encoded. 33 | * FEATURE: Convenience function for decoding logged buffers, see ``loggedfs.decode_buffer`` 34 | * FIX: LoggedFS-python would have crashed if no XML configuration file had been specified. 35 | * FIX: **Directory listing (``ls``) was broken.** 36 | * FIX: Testing infrastructure did not catch all exceptions in tests. 37 | * FIX: Testing infrastructure did not handle timeouts on individual tests correctly. 38 | 39 | 0.0.2 (2019-04-23) 40 | ------------------ 41 | 42 | * FEATURE: New flag ``-j`` for JSON-formatted log output 43 | * FEATURE: New field ``command`` allowed in XML configuration files for filtering for command strings with regular expressions 44 | * FEATURE: All fields in ``include`` and ``exclude`` tags, e.g. ``extension`` or ``uid``, become optional / implicit and can be omitted. 45 | * FEATURE: (UNTESTED) Mac OS X support. Test framework still relies on Linux. 46 | * FIX: Several implementations of FUSE calls such as truncate did rely on the assumption that the current working directory of the file system process would not change. This was risky. LoggedFS-python does also NOT change the current working directory anymore on its own. 47 | * Code cleanup 48 | 49 | 0.0.1 (2019-04-11) 50 | ------------------ 51 | 52 | * First official BETA-release of *LoggedFS-python* 53 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/conftest.py: Configures the tests 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import argparse 33 | import os 34 | 35 | from .lib import ( 36 | fstest_parameters, 37 | TEST_ROOT_PATH, 38 | TEST_FSTEST_PATH, 39 | TEST_FSTEST_TESTS_SUBPATH 40 | ) 41 | 42 | 43 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 44 | # ROUTINES: PYTEST API 45 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 46 | 47 | def pytest_addoption(parser): 48 | 49 | parser.addoption( 50 | '-T', 51 | action = 'append', 52 | help = 'run specified test file', 53 | nargs = 1, 54 | type = __arg_type_testfile__ 55 | ) 56 | parser.addoption( 57 | '-M', 58 | action = 'store', 59 | default = 'loggedfs', 60 | help = 'specify tested filesystem' 61 | ) 62 | 63 | 64 | def pytest_generate_tests(metafunc): 65 | 66 | if 'fstest_group_path' in metafunc.fixturenames: 67 | if metafunc.config.getoption('T'): 68 | test_list = [a[0] for a in metafunc.config.getoption('T')] 69 | else: 70 | test_list = fstest_parameters() 71 | metafunc.parametrize('fstest_group_path', test_list) 72 | 73 | 74 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 75 | # ROUTINES: HELPER 76 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 77 | 78 | def __arg_type_testfile__(filename): 79 | 80 | file_path = os.path.join( 81 | TEST_ROOT_PATH, 82 | TEST_FSTEST_PATH, 83 | TEST_FSTEST_TESTS_SUBPATH, 84 | filename 85 | ) 86 | 87 | if os.path.isfile(file_path) and file_path.endswith('.t'): 88 | return os.path.abspath(file_path) 89 | 90 | raise argparse.ArgumentTypeError('No testfile: "%s"' % filename) 91 | -------------------------------------------------------------------------------- /tests/lib/mount.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/mount.py: Mount & umount routines 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | 34 | from .procio import run_command 35 | 36 | 37 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 38 | # ROUTINES 39 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 40 | 41 | def attach_loop_device(in_abs_path): 42 | 43 | return run_command(['losetup', '-f', in_abs_path], sudo = True) 44 | 45 | 46 | def detach_loop_device(device_path): 47 | 48 | return run_command(['losetup', '-d', device_path], sudo = True) 49 | 50 | 51 | def find_loop_devices(in_abs_path): 52 | 53 | status, out, err = run_command(['losetup', '-j', in_abs_path], return_output = True, sudo = True) 54 | if status: 55 | return [line.strip().split(':')[0] for line in out.split('\n') if line.strip() != ''] 56 | else: 57 | return None 58 | 59 | 60 | def is_path_mountpoint(in_abs_path): 61 | 62 | return run_command(['mountpoint', '-q', in_abs_path]) 63 | 64 | 65 | def mount(in_abs_path, device_path): 66 | 67 | return run_command(['mount', device_path, in_abs_path], sudo = True) 68 | 69 | 70 | def mount_loggedfs_python(in_abs_path, logfile, cfgfile, sudo = False): 71 | 72 | return run_command( 73 | [ 74 | 'coverage', 'run', 75 | os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'loggedfs'), 76 | '-l', logfile, 77 | '-c', cfgfile, 78 | '-p', 79 | '-s', 80 | '-b', 81 | # '-j', # optional: JSON 82 | in_abs_path, 83 | ], 84 | return_output = True, sudo = sudo, sudo_env = sudo 85 | ) 86 | 87 | 88 | def umount(in_abs_path, sudo = False, force = False): 89 | 90 | cmd_list = ['umount'] 91 | if force: 92 | cmd_list.append('-f') 93 | cmd_list.append(in_abs_path) 94 | 95 | return run_command(cmd_list, sudo = sudo) 96 | 97 | 98 | def umount_fuse(in_abs_path, sudo): 99 | 100 | return run_command(['fusermount', '-u', in_abs_path], sudo = sudo) 101 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | How to contribute to *LoggedFS-python* 2 | ====================================== 3 | 4 | Thank you for considering contributing to *LoggedFS-python*! 5 | **Contributions are highly welcomed!** 6 | 7 | Reporting issues 8 | ---------------- 9 | 10 | Issues are tracked on `Gitbub`_. 11 | 12 | - Describe what you expected to happen. 13 | - If possible, include a `minimal, complete, and verifiable example`_ to help 14 | identify the issue. This also helps check that the issue is not with your 15 | own code. 16 | - Describe what actually happened. Include the full traceback if there was an 17 | exception. 18 | - Add log files if possible. Careful, they tend to be huge. 19 | - List your operating system, *Python*, *FUSE* and *LoggedFS-python* versions. 20 | If possible, check if this issue is already fixed in the repository 21 | (development branch). 22 | 23 | .. _Gitbub: https://github.com/pleiszenburg/loggedfs-python/issues 24 | .. _minimal, complete, and verifiable example: https://stackoverflow.com/help/mcve 25 | 26 | Submitting patches 27 | ------------------ 28 | 29 | - Run ``make test`` before submission and indicate which tests the file-system now passes that did worked before. 30 | - Use **tabs** for indentation. 31 | - No, there is no line limit. Let your editor wrap the lines for you, if you want. 32 | - Add as many comments as you can - code-readability matters. 33 | - The ``master`` branch is supposed to be stable - request merges into the ``develop`` branch instead. 34 | - Commits are preferred to be signed (GPG). Seriously, sign your code. 35 | 36 | First time setup 37 | ---------------- 38 | 39 | - Make sure you have *FUSE* installed and working. 40 | - Download and install the `latest version of git`_. 41 | - Configure git with your `username`_ and `email`_: 42 | 43 | .. code:: bash 44 | 45 | git config --global user.name 'your name' 46 | git config --global user.email 'your email' 47 | 48 | - Make sure you have a `GitHub account`_. 49 | - Fork *LoggedFS-python* to your GitHub account by clicking the `Fork`_ button. 50 | - `Clone`_ your GitHub fork locally: 51 | 52 | .. code:: bash 53 | 54 | git clone https://github.com/{username}/loggedfs-python 55 | cd loggedfs-python 56 | 57 | - Add the main repository as a remote to update later: 58 | 59 | .. code:: bash 60 | 61 | git remote add pleiszenburg https://github.com/pleiszenburg/loggedfs-python 62 | git fetch pleiszenburg 63 | 64 | - Create a virtualenv: 65 | 66 | .. code:: bash 67 | 68 | python3 -m venv env 69 | . env/bin/activate 70 | 71 | - Install *LoggedFS-python* in editable mode with development dependencies. 72 | 73 | .. code:: bash 74 | 75 | make install_link 76 | 77 | - Check the installation by testing it: 78 | 79 | .. code:: bash 80 | 81 | make test 82 | 83 | .. _GitHub account: https://github.com/join 84 | .. _latest version of git: https://git-scm.com/downloads 85 | .. _username: https://help.github.com/articles/setting-your-username-in-git/ 86 | .. _email: https://help.github.com/articles/setting-your-email-in-git/ 87 | .. _Fork: https://github.com/pleiszenburg/loggedfs-python#fork-destination-box 88 | .. _Clone: https://help.github.com/articles/fork-a-repo/#step-2-create-a-local-clone-of-your-fork 89 | -------------------------------------------------------------------------------- /tests/lib/const.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/const.py: Const items 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # URLs & SOURCES 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | TEST_FSTEST_GITREPO = 'https://github.com/pjd/pjdfstest.git' 33 | TEST_FSX_SOURCE_URL = 'https://github.com/linux-test-project/ltp/raw/master/testcases/kernel/fs/fsx-linux/fsx-linux.c' 34 | 35 | 36 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 37 | # FOLDER & FILE NAMES 38 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 39 | 40 | TEST_ROOT_PATH = 'tests' 41 | 42 | TEST_FSTEST_PATH = 'test_suite' 43 | TEST_FSX_PATH = 'test_fsx' 44 | TEST_MOUNT_PARENT_PATH = 'test_mount' 45 | TEST_MOUNT_CHILD_PATH = 'test_child' 46 | TEST_LOG_PATH = 'test_logs' 47 | 48 | TEST_FSTEST_TESTS_SUBPATH = 'tests' 49 | TEST_FSTEST_CONF_FN = 'conf' 50 | TEST_FSTEST_LOG_FN = 'fstest.log' 51 | 52 | TEST_IMAGE_FN = 'test_image.bin' 53 | TEST_FSCK_FN = 'fsck.log' 54 | 55 | TEST_LOGGEDFS_CFG_FN = 'test_loggedfs_cfg.xml' 56 | TEST_LOGGEDFS_LOG_FN = 'loggedfs.log' 57 | TEST_LOGGEDFS_OUT_FN = 'loggedfs_out.log' 58 | TEST_LOGGEDFS_ERR_FN = 'loggedfs_err.log' 59 | 60 | TEST_RESULTS_FN = 'test_fstest_results.log' 61 | TEST_ERRORS_FN = 'test_fstest_errors.log' 62 | TEST_STATUS_CURRENT_FN = 'test_status_current.yaml' 63 | TEST_STATUS_DIFF_FN = 'test_status_diff.yaml' 64 | TEST_STATUS_FROZEN_FN = 'test_status_frozen.yaml' 65 | 66 | 67 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 68 | # MODES 69 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 70 | 71 | TEST_FS_EXT4 = 'ext4' 72 | TEST_FS_LOGGEDFS = 'loggedfs' 73 | TEST_IMAGE_SIZE_MB = 150 # > 129 74 | 75 | 76 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 77 | # SNIPPETS 78 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 79 | 80 | TEST_LOG_HEAD = """ 81 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 82 | # %s 83 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 84 | 85 | """ 86 | 87 | TEST_LOG_STATS = 'Stats: expected %d | %d passed | %d failed' 88 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | # LoggedFS-python 2 | # Filesystem monitoring with Fuse and Python 3 | # https://github.com/pleiszenburg/loggedfs-python 4 | # 5 | # makefile: GNU makefile for project management 6 | # 7 | # Copyright (C) 2017-2020 Sebastian M. Ernst 8 | # 9 | # 10 | # The contents of this file are subject to the Apache License 11 | # Version 2 ("License"). You may not use this file except in 12 | # compliance with the License. You may obtain a copy of the License at 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | # https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 15 | # 16 | # Software distributed under the License is distributed on an "AS IS" basis, 17 | # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing rights and limitations under the License. 19 | # 20 | 21 | 22 | T = "" 23 | 24 | clean: 25 | -rm -r build/* 26 | -rm -r dist/* 27 | -rm -r src/*.egg-info 28 | # -rm -r htmlconv/* 29 | # -rm .coverage* 30 | coverage erase 31 | find src/ tests/ -name '*.pyc' -exec rm -f {} + 32 | find src/ tests/ -name '*.pyo' -exec rm -f {} + 33 | # find src/ tests/ -name '*~' -exec rm -f {} + 34 | find src/ tests/ -name '__pycache__' -exec rm -fr {} + 35 | # find src/ tests/ -name '*.htm' -exec rm -f {} + 36 | # find src/ tests/ -name '*.html' -exec rm -f {} + 37 | # find src/ tests/ -name '*.so' -exec rm -f {} + 38 | 39 | # docu: 40 | # @(cd docs; make clean; make html) 41 | 42 | release: 43 | make clean 44 | python setup.py sdist bdist_wheel 45 | gpg --detach-sign -a dist/loggedfs*.whl 46 | gpg --detach-sign -a dist/loggedfs*.tar.gz 47 | 48 | upload: 49 | for filename in $$(ls dist/*.tar.gz dist/*.whl) ; do \ 50 | twine upload $$filename $$filename.asc ; \ 51 | done 52 | 53 | upload_test: 54 | for filename in $$(ls dist/*.tar.gz dist/*.whl) ; do \ 55 | twine upload $$filename $$filename.asc -r pypitest ; \ 56 | done 57 | 58 | install: 59 | pip install .[dev] 60 | make install_fstest 61 | make install_fsx 62 | 63 | install_link: 64 | pip install -e .[dev] 65 | make install_fstest 66 | make install_fsx 67 | 68 | install_fstest: 69 | python3 -c 'import tests; tests.lib.install_fstest()' 70 | 71 | install_fsx: 72 | python3 -c 'import tests; tests.lib.install_fsx()' 73 | 74 | cleanup: 75 | python3 -c 'import tests; tests.lib.quick_cli_clean()' 76 | init: 77 | python3 -c 'import tests; tests.lib.quick_cli_init()' 78 | init_parentfs: 79 | python3 -c 'import tests; tests.lib.quick_cli_init_parentfs()' 80 | init_childfs: 81 | python3 -c 'import tests; tests.lib.quick_cli_init_childfs()' 82 | destroy: 83 | python3 -c 'import tests; tests.lib.quick_cli_destroy()' 84 | destroy_parentfs: 85 | python3 -c 'import tests; tests.lib.quick_cli_destroy_parentfs()' 86 | destroy_childfs: 87 | python3 -c 'import tests; tests.lib.quick_cli_destroy_childfs()' 88 | destroy_force: 89 | -sudo fusermount -u tests/test_mount/test_child 90 | -sudo umount tests/test_mount 91 | 92 | test: 93 | make test_posix 94 | make test_stress 95 | 96 | test_posix: 97 | # make docu 98 | -rm tests/__pycache__/*.pyc 99 | -rm tests/lib/__pycache__/*.pyc 100 | # USAGE: make test T="-T chmod/01.t -T chmod/02.t" 101 | # REFERENCE TEST WITH EXT4: make test T="-M ext4" 102 | pytest $(T) 103 | 104 | test_stress: 105 | tests/scripts/fsx 106 | -------------------------------------------------------------------------------- /tests/lib/install.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/install.py: Install software required for tests 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | import shutil 34 | 35 | from .const import ( 36 | TEST_FSTEST_GITREPO, 37 | TEST_FSTEST_PATH, 38 | TEST_FSTEST_CONF_FN, 39 | TEST_FSTEST_TESTS_SUBPATH, 40 | TEST_FSX_PATH, 41 | TEST_FSX_SOURCE_URL, 42 | TEST_ROOT_PATH 43 | ) 44 | from .procio import ( 45 | download_file, 46 | read_file, 47 | run_command, 48 | write_file 49 | ) 50 | 51 | 52 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 53 | # ROUTINES: FSTEST 54 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 55 | 56 | def install_fstest(): 57 | """PUBLIC: Called from project root 58 | """ 59 | 60 | install_path = os.path.join(TEST_ROOT_PATH, TEST_FSTEST_PATH) 61 | if os.path.isdir(install_path): 62 | shutil.rmtree(install_path, ignore_errors = True) 63 | git_clone_result = run_command(['git', 'clone', TEST_FSTEST_GITREPO, install_path]) 64 | assert git_clone_result 65 | __build_fstest__(install_path) 66 | 67 | 68 | def install_fsx(): 69 | """PUBLIC: Called from project root 70 | """ 71 | 72 | install_path = os.path.join(TEST_ROOT_PATH, TEST_FSX_PATH) 73 | source_path = os.path.join(install_path, TEST_FSX_SOURCE_URL.split('/')[-1]) 74 | 75 | if os.path.isfile(source_path): 76 | os.remove(source_path) 77 | write_file(source_path, download_file(TEST_FSX_SOURCE_URL).decode('utf-8')) 78 | 79 | old_path = os.getcwd() 80 | os.chdir(install_path) 81 | 82 | compile_status = run_command(['make', 'all']) 83 | assert compile_status 84 | install_status = run_command(['make', 'install']) 85 | assert install_status 86 | 87 | os.chdir(old_path) 88 | 89 | 90 | def __build_fstest__(abs_in_path, filesystem = 'ext4'): 91 | 92 | old_path = os.getcwd() 93 | os.chdir(abs_in_path) 94 | 95 | # Fix filesystem in test config 96 | conf_rel_path = os.path.join(TEST_FSTEST_TESTS_SUBPATH, TEST_FSTEST_CONF_FN) 97 | fstest_conf = read_file(conf_rel_path).split('\n') 98 | for index, line in enumerate(fstest_conf): 99 | if line.startswith('fs=') or line.startswith('#fs='): 100 | fstest_conf[index] = 'fs="%s"' % filesystem 101 | break 102 | write_file(conf_rel_path, '\n'.join(fstest_conf)) 103 | 104 | autoreconf_status = run_command(['autoreconf', '-ifs']) 105 | assert autoreconf_status 106 | configure_status = run_command(['./configure']) 107 | assert configure_status 108 | build_status, out, err = run_command(['make', 'pjdfstest'], return_output = True) 109 | assert build_status 110 | 111 | os.chdir(old_path) 112 | -------------------------------------------------------------------------------- /docs/library_example.rst: -------------------------------------------------------------------------------- 1 | Using LoggedFS-python as a library 2 | ================================== 3 | 4 | Create a new directory, for instance in your current working directory, named ``demo_dir``. Then fire up an interactive Python shell such as Jupyter Notebook or Jupyter Lab. Now you can try the following: 5 | 6 | .. code:: python 7 | 8 | import re 9 | import loggedfs 10 | 11 | demo_filter = loggedfs.filter_pipeline_class( 12 | include_list = [loggedfs.filter_item_class([loggedfs.filter_field_class( 13 | name = 'proc_cmd', value = re.compile('.*kate.*').match 14 | )])] 15 | ) 16 | demo_data = [] 17 | demo_err = [] 18 | 19 | demo = loggedfs.loggedfs_notify( 20 | 'demo_dir', 21 | background = True, 22 | log_filter = demo_filter, 23 | consumer_out_func = demo_data.append, 24 | consumer_err_func = demo_err.append 25 | ) 26 | 27 | You have just started recording all filesystem events that involve a command containing the string ``kate``. Put the Python shell aside and write some stuff into the ``demo_dir`` using ``Kate``, the KDE text editor. Once you are finished, go back to your Python shell and terminate the recording. 28 | 29 | .. code:: python 30 | 31 | demo.terminate() 32 | 33 | Notice that the recorded data ends with an "end of transmission" marker. For convenience, remove it first: 34 | 35 | .. code:: python 36 | 37 | assert isinstance(demo_data[-1], loggedfs.end_of_transmission) 38 | demo_data = demo_data[:-1] 39 | 40 | Let's have a look at what you have recorded: 41 | 42 | .. code:: python 43 | 44 | print(demo_data[44]) # index 44 might show something different in your case 45 | 46 | :: 47 | 48 | {'proc_cmd': '/usr/bin/kate -b /test/demo_dir/demo_file.txt', 49 | 'proc_uid': 1000, 50 | 'proc_uid_name': 'ernst', 51 | 'proc_gid': 100, 52 | 'proc_gid_name': 'users', 53 | 'proc_pid': 11716, 54 | 'action': 'read', 55 | 'status': True, 56 | 'param_path': '/test/demo_dir/demo_file.txt', 57 | 'param_length': 4096, 58 | 'param_offset': 0, 59 | 'param_fip': 5, 60 | 'return_len': 1486, 61 | 'return': '', 62 | 'time': 1556562162704772619} 63 | 64 | Every single event is represented as a dictionary. ``demo_data`` is therefore a list of dictionaries. The following columns / keys are always present: 65 | 66 | - proc_cmd: Command line of the process ordering the operation. 67 | - proc_uid: UID (user ID) of the owner of the process ordering the operation. 68 | - proc_uid_name: User name of the owner of the process ordering the operation. 69 | - proc_gid: GID (group ID) of the owner of the process ordering the operation. 70 | - proc_gid_name: Group name of the owner of the process ordering the operation. 71 | - proc_pid: PID (process ID) of the process ordering the operation. 72 | - action: Name of filesystem operation, such as ``open``, ``read`` or ``write``. 73 | - status: Boolean, describing the success of the operation. 74 | - return: Return value of operation. ``None`` if there is none. 75 | - time: System time, nanoseconds, UTC 76 | 77 | Other columns / keys are optional and depend on the operation and its status. With this knowledge, you can run typical Python data analysis frameworks across this data. Pandas for instance: 78 | 79 | .. code:: python 80 | 81 | import pandas as pd 82 | data_df = pd.DataFrame.from_records(demo_data, index = 'time') 83 | 84 | data_df[data_df['action'] == 'write'][['param_buf_len', 'param_offset', 'return']] 85 | 86 | :: 87 | 88 | param_buf_len param_offset return 89 | time 90 | 1556562164301499774 57.0 0.0 57 91 | 1556562164304043463 2.0 57.0 2 92 | 1556562164621417400 1487.0 0.0 1487 93 | 1556562165260276486 53.0 0.0 53 94 | 1556562165532797611 1486.0 0.0 1486 95 | -------------------------------------------------------------------------------- /src/loggedfs/_core/log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/log.py: Logging 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import json 33 | import logging 34 | import logging.handlers 35 | import os 36 | import platform 37 | 38 | from .timing import time 39 | 40 | 41 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 42 | # CONST 43 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 44 | 45 | SYSLOG_ADDRESS = { 46 | 'Linux': '/dev/log', 47 | 'Darwin': '/var/run/syslog' 48 | } 49 | 50 | 51 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 52 | # LOGGING: Support nano-second timestamps 53 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 54 | 55 | class _LogRecord_ns_(logging.LogRecord): 56 | 57 | def __init__(self, *args, **kwargs): 58 | 59 | self.created_ns = time.time_ns() # Fetch precise timestamp 60 | super().__init__(*args, **kwargs) 61 | 62 | 63 | class _Formatter_ns_(logging.Formatter): 64 | 65 | default_nsec_format = '%s,%09d' 66 | 67 | def formatTime(self, record, datefmt=None): 68 | 69 | if datefmt is not None: # Do not handle custom formats here ... 70 | return super().formatTime(record, datefmt) # ... leave to original implementation 71 | ct = self.converter(record.created_ns / 1e9) 72 | t = time.strftime(self.default_time_format, ct) 73 | s = self.default_nsec_format % (t, record.created_ns - (record.created_ns // 10**9) * 10**9) 74 | return s 75 | 76 | 77 | logging.setLogRecordFactory(_LogRecord_ns_) 78 | 79 | 80 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 81 | # ROUTINES 82 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 83 | 84 | 85 | def get_logger(name, log_enabled, log_file, log_syslog, log_json): 86 | 87 | if log_json: 88 | log_formater = _Formatter_ns_('{"time": "%(asctime)s", "logger": "%(name)s", %(message)s}') 89 | log_formater_short = _Formatter_ns_('{%(message)s}') 90 | else: 91 | log_formater = _Formatter_ns_('%(asctime)s (%(name)s) %(message)s') 92 | log_formater_short = _Formatter_ns_('%(message)s') 93 | 94 | logger = logging.getLogger(name) 95 | 96 | if not bool(log_enabled): 97 | logger.setLevel(logging.CRITICAL) 98 | return logger 99 | logger.setLevel(logging.DEBUG) 100 | 101 | ch = logging.StreamHandler() 102 | ch.setLevel(logging.DEBUG) 103 | ch.setFormatter(log_formater) 104 | logger.addHandler(ch) 105 | 106 | if bool(log_syslog): 107 | try: 108 | sl = logging.handlers.SysLogHandler(address = SYSLOG_ADDRESS[platform.system()]) 109 | except KeyError: 110 | raise NotImplementedError('unsupported operating system') 111 | sl.setLevel(logging.DEBUG) 112 | sl.setFormatter(log_formater_short) 113 | logger.addHandler(sl) 114 | 115 | if log_file is None: 116 | return logger 117 | 118 | fh = logging.FileHandler(os.path.join(log_file)) # TODO 119 | fh.setLevel(logging.DEBUG) 120 | fh.setFormatter(log_formater) 121 | logger.addHandler(fh) 122 | 123 | return logger 124 | 125 | 126 | def log_msg(log_json, msg): 127 | 128 | if log_json: 129 | return '"msg": %s' % json.dumps(msg) 130 | else: 131 | return msg 132 | -------------------------------------------------------------------------------- /tests/lib/param.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/param.py: Generate parameter tuple for tests 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | 34 | # Use the built-in version of scandir/walk if possible 35 | # Otherwise: https://github.com/benhoyt/scandir 36 | try: 37 | from os import scandir 38 | except ImportError: 39 | from scandir import scandir 40 | 41 | import pytest 42 | 43 | from .const import ( 44 | TEST_ROOT_PATH, 45 | TEST_FSTEST_PATH, 46 | TEST_FSTEST_TESTS_SUBPATH 47 | ) 48 | 49 | 50 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 51 | # ROUTINES 52 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 53 | 54 | def fstest_parameters(): 55 | """Can be called from anywhere in project tree ... 56 | """ 57 | 58 | fstests_root_abs_path = os.path.join( 59 | __find_root_path__(os.getcwd()), 60 | TEST_ROOT_PATH, 61 | TEST_FSTEST_PATH, 62 | TEST_FSTEST_TESTS_SUBPATH 63 | ) 64 | 65 | test_group_list = [] 66 | __get_recursive_inventory_list__(fstests_root_abs_path, fstests_root_abs_path, test_group_list) 67 | test_group_list = __ignore_tests__(test_group_list) 68 | 69 | return test_group_list 70 | 71 | 72 | def __find_root_path__(in_path): 73 | 74 | if os.path.isfile(os.path.join(in_path, 'setup.py')): 75 | return in_path 76 | 77 | while True: 78 | 79 | # Go one up 80 | new_path = os.path.abspath(os.path.join(in_path, '..')) 81 | # Can't go futher up 82 | if new_path == in_path: 83 | break 84 | # Set path 85 | in_path = new_path 86 | 87 | # Check for repo folder 88 | if os.path.isfile(os.path.join(in_path, 'setup.py')): 89 | return in_path 90 | 91 | # Nothing found 92 | raise 93 | 94 | 95 | def __get_recursive_inventory_list__(root_path, scan_root_path, test_group_list): 96 | 97 | relative_path = os.path.relpath(scan_root_path, root_path) 98 | for item in scandir(scan_root_path): 99 | if item.is_file(): 100 | if not item.name.endswith('.t'): 101 | continue 102 | test_group_list.append(item.path) 103 | elif item.is_dir(): 104 | __get_recursive_inventory_list__(root_path, item.path, test_group_list) 105 | elif item.is_symlink(): 106 | pass 107 | else: 108 | raise # TODO 109 | 110 | 111 | def __ignore_tests__(old_group_list): 112 | 113 | new_group_list = [] 114 | 115 | ignore_group_list = [] 116 | # [ 117 | # ('truncate', 3), 118 | # ('open', 3), 119 | # ('mkdir', 3), 120 | # ('chmod', 3), 121 | # ('mknod', 3), 122 | # ('mkfifo', 3), 123 | # ('unlink', 3), 124 | # ('symlink', 3), 125 | # ('link', 3), 126 | # ('chown', 3), 127 | # ('ftruncate', 3), 128 | # ('rmdir', 3), 129 | # ('rename', 2) 130 | # ] # The original LoggedFS crashes when tested against those. 131 | xfail_group_list = [] 132 | 133 | for group_path in old_group_list: 134 | 135 | group_path_segment, group_number_str = os.path.split(group_path) 136 | group_number = int(group_number_str.split('.')[0]) 137 | _, group_folder = os.path.split(group_path_segment) 138 | 139 | if (group_folder, group_number) not in ignore_group_list and (group_folder, group_number) not in xfail_group_list: 140 | new_group_list.append(group_path) 141 | elif (group_folder, group_number) in xfail_group_list and (group_folder, group_number) not in ignore_group_list: 142 | new_group_list.append(pytest.param(group_path, marks = pytest.mark.xfail)) 143 | 144 | return new_group_list 145 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | setup.py: Used for package distribution 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | from setuptools import ( 34 | find_packages, 35 | setup 36 | ) 37 | import sys 38 | 39 | 40 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 41 | # SETUP 42 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 43 | 44 | 45 | # Bump version HERE! 46 | _version_ = '0.0.6' 47 | 48 | 49 | # List all versions of Python which are supported 50 | python_minor_min = 5 51 | python_minor_max = 8 52 | confirmed_python_versions = [ 53 | 'Programming Language :: Python :: 3.{MINOR:d}'.format(MINOR = minor) 54 | for minor in range(python_minor_min, python_minor_max + 1) 55 | ] 56 | 57 | 58 | # Fetch readme file 59 | with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: 60 | long_description = f.read() 61 | 62 | 63 | # Development dependencies 64 | development_deps_list = [ 65 | 'coverage', 66 | 'psutil', 67 | 'pytest', 68 | 'python-language-server[all]', 69 | 'PyYAML', 70 | 'setuptools', 71 | 'Sphinx', 72 | 'sphinx_rtd_theme', 73 | 'tap.py', 74 | 'twine', 75 | 'wheel' 76 | ] 77 | 78 | 79 | # Get Python interpreter version 80 | py_gen, py_ver, *_ = sys.version_info 81 | # Raise an error if this is running on Python 2 (legacy) 82 | if py_gen < 3: 83 | raise NotImplementedError 84 | # Handle CPython 3.4 and below extra dependency 85 | if py_ver < 5: 86 | development_deps_list.append('scandir') # See https://github.com/benhoyt/scandir 87 | 88 | 89 | setup( 90 | name = 'loggedfs', 91 | packages = find_packages('src'), 92 | package_dir = {'': 'src'}, 93 | version = _version_, 94 | description = 'Filesystem monitoring with Fuse and Python', 95 | long_description = long_description, 96 | author = 'Sebastian M. Ernst', 97 | author_email = 'ernst@pleiszenburg.de', 98 | url = 'https://github.com/pleiszenburg/loggedfs-python', 99 | download_url = 'https://github.com/pleiszenburg/loggedfs-python/archive/v%s.tar.gz' % _version_, 100 | license = 'Apache License 2.0', 101 | keywords = ['filesystem', 'fuse', 'logging', 'monitoring'], 102 | include_package_data = True, 103 | python_requires = '>=3.{MINOR:d}'.format(MINOR = python_minor_min), 104 | install_requires = [ 105 | 'click>=7.0', 106 | 'refuse==0.0.4', 107 | 'xmltodict' 108 | ], 109 | extras_require = {'dev': development_deps_list}, 110 | entry_points = ''' 111 | [console_scripts] 112 | loggedfs = loggedfs:cli_entry 113 | ''', 114 | zip_safe = False, 115 | classifiers = [ 116 | 'Development Status :: 4 - Beta', 117 | 'Environment :: Console', 118 | 'Intended Audience :: Developers', 119 | 'Intended Audience :: Information Technology', 120 | 'Intended Audience :: Science/Research', 121 | 'Intended Audience :: System Administrators', 122 | 'License :: OSI Approved :: Apache Software License', 123 | 'Operating System :: POSIX :: Linux', 124 | 'Operating System :: MacOS', 125 | 'Programming Language :: Python :: 3' 126 | ] + confirmed_python_versions + [ 127 | 'Programming Language :: Python :: 3 :: Only', 128 | 'Programming Language :: Python :: Implementation :: CPython', 129 | 'Topic :: Scientific/Engineering', 130 | 'Topic :: Software Development :: Libraries', 131 | 'Topic :: Software Development :: Testing', 132 | 'Topic :: System :: Archiving', 133 | 'Topic :: System :: Filesystems', 134 | 'Topic :: System :: Logging', 135 | 'Topic :: System :: Monitoring', 136 | 'Topic :: System :: Systems Administration', 137 | 'Topic :: Utilities' 138 | ] 139 | ) 140 | -------------------------------------------------------------------------------- /src/loggedfs/_core/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/cli.py: Command line interface 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import click 33 | 34 | from .defaults import LOG_ENABLED_DEFAULT, LOG_PRINTPROCESSNAME_DEFAULT 35 | from .fs import loggedfs_factory 36 | from .filter import filter_pipeline_class 37 | 38 | 39 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 40 | # ROUTINE 41 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 42 | 43 | @click.command() 44 | @click.option( 45 | '-f', 46 | is_flag = True, 47 | help = 'Do not start as a daemon. Write logs to stdout if no log file is specified.' 48 | ) 49 | @click.option( 50 | '-p', 51 | is_flag = True, 52 | help = 'Allow every user to see the new loggedfs.' 53 | ) 54 | @click.option( 55 | '-c', 56 | type = click.File(mode = 'r'), 57 | help = 'Use the "config-file" to filter what you want to log.' 58 | ) 59 | @click.option( 60 | '-s', 61 | is_flag = True, 62 | help = 'Deactivate logging to syslog.' 63 | ) 64 | @click.option( 65 | '-l', 66 | type = click.Path(file_okay = True, dir_okay = False, resolve_path = True), 67 | help = ('Use the "log-file" to write logs to.') 68 | ) 69 | @click.option( 70 | '-j', '--json', 71 | is_flag = True, 72 | help = 'Format output as JSON instead of traditional loggedfs format.' 73 | ) 74 | @click.option( 75 | '-b', '--buffers', 76 | is_flag = True, 77 | help = 'Include read/write-buffers (compressed, BASE64) in log.' 78 | ) 79 | @click.option( 80 | '--lib', 81 | is_flag = True, 82 | help = 'Run in library mode. DO NOT USE THIS FROM THE COMMAND LINE!', 83 | hidden = True 84 | ) 85 | @click.option( 86 | '-m', '--only-modify-operations', 87 | is_flag = True, 88 | help = 'Exclude logging of all operations that can not cause changes in the filesystem. Convenience flag for accelerated logging.' 89 | ) 90 | @click.argument( 91 | 'directory', 92 | type = click.Path(exists = True, file_okay = False, dir_okay = True, resolve_path = True) 93 | ) 94 | def cli_entry(f, p, c, s, l, json, buffers, lib, only_modify_operations, directory): 95 | """LoggedFS-python is a transparent fuse-filesystem which allows to log 96 | every operation that happens in the backend filesystem. Logs can be written 97 | to syslog, to a file, or to the standard output. LoggedFS-python allows to specify an XML 98 | configuration file in which you can choose exactly what you want to log and 99 | what you don't want to log. You can add filters on users, operations (open, 100 | read, write, chown, chmod, etc.), filenames, commands and return code. 101 | """ 102 | 103 | loggedfs_factory( 104 | directory, 105 | **__process_config__(c, l, s, f, p, json, buffers, lib, only_modify_operations) 106 | ) 107 | 108 | 109 | def __process_config__( 110 | config_fh, 111 | log_file, 112 | log_syslog_off, 113 | fuse_foreground, 114 | fuse_allowother, 115 | log_json, 116 | log_buffers, 117 | lib_mode, 118 | log_only_modify_operations 119 | ): 120 | 121 | if config_fh is not None: 122 | config_data = config_fh.read() 123 | config_fh.close() 124 | ( 125 | log_enabled, log_printprocessname, filter_obj 126 | ) = filter_pipeline_class.from_xmlstring(config_data) 127 | config_file = config_fh.name 128 | else: 129 | log_enabled = LOG_ENABLED_DEFAULT 130 | log_printprocessname = LOG_PRINTPROCESSNAME_DEFAULT 131 | filter_obj = filter_pipeline_class() 132 | config_file = None 133 | 134 | return { 135 | 'fuse_foreground': fuse_foreground, 136 | 'fuse_allowother': fuse_allowother, 137 | 'lib_mode': lib_mode, 138 | 'log_buffers': log_buffers, 139 | '_log_configfile' : config_file, 140 | 'log_enabled': log_enabled, 141 | 'log_file': log_file, 142 | 'log_filter': filter_obj, 143 | 'log_json': log_json, 144 | 'log_only_modify_operations': log_only_modify_operations, 145 | 'log_printprocessname': log_printprocessname, 146 | 'log_syslog': not log_syslog_off 147 | } 148 | -------------------------------------------------------------------------------- /src/loggedfs/_core/ipc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/ipc.py: Fast IPC through a pipe 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import pickle 33 | import queue 34 | import struct 35 | import subprocess 36 | import sys 37 | import threading 38 | import time 39 | 40 | 41 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 42 | # CONST 43 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 44 | 45 | PREFIX = b'\xBA\xDE\xAF\xFE' 46 | LEN_DTYPE = 'Q' # uint64 47 | WAIT_TIMEOUT = 0.1 # seconds 48 | 49 | 50 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 51 | # CLASS: END OF TRANSMISSION 52 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 53 | 54 | class end_of_transmission: 55 | 56 | 57 | def __init__(self, id): 58 | 59 | self._id = id 60 | 61 | 62 | @property 63 | def id(self): 64 | return self._id 65 | 66 | 67 | def __repr__(self): 68 | 69 | return ''.format(ID = self._id) 70 | 71 | 72 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 73 | # CLASS: RECEIVER (SINGLE STREAM) 74 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 75 | 76 | class _receiver_class: 77 | 78 | 79 | def __init__(self, stream_id, in_stream, decoder_func, processing_func): 80 | 81 | self._id = stream_id 82 | self._s = in_stream 83 | self._f = processing_func 84 | self._q = queue.Queue() 85 | self._t = threading.Thread( 86 | target = decoder_func, 87 | args = (self._id, self._s, self._q), 88 | daemon = True 89 | ) 90 | self._t.start() 91 | self.join = self._t.join 92 | 93 | 94 | def flush(self): 95 | 96 | while not self._q.empty(): 97 | try: 98 | data = self._q.get_nowait() 99 | except queue.Empty: 100 | pass 101 | else: 102 | self._f(data) 103 | self._q.task_done() 104 | 105 | 106 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 107 | # ROUTINES: DECODER FUNCTIONS FOR RECEIVER 108 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 109 | 110 | def _out_decoder(_id, _s, _q): 111 | 112 | prefix_len = len(PREFIX) 113 | while True: 114 | prefix = _s.read(prefix_len) 115 | if len(prefix) == 0: # end of pipe 116 | _q.put(end_of_transmission(_id)) 117 | break 118 | data_len_encoded = _s.read(8) 119 | data_len = struct.unpack(LEN_DTYPE, data_len_encoded)[0] 120 | data_bin = _s.read(data_len) 121 | _q.put(pickle.loads(data_bin)) 122 | 123 | 124 | def _err_decoder(_id, _s, _q): 125 | 126 | while True: 127 | msg = _s.readline() 128 | if len(msg) == 0: 129 | _q.put(end_of_transmission(_id)) 130 | break 131 | _q.put(msg.decode('utf-8')) 132 | 133 | 134 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 135 | # ROUTINES: SEND AND RECEIVE 136 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 137 | 138 | def receive(cmd_list, out_func, err_func, post_exit_func): 139 | 140 | proc = subprocess.Popen(cmd_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE) 141 | proc_alive = True 142 | out_r = _receiver_class('out', proc.stdout, _out_decoder, out_func) 143 | err_r = _receiver_class('err', proc.stderr, _err_decoder, err_func) 144 | 145 | while proc_alive: 146 | time.sleep(WAIT_TIMEOUT) 147 | out_r.flush() 148 | err_r.flush() 149 | proc_alive = proc.poll() is None 150 | 151 | out_r.join() 152 | err_r.join() 153 | post_exit_func() 154 | 155 | 156 | def send(data): 157 | 158 | data_bin = pickle.dumps(data) 159 | data_len = struct.pack(LEN_DTYPE, len(data_bin)) 160 | sys.stdout.buffer.write(PREFIX) 161 | sys.stdout.buffer.write(data_len) 162 | sys.stdout.buffer.write(data_bin) 163 | sys.stdout.flush() 164 | -------------------------------------------------------------------------------- /tests/scripts/fsx_analyze.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | 6 | LoggedFS-python 7 | Filesystem monitoring with Fuse and Python 8 | https://github.com/pleiszenburg/loggedfs-python 9 | 10 | tests/scripts/fsx_analyze.py: Merge the output of fsx-linux and LoggedFS, then display result 11 | 12 | Copyright (C) 2017-2020 Sebastian M. Ernst 13 | 14 | 15 | The contents of this file are subject to the Apache License 16 | Version 2 ("License"). You may not use this file except in 17 | compliance with the License. You may obtain a copy of the License at 18 | https://www.apache.org/licenses/LICENSE-2.0 19 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 20 | 21 | Software distributed under the License is distributed on an "AS IS" basis, 22 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 23 | specific language governing rights and limitations under the License. 24 | 25 | 26 | """ 27 | 28 | 29 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 30 | # IMPORT 31 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 32 | 33 | import datetime 34 | import os 35 | import sys 36 | 37 | 38 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 39 | # CONST 40 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 41 | 42 | # https://en.wikipedia.org/wiki/ANSI_escape_code 43 | c = { 44 | 'RESET': '\033[0;0m', 45 | 'BOLD': '\033[;1m', 46 | 'REVERSE': '\033[;7m', 47 | 'GREY': '\033[1;30m', 48 | 'RED': '\033[1;31m', 49 | 'GREEN': '\033[1;32m', 50 | 'YELLOW': '\033[1;33m', 51 | 'BLUE': '\033[1;34m', 52 | 'MAGENTA': '\033[1;35m', 53 | 'CYAN': '\033[1;36m', 54 | 'WHITE': '\033[1;37m' 55 | } 56 | 57 | 58 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 59 | # ROUTINES 60 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 61 | 62 | def print_line(line_dict): 63 | t = str(line_dict['t']) 64 | sys.stdout.write(c['GREY'] + t[:-9] + '.' + t[-9:] + c['RESET'] + ' ') 65 | if line_dict['s'] == 'FS': 66 | sys.stdout.write(c['RED']) 67 | elif line_dict['s'] == 'IN': 68 | sys.stdout.write(c['MAGENTA']) 69 | elif line_dict['s'] == '??': 70 | sys.stdout.write(c['YELLOW']) 71 | else: 72 | sys.stdout.write(c['GREEN']) 73 | sys.stdout.write(line_dict['c'] + c['RESET'] + ' ') 74 | sys.stdout.write(c['WHITE'] + line_dict['p'] + c['RESET']) 75 | sys.stdout.write('\n') 76 | sys.stdout.flush() 77 | 78 | 79 | def parse_iso_datestring(date_str, original_loggedfs): 80 | 81 | return ( 82 | int(datetime.datetime( 83 | *(int(v) for v in date_str[:19].replace('-', ' ').replace(':', ' ').split(' ')) 84 | ).timestamp()) 85 | * 10**9 + (int(date_str[20:]) * 10**6 if original_loggedfs else int(date_str[20:])) 86 | ) 87 | 88 | 89 | def parse_timestamp(date_str): 90 | 91 | seconds, microseconds = date_str.split('.') 92 | seconds = int(seconds) 93 | microseconds = int(microseconds) 94 | 95 | return seconds * 10**9 + microseconds * 10**3 96 | 97 | 98 | def hex_to_dec(hex_str): 99 | 100 | return str(int(hex_str[2:], 16)) 101 | 102 | 103 | def split_at(in_str, separator, position): 104 | 105 | split_list = in_str.split(separator) 106 | return [ 107 | separator.join(split_list[:position]), 108 | separator.join(split_list[position:]) 109 | ] 110 | 111 | 112 | def parse_fs_line(line): 113 | 114 | ts, payload = split_at(line, ' ', 2) 115 | if payload.startswith('INFO [default] '): # original LoggedFS 116 | _, payload = split_at(payload, ' ', 2) 117 | _original_loggedfs = True 118 | else: # LoggedFS-python 119 | _, payload = split_at(payload, ' ', 1) 120 | _original_loggedfs = False 121 | command, payload = payload.split(' ', 1) 122 | if '{SUCCESS}' in payload: 123 | param, remaining = payload.split(' {SUCCESS} ', 1) 124 | else: 125 | param, remaining = payload.split(' {FAILURE} ', 1) 126 | 127 | return { 128 | 't': parse_iso_datestring(ts, _original_loggedfs), # time 129 | 'c': command, # command 130 | 'p': param, # param 131 | 's': 'FS' if ' fsx-linux ' in line else ('IN' if ' pid = 0 uid = 0 ' in line else '??') # log source 132 | } 133 | 134 | 135 | def parse_fsx_line(line): 136 | 137 | _, payload = line.split(' ', 1) 138 | ts, payload = payload.split(' ', 1) 139 | command, param = payload.split(' ', 1) 140 | 141 | param = param.replace('\t', ' ').replace(' ', ' ').strip() 142 | param = [item.strip(' ()') for item in param.split(' ')] 143 | param = [hex_to_dec(item) if item.startswith('0x') else item for item in param] 144 | param = [str(int((item))) if item.isdigit() else item for item in param] 145 | param = ' '.join(param) 146 | 147 | return { 148 | 't': parse_timestamp(ts), # time 149 | 'c': command, # command 150 | 'p': param, # param 151 | 's': 'XX' # log source 152 | } 153 | 154 | 155 | def main(): 156 | 157 | with open('tests/test_logs/loggedfs.log', 'r', encoding = 'utf-8') as f: 158 | log_fs = f.read() 159 | 160 | with open('tests/test_logs/iotest.fsxlog', 'r', encoding = 'utf-8') as f: 161 | log_fsx = f.read() 162 | 163 | log_fs_lines = [ 164 | parse_fs_line(line) for line in 165 | log_fs.replace(os.path.join(os.path.abspath('.'), 'tests/test_mount/test_child'), '').split('\n') 166 | if ( 167 | ' fsx-linux ' in line 168 | or ' pid = 0 uid = 0 ' in line 169 | or ' write ' in line 170 | or ' read ' in line 171 | or ' truncate ' in line 172 | ) and ' getattr ' not in line 173 | ] 174 | 175 | log_fsx_lines = [ 176 | parse_fsx_line(line) for line in 177 | log_fsx.split('\n') 178 | if line[:6].isdigit() and line[6] == ' ' 179 | ] 180 | 181 | log_lines = sorted(log_fs_lines + log_fsx_lines, key = lambda k: k['t']) 182 | 183 | for line in log_lines: 184 | print_line(line) 185 | 186 | 187 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 188 | # ENTRY POINT 189 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 190 | 191 | if __name__ == '__main__': 192 | 193 | main() 194 | -------------------------------------------------------------------------------- /tests/lib/prove.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/prove.py: Stuff happening during test(s) 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import base64 33 | import os 34 | import time 35 | import zlib 36 | 37 | from .const import ( 38 | TEST_FS_LOGGEDFS, 39 | TEST_LOG_HEAD, 40 | TEST_LOG_STATS 41 | ) 42 | from .mount import is_path_mountpoint 43 | from .procio import ( 44 | append_to_file, 45 | format_yaml, 46 | read_file, 47 | run_command, 48 | write_file 49 | ) 50 | 51 | import tap.parser 52 | 53 | 54 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 55 | # CLASS: (2/3) PROVE 56 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 57 | 58 | class fstest_prove_class: 59 | 60 | 61 | def prove(self, test_path): 62 | """Called from mountpoint! 63 | """ 64 | 65 | if self.fs_type == TEST_FS_LOGGEDFS: 66 | assert is_path_mountpoint(self.mount_child_abs_path) 67 | 68 | append_to_file(self.fstest_log_abs_path, test_path + '\n') 69 | 70 | status, out, err = self.__run_fstest__(test_path) 71 | ( 72 | len_expected, 73 | len_passed, 74 | len_passed_todo, 75 | len_failed, 76 | len_failed_todo, 77 | res_dict 78 | ) = self.__process_raw_results__(out) 79 | 80 | grp_dir, grp_nr = self.__get_group_id_from_path__(test_path) 81 | grp_code = read_file(test_path) 82 | grp_log = read_file(self.loggedfs_log_abs_path) 83 | 84 | pass_condition = all([ 85 | status, # ASSERT BELOW! 86 | len_failed == 0, # ASSERT BELOW! 87 | len_expected == (len_passed + len_passed_todo + len_failed + len_failed_todo), # ASSERT BELOW! 88 | len_expected != 0, # ASSERT BELOW! 89 | 'Traceback (most recent call last)' not in grp_log # ASSERT BELOW! 90 | ]) 91 | # pass_condition_err = err.strip() == '' 92 | 93 | if pass_condition: # and pass_condition_err: 94 | self.__clear_loggedfs_log__() 95 | assert True # Test is good, nothing more to do 96 | return # Get out of here ... 97 | 98 | report = [] 99 | 100 | report.append(TEST_LOG_HEAD % 'TEST SUITE LOG') 101 | report.append(test_path) 102 | report.append(TEST_LOG_STATS % (len_expected, len_passed, len_failed)) 103 | report.append(format_yaml(res_dict)) 104 | 105 | report.append(TEST_LOG_HEAD % 'TEST SUITE LOG RAW') 106 | report.append(out) 107 | 108 | if err.strip() != '': 109 | report.append(TEST_LOG_HEAD % 'TEST SUITE ERR') 110 | report.append(err) 111 | 112 | report.append(TEST_LOG_HEAD % 'TEST SUITE CODE') 113 | report.append(grp_code) 114 | 115 | report.append(TEST_LOG_HEAD % 'LOGGEDFS LOG') 116 | report.append(grp_log) 117 | 118 | report_fn = 'test_%s_%02d_err.log' % (grp_dir, grp_nr) 119 | report_str = '\n'.join(report) 120 | 121 | write_file(os.path.join(self.logs_abs_path, report_fn), report_str) 122 | 123 | print('=== %s ===' % report_fn) 124 | print('len_expected = {:d}\nlen_passed = {:d}\nlen_passed_todo = {:d}\nlen_failed = {:d}\nlen_failed_todo = {:d}'.format( 125 | len_expected, len_passed, len_passed_todo, len_failed, len_failed_todo 126 | )) 127 | report_str = base64.encodebytes(zlib.compress(report_str.encode('utf-8'), 7)).decode('utf-8') # compress level 7 (high) 128 | for line in report_str.split('\n'): 129 | print(line) 130 | time.sleep(0.002) # HACK avoid Travis buffering issues 131 | 132 | self.__clear_loggedfs_log__() 133 | 134 | assert status # ASSERT ABOVE! 135 | assert len_failed == 0 # ASSERT ABOVE! 136 | assert len_expected == (len_passed + len_passed_todo + len_failed + len_failed_todo) # ASSERT ABOVE! 137 | assert len_expected != 0 # ASSERT ABOVE! 138 | assert 'Traceback (most recent call last)' not in grp_log # ASSERT ABOVE! 139 | 140 | 141 | def __clear_loggedfs_log__(self): 142 | 143 | run_command(['truncate', '-s', '0', self.loggedfs_log_abs_path], sudo = self.with_sudo) 144 | 145 | 146 | def __get_group_id_from_path__(self, in_path): 147 | 148 | group_path, group_nr = os.path.split(in_path) 149 | _, group_dir = os.path.split(group_path) 150 | 151 | return (group_dir, int(group_nr.split('.')[0])) 152 | 153 | 154 | def __process_raw_results__(self, in_str): 155 | 156 | ret_dict = {} 157 | tap_parser = tap.parser.Parser() 158 | tap_lines_generator = tap_parser.parse_text(in_str) 159 | 160 | len_passed = 0 161 | len_passed_todo = 0 162 | len_failed = 0 163 | len_failed_todo = 0 164 | len_expected = 0 165 | for line in tap_lines_generator: 166 | if line.category == 'plan': 167 | len_expected = line.expected_tests 168 | continue 169 | if not hasattr(line, 'ok'): 170 | continue 171 | if line.number is None: 172 | continue 173 | if line.todo: 174 | if line.ok: 175 | len_passed_todo += 1 176 | else: 177 | len_failed_todo += 1 178 | else: 179 | if line.ok: 180 | len_passed += 1 181 | else: 182 | len_failed += 1 183 | ret_dict.update({line.number: { 184 | 'ok': line.ok, 185 | 'description' : line.description 186 | }}) 187 | 188 | return len_expected, len_passed, len_passed_todo, len_failed, len_failed_todo, ret_dict 189 | 190 | 191 | def __run_fstest__(self, abs_test_path): 192 | 193 | return run_command( 194 | # ['prove', '-v', abs_test_path], 195 | ['bash', abs_test_path], 196 | return_output = True, sudo = self.with_sudo, timeout = 120, setsid = True 197 | ) 198 | -------------------------------------------------------------------------------- /tests/lib/procio.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/procio.py: Library routines, processes, I/O, ... 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | import signal 34 | import subprocess 35 | import urllib.request 36 | 37 | import psutil 38 | from yaml import load, dump 39 | try: 40 | from yaml import CLoader as Loader, CDumper as Dumper 41 | except ImportError: 42 | from yaml import Loader, Dumper 43 | 44 | from .const import TEST_FS_EXT4 45 | 46 | 47 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 48 | # ROUTINES: SHELL OUT 49 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 50 | 51 | def kill_proc(pid, k_signal = signal.SIGINT, entire_group = False, sudo = False): 52 | 53 | if not sudo: 54 | if entire_group: 55 | os.killpg(os.getpgid(pid), k_signal) 56 | else: 57 | os.kill(pid, k_signal) 58 | else: 59 | if entire_group: 60 | run_command(['kill', '-%d' % k_signal, '--', '-%d' % os.getpgid(pid)], sudo = sudo) 61 | else: 62 | run_command(['kill', '-%d' % k_signal, '%d' % pid], sudo = sudo) 63 | 64 | 65 | def run_command( 66 | cmd_list, return_output = False, sudo = False, sudo_env = False, timeout = None, setsid = False, return_status_code = False 67 | ): 68 | 69 | cmd_prefix = [] 70 | 71 | if sudo: 72 | cmd_prefix.append('sudo') 73 | if setsid: 74 | cmd_prefix.append('-b') 75 | if sudo_env: 76 | cmd_prefix.append('env') 77 | cmd_prefix.append('%s=%s' % ('VIRTUAL_ENV', os.environ['VIRTUAL_ENV'])) 78 | cmd_prefix.append('%s=%s:%s' % ('PATH', os.path.join(os.environ['VIRTUAL_ENV'], 'bin'), os.environ['PATH'])) 79 | elif setsid: 80 | cmd_prefix.append('setsid') # TODO untested codepath 81 | 82 | full_cmd = cmd_prefix + cmd_list 83 | 84 | proc = subprocess.Popen( 85 | full_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE 86 | ) 87 | 88 | timeout_alert = '' 89 | if timeout is not None: 90 | try: 91 | outs, errs = proc.communicate(timeout = timeout) 92 | except subprocess.TimeoutExpired: 93 | timeout_alert = '\n\nTEST_LIB: COMMAND TIMED OUT AND WAS KILLED!' 94 | if setsid: 95 | kill_pid = __get_pid__(full_cmd) # proc.pid will deliver wrong pid! 96 | else: 97 | kill_pid = proc.pid 98 | kill_proc(kill_pid, k_signal = signal.SIGINT, entire_group = setsid, sudo = sudo) 99 | outs, errs = proc.communicate() # (proc.stdout.read(), proc.stderr.read()) 100 | else: 101 | outs, errs = proc.communicate() 102 | 103 | status_value = proc.returncode if return_status_code else not bool(proc.returncode) 104 | 105 | if return_output: 106 | return (status_value, outs.decode('utf-8'), errs.decode('utf-8') + timeout_alert) 107 | return status_value 108 | 109 | 110 | def __get_pid__(cmd_line_list): 111 | 112 | for pid in psutil.pids(): 113 | proc = psutil.Process(pid) 114 | if cmd_line_list == proc.cmdline(): 115 | return proc.pid 116 | return None 117 | 118 | 119 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 120 | # ROUTINES: FILESYSTEM 121 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 122 | 123 | def ck_filesystem(filename, file_system = TEST_FS_EXT4): 124 | 125 | assert file_system == TEST_FS_EXT4 # TODO add support for other filesystems? 126 | return run_command( 127 | ['fsck.ext4', '-f', '-F', '-n', '-v', filename], 128 | return_output = True, sudo = True, return_status_code = True 129 | ) 130 | 131 | 132 | def create_zero_file(filename, size_in_mb): 133 | 134 | assert not os.path.isfile(filename) 135 | status = run_command(['dd', 'if=/dev/zero', 'of=%s' % filename, 'bs=1M', 'count=%d' % size_in_mb]) 136 | assert status 137 | assert os.path.isfile(filename) 138 | 139 | 140 | def mk_filesystem(filename, file_system = TEST_FS_EXT4): 141 | 142 | assert file_system == TEST_FS_EXT4 # TODO add support for other filesystems? 143 | # https://github.com/pjd/pjdfstest/issues/24 144 | # Block size must be equal or greater than PATH_MAX or symlink/03.t tests 1&2 fail 145 | status = run_command( 146 | ['mke2fs', '-b', '4096', '-I', '256', '-t', file_system, '-E', 'lazy_itable_init=0', '-O', '^has_journal', filename], 147 | sudo = True 148 | ) 149 | assert status 150 | 151 | 152 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 153 | # ROUTINES: I/O 154 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 155 | 156 | def append_to_file(filename, data): 157 | 158 | f = open(filename, 'a') 159 | f.write(data) 160 | f.close() 161 | 162 | 163 | def format_yaml(data): 164 | 165 | return dump(data, Dumper = Dumper, default_flow_style = False) 166 | 167 | 168 | def download_file(in_url): 169 | 170 | req = urllib.request.urlopen(in_url) 171 | data = req.read() 172 | req.close() 173 | 174 | return data 175 | 176 | 177 | def dump_yaml(filename, data): 178 | 179 | f = open(filename, 'w') 180 | dump(data, f, Dumper = Dumper, default_flow_style = False) 181 | f.close() 182 | 183 | 184 | def load_yaml(filename): 185 | 186 | f = open(filename, 'r') 187 | data = load(f) 188 | f.close() 189 | return data 190 | 191 | 192 | def read_file(filename): 193 | 194 | f = open(filename, 'r') 195 | data = f.read() 196 | f.close() 197 | return data 198 | 199 | 200 | def write_file(filename, data): 201 | 202 | f = open(filename, 'w') 203 | f.write(data) 204 | f.close() 205 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. |build_master| image:: https://img.shields.io/travis/pleiszenburg/loggedfs-python/master.svg?style=flat-square 2 | :target: https://travis-ci.org/pleiszenburg/loggedfs-python 3 | :alt: Build Status: master / release 4 | 5 | .. |build_develop| image:: https://img.shields.io/travis/pleiszenburg/loggedfs-python/develop.svg?style=flat-square 6 | :target: https://travis-ci.org/pleiszenburg/loggedfs-python 7 | :alt: Build Status: development branch 8 | 9 | .. |license| image:: https://img.shields.io/pypi/l/loggedfs.svg?style=flat-square 10 | :target: https://github.com/pleiszenburg/loggedfs/blob/master/LICENSE 11 | :alt: Project License: Apache License v2 12 | 13 | .. |status| image:: https://img.shields.io/pypi/status/loggedfs.svg?style=flat-square 14 | :target: https://github.com/pleiszenburg/loggedfs-python/milestone/1 15 | :alt: Project Development Status 16 | 17 | .. |pypi_version| image:: https://img.shields.io/pypi/v/loggedfs.svg?style=flat-square 18 | :target: https://pypi.python.org/pypi/loggedfs 19 | :alt: Available on PyPi - the Python Package Index 20 | 21 | .. |pypi_versions| image:: https://img.shields.io/pypi/pyversions/loggedfs.svg?style=flat-square 22 | :target: https://pypi.python.org/pypi/loggedfs 23 | :alt: Available on PyPi - the Python Package Index 24 | 25 | .. |loggedfs_python_logo| image:: http://www.pleiszenburg.de/loggedfs-python_logo.png 26 | :target: https://github.com/pleiszenburg/loggedfs-python 27 | :alt: LoggedFS-python repository 28 | 29 | |build_master| |build_develop| |license| |status| |pypi_version| |pypi_versions| 30 | 31 | |loggedfs_python_logo| 32 | 33 | Synopsis 34 | ======== 35 | 36 | LoggedFS-python is a FUSE-based filesystem which can log every operation that happens in it. 37 | It is a pure Python re-implementation of `LoggedFS`_ by `Rémi Flament`_ maintaining CLI compatibility. 38 | The project is heavily inspired by `Stavros Korokithakis`_' 2013 blog post entitled 39 | "`Writing a FUSE filesystem in Python`_" (`source code repository`_). 40 | The filesystem is fully `POSIX`_ compliant, passing the `pjdfstest test-suite`_, a descendant of FreeBSD's `fstest`_. 41 | It furthermore passes stress tests with fsx-linux based on the `fsx-flavor`_ released by the `Linux Test Project`_. 42 | It is intended to be suitable for production systems. 43 | 44 | .. _LoggedFS: https://github.com/rflament/loggedfs 45 | .. _Rémi Flament: https://github.com/rflament 46 | .. _Stavros Korokithakis: https://github.com/skorokithakis 47 | .. _Writing a FUSE filesystem in Python: https://www.stavros.io/posts/python-fuse-filesystem/ 48 | .. _source code repository: https://github.com/skorokithakis/python-fuse-sample 49 | .. _POSIX: https://en.wikipedia.org/wiki/POSIX 50 | .. _pjdfstest test-suite: https://github.com/pjd/pjdfstest 51 | .. _fstest: https://github.com/zfsonlinux/fstest 52 | .. _fsx-flavor: http://codemonkey.org.uk/projects/fsx/ 53 | .. _Linux Test Project: https://github.com/linux-test-project/ltp 54 | 55 | 56 | CAVEATS 57 | ======= 58 | 59 | * PROJECT STATUS: **BETA** 60 | * THE FILESYSTEM IS CURRENTLY **ONLY** BEING DEVELOPED FOR AND TESTED ON **LINUX**. 61 | ANYONE INTERESTED IN CONFIRMING MAC OS X AND/OR ADDING BSD SUPPORT? 62 | 63 | 64 | Installation 65 | ============ 66 | 67 | From the `Python Package Index`_ (PyPI): 68 | 69 | .. code:: bash 70 | 71 | pip install loggedfs 72 | 73 | From GitHub: 74 | 75 | .. code:: bash 76 | 77 | pip install git+https://github.com/pleiszenburg/loggedfs-python.git@master 78 | 79 | **Supports Python 3.{5,6,7,8}.** 80 | 81 | **Supports Linux.** 82 | Support for MAC OS X and BSD is implemented but has yet not been tested. 83 | 84 | .. _Python Package Index: https://pypi.org/ 85 | 86 | 87 | Simple usage example 88 | ==================== 89 | 90 | To start recording access to ``/tmp/TEST`` into ``/root/log.txt``, just do: 91 | 92 | .. code:: bash 93 | 94 | sudo loggedfs -p -s -l /root/log.txt /tmp/TEST 95 | 96 | To stop recording, just unmount as usual: 97 | 98 | .. code:: bash 99 | 100 | sudo fusermount -u /tmp/TEST 101 | 102 | 103 | CLI usage 104 | ========= 105 | 106 | .. code:: bash 107 | 108 | loggedfs --help 109 | Usage: loggedfs [OPTIONS] DIRECTORY 110 | 111 | Options: 112 | -f Do not start as a daemon. Write logs to stdout 113 | if no log file is specified. 114 | 115 | -p Allow every user to see the new loggedfs. 116 | -c FILENAME Use the "config-file" to filter what you want 117 | to log. 118 | 119 | -s Deactivate logging to syslog. 120 | -l FILE Use the "log-file" to write logs to. 121 | -j, --json Format output as JSON instead of traditional 122 | loggedfs format. 123 | 124 | -b, --buffers Include read/write-buffers (compressed, 125 | BASE64) in log. 126 | 127 | -m, --only-modify-operations Exclude logging of all operations that can not 128 | cause changes in the filesystem. Convenience 129 | flag for accelerated logging. 130 | 131 | --help Show this message and exit. 132 | 133 | 134 | Configuration 135 | ============= 136 | 137 | LoggedFS-python can use an XML configuration file if you want it to log operations only for certain files, for certain users, or for certain operations. LoggedFS-python is fully compatible with configuration files in LoggedFS' original format. Yet it can also handle additional fields (e.g. the ``command`` field). 138 | 139 | Here is a sample configuration file : 140 | 141 | .. code:: xml 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | This configuration can be used to log everything except if it concerns a 157 | ``*.bak`` file, or if the ``uid`` is 1000, or if the operation is ``getattr``. 158 | 159 | 160 | Need help? 161 | ========== 162 | 163 | Feel free to post questions in the `GitHub issue tracker`_ of this project. 164 | 165 | .. _GitHub issue tracker: https://github.com/pleiszenburg/loggedfs-python/issues 166 | 167 | 168 | Bugs & issues 169 | ============= 170 | 171 | Please report bugs in LoggedFS-python here in its `GitHub issue tracker`_. 172 | 173 | 174 | Miscellaneous 175 | ============= 176 | 177 | - Library documentation: `LoggedFS-python Jupyter Notebook`_ 178 | - `License`_ (**Apache License 2.0**) 179 | - `Contributing`_ (**Contributions are highly welcomed!**) 180 | - `Authors`_ 181 | - `Changes`_ 182 | - `Long-term ideas`_ 183 | - `Upstream issues`_ (relevant bugs in dependencies) 184 | 185 | .. _LoggedFS-python Jupyter Notebook: https://github.com/pleiszenburg/loggedfs-python/blob/master/docs/library_demo.ipynb 186 | .. _License: https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 187 | .. _Contributing: https://github.com/pleiszenburg/loggedfs-python/blob/master/CONTRIBUTING.rst 188 | .. _Authors: https://github.com/pleiszenburg/loggedfs-python/blob/master/AUTHORS.rst 189 | .. _Changes: https://github.com/pleiszenburg/loggedfs-python/blob/master/CHANGES.rst 190 | .. _Long-term ideas: https://github.com/pleiszenburg/loggedfs-python/milestone/2 191 | .. _Upstream issues: https://github.com/pleiszenburg/loggedfs-python/issues?q=is%3Aissue+is%3Aopen+label%3Aupstream 192 | -------------------------------------------------------------------------------- /src/loggedfs/_core/notify.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/notify.py: Notification backend - LoggedFS as a library 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import atexit 33 | import inspect 34 | import os 35 | import subprocess 36 | import sys 37 | import threading 38 | 39 | from .defaults import ( 40 | FUSE_ALLOWOTHER_DEFAULT, 41 | LOG_BUFFERS_DEFAULT, 42 | LOG_ONLYMODIFYOPERATIONS_DEFAULT 43 | ) 44 | from .filter import filter_pipeline_class 45 | from .ipc import receive, end_of_transmission 46 | 47 | 48 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 49 | # CONST 50 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 51 | 52 | # https://en.wikipedia.org/wiki/ANSI_escape_code 53 | c = { 54 | 'RESET': '\033[0;0m', 55 | 'BOLD': '\033[;1m', 56 | 'REVERSE': '\033[;7m', 57 | 'GREY': '\033[1;30m', 58 | 'RED': '\033[1;31m', 59 | 'GREEN': '\033[1;32m', 60 | 'YELLOW': '\033[1;33m', 61 | 'BLUE': '\033[1;34m', 62 | 'MAGENTA': '\033[1;35m', 63 | 'CYAN': '\033[1;36m', 64 | 'WHITE': '\033[1;37m' 65 | } 66 | 67 | 68 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 69 | # CLASS: NOTIFY 70 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 71 | 72 | class notify_class: 73 | """Simple wrapper for using LoggedFS-python as a library. 74 | Attach a method of your choice to filesystem events. 75 | """ 76 | 77 | 78 | def __init__(self, 79 | directory, 80 | consumer_out_func = None, # consumes signals 81 | consumer_err_func = None, # consumes anything on stderr 82 | post_exit_func = None, # called on exit 83 | log_filter = None, 84 | log_buffers = LOG_BUFFERS_DEFAULT, 85 | log_only_modify_operations = LOG_ONLYMODIFYOPERATIONS_DEFAULT, 86 | fuse_allowother = FUSE_ALLOWOTHER_DEFAULT, 87 | background = False # thread in background 88 | ): 89 | """Creates a filesystem notifier object. 90 | 91 | - directory: Relative or absolute path as a string 92 | - consumer_out_func: None or callable, consumes events provided as a dictionary 93 | - consumer_err_func: None or callable, consumes output from stderr 94 | - post_exit_func: None or callable, called when notifier was terminated 95 | - log_filter: None or instance of filter_pipeline_class 96 | - log_buffers: Boolean, activates logging of read and write buffers 97 | - fuse_allowother: Boolean, allows other users to see the LoggedFS filesystem 98 | - background: Boolean, starts notifier in a thread 99 | """ 100 | 101 | if log_filter is None: 102 | log_filter = filter_pipeline_class() 103 | 104 | if not isinstance(directory, str): 105 | raise TypeError('directory must be of type string') 106 | if not os.path.isdir(directory): 107 | raise ValueError('directory must be a path to an existing directory') 108 | if not os.access(directory, os.W_OK | os.R_OK): 109 | raise ValueError('not sufficient permissions on directory') 110 | 111 | if consumer_out_func is not None and not hasattr(consumer_out_func, '__call__'): 112 | raise TypeError('consumer_out_func must either be None or callable') 113 | if hasattr(consumer_out_func, '__call__'): 114 | if len(inspect.signature(consumer_out_func).parameters.keys()) != 1: 115 | raise ValueError('consumer_out_func must have one parameter') 116 | if consumer_err_func is not None and not hasattr(consumer_err_func, '__call__'): 117 | raise TypeError('consumer_err_func must either be None or callable') 118 | if hasattr(consumer_err_func, '__call__'): 119 | if len(inspect.signature(consumer_err_func).parameters.keys()) != 1: 120 | raise ValueError('consumer_err_func must have one parameter') 121 | if post_exit_func is not None and not hasattr(post_exit_func, '__call__'): 122 | raise TypeError('post_exit_func must either be None or callable') 123 | if not isinstance(log_filter, filter_pipeline_class): 124 | raise TypeError('log_filter must either be None or of type filter_pipeline_class') 125 | if not isinstance(log_buffers, bool): 126 | raise TypeError('log_buffers must be of type bool') 127 | if not isinstance(log_only_modify_operations, bool): 128 | raise TypeError('log_only_modify_operations must be of type bool') 129 | if not isinstance(fuse_allowother, bool): 130 | raise TypeError('fuse_allowother must be of type bool') 131 | if not isinstance(background, bool): 132 | raise TypeError('background must be of type bool') 133 | 134 | self._directory = os.path.abspath(directory) 135 | self._post_exit_func = post_exit_func 136 | self._consumer_out_func = consumer_out_func 137 | self._consumer_err_func = consumer_err_func 138 | self._log_filter = log_filter 139 | self._log_buffers = log_buffers 140 | self._log_only_modify_operations = log_only_modify_operations 141 | self._fuse_allowother = fuse_allowother 142 | self._background = background 143 | 144 | self._up = True 145 | 146 | command = ['loggedfs', 147 | '-f', # foreground 148 | '-s', # no syslog 149 | '--lib' # "hidden" library mode 150 | ] 151 | if self._log_buffers: 152 | command.append('-b') # also log read and write buffers 153 | if self._log_only_modify_operations: 154 | command.append('-m') 155 | if self._fuse_allowother: 156 | command.append('-p') 157 | command.append(self._directory) 158 | 159 | args = (command, self._handle_stdout, self._handle_stderr, self._handle_exit) 160 | 161 | atexit.register(self.terminate) 162 | if self._background: 163 | self._t = threading.Thread(target = receive, args = args, daemon = False) 164 | self._t.start() 165 | else: 166 | receive(*args) 167 | 168 | 169 | def _handle_stderr(self, msg): 170 | 171 | if self._consumer_err_func is not None: 172 | self._consumer_err_func(msg) 173 | else: 174 | sys.stderr.write(c['RED'] + str(msg).rstrip('\n') + c['RESET'] + '\n') 175 | sys.stderr.flush() 176 | 177 | 178 | def _handle_stdout(self, msg): 179 | 180 | if not isinstance(msg, end_of_transmission): 181 | if not self._log_filter.match(msg): 182 | return 183 | 184 | if self._consumer_out_func is not None: 185 | self._consumer_out_func(msg) 186 | else: 187 | sys.stdout.write(c['GREEN'] + str(msg) + c['RESET'] + '\n') 188 | sys.stdout.flush() 189 | 190 | 191 | def _handle_exit(self): 192 | 193 | if self._post_exit_func is not None: 194 | self._post_exit_func() 195 | 196 | 197 | def terminate(self): 198 | 199 | if not self._up: 200 | return 201 | 202 | self._up = False 203 | 204 | command = ['fusermount', '-u', self._directory] 205 | proc = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE) 206 | proc.wait() 207 | 208 | if self._background: 209 | self._t.join() 210 | -------------------------------------------------------------------------------- /src/loggedfs/_core/out.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/out.py: Log output formatting and filtering 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import base64 33 | import errno 34 | from functools import wraps 35 | import grp 36 | import inspect 37 | import json 38 | import pwd 39 | import zlib 40 | 41 | from refuse.high import ( 42 | fuse_get_context, 43 | FuseOSError, 44 | ) 45 | 46 | from .ipc import send 47 | from .log import log_msg 48 | from .timing import time 49 | 50 | 51 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 52 | # CONST 53 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 54 | 55 | STATUS_DICT = { 56 | True: 'SUCCESS', 57 | False: 'FAILURE' 58 | } 59 | 60 | ATTR_NAME = '__name__' 61 | ATTR_ERRNO = 'errno' 62 | 63 | NAME_OSERROR = 'OSError' 64 | NAME_FUSEOSERROR = 'FuseOSError' 65 | NAME_UNKNOWN = 'Unknown Exception' 66 | 67 | ERROR_STAGE1 = 'UNEXPECTED in operation stage (1)' 68 | ERROR_STAGE2 = 'UNEXPECTED in log stage (2)' 69 | 70 | 71 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 72 | # ROUTINES 73 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 74 | 75 | def event(format_pattern = ''): 76 | 77 | def wrapper(func): 78 | 79 | _func_param = inspect.signature(func).parameters 80 | func_arg_names = tuple(_func_param.keys())[1:] 81 | func_arg_abspath = func_arg_names[[item.endswith('path') for item in func_arg_names].index(True)] 82 | func_arg_defaults = { 83 | k: _func_param[k].default 84 | for k in func_arg_names 85 | if _func_param[k].default != inspect._empty 86 | } 87 | del _func_param 88 | 89 | @wraps(func) 90 | def wrapped(self, *func_args, **func_kwargs): 91 | 92 | ret_value = None 93 | ret_status = False 94 | try: 95 | ret_value = func(self, *func_args, **func_kwargs) 96 | ret_status = True 97 | except FuseOSError as e: 98 | ret_value = (NAME_FUSEOSERROR, e.errno) 99 | raise e 100 | except OSError as e: 101 | ret_value = (NAME_OSERROR, e.errno) 102 | raise FuseOSError(e.errno) 103 | except Exception as e: 104 | if hasattr(e, ATTR_ERRNO): # all subclasses of OSError 105 | ret_value = (getattr(type(e), ATTR_NAME, NAME_UNKNOWN), e.errno) 106 | raise FuseOSError(e.errno) 107 | else: 108 | self._logger.exception(log_msg(self._log_json, ERROR_STAGE1)) 109 | raise e 110 | else: 111 | return ret_value 112 | finally: 113 | try: 114 | _log_event_( 115 | self, 116 | func, func_arg_names, func_arg_abspath, func_arg_defaults, func_args, func_kwargs, 117 | format_pattern, 118 | ret_status, ret_value 119 | ) 120 | except Exception as e: 121 | self._logger.exception(log_msg(self._log_json, ERROR_STAGE2)) 122 | raise e 123 | 124 | return wrapped 125 | 126 | return wrapper 127 | 128 | 129 | def decode_buffer(in_buffer): 130 | 131 | if not isinstance(in_buffer, str): 132 | raise TypeError('in_buffer must be a string') 133 | 134 | return zlib.decompress(base64.b64decode(in_buffer.encode('utf-8'))) 135 | 136 | 137 | def _encode_buffer_(in_bytes): 138 | 139 | return base64.b64encode(zlib.compress(in_bytes, 1)).decode('utf-8') # compress level 1 (weak) 140 | 141 | 142 | def _get_fh_from_fip_(fip): 143 | 144 | if fip is None: 145 | return -1 146 | if not hasattr(fip, 'fh'): 147 | return -2 148 | if not isinstance(fip.fh, int): 149 | return -3 150 | return fip.fh 151 | 152 | 153 | def _get_group_name_from_gid_(gid): 154 | 155 | try: 156 | return grp.getgrgid(gid).gr_name 157 | except KeyError: 158 | return '[gid: omitted argument]' 159 | 160 | 161 | def _get_process_cmdline_(pid): 162 | 163 | try: 164 | with open('/proc/%d/cmdline' % pid, 'r') as f: # TODO encoding, bytes? 165 | cmdline = f.read() 166 | return cmdline.replace('\x00', ' ').strip() 167 | except FileNotFoundError: 168 | return '' 169 | 170 | 171 | def _get_user_name_from_uid_(uid): 172 | 173 | try: 174 | return pwd.getpwuid(uid).pw_name 175 | except KeyError: 176 | return '[uid: omitted argument]' 177 | 178 | 179 | def _log_event_( 180 | self, 181 | func, func_arg_names, func_arg_abspath, func_arg_defaults, func_args, func_kwargs, 182 | format_pattern, 183 | ret_status, ret_value 184 | ): 185 | 186 | if self._log_only_modify_operations: 187 | if func.__name__ not in ( 188 | 'chmod', 189 | 'chown', 190 | 'link', 191 | 'mkdir', 192 | 'mknod', 193 | 'rename', 194 | 'rmdir', 195 | 'symlink', 196 | 'truncate', 197 | 'unlink', 198 | 'utimens', 199 | 'write' 200 | ): 201 | return 202 | 203 | uid, gid, pid = fuse_get_context() 204 | 205 | p_cmdname = '' 206 | if self._log_printprocessname or self._log_json: 207 | p_cmdname = _get_process_cmdline_(pid).strip() 208 | 209 | log_dict = { 210 | 'proc_cmd': p_cmdname, 211 | 'proc_uid': uid, 212 | 'proc_uid_name': _get_user_name_from_uid_(uid), 213 | 'proc_gid': gid, 214 | 'proc_gid_name': _get_group_name_from_gid_(gid), 215 | 'proc_pid': pid, 216 | 'action': func.__name__, 217 | 'status': ret_status, 218 | } 219 | 220 | arg_dict = { 221 | arg_name: arg 222 | for arg_name, arg in zip(func_arg_names, func_args) 223 | } 224 | arg_dict.update({ 225 | arg_name: func_kwargs.get(arg_name, func_arg_defaults[arg_name]) 226 | for arg_name in func_arg_names[len(func_args):] 227 | }) 228 | 229 | try: 230 | arg_dict['uid_name'] = _get_user_name_from_uid_(arg_dict['uid']) 231 | except KeyError: 232 | pass 233 | try: 234 | arg_dict['gid_name'] = _get_group_name_from_gid_(arg_dict['gid']) 235 | except KeyError: 236 | pass 237 | try: 238 | arg_dict['fip'] = _get_fh_from_fip_(arg_dict['fip']) 239 | except KeyError: 240 | pass 241 | for k in arg_dict.keys(): 242 | if k.endswith('path'): 243 | arg_dict[k] = self._full_path(arg_dict[k]) 244 | try: 245 | arg_dict['buf_len'] = len(arg_dict['buf']) 246 | arg_dict['buf'] = _encode_buffer_(arg_dict['buf']) if self._log_buffers else '' 247 | except KeyError: 248 | pass 249 | 250 | log_dict.update({'param_%s' % k: v for k, v in arg_dict.items()}) 251 | 252 | if log_dict['status']: # SUCCESS 253 | if any(( 254 | ret_value is None, 255 | isinstance(ret_value, int), 256 | isinstance(ret_value, str), 257 | isinstance(ret_value, dict), 258 | isinstance(ret_value, list) 259 | )): 260 | log_dict['return'] = ret_value 261 | elif isinstance(ret_value, bytes): 262 | log_dict['return_len'] = len(ret_value) 263 | log_dict['return'] = _encode_buffer_(ret_value) if self._log_buffers else '' 264 | 265 | else: # FAILURE 266 | log_dict.update({ 267 | 'return': None, 268 | 'return_exception': ret_value[0], 269 | 'return_errno': ret_value[1], 270 | 'return_errorcode': errno.errorcode[ret_value[1]] 271 | }) 272 | 273 | if not self._lib_mode: 274 | if not self._log_filter.match(log_dict): 275 | return 276 | 277 | if self._log_json and not self._lib_mode: 278 | self._logger.info( json.dumps(log_dict, sort_keys = True)[1:-1] ) 279 | return 280 | elif self._lib_mode: 281 | log_dict['time'] = time.time_ns() 282 | send(log_dict) 283 | return 284 | 285 | log_out = ' '.join([ 286 | '%s %s' % (func.__name__, format_pattern.format(**log_dict)), 287 | '{%s}' % STATUS_DICT[ret_status], 288 | '[ pid = %d %suid = %d ]' % (pid, ('%s ' % p_cmdname) if len(p_cmdname) > 0 else '', uid), 289 | '( r = %s )' % str(log_dict['return']) 290 | if log_dict['status'] else 291 | '( %s = %d )' % ret_value 292 | ]) 293 | 294 | self._logger.info(log_out) 295 | -------------------------------------------------------------------------------- /tests/lib/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | tests/lib/base.py: Base class for test infrastructure 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import os 33 | import shutil 34 | import time 35 | 36 | from .const import ( 37 | TEST_FS_EXT4, 38 | TEST_FS_LOGGEDFS, 39 | TEST_FSCK_FN, 40 | TEST_FSTEST_LOG_FN, 41 | TEST_IMAGE_FN, 42 | TEST_IMAGE_SIZE_MB, 43 | TEST_MOUNT_CHILD_PATH, 44 | TEST_MOUNT_PARENT_PATH, 45 | TEST_LOG_HEAD, 46 | TEST_LOG_PATH, 47 | TEST_ROOT_PATH, 48 | TEST_LOGGEDFS_CFG_FN, 49 | TEST_LOGGEDFS_ERR_FN, 50 | TEST_LOGGEDFS_LOG_FN, 51 | TEST_LOGGEDFS_OUT_FN, 52 | ) 53 | from .mount import ( 54 | attach_loop_device, 55 | detach_loop_device, 56 | find_loop_devices, 57 | is_path_mountpoint, 58 | mount, 59 | mount_loggedfs_python, 60 | umount, 61 | umount_fuse, 62 | ) 63 | from .procio import ( 64 | ck_filesystem, 65 | create_zero_file, 66 | mk_filesystem, 67 | run_command, 68 | write_file, 69 | ) 70 | 71 | 72 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 73 | # CLASS: (1/3) PRE 74 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 75 | 76 | class fstest_base_class(): 77 | 78 | 79 | with_sudo = True 80 | fs_type = TEST_FS_LOGGEDFS 81 | travis = False 82 | 83 | 84 | def init_a_members(self, fs_type = TEST_FS_LOGGEDFS): 85 | 86 | self.fs_type = fs_type 87 | 88 | self.prj_abs_path = os.getcwd() 89 | self.root_abs_path = os.path.abspath(os.path.join(self.prj_abs_path, TEST_ROOT_PATH)) 90 | assert os.path.isdir(self.root_abs_path) 91 | 92 | self.logs_abs_path = os.path.join(self.root_abs_path, TEST_LOG_PATH) 93 | 94 | self.image_abs_path = os.path.join('/dev/shm', TEST_IMAGE_FN) 95 | 96 | self.mount_parent_abs_path = os.path.join(self.root_abs_path, TEST_MOUNT_PARENT_PATH) 97 | self.mount_child_abs_path = os.path.join(self.mount_parent_abs_path, TEST_MOUNT_CHILD_PATH) 98 | 99 | self.loggedfs_log_abs_path = os.path.join(self.logs_abs_path, TEST_LOGGEDFS_LOG_FN) 100 | self.loggedfs_cfg_abs_path = os.path.join(self.root_abs_path, TEST_LOGGEDFS_CFG_FN) 101 | 102 | if 'TRAVIS' in os.environ.keys(): 103 | self.travis = os.environ['TRAVIS'] == 'true' 104 | 105 | self.fstest_log_abs_path = os.path.join(self.logs_abs_path, TEST_FSTEST_LOG_FN) 106 | 107 | 108 | def init_b_cleanup(self): 109 | 110 | if self.travis: 111 | return 112 | 113 | self.__cleanup_logfiles__() # rm -r log_dir 114 | self.__cleanup_mountpoint__(self.mount_child_abs_path) # umount & rmdir 115 | self.__cleanup_mountpoint__(self.mount_parent_abs_path) # umount & rmdir 116 | self.__cleanup_loop_devices__() # losetup -d /dev/loopX 117 | self.__cleanup_image__() # rm file 118 | 119 | 120 | def init_c_parentfs(self): 121 | 122 | if not self.travis: 123 | create_zero_file(self.image_abs_path, TEST_IMAGE_SIZE_MB) 124 | self.__attach_loop_device__() 125 | mk_filesystem(self.loop_device_path, file_system = TEST_FS_EXT4) 126 | self.__mk_dir__(self.mount_parent_abs_path) 127 | if not self.travis: 128 | self.__mount_parent_fs__() 129 | self.__mk_dir__(self.logs_abs_path, allow_preexisting = True) 130 | 131 | 132 | def init_d_childfs(self): 133 | 134 | open(self.loggedfs_log_abs_path, 'a').close() # HACK create empty loggedfs log file 135 | self.__mk_dir__(self.mount_child_abs_path, in_fs_root = True) 136 | if self.fs_type == TEST_FS_LOGGEDFS: 137 | self.__mount_child_fs__() 138 | open(self.fstest_log_abs_path, 'a').close() # HACK create empty fstest log file 139 | 140 | 141 | def assert_childfs_mountpoint(self): 142 | 143 | assert is_path_mountpoint(self.mount_child_abs_path) 144 | 145 | 146 | def assert_parentfs_mountpoint(self): 147 | 148 | if self.travis: 149 | return 150 | 151 | assert is_path_mountpoint(self.mount_parent_abs_path) 152 | 153 | 154 | def destroy_a_childfs(self): 155 | 156 | if not self.fs_type == TEST_FS_LOGGEDFS: 157 | return 158 | 159 | if is_path_mountpoint(self.mount_child_abs_path): 160 | umount_child_status = umount_fuse(self.mount_child_abs_path, sudo = self.with_sudo) 161 | assert umount_child_status 162 | assert not is_path_mountpoint(self.mount_child_abs_path) 163 | 164 | self.__rm_tree__(self.mount_child_abs_path, in_fs_root = True) 165 | 166 | time.sleep(0.1) # HACK ... otherwise parent will be busy 167 | 168 | 169 | def destroy_b_parentfs(self): 170 | 171 | if not self.travis: 172 | umount_parent_status = umount(self.mount_parent_abs_path, sudo = True) 173 | assert umount_parent_status 174 | assert not is_path_mountpoint(self.mount_parent_abs_path) 175 | 176 | self.__rm_tree__(self.mount_parent_abs_path) 177 | 178 | if self.travis: 179 | return 180 | 181 | loop_device_list = find_loop_devices(self.image_abs_path) 182 | assert isinstance(loop_device_list, list) 183 | assert len(loop_device_list) == 1 184 | loop_device_path = loop_device_list[0] 185 | 186 | ck_status_code, ck_out, ck_err = ck_filesystem(loop_device_path) 187 | write_file( 188 | os.path.join(self.logs_abs_path, TEST_FSCK_FN), 189 | ''.join([ 190 | TEST_LOG_HEAD % 'EXIT STATUS', 191 | '%d\n' % ck_status_code, 192 | TEST_LOG_HEAD % 'OUT', 193 | ck_out, 194 | TEST_LOG_HEAD % 'ERR', 195 | ck_err 196 | ]) 197 | ) 198 | 199 | detach_status = detach_loop_device(loop_device_path) 200 | assert detach_status 201 | 202 | assert not bool(ck_status_code) # not 0 for just about any type of error! Therefore asserting at the very end. 203 | 204 | os.remove(self.image_abs_path) 205 | assert not os.path.exists(self.image_abs_path) 206 | 207 | 208 | def __attach_loop_device__(self): 209 | 210 | loop_status = attach_loop_device(self.image_abs_path) 211 | assert loop_status 212 | loop_device_list = find_loop_devices(self.image_abs_path) 213 | assert isinstance(loop_device_list, list) 214 | assert len(loop_device_list) == 1 215 | 216 | self.loop_device_path = loop_device_list[0] 217 | 218 | 219 | def __cleanup_image__(self): 220 | 221 | if os.path.isfile(self.image_abs_path): 222 | os.remove(self.image_abs_path) 223 | assert not os.path.isfile(self.image_abs_path) 224 | 225 | 226 | def __cleanup_mountpoint__(self, in_path): 227 | 228 | if is_path_mountpoint(in_path): 229 | umount_status = umount(in_path, sudo = True, force = True) 230 | if not umount_status: 231 | fumount_status = umount_fuse(in_path, sudo = True) 232 | assert not is_path_mountpoint(in_path) 233 | 234 | if os.path.isdir(in_path): 235 | assert in_path != '/' 236 | run_command(['rmdir', in_path], sudo = self.with_sudo) 237 | assert not os.path.isdir(in_path) 238 | 239 | 240 | def __cleanup_logfiles__(self): 241 | 242 | if os.path.isdir(self.logs_abs_path): 243 | assert self.logs_abs_path != '/' 244 | run_command(['rm', '-r', self.logs_abs_path], sudo = self.with_sudo) 245 | 246 | 247 | def __cleanup_loop_devices__(self): 248 | 249 | if not os.path.isfile(self.image_abs_path): 250 | assert find_loop_devices(self.image_abs_path) == [] 251 | return 252 | 253 | loop_dev_list = find_loop_devices(self.image_abs_path) 254 | assert loop_dev_list is not None 255 | 256 | for loop_dev in loop_dev_list: 257 | detach_status = detach_loop_device(loop_dev) 258 | assert detach_status 259 | 260 | assert find_loop_devices(self.image_abs_path) == [] 261 | 262 | 263 | def __mk_dir__(self, in_path, in_fs_root = False, allow_preexisting = False): 264 | 265 | if allow_preexisting: 266 | if os.path.isdir(in_path): 267 | return 268 | 269 | if not in_fs_root: 270 | os.mkdir(in_path) 271 | else: 272 | mkdir_status = run_command(['mkdir', in_path], sudo = True) 273 | assert mkdir_status 274 | chown_status = run_command(['chown', '%d:%d' % (os.getuid(), os.getgid()), in_path], sudo = True) 275 | assert chown_status 276 | 277 | assert os.path.isdir(in_path) 278 | 279 | 280 | def __mount_child_fs__(self): 281 | 282 | assert not is_path_mountpoint(self.mount_child_abs_path) 283 | 284 | loggedfs_status, loggedfs_out, loggedfs_err = mount_loggedfs_python( 285 | self.mount_child_abs_path, 286 | self.loggedfs_log_abs_path, 287 | self.loggedfs_cfg_abs_path, 288 | sudo = self.with_sudo 289 | ) 290 | 291 | write_file(os.path.join(self.root_abs_path, TEST_LOG_PATH, TEST_LOGGEDFS_OUT_FN), loggedfs_out) 292 | write_file(os.path.join(self.root_abs_path, TEST_LOG_PATH, TEST_LOGGEDFS_ERR_FN), loggedfs_err) 293 | 294 | assert loggedfs_status 295 | assert is_path_mountpoint(self.mount_child_abs_path) 296 | 297 | 298 | def __mount_parent_fs__(self): 299 | 300 | assert not is_path_mountpoint(self.mount_parent_abs_path) 301 | mount_status = mount(self.mount_parent_abs_path, self.loop_device_path) 302 | assert mount_status 303 | assert is_path_mountpoint(self.mount_parent_abs_path) 304 | 305 | 306 | def __rm_tree__(self, in_path, in_fs_root = False): 307 | 308 | if not in_fs_root: 309 | shutil.rmtree(in_path) 310 | else: 311 | rmdir_status = run_command(['rm', '-r', in_path], sudo = True) 312 | assert rmdir_status 313 | 314 | assert not os.path.exists(in_path) 315 | -------------------------------------------------------------------------------- /src/loggedfs/_core/filter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/filter.py: Filtering events by criteria 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | from collections import OrderedDict 33 | import re 34 | 35 | import xmltodict 36 | 37 | from .defaults import LOG_ENABLED_DEFAULT, LOG_PRINTPROCESSNAME_DEFAULT 38 | 39 | 40 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 41 | # FILTER FIELD CLASS 42 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 43 | 44 | class filter_field_class: 45 | 46 | 47 | def __init__(self, name, value): 48 | 49 | self._name_is_func = hasattr(name, '__call__') 50 | 51 | if not self._name_is_func and not isinstance(name, str): 52 | raise TypeError('name must either be callable or a string') 53 | if not hasattr(value, '__call__'): 54 | raise TypeError('value must either be callable') 55 | 56 | self._name = name 57 | self._value = value 58 | 59 | 60 | def __repr__(self): 61 | 62 | return '' % ( 63 | self._name if not self._name_is_func else '{callable:' + getattr(self._name, '__name__', 'NONAME') + '}', 64 | '{callable:' + getattr(self._value, '__name__', 'NONAME') + '}' 65 | ) 66 | 67 | 68 | @property 69 | def name_is_func(self): 70 | 71 | return self._name_is_func 72 | 73 | 74 | @property 75 | def name(self): 76 | 77 | return self._name 78 | 79 | 80 | @property 81 | def value(self): 82 | 83 | return self._value 84 | 85 | 86 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 87 | # FILTER ITEM CLASS 88 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 89 | 90 | class filter_item_class: 91 | 92 | 93 | def __init__(self, fields_list): 94 | 95 | if not isinstance(fields_list, list): 96 | raise TypeError('fields_list must be of type list') 97 | if len(fields_list) == 0: 98 | raise ValueError('at least one field is required for setting up a filter') 99 | if any((not isinstance(item, filter_field_class) for item in fields_list)): 100 | raise TypeError('fields_list must only contain type filter_field_class') 101 | 102 | self._fields_list = fields_list 103 | self._field_names = {field.name for field in self._fields_list if not field.name_is_func} 104 | self._field_nofuncs = {field for field in self._fields_list if not field.name_is_func} 105 | self._field_funcs = {field for field in self._fields_list if field.name_is_func} 106 | 107 | 108 | def __repr__(self): 109 | 110 | return ( 111 | '\n\t' + 112 | '\n\t'.join((repr(field) for field in self._fields_list)) 113 | + '\n' 114 | ) 115 | 116 | 117 | def match(self, event_dict): 118 | 119 | if not isinstance(event_dict, dict): 120 | raise TypeError('event_dict must be of type dict') 121 | 122 | if len(self._field_names - event_dict.keys()) > 0: 123 | return False 124 | 125 | if any((not field.value(event_dict[field.name]) for field in self._field_nofuncs)): 126 | return False 127 | 128 | for field in self._field_funcs: 129 | key_match = None 130 | for key in event_dict.keys(): 131 | if field.name(key): 132 | key_match = key 133 | break 134 | if key_match is None: 135 | return False 136 | if not field.value(event_dict[key_match]): 137 | return False 138 | 139 | return True 140 | 141 | 142 | @staticmethod 143 | def _from_xmldict(xml_dict): 144 | 145 | if not isinstance(xml_dict, OrderedDict) and not isinstance(xml_dict, dict): 146 | raise TypeError('can not construct filter item from non-dict type') 147 | if any((not isinstance(item, str) for item in xml_dict.keys())): 148 | raise TypeError('non-string key in dict') 149 | if any((not isinstance(item, str) for item in xml_dict.values())): 150 | raise TypeError('non-string value in dict') 151 | 152 | fields_list = [] 153 | 154 | try: 155 | if xml_dict['@retname'] == 'SUCCESS': 156 | fields_list.append(filter_field_class('status', lambda x: x == True)) 157 | elif xml_dict['@retname'] == 'FAILURE': 158 | fields_list.append(filter_field_class('status', lambda x: x == False)) 159 | elif xml_dict['@retname'] != '.*': 160 | raise ValueError('unexpected value for "retname"') 161 | except KeyError: 162 | pass 163 | 164 | try: 165 | if xml_dict['@extension'] != '.*': 166 | fields_list.append(filter_field_class( 167 | lambda x: x.endswith('path'), re.compile(xml_dict['@extension']).match 168 | )) 169 | except KeyError: 170 | pass 171 | 172 | try: 173 | if xml_dict['@uid'].isdecimal(): 174 | fields_list.append(filter_field_class( 175 | 'proc_uid', lambda x: x == int(xml_dict['@uid']) 176 | )) 177 | elif xml_dict['@uid'] != '*': 178 | raise ValueError('unexpected value for "uid"') 179 | except KeyError: 180 | pass 181 | 182 | try: 183 | if xml_dict['@action'] != '.*': 184 | fields_list.append(filter_field_class( 185 | 'action', re.compile(xml_dict['@action']).match 186 | )) 187 | except KeyError: 188 | pass 189 | 190 | try: 191 | if xml_dict['@command'] != '.*': 192 | fields_list.append(filter_field_class( 193 | 'proc_cmd', re.compile(xml_dict['@command']).match 194 | )) 195 | except KeyError: 196 | pass 197 | 198 | if len(fields_list) > 0: 199 | return filter_item_class(fields_list) 200 | else: 201 | return None 202 | 203 | 204 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 205 | # FILTER PIPELINE CLASS 206 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 207 | 208 | class filter_pipeline_class: 209 | 210 | 211 | VALID_XML_BLOCKS = ('@logEnabled', '@printProcessName', 'includes', 'excludes') 212 | 213 | 214 | def __init__(self, include_list = None, exclude_list = None): 215 | 216 | if include_list is None: 217 | include_list = [] 218 | if exclude_list is None: 219 | exclude_list = [] 220 | 221 | if not isinstance(include_list, list): 222 | raise TypeError('include_list must have type list') 223 | if not isinstance(exclude_list, list): 224 | raise TypeError('exclude_list must have type list') 225 | if any((not isinstance(item, filter_item_class) for item in include_list)): 226 | raise TypeError('include_list must only contain type filter_item_class') 227 | if any((not isinstance(item, filter_item_class) for item in exclude_list)): 228 | raise TypeError('exclude_list must only contain type filter_item_class') 229 | 230 | self._include_list = include_list 231 | self._exclude_list = exclude_list 232 | 233 | 234 | def __repr__(self): 235 | 236 | return ( 237 | '\n' 238 | + '\t\n' 239 | + '\n'.join( ( 240 | '\n'.join(('\t\t' + line for line in repr(item).split('\n') )) 241 | for item in self._include_list) 242 | ) + ('\n' if len(self._include_list) > 0 else '') 243 | + '\t\n' 244 | + '\t\n' 245 | + '\n'.join( ( 246 | '\n'.join(('\t\t' + line for line in repr(item).split('\n') )) 247 | for item in self._exclude_list) 248 | ) + ('\n' if len(self._exclude_list) > 0 else '') 249 | + '\t\n' 250 | + '' 251 | ) 252 | 253 | 254 | def match(self, event_dict): 255 | 256 | if not isinstance(event_dict, dict): 257 | raise TypeError('event_dict must be of type dict') 258 | 259 | if len(self._include_list) > 0: 260 | if not any((item.match(event_dict) for item in self._include_list)): 261 | return False 262 | 263 | if any((item.match(event_dict) for item in self._exclude_list)): 264 | return False 265 | 266 | return True 267 | 268 | 269 | @staticmethod 270 | def from_xmlstring(xml_str): 271 | """Parse XML configuration string and return instance of filter_pipeline_class. 272 | Compatibility layer for original LoggedFS XML configuration file format. 273 | """ 274 | 275 | if not isinstance(xml_str, str): 276 | raise TypeError('xml_str must have type str') 277 | if len(xml_str) == 0: 278 | raise ValueError('xml_str must not be empty') 279 | try: 280 | xml_dict = xmltodict.parse(xml_str) 281 | except: 282 | raise ValueError('xml_str does not contain valid XML') 283 | try: 284 | xml_dict = xml_dict['loggedFS'] 285 | except KeyError: 286 | raise ValueError('XML tree does not have loggedFS top-level tag') 287 | if len(xml_dict.keys() - set(filter_pipeline_class.VALID_XML_BLOCKS)) > 0: 288 | raise ValueError('unexpected tags and/or parameters in XML tree') 289 | 290 | log_enabled = xml_dict.pop('@logEnabled', None) 291 | if log_enabled is None: 292 | log_enabled = LOG_ENABLED_DEFAULT 293 | else: 294 | log_enabled = log_enabled.lower() == 'true' 295 | log_printprocessname = xml_dict.pop('@printProcessName', None) 296 | if log_printprocessname is None: 297 | log_printprocessname = LOG_PRINTPROCESSNAME_DEFAULT 298 | else: 299 | log_printprocessname = log_printprocessname.lower() == 'true' 300 | 301 | group_list = [] 302 | for f_type in ('includes', 'excludes'): 303 | group = xml_dict.pop(f_type, None) 304 | if group is None: 305 | group_list.append([]) 306 | continue 307 | if not isinstance(group, OrderedDict) and not isinstance(group, dict): 308 | raise TypeError('malformed XML tree for %s' % f_type) 309 | group = group.get(f_type[:-1], None) 310 | if group is None: 311 | group_list.append([]) 312 | continue 313 | if not any(( 314 | isinstance(group, list), isinstance(group, OrderedDict), isinstance(group, dict) 315 | )): 316 | raise TypeError('malformed XML tree for %s' % f_type[:-1]) 317 | if isinstance(group, list): 318 | tmp = [filter_item_class._from_xmldict(item) for item in group] 319 | tmp = [item for item in tmp if item is not None] 320 | group_list.append(tmp) 321 | else: 322 | tmp = filter_item_class._from_xmldict(group) 323 | if tmp is not None: 324 | group_list.append([tmp]) 325 | else: 326 | group_list.append([]) 327 | 328 | return log_enabled, log_printprocessname, filter_pipeline_class(*group_list) 329 | -------------------------------------------------------------------------------- /docs/library_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import re\n", 10 | "import pandas as pd\n", 11 | "\n", 12 | "import loggedfs # Python pass-through filesystem: pip install loggedfs" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "demo_filter = loggedfs.filter_pipeline_class(include_list = [loggedfs.filter_item_class([loggedfs.filter_field_class(\n", 22 | " name = 'proc_cmd', value = re.compile('.*kate.*').match\n", 23 | " )])]) # filter pipeline for filesystem actions coming from commands containing the string \"kate\" (the KDE text editor)\n", 24 | "\n", 25 | "demo_data, demo_err = [], [] # dump logging data here\n", 26 | "\n", 27 | "demo = loggedfs.loggedfs_notify(\n", 28 | " 'demo_dir', # relative path to mountpoint\n", 29 | " background = True, # log in background thread\n", 30 | " log_filter = demo_filter, # apply filter pipeline\n", 31 | " log_only_modify_operations = True, # only log operations that have the potential to modify the filesystem\n", 32 | " log_buffers = True, # include read and write buffers into log\n", 33 | " consumer_out_func = demo_data.append, # consume logging data\n", 34 | " consumer_err_func = demo_err.append # consume potential errors\n", 35 | " ) # MOUNT (and start logging)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 3, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "data": { 45 | "text/plain": [ 46 | "27" 47 | ] 48 | }, 49 | "execution_count": 3, 50 | "metadata": {}, 51 | "output_type": "execute_result" 52 | } 53 | ], 54 | "source": [ 55 | "demo.terminate() # UMOUNT (and stop logging)\n", 56 | "\n", 57 | "assert isinstance(demo_data[-1], loggedfs.end_of_transmission)\n", 58 | "demo_data = demo_data[:-1]\n", 59 | "\n", 60 | "len(demo_data)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 4, 66 | "metadata": {}, 67 | "outputs": [ 68 | { 69 | "data": { 70 | "text/html": [ 71 | "
\n", 72 | "\n", 85 | "\n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | "
actionproc_pidparam_pathparam_old_pathparam_new_pathparam_fipparam_buf_lenparam_offsetparam_modereturnstatus
time
1556736520813215826mknod30977/tmp/ernst/demo_dir/.demo_file.txt.kate-swpNaNNaNNaNNaNNaN33188.0NaNTrue
1556736520814149530chmod30977/tmp/ernst/demo_dir/.demo_file.txt.kate-swpNaNNaNNaNNaNNaN33152.0NaNTrue
1556736524870363873mknod30977/tmp/ernst/demo_dir/demo_file.txt.X30977NaNNaNNaNNaNNaN33152.0NaNTrue
1556736524871304006chmod30977/tmp/ernst/demo_dir/demo_file.txt.X30977NaNNaNNaNNaNNaN33188.0NaNTrue
1556736524872197836rename30977NaN/tmp/ernst/demo_dir/demo_file.txt.X30977/tmp/ernst/demo_dir/demo_file.txtNaNNaNNaNNaNNaNTrue
1556736524872611868unlink30977/tmp/ernst/demo_dir/.demo_file.txt.kate-swpNaNNaNNaNNaNNaNNaNNaNTrue
\n", 203 | "
" 204 | ], 205 | "text/plain": [ 206 | " action proc_pid \\\n", 207 | "time \n", 208 | "1556736520813215826 mknod 30977 \n", 209 | "1556736520814149530 chmod 30977 \n", 210 | "1556736524870363873 mknod 30977 \n", 211 | "1556736524871304006 chmod 30977 \n", 212 | "1556736524872197836 rename 30977 \n", 213 | "1556736524872611868 unlink 30977 \n", 214 | "\n", 215 | " param_path \\\n", 216 | "time \n", 217 | "1556736520813215826 /tmp/ernst/demo_dir/.demo_file.txt.kate-swp \n", 218 | "1556736520814149530 /tmp/ernst/demo_dir/.demo_file.txt.kate-swp \n", 219 | "1556736524870363873 /tmp/ernst/demo_dir/demo_file.txt.X30977 \n", 220 | "1556736524871304006 /tmp/ernst/demo_dir/demo_file.txt.X30977 \n", 221 | "1556736524872197836 NaN \n", 222 | "1556736524872611868 /tmp/ernst/demo_dir/.demo_file.txt.kate-swp \n", 223 | "\n", 224 | " param_old_path \\\n", 225 | "time \n", 226 | "1556736520813215826 NaN \n", 227 | "1556736520814149530 NaN \n", 228 | "1556736524870363873 NaN \n", 229 | "1556736524871304006 NaN \n", 230 | "1556736524872197836 /tmp/ernst/demo_dir/demo_file.txt.X30977 \n", 231 | "1556736524872611868 NaN \n", 232 | "\n", 233 | " param_new_path param_fip \\\n", 234 | "time \n", 235 | "1556736520813215826 NaN NaN \n", 236 | "1556736520814149530 NaN NaN \n", 237 | "1556736524870363873 NaN NaN \n", 238 | "1556736524871304006 NaN NaN \n", 239 | "1556736524872197836 /tmp/ernst/demo_dir/demo_file.txt NaN \n", 240 | "1556736524872611868 NaN NaN \n", 241 | "\n", 242 | " param_buf_len param_offset param_mode return status \n", 243 | "time \n", 244 | "1556736520813215826 NaN NaN 33188.0 NaN True \n", 245 | "1556736520814149530 NaN NaN 33152.0 NaN True \n", 246 | "1556736524870363873 NaN NaN 33152.0 NaN True \n", 247 | "1556736524871304006 NaN NaN 33188.0 NaN True \n", 248 | "1556736524872197836 NaN NaN NaN NaN True \n", 249 | "1556736524872611868 NaN NaN NaN NaN True " 250 | ] 251 | }, 252 | "execution_count": 4, 253 | "metadata": {}, 254 | "output_type": "execute_result" 255 | } 256 | ], 257 | "source": [ 258 | "data_df = pd.DataFrame.from_records(demo_data, index = 'time')\n", 259 | "\n", 260 | "data_df[data_df['action'] != 'write'][['action', 'proc_pid', 'param_path', 'param_old_path', 'param_new_path', 'param_fip', 'param_buf_len', 'param_offset', 'param_mode', 'return', 'status']]" 261 | ] 262 | } 263 | ], 264 | "metadata": { 265 | "kernelspec": { 266 | "display_name": "Python 3", 267 | "language": "python", 268 | "name": "python3" 269 | }, 270 | "language_info": { 271 | "codemirror_mode": { 272 | "name": "ipython", 273 | "version": 3 274 | }, 275 | "file_extension": ".py", 276 | "mimetype": "text/x-python", 277 | "name": "python", 278 | "nbconvert_exporter": "python", 279 | "pygments_lexer": "ipython3", 280 | "version": "3.7.0" 281 | } 282 | }, 283 | "nbformat": 4, 284 | "nbformat_minor": 2 285 | } 286 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /src/loggedfs/_core/fs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | LoggedFS-python 6 | Filesystem monitoring with Fuse and Python 7 | https://github.com/pleiszenburg/loggedfs-python 8 | 9 | src/loggedfs/_core/fs.py: File system core 10 | 11 | Copyright (C) 2017-2020 Sebastian M. Ernst 12 | 13 | 14 | The contents of this file are subject to the Apache License 15 | Version 2 ("License"). You may not use this file except in 16 | compliance with the License. You may obtain a copy of the License at 17 | https://www.apache.org/licenses/LICENSE-2.0 18 | https://github.com/pleiszenburg/loggedfs-python/blob/master/LICENSE 19 | 20 | Software distributed under the License is distributed on an "AS IS" basis, 21 | WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the 22 | specific language governing rights and limitations under the License. 23 | 24 | 25 | """ 26 | 27 | 28 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 29 | # IMPORT 30 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | import errno 33 | import os 34 | import stat 35 | 36 | from refuse.high import ( 37 | FUSE, 38 | fuse_get_context, 39 | FuseOSError, 40 | Operations 41 | ) 42 | 43 | from .defaults import ( 44 | FUSE_ALLOWOTHER_DEFAULT, 45 | FUSE_FOREGROUND_DEFAULT, 46 | LIB_MODE_DEFAULT, 47 | LOG_BUFFERS_DEFAULT, 48 | LOG_ENABLED_DEFAULT, 49 | LOG_JSON_DEFAULT, 50 | LOG_ONLYMODIFYOPERATIONS_DEFAULT, 51 | LOG_PRINTPROCESSNAME_DEFAULT, 52 | LOG_SYSLOG_DEFAULT 53 | ) 54 | from .filter import filter_pipeline_class 55 | from .log import get_logger, log_msg 56 | from .out import event 57 | from .timing import time 58 | 59 | 60 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 61 | # ROUTINES 62 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 63 | 64 | def loggedfs_factory(directory, **kwargs): 65 | 66 | if not isinstance(directory, str): 67 | raise TypeError('directory must be of type string') 68 | if not os.path.isdir(directory): 69 | raise ValueError('directory must be a path to an existing directory') 70 | 71 | if not isinstance(kwargs.get('fuse_foreground', FUSE_FOREGROUND_DEFAULT), bool): 72 | raise TypeError('fuse_foreground must be of type bool') 73 | if not isinstance(kwargs.get('fuse_allowother', FUSE_ALLOWOTHER_DEFAULT), bool): 74 | raise TypeError('fuse_allowother must be of type bool') 75 | 76 | return FUSE( 77 | _loggedfs( 78 | directory, 79 | **kwargs 80 | ), 81 | directory, 82 | raw_fi = True, 83 | nothreads = True, 84 | foreground = kwargs.get('fuse_foreground', FUSE_FOREGROUND_DEFAULT), 85 | allow_other = kwargs.get('fuse_allowother', FUSE_ALLOWOTHER_DEFAULT), 86 | default_permissions = kwargs.get('fuse_allowother', FUSE_ALLOWOTHER_DEFAULT), 87 | attr_timeout = 0, 88 | entry_timeout = 0, 89 | negative_timeout = 0, 90 | sync_read = False, # relying on fuse.Operations class defaults? 91 | # max_readahead = 0, # relying on fuse.Operations class defaults? 92 | # direct_io = True, # relying on fuse.Operations class defaults? 93 | nonempty = True, # common options taken from LoggedFS 94 | use_ino = True # common options taken from LoggedFS 95 | ) 96 | 97 | 98 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 99 | # CORE CLASS: Init and internal routines 100 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 101 | 102 | class _loggedfs(Operations): 103 | 104 | 105 | flag_utime_omit_ok = 1 106 | use_ns = True 107 | 108 | 109 | _ST_FIELDS = tuple(i for i in dir(os.stat_result) if i.startswith('st_')) 110 | _STVFS_FIELDS = tuple(i for i in dir(os.statvfs_result) if i.startswith('f_')) 111 | 112 | 113 | def __init__(self, 114 | directory, 115 | fuse_foreground = FUSE_FOREGROUND_DEFAULT, 116 | fuse_allowother = FUSE_ALLOWOTHER_DEFAULT, 117 | lib_mode = LIB_MODE_DEFAULT, 118 | log_buffers = LOG_BUFFERS_DEFAULT, 119 | log_enabled = LOG_ENABLED_DEFAULT, 120 | log_file = None, 121 | log_filter = None, 122 | log_json = LOG_JSON_DEFAULT, 123 | log_only_modify_operations = LOG_ONLYMODIFYOPERATIONS_DEFAULT, 124 | log_printprocessname = LOG_PRINTPROCESSNAME_DEFAULT, 125 | log_syslog = LOG_SYSLOG_DEFAULT, 126 | **kwargs 127 | ): 128 | 129 | if log_filter is None: 130 | log_filter = filter_pipeline_class() 131 | 132 | if not isinstance(directory, str): 133 | raise TypeError('directory must be of type string') 134 | if not os.path.isdir(directory): 135 | raise ValueError('directory must be a path to an existing directory') 136 | if not os.access(directory, os.W_OK | os.R_OK): 137 | raise ValueError('not sufficient permissions on "directory"') 138 | 139 | if not isinstance(log_filter, filter_pipeline_class): 140 | raise TypeError('log_filter must either be None or of type filter_pipeline_class') 141 | if log_file is not None: 142 | if not os.path.isdir(os.path.dirname(log_file)): 143 | raise ValueError('path to logfile directory does not exist') 144 | if os.path.exists(log_file) and not os.path.isfile(log_file): 145 | raise ValueError('logfile exists and is not a file') 146 | if os.path.isfile(log_file) and not os.access(log_file, os.W_OK): 147 | raise ValueError('logfile exists and is not writeable') 148 | if not os.path.exists(log_file) and not os.access(directory, os.W_OK): 149 | raise ValueError('path to logfile directory is not writeable') 150 | if not isinstance(log_syslog, bool): 151 | raise TypeError('log_syslog must be of type bool') 152 | if not isinstance(log_enabled, bool): 153 | raise TypeError('log_enabled must be of type bool') 154 | if not isinstance(log_printprocessname, bool): 155 | raise TypeError('log_printprocessname must be of type bool') 156 | if not isinstance(log_json, bool): 157 | raise TypeError('log_json must be of type bool') 158 | if not isinstance(log_buffers, bool): 159 | raise TypeError('log_buffers must be of type bool') 160 | if not isinstance(lib_mode, bool): 161 | raise TypeError('lib_mode must be of type bool') 162 | if not isinstance(log_only_modify_operations, bool): 163 | raise TypeError('log_only_modify_operations must be of type bool') 164 | 165 | if not isinstance(fuse_foreground, bool): 166 | raise TypeError('fuse_foreground must be of type bool') 167 | if not isinstance(fuse_allowother, bool): 168 | raise TypeError('fuse_allowother must be of type bool') 169 | 170 | self._root_path = directory 171 | self._log_printprocessname = log_printprocessname 172 | self._log_json = log_json 173 | self._log_buffers = log_buffers 174 | self._log_filter = log_filter 175 | self._lib_mode = lib_mode 176 | self._log_only_modify_operations = log_only_modify_operations 177 | 178 | self._logger = get_logger('LoggedFS-python', log_enabled, log_file, log_syslog, self._log_json) 179 | 180 | if fuse_foreground: 181 | self._logger.info(log_msg(self._log_json, 'LoggedFS-python not running as a daemon')) 182 | if fuse_allowother: 183 | self._logger.info(log_msg(self._log_json, 'LoggedFS-python running as a public filesystem')) 184 | if log_file is not None: 185 | self._logger.info(log_msg(self._log_json, 'LoggedFS-python log file: %s' % log_file)) 186 | 187 | self._logger.info(log_msg(self._log_json, 'LoggedFS-python starting at %s' % directory)) 188 | 189 | try: 190 | self._root_path_fd = os.open(directory, os.O_RDONLY) 191 | except Exception as e: 192 | self._logger.exception('Directory access failed.') 193 | raise e 194 | 195 | log_configfile = kwargs.pop('_log_configfile', None) 196 | if log_configfile is not None: 197 | self._logger.info(log_msg(self._log_json, 198 | 'LoggedFS-python using configuration file %s' % log_configfile 199 | )) 200 | 201 | if len(kwargs) > 0: 202 | raise ValueError('unknown keyword argument(s)') 203 | 204 | 205 | def _full_path(self, partial_path): 206 | 207 | if partial_path.startswith('/'): 208 | partial_path = partial_path[1:] 209 | path = os.path.join(self._root_path, partial_path) 210 | return path 211 | 212 | 213 | @staticmethod 214 | def _rel_path(partial_path): 215 | 216 | if len(partial_path) == 0: 217 | return '.' 218 | elif partial_path == '/': 219 | return '.' 220 | elif partial_path.startswith('/'): 221 | return partial_path[1:] 222 | elif partial_path.startswith('./'): 223 | return partial_path[2:] 224 | else: 225 | return partial_path 226 | 227 | 228 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 229 | # CORE CLASS: Filesystem & file methods - STUBS 230 | # ... addressing https://github.com/fusepy/fusepy/issues/81 231 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 232 | 233 | def create(self, path, mode, fi = None): 234 | 235 | raise FuseOSError(errno.ENOSYS) 236 | 237 | 238 | def flush(self, path, fip): 239 | 240 | raise FuseOSError(errno.ENOSYS) 241 | 242 | 243 | def fsync(self, path, datasync, fip): 244 | 245 | raise FuseOSError(errno.ENOSYS) # the original loggedfs just returns 0 246 | 247 | 248 | def ioctl(self, path, cmd, arg, fh, flags, data): 249 | 250 | raise FuseOSError(errno.ENOSYS) 251 | 252 | 253 | def lock(self, path, fh, cmd, lock): 254 | 255 | raise FuseOSError(errno.ENOSYS) 256 | 257 | 258 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 259 | # CORE CLASS: Filesystem & file methods - IMPLEMENTATION 260 | # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 261 | 262 | @event(format_pattern = '{param_path}') 263 | def access(self, path, mode): 264 | 265 | if not os.access(self._rel_path(path), mode, dir_fd = self._root_path_fd): 266 | raise FuseOSError(errno.EACCES) 267 | 268 | 269 | @event(format_pattern = '{param_path} to {param_mode}') 270 | def chmod(self, path, mode): 271 | 272 | os.chmod(self._rel_path(path), mode, dir_fd = self._root_path_fd) 273 | 274 | 275 | @event(format_pattern = '{param_path} to {param_uid_name}({param_uid}):{param_gid_name}({param_gid})') 276 | def chown(self, path, uid, gid): 277 | 278 | os.chown(self._rel_path(path), uid, gid, dir_fd = self._root_path_fd, follow_symlinks = False) 279 | 280 | 281 | @event(format_pattern = '{param_path}') 282 | def destroy(self, path): 283 | 284 | os.close(self._root_path_fd) 285 | 286 | 287 | @event(format_pattern = '{param_path} (fh={param_fip})') 288 | def getattr(self, path, fip): 289 | 290 | if not fip: 291 | try: 292 | st = os.lstat(self._rel_path(path), dir_fd = self._root_path_fd) 293 | except FileNotFoundError: 294 | raise FuseOSError(errno.ENOENT) 295 | else: 296 | st = os.fstat(fip.fh) 297 | 298 | ret_dict = {key: getattr(st, key) for key in self._ST_FIELDS} 299 | 300 | for key in ['st_atime', 'st_ctime', 'st_mtime']: 301 | ret_dict[key] = ret_dict.pop(key + '_ns') 302 | 303 | return ret_dict 304 | 305 | 306 | @event(format_pattern = '{param_path}') 307 | def init(self, path): 308 | 309 | pass 310 | 311 | 312 | @event(format_pattern = '{param_source_path} to {param_target_path}') 313 | def link(self, target_path, source_path): 314 | 315 | target_rel_path = self._rel_path(target_path) 316 | 317 | os.link( 318 | self._rel_path(source_path), target_rel_path, 319 | src_dir_fd = self._root_path_fd, dst_dir_fd = self._root_path_fd 320 | ) 321 | 322 | uid, gid, pid = fuse_get_context() 323 | os.chown(target_rel_path, uid, gid, dir_fd = self._root_path_fd, follow_symlinks = False) 324 | 325 | 326 | @event(format_pattern = '{param_path} {param_mode}') 327 | def mkdir(self, path, mode): 328 | 329 | rel_path = self._rel_path(path) 330 | 331 | os.mkdir(rel_path, mode, dir_fd = self._root_path_fd) 332 | 333 | uid, gid, pid = fuse_get_context() 334 | 335 | os.chown(rel_path, uid, gid, dir_fd = self._root_path_fd, follow_symlinks = False) 336 | os.chmod(rel_path, mode, dir_fd = self._root_path_fd) # follow_symlinks = False 337 | 338 | 339 | @event(format_pattern = '{param_path} {param_mode}') 340 | def mknod(self, path, mode, dev): 341 | 342 | rel_path = self._rel_path(path) 343 | 344 | if stat.S_ISREG(mode): 345 | res = os.open( 346 | rel_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, mode, 347 | dir_fd = self._root_path_fd 348 | ) # TODO broken, applies umask to mode no matter what ... 349 | if res >= 0: 350 | os.close(res) 351 | elif stat.S_ISFIFO(mode): 352 | os.mkfifo(rel_path, mode, dir_fd = self._root_path_fd) 353 | else: 354 | os.mknod(rel_path, mode, dev, dir_fd = self._root_path_fd) 355 | 356 | uid, gid, pid = fuse_get_context() 357 | os.chown(rel_path, uid, gid, dir_fd = self._root_path_fd, follow_symlinks = False) 358 | os.chmod(rel_path, mode, dir_fd = self._root_path_fd) # follow_symlinks = False 359 | 360 | 361 | @event(format_pattern = '({param_fip}) {param_path} (fh={param_fip})') 362 | def open(self, path, fip): 363 | 364 | fip.fh = os.open(self._rel_path(path), fip.flags, dir_fd = self._root_path_fd) 365 | 366 | return 0 367 | 368 | 369 | @event(format_pattern = '{param_length} bytes from {param_path} at offset {param_offset} (fh={param_fip})') 370 | def read(self, path, length, offset, fip): 371 | 372 | ret = os.pread(fip.fh, length, offset) 373 | 374 | return ret 375 | 376 | 377 | @event(format_pattern = '{param_path}') 378 | def readdir(self, path, fh): 379 | 380 | rel_path = self._rel_path(path) 381 | 382 | dirents = ['.', '..'] 383 | if stat.S_ISDIR(os.lstat(rel_path, dir_fd = self._root_path_fd).st_mode): 384 | dir_fd = os.open(rel_path, os.O_RDONLY, dir_fd = self._root_path_fd) 385 | dirents.extend(os.listdir(dir_fd)) 386 | os.close(dir_fd) 387 | 388 | return dirents 389 | 390 | 391 | @event(format_pattern = '{param_path}') 392 | def readlink(self, path): 393 | 394 | pathname = os.readlink(self._rel_path(path), dir_fd = self._root_path_fd) 395 | 396 | if pathname.startswith('/'): # TODO check this ... actually required? 397 | return os.path.relpath(pathname, self._root_path) 398 | else: 399 | return pathname 400 | 401 | 402 | @event(format_pattern = '{param_path} (fh={param_fip})') 403 | def release(self, path, fip): 404 | 405 | os.close(fip.fh) 406 | 407 | 408 | @event(format_pattern = '{param_old_path} to {param_new_path}') 409 | def rename(self, old_path, new_path): 410 | 411 | os.rename( 412 | self._rel_path(old_path), self._rel_path(new_path), 413 | src_dir_fd = self._root_path_fd, dst_dir_fd = self._root_path_fd 414 | ) 415 | 416 | 417 | @event(format_pattern = '{param_path}') 418 | def rmdir(self, path): 419 | 420 | os.rmdir(self._rel_path(path), dir_fd = self._root_path_fd) 421 | 422 | 423 | @event(format_pattern = '{param_path}') 424 | def statfs(self, path): 425 | 426 | fd = os.open(self._rel_path(path), os.O_RDONLY, dir_fd = self._root_path_fd) 427 | stv = os.statvfs(fd) 428 | os.close(fd) 429 | 430 | return {key: getattr(stv, key) for key in self._STVFS_FIELDS} 431 | 432 | 433 | @event(format_pattern = 'from {param_source_path} to {param_target_path_}') 434 | def symlink(self, target_path_, source_path): 435 | 436 | target_rel_path = self._rel_path(target_path_) 437 | 438 | os.symlink(source_path, target_rel_path, dir_fd = self._root_path_fd) 439 | 440 | uid, gid, pid = fuse_get_context() 441 | os.chown(target_rel_path, uid, gid, dir_fd = self._root_path_fd, follow_symlinks = False) 442 | 443 | 444 | @event(format_pattern = '{param_path} to {param_length} bytes (fh={param_fip})') 445 | def truncate(self, path, length, fip = None): 446 | 447 | if fip is None: 448 | 449 | fd = os.open(self._rel_path(path), flags = os.O_WRONLY, dir_fd = self._root_path_fd) 450 | ret = os.ftruncate(fd, length) 451 | os.close(fd) 452 | return ret 453 | 454 | else: 455 | 456 | return os.ftruncate(fip.fh, length) 457 | 458 | 459 | @event(format_pattern = '{param_path}') 460 | def unlink(self, path): 461 | 462 | os.unlink(self._rel_path(path), dir_fd = self._root_path_fd) 463 | 464 | 465 | @event(format_pattern = '{param_path}') 466 | def utimens(self, path, times = None): 467 | 468 | def _fix_time_(atime, mtime): 469 | if None in (atime, mtime): 470 | st = os.lstat(relpath, dir_fd = self._root_path_fd) 471 | if atime is None: 472 | atime = st.st_atime_ns 473 | if mtime is None: 474 | mtime = st.st_mtime_ns 475 | return (atime, mtime) 476 | 477 | relpath = self._rel_path(path) 478 | 479 | os.utime(relpath, ns = _fix_time_(*times), dir_fd = self._root_path_fd, follow_symlinks = False) 480 | 481 | 482 | @event(format_pattern = '{param_buf_len} bytes to {param_path} at offset {param_offset} (fh={param_fip})') 483 | def write(self, path, buf, offset, fip): 484 | 485 | res = os.pwrite(fip.fh, buf, offset) 486 | 487 | return res 488 | --------------------------------------------------------------------------------