├── src ├── lock.py ├── logger.py └── mover.py ├── .gitignore ├── start.sh ├── index.py ├── .deepsource.toml ├── .flake8 ├── config-example.yaml ├── pyproject.toml ├── install.sh ├── LICENSE ├── README.md └── poetry.lock /src/lock.py: -------------------------------------------------------------------------------- 1 | class Lock: 2 | plot: list = [] 3 | dest: list = [] 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | .idea 3 | /config.yaml 4 | /src/__pycache__ 5 | /source/*.plot 6 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source .venv/bin/activate 4 | 5 | python index.py 6 | -------------------------------------------------------------------------------- /index.py: -------------------------------------------------------------------------------- 1 | from src.mover import PlotMover 2 | 3 | if __name__ == '__main__': 4 | PlotMover().main() 5 | -------------------------------------------------------------------------------- /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | [[analyzers]] 4 | name = "shell" 5 | 6 | [[analyzers]] 7 | name = "python" 8 | 9 | [analyzers.meta] 10 | runtime_version = "3.x.x" -------------------------------------------------------------------------------- /src/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%H:%M:%S') 4 | logger = logging.getLogger('plot-mover') 5 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = 3 | .git, 4 | __pycache__, 5 | pip-wheel-metadata, 6 | *.egg-info, 7 | .ipynb_checkpoints, 8 | .venv, 9 | .cache 10 | max-complexity = 10 11 | max-line-length = 120 12 | -------------------------------------------------------------------------------- /config-example.yaml: -------------------------------------------------------------------------------- 1 | # time (sec) between checking for new plots 2 | sleep: 60 3 | # time (sec) between spawning new copy threads 4 | debounce: 2 5 | 6 | source: 7 | - /source/path 8 | dest: 9 | - /destination/path1 10 | - /destination/path2 11 | - /destination/path3 12 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "plot_mover" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Maksym Leonov "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | PyYAML = "^5.4.1" 10 | 11 | [tool.poetry.dev-dependencies] 12 | flake8 = "^3.9.2" 13 | 14 | [build-system] 15 | requires = ["poetry-core>=1.0.0"] 16 | build-backend = "poetry.core.masonry.api" 17 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if ! python3 -c 'import sys; assert sys.version_info >= (3,7)' 2> /dev/null; 4 | then 5 | echo 'Found an unsupported version of Python' 6 | echo 'Python 3.7+ required. Update before proceeding with the installation' 7 | exit 1 8 | fi 9 | 10 | # Create virtual environment 11 | python3 -m venv .venv 12 | 13 | source .venv/bin/activate 14 | 15 | python -m pip install poetry 16 | poetry install --no-dev 17 | 18 | deactivate -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Maksym Leonov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://vshymanskyy.github.io/StandWithUkraine) 2 | 3 | # Chia Plot Mover 4 | Little tool to help to move plots across hard drives. 5 | Assuming you are plotting to one (or multiple) drive and want to move plots to multiple destination drives. 6 | Script is automatically look for space across specified list of destinations to move plots on. 7 | Can work with multiple plots at the same time, it will be helpful if speed of creating plots is higher then speed of moving plots to the destinaton 8 | directories. 9 | 10 | ## Install 11 | Python 3.7 or newer is required. Should be in place if you already using official chia client. 12 | Tested on Ubuntu 20.04 13 | 14 | Semi automated: 15 | ```bash 16 | git clone https://github.com/maxbanton/chia-plot-mover.git 17 | cd chia-plot-mover 18 | sh install.sh 19 | cp config-example.yaml config.yaml # Fill config.yaml with your values 20 | ``` 21 | 22 | Manual: 23 | ```bash 24 | git clone https://github.com/maxbanton/chia-plot-mover.git 25 | cd chia-plot-mover 26 | python -m venv .venv 27 | source .venv/bin/activate 28 | pip install poetry 29 | poetry install 30 | cp config-example.yaml config.yaml # Fill config.yaml with your values 31 | ``` 32 | 33 | ## Run 34 | Tip: Since plotting is long running process, running plot mover in screen session would be preferable 35 | 36 | ```bash 37 | sh start.sh 38 | ``` 39 | 40 | ## Donate 41 | You can send any amount of chia if you find this tool useful. Thanks! 42 | ```bash 43 | xch122s5zr5y3d53q348dnx2htaktq5huqmme3xvsmsqgte2rqjcev4srzttyw 44 | ``` 45 | -------------------------------------------------------------------------------- /src/mover.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import threading 4 | import time 5 | from typing import Dict, List 6 | 7 | import yaml 8 | 9 | from src.lock import Lock 10 | from src.logger import logger 11 | 12 | 13 | class PlotMover: 14 | CONFIG_FILE_NAME = 'config.yaml' 15 | MIN_K32_PLOT_SIZE = 108 * 10 ** 9 16 | 17 | _config: Dict 18 | 19 | def __init__(self): 20 | self._config = self._read_config() 21 | self._lock = Lock() 22 | self._mutex = threading.Lock() 23 | 24 | def _read_config(self): 25 | current_dir = os.path.dirname(__file__) 26 | config_path = os.path.join(current_dir, '..', self.CONFIG_FILE_NAME) 27 | filename = os.path.abspath(os.path.realpath(config_path)) 28 | 29 | with open(filename, 'r') as stream: 30 | try: 31 | return yaml.safe_load(stream) 32 | except yaml.YAMLError as exc: 33 | print(exc) 34 | 35 | def _look_for_plots(self) -> List[Dict]: 36 | result = [] 37 | 38 | for dir_ in self._config.get('source'): 39 | for file in os.listdir(dir_): 40 | if file.endswith(".plot") and file not in self._lock.plot: 41 | plot_path = os.path.join(dir_, file) 42 | size = os.path.getsize(plot_path) 43 | 44 | if size < self.MIN_K32_PLOT_SIZE: 45 | logger.warning(f'Main thread: Plot file {plot_path} size is to small. Is it real plot?') 46 | else: 47 | result.append({'dir': dir_, 'file': file, 'size': size}) 48 | 49 | return result 50 | 51 | def _look_for_destination(self, needed_space) -> str: 52 | emptiest_dir = None 53 | emptiest_dir_free = 0 54 | for dir_ in self._config.get('dest'): 55 | _, _, free = shutil.disk_usage(dir_) 56 | if free > needed_space and dir_ not in self._lock.dest and free > emptiest_dir_free: 57 | emptiest_dir = dir_ 58 | emptiest_dir_free = free 59 | return emptiest_dir 60 | 61 | @staticmethod 62 | def move_plot(self, src_dir, plot_file, dst_dir, size, lock): 63 | src_path = os.path.join(src_dir, plot_file) 64 | dst_path = os.path.join(dst_dir, plot_file) 65 | temp_dst_path = dst_path + '.move' 66 | 67 | if os.path.isfile(dst_path): 68 | raise Exception(f'Copy thread: Plot file {dst_path} already exists. Duplicate?') 69 | 70 | self._mutex.acquire() 71 | if dst_dir not in self._lock.dest: 72 | lock.plot.append(plot_file) 73 | lock.dest.append(dst_dir) 74 | self._mutex.release() 75 | 76 | logger.info(f'Copy thread: Starting to move plot from {src_path} to {dst_path}') 77 | start = time.time() 78 | shutil.move(src_path, temp_dst_path) 79 | duration = round(time.time() - start, 1) 80 | shutil.move(temp_dst_path, dst_path) 81 | speed = (size / duration) // (2 ** 20) 82 | logger.info(f'Copy thread: Plot file {src_path} moved, time: {duration} s, avg speed: {speed} MiB/s') 83 | 84 | lock.plot.remove(plot_file) 85 | lock.dest.remove(dst_dir) 86 | 87 | def main(self): 88 | while True: 89 | plots = self._look_for_plots() 90 | 91 | for plot in plots: 92 | src_dir = plot.get("dir") 93 | file = plot.get("file") 94 | size = plot.get("size") 95 | plot_path = os.path.join(src_dir, file) 96 | 97 | logger.info(f'Main thread: Found plot {plot_path} of size {size // (2 ** 30)} GiB') 98 | 99 | time.sleep(self._config.get('debounce')) 100 | 101 | dst_dir = self._look_for_destination(size) 102 | 103 | if dst_dir: 104 | thread = threading.Thread(target=self.move_plot, args=(self, src_dir, file, dst_dir, size, self._lock)) 105 | thread.start() 106 | else: 107 | logger.warning(f'Main thread: No destination available for plot {plot_path}') 108 | time.sleep(self._config.get('sleep')) 109 | else: 110 | logger.info(f"Main thread: No plots found. Sleep for {self._config.get('sleep')}s") 111 | time.sleep(self._config.get('sleep')) 112 | -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- 1 | [[package]] 2 | name = "flake8" 3 | version = "3.9.2" 4 | description = "the modular source code checker: pep8 pyflakes and co" 5 | category = "dev" 6 | optional = false 7 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" 8 | 9 | [package.dependencies] 10 | mccabe = ">=0.6.0,<0.7.0" 11 | pycodestyle = ">=2.7.0,<2.8.0" 12 | pyflakes = ">=2.3.0,<2.4.0" 13 | 14 | [[package]] 15 | name = "mccabe" 16 | version = "0.6.1" 17 | description = "McCabe checker, plugin for flake8" 18 | category = "dev" 19 | optional = false 20 | python-versions = "*" 21 | 22 | [[package]] 23 | name = "pycodestyle" 24 | version = "2.7.0" 25 | description = "Python style guide checker" 26 | category = "dev" 27 | optional = false 28 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 29 | 30 | [[package]] 31 | name = "pyflakes" 32 | version = "2.3.1" 33 | description = "passive checker of Python programs" 34 | category = "dev" 35 | optional = false 36 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 37 | 38 | [[package]] 39 | name = "pyyaml" 40 | version = "5.4.1" 41 | description = "YAML parser and emitter for Python" 42 | category = "main" 43 | optional = false 44 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" 45 | 46 | [metadata] 47 | lock-version = "1.1" 48 | python-versions = "^3.8" 49 | content-hash = "4119908737f86f9562d604283083b87c6a82e1b52e78db95671b3931492f4234" 50 | 51 | [metadata.files] 52 | flake8 = [ 53 | {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, 54 | {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, 55 | ] 56 | mccabe = [ 57 | {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, 58 | {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, 59 | ] 60 | pycodestyle = [ 61 | {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, 62 | {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, 63 | ] 64 | pyflakes = [ 65 | {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, 66 | {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, 67 | ] 68 | pyyaml = [ 69 | {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, 70 | {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, 71 | {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, 72 | {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, 73 | {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, 74 | {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, 75 | {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, 76 | {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, 77 | {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, 78 | {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, 79 | {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, 80 | {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, 81 | {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, 82 | {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, 83 | {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, 84 | {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, 85 | {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, 86 | {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, 87 | {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, 88 | {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, 89 | {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, 90 | {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, 91 | {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, 92 | {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, 93 | {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, 94 | {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, 95 | {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, 96 | {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, 97 | {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, 98 | ] 99 | --------------------------------------------------------------------------------