├── scripts ├── create_archive.sh └── deploy.sh ├── README.md └── main.py /scripts/create_archive.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Create an archive named with "$2", containing all files under "$1" 4 | # and no subdir containing all others 5 | 6 | if (( $# != 2 )) ; then 7 | echo "parameters not meet" 8 | exit 1 9 | fi 10 | 11 | find "$1" -mindepth 1 -not -type d -printf "%P\n" | tar -C "$1" -cf "$2" --gzip -T - 12 | -------------------------------------------------------------------------------- /scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # The http server not support data-form or x-www-url-encoded-form, 4 | # so the archive file must just in the post body, so use --data-binary 5 | # as curl argument. 6 | # 7 | # $1 is the archive path to be sent to http server 8 | # $2 is the http server (including schema, host, port and path) 9 | # 10 | # Example: 11 | # curl --data-binary @public.tar.gz http://localhost:8080/ 12 | 13 | if (( $# != 2 )) ; then 14 | echo "parameter not meet" 15 | exit 1 16 | fi 17 | 18 | curl --data-binary @"$1" "$2" 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Static Deplyer 2 | 3 | A simple daemon help with extracting tar files to specific place 4 | and create a symbolic link to it, the tar files should be transferred 5 | to this daemon by http. 6 | 7 | ## Caution 8 | 9 | No `data-form` or `x-www-urlencoded-form` supported, so the tar must be transferred 10 | just in POST body, ie: `curl --data-binary @file.tar.gz`. 11 | 12 | No authorization implemented, so a reverse proxy with auth is recommanded. 13 | 14 | Tar file with sub directories with be extracted as is, so do not contain parent 15 | directory when you create tar. 16 | 17 | ## Usage 18 | 19 | ``` 20 | $ ./main.py --help 21 | usage: main.py [-h] --archive-dir ARCHIVE_DIR --extract-dir EXTRACT_DIR --symlink-path 22 | SYMLINK_PATH [--keep-extract KEEP_EXTRACT] [--keep-archive KEEP_ARCHIVE] 23 | --port PORT [--temp-dir TEMP_DIR] 24 | 25 | options: 26 | -h, --help show this help message and exit 27 | --archive-dir ARCHIVE_DIR 28 | directory to save archives 29 | --extract-dir EXTRACT_DIR 30 | directory to save extracted files 31 | --symlink-path SYMLINK_PATH 32 | path of symlink which redirect to extracted archive 33 | --keep-extract KEEP_EXTRACT 34 | Number of extracted archives to keep, 0 mean never vacuum 35 | --keep-archive KEEP_ARCHIVE 36 | Number of archives to keep, 0 mean never vacuum 37 | --port PORT listen port on 127.0.0.1, no authorization implemented so only 38 | listen on 127.0.0.1 for safety 39 | --temp-dir TEMP_DIR path to save in-delivery archive 40 | ``` 41 | 42 | ## Example 43 | 44 | First start the daemon. 45 | 46 | ``` 47 | $ ./main.py --port 8080 --archive-dir archive --extract-dir extracted --symlink-path serve 48 | INFO:root:Listening on 127.0.0.1:8080 49 | INFO:root:Archive saves under: archive 50 | INFO:root:Extract tar under: extracted 51 | INFO:root:Keep 8 archives at most 52 | INFO:root:Keep 4 extracted at most 53 | INFO:root:Symbolic link location: serve 54 | INFO:root:Temperory directory: /tmp 55 | INFO:root:Starting httpd... 56 | ``` 57 | 58 | Then create a tar and upload it to this daemon. 59 | 60 | **Note**: the tar shouldn't contain its parent directory, but `.` as parent 61 | is acceptable. Or you can follow [this step](https://stackoverflow.com/a/39530409) 62 | to create a more elegant tar. 63 | 64 | ``` 65 | $ mkdir tmp 66 | $ cd tmp 67 | $ echo 'Hello, world!' > index.html 68 | $ tar --gzip -cf ../tmp.tar.gz . 69 | $ tar -tf ../tmp.tar.gz 70 | ./ 71 | ./index.html 72 | $ curl --data-binary @../tmp.tar.gz http://localhost:8080/ 73 | Success 74 | ``` 75 | 76 | And the server side shows 77 | 78 | ``` 79 | ... 80 | INFO:root:Starting httpd... 81 | INFO:FileManager:Temporarily save to /tmp/archive_2023-04-04T10:35:49.tar.gz 82 | INFO:FileManager:Moving saved archive to archive/archive_2023-04-04T10:35:49.tar.gz 83 | INFO:FileManager:Extracting to extracted/archive_2023-04-04T10:35:49 84 | INFO:FileManager:Recreating symlink point to extracted/archive_2023-04-04T10:35:49 85 | INFO:FileManager:Vacuuming archive, keep the 8 lastest 86 | INFO:FileManager:Vacuuming extract, keep the 4 lastest 87 | INFO:FileManager:Deploy success 88 | 127.0.0.1 - - [04/Apr/2023 10:35:49] "POST / HTTP/1.1" 200 - 89 | ``` 90 | 91 | Finally the directory looks like (omit unrelated directories): 92 | 93 | ``` 94 | . 95 | ├── archive 96 | │ └── archive_2023-04-04T10:35:49.tar.gz 97 | ├── extracted 98 | │ └── archive_2023-04-04T10:35:49 99 | │ └── index.html 100 | └── serve -> extracted/archive_2023-04-04T10:35:49 101 | ``` 102 | 103 | ## Use Case 104 | 105 | When you hold a static site and want to update its content easily, 106 | like just uploading a tar and automatically deployed. 107 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | This Listen a port on 127.0.0.1 or [::1], (so no authorization 5 | implement required), receive a .tar.gz file and extract it to 6 | a specific path, and make it a soft link to the extracted files. 7 | 8 | Use the following command to upload the file content: 9 | curl -H "Content-Type: application/octet-stream" --data-binary @main.py http://localhost:8080 10 | 11 | The file content must be a legal .tar.gz file. There must not be 12 | a subdirectory to contain other files 13 | """ 14 | 15 | import os 16 | import tarfile 17 | import shutil 18 | from pathlib import Path 19 | from typing import BinaryIO, Union 20 | from http.server import BaseHTTPRequestHandler, HTTPServer 21 | from datetime import datetime 22 | import argparse 23 | import logging 24 | 25 | class NotSymlinkException(Exception): 26 | pass 27 | class NotDirectoryException(Exception): 28 | pass 29 | 30 | class FileManager: 31 | logger = logging.getLogger("FileManager") 32 | 33 | archive_ext = ".tar.gz" 34 | 35 | def __init__(self, archive_dir: str, extract_dir: str, symlink_path: str, 36 | keep_archive: int, keep_extract: int, temp_dir: str = "/tmp"): 37 | self.archive_dir = archive_dir 38 | self.extract_dir = extract_dir 39 | self.symlink_path = symlink_path 40 | 41 | self.keep_archive = keep_archive 42 | self.keep_extract = keep_extract 43 | 44 | self.temp_dir = temp_dir 45 | 46 | self._check_dirs() 47 | 48 | def _check_dirs(self): 49 | def check_dir(path): 50 | p = Path(path) 51 | if p.exists(): 52 | if not p.is_dir(): 53 | raise NotDirectoryException("{} exists and is not a directory".format(path)) 54 | else: 55 | os.makedirs(path) 56 | 57 | def check_symlink(path): 58 | p = Path(path) 59 | if p.exists(): 60 | if not p.is_symlink(): 61 | raise NotSymlinkException("{} exists and is not a symlink".format(path)) 62 | else: 63 | check_dir(os.path.dirname(path)) 64 | 65 | check_dir(self.archive_dir) 66 | check_dir(self.extract_dir) 67 | check_dir(self.temp_dir) 68 | check_symlink(self.symlink_path) 69 | 70 | def _get_archive_name(self) -> str: 71 | time_str = datetime.now().isoformat(timespec="seconds") 72 | return f"archive_{time_str}{self.archive_ext}" 73 | 74 | def _get_basename(self, filename: str) -> str: 75 | if filename.endswith(self.archive_ext): 76 | return filename[:-len(self.archive_ext)] 77 | return filename 78 | 79 | def _extract(self, archive_path: str, target_path: str) -> bool: 80 | try: 81 | with tarfile.open(archive_path, mode="r:gz") as tf: 82 | tf.extractall(target_path) 83 | except Exception as e: 84 | self.logger.error("Failed to extract tar file: {}".format(e)) 85 | return False 86 | return True 87 | 88 | 89 | def save_file(self, src: BinaryIO, content_length: int) -> Union[str, None]: 90 | archive_name = self._get_archive_name() 91 | tgt_file = os.path.join(self.temp_dir, archive_name) 92 | 93 | self.logger.info("Temporarily save to {}".format(tgt_file)) 94 | 95 | try: 96 | f = open(tgt_file, "bw") 97 | redirect_stream(src, f, content_length) 98 | f.close() 99 | except: 100 | os.remove(tgt_file) 101 | return None 102 | 103 | final_file = os.path.join(self.archive_dir, archive_name) 104 | self.logger.info("Moving saved archive to {}".format(final_file)) 105 | shutil.move(tgt_file, final_file) 106 | return final_file 107 | 108 | def deploy(self, archive_path: str) -> bool: 109 | extract_dir = os.path.join(self.extract_dir, 110 | self._get_basename(os.path.basename(archive_path))) 111 | 112 | self.logger.info("Extracting to {}".format(extract_dir)) 113 | 114 | os.mkdir(extract_dir) 115 | if not self._extract(archive_path, extract_dir): 116 | self.logger.error("Failed to extract archive {} to {}" 117 | .format(archive_path, extract_dir)) 118 | return False 119 | 120 | self.logger.info("Recreating symlink point to {}".format(extract_dir)) 121 | if Path(self.symlink_path).exists(): 122 | os.remove(self.symlink_path) 123 | os.symlink(extract_dir, self.symlink_path) 124 | 125 | return True 126 | 127 | def _vacuum_single(self, dirname: str, keep_count: int, rm_dir: bool) -> None: 128 | files = os.listdir(dirname) 129 | files.sort() 130 | for f in files[:-keep_count]: 131 | full_path = os.path.join(dirname, f) 132 | 133 | self.logger.info("Removing {}".format(full_path)) 134 | if rm_dir: 135 | shutil.rmtree(full_path) 136 | else: 137 | os.remove(full_path) 138 | 139 | def vacuum(self) -> None: 140 | if self.keep_archive > 0: 141 | self.logger.info("Vacuuming archive, keep the {} lastest".format(self.keep_archive)) 142 | self._vacuum_single(self.archive_dir, self.keep_archive, False) 143 | if self.keep_extract > 0: 144 | self.logger.info("Vacuuming extract, keep the {} lastest".format(self.keep_extract)) 145 | self._vacuum_single(self.extract_dir, self.keep_extract, True) 146 | 147 | def handle(self, instream: BinaryIO, content_length: int) -> bool: 148 | archive_path = self.save_file(instream, content_length) 149 | if archive_path is None: 150 | self.logger.error("Failed to save file. Aborted!") 151 | return False 152 | 153 | if not self.deploy(archive_path): 154 | self.logger.error("Failed to extract or create symlink. Aborted!") 155 | return False 156 | self.vacuum() 157 | self.logger.info("Deploy success") 158 | return True 159 | 160 | global_mgr: FileManager 161 | 162 | 163 | def redirect_stream(src: BinaryIO, tgt: BinaryIO, size: int) -> None: 164 | block_size = 4 * 1024 * 1024 # 4MB 165 | 166 | cache = src.read(size % block_size) 167 | tgt.write(cache) 168 | size -= size % block_size 169 | 170 | while size > 0: 171 | cache = src.read(block_size) 172 | tgt.write(cache) 173 | size -= block_size 174 | 175 | 176 | class S(BaseHTTPRequestHandler): 177 | protocol_version = 'HTTP/1.1' 178 | logger = logging.getLogger("HttpHandler") 179 | 180 | def __init__(self, *args, **kwargs): 181 | super(S, self).__init__(*args, **kwargs) 182 | 183 | def _set_response(self): 184 | self.send_response(200) 185 | self.send_header("Content-Type", "text/plaintext") 186 | self.end_headers() 187 | 188 | def do_GET(self): 189 | self.logger.info("Received GET request, Path: %s", str(self.path)) 190 | content = "Non-implemented".encode("utf-8") 191 | self._write_response(403, "text/plaintext", content) 192 | 193 | def _write_response(self, status_code: int, content_type: str, content: bytes): 194 | self.send_response(status_code) 195 | self.send_header("Content-Type", content_type) 196 | self.send_header("Content-Length", str(len(content))) 197 | self.end_headers() 198 | 199 | self.wfile.write(content) 200 | 201 | def do_POST(self): 202 | content_length = int(self.headers["Content-Length"]) 203 | if global_mgr.handle(self.rfile, content_length): 204 | content = "Success".encode("utf-8") 205 | self._write_response(200, "text/plaintext", content) 206 | else: 207 | content = "Failed".encode("utf-8") 208 | self._write_response(200, "text/plaintext", content) 209 | self.wfile.flush() 210 | 211 | 212 | def run(archive_dir: str, extract_dir: str, symlink_path: str, 213 | keep_archive: int, keep_extract: int, 214 | port: int = 8080, temp_dir: str = "/tmp"): 215 | logging.basicConfig(level=logging.DEBUG) 216 | 217 | address = "127.0.0.1" 218 | 219 | logging.info("Listening on {}:{}".format(address, port)) 220 | logging.info("Archive saves under: {}".format(archive_dir)) 221 | logging.info("Extract tar under: {}".format(extract_dir)) 222 | logging.info("Keep {} archives at most".format(keep_archive)) 223 | logging.info("Keep {} extracted at most".format(keep_extract)) 224 | logging.info("Symbolic link location: {}".format(symlink_path)) 225 | logging.info("Temperory directory: {}".format(temp_dir)) 226 | 227 | global global_mgr 228 | global_mgr = FileManager(archive_dir=archive_dir, extract_dir=extract_dir, 229 | symlink_path=symlink_path, temp_dir=temp_dir, 230 | keep_archive=keep_archive, keep_extract=keep_extract) 231 | 232 | httpd = HTTPServer((address, port), S) 233 | logging.info("Starting httpd...") 234 | 235 | try: 236 | httpd.serve_forever() 237 | except KeyboardInterrupt: 238 | pass 239 | 240 | httpd.server_close() 241 | logging.info("Stopping httpd...") 242 | 243 | 244 | if __name__ == "__main__": 245 | ap = argparse.ArgumentParser() 246 | ap.add_argument("--archive-dir", dest="archive_dir", type=str, 247 | required=True, help="directory to save archives") 248 | ap.add_argument("--extract-dir", dest="extract_dir", type=str, 249 | required=True, help="directory to save extracted files") 250 | ap.add_argument("--symlink-path", dest="symlink_path", type=str, 251 | required=True, help="path of symlink which redirect to extracted archive") 252 | 253 | ap.add_argument("--keep-extract", dest="keep_extract", type=int, 254 | default=4, help="Number of extracted archives to keep, 0 mean never vacuum") 255 | ap.add_argument("--keep-archive", dest="keep_archive", type=int, 256 | default=8, help="Number of archives to keep, 0 mean never vacuum") 257 | 258 | ap.add_argument("--port", dest="port", type=int, 259 | required=True, help="listen port on 127.0.0.1, " + 260 | "no authorization implemented so only listen on 127.0.0.1 for safety") 261 | ap.add_argument("--temp-dir", dest="temp_dir", type=str, 262 | default="/tmp", help="path to save in-delivery archive") 263 | 264 | args = ap.parse_args() 265 | 266 | run(archive_dir=args.archive_dir, 267 | extract_dir=args.extract_dir, 268 | symlink_path=args.symlink_path, 269 | keep_archive=args.keep_archive, 270 | keep_extract=args.keep_extract, 271 | temp_dir=args.temp_dir, 272 | port=args.port) 273 | --------------------------------------------------------------------------------