├── requirements.txt ├── demo_screenshot.png ├── .envREMOVETHISPART ├── .gitignore ├── README.md ├── sample_log_files_to_understand_the_parsing_functions ├── TestnetSupernode01__154_12_235_41__systemd_log.txt └── TestnetSupernode01__154_12_235_41__entry.log └── automatic_log_collector_and_analyzer.py /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | paramiko 3 | sqlalchemy 4 | tqdm 5 | redis 6 | ruff 7 | pandas 8 | pyyaml 9 | python-decouple -------------------------------------------------------------------------------- /demo_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dicklesworthstone/automatic_log_collector_and_analyzer/HEAD/demo_screenshot.png -------------------------------------------------------------------------------- /.envREMOVETHISPART: -------------------------------------------------------------------------------- 1 | SSH_TIMEOUT_SECONDS = 25 2 | BASE_PATH =/home/ubuntu/automatic_log_collector_and_analyzer/ 3 | AWS_ACCESS_KEY_ID=your_aws_access_key_id 4 | AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key 5 | AWS_REGION=us-east-2 6 | SSH_USER=ubuntu 7 | SSH_KEY_NAME=my-secret-ssh-key.pem 8 | ANSIBLE_INVENTORY_FILE=my_inventory_file.yml 9 | DOWNLOADED_LOG_FILES_PATH=downloaded_log_files/ 10 | SQLITE_FILE_PATH=log_analysis_tool_db.sqlite 11 | NODE_STATUS_DATA_BACKUP_PATH=node_status_data_backup 12 | BACKUP_DATABASE_TABLE_CSV_FILES_DIR_NAME=backup_database_table_csv_files 13 | NUMBER_OF_BACKUP_CSV_FILES_TO_KEEP=10 14 | INSTANCE_NAME_PREFIX=NewTestnet testnet - SuperNode 15 | DATASETTE_PATH=/home/ubuntu/.local/bin/datasette 16 | HOST=0.0.0.0 17 | PORT=5555 18 | TIME_LIMIT_MS=30000 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | # *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | #ignore .bin model files and .sqlite files 163 | *.bin 164 | *.sqlite 165 | downloaded_log_files* 166 | backup_database_table_csv_files* 167 | node_status_data_backup* 168 | *.csv 169 | *.lock 170 | *.yml 171 | *.pem -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Automatically Download and Analyze Log Files from Remote Machines 2 | 3 | This application is designed to collect and analyze logs from remote machines hosted on Amazon Web Services (AWS) and other cloud hosting services. 4 | 5 | **Note**: This application was specifically designed for use with Pastel Network's log files. However, it can be easily adapted to work with any log files by modifying the parsing functions, data models, and specifying the location and names of the log files to be downloaded. It is compatible with log files stored in a standard format, where each entry is on a separate line and contains a timestamp, a log level, and a message. The application has been tested with log files several gigabytes in size from dozens of machines and can process all of it in minutes. It is designed for Ubuntu 22.04+, but can be adapted for other Linux distributions. 6 | 7 | ![Demo Screenshot:](https://raw.githubusercontent.com/Dicklesworthstone/automatic_log_collector_and_analyzer/main/demo_screenshot.png) 8 | 9 | ## Customization 10 | 11 | To adapt this application for your own use case, refer to the included sample log files and compare them to the parsing functions in the code. You can also modify the data models to store log entries as desired. 12 | 13 | ## Features 14 | 15 | The application consists of various Python scripts that perform the following functions: 16 | 17 | * **Connect to Remote Machines**: Using the boto3 library for AWS instances and an Ansible inventory file for non-AWS instances, the application establishes SSH connections to each remote machine. 18 | * **Download and Parse Log Files**: Downloads specified log files from each remote machine and parses them. The parsed log entries are then queued for database insertion. 19 | * **Insert Log Entries into Database**: Uses SQLAlchemy to insert the parsed log entries from the queue into an SQLite database. 20 | * **Process and Analyze Log Entries**: Processes and analyzes log entries stored in the database, offering functions to find error entries and create views of aggregated data based on specified criteria. 21 | * **Generate Network Activity Data**: Fetches and processes network activity data from each remote machine. 22 | * **Expose Database via Web App using Datasette**: Once the database is generated, it can be shared over the web using Datasette. 23 | 24 | ## Compatibility 25 | 26 | The tool is compatible with both AWS-hosted instances and any list of Linux instances stored in a standard Ansible inventory file with the following structure: 27 | 28 | ```yaml 29 | all: 30 | vars: 31 | ansible_connection: ssh 32 | ansible_user: ubuntu 33 | ansible_ssh_private_key_file: /path/to/ssh/key/file.pem 34 | hosts: 35 | MyCoolMachine01: 36 | ansible_host: 1.2.3.41 37 | MyCoolMachine02: 38 | ansible_host: 1.2.3.19 39 | ``` 40 | 41 | (Both can be used seamlessly.) 42 | 43 | ## Warning 44 | 45 | To simplify the code, the tool is designed to delete all downloaded log files and generated databases each time it runs. Consequently, this can consume significant bandwidth depending on your log files' size. However, the design's high level of parallel processing and concurrency allows it to run quickly, even when connecting to dozens of remote machines and downloading hundreds of log files. 46 | 47 | ## Usage 48 | 49 | Designed for Ubuntu 22.04+, first install the requirements: 50 | 51 | ```bash 52 | python3 -m venv venv 53 | source venv/bin/activate 54 | python3 -m pip install --upgrade pip 55 | python3 -m pip install wheel 56 | pip install -r requirements.txt 57 | ``` 58 | 59 | You will also need to install Redis: 60 | 61 | ```bash 62 | sudo apt install redis -y 63 | ``` 64 | 65 | And install Datasette to expose the results as a website: 66 | 67 | ```bash 68 | sudo apt install pipx -y && pipx ensurepath && pipx install datasette 69 | ``` 70 | 71 | To run the application every 30 minutes as a cron job, execute: 72 | 73 | ```bash 74 | crontab -e 75 | ``` 76 | 77 | And add the following line: 78 | 79 | ```bash 80 | */30 * * * * . $HOME/.profile; /home/ubuntu/automatic_log_collector_and_analyzer/venv/bin/python /home/ubuntu/automatic_log_collector_and_analyzer/automatic_log_collector_and_analyzer.py >> /home/ubuntu/automatic_log_collector_and_analyzer/log_$(date +\%Y-\%m-\%dT\%H_\%M_\%S).log 2>&1 81 | ``` 82 | --- 83 | 84 | Thanks for your interest in my open-source project! I hope you find it useful. You might also find my commercial web apps useful, and I would really appreciate it if you checked them out: 85 | 86 | **[YoutubeTranscriptOptimizer.com](https://youtubetranscriptoptimizer.com)** makes it really quick and easy to paste in a YouTube video URL and have it automatically generate not just a really accurate direct transcription, but also a super polished and beautifully formatted written document that can be used independently of the video. 87 | 88 | The document basically sticks to the same material as discussed in the video, but it sounds much more like a real piece of writing and not just a transcript. It also lets you optionally generate quizzes based on the contents of the document, which can be either multiple choice or short-answer quizzes, and the multiple choice quizzes get turned into interactive HTML files that can be hosted and easily shared, where you can actually take the quiz and it will grade your answers and score the quiz for you. 89 | 90 | **[FixMyDocuments.com](https://fixmydocuments.com/)** lets you submit any kind of document— PDFs (including scanned PDFs that require OCR), MS Word and Powerpoint files, images, audio files (mp3, m4a, etc.) —and turn them into highly optimized versions in nice markdown formatting, from which HTML and PDF versions are automatically generated. Once converted, you can also edit them directly in the site using the built-in markdown editor, where it saves a running revision history and regenerates the PDF/HTML versions. 91 | 92 | In addition to just getting the optimized version of the document, you can also generate many other kinds of "derived documents" from the original: interactive multiple-choice quizzes that you can actually take and get graded on; slick looking presentation slides as PDF or HTML (using LaTeX and Reveal.js), an in-depth summary, a concept mind map (using Mermaid diagrams) and outline, custom lesson plans where you can select your target audience, a readability analysis and grade-level versions of your original document (good for simplifying concepts for students), Anki Flashcards that you can import directly into the Anki app or use on the site in a nice interface, and more. 93 | -------------------------------------------------------------------------------- /sample_log_files_to_understand_the_parsing_functions/TestnetSupernode01__154_12_235_41__systemd_log.txt: -------------------------------------------------------------------------------- 1 | Sep 17 07:25:04 TestnetSupernode01 kernel: Command line: BOOT_IMAGE=/vmlinuz-5.15.0-79-generic root=UUID=11c5afcd-dda3-45f8-b3e4-28d334354465 ro net.ifnames=0 biosdevname=0 nomodeset 2 | Sep 17 07:25:04 TestnetSupernode01 kernel: KERNEL supported cpus: 3 | Sep 17 07:25:04 TestnetSupernode01 kernel: Intel GenuineIntel 4 | Sep 17 07:25:04 TestnetSupernode01 kernel: AMD AuthenticAMD 5 | Sep 17 07:25:04 TestnetSupernode01 kernel: Hygon HygonGenuine 6 | Sep 17 07:25:04 TestnetSupernode01 kernel: Centaur CentaurHauls 7 | Sep 17 07:25:04 TestnetSupernode01 kernel: zhaoxin Shanghai 8 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers' 9 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers' 10 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/fpu: Supporting XSAVE feature 0x004: 'AVX registers' 11 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/fpu: xstate_offset[2]: 576, xstate_sizes[2]: 256 12 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/fpu: Enabled xstate features 0x7, context size is 832 bytes, using 'standard' format. 13 | Sep 17 07:25:04 TestnetSupernode01 kernel: signal: max sigframe size: 1776 14 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-provided physical RAM map: 15 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable 16 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved 17 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x00000000000f0000-0x00000000000fffff] reserved 18 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x0000000000100000-0x00000000bffd9fff] usable 19 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x00000000bffda000-0x00000000bfffffff] reserved 20 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x00000000feffc000-0x00000000feffffff] reserved 21 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x00000000fffc0000-0x00000000ffffffff] reserved 22 | Sep 17 07:25:04 TestnetSupernode01 kernel: BIOS-e820: [mem 0x0000000100000000-0x00000007bfffffff] usable 23 | Sep 17 07:25:04 TestnetSupernode01 kernel: NX (Execute Disable) protection: active 24 | Sep 17 07:25:04 TestnetSupernode01 kernel: SMBIOS 2.8 present. 25 | Sep 17 07:25:04 TestnetSupernode01 kernel: DMI: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014 26 | Sep 17 07:25:04 TestnetSupernode01 kernel: Hypervisor detected: KVM 27 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: Using msrs 4b564d01 and 4b564d00 28 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 0, msr 5c9a01001, primary cpu clock 29 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: using sched offset of 15397846816 cycles 30 | Sep 17 07:25:04 TestnetSupernode01 kernel: clocksource: kvm-clock: mask: 0xffffffffffffffff max_cycles: 0x1cd42e4dffb, max_idle_ns: 881590591483 ns 31 | Sep 17 07:25:04 TestnetSupernode01 kernel: tsc: Detected 2794.748 MHz processor 32 | Sep 17 07:25:04 TestnetSupernode01 kernel: e820: update [mem 0x00000000-0x00000fff] usable ==> reserved 33 | Sep 17 07:25:04 TestnetSupernode01 kernel: e820: remove [mem 0x000a0000-0x000fffff] usable 34 | Sep 17 07:25:04 TestnetSupernode01 kernel: last_pfn = 0x7c0000 max_arch_pfn = 0x400000000 35 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/PAT: Configuration [0-7]: WB WC UC- UC WB WP UC- WT 36 | Sep 17 07:25:04 TestnetSupernode01 kernel: last_pfn = 0xbffda max_arch_pfn = 0x400000000 37 | Sep 17 07:25:04 TestnetSupernode01 kernel: found SMP MP-table at [mem 0x000f5a80-0x000f5a8f] 38 | Sep 17 07:25:04 TestnetSupernode01 kernel: Using GB pages for direct mapping 39 | Sep 17 07:25:04 TestnetSupernode01 kernel: RAMDISK: [mem 0x2ffd3000-0x33fe0fff] 40 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Early table checksum verification disabled 41 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: RSDP 0x00000000000F5850 000014 (v00 BOCHS ) 42 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: RSDT 0x00000000BFFE1693 000038 (v01 BOCHS BXPCRSDT 00000001 BXPC 00000001) 43 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: FACP 0x00000000BFFE1445 000074 (v01 BOCHS BXPCFACP 00000001 BXPC 00000001) 44 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: DSDT 0x00000000BFFDF040 002405 (v01 BOCHS BXPCDSDT 00000001 BXPC 00000001) 45 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: FACS 0x00000000BFFDF000 000040 46 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: APIC 0x00000000BFFE14B9 0000B0 (v01 BOCHS BXPCAPIC 00000001 BXPC 00000001) 47 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: SSDT 0x00000000BFFE1569 0000CA (v01 BOCHS VMGENID 00000001 BXPC 00000001) 48 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: HPET 0x00000000BFFE1633 000038 (v01 BOCHS BXPCHPET 00000001 BXPC 00000001) 49 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: WAET 0x00000000BFFE166B 000028 (v01 BOCHS BXPCWAET 00000001 BXPC 00000001) 50 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Reserving FACP table memory at [mem 0xbffe1445-0xbffe14b8] 51 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Reserving DSDT table memory at [mem 0xbffdf040-0xbffe1444] 52 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Reserving FACS table memory at [mem 0xbffdf000-0xbffdf03f] 53 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Reserving APIC table memory at [mem 0xbffe14b9-0xbffe1568] 54 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Reserving SSDT table memory at [mem 0xbffe1569-0xbffe1632] 55 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Reserving HPET table memory at [mem 0xbffe1633-0xbffe166a] 56 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Reserving WAET table memory at [mem 0xbffe166b-0xbffe1692] 57 | Sep 17 07:25:04 TestnetSupernode01 kernel: No NUMA configuration found 58 | Sep 17 07:25:04 TestnetSupernode01 kernel: Faking a node at [mem 0x0000000000000000-0x00000007bfffffff] 59 | Sep 17 07:25:04 TestnetSupernode01 kernel: NODE_DATA(0) allocated [mem 0x7bffd6000-0x7bfffffff] 60 | Sep 17 07:25:04 TestnetSupernode01 kernel: Zone ranges: 61 | Sep 17 07:25:04 TestnetSupernode01 kernel: DMA [mem 0x0000000000001000-0x0000000000ffffff] 62 | Sep 17 07:25:04 TestnetSupernode01 kernel: DMA32 [mem 0x0000000001000000-0x00000000ffffffff] 63 | Sep 17 07:25:04 TestnetSupernode01 kernel: Normal [mem 0x0000000100000000-0x00000007bfffffff] 64 | Sep 17 07:25:04 TestnetSupernode01 kernel: Device empty 65 | Sep 17 07:25:04 TestnetSupernode01 kernel: Movable zone start for each node 66 | Sep 17 07:25:04 TestnetSupernode01 kernel: Early memory node ranges 67 | Sep 17 07:25:04 TestnetSupernode01 kernel: node 0: [mem 0x0000000000001000-0x000000000009efff] 68 | Sep 17 07:25:04 TestnetSupernode01 kernel: node 0: [mem 0x0000000000100000-0x00000000bffd9fff] 69 | Sep 17 07:25:04 TestnetSupernode01 kernel: node 0: [mem 0x0000000100000000-0x00000007bfffffff] 70 | Sep 17 07:25:04 TestnetSupernode01 kernel: Initmem setup node 0 [mem 0x0000000000001000-0x00000007bfffffff] 71 | Sep 17 07:25:04 TestnetSupernode01 kernel: On node 0, zone DMA: 1 pages in unavailable ranges 72 | Sep 17 07:25:04 TestnetSupernode01 kernel: On node 0, zone DMA: 97 pages in unavailable ranges 73 | Sep 17 07:25:04 TestnetSupernode01 kernel: On node 0, zone Normal: 38 pages in unavailable ranges 74 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PM-Timer IO Port: 0x608 75 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1]) 76 | Sep 17 07:25:04 TestnetSupernode01 kernel: IOAPIC[0]: apic_id 0, version 17, address 0xfec00000, GSI 0-23 77 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl) 78 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: INT_SRC_OVR (bus 0 bus_irq 5 global_irq 5 high level) 79 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level) 80 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: INT_SRC_OVR (bus 0 bus_irq 10 global_irq 10 high level) 81 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: INT_SRC_OVR (bus 0 bus_irq 11 global_irq 11 high level) 82 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Using ACPI (MADT) for SMP configuration information 83 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: HPET id: 0x8086a201 base: 0xfed00000 84 | Sep 17 07:25:04 TestnetSupernode01 kernel: TSC deadline timer available 85 | Sep 17 07:25:04 TestnetSupernode01 kernel: smpboot: Allowing 8 CPUs, 0 hotplug CPUs 86 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: KVM setup pv remote TLB flush 87 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: setup PV sched yield 88 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0x00000000-0x00000fff] 89 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0x0009f000-0x0009ffff] 90 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0x000a0000-0x000effff] 91 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0x000f0000-0x000fffff] 92 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0xbffda000-0xbfffffff] 93 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0xc0000000-0xfeffbfff] 94 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0xfeffc000-0xfeffffff] 95 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0xff000000-0xfffbffff] 96 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: hibernation: Registered nosave memory: [mem 0xfffc0000-0xffffffff] 97 | Sep 17 07:25:04 TestnetSupernode01 kernel: [mem 0xc0000000-0xfeffbfff] available for PCI devices 98 | Sep 17 07:25:04 TestnetSupernode01 kernel: Booting paravirtualized kernel on KVM 99 | Sep 17 07:25:04 TestnetSupernode01 kernel: clocksource: refined-jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645519600211568 ns 100 | Sep 17 07:25:04 TestnetSupernode01 kernel: setup_percpu: NR_CPUS:8192 nr_cpumask_bits:8 nr_cpu_ids:8 nr_node_ids:1 101 | Sep 17 07:25:04 TestnetSupernode01 kernel: percpu: Embedded 61 pages/cpu s212992 r8192 d28672 u262144 102 | Sep 17 07:25:04 TestnetSupernode01 kernel: pcpu-alloc: s212992 r8192 d28672 u262144 alloc=1*2097152 103 | Sep 17 07:25:04 TestnetSupernode01 kernel: pcpu-alloc: [0] 0 1 2 3 4 5 6 7 104 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 0, msr 7a1c33080 105 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: PV spinlocks enabled 106 | Sep 17 07:25:04 TestnetSupernode01 kernel: PV qspinlock hash table entries: 256 (order: 0, 4096 bytes, linear) 107 | Sep 17 07:25:04 TestnetSupernode01 kernel: Built 1 zonelists, mobility grouping on. Total pages: 7741146 108 | Sep 17 07:25:04 TestnetSupernode01 kernel: Policy zone: Normal 109 | Sep 17 07:25:04 TestnetSupernode01 kernel: Kernel command line: BOOT_IMAGE=/vmlinuz-5.15.0-79-generic root=UUID=11c5afcd-dda3-45f8-b3e4-28d334354465 ro net.ifnames=0 biosdevname=0 nomodeset 110 | Sep 17 07:25:04 TestnetSupernode01 kernel: You have booted with nomodeset. This means your GPU drivers are DISABLED 111 | Sep 17 07:25:04 TestnetSupernode01 kernel: Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly 112 | Sep 17 07:25:04 TestnetSupernode01 kernel: Unless you actually understand what nomodeset does, you should reboot without enabling it 113 | Sep 17 07:25:04 TestnetSupernode01 kernel: Unknown kernel command line parameters "BOOT_IMAGE=/vmlinuz-5.15.0-79-generic biosdevname=0", will be passed to user space. 114 | Sep 17 07:25:04 TestnetSupernode01 kernel: Dentry cache hash table entries: 4194304 (order: 13, 33554432 bytes, linear) 115 | Sep 17 07:25:04 TestnetSupernode01 kernel: Inode-cache hash table entries: 2097152 (order: 12, 16777216 bytes, linear) 116 | Sep 17 07:25:04 TestnetSupernode01 kernel: mem auto-init: stack:off, heap alloc:on, heap free:off 117 | Sep 17 07:25:04 TestnetSupernode01 kernel: Memory: 30723624K/31456736K available (16393K kernel code, 4390K rwdata, 10860K rodata, 3352K init, 18720K bss, 732852K reserved, 0K cma-reserved) 118 | Sep 17 07:25:04 TestnetSupernode01 kernel: SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=8, Nodes=1 119 | Sep 17 07:25:04 TestnetSupernode01 kernel: ftrace: allocating 50633 entries in 198 pages 120 | Sep 17 07:25:04 TestnetSupernode01 kernel: ftrace: allocated 198 pages with 4 groups 121 | Sep 17 07:25:04 TestnetSupernode01 kernel: rcu: Hierarchical RCU implementation. 122 | Sep 17 07:25:04 TestnetSupernode01 kernel: rcu: RCU restricting CPUs from NR_CPUS=8192 to nr_cpu_ids=8. 123 | Sep 17 07:25:04 TestnetSupernode01 kernel: Rude variant of Tasks RCU enabled. 124 | Sep 17 07:25:04 TestnetSupernode01 kernel: Tracing variant of Tasks RCU enabled. 125 | Sep 17 07:25:04 TestnetSupernode01 kernel: rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies. 126 | Sep 17 07:25:04 TestnetSupernode01 kernel: rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=8 127 | Sep 17 07:25:04 TestnetSupernode01 kernel: NR_IRQS: 524544, nr_irqs: 488, preallocated irqs: 16 128 | Sep 17 07:25:04 TestnetSupernode01 kernel: random: crng init done 129 | Sep 17 07:25:04 TestnetSupernode01 kernel: Console: colour dummy device 80x25 130 | Sep 17 07:25:04 TestnetSupernode01 kernel: printk: console [tty0] enabled 131 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Core revision 20210730 132 | Sep 17 07:25:04 TestnetSupernode01 kernel: clocksource: hpet: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 19112604467 ns 133 | Sep 17 07:25:04 TestnetSupernode01 kernel: APIC: Switch to symmetric I/O mode setup 134 | Sep 17 07:25:04 TestnetSupernode01 kernel: x2apic enabled 135 | Sep 17 07:25:04 TestnetSupernode01 kernel: Switched APIC routing to physical x2apic. 136 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: setup PV IPIs 137 | Sep 17 07:25:04 TestnetSupernode01 kernel: ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1 138 | Sep 17 07:25:04 TestnetSupernode01 kernel: tsc: Marking TSC unstable due to TSCs unsynchronized 139 | Sep 17 07:25:04 TestnetSupernode01 kernel: Calibrating delay loop (skipped) preset value.. 5589.49 BogoMIPS (lpj=11178992) 140 | Sep 17 07:25:04 TestnetSupernode01 kernel: pid_max: default: 32768 minimum: 301 141 | Sep 17 07:25:04 TestnetSupernode01 kernel: LSM: Security Framework initializing 142 | Sep 17 07:25:04 TestnetSupernode01 kernel: landlock: Up and running. 143 | Sep 17 07:25:04 TestnetSupernode01 kernel: Yama: becoming mindful. 144 | Sep 17 07:25:04 TestnetSupernode01 kernel: AppArmor: AppArmor initialized 145 | Sep 17 07:25:04 TestnetSupernode01 kernel: Mount-cache hash table entries: 65536 (order: 7, 524288 bytes, linear) 146 | Sep 17 07:25:04 TestnetSupernode01 kernel: Mountpoint-cache hash table entries: 65536 (order: 7, 524288 bytes, linear) 147 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/cpu: User Mode Instruction Prevention (UMIP) activated 148 | Sep 17 07:25:04 TestnetSupernode01 kernel: Last level iTLB entries: 4KB 512, 2MB 255, 4MB 127 149 | Sep 17 07:25:04 TestnetSupernode01 kernel: Last level dTLB entries: 4KB 512, 2MB 255, 4MB 127, 1GB 0 150 | Sep 17 07:25:04 TestnetSupernode01 kernel: Spectre V1 : Mitigation: usercopy/swapgs barriers and __user pointer sanitization 151 | Sep 17 07:25:04 TestnetSupernode01 kernel: Spectre V2 : Mitigation: Retpolines 152 | Sep 17 07:25:04 TestnetSupernode01 kernel: Spectre V2 : Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch 153 | Sep 17 07:25:04 TestnetSupernode01 kernel: Spectre V2 : Spectre v2 / SpectreRSB : Filling RSB on VMEXIT 154 | Sep 17 07:25:04 TestnetSupernode01 kernel: Spectre V2 : Enabling Speculation Barrier for firmware calls 155 | Sep 17 07:25:04 TestnetSupernode01 kernel: RETBleed: Mitigation: untrained return thunk 156 | Sep 17 07:25:04 TestnetSupernode01 kernel: Spectre V2 : mitigation: Enabling conditional Indirect Branch Prediction Barrier 157 | Sep 17 07:25:04 TestnetSupernode01 kernel: Speculative Store Bypass: Mitigation: Speculative Store Bypass disabled via prctl and seccomp 158 | Sep 17 07:25:04 TestnetSupernode01 kernel: Freeing SMP alternatives memory: 44K 159 | Sep 17 07:25:04 TestnetSupernode01 kernel: smpboot: CPU0: AMD EPYC 7282 16-Core Processor (family: 0x17, model: 0x31, stepping: 0x0) 160 | Sep 17 07:25:04 TestnetSupernode01 kernel: Performance Events: Fam17h+ core perfctr, AMD PMU driver. 161 | Sep 17 07:25:04 TestnetSupernode01 kernel: ... version: 0 162 | Sep 17 07:25:04 TestnetSupernode01 kernel: ... bit width: 48 163 | Sep 17 07:25:04 TestnetSupernode01 kernel: ... generic registers: 6 164 | Sep 17 07:25:04 TestnetSupernode01 kernel: ... value mask: 0000ffffffffffff 165 | Sep 17 07:25:04 TestnetSupernode01 kernel: ... max period: 00007fffffffffff 166 | Sep 17 07:25:04 TestnetSupernode01 kernel: ... fixed-purpose events: 0 167 | Sep 17 07:25:04 TestnetSupernode01 kernel: ... event mask: 000000000000003f 168 | Sep 17 07:25:04 TestnetSupernode01 kernel: rcu: Hierarchical SRCU implementation. 169 | Sep 17 07:25:04 TestnetSupernode01 kernel: smp: Bringing up secondary CPUs ... 170 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86: Booting SMP configuration: 171 | Sep 17 07:25:04 TestnetSupernode01 kernel: .... node #0, CPUs: #1 172 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 1, msr 5c9a01041, secondary cpu clock 173 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 1, msr 7a1c73080 174 | Sep 17 07:25:04 TestnetSupernode01 kernel: #2 175 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 2, msr 5c9a01081, secondary cpu clock 176 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 2, msr 7a1cb3080 177 | Sep 17 07:25:04 TestnetSupernode01 kernel: #3 178 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 3, msr 5c9a010c1, secondary cpu clock 179 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 3, msr 7a1cf3080 180 | Sep 17 07:25:04 TestnetSupernode01 kernel: #4 181 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 4, msr 5c9a01101, secondary cpu clock 182 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 4, msr 7a1d33080 183 | Sep 17 07:25:04 TestnetSupernode01 kernel: #5 184 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 5, msr 5c9a01141, secondary cpu clock 185 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 5, msr 7a1d73080 186 | Sep 17 07:25:04 TestnetSupernode01 kernel: #6 187 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 6, msr 5c9a01181, secondary cpu clock 188 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 6, msr 7a1db3080 189 | Sep 17 07:25:04 TestnetSupernode01 kernel: #7 190 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-clock: cpu 7, msr 5c9a011c1, secondary cpu clock 191 | Sep 17 07:25:04 TestnetSupernode01 kernel: kvm-guest: stealtime: cpu 7, msr 7a1df3080 192 | Sep 17 07:25:04 TestnetSupernode01 kernel: smp: Brought up 1 node, 8 CPUs 193 | Sep 17 07:25:04 TestnetSupernode01 kernel: smpboot: Max logical packages: 1 194 | Sep 17 07:25:04 TestnetSupernode01 kernel: smpboot: Total of 8 processors activated (44715.96 BogoMIPS) 195 | Sep 17 07:25:04 TestnetSupernode01 kernel: devtmpfs: initialized 196 | Sep 17 07:25:04 TestnetSupernode01 kernel: x86/mm: Memory block size: 128MB 197 | Sep 17 07:25:04 TestnetSupernode01 kernel: clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns 198 | Sep 17 07:25:04 TestnetSupernode01 kernel: futex hash table entries: 2048 (order: 5, 131072 bytes, linear) 199 | Sep 17 07:25:04 TestnetSupernode01 kernel: pinctrl core: initialized pinctrl subsystem 200 | Sep 17 07:25:04 TestnetSupernode01 kernel: PM: RTC time: 11:24:57, date: 2023-09-17 201 | Sep 17 07:25:04 TestnetSupernode01 kernel: NET: Registered PF_NETLINK/PF_ROUTE protocol family 202 | Sep 17 07:25:04 TestnetSupernode01 kernel: DMA: preallocated 4096 KiB GFP_KERNEL pool for atomic allocations 203 | Sep 17 07:25:04 TestnetSupernode01 kernel: DMA: preallocated 4096 KiB GFP_KERNEL|GFP_DMA pool for atomic allocations 204 | Sep 17 07:25:04 TestnetSupernode01 kernel: DMA: preallocated 4096 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations 205 | Sep 17 07:25:04 TestnetSupernode01 kernel: audit: initializing netlink subsys (disabled) 206 | Sep 17 07:25:04 TestnetSupernode01 kernel: thermal_sys: Registered thermal governor 'fair_share' 207 | Sep 17 07:25:04 TestnetSupernode01 kernel: thermal_sys: Registered thermal governor 'bang_bang' 208 | Sep 17 07:25:04 TestnetSupernode01 kernel: thermal_sys: Registered thermal governor 'step_wise' 209 | Sep 17 07:25:04 TestnetSupernode01 kernel: thermal_sys: Registered thermal governor 'user_space' 210 | Sep 17 07:25:04 TestnetSupernode01 kernel: thermal_sys: Registered thermal governor 'power_allocator' 211 | Sep 17 07:25:04 TestnetSupernode01 kernel: EISA bus registered 212 | Sep 17 07:25:04 TestnetSupernode01 kernel: audit: type=2000 audit(1694949898.995:1): state=initialized audit_enabled=0 res=1 213 | Sep 17 07:25:04 TestnetSupernode01 kernel: cpuidle: using governor ladder 214 | Sep 17 07:25:04 TestnetSupernode01 kernel: cpuidle: using governor menu 215 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: bus type PCI registered 216 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5 217 | Sep 17 07:25:04 TestnetSupernode01 kernel: PCI: Using configuration type 1 for base access 218 | Sep 17 07:25:04 TestnetSupernode01 kernel: PCI: Using configuration type 1 for extended access 219 | Sep 17 07:25:04 TestnetSupernode01 kernel: kprobes: kprobe jump-optimization is enabled. All kprobes are optimized if possible. 220 | Sep 17 07:25:04 TestnetSupernode01 kernel: HugeTLB registered 1.00 GiB page size, pre-allocated 0 pages 221 | Sep 17 07:25:04 TestnetSupernode01 kernel: HugeTLB registered 2.00 MiB page size, pre-allocated 0 pages 222 | Sep 17 07:25:04 TestnetSupernode01 kernel: fbcon: Taking over console 223 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Added _OSI(Module Device) 224 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Added _OSI(Processor Device) 225 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Added _OSI(3.0 _SCP Extensions) 226 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Added _OSI(Processor Aggregator Device) 227 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Added _OSI(Linux-Dell-Video) 228 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Added _OSI(Linux-Lenovo-NV-HDMI-Audio) 229 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Added _OSI(Linux-HPI-Hybrid-Graphics) 230 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: 2 ACPI AML tables successfully acquired and loaded 231 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Interpreter enabled 232 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PM: (supports S0 S3 S4 S5) 233 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Using IOAPIC for interrupt routing 234 | Sep 17 07:25:04 TestnetSupernode01 kernel: PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug 235 | Sep 17 07:25:04 TestnetSupernode01 kernel: PCI: Using E820 reservations for host bridge windows 236 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: Enabled 3 GPEs in block 00 to 0F 237 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff]) 238 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpi PNP0A03:00: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI EDR HPX-Type3] 239 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [3] registered 240 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [4] registered 241 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [5] registered 242 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [6] registered 243 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [7] registered 244 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [8] registered 245 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [9] registered 246 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [10] registered 247 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [11] registered 248 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [12] registered 249 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [13] registered 250 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [14] registered 251 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [15] registered 252 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [16] registered 253 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [17] registered 254 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [18] registered 255 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [19] registered 256 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [20] registered 257 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [21] registered 258 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [22] registered 259 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [23] registered 260 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [24] registered 261 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [25] registered 262 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [26] registered 263 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [27] registered 264 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [28] registered 265 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [29] registered 266 | Sep 17 07:25:04 TestnetSupernode01 kernel: PCI host bridge to bus 0000:00 267 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:00: root bus resource [bus 00-ff] 268 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:00: root bus resource [io 0x0000-0x0cf7 window] 269 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:00: root bus resource [io 0x0d00-0xffff window] 270 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window] 271 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:00: root bus resource [mem 0xc0000000-0xfebfffff window] 272 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:00: root bus resource [mem 0x7c0000000-0x83fffffff window] 273 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:00.0: [8086:1237] type 00 class 0x060000 274 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.0: [8086:7000] type 00 class 0x060100 275 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.1: [8086:7010] type 00 class 0x010180 276 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.1: reg 0x20: [io 0xe0c0-0xe0cf] 277 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.1: legacy IDE quirk: reg 0x10: [io 0x01f0-0x01f7] 278 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.1: legacy IDE quirk: reg 0x14: [io 0x03f6] 279 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.1: legacy IDE quirk: reg 0x18: [io 0x0170-0x0177] 280 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.1: legacy IDE quirk: reg 0x1c: [io 0x0376] 281 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.2: [8086:7020] type 00 class 0x0c0300 282 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.2: reg 0x20: [io 0xe080-0xe09f] 283 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.3: [8086:7113] type 00 class 0x068000 284 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.3: quirk: [io 0x0600-0x063f] claimed by PIIX4 ACPI 285 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:01.3: quirk: [io 0x0700-0x070f] claimed by PIIX4 SMB 286 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: [1234:1111] type 00 class 0x030000 287 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: reg 0x10: [mem 0xfd000000-0xfdffffff pref] 288 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: reg 0x18: [mem 0xfea50000-0xfea50fff] 289 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: reg 0x30: [mem 0xfea40000-0xfea4ffff pref] 290 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: BAR 0: assigned to efifb 291 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: Video device with shadowed ROM at [mem 0x000c0000-0x000dffff] 292 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:03.0: [1af4:1002] type 00 class 0x00ff00 293 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:03.0: reg 0x10: [io 0xe000-0xe03f] 294 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:03.0: reg 0x20: [mem 0xfe400000-0xfe403fff 64bit pref] 295 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:05.0: [1af4:1004] type 00 class 0x010000 296 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:05.0: reg 0x10: [io 0xe040-0xe07f] 297 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:05.0: reg 0x14: [mem 0xfea51000-0xfea51fff] 298 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:05.0: reg 0x20: [mem 0xfe404000-0xfe407fff 64bit pref] 299 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:12.0: [1af4:1000] type 00 class 0x020000 300 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:12.0: reg 0x10: [io 0xe0a0-0xe0bf] 301 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:12.0: reg 0x14: [mem 0xfea52000-0xfea52fff] 302 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:12.0: reg 0x20: [mem 0xfe408000-0xfe40bfff 64bit pref] 303 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:12.0: reg 0x30: [mem 0xfea00000-0xfea3ffff pref] 304 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1e.0: [1b36:0001] type 01 class 0x060400 305 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1e.0: reg 0x10: [mem 0xfea53000-0xfea530ff 64bit] 306 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1f.0: [1b36:0001] type 01 class 0x060400 307 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1f.0: reg 0x10: [mem 0xfea54000-0xfea540ff 64bit] 308 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:01: extended config space not accessible 309 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [0] registered 310 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [1] registered 311 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [2] registered 312 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [3-2] registered 313 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [4-2] registered 314 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [5-2] registered 315 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [6-2] registered 316 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [7-2] registered 317 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [8-2] registered 318 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [9-2] registered 319 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [10-2] registered 320 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [11-2] registered 321 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [12-2] registered 322 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [13-2] registered 323 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [14-2] registered 324 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [15-2] registered 325 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [16-2] registered 326 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [17-2] registered 327 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [18-2] registered 328 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [19-2] registered 329 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [20-2] registered 330 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [21-2] registered 331 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [22-2] registered 332 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [23-2] registered 333 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [24-2] registered 334 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [25-2] registered 335 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [26-2] registered 336 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [27-2] registered 337 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [28-2] registered 338 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [29-2] registered 339 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [30] registered 340 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [31] registered 341 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1e.0: PCI bridge to [bus 01] 342 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1e.0: bridge window [io 0xd000-0xdfff] 343 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1e.0: bridge window [mem 0xfe800000-0xfe9fffff] 344 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1e.0: bridge window [mem 0xfe200000-0xfe3fffff 64bit pref] 345 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci_bus 0000:02: extended config space not accessible 346 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [0-2] registered 347 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [1-2] registered 348 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [2-2] registered 349 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [3-3] registered 350 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [4-3] registered 351 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [5-3] registered 352 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [6-3] registered 353 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [7-3] registered 354 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [8-3] registered 355 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [9-3] registered 356 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [10-3] registered 357 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [11-3] registered 358 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [12-3] registered 359 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [13-3] registered 360 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [14-3] registered 361 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [15-3] registered 362 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [16-3] registered 363 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [17-3] registered 364 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [18-3] registered 365 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [19-3] registered 366 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [20-3] registered 367 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [21-3] registered 368 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [22-3] registered 369 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [23-3] registered 370 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [24-3] registered 371 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [25-3] registered 372 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [26-3] registered 373 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [27-3] registered 374 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [28-3] registered 375 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [29-3] registered 376 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [30-2] registered 377 | Sep 17 07:25:04 TestnetSupernode01 kernel: acpiphp: Slot [31-2] registered 378 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1f.0: PCI bridge to [bus 02] 379 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1f.0: bridge window [io 0xc000-0xcfff] 380 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1f.0: bridge window [mem 0xfe600000-0xfe7fffff] 381 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:1f.0: bridge window [mem 0xfe000000-0xfe1fffff 64bit pref] 382 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PCI: Interrupt link LNKA configured for IRQ 10 383 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PCI: Interrupt link LNKB configured for IRQ 10 384 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PCI: Interrupt link LNKC configured for IRQ 11 385 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PCI: Interrupt link LNKD configured for IRQ 11 386 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: PCI: Interrupt link LNKS configured for IRQ 9 387 | Sep 17 07:25:04 TestnetSupernode01 kernel: iommu: Default domain type: Translated 388 | Sep 17 07:25:04 TestnetSupernode01 kernel: iommu: DMA domain TLB invalidation policy: lazy mode 389 | Sep 17 07:25:04 TestnetSupernode01 kernel: SCSI subsystem initialized 390 | Sep 17 07:25:04 TestnetSupernode01 kernel: libata version 3.00 loaded. 391 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: vgaarb: setting as boot VGA device 392 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: vgaarb: VGA device added: decodes=io+mem,owns=io+mem,locks=none 393 | Sep 17 07:25:04 TestnetSupernode01 kernel: pci 0000:00:02.0: vgaarb: bridge control possible 394 | Sep 17 07:25:04 TestnetSupernode01 kernel: vgaarb: loaded 395 | Sep 17 07:25:04 TestnetSupernode01 kernel: ACPI: bus type USB registered 396 | Sep 17 07:25:04 TestnetSupernode01 kernel: usbcore: registered new interface driver usbfs 397 | Sep 17 07:25:04 TestnetSupernode01 kernel: usbcore: registered new interface driver hub 398 | Sep 17 07:25:04 TestnetSupernode01 kernel: usbcore: registered new device driver usb 399 | Sep 17 07:25:04 TestnetSupernode01 kernel: pps_core: LinuxPPS API ver. 1 registered 400 | Sep 17 07:25:04 TestnetSupernode01 kernel: pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti 401 | -------------------------------------------------------------------------------- /automatic_log_collector_and_analyzer.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import sys 4 | import subprocess 5 | import mmap 6 | import glob 7 | import re 8 | import hashlib 9 | import json 10 | import signal 11 | import sqlite3 12 | import shlex 13 | import shutil 14 | import threading 15 | from functools import lru_cache 16 | import datetime as dt 17 | from datetime import datetime, timedelta 18 | from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor 19 | from multiprocessing import Pool, Manager 20 | from typing import List, Dict, Any, Tuple 21 | from tqdm import tqdm 22 | import boto3 23 | import paramiko 24 | import sqlalchemy as sa 25 | from sqlalchemy import and_, Column, Integer, JSON, ForeignKey 26 | from sqlalchemy.orm import relationship, sessionmaker, Session 27 | from sqlalchemy.exc import OperationalError, IntegrityError 28 | from sqlalchemy.sql.expression import or_ 29 | import redis 30 | import cProfile 31 | import pstats 32 | import pandas as pd 33 | import concurrent.futures 34 | from collections import Counter 35 | import yaml 36 | from decouple import config 37 | 38 | SSH_TIMEOUT_SECONDS = config("SSH_TIMEOUT_SECONDS", cast=int) 39 | BASE_PATH = config("BASE_PATH", cast=str) 40 | 41 | lock_file = "automatic_log_collector_and_analyzer.lock" 42 | if os.path.exists(lock_file): 43 | print("Lock file exists, another instance of the script is running. Exiting.") 44 | sys.exit() 45 | open(lock_file, 'w').close() 46 | Base = sa.orm.declarative_base() 47 | 48 | # Prerequisites under Ubuntu 20+: 49 | # sudo apt update && sudo apt install redis pipx -y && pipx ensurepath && pipx install datasette 50 | 51 | class LogEntry(Base): 52 | __tablename__ = 'log_entries' 53 | id = sa.Column(sa.Integer, primary_key=True) 54 | hash_id = sa.Column(sa.String, index=True) 55 | instance_id = sa.Column(sa.String) 56 | machine_name = sa.Column(sa.String, index=True) 57 | public_ip = sa.Column(sa.String, index=True) 58 | log_file_source = sa.Column(sa.String, index=True) 59 | timestamp = sa.Column(sa.DateTime) 60 | message = sa.Column(sa.String) 61 | 62 | 63 | class SNStatus(Base): 64 | __tablename__ = 'sn_status' 65 | id = sa.Column(sa.Integer, primary_key=True) 66 | public_ip = sa.Column(sa.String) 67 | instance_name = sa.Column(sa.String) 68 | datetime_of_data = sa.Column(sa.String) 69 | version = sa.Column(sa.Integer) 70 | protocolversion = sa.Column(sa.Integer) 71 | walletversion = sa.Column(sa.Integer) 72 | chain = sa.Column(sa.String) 73 | balance = sa.Column(sa.Float) 74 | blocks = sa.Column(sa.Integer) 75 | timeoffset = sa.Column(sa.Integer) 76 | connections = sa.Column(sa.Integer) 77 | proxy = sa.Column(sa.String) 78 | difficulty = sa.Column(sa.Float) 79 | testnet = sa.Column(sa.Boolean) # Add this line 80 | keypoololdest = sa.Column(sa.Integer) 81 | keypoolsize = sa.Column(sa.Integer) 82 | paytxfee = sa.Column(sa.Float) 83 | relayfee = sa.Column(sa.Float) 84 | errors = sa.Column(sa.String) 85 | masternode_collateral_txid_and_outpoint = sa.Column(sa.String) 86 | masternode_collateral_address = sa.Column(sa.String) 87 | sn_pastelid_pubkey = sa.Column(sa.String) 88 | sn_alias = sa.Column(sa.String) 89 | sn_status = sa.Column(sa.String) 90 | def to_dict(self): 91 | return {c.name: getattr(self, c.name) for c in self.__table__.columns} 92 | 93 | 94 | class SNMasternodeStatus(Base): 95 | __tablename__ = 'sn_masternode_status' 96 | public_ip = sa.Column(sa.String) 97 | instance_name = sa.Column(sa.String) 98 | datetime_of_data = sa.Column(sa.String) 99 | id = sa.Column(sa.Integer, primary_key=True) 100 | masternode_collateral_txid_and_outpoint = sa.Column(sa.String) 101 | masternode_status_message = sa.Column(sa.String) 102 | protocol_version = sa.Column(sa.String) 103 | masternode_collateral_address = sa.Column(sa.String) 104 | datetime_last_seen = sa.Column(sa.String) 105 | active_seconds = sa.Column(sa.String) 106 | datetime_last_paid = sa.Column(sa.String) 107 | last_paid_blockheight = sa.Column(sa.String) 108 | ip_address_and_port = sa.Column(sa.String) 109 | rank_as_of_block_height = Column(sa.Integer) 110 | masternode_rank = Column(sa.Integer) 111 | sn_pastelid_pubkey = Column(sa.String) 112 | def to_dict(self): 113 | return {c.name: getattr(self, c.name) for c in self.__table__.columns} 114 | 115 | 116 | class SNNetworkActivityNetstat(Base): 117 | __tablename__ = 'sn_network_activity_netstat' 118 | id = sa.Column(sa.Integer, primary_key=True) 119 | public_ip = sa.Column(sa.String) 120 | instance_name = sa.Column(sa.String) 121 | datetime_of_data = sa.Column(sa.String) 122 | netstat__proto = sa.Column(sa.String) 123 | netstat__recv_q = sa.Column(sa.Integer) 124 | netstat__send_q = sa.Column(sa.Integer) 125 | netstat__local_address = sa.Column(sa.String) 126 | netstat__foreign_address = sa.Column(sa.String) 127 | netstat__state = sa.Column(sa.String) 128 | netstat__pid_program_name = sa.Column(sa.String) 129 | def to_dict(self): 130 | return {c.name: getattr(self, c.name) for c in self.__table__.columns} 131 | 132 | 133 | class SNNetworkActivityLSOF(Base): 134 | __tablename__ = 'sn_network_activity_lsof' 135 | id = sa.Column(sa.Integer, primary_key=True) 136 | public_ip = sa.Column(sa.String) 137 | instance_name = sa.Column(sa.String) 138 | datetime_of_data = sa.Column(sa.String) 139 | lsof__command = sa.Column(sa.String) 140 | lsof__pid = sa.Column(sa.Integer) 141 | lsof__user = sa.Column(sa.String) 142 | lsof__fd = sa.Column(sa.String) 143 | lsof__type = sa.Column(sa.String) 144 | lsof__device = sa.Column(sa.String) 145 | lsof__size_off = sa.Column(sa.String) 146 | lsof__node = sa.Column(sa.String) 147 | lsof__name = sa.Column(sa.String) 148 | def to_dict(self): 149 | return {c.name: getattr(self, c.name) for c in self.__table__.columns} 150 | 151 | 152 | class SNNetworkActivitySS(Base): 153 | __tablename__ = 'sn_network_activity_ss' 154 | id = sa.Column(sa.Integer, primary_key=True) 155 | public_ip = sa.Column(sa.String) 156 | instance_name = sa.Column(sa.String) 157 | datetime_of_data = sa.Column(sa.String) 158 | ss__state = sa.Column(sa.String) 159 | ss__recv_q = sa.Column(sa.Integer) 160 | ss__send_q = sa.Column(sa.Integer) 161 | ss__local_address_port = sa.Column(sa.String) 162 | ss__peer_address_port = sa.Column(sa.String) 163 | ss__process = sa.Column(sa.String) 164 | def to_dict(self): 165 | return {c.name: getattr(self, c.name) for c in self.__table__.columns} 166 | 167 | 168 | class NodeHealthChecks(Base): 169 | __tablename__ = 'node_health_checks' 170 | id = Column(Integer, primary_key=True) 171 | missing_status_responses = Column(JSON) # JSON type is used for list of strings 172 | out_of_sync_nodes = Column(JSON) # JSON type is used for complex nested structure 173 | nodes_with_zero_connections = Column(JSON) # JSON type is used for list of strings 174 | 175 | 176 | class NodeMasternodeHealthChecks(Base): 177 | __tablename__ = 'node_masternode_health_checks' 178 | id = Column(Integer, primary_key=True) 179 | masternode_rank_outlier_report_explanations = Column(JSON, nullable=True) 180 | all_new_start_required = Column(JSON, nullable=True) 181 | supernodes_reported_to_be_in_new_start_required_mode = Column(JSON, nullable=True) 182 | 183 | 184 | class EntriesBeforeAndAfterPanics(Base): 185 | __tablename__ = 'entries_before_and_after_panics' 186 | id = Column(Integer, primary_key=True) 187 | log_entry_id = Column(Integer, ForeignKey('log_entries.id')) 188 | log_entry = relationship("LogEntry") 189 | 190 | class MiscErrorEntries(Base): 191 | __tablename__ = 'misc_error_entries' 192 | id = Column(Integer, primary_key=True) 193 | log_entry_id = Column(Integer, ForeignKey('log_entries.id')) 194 | log_entry = relationship("LogEntry") 195 | 196 | def get_instance_name(tags): 197 | for tag in tags: 198 | if tag['Key'] == 'Name': 199 | return tag['Value'] 200 | return None 201 | 202 | def get_instances_with_name_prefix(name_prefix, aws_access_key_id, aws_secret_access_key, aws_region): 203 | ec2 = boto3.resource('ec2', region_name=aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) 204 | instances = ec2.instances.filter( 205 | Filters=[ 206 | {'Name': 'tag:Name', 'Values': [f'{name_prefix}*']}, 207 | {'Name': 'instance-state-name', 'Values': ['running']} 208 | ] 209 | ) 210 | instances = sorted(instances, key=lambda instance: get_instance_name(instance.tags)) 211 | return instances 212 | 213 | def get_inventory(): 214 | with open(config("ANSIBLE_INVENTORY_FILE", cast=str), 'r') as f: 215 | return yaml.safe_load(f) 216 | 217 | def get_ssh_key_and_user(instance_name): 218 | inventory = get_inventory() 219 | hosts = inventory.get('all', {}).get('hosts', {}) 220 | if instance_name in hosts: 221 | host_data = hosts[instance_name] 222 | return host_data.get('ansible_ssh_private_key_file'), inventory['all']['vars'].get('ansible_user', 'ubuntu') 223 | return None, None 224 | 225 | def ssh_connect(ip, user, key_path): 226 | ssh = paramiko.SSHClient() 227 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 228 | try: 229 | ssh.connect(ip, username=user, key_filename=key_path, timeout=5) 230 | ssh.close() 231 | return (True, None) 232 | except paramiko.AuthenticationException: 233 | return (False, "Authentication failed.") 234 | except Exception as e: 235 | return (False, str(e)) 236 | 237 | def calculate_hash_id(log_entry): 238 | hash_content = f"{log_entry['instance_id']}{log_entry['machine_name']}{log_entry['public_ip']}{log_entry['log_file_source']}{log_entry['message']}" 239 | return hashlib.sha256(hash_content.encode()).hexdigest() 240 | 241 | def execute_network_commands_func(command: str) -> str: 242 | result = subprocess.run(command, shell=True, text=True, capture_output=True) 243 | return result.stdout 244 | 245 | def parse_netstat_output(output: str) -> Dict: 246 | lines = output.split('\n')[2:] 247 | records = [] 248 | for line in lines: 249 | if line: 250 | fields = line.split() 251 | record = { 252 | 'netstat__proto': fields[0], 253 | 'netstat__recv_q': int(fields[1]), 254 | 'netstat__send_q': int(fields[2]), 255 | 'netstat__local_address': fields[3], 256 | 'netstat__foreign_address': fields[4], 257 | 'netstat__state': fields[5] if len(fields) > 5 else None, 258 | 'netstat__pid_program_name': fields[6] if len(fields) > 6 else None 259 | } 260 | records.append(record) 261 | return records 262 | 263 | def parse_lsof_output(output: str) -> Dict: 264 | lines = output.split('\n')[1:] 265 | records = [] 266 | for line in lines: 267 | if line: 268 | fields = line.split() 269 | record = { 270 | 'lsof__command': fields[0], 271 | 'lsof__pid': int(fields[1]), 272 | 'lsof__user': fields[2], 273 | 'lsof__fd': fields[3], 274 | 'lsof__type': fields[4], 275 | 'lsof__device': fields[5], 276 | 'lsof__size_off': fields[6], 277 | 'lsof__node': fields[7], 278 | 'lsof__name': fields[8] 279 | } 280 | records.append(record) 281 | return records 282 | 283 | def parse_ss_output(output: str) -> Dict: 284 | lines = output.split('\n')[1:] 285 | records = [] 286 | for line in lines: 287 | if line: 288 | fields = line.split() 289 | record = { 290 | 'ss__state': fields[0], 291 | 'ss__recv_q': int(fields[1]), 292 | 'ss__send_q': int(fields[2]), 293 | 'ss__local_address_port': fields[3], 294 | 'ss__peer_address_port': fields[4], 295 | 'ss__process': ' '.join(fields[5:]) 296 | } 297 | records.append(record) 298 | return records 299 | 300 | def get_sn_network_data(remote_ip, user, key_path, instance_name): 301 | global engine 302 | ssh = paramiko.SSHClient() 303 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 304 | Session = sessionmaker(bind=engine) 305 | session = Session() 306 | try: 307 | ssh.connect(remote_ip, username=user, key_filename=key_path, timeout=SSH_TIMEOUT_SECONDS) 308 | commands_and_parsers = [ 309 | ('sudo netstat -tulnp', parse_netstat_output, SNNetworkActivityNetstat), 310 | ('sudo lsof -i', parse_lsof_output, SNNetworkActivityLSOF), 311 | ('sudo ss -tnp', parse_ss_output, SNNetworkActivitySS), 312 | ] 313 | for command, parser, model in commands_and_parsers: 314 | stdin, stdout, stderr = ssh.exec_command(command) 315 | output = stdout.read().decode('utf-8') 316 | parsed_records = parser(output) 317 | for record in parsed_records: 318 | record.update({ 319 | 'public_ip': remote_ip, 320 | 'instance_name': instance_name, 321 | 'datetime_of_data': datetime.now().isoformat() 322 | }) 323 | sn_network_activity = model(**record) 324 | session.add(sn_network_activity) 325 | session.commit() 326 | except Exception as e: 327 | print(f"Error while getting network data: {str(e)}") 328 | finally: 329 | ssh.close() 330 | session.close() 331 | 332 | def download_logs(remote_ip, user, key_path, instance_name, log_files): 333 | log_files_directory = "downloaded_log_files" 334 | os.makedirs(log_files_directory, exist_ok=True) 335 | current_time = dt.datetime.now() 336 | list_of_local_log_file_names = [] 337 | def is_recently_downloaded(file_name): 338 | if os.path.exists(file_name): 339 | file_modification_time = dt.datetime.fromtimestamp(os.path.getmtime(file_name)) 340 | time_difference = current_time - file_modification_time 341 | return time_difference < dt.timedelta(minutes=5) 342 | return False 343 | def download_log_file(log_file, local_file_name, sudo_prefix="sudo ", skip_first_line=False): 344 | try: 345 | cat_command = f"{sudo_prefix}cat {log_file}" 346 | if skip_first_line: 347 | cat_command = f"{sudo_prefix}tail -n +2 {log_file}" 348 | subprocess.run(['bash', '-c', f'ssh -i {key_path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {user}@{remote_ip} "{cat_command}" > {local_file_name}'], check=True) 349 | print(f"Downloaded log file {log_file} from {instance_name} ({remote_ip}) to {local_file_name}") 350 | except subprocess.CalledProcessError as e: 351 | print(f"Error downloading log file {log_file} from {instance_name} ({remote_ip}): {e}") 352 | with ThreadPoolExecutor() as executor: 353 | futures = [] 354 | for log_file in log_files: 355 | local_file_name = os.path.join(log_files_directory, f"{instance_name.replace(' ', '_')}__{remote_ip.replace('.','_')}__{os.path.basename(log_file)}") 356 | list_of_local_log_file_names += [local_file_name] 357 | if is_recently_downloaded(local_file_name): 358 | print(f"Log file {local_file_name} was downloaded within the past 5 minutes, skipping download.") 359 | continue 360 | futures.append(executor.submit(download_log_file, log_file, local_file_name)) 361 | remote_journalctl_output = f"/home/{user}/journalctl_output.txt" 362 | try: 363 | subprocess.run(["ssh", "-i", key_path, "-o", "StrictHostKeyChecking=no", f"{user}@{remote_ip}", f"sudo journalctl --since '-1 weeks' > {remote_journalctl_output}"], check=True) 364 | except subprocess.CalledProcessError as e: 365 | print(f"Error running journalctl command on {instance_name} ({remote_ip}): {e}") 366 | local_journalctl_file_name = os.path.join(log_files_directory, f"{instance_name.replace(' ', '_')}__{remote_ip.replace('.','_')}__systemd_log.txt") 367 | list_of_local_log_file_names += [local_journalctl_file_name] 368 | if not is_recently_downloaded(local_journalctl_file_name): 369 | futures.append(executor.submit(download_log_file, remote_journalctl_output, local_journalctl_file_name, skip_first_line=True)) 370 | for future in futures: 371 | future.result() 372 | print(f"Finished downloading all log files from {instance_name} ({remote_ip})") 373 | return list_of_local_log_file_names 374 | 375 | def check_sn_status(remote_ip, user, key_path, instance_name): 376 | global engine 377 | ssh = paramiko.SSHClient() 378 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 379 | Session = sessionmaker(bind=engine) 380 | session = Session() 381 | output_dict = {'Error': f'Could not connect to instance {instance_name}.'} 382 | try: 383 | ssh.connect(remote_ip, username=user, key_filename=key_path, timeout=SSH_TIMEOUT_SECONDS) 384 | stdin, stdout, stderr = ssh.exec_command('/home/ubuntu/pastel/pastel-cli masternode status') 385 | mn_status_output = stdout.read().decode('utf-8') 386 | if mn_status_output: 387 | mn_status_output_dict = json.loads(mn_status_output) 388 | else: 389 | mn_status_output_dict = {} 390 | print(f"mn_status_output_dict for {instance_name} ({remote_ip}): {mn_status_output_dict}") 391 | 392 | stdin, stdout, stderr = ssh.exec_command('/home/ubuntu/pastel/pastel-cli getinfo') 393 | output = stdout.read().decode('utf-8') 394 | if output: 395 | output_dict = json.loads(output) 396 | output_dict['public_ip'] = remote_ip 397 | output_dict['instance_name'] = instance_name 398 | output_dict['datetime_of_data'] = datetime.now().isoformat() 399 | if len(mn_status_output_dict) > 0: 400 | output_dict['masternode_collateral_txid_and_outpoint'] = mn_status_output_dict['outpoint'] 401 | output_dict['masternode_collateral_address'] = mn_status_output_dict['payee'] 402 | output_dict['sn_pastelid_pubkey'] = mn_status_output_dict['extKey'] 403 | output_dict['sn_alias'] = mn_status_output_dict['alias'] 404 | output_dict['sn_status'] = mn_status_output_dict['status'] 405 | print(f"output_dict for {instance_name} ({remote_ip}): {output_dict}") 406 | sn_status = SNStatus(**output_dict) 407 | session.add(sn_status) 408 | session.commit() 409 | print(f"Data collected and inserted for {instance_name} ({remote_ip})") 410 | else: 411 | print(f"No output from getinfo command for {instance_name} ({remote_ip})") 412 | except Exception as e: 413 | print(f"Error while checking sn status for {instance_name} ({remote_ip}): {str(e)}") 414 | finally: 415 | ssh.close() 416 | session.close() 417 | return output_dict 418 | 419 | def check_sn_masternode_status(remote_ip, user, key_path, instance_name): 420 | global engine 421 | cmd1 = '/home/ubuntu/pastel/pastel-cli masternode list full' 422 | result1 = subprocess.run(['ssh', '-i', key_path, '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', '-o', f'ConnectTimeout={SSH_TIMEOUT_SECONDS}', f'{user}@{remote_ip}', cmd1], capture_output=True, text=True) 423 | if len(result1.stdout) > 0: 424 | data1 = json.loads(result1.stdout) 425 | cmd2 = '/home/ubuntu/pastel/pastel-cli masternode top' 426 | result2 = subprocess.run(['ssh', '-i', key_path, '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null','-o', f'ConnectTimeout={SSH_TIMEOUT_SECONDS}', f'{user}@{remote_ip}', cmd2], capture_output=True, text=True) 427 | data2 = json.loads(result2.stdout) 428 | rank_as_of_block_height = int(list(data2.keys())[0]) 429 | data2_values = [x for x in list(data2.values())[0]] 430 | masternode_top_dict = {} 431 | for current_value in data2_values: 432 | masternode_collateral_txid_and_outpoint = current_value['outpoint'] 433 | masternode_rank = int(current_value['rank']) 434 | sn_pastelid_pubkey = current_value['extKey'] 435 | masternode_top_dict[masternode_collateral_txid_and_outpoint] = {'masternode_rank': masternode_rank, 'sn_pastelid_pubkey': sn_pastelid_pubkey, 'rank_as_of_block_height': rank_as_of_block_height} 436 | cmd3 = '/home/ubuntu/pastel/pastel-cli masternode list extra' 437 | result3 = subprocess.run(['ssh', '-i', key_path, '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null','-o', f'ConnectTimeout={SSH_TIMEOUT_SECONDS}', f'{user}@{remote_ip}', cmd3], capture_output=True, text=True) 438 | data3 = json.loads(result3.stdout) 439 | Session = sessionmaker(bind=engine) 440 | session = Session() 441 | combined_masternode_status_dict = {} 442 | for key, value in data1.items(): 443 | extra = masternode_top_dict.get(key, {}) 444 | if len(extra) > 0: 445 | sn_pastelid_pubkey = extra.get('sn_pastelid_pubkey') 446 | masternode_rank = extra.get('masternode_rank') 447 | rank_as_of_block_height = extra.get('rank_as_of_block_height') 448 | else: 449 | extra = data3.get(key, {}) 450 | sn_pastelid_pubkey = extra.get('extKey') 451 | masternode_rank = -1 452 | rank_as_of_block_height = -1 453 | values = value.split() 454 | masternode_status_message = values[0] 455 | protocol_version = values[1] 456 | masternode_collateral_address = values[2] 457 | datetime_last_seen = values[3] 458 | active_seconds = values[4] 459 | datetime_last_paid = values[5] 460 | last_paid_blockheight = values[6] 461 | ip_address_and_port = values[7] 462 | status = SNMasternodeStatus( 463 | masternode_collateral_txid_and_outpoint=key, 464 | masternode_status_message=masternode_status_message, 465 | protocol_version=protocol_version, 466 | masternode_collateral_address=masternode_collateral_address, 467 | datetime_last_seen=datetime_last_seen, 468 | active_seconds=active_seconds, 469 | datetime_last_paid=datetime_last_paid, 470 | last_paid_blockheight=last_paid_blockheight, 471 | ip_address_and_port=ip_address_and_port, 472 | sn_pastelid_pubkey=sn_pastelid_pubkey, 473 | masternode_rank = masternode_rank, 474 | rank_as_of_block_height = rank_as_of_block_height, 475 | public_ip=remote_ip, 476 | instance_name=instance_name, 477 | datetime_of_data= datetime.now().isoformat(), 478 | ) 479 | combined_masternode_status_dict[key] = status.to_dict() 480 | session.add(status) 481 | session.commit() 482 | session.close() 483 | else: 484 | combined_masternode_status_dict = {'Error': f'Could not connect to instance {instance_name}.'} 485 | return combined_masternode_status_dict 486 | 487 | @lru_cache(maxsize=None) 488 | def get_current_year(): 489 | return datetime.now().year 490 | 491 | cnode_pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}") 492 | gonode_pattern = re.compile(r"\[.*\]") 493 | dd_service_pattern = re.compile(r"(\d{1,7}) - (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})? \[.*\]") 494 | dd_entry_pattern = re.compile(r'^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - (.*)$') 495 | systemd_pattern = re.compile(r"(\S+)\s(\d+)\s(\d+):(\d+):(\d+)\s(\S+)\s(.*)") 496 | inference_layer_pattern = re.compile(r"(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),%f - pastel_supernode_inference_layer - (?P\w+) - (?P.+)") 497 | 498 | def parse_cnode_log(log_line): 499 | match = cnode_pattern.search(log_line) 500 | if match: 501 | try: 502 | timestamp = datetime.strptime(match.group(), "%Y-%m-%d %H:%M:%S") 503 | message = log_line[match.end():].strip() 504 | return {'timestamp': timestamp, 'message': message} 505 | except ValueError: 506 | return None 507 | return None 508 | 509 | def parse_systemd_log(log_line): 510 | match = systemd_pattern.match(log_line) 511 | if match: 512 | log_parts = match.groups() 513 | timestamp_str = f"{log_parts[0]} {log_parts[1]} {log_parts[2]}:{log_parts[3]}:{log_parts[4]}" 514 | timestamp = datetime.strptime(timestamp_str, "%b %d %H:%M:%S") 515 | current_year = get_current_year() 516 | timestamp = timestamp.replace(year=current_year) 517 | message = log_parts[6] 518 | return {'timestamp': timestamp, 'message': message} 519 | return None 520 | 521 | def parse_gonode_log(log_line): 522 | match = gonode_pattern.search(log_line) 523 | if match: 524 | datetime_str = match.group().strip("[]") 525 | timestamp = datetime.strptime(datetime_str, "%b %d %H:%M:%S.%f") 526 | current_year = get_current_year() 527 | timestamp = timestamp.replace(year=current_year) 528 | message = log_line[match.end():].strip() 529 | return {'timestamp': timestamp, 'message': message} 530 | return None 531 | 532 | def parse_dd_service_log(log_line, last_timestamp=None): 533 | match = dd_service_pattern.search(log_line) 534 | if match: 535 | if match.group(2) is not None: 536 | datetime_str = match.group(2).strip("[]") 537 | timestamp = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S.%f") 538 | elif last_timestamp is not None: 539 | timestamp = last_timestamp 540 | else: 541 | return None 542 | message = log_line[match.end():].strip() 543 | return {'timestamp': timestamp, 'message': message} 544 | return None 545 | 546 | def parse_dd_entry_log(entry_line, last_timestamp=None): 547 | match = dd_entry_pattern.search(entry_line) 548 | if match: 549 | datetime_str = match.group(1) 550 | message = match.group(2).strip() 551 | try: 552 | timestamp = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S.%f") 553 | except ValueError: 554 | timestamp = last_timestamp 555 | return {'timestamp': timestamp, 'message': message} 556 | return None 557 | 558 | def parse_inference_layer_log(line): 559 | match = inference_layer_pattern.match(line) 560 | if match: 561 | try: 562 | timestamp_str = match.group("timestamp") 563 | timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") 564 | level = match.group("level") 565 | message = match.group("message").strip() 566 | return {"timestamp": timestamp, "message": message} 567 | except ValueError as e: 568 | print(f"Error parsing line: {line}\n{e}") 569 | return None 570 | return None 571 | 572 | def parse_logs(local_file_name: str, instance_id: str, machine_name: str, public_ip: str, log_file_source: str) -> List[Dict]: 573 | global earliest_date_cutoff 574 | list_of_ignored_strings = ['sshd['] 575 | log_entries = [] 576 | if "debug.log" in local_file_name: 577 | parse_function = parse_cnode_log 578 | elif ("supernode.log" in local_file_name) or ("hermes.log" in local_file_name): 579 | parse_function = parse_gonode_log 580 | elif "dd-service-log.txt" in local_file_name: 581 | parse_function = parse_dd_service_log 582 | elif "entry.log" in local_file_name: 583 | parse_function = parse_dd_entry_log 584 | elif "systemd_log.txt" in local_file_name: 585 | parse_function = parse_systemd_log 586 | elif "pastel_supernode_inference_layer.log" in local_file_name: 587 | parse_function = parse_inference_layer_log 588 | else: 589 | raise ValueError("Unsupported log file format") 590 | if os.path.getsize(local_file_name) == 0: 591 | print(f"File '{local_file_name}' is empty. Skipping parsing.") 592 | return log_entries 593 | with open(local_file_name, 'r') as f: 594 | mmapped_file = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) 595 | for line in tqdm(iter(mmapped_file.readline, b'')): 596 | line = line.decode('utf-8') 597 | try: 598 | parsed_log = parse_function(line) 599 | if parsed_log: 600 | if len(parsed_log['message']) > 0 and not any(ignored_string in parsed_log['message'] for ignored_string in list_of_ignored_strings): 601 | if parsed_log['timestamp'] < earliest_date_cutoff: 602 | continue 603 | log_entries.append({ 604 | 'instance_id': instance_id, 605 | 'machine_name': machine_name, 606 | 'public_ip': public_ip, 607 | 'log_file_source': log_file_source, 608 | 'timestamp': parsed_log['timestamp'], 609 | 'message': parsed_log['message'] 610 | }) 611 | except Exception as e: # noqa: F841 612 | pass 613 | mmapped_file.close() 614 | print(f"Finished parsing log file '{local_file_name}' with {len(log_entries):,} log entries!") 615 | return log_entries 616 | 617 | def parse_and_append_logs(local_file_name, instance_id, instance_name, public_ip, log_file_source): 618 | parsed_log_entries = parse_logs(local_file_name, instance_id, instance_name, public_ip, log_file_source) 619 | return parsed_log_entries 620 | 621 | def remove_dupes_from_list_but_preserve_order_func(list_of_items): 622 | deduplicated_list = list(dict.fromkeys(list_of_items).keys()) 623 | return deduplicated_list 624 | 625 | def insert_log_entries(parsed_log_entries: List[Dict], session: Session, chunk_size: int = 250000, max_retries: int = 3): 626 | def commit_with_retry(session): 627 | for retry_count in range(max_retries): 628 | try: 629 | session.commit() 630 | break 631 | except OperationalError as e: 632 | if 'database is locked' in str(e).lower() and retry_count < max_retries - 1: 633 | sleep_time = 2 ** retry_count 634 | time.sleep(sleep_time) 635 | else: 636 | raise 637 | except IntegrityError as e: 638 | session.rollback() 639 | print(f"IntegrityError occurred: {e}. Skipping this insert.") 640 | for idx in range(0, len(parsed_log_entries), chunk_size): 641 | chunk = parsed_log_entries[idx:idx + chunk_size] 642 | hash_ids = [calculate_hash_id(log_entry) for log_entry in chunk] 643 | hash_id_existence = redis_client.mget(hash_ids) 644 | hash_id_exists_map = {hash_id: exists for hash_id, exists in zip(hash_ids, hash_id_existence)} 645 | new_log_entries = [] 646 | for log_entry, log_entry_hash_id in zip(chunk, hash_ids): 647 | if not hash_id_exists_map.get(log_entry_hash_id): 648 | redis_client.set(log_entry_hash_id, 1) 649 | new_log_entries.append(LogEntry(hash_id=log_entry_hash_id, **log_entry)) 650 | new_log_entries = remove_dupes_from_list_but_preserve_order_func(new_log_entries) 651 | session.add_all(new_log_entries) 652 | commit_with_retry(session) 653 | 654 | def get_status_info_for_instance(instance_id): 655 | global aws_region, aws_access_key_id, aws_secret_access_key, ssh_key_path, ansible_inventory_file 656 | print(f"Checking {instance_id}...") 657 | with open(ansible_inventory_file, 'r') as f: 658 | ansible_inventory = yaml.safe_load(f) 659 | public_ip, key_path = None, None 660 | instance_name = instance_id 661 | ssh_user = ansible_inventory['all']['vars']['ansible_user'] 662 | hosts = ansible_inventory.get('all', {}).get('hosts', {}) 663 | if instance_id in hosts: 664 | host_info = hosts[instance_id] 665 | public_ip = host_info.get('ansible_host') 666 | key_path = host_info.get('ansible_ssh_private_key_file') 667 | if not public_ip or not key_path: 668 | print(f"Instance {instance_id} is not in the Ansible inventory file, using AWS API to get public IP address and instance name.") 669 | ec2 = boto3.client('ec2', region_name=aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) 670 | instance = ec2.describe_instances(InstanceIds=[instance_id])['Reservations'][0]['Instances'][0] 671 | public_ip = instance['PublicIpAddress'] 672 | instance_name = get_instance_name(instance['Tags']) 673 | print(f"Now checking for SSH connectivity on instance named {instance_name} with instance ID of {instance_id} and public IP address of {public_ip}...") 674 | ssh_status, ssh_error = ssh_connect(public_ip, ssh_user, key_path if key_path else ssh_key_path) 675 | if ssh_status: 676 | print(f"{instance_id} ({instance_name}) is reachable by SSH") 677 | print(f"Now checking the status of the Pastel node on {instance_name} using `pastel-cli getinfo`...") 678 | output_dict = check_sn_status(public_ip, ssh_user, key_path if key_path else ssh_key_path, instance_name) 679 | print(f"Result of `getinfo` command on {instance_name}: {output_dict}") 680 | print(f"Now checking the masternode status of {instance_name} using `pastel-cli masternode list full` and `list extra`...") 681 | combined_masternode_status_dict = check_sn_masternode_status(public_ip, ssh_user, key_path if key_path else ssh_key_path, instance_name) 682 | print(f"Result of `masternode list full` and `list extra` command on {instance_name}: {combined_masternode_status_dict}") 683 | print(f"Now getting network data for {instance_name} using various network commands...") 684 | get_sn_network_data(public_ip, ssh_user, key_path if key_path else ssh_key_path, instance_name) 685 | print('Done getting network data!') 686 | else: 687 | if ssh_error == "Authentication failed.": 688 | print(f"{instance_id} ({instance_name}) has an authentication issue!") 689 | else: 690 | print(f"{instance_id} ({instance_name}) is not reachable by SSH!") 691 | 692 | def insert_log_entries_worker(db_write_queue): 693 | Session = sessionmaker(bind=engine) 694 | none_count = 0 695 | while none_count < len(instance_ids): # Continue processing until all instances have indicated they are done 696 | logs_to_insert = db_write_queue.get() 697 | if logs_to_insert is None: 698 | none_count += 1 699 | else: 700 | session = Session() 701 | try: 702 | session.bulk_insert_mappings(LogEntry, logs_to_insert) 703 | session.commit() 704 | except Exception as e: 705 | print(f"Error inserting log entries: {e}") 706 | finally: 707 | session.close() 708 | db_write_queue.task_done() 709 | 710 | def process_instance(instance_id, db_write_queue): 711 | global aws_region, aws_access_key_id, aws_secret_access_key, ansible_inventory_file 712 | num_cores = os.cpu_count() 713 | if num_cores is not None: 714 | num_cores = max(1, num_cores - 2) 715 | with open(ansible_inventory_file, 'r') as f: 716 | ansible_inventory = yaml.safe_load(f) 717 | print(f"Checking {instance_id}...") 718 | public_ip, key_path = None, None 719 | instance_name = instance_id 720 | ssh_user = ansible_inventory['all']['vars']['ansible_user'] 721 | hosts = ansible_inventory.get('all', {}).get('hosts', {}) 722 | if instance_id in hosts: 723 | host_info = hosts[instance_id] 724 | public_ip = host_info.get('ansible_host') 725 | key_path = host_info.get('ansible_ssh_private_key_file') 726 | if not public_ip or not key_path: 727 | print('Instance is not in the Ansible inventory file, using AWS API to get public IP address and instance name.') 728 | ec2 = boto3.client('ec2', region_name=aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) 729 | instance = ec2.describe_instances(InstanceIds=[instance_id])['Reservations'][0]['Instances'][0] 730 | public_ip = instance['PublicIpAddress'] 731 | instance_name = get_instance_name(instance['Tags']) 732 | Session = sessionmaker(bind=engine) 733 | session = Session() 734 | user = ssh_user 735 | key_path = key_path if key_path else ssh_key_path 736 | print(f"Now checking for SSH connectivity on instance named {instance_name} with instance ID of {instance_id} and public IP address of {public_ip}...") 737 | ssh_status, ssh_error = ssh_connect(public_ip, user, key_path) 738 | if ssh_status: 739 | print(f"{instance_id} ({instance_name}) is reachable by SSH") 740 | log_files = [ 741 | "/home/ubuntu/.pastel/debug.log", 742 | "/home/ubuntu/.pastel/supernode.log", 743 | "/home/ubuntu/.pastel/hermes.log", 744 | "/home/ubuntu/pastel_dupe_detection_service/logs/dd-service-log.txt", 745 | "/home/ubuntu/pastel_dupe_detection_service/logs/entry/entry.log", 746 | "/home/ubuntu/python_inference_layer_server/pastel_supernode_inference_layer.log", 747 | ] 748 | list_of_local_log_file_names = download_logs(public_ip, user, key_path, instance_name, log_files) 749 | print('Now parsing log files...') 750 | all_parsed_log_entries = [] 751 | with ThreadPoolExecutor(max_workers=num_cores) as executor: 752 | futures = [executor.submit(parse_and_append_logs, local_file_name, instance_id, instance_name, public_ip, 753 | os.path.basename(local_file_name).split('.')[0].split('__')[-1].replace('debug', 'cnode').replace('entry', 'dd_entry')) 754 | for local_file_name in list_of_local_log_file_names] 755 | for future in futures: 756 | all_parsed_log_entries.extend(future.result()) 757 | print(f'Done parsing log files on {instance_name}! Total number of log entries: {len(all_parsed_log_entries):,}') 758 | db_write_queue.put(all_parsed_log_entries) 759 | session.close() 760 | print(f'Done inserting log entries into database on {instance_name}! Number of log entries inserted: {len(all_parsed_log_entries):,}') 761 | else: 762 | if ssh_error == "Authentication failed.": 763 | print(f"{instance_id} ({instance_name}) has an authentication issue!") 764 | else: 765 | print(f"{instance_id} ({instance_name}) is not reachable by SSH!") 766 | db_write_queue.put(None) 767 | 768 | def process_panic(panic, k_delta, session): 769 | time_start = panic.timestamp - k_delta 770 | time_end = panic.timestamp + k_delta 771 | related_logs = session.query(LogEntry).filter( 772 | LogEntry.instance_id == panic.instance_id, 773 | LogEntry.timestamp.between(time_start, time_end), 774 | LogEntry.log_file_source.in_(["systemd_log", "supernode", "hermes", "cnode", "dd-service-log", "dd_entry"]) 775 | ).all() 776 | for log in related_logs: 777 | entry = EntriesBeforeAndAfterPanics(log_entry_id=log.id) 778 | session.add(entry) 779 | session.commit() 780 | 781 | def analyze_panics(n_days=2, k_minutes=10): 782 | global engine 783 | print(f'Now analyzing panics over the previous {n_days} days and {k_minutes} minutes before and after the panic...') 784 | Session = sessionmaker(bind=engine) 785 | now = datetime.now() 786 | start_date = now - timedelta(days=n_days) 787 | k_delta = timedelta(minutes=k_minutes) 788 | session = Session() 789 | panics = session.query(LogEntry).filter( 790 | LogEntry.log_file_source == "systemd_log", 791 | LogEntry.timestamp.between(start_date, now), 792 | LogEntry.message.contains("panic") 793 | ).all() 794 | num_cores = os.cpu_count() 795 | if num_cores is not None: 796 | num_cores = max(1, num_cores - 2) 797 | with Pool(processes=num_cores) as pool: 798 | for panic in panics: 799 | session = Session() 800 | pool.apply_async(process_panic, (panic, k_delta, session)) 801 | session.commit() 802 | pool.close() 803 | pool.join() 804 | create_view_sql = """ 805 | CREATE VIEW entries_before_and_after_panics_view AS 806 | SELECT log_entries.* FROM log_entries 807 | JOIN entries_before_and_after_panics 808 | ON log_entries.id = entries_before_and_after_panics.log_entry_id 809 | """ 810 | with engine.connect() as connection: 811 | try: 812 | connection.execute(sa.DDL(create_view_sql)) 813 | except OperationalError as e: 814 | if "table entries_before_and_after_panics_view already exists" in str(e): 815 | pass 816 | else: 817 | raise e 818 | if len(panics) > 0: 819 | print(f'Done analyzing panics over the previous {n_days} days and {k_minutes} minutes before and after the panic! Found {len(panics):,} panics.') 820 | 821 | def find_error_entries(n_days=3): 822 | global engine 823 | print('Now finding error entries...') 824 | Session = sessionmaker(bind=engine) 825 | session = Session() 826 | now = datetime.now() 827 | start_date = now - timedelta(days=n_days) 828 | error_keywords = ['error', 'invalid', 'failed', 'failure', 'unable', 'panic', 'segfault', 'exception', 'crash', 'corrupt', 'missing', 'not found', 'not responding'] 829 | error_filters = or_(*(LogEntry.message.ilike(f"%{keyword}%") for keyword in error_keywords)) 830 | error_entries = session.query(LogEntry).filter( 831 | error_filters, 832 | LogEntry.timestamp.between(start_date, now) 833 | ).all() 834 | Base.metadata.create_all(engine) 835 | error_entries_mappings = [{'log_entry_id': entry.id} for entry in error_entries] 836 | session.bulk_insert_mappings(MiscErrorEntries, error_entries_mappings) 837 | session.commit() 838 | create_view_sql = """ 839 | CREATE VIEW misc_error_entries_view AS 840 | SELECT log_entries.* FROM log_entries 841 | JOIN misc_error_entries 842 | ON log_entries.id = misc_error_entries.log_entry_id 843 | """ 844 | with engine.connect() as connection: 845 | try: 846 | connection.execute(sa.DDL(create_view_sql)) 847 | except OperationalError as e: 848 | if "table misc_error_entries_view already exists" in str(e): 849 | pass 850 | else: 851 | raise e 852 | print(f'Done finding error entries! Found {len(error_entries):,} error entries and inserted them into the database.') 853 | 854 | def get_latest_sn_statuses_func(): 855 | global engine 856 | Session = sessionmaker(bind=engine) 857 | session = Session() 858 | try: 859 | # Check if the SNStatus table has any data 860 | sn_status_count = session.query(SNStatus).count() 861 | print(f"SNStatus table row count: {sn_status_count}") 862 | if sn_status_count == 0: 863 | print("SNStatus table is empty.") 864 | return pd.DataFrame() 865 | # Execute the subquery and main query 866 | subq = session.query(SNStatus.instance_name, sa.func.max(SNStatus.datetime_of_data).label('max_datetime')).group_by(SNStatus.instance_name).subquery() 867 | latest_statuses = session.query(SNStatus).join(subq, and_(SNStatus.instance_name == subq.c.instance_name, SNStatus.datetime_of_data == subq.c.max_datetime)).all() 868 | latest_sn_statuses_df = pd.DataFrame([status.to_dict() for status in latest_statuses]) 869 | # Print the DataFrame to check the results 870 | print(f"Latest SN statuses DataFrame:\n{latest_sn_statuses_df}") 871 | return latest_sn_statuses_df 872 | except Exception as e: 873 | print(f"Error while fetching latest SNStatus: {str(e)}") 874 | finally: 875 | session.close() 876 | 877 | def run_checks_on_latest_sn_statuses_data_func(latest_sn_statuses_df, instances=None): 878 | global engine 879 | Session = sessionmaker(bind=engine) 880 | session = Session() 881 | result = {} 882 | instances_with_status = set(latest_sn_statuses_df['instance_name']) 883 | if instances: 884 | instance_names = [get_instance_name(instance.tags) for instance in instances] 885 | else: 886 | instance_names = instances_with_status 887 | missing_status_responses = list(set(instance_names) - instances_with_status) 888 | if len(missing_status_responses) > 0: 889 | result['missing_status_responses'] = missing_status_responses 890 | else: 891 | result['missing_status_responses'] = "OK" 892 | block_heights = latest_sn_statuses_df['blocks'].values 893 | latest_block_height_reported_by_any_node = int(max(block_heights)) 894 | block_heights_dict = latest_sn_statuses_df[['instance_name', 'blocks']].set_index('instance_name').to_dict()['blocks'] 895 | out_of_sync_nodes = { 896 | instance_name: { 897 | "reported_block_height": int(reported_block_height), 898 | "number_of_blocks_out_of_sync": int(latest_block_height_reported_by_any_node) - int(reported_block_height) 899 | } 900 | for instance_name, reported_block_height in block_heights_dict.items() 901 | if reported_block_height != latest_block_height_reported_by_any_node 902 | } 903 | if len(out_of_sync_nodes) > 0: 904 | result['out_of_sync_nodes'] = { 905 | "latest_block_height_reported_by_any_node": int(latest_block_height_reported_by_any_node), 906 | "nodes": out_of_sync_nodes 907 | } 908 | else: 909 | result['out_of_sync_nodes'] = "OK" 910 | connections_dict = latest_sn_statuses_df[['instance_name', 'connections']].set_index('instance_name').to_dict()['connections'] 911 | nodes_with_zero_connections = [instance_name for instance_name, connection in connections_dict.items() if connection == 0] 912 | if len(nodes_with_zero_connections) > 0: 913 | result['nodes_with_zero_connections'] = nodes_with_zero_connections 914 | else: 915 | result['nodes_with_zero_connections'] = "OK" 916 | health_check = NodeHealthChecks( 917 | missing_status_responses=result['missing_status_responses'], 918 | out_of_sync_nodes=result['out_of_sync_nodes'], 919 | nodes_with_zero_connections=result['nodes_with_zero_connections'] 920 | ) 921 | try: 922 | session.add(health_check) 923 | session.commit() 924 | except Exception as e: 925 | print(f"Error while inserting NodeHealthChecks: {str(e)}") 926 | finally: 927 | session.close() 928 | return result 929 | 930 | def get_latest_sn_masternode_statuses_func(): 931 | global engine 932 | Session = sessionmaker(bind=engine) 933 | session = Session() 934 | try: 935 | subq = session.query( 936 | SNMasternodeStatus.instance_name, 937 | sa.func.max(SNMasternodeStatus.datetime_of_data).label('max_datetime') 938 | ).group_by(SNMasternodeStatus.instance_name).subquery() 939 | latest_masternode_statuses = session.query(SNMasternodeStatus).filter( 940 | SNMasternodeStatus.instance_name == subq.c.instance_name, 941 | sa.func.julianday(subq.c.max_datetime) - sa.func.julianday(SNMasternodeStatus.datetime_of_data) <= 2 / (24*60*60) 942 | ).all() 943 | latest_masternode_statuses_df = pd.DataFrame([status.to_dict() for status in latest_masternode_statuses]) 944 | return latest_masternode_statuses_df 945 | except Exception as e: 946 | print(f"Error while fetching latest SNMasternodeStatus: {str(e)}") 947 | finally: 948 | session.close() 949 | 950 | def run_checks_on_latest_sn_masternode_statuses_data_func(latest_masternode_statuses_df, latest_sn_statuses_df): 951 | global engine 952 | Session = sessionmaker(bind=engine) 953 | session = Session() 954 | results = {} 955 | masternode_collateral_txid_and_outpoint_to_instance_name_dict = dict(zip(latest_sn_statuses_df['masternode_collateral_txid_and_outpoint'], latest_sn_statuses_df['instance_name'])) 956 | outlier_reports = {"reporting_SNs": {}} 957 | grouped = latest_masternode_statuses_df[latest_masternode_statuses_df.masternode_rank != -1].groupby(['instance_name', 'masternode_collateral_txid_and_outpoint'])['masternode_rank'].apply(list) 958 | ranks = grouped.values 959 | majority_ranks = [Counter(rank).most_common(1)[0][0] for rank in zip(*ranks)] 960 | for (reporting_sn, reported_sn), reported_ranks in grouped.items(): 961 | for reported_rank, majority in zip(reported_ranks, majority_ranks): 962 | if reported_rank != majority: 963 | if reporting_sn not in outlier_reports["reporting_SNs"]: 964 | outlier_reports["reporting_SNs"][reporting_sn] = {"disagreements_with_majority_rank": {}} 965 | if majority not in outlier_reports["reporting_SNs"][reporting_sn]["disagreements_with_majority_rank"]: 966 | outlier_reports["reporting_SNs"][reporting_sn]["disagreements_with_majority_rank"][majority] = {"reported_ranks_instead": []} 967 | outlier_reports["reporting_SNs"][reporting_sn]["disagreements_with_majority_rank"][majority]["reported_ranks_instead"].append((reported_rank, reported_sn)) 968 | explanations = [] 969 | for reporting_sn, sn_data in outlier_reports["reporting_SNs"].items(): 970 | for majority_rank, rank_data in sn_data["disagreements_with_majority_rank"].items(): 971 | for reported_rank, reported_sn in rank_data["reported_ranks_instead"]: 972 | try: 973 | reported_sn_instance_name = masternode_collateral_txid_and_outpoint_to_instance_name_dict[reported_sn] 974 | reported_sn_instance_name = reported_sn_instance_name.split(" - ")[-1] 975 | reporting_sn_instance_name = reporting_sn.split(" - ")[-1] 976 | explanation = f"Reporting SN {reporting_sn_instance_name} disagreed with the majority rank {majority_rank} and reported the rank {reported_rank} instead for SN {reported_sn_instance_name}." 977 | explanations.append(explanation) 978 | except KeyError: 979 | pass 980 | results['outlier_reports_explanations'] = explanations 981 | all_new_start_required = latest_masternode_statuses_df.groupby('instance_name')['masternode_status_message'].all() == "NEW_START_REQUIRED" 982 | if not any(all_new_start_required.values.tolist()): 983 | results['all_new_start_required'] = 'OK' 984 | else: 985 | results['all_new_start_required'] = all_new_start_required.to_dict() 986 | new_start_required_sns = latest_masternode_statuses_df[latest_masternode_statuses_df.masternode_status_message == "NEW_START_REQUIRED"]["masternode_collateral_txid_and_outpoint"].unique() 987 | sn_new_start_required_dict = {} 988 | for sn in new_start_required_sns: 989 | reporting_df = latest_masternode_statuses_df[(latest_masternode_statuses_df.masternode_collateral_txid_and_outpoint == sn) & (latest_masternode_statuses_df.masternode_status_message == "NEW_START_REQUIRED")] 990 | reporting_sns = reporting_df['instance_name'].unique().tolist() 991 | reporting_sns.sort() 992 | if len(reporting_sns) > 1: 993 | try: 994 | sn_instance_name = masternode_collateral_txid_and_outpoint_to_instance_name_dict[sn] 995 | sn_new_start_required_dict[sn_instance_name] = {"instance_name_of_reporting_sn": reporting_sns} 996 | except Exception as e: # noqa: F841 997 | pass 998 | results['supernodes_reported_to_be_in_new_start_required_mode'] = sn_new_start_required_dict 999 | masternode_health_check = NodeMasternodeHealthChecks( 1000 | masternode_rank_outlier_report_explanations=results['outlier_reports_explanations'], 1001 | all_new_start_required=results['all_new_start_required'], 1002 | supernodes_reported_to_be_in_new_start_required_mode=results['supernodes_reported_to_be_in_new_start_required_mode'] 1003 | ) 1004 | try: 1005 | session.add(masternode_health_check) 1006 | session.commit() 1007 | session.close() 1008 | except Exception as e: 1009 | print(f"Error while inserting NodeMasternodeHealthChecks: {str(e)}") 1010 | finally: 1011 | session.close() 1012 | return results 1013 | 1014 | def create_view_of_connection_counts(): 1015 | global engine 1016 | create_view_sql = """ 1017 | CREATE VIEW connection_count_per_service_view AS 1018 | SELECT public_ip, instance_name, lsof__command, datetime_of_data_truncated, COUNT(*) as count 1019 | FROM ( 1020 | SELECT sn.*, SUBSTR(sn.datetime_of_data, 1, 15) as datetime_of_data_truncated 1021 | FROM sn_network_activity_lsof sn 1022 | WHERE sn.lsof__type = 'IPv4' OR sn.lsof__type = 'IPv6' 1023 | ) 1024 | WHERE SUBSTR(datetime_of_data_truncated, 1, 15) >= ( 1025 | SELECT SUBSTR(MAX(datetime_of_data), 1, 15) 1026 | FROM sn_network_activity_lsof 1027 | ) 1028 | GROUP BY public_ip, instance_name, lsof__command, datetime_of_data_truncated; 1029 | """ 1030 | with engine.connect() as connection: 1031 | try: 1032 | connection.execute(sa.DDL(create_view_sql)) 1033 | except OperationalError as e: 1034 | if "table connection_count_per_service_view already exists" in str(e): 1035 | pass 1036 | else: 1037 | raise e 1038 | print('View created!') 1039 | 1040 | def create_view_of_connection_counts(): 1041 | global engine 1042 | create_view_sql = """ 1043 | CREATE VIEW connection_count_per_service_view AS 1044 | SELECT public_ip, instance_name, lsof__command, datetime_of_data_truncated, COUNT(*) as count 1045 | FROM ( 1046 | SELECT sn.*, SUBSTR(sn.datetime_of_data, 1, 15) as datetime_of_data_truncated 1047 | FROM sn_network_activity_lsof sn 1048 | WHERE sn.lsof__type = 'IPv4' OR sn.lsof__type = 'IPv6' 1049 | ) 1050 | WHERE SUBSTR(datetime_of_data_truncated, 1, 15) >= ( 1051 | SELECT SUBSTR(MAX(datetime_of_data), 1, 15) 1052 | FROM sn_network_activity_lsof 1053 | ) 1054 | GROUP BY public_ip, instance_name, lsof__command, datetime_of_data_truncated; 1055 | """ 1056 | with engine.connect() as connection: 1057 | try: 1058 | connection.execute(sa.DDL(create_view_sql)) 1059 | except OperationalError as e: 1060 | if "table connection_count_per_service_view already exists" in str(e): 1061 | pass 1062 | else: 1063 | raise e 1064 | print('View created!') 1065 | 1066 | def add_summary_statistic_views(engine): 1067 | views = [ 1068 | { 1069 | "name": "log_entry_count_by_node_and_source", 1070 | "query": """ 1071 | SELECT 1072 | machine_name, 1073 | log_file_source, 1074 | COUNT(*) as entry_count 1075 | FROM log_entries 1076 | GROUP BY machine_name, log_file_source 1077 | """ 1078 | }, 1079 | { 1080 | "name": "error_count_by_node_and_source", 1081 | "query": """ 1082 | SELECT 1083 | machine_name, 1084 | log_file_source, 1085 | COUNT(*) as error_count 1086 | FROM log_entries 1087 | WHERE LOWER(message) LIKE '%error%' OR LOWER(message) LIKE '%failed%' OR LOWER(message) LIKE '%exception%' 1088 | GROUP BY machine_name, log_file_source 1089 | """ 1090 | }, 1091 | { 1092 | "name": "daily_log_entry_count", 1093 | "query": """ 1094 | SELECT 1095 | machine_name, 1096 | log_file_source, 1097 | DATE(timestamp) as log_date, 1098 | COUNT(*) as daily_entry_count 1099 | FROM log_entries 1100 | GROUP BY machine_name, log_file_source, log_date 1101 | """ 1102 | }, 1103 | { 1104 | "name": "most_frequent_log_messages", 1105 | "query": """ 1106 | SELECT 1107 | machine_name, 1108 | log_file_source, 1109 | message, 1110 | COUNT(*) as message_count 1111 | FROM log_entries 1112 | GROUP BY machine_name, log_file_source, message 1113 | ORDER BY message_count DESC 1114 | LIMIT 100 1115 | """ 1116 | }, 1117 | { 1118 | "name": "recent_errors", 1119 | "query": """ 1120 | SELECT 1121 | machine_name, 1122 | log_file_source, 1123 | timestamp, 1124 | message 1125 | FROM log_entries 1126 | WHERE (LOWER(message) LIKE '%error%' OR LOWER(message) LIKE '%failed%' OR LOWER(message) LIKE '%exception%') 1127 | AND timestamp >= datetime('now', '-1 day') 1128 | ORDER BY timestamp DESC 1129 | LIMIT 100 1130 | """ 1131 | }, 1132 | { 1133 | "name": "log_entry_time_distribution", 1134 | "query": """ 1135 | SELECT 1136 | machine_name, 1137 | log_file_source, 1138 | strftime('%H', timestamp) as hour_of_day, 1139 | COUNT(*) as entry_count 1140 | FROM log_entries 1141 | GROUP BY machine_name, log_file_source, hour_of_day 1142 | """ 1143 | }, 1144 | { 1145 | "name": "node_status_summary", 1146 | "query": """ 1147 | SELECT 1148 | instance_name, 1149 | MAX(datetime_of_data) as last_update, 1150 | MAX(blocks) as latest_block_height, 1151 | AVG(connections) as avg_connections, 1152 | MAX(balance) as latest_balance 1153 | FROM sn_status 1154 | GROUP BY instance_name 1155 | """ 1156 | }, 1157 | { 1158 | "name": "masternode_status_summary", 1159 | "query": """ 1160 | SELECT 1161 | instance_name, 1162 | MAX(datetime_of_data) as last_update, 1163 | COUNT(DISTINCT masternode_collateral_txid_and_outpoint) as unique_masternodes, 1164 | COUNT(CASE WHEN masternode_status_message = 'ENABLED' THEN 1 END) as enabled_count, 1165 | COUNT(CASE WHEN masternode_status_message = 'NEW_START_REQUIRED' THEN 1 END) as new_start_required_count 1166 | FROM sn_masternode_status 1167 | GROUP BY instance_name 1168 | """ 1169 | }, 1170 | { 1171 | "name": "node_sync_status_over_time", 1172 | "query": """ 1173 | SELECT 1174 | instance_name, 1175 | datetime_of_data, 1176 | blocks, 1177 | LAG(blocks) OVER (PARTITION BY instance_name ORDER BY datetime_of_data) as previous_blocks, 1178 | JULIANDAY(datetime_of_data) - JULIANDAY(LAG(datetime_of_data) OVER (PARTITION BY instance_name ORDER BY datetime_of_data)) as days_since_last_update, 1179 | (blocks - LAG(blocks) OVER (PARTITION BY instance_name ORDER BY datetime_of_data)) / 1180 | NULLIF((JULIANDAY(datetime_of_data) - JULIANDAY(LAG(datetime_of_data) OVER (PARTITION BY instance_name ORDER BY datetime_of_data))), 0) as blocks_per_day 1181 | FROM sn_status 1182 | ORDER BY instance_name, datetime_of_data 1183 | """ 1184 | }, 1185 | { 1186 | "name": "connection_count_anomalies", 1187 | "query": """ 1188 | SELECT 1189 | instance_name, 1190 | datetime_of_data, 1191 | connections, 1192 | AVG(connections) OVER (PARTITION BY instance_name ORDER BY datetime_of_data ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as avg_connections, 1193 | connections - AVG(connections) OVER (PARTITION BY instance_name ORDER BY datetime_of_data ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as connection_anomaly 1194 | FROM sn_status 1195 | ORDER BY ABS(connections - AVG(connections) OVER (PARTITION BY instance_name ORDER BY datetime_of_data ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING)) DESC 1196 | """ 1197 | }, 1198 | { 1199 | "name": "error_frequency", 1200 | "query": """ 1201 | SELECT 1202 | machine_name, 1203 | DATE(timestamp) as error_date, 1204 | COUNT(*) as error_count 1205 | FROM log_entries 1206 | WHERE LOWER(message) LIKE '%error%' OR LOWER(message) LIKE '%failed%' OR LOWER(message) LIKE '%exception%' 1207 | GROUP BY machine_name, error_date 1208 | ORDER BY machine_name, error_date 1209 | """ 1210 | }, 1211 | { 1212 | "name": "masternode_status_changes", 1213 | "query": """ 1214 | SELECT 1215 | instance_name, 1216 | masternode_collateral_txid_and_outpoint, 1217 | datetime_of_data, 1218 | masternode_status_message, 1219 | LAG(masternode_status_message) OVER (PARTITION BY instance_name, masternode_collateral_txid_and_outpoint ORDER BY datetime_of_data) as previous_status, 1220 | CASE 1221 | WHEN masternode_status_message != LAG(masternode_status_message) OVER (PARTITION BY instance_name, masternode_collateral_txid_and_outpoint ORDER BY datetime_of_data) 1222 | THEN 1 1223 | ELSE 0 1224 | END as status_changed 1225 | FROM sn_masternode_status 1226 | ORDER BY instance_name, masternode_collateral_txid_and_outpoint, datetime_of_data 1227 | """ 1228 | }, 1229 | { 1230 | "name": "network_activity_summary", 1231 | "query": """ 1232 | SELECT 1233 | instance_name, 1234 | datetime_of_data, 1235 | COUNT(DISTINCT netstat__foreign_address) as unique_connections, 1236 | SUM(CASE WHEN netstat__state = 'ESTABLISHED' THEN 1 ELSE 0 END) as established_connections, 1237 | SUM(CASE WHEN netstat__state = 'LISTEN' THEN 1 ELSE 0 END) as listening_ports 1238 | FROM sn_network_activity_netstat 1239 | GROUP BY instance_name, datetime_of_data 1240 | ORDER BY instance_name, datetime_of_data 1241 | """ 1242 | }, 1243 | { 1244 | "name": "log_message_patterns", 1245 | "query": """ 1246 | WITH message_patterns AS ( 1247 | SELECT 1248 | machine_name, 1249 | log_file_source, 1250 | REGEXP_REPLACE(message, '[0-9]+', '#') as pattern, 1251 | COUNT(*) as pattern_count 1252 | FROM log_entries 1253 | GROUP BY machine_name, log_file_source, pattern 1254 | ) 1255 | SELECT 1256 | machine_name, 1257 | log_file_source, 1258 | pattern, 1259 | pattern_count, 1260 | ROW_NUMBER() OVER (PARTITION BY machine_name, log_file_source ORDER BY pattern_count DESC) as pattern_rank 1261 | FROM message_patterns 1262 | ORDER BY machine_name, log_file_source, pattern_count DESC 1263 | """ 1264 | }, 1265 | { 1266 | "name": "node_performance_metrics", 1267 | "query": """ 1268 | SELECT 1269 | instance_name, 1270 | datetime_of_data, 1271 | blocks, 1272 | connections, 1273 | balance, 1274 | timeoffset, 1275 | JULIANDAY(datetime_of_data) - JULIANDAY(LAG(datetime_of_data) OVER (PARTITION BY instance_name ORDER BY datetime_of_data)) as days_since_last_update, 1276 | (blocks - LAG(blocks) OVER (PARTITION BY instance_name ORDER BY datetime_of_data)) / 1277 | NULLIF((JULIANDAY(datetime_of_data) - JULIANDAY(LAG(datetime_of_data) OVER (PARTITION BY instance_name ORDER BY datetime_of_data))), 0) as blocks_per_day, 1278 | (balance - LAG(balance) OVER (PARTITION BY instance_name ORDER BY datetime_of_data)) / 1279 | NULLIF((JULIANDAY(datetime_of_data) - JULIANDAY(LAG(datetime_of_data) OVER (PARTITION BY instance_name ORDER BY datetime_of_data))), 0) as balance_change_per_day 1280 | FROM sn_status 1281 | ORDER BY instance_name, datetime_of_data 1282 | """ 1283 | }, 1284 | { 1285 | "name": "correlated_log_entries", 1286 | "query": """ 1287 | WITH ranked_logs AS ( 1288 | SELECT 1289 | *, 1290 | ROW_NUMBER() OVER (PARTITION BY machine_name ORDER BY timestamp) as row_num 1291 | FROM log_entries 1292 | ) 1293 | SELECT 1294 | a.machine_name, 1295 | a.log_file_source as source_1, 1296 | a.timestamp as time_1, 1297 | a.message as message_1, 1298 | b.log_file_source as source_2, 1299 | b.timestamp as time_2, 1300 | b.message as message_2, 1301 | (JULIANDAY(b.timestamp) - JULIANDAY(a.timestamp)) * 86400 as seconds_between 1302 | FROM ranked_logs a 1303 | JOIN ranked_logs b ON a.machine_name = b.machine_name AND a.row_num < b.row_num 1304 | WHERE (JULIANDAY(b.timestamp) - JULIANDAY(a.timestamp)) * 86400 <= 60 -- Within 60 seconds 1305 | AND a.log_file_source != b.log_file_source 1306 | AND ( 1307 | (LOWER(a.message) LIKE '%error%' AND LOWER(b.message) LIKE '%error%') 1308 | OR (LOWER(a.message) LIKE '%failed%' AND LOWER(b.message) LIKE '%failed%') 1309 | OR (LOWER(a.message) LIKE '%exception%' AND LOWER(b.message) LIKE '%exception%') 1310 | ) 1311 | ORDER BY a.machine_name, a.timestamp 1312 | """ 1313 | } 1314 | ] 1315 | with engine.connect() as connection: 1316 | for view in views: 1317 | try: 1318 | connection.execute(sa.text(f"DROP VIEW IF EXISTS {view['name']}")) 1319 | print(f"Dropped existing view: {view['name']}") 1320 | create_view_sql = f"CREATE VIEW {view['name']} AS {view['query']}" 1321 | connection.execute(sa.text(create_view_sql)) 1322 | print(f"Created view: {view['name']}") 1323 | except Exception as e: 1324 | print(f"Error creating view {view['name']}: {str(e)}") 1325 | 1326 | print("Finished creating all summary statistic views.") 1327 | 1328 | def stop_existing_datasette(): 1329 | ps_output = subprocess.check_output(['ps', 'aux']).decode('utf-8') 1330 | datasette_processes = [line for line in ps_output.split('\n') if 'datasette' in line and not 'grep' in line] 1331 | for process in datasette_processes: 1332 | pid = int(process.split()[1]) 1333 | os.kill(pid, signal.SIGTERM) 1334 | print(f"Stopped existing Datasette process with PID {pid}") 1335 | 1336 | def restart_datasette(datasette_path: str, sqlite_path: str, host: str, port: int, time_limit_ms: int): 1337 | stop_existing_datasette() 1338 | metadata = { 1339 | "title": "Log Analysis Dashboard", 1340 | "description": "Summary statistics and log analysis for Pastel nodes", 1341 | "databases": { 1342 | "log_analysis": { 1343 | "tables": { 1344 | "log_entry_count_by_node_and_source": {"title": "Log Entry Count by Node and Source"}, 1345 | "error_count_by_node_and_source": {"title": "Error Count by Node and Source"}, 1346 | "daily_log_entry_count": {"title": "Daily Log Entry Count"}, 1347 | "most_frequent_log_messages": {"title": "Most Frequent Log Messages"}, 1348 | "recent_errors": {"title": "Recent Errors"}, 1349 | "log_entry_time_distribution": {"title": "Log Entry Time Distribution"}, 1350 | "node_status_summary": {"title": "Node Status Summary"}, 1351 | "masternode_status_summary": {"title": "Masternode Status Summary"}, 1352 | "node_sync_status_over_time": {"title": "Node Sync Status Over Time"}, 1353 | "connection_count_anomalies": {"title": "Connection Count Anomalies"}, 1354 | "error_frequency": {"title": "Error Frequency by Node and Time"}, 1355 | "masternode_status_changes": {"title": "Masternode Status Changes"}, 1356 | "network_activity_summary": {"title": "Network Activity Summary"}, 1357 | "log_message_patterns": {"title": "Log Message Patterns"}, 1358 | "node_performance_metrics": {"title": "Node Performance Metrics"}, 1359 | "correlated_log_entries": {"title": "Correlated Log Entries"} 1360 | } 1361 | } 1362 | } 1363 | } 1364 | metadata_file = "datasette_metadata.json" 1365 | with open(metadata_file, "w") as f: 1366 | json.dump(metadata, f) 1367 | cmd = f"{datasette_path} {sqlite_path} -h {host} -p {port} --metadata {metadata_file} --setting sql_time_limit_ms {time_limit_ms}" 1368 | with open(os.devnull, 'w') as dev_null: 1369 | subprocess.Popen(shlex.split(cmd), stdout=dev_null, stderr=dev_null) 1370 | print(f"Datasette started with the new SQLite file and views at http://{host}:{port}") 1371 | 1372 | def backup_table(table_name): 1373 | with sqlite3.connect(sqlite_file_path) as conn: 1374 | df = pd.read_sql(f'SELECT * FROM {table_name}', conn) 1375 | backup_file_name = f'{backup_base_path}_{table_name}.csv' 1376 | df.to_csv(backup_file_name, index=False) 1377 | timestamp = datetime.now().strftime('__%Y_%m_%d__%H_%M_%S') 1378 | backup_dir = BASE_PATH + config("BACKUP_DATABASE_TABLE_CSV_FILES_DIR_NAME", cast=str) 1379 | os.makedirs(backup_dir, exist_ok=True) 1380 | backup_file_name_timestamped = f'{table_name}{timestamp}.csv' 1381 | backup_file_path_timestamped = os.path.join(backup_dir, backup_file_name_timestamped) 1382 | shutil.copy2(backup_file_name, backup_file_path_timestamped) 1383 | print(f"Backed up {table_name} to {backup_file_name} and {backup_file_path_timestamped}") 1384 | 1385 | def load_table(table_name): 1386 | global earliest_date_cutoff 1387 | backup_path = f'{backup_base_path}_{table_name}.csv' 1388 | if os.path.exists(backup_path): 1389 | df = pd.read_csv(backup_path) 1390 | initial_len = len(df) 1391 | print(f"Loaded {table_name} from {backup_path}") 1392 | df = df[pd.to_datetime(df['datetime_of_data']) > earliest_date_cutoff] 1393 | print(f"Filtered {table_name} to only include entries after {earliest_date_cutoff} ({len(df)} entries retained, {initial_len - len(df)} entries removed for being too old)") 1394 | with sqlite3.connect(sqlite_file_path) as conn: 1395 | df.to_sql(table_name, conn, if_exists='append', index=False) 1396 | 1397 | def get_status_data_in_parallel_func(instance_ids): 1398 | if use_sequential_data_collection: 1399 | print('\n\nNOTE: USING SEQUENTIAL DATA COLLECTION\n\n') 1400 | for instance_id in instance_ids: 1401 | get_status_info_for_instance(instance_id) 1402 | else: 1403 | with concurrent.futures.ThreadPoolExecutor() as executor: 1404 | executor.map(get_status_info_for_instance, instance_ids) 1405 | 1406 | def get_instance_ids_from_inventory(inventory_file): 1407 | with open(inventory_file, 'r') as file: 1408 | inventory = yaml.safe_load(file) 1409 | instance_ids = [] 1410 | hosts = inventory.get('all', {}).get('hosts', {}) 1411 | instance_ids.extend(hosts.keys()) 1412 | return instance_ids 1413 | 1414 | def main(): 1415 | num_cores = os.cpu_count() 1416 | if num_cores is not None: 1417 | num_cores = max(1, num_cores - 2) 1418 | manager = Manager() 1419 | db_write_queue = manager.JoinableQueue() 1420 | db_writer_thread = threading.Thread(target=insert_log_entries_worker, args=(db_write_queue,)) 1421 | db_writer_thread.start() 1422 | with ProcessPoolExecutor(max_workers=num_cores) as executor: 1423 | futures = [] 1424 | for instance_id in instance_ids: 1425 | future = executor.submit(process_instance, instance_id, db_write_queue) 1426 | futures.append(future) 1427 | for future in futures: 1428 | try: 1429 | future.result() 1430 | except Exception as e: 1431 | print(f"Error processing instance: {e}") 1432 | db_write_queue.put(None) 1433 | db_writer_thread.join() 1434 | 1435 | redis_client = redis.StrictRedis(host='localhost', port=6379, db=0) 1436 | 1437 | if __name__ == "__main__": 1438 | profile_code = 0 1439 | profile_output_file = "profiling_data.prof" 1440 | os.system('clear') 1441 | print('Clearing existing Redis contents...') 1442 | redis_client.flushall() 1443 | print('Done clearing Redis contents!') 1444 | aws_access_key_id = config("AWS_ACCESS_KEY_ID", cast=str) 1445 | aws_secret_access_key = config("AWS_SECRET_ACCESS_KEY", cast=str) 1446 | aws_region = config("AWS_REGION", cast=str) 1447 | ansible_inventory_file = config("ANSIBLE_INVENTORY_FILE", cast=str) 1448 | earliest_date_cutoff = datetime.now() - timedelta(days=7) 1449 | print('Clearing existing log files and database...') 1450 | downloaded_log_files_path = config("DOWNLOADED_LOG_FILES_PATH", cast=str) 1451 | existing_log_files = glob.glob(downloaded_log_files_path + '/*') 1452 | sqlite_file_path = config("SQLITE_FILE_PATH", cast=str) 1453 | node_status_data_backup_path = config("NODE_STATUS_DATA_BACKUP_PATH", cast=str) 1454 | backup_base_path = BASE_PATH + node_status_data_backup_path 1455 | for file in existing_log_files: 1456 | os.remove(file) 1457 | try: 1458 | os.remove(sqlite_file_path) 1459 | except FileNotFoundError: 1460 | pass 1461 | log_directory = BASE_PATH 1462 | log_files_pattern = os.path.join(log_directory, "*.log") 1463 | log_files = glob.glob(log_files_pattern) 1464 | if len(log_files) > 5: 1465 | sorted_log_files = sorted(log_files, key=os.path.getctime, reverse=True) 1466 | for log_file in sorted_log_files[5:]: 1467 | os.remove(log_file) 1468 | print('Done clearing existing log files and database!') 1469 | print('\n\nNow starting log analysis...') 1470 | engine = sa.create_engine(f'sqlite:///{sqlite_file_path}', connect_args={'timeout': 20}, pool_size=10, max_overflow=20) 1471 | Base.metadata.create_all(engine) 1472 | print('Now attempting to load historical node status data from backup...') 1473 | try: 1474 | load_table('sn_masternode_status') 1475 | load_table('sn_status') 1476 | load_table('sn_network_activity_netstat') 1477 | load_table('sn_network_activity_lsof') 1478 | load_table('sn_network_activity_ss') 1479 | print('Done loading historical node status data from backup!') 1480 | except Exception as e: 1481 | print(f"Error loading historical node status data from backup: {e}") 1482 | backup_database_table_csv_files_dir_name = config("BACKUP_DATABASE_TABLE_CSV_FILES_DIR_NAME", cast=str) 1483 | backup_csv_files = glob.glob(backup_database_table_csv_files_dir_name + '/*.csv') 1484 | number_of_backup_csv_files_to_keep = config("NUMBER_OF_BACKUP_CSV_FILES_TO_KEEP", cast=int) 1485 | if len(backup_csv_files) > number_of_backup_csv_files_to_keep: 1486 | sorted_backup_csv_files = sorted(backup_csv_files, key=os.path.getctime, reverse=True) 1487 | for backup_csv_file in sorted_backup_csv_files[number_of_backup_csv_files_to_keep:]: 1488 | os.remove(backup_csv_file) 1489 | print(f"Deleted backup CSV file {backup_csv_file} because it was too old (more than {number_of_backup_csv_files_to_keep} files in the backup directory)") 1490 | instance_name_prefix = config("INSTANCE_NAME_PREFIX", cast=str) 1491 | non_aws_instance_ids = get_instance_ids_from_inventory(ansible_inventory_file) 1492 | print('Non-AWS instance IDs:', non_aws_instance_ids) 1493 | if 0: 1494 | print('Only using first non-AWS instance for debug purposes...') 1495 | non_aws_instance_ids = non_aws_instance_ids[:1] 1496 | try: 1497 | instances = get_instances_with_name_prefix(instance_name_prefix, aws_access_key_id, aws_secret_access_key, aws_region) 1498 | aws_instance_ids = [instance.id for instance in instances] 1499 | except Exception as e: 1500 | print("Error getting instances from AWS API: ", e) 1501 | aws_instance_ids = [] 1502 | instances = None 1503 | instance_ids = aws_instance_ids + non_aws_instance_ids 1504 | print(f'Now collecting status info for {len(instance_ids)} instances...') 1505 | use_sequential_data_collection = 0 1506 | get_status_data_in_parallel_func(instance_ids) 1507 | print('Done collecting status info for all instances!') 1508 | latest_sn_statuses_df = get_latest_sn_statuses_func() 1509 | latest_masternode_statuses_df = get_latest_sn_masternode_statuses_func() 1510 | sn_status_check_results_dict = run_checks_on_latest_sn_statuses_data_func(latest_sn_statuses_df, instances) 1511 | sn_status_check_results_json = json.dumps(sn_status_check_results_dict, indent=4) 1512 | print(f'\n\nResults of checks on latest SN status data:\n{sn_status_check_results_json}') 1513 | sn_masternode_status_check_results_dict = run_checks_on_latest_sn_masternode_statuses_data_func(latest_masternode_statuses_df, latest_sn_statuses_df) 1514 | sn_masternode_status_check_results_json = json.dumps(sn_masternode_status_check_results_dict, indent=4) 1515 | print(f'\n\nResults of checks on latest SN masternode status data:\n{sn_masternode_status_check_results_json}') 1516 | if not profile_code: 1517 | main() 1518 | else: 1519 | cProfile.run("main()", profile_output_file) 1520 | print('\n\nDone ingesting log files!') 1521 | analyze_panics(n_days=1, k_minutes=10) 1522 | find_error_entries() 1523 | create_view_of_connection_counts() 1524 | add_summary_statistic_views(engine) 1525 | print('\n\nNow backing up database tables for `sn_masternode_status` and `sn_status` ...') 1526 | backup_table('sn_masternode_status') 1527 | backup_table('sn_status') 1528 | backup_table('sn_network_activity_netstat') 1529 | backup_table('sn_network_activity_lsof') 1530 | backup_table('sn_network_activity_ss') 1531 | print('Done backing up database tables!') 1532 | print('All Completed!') 1533 | datasette_path = config("DATASETTE_PATH", cast=str) 1534 | sqlite_path = BASE_PATH + sqlite_file_path 1535 | host = config("HOST", cast=str) 1536 | port = config("PORT", cast=int) 1537 | time_limit_ms = config("TIME_LIMIT_MS", cast=int) 1538 | restart_datasette(datasette_path, sqlite_path, host, port, time_limit_ms) 1539 | try: 1540 | os.remove(lock_file) 1541 | except Exception as e: # noqa: F841 1542 | pass 1543 | if profile_code: 1544 | with open("profiling_report.txt", "w") as f: 1545 | stats = pstats.Stats(profile_output_file, stream=f) 1546 | stats.sort_stats("cumulative") 1547 | stats.print_stats() 1548 | -------------------------------------------------------------------------------- /sample_log_files_to_understand_the_parsing_functions/TestnetSupernode01__154_12_235_41__entry.log: -------------------------------------------------------------------------------- 1 | 2023-07-24 10:39:17.922 - [37] RECEIVED, request_size=495 bytes, height=326400 2 | 2023-07-24 10:39:17.925 - [37] QUEUED, queue_size=1 3 | 2023-07-24 10:39:17.931 - [37] STARTED, queue_wait_time=0.006175 secs 4 | 2023-07-24 10:44:12.112 - [37] SUCCEEDED, execution_time=294.028 secs, response_size=2531630 bytes, image_hash=8c26b8fe1f22e2078fbfd4a6d86e5728da2a58f749da83d210a341be081f35ee 5 | 2023-07-24 11:26:00.256 - [38] RECEIVED, request_size=495 bytes, height=326421 6 | 2023-07-24 11:26:00.257 - [38] QUEUED, queue_size=1 7 | 2023-07-24 11:26:00.260 - [38] STARTED, queue_wait_time=0.003543 secs 8 | 2023-07-24 11:32:16.473 - [38] SUCCEEDED, execution_time=376.044 secs, response_size=2915406 bytes, image_hash=e3567f0081cf92013beb8131a9012fc758931eb478dc052ab90e4a7c851d1729 9 | 2023-07-24 16:17:28.202 - -------------------------------------------------- v2.3.27 -------------------------------------------------- 10 | 2023-07-24 16:47:02.504 - -------------------------------------------------- v2.3.27 -------------------------------------------------- 11 | 2023-07-24 17:11:03.921 - [1] RECEIVED, request_size=494 bytes, height=326565 12 | 2023-07-24 17:11:03.923 - [1] QUEUED, queue_size=1 13 | 2023-07-24 17:11:04.120 - [1] STARTED, queue_wait_time=0.198139 secs 14 | 2023-07-24 17:11:31.186 - [2] RECEIVED, request_size=494 bytes, height=326565 15 | 2023-07-24 17:11:31.186 - [2] QUEUED, queue_size=1 16 | 2023-07-24 17:12:54.229 - [1] SUCCEEDED, execution_time=110.014 secs, response_size=1631686 bytes, image_hash=825723e242d2f0b1fec19749b04812d0bb23fc91f6ec375ffd1907210b1283d9 17 | 2023-07-24 17:12:54.478 - [2] STARTED, queue_wait_time=83.292199 secs 18 | 2023-07-24 17:13:31.705 - [3] RECEIVED, request_size=495 bytes, height=326565 19 | 2023-07-24 17:13:31.706 - [3] QUEUED, queue_size=1 20 | 2023-07-24 17:15:39.657 - [2] SUCCEEDED, execution_time=165.034 secs, response_size=2094766 bytes, image_hash=42b6644e34ab203d4311f495624fef5f5427cad88cc14d591158b2daf34715ec 21 | 2023-07-24 17:15:39.868 - [3] STARTED, queue_wait_time=128.162280 secs 22 | 2023-07-24 17:16:33.025 - [4] RECEIVED, request_size=495 bytes, height=326565 23 | 2023-07-24 17:16:33.026 - [4] QUEUED, queue_size=1 24 | 2023-07-24 17:20:12.102 - [3] SUCCEEDED, execution_time=272.049 secs, response_size=2407517 bytes, image_hash=0412cb7b2104d3f53e5dbe877bafe865a3144498c8dc79a4f1d07c21e91d56f8 25 | 2023-07-24 17:20:12.304 - [4] STARTED, queue_wait_time=219.278318 secs 26 | 2023-07-24 17:27:00.768 - [4] SUCCEEDED, execution_time=408.073 secs, response_size=5830175 bytes, image_hash=e3567f0081cf92013beb8131a9012fc758931eb478dc052ab90e4a7c851d1729 27 | 2023-07-24 18:02:18.943 - [5] RECEIVED, request_size=495 bytes, height=326583 28 | 2023-07-24 18:02:18.944 - [5] QUEUED, queue_size=1 29 | 2023-07-24 18:02:19.315 - [5] STARTED, queue_wait_time=0.371398 secs 30 | 2023-07-24 18:02:25.347 - [6] RECEIVED, request_size=495 bytes, height=326583 31 | 2023-07-24 18:02:25.348 - [6] QUEUED, queue_size=1 32 | 2023-07-24 18:07:49.684 - [5] SUCCEEDED, execution_time=330.028 secs, response_size=5995736 bytes, image_hash=1e7241b7cfb5085d42c2a0501e6bbc75b15c3924307f8019afb8f45431f270e6 33 | 2023-07-24 18:07:49.934 - [6] STARTED, queue_wait_time=324.585915 secs 34 | 2023-07-24 18:12:40.207 - [6] SUCCEEDED, execution_time=290.018 secs, response_size=4359419 bytes, image_hash=653843d02c9635e5a049a0e1e5dcb335c49b3c12aaa0ac95c05f385fcf9bf009 35 | 2023-07-24 18:14:05.641 - [7] RECEIVED, request_size=495 bytes, height=326590 36 | 2023-07-24 18:14:05.642 - [7] QUEUED, queue_size=1 37 | 2023-07-24 18:14:05.919 - [7] STARTED, queue_wait_time=0.277769 secs 38 | 2023-07-24 18:24:46.147 - [7] SUCCEEDED, execution_time=640.060 secs, response_size=3145125 bytes, image_hash=eed61d9929cbf41a30ef42e5d6461fae8f409c70672c462170d6185aabb50a1e 39 | 2023-07-24 18:37:23.134 - [8] RECEIVED, request_size=494 bytes, height=326599 40 | 2023-07-24 18:37:23.136 - [8] QUEUED, queue_size=1 41 | 2023-07-24 18:37:23.465 - [8] STARTED, queue_wait_time=0.330333 secs 42 | 2023-07-24 18:37:24.226 - [9] RECEIVED, request_size=495 bytes, height=326599 43 | 2023-07-24 18:37:24.227 - [9] QUEUED, queue_size=1 44 | 2023-07-24 18:37:24.540 - [9] STARTED, queue_wait_time=0.313238 secs 45 | 2023-07-24 18:37:26.963 - [10] RECEIVED, request_size=495 bytes, height=326599 46 | 2023-07-24 18:37:26.963 - [10] QUEUED, queue_size=1 47 | 2023-07-24 18:37:27.160 - [10] STARTED, queue_wait_time=0.196738 secs 48 | 2023-07-24 18:37:28.432 - [11] RECEIVED, request_size=494 bytes, height=326599 49 | 2023-07-24 18:37:28.432 - [11] QUEUED, queue_size=1 50 | 2023-07-24 18:37:28.644 - [11] STARTED, queue_wait_time=0.211709 secs 51 | 2023-07-24 18:52:25.919 - [8] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/24ca0b11.png] timed out after 901.066 secs 52 | 2023-07-24 18:52:26.288 - [9] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/fb46e9f5.jpeg] timed out after 901.044 secs 53 | 2023-07-24 18:52:28.567 - [10] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/280c3129.jpeg] timed out after 901.058 secs 54 | 2023-07-24 18:52:29.937 - [11] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/527eab25.png] timed out after 901.040 secs 55 | 2023-07-24 19:00:43.256 - [12] RECEIVED, request_size=495 bytes, height=326607 56 | 2023-07-24 19:00:43.257 - [12] QUEUED, queue_size=1 57 | 2023-07-24 19:00:43.546 - [12] STARTED, queue_wait_time=0.289165 secs 58 | 2023-07-24 19:00:43.891 - [13] RECEIVED, request_size=495 bytes, height=326607 59 | 2023-07-24 19:00:43.892 - [13] QUEUED, queue_size=1 60 | 2023-07-24 19:00:43.896 - [14] RECEIVED, request_size=495 bytes, height=326607 61 | 2023-07-24 19:00:43.896 - [14] QUEUED, queue_size=1 62 | 2023-07-24 19:00:44.135 - [13] STARTED, queue_wait_time=0.242743 secs 63 | 2023-07-24 19:00:44.165 - [15] RECEIVED, request_size=495 bytes, height=326607 64 | 2023-07-24 19:00:44.166 - [15] QUEUED, queue_size=1 65 | 2023-07-24 19:00:44.365 - [14] STARTED, queue_wait_time=0.468694 secs 66 | 2023-07-24 19:00:44.597 - [15] STARTED, queue_wait_time=0.431631 secs 67 | 2023-07-24 19:00:47.443 - [16] RECEIVED, request_size=494 bytes, height=326607 68 | 2023-07-24 19:00:47.444 - [16] QUEUED, queue_size=1 69 | 2023-07-24 19:00:47.713 - [16] STARTED, queue_wait_time=0.269240 secs 70 | 2023-07-24 19:00:48.181 - [17] RECEIVED, request_size=495 bytes, height=326607 71 | 2023-07-24 19:00:48.181 - [17] QUEUED, queue_size=1 72 | 2023-07-24 19:15:46.359 - [12] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/cf284614.jpeg] timed out after 901.047 secs 73 | 2023-07-24 19:15:46.525 - [14] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/888c5420.jpeg] timed out after 901.048 secs 74 | 2023-07-24 19:15:46.688 - [15] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/32604199.jpeg] timed out after 901.037 secs 75 | 2023-07-24 19:15:46.841 - [13] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/0719e42a.jpeg] timed out after 901.032 secs 76 | 2023-07-24 19:15:46.978 - [17] STARTED, queue_wait_time=898.797113 secs 77 | 2023-07-24 19:15:49.056 - [16] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/b36a2d62.png] timed out after 901.041 secs 78 | 2023-07-24 19:30:48.042 - [17] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/20901119.jpeg] timed out after 901.037 secs 79 | 2023-07-24 22:36:50.693 - -------------------------------------------------- v2.3.27 -------------------------------------------------- 80 | 2023-07-25 19:07:04.458 - [1] RECEIVED, request_size=502 bytes, height=327153 81 | 2023-07-25 19:07:04.464 - [1] QUEUED, queue_size=1 82 | 2023-07-25 19:07:04.864 - [1] STARTED, queue_wait_time=0.403349 secs 83 | 2023-07-25 19:14:00.386 - [1] SUCCEEDED, execution_time=415.091 secs, response_size=6645148 bytes, image_hash=11f0b9e9c946aa6c29dea0a56c021f0733c219bdbc8d5bccb64e16eadf6ea0f1 84 | 2023-07-26 00:41:03.947 - [2] RECEIVED, request_size=494 bytes, height=327293 85 | 2023-07-26 00:41:03.951 - [2] QUEUED, queue_size=1 86 | 2023-07-26 00:41:04.178 - [2] STARTED, queue_wait_time=0.227719 secs 87 | 2023-07-26 00:41:06.130 - [3] RECEIVED, request_size=494 bytes, height=327293 88 | 2023-07-26 00:41:06.130 - [3] QUEUED, queue_size=1 89 | 2023-07-26 00:43:01.387 - [2] SUCCEEDED, execution_time=117.012 secs, response_size=2790683 bytes, image_hash=6312abea9b69186439c2255f185b99a98c4ee0fae75a0d3b0bd6808d7c0f68fa 90 | 2023-07-26 00:43:01.518 - [3] STARTED, queue_wait_time=115.387608 secs 91 | 2023-07-26 00:43:17.299 - [4] RECEIVED, request_size=495 bytes, height=327293 92 | 2023-07-26 00:43:17.299 - [4] QUEUED, queue_size=1 93 | 2023-07-26 00:44:45.670 - [3] SUCCEEDED, execution_time=104.017 secs, response_size=2457470 bytes, image_hash=b697c45c50b7ee856ede30e9fde45568685773b96330697b2305c4d8d0686a28 94 | 2023-07-26 00:44:45.879 - [4] STARTED, queue_wait_time=88.579969 secs 95 | 2023-07-26 00:44:55.503 - [5] RECEIVED, request_size=495 bytes, height=327293 96 | 2023-07-26 00:44:55.503 - [5] QUEUED, queue_size=1 97 | 2023-07-26 00:49:34.131 - [4] SUCCEEDED, execution_time=288.028 secs, response_size=3664071 bytes, image_hash=a21efe718cd7bb6fff2baee26d838f4d09eede1d831593d08c4a91314552ab4d 98 | 2023-07-26 00:49:34.312 - [5] STARTED, queue_wait_time=278.809194 secs 99 | 2023-07-26 00:49:40.515 - [6] RECEIVED, request_size=495 bytes, height=327293 100 | 2023-07-26 00:49:40.516 - [6] QUEUED, queue_size=1 101 | 2023-07-26 00:56:55.702 - [5] SUCCEEDED, execution_time=441.067 secs, response_size=5554311 bytes, image_hash=6940d810012d1a815379f80d7da6228948a104f423710bcd7e4050fea0286684 102 | 2023-07-26 00:56:55.830 - [6] STARTED, queue_wait_time=435.314985 secs 103 | 2023-07-26 01:02:26.026 - [6] SUCCEEDED, execution_time=330.028 secs, response_size=2958065 bytes, image_hash=98fb037b621117c2b960a9264aed4d7f336eb98ffbd9d385b5408e350a65f76d 104 | 2023-07-26 01:23:54.031 - [7] RECEIVED, request_size=495 bytes, height=327314 105 | 2023-07-26 01:23:54.032 - [7] QUEUED, queue_size=1 106 | 2023-07-26 01:23:54.357 - [7] STARTED, queue_wait_time=0.324816 secs 107 | 2023-07-26 01:27:56.619 - [7] SUCCEEDED, execution_time=242.014 secs, response_size=4812167 bytes, image_hash=b2306f8c02aabc21a232bf07d67dd558b7257b4e03da27419aef3bcb1771dad6 108 | 2023-07-26 02:22:20.983 - [8] RECEIVED, request_size=495 bytes, height=327337 109 | 2023-07-26 02:22:20.986 - [8] QUEUED, queue_size=1 110 | 2023-07-26 02:22:21.584 - [8] STARTED, queue_wait_time=0.598931 secs 111 | 2023-07-26 02:22:27.215 - [9] RECEIVED, request_size=495 bytes, height=327337 112 | 2023-07-26 02:22:27.216 - [9] QUEUED, queue_size=1 113 | 2023-07-26 02:22:27.421 - [9] STARTED, queue_wait_time=0.205772 secs 114 | 2023-07-26 02:33:54.326 - [10] RECEIVED, request_size=495 bytes, height=327342 115 | 2023-07-26 02:33:54.328 - [10] QUEUED, queue_size=1 116 | 2023-07-26 02:33:54.915 - [10] STARTED, queue_wait_time=0.588217 secs 117 | 2023-07-26 02:37:23.019 - [8] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/d1bebf50.jpeg] timed out after 901.070 secs 118 | 2023-07-26 02:37:29.021 - [9] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/51df115c.jpeg] timed out after 901.091 secs 119 | 2023-07-26 02:41:07.181 - [10] SUCCEEDED, execution_time=432.053 secs, response_size=3466685 bytes, image_hash=4e016abd79d7d1e4e03e9e37807dae8eb45d1ed211f77a324a0d2482dade4e1a 120 | 2023-07-26 02:45:31.449 - [11] RECEIVED, request_size=495 bytes, height=327346 121 | 2023-07-26 02:45:31.450 - [11] QUEUED, queue_size=1 122 | 2023-07-26 02:45:31.789 - [11] STARTED, queue_wait_time=0.339726 secs 123 | 2023-07-26 02:54:20.027 - [11] SUCCEEDED, execution_time=528.033 secs, response_size=3584919 bytes, image_hash=957083ffa4b4d5e91ad8502510490fb236159ff04e784a1953d14cb7d2d9da6a 124 | 2023-07-26 02:57:11.978 - [12] RECEIVED, request_size=495 bytes, height=327353 125 | 2023-07-26 02:57:11.979 - [12] QUEUED, queue_size=1 126 | 2023-07-26 02:57:12.331 - [12] STARTED, queue_wait_time=0.351882 secs 127 | 2023-07-26 02:57:19.578 - [13] RECEIVED, request_size=495 bytes, height=327353 128 | 2023-07-26 02:57:19.579 - [13] QUEUED, queue_size=1 129 | 2023-07-26 02:57:19.865 - [13] STARTED, queue_wait_time=0.286193 secs 130 | 2023-07-26 02:57:20.670 - [14] RECEIVED, request_size=495 bytes, height=327353 131 | 2023-07-26 02:57:20.670 - [14] QUEUED, queue_size=1 132 | 2023-07-26 02:57:20.856 - [14] STARTED, queue_wait_time=0.185397 secs 133 | 2023-07-26 03:12:25.231 - [12] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5c370584.jpeg] timed out after 911.057 secs 134 | 2023-07-26 03:12:31.624 - [13] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/f63c257a.jpeg] timed out after 911.040 secs 135 | 2023-07-26 03:12:32.208 - [14] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a34198e5.jpeg] timed out after 911.049 secs 136 | 2023-07-26 03:43:56.221 - [15] RECEIVED, request_size=495 bytes, height=327370 137 | 2023-07-26 03:43:56.223 - [15] QUEUED, queue_size=1 138 | 2023-07-26 03:43:56.602 - [15] STARTED, queue_wait_time=0.379277 secs 139 | 2023-07-26 03:43:57.449 - [16] RECEIVED, request_size=495 bytes, height=327370 140 | 2023-07-26 03:43:57.449 - [16] QUEUED, queue_size=1 141 | 2023-07-26 03:43:57.794 - [17] RECEIVED, request_size=495 bytes, height=327370 142 | 2023-07-26 03:43:57.796 - [17] QUEUED, queue_size=1 143 | 2023-07-26 03:43:57.801 - [16] STARTED, queue_wait_time=0.351202 secs 144 | 2023-07-26 03:43:58.006 - [17] STARTED, queue_wait_time=0.211235 secs 145 | 2023-07-26 03:55:34.674 - [18] RECEIVED, request_size=495 bytes, height=327375 146 | 2023-07-26 03:55:34.675 - [18] QUEUED, queue_size=1 147 | 2023-07-26 03:55:35.337 - [18] STARTED, queue_wait_time=0.662120 secs 148 | 2023-07-26 03:55:38.389 - [19] RECEIVED, request_size=495 bytes, height=327375 149 | 2023-07-26 03:55:38.390 - [19] QUEUED, queue_size=1 150 | 2023-07-26 03:55:38.587 - [20] RECEIVED, request_size=494 bytes, height=327375 151 | 2023-07-26 03:55:38.588 - [20] QUEUED, queue_size=1 152 | 2023-07-26 03:55:39.098 - [19] STARTED, queue_wait_time=0.708820 secs 153 | 2023-07-26 03:59:08.787 - [15] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/57af5ec0.jpeg] timed out after 911.048 secs 154 | 2023-07-26 03:59:09.299 - [16] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9c6853bf.jpeg] timed out after 911.044 secs 155 | 2023-07-26 03:59:09.364 - [20] STARTED, queue_wait_time=210.776038 secs 156 | 2023-07-26 03:59:09.457 - [17] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5c55f742.jpeg] timed out after 911.062 secs 157 | 2023-07-26 04:01:43.451 - [19] SUCCEEDED, execution_time=364.043 secs, response_size=1382112 bytes, image_hash=38a080e07f4014481647e640294c2daf0570d442cef1a33d0bdce73326bfcf36 158 | 2023-07-26 04:10:47.174 - [18] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/696afb50.jpeg] timed out after 911.068 secs 159 | 2023-07-26 04:11:03.593 - [20] SUCCEEDED, execution_time=714.111 secs, response_size=1964674 bytes, image_hash=229fe9852b8d5c6d556104c9be5d5f9202047df38a2f8451d4c290c46052a56e 160 | 2023-07-26 04:42:10.532 - [21] RECEIVED, request_size=495 bytes, height=327388 161 | 2023-07-26 04:42:10.535 - [21] QUEUED, queue_size=1 162 | 2023-07-26 04:42:10.876 - [21] STARTED, queue_wait_time=0.341878 secs 163 | 2023-07-26 04:42:13.217 - [22] RECEIVED, request_size=494 bytes, height=327388 164 | 2023-07-26 04:42:13.218 - [22] QUEUED, queue_size=1 165 | 2023-07-26 04:42:13.416 - [22] STARTED, queue_wait_time=0.197953 secs 166 | 2023-07-26 04:42:14.360 - [23] RECEIVED, request_size=495 bytes, height=327388 167 | 2023-07-26 04:42:14.361 - [23] QUEUED, queue_size=1 168 | 2023-07-26 04:42:14.583 - [23] STARTED, queue_wait_time=0.222933 secs 169 | 2023-07-26 04:57:12.772 - [21] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c4f42dcf.jpeg] timed out after 901.029 secs 170 | 2023-07-26 04:57:15.127 - [22] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/d6596625.png] timed out after 901.024 secs 171 | 2023-07-26 04:57:15.955 - [23] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/fe85d1e7.jpeg] timed out after 901.038 secs 172 | 2023-07-26 07:02:11.058 - [24] RECEIVED, request_size=495 bytes, height=327428 173 | 2023-07-26 07:02:11.060 - [24] QUEUED, queue_size=1 174 | 2023-07-26 07:02:11.446 - [24] STARTED, queue_wait_time=0.386023 secs 175 | 2023-07-26 07:02:15.514 - [25] RECEIVED, request_size=495 bytes, height=327428 176 | 2023-07-26 07:02:15.517 - [25] QUEUED, queue_size=1 177 | 2023-07-26 07:02:15.798 - [25] STARTED, queue_wait_time=0.280671 secs 178 | 2023-07-26 07:17:13.035 - [24] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/039cc0b9.jpeg] timed out after 901.038 secs 179 | 2023-07-26 07:17:17.147 - [25] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/258f8f42.jpeg] timed out after 901.047 secs 180 | 2023-07-26 07:25:31.781 - [26] RECEIVED, request_size=495 bytes, height=327435 181 | 2023-07-26 07:25:31.782 - [26] QUEUED, queue_size=1 182 | 2023-07-26 07:25:32.131 - [26] STARTED, queue_wait_time=0.349525 secs 183 | 2023-07-26 07:31:32.296 - [26] SUCCEEDED, execution_time=360.030 secs, response_size=2087796 bytes, image_hash=fce4c5c17aaf959f2ab0cd10e3ba48e89b95832198c6b781f22920d63114dc9b 184 | 2023-07-26 07:48:53.502 - [27] RECEIVED, request_size=494 bytes, height=327435 185 | 2023-07-26 07:48:53.503 - [27] QUEUED, queue_size=1 186 | 2023-07-26 07:48:53.748 - [27] STARTED, queue_wait_time=0.244654 secs 187 | 2023-07-26 08:00:32.125 - [28] RECEIVED, request_size=495 bytes, height=327435 188 | 2023-07-26 08:00:32.128 - [28] QUEUED, queue_size=1 189 | 2023-07-26 08:00:32.444 - [28] STARTED, queue_wait_time=0.317919 secs 190 | 2023-07-26 08:03:55.322 - [27] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/260c6782.png] timed out after 901.045 secs 191 | 2023-07-26 08:15:34.047 - [28] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/25b18cc5.jpeg] timed out after 901.060 secs 192 | 2023-07-26 15:58:53.848 - [29] RECEIVED, request_size=495 bytes, height=327435 193 | 2023-07-26 15:58:53.852 - [29] QUEUED, queue_size=1 194 | 2023-07-26 15:58:54.279 - [29] STARTED, queue_wait_time=0.428467 secs 195 | 2023-07-26 16:06:24.651 - [29] SUCCEEDED, execution_time=450.045 secs, response_size=6013207 bytes, image_hash=e3567f0081cf92013beb8131a9012fc758931eb478dc052ab90e4a7c851d1729 196 | 2023-07-26 16:57:17.432 - [30] RECEIVED, request_size=495 bytes, height=327435 197 | 2023-07-26 16:57:17.435 - [30] QUEUED, queue_size=1 198 | 2023-07-26 16:57:17.779 - [30] STARTED, queue_wait_time=0.344181 secs 199 | 2023-07-26 17:02:18.161 - [30] SUCCEEDED, execution_time=300.018 secs, response_size=6603468 bytes, image_hash=f35171c21943600b61e261adabcede83080389d55457bec100185ff86215682e 200 | 2023-07-26 17:43:56.862 - [31] RECEIVED, request_size=495 bytes, height=327435 201 | 2023-07-26 17:43:56.865 - [31] QUEUED, queue_size=1 202 | 2023-07-26 17:43:57.376 - [31] STARTED, queue_wait_time=0.511793 secs 203 | 2023-07-26 17:54:57.648 - [31] SUCCEEDED, execution_time=660.051 secs, response_size=3700590 bytes, image_hash=c6768e3d4c58b3724bb79e2e33619c53c0614c81b6e2d58d483f49263c3472c9 204 | 2023-07-26 17:55:38.904 - [32] RECEIVED, request_size=494 bytes, height=327436 205 | 2023-07-26 17:55:38.905 - [32] QUEUED, queue_size=1 206 | 2023-07-26 17:55:39.150 - [32] STARTED, queue_wait_time=0.245342 secs 207 | 2023-07-26 17:55:39.788 - [33] RECEIVED, request_size=495 bytes, height=327436 208 | 2023-07-26 17:55:39.789 - [33] QUEUED, queue_size=1 209 | 2023-07-26 17:55:39.928 - [34] RECEIVED, request_size=495 bytes, height=327436 210 | 2023-07-26 17:55:39.929 - [34] QUEUED, queue_size=1 211 | 2023-07-26 17:55:40.005 - [33] STARTED, queue_wait_time=0.216400 secs 212 | 2023-07-26 17:55:40.296 - [34] STARTED, queue_wait_time=0.367725 secs 213 | 2023-07-26 17:55:42.325 - [35] RECEIVED, request_size=495 bytes, height=327436 214 | 2023-07-26 17:55:42.326 - [35] QUEUED, queue_size=1 215 | 2023-07-26 17:55:42.541 - [35] STARTED, queue_wait_time=0.214316 secs 216 | 2023-07-26 18:10:43.216 - [32] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/2f102c82.png] timed out after 902.042 secs 217 | 2023-07-26 18:10:43.441 - [34] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/967a2ee4.jpeg] timed out after 902.039 secs 218 | 2023-07-26 18:10:43.627 - [33] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/2423bef2.jpeg] timed out after 902.034 secs 219 | 2023-07-26 18:10:44.967 - [35] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/6055045f.jpeg] timed out after 902.066 secs 220 | 2023-07-26 18:18:58.119 - [36] RECEIVED, request_size=495 bytes, height=327439 221 | 2023-07-26 18:18:58.119 - [36] QUEUED, queue_size=1 222 | 2023-07-26 18:18:58.455 - [36] STARTED, queue_wait_time=0.335344 secs 223 | 2023-07-26 18:19:00.840 - [37] RECEIVED, request_size=495 bytes, height=327439 224 | 2023-07-26 18:19:00.841 - [37] QUEUED, queue_size=1 225 | 2023-07-26 18:19:01.068 - [37] STARTED, queue_wait_time=0.226985 secs 226 | 2023-07-26 18:19:01.306 - [38] RECEIVED, request_size=495 bytes, height=327439 227 | 2023-07-26 18:19:01.306 - [38] QUEUED, queue_size=1 228 | 2023-07-26 18:19:01.550 - [38] STARTED, queue_wait_time=0.243462 secs 229 | 2023-07-26 18:19:01.764 - [39] RECEIVED, request_size=494 bytes, height=327439 230 | 2023-07-26 18:19:01.764 - [39] QUEUED, queue_size=1 231 | 2023-07-26 18:19:02.003 - [39] STARTED, queue_wait_time=0.238575 secs 232 | 2023-07-26 18:19:08.585 - [40] RECEIVED, request_size=494 bytes, height=327439 233 | 2023-07-26 18:19:08.586 - [40] QUEUED, queue_size=1 234 | 2023-07-26 18:19:08.817 - [40] STARTED, queue_wait_time=0.231476 secs 235 | 2023-07-26 18:22:33.008 - [40] SUCCEEDED, execution_time=204.014 secs, response_size=1620391 bytes, image_hash=651f8a0dac836a621d08cf3cd91fa5d2d5419db3c05ad1ccaa2514f5cb0fe5f7 236 | 2023-07-26 18:34:02.086 - [36] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/38dc9e81.jpeg] timed out after 902.053 secs 237 | 2023-07-26 18:34:03.823 - [37] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/79ed9376.jpeg] timed out after 902.036 secs 238 | 2023-07-26 18:34:04.279 - [38] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e63703d4.jpeg] timed out after 902.028 secs 239 | 2023-07-26 18:34:04.500 - [39] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e8523752.png] timed out after 902.029 secs 240 | 2023-07-26 20:27:20.648 - [41] RECEIVED, request_size=495 bytes, height=327444 241 | 2023-07-26 20:27:20.652 - [41] QUEUED, queue_size=1 242 | 2023-07-26 20:27:20.917 - [41] STARTED, queue_wait_time=0.265524 secs 243 | 2023-07-26 20:32:21.087 - [41] SUCCEEDED, execution_time=300.027 secs, response_size=2240385 bytes, image_hash=b2d1fc64ef7bf112f23095d713814a39c46d73f208f2fd217af127563cb72a13 244 | 2023-07-26 20:38:55.502 - [42] RECEIVED, request_size=495 bytes, height=327450 245 | 2023-07-26 20:38:55.504 - [42] QUEUED, queue_size=1 246 | 2023-07-26 20:38:55.891 - [42] STARTED, queue_wait_time=0.387414 secs 247 | 2023-07-26 20:38:58.062 - [43] RECEIVED, request_size=495 bytes, height=327450 248 | 2023-07-26 20:38:58.062 - [43] QUEUED, queue_size=1 249 | 2023-07-26 20:38:58.090 - [44] RECEIVED, request_size=495 bytes, height=327450 250 | 2023-07-26 20:38:58.091 - [44] QUEUED, queue_size=1 251 | 2023-07-26 20:38:58.321 - [43] STARTED, queue_wait_time=0.257987 secs 252 | 2023-07-26 20:38:58.582 - [44] STARTED, queue_wait_time=0.491358 secs 253 | 2023-07-26 20:50:36.870 - [45] RECEIVED, request_size=494 bytes, height=327457 254 | 2023-07-26 20:50:36.872 - [45] QUEUED, queue_size=1 255 | 2023-07-26 20:50:37.842 - [45] STARTED, queue_wait_time=0.956145 secs 256 | 2023-07-26 20:50:40.363 - [46] RECEIVED, request_size=494 bytes, height=327457 257 | 2023-07-26 20:50:40.363 - [46] QUEUED, queue_size=1 258 | 2023-07-26 20:50:41.330 - [46] STARTED, queue_wait_time=0.956772 secs 259 | 2023-07-26 20:54:01.482 - [42] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c506c105.jpeg] timed out after 903.057 secs 260 | 2023-07-26 20:54:01.984 - [43] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/b785df1a.jpeg] timed out after 903.052 secs 261 | 2023-07-26 20:54:02.557 - [44] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/2738fd5f.jpeg] timed out after 903.066 secs 262 | 2023-07-26 20:55:27.656 - [46] SUCCEEDED, execution_time=286.105 secs, response_size=1685175 bytes, image_hash=61d6415d10bcc6c83df6fb568f8a4562f85e5ccf8d574af8ca7918b2faeb6591 263 | 2023-07-26 20:56:07.934 - [45] SUCCEEDED, execution_time=330.070 secs, response_size=473249 bytes, image_hash=33c97619a15187de844610770b2357b5fb7b969e7e3baaced4586a5817e5a702 264 | 2023-07-26 21:02:16.849 - [47] RECEIVED, request_size=494 bytes, height=327466 265 | 2023-07-26 21:02:16.850 - [47] QUEUED, queue_size=1 266 | 2023-07-26 21:02:17.058 - [47] STARTED, queue_wait_time=0.207702 secs 267 | 2023-07-26 21:02:28.382 - [48] RECEIVED, request_size=495 bytes, height=327466 268 | 2023-07-26 21:02:28.383 - [48] QUEUED, queue_size=1 269 | 2023-07-26 21:02:28.591 - [48] STARTED, queue_wait_time=0.208466 secs 270 | 2023-07-26 21:03:28.389 - [49] RECEIVED, request_size=495 bytes, height=327466 271 | 2023-07-26 21:03:28.390 - [49] QUEUED, queue_size=1 272 | 2023-07-26 21:03:28.705 - [49] STARTED, queue_wait_time=0.314121 secs 273 | 2023-07-26 21:17:19.215 - [47] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/16270591.png] timed out after 901.098 secs 274 | 2023-07-26 21:17:30.267 - [48] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/df82e67b.jpeg] timed out after 901.099 secs 275 | 2023-07-26 21:18:30.076 - [49] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c5e0ca21.jpeg] timed out after 901.070 secs 276 | 2023-07-26 21:25:38.422 - [50] RECEIVED, request_size=494 bytes, height=327478 277 | 2023-07-26 21:25:38.422 - [50] QUEUED, queue_size=1 278 | 2023-07-26 21:25:38.558 - [51] RECEIVED, request_size=495 bytes, height=327478 279 | 2023-07-26 21:25:38.558 - [51] QUEUED, queue_size=1 280 | 2023-07-26 21:25:38.772 - [50] STARTED, queue_wait_time=0.349401 secs 281 | 2023-07-26 21:25:38.987 - [51] STARTED, queue_wait_time=0.428995 secs 282 | 2023-07-26 21:25:41.009 - [52] RECEIVED, request_size=495 bytes, height=327478 283 | 2023-07-26 21:25:41.009 - [52] QUEUED, queue_size=1 284 | 2023-07-26 21:25:41.254 - [52] STARTED, queue_wait_time=0.245398 secs 285 | 2023-07-26 21:25:42.397 - [53] RECEIVED, request_size=495 bytes, height=327478 286 | 2023-07-26 21:25:42.397 - [53] QUEUED, queue_size=1 287 | 2023-07-26 21:25:42.628 - [53] STARTED, queue_wait_time=0.231254 secs 288 | 2023-07-26 21:25:47.578 - [54] RECEIVED, request_size=494 bytes, height=327478 289 | 2023-07-26 21:25:47.578 - [54] QUEUED, queue_size=1 290 | 2023-07-26 21:25:47.945 - [54] STARTED, queue_wait_time=0.355702 secs 291 | 2023-07-26 21:29:08.355 - [54] SUCCEEDED, execution_time=200.054 secs, response_size=1846223 bytes, image_hash=86a833c1db815502c58a761cffe875303f55e18d4afdeba2d519bd61017d6c1d 292 | 2023-07-26 21:37:16.863 - [55] RECEIVED, request_size=495 bytes, height=327485 293 | 2023-07-26 21:37:16.866 - [55] QUEUED, queue_size=1 294 | 2023-07-26 21:37:17.176 - [56] RECEIVED, request_size=495 bytes, height=327485 295 | 2023-07-26 21:37:17.177 - [56] QUEUED, queue_size=1 296 | 2023-07-26 21:37:18.160 - [55] STARTED, queue_wait_time=1.277590 secs 297 | 2023-07-26 21:40:41.558 - [50] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9fbcd4ab.png] timed out after 901.059 secs 298 | 2023-07-26 21:40:41.758 - [51] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/198347f6.jpeg] timed out after 901.093 secs 299 | 2023-07-26 21:40:42.172 - [56] STARTED, queue_wait_time=204.995329 secs 300 | 2023-07-26 21:40:43.516 - [52] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/11890a9c.jpeg] timed out after 901.087 secs 301 | 2023-07-26 21:40:44.243 - [53] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/983a2090.jpeg] timed out after 901.071 secs 302 | 2023-07-26 21:41:18.762 - [57] RECEIVED, request_size=495 bytes, height=327485 303 | 2023-07-26 21:41:18.763 - [57] QUEUED, queue_size=1 304 | 2023-07-26 21:41:19.088 - [57] STARTED, queue_wait_time=0.323415 secs 305 | 2023-07-26 21:48:58.009 - [58] RECEIVED, request_size=495 bytes, height=327491 306 | 2023-07-26 21:48:58.010 - [58] QUEUED, queue_size=1 307 | 2023-07-26 21:48:58.913 - [58] STARTED, queue_wait_time=0.903231 secs 308 | 2023-07-26 21:48:59.298 - [59] RECEIVED, request_size=495 bytes, height=327491 309 | 2023-07-26 21:48:59.299 - [59] QUEUED, queue_size=1 310 | 2023-07-26 21:48:59.527 - [60] RECEIVED, request_size=495 bytes, height=327491 311 | 2023-07-26 21:48:59.527 - [60] QUEUED, queue_size=1 312 | 2023-07-26 21:49:00.066 - [59] STARTED, queue_wait_time=0.767576 secs 313 | 2023-07-26 21:52:25.389 - [55] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/28f5bbff.jpeg] timed out after 905.074 secs 314 | 2023-07-26 21:52:26.131 - [60] STARTED, queue_wait_time=206.603373 secs 315 | 2023-07-26 21:53:00.224 - [61] RECEIVED, request_size=495 bytes, height=327491 316 | 2023-07-26 21:53:00.230 - [61] QUEUED, queue_size=1 317 | 2023-07-26 21:55:48.977 - [56] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/410aaa52.jpeg] timed out after 905.127 secs 318 | 2023-07-26 21:55:50.109 - [61] STARTED, queue_wait_time=169.880029 secs 319 | 2023-07-26 21:56:01.231 - [62] RECEIVED, request_size=495 bytes, height=327491 320 | 2023-07-26 21:56:01.232 - [62] QUEUED, queue_size=1 321 | 2023-07-26 21:56:24.941 - [57] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a2d745dd.jpeg] timed out after 905.055 secs 322 | 2023-07-26 21:56:25.516 - [62] STARTED, queue_wait_time=24.283883 secs 323 | 2023-07-26 22:04:07.094 - [58] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/dca5a8e6.jpeg] timed out after 905.132 secs 324 | 2023-07-26 22:04:07.750 - [59] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/af0b1483.jpeg] timed out after 905.123 secs 325 | 2023-07-26 22:07:32.433 - [60] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/aeccd0cb.jpeg] timed out after 905.114 secs 326 | 2023-07-26 22:10:55.869 - [61] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/6767277e.jpeg] timed out after 905.101 secs 327 | 2023-07-26 22:11:31.124 - [62] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/cd28f285.jpeg] timed out after 905.160 secs 328 | 2023-07-26 22:12:19.159 - [63] RECEIVED, request_size=494 bytes, height=327502 329 | 2023-07-26 22:12:19.160 - [63] QUEUED, queue_size=1 330 | 2023-07-26 22:12:19.515 - [63] STARTED, queue_wait_time=0.354383 secs 331 | 2023-07-26 22:12:21.137 - [64] RECEIVED, request_size=495 bytes, height=327502 332 | 2023-07-26 22:12:21.138 - [64] QUEUED, queue_size=1 333 | 2023-07-26 22:12:21.332 - [64] STARTED, queue_wait_time=0.193771 secs 334 | 2023-07-26 22:12:21.708 - [65] RECEIVED, request_size=495 bytes, height=327502 335 | 2023-07-26 22:12:21.709 - [65] QUEUED, queue_size=1 336 | 2023-07-26 22:12:21.908 - [65] STARTED, queue_wait_time=0.199350 secs 337 | 2023-07-26 22:23:58.598 - [66] RECEIVED, request_size=495 bytes, height=327508 338 | 2023-07-26 22:23:58.599 - [66] QUEUED, queue_size=1 339 | 2023-07-26 22:23:58.841 - [67] RECEIVED, request_size=495 bytes, height=327508 340 | 2023-07-26 22:23:58.842 - [67] QUEUED, queue_size=1 341 | 2023-07-26 22:23:59.534 - [66] STARTED, queue_wait_time=0.930184 secs 342 | 2023-07-26 22:24:00.413 - [67] STARTED, queue_wait_time=1.571480 secs 343 | 2023-07-26 22:24:02.494 - [68] RECEIVED, request_size=494 bytes, height=327508 344 | 2023-07-26 22:24:02.495 - [68] QUEUED, queue_size=1 345 | 2023-07-26 22:27:26.712 - [63] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/3bd8ba8a.png] timed out after 905.150 secs 346 | 2023-07-26 22:27:27.267 - [68] STARTED, queue_wait_time=204.768940 secs 347 | 2023-07-26 22:27:27.799 - [64] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a8bd3a86.jpeg] timed out after 905.068 secs 348 | 2023-07-26 22:27:28.322 - [65] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/abec0eb7.jpeg] timed out after 905.111 secs 349 | 2023-07-26 22:39:06.902 - [67] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/690f03bc.jpeg] timed out after 905.073 secs 350 | 2023-07-26 22:39:06.993 - [66] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/db521b2f.jpeg] timed out after 905.106 secs 351 | 2023-07-26 22:39:27.499 - [68] SUCCEEDED, execution_time=720.089 secs, response_size=2531235 bytes, image_hash=c0e762d2b6a7512f9805b6320c35247f35343d5e1e956f3058b5762245514a8d 352 | 2023-07-26 22:58:57.240 - [69] RECEIVED, request_size=495 bytes, height=327522 353 | 2023-07-26 22:58:57.240 - [69] QUEUED, queue_size=1 354 | 2023-07-26 22:58:57.575 - [69] STARTED, queue_wait_time=0.334831 secs 355 | 2023-07-26 23:04:01.855 - [69] SUCCEEDED, execution_time=304.025 secs, response_size=3003522 bytes, image_hash=049b9874c0534e107e06460152114b575f29bd062b2a1be3440b9747d2c77fb7 356 | 2023-07-26 23:22:15.482 - [70] RECEIVED, request_size=494 bytes, height=327533 357 | 2023-07-26 23:22:15.483 - [70] QUEUED, queue_size=1 358 | 2023-07-26 23:22:15.728 - [70] STARTED, queue_wait_time=0.245333 secs 359 | 2023-07-26 23:28:19.867 - [70] SUCCEEDED, execution_time=364.020 secs, response_size=1832892 bytes, image_hash=2c2d4db6476f4ed6d9946e399bc4eac93ad5c164238f06684411ce2cd9fda606 360 | 2023-07-26 23:33:56.482 - [71] RECEIVED, request_size=494 bytes, height=327538 361 | 2023-07-26 23:33:56.486 - [71] QUEUED, queue_size=1 362 | 2023-07-26 23:33:56.836 - [71] STARTED, queue_wait_time=0.351136 secs 363 | 2023-07-26 23:41:23.043 - [71] SUCCEEDED, execution_time=444.042 secs, response_size=39670258 bytes, image_hash=4310736a1195c5acc63ac8860e291aeb99700676a0c0c833cd2336ec6310eedd 364 | 2023-07-27 00:08:57.300 - [72] RECEIVED, request_size=494 bytes, height=327553 365 | 2023-07-27 00:08:57.301 - [72] QUEUED, queue_size=1 366 | 2023-07-27 00:08:57.491 - [73] RECEIVED, request_size=495 bytes, height=327553 367 | 2023-07-27 00:08:57.491 - [73] QUEUED, queue_size=1 368 | 2023-07-27 00:08:57.533 - [72] STARTED, queue_wait_time=0.232865 secs 369 | 2023-07-27 00:08:57.826 - [73] STARTED, queue_wait_time=0.334580 secs 370 | 2023-07-27 00:08:58.359 - [74] RECEIVED, request_size=495 bytes, height=327553 371 | 2023-07-27 00:08:58.359 - [74] QUEUED, queue_size=1 372 | 2023-07-27 00:08:58.570 - [74] STARTED, queue_wait_time=0.211135 secs 373 | 2023-07-27 00:20:57.219 - [75] RECEIVED, request_size=495 bytes, height=327558 374 | 2023-07-27 00:20:57.224 - [75] QUEUED, queue_size=1 375 | 2023-07-27 00:20:58.019 - [75] STARTED, queue_wait_time=0.795838 secs 376 | 2023-07-27 00:24:09.282 - [72] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9ee1a1eb.png] timed out after 911.053 secs 377 | 2023-07-27 00:24:09.702 - [73] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/f46b9774.jpeg] timed out after 911.040 secs 378 | 2023-07-27 00:24:10.010 - [74] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/da546a62.jpeg] timed out after 911.040 secs 379 | 2023-07-27 00:30:56.438 - [75] SUCCEEDED, execution_time=598.058 secs, response_size=6457080 bytes, image_hash=76ddc77f8d428a1ca845c9d4c1b6d3a86fcda3ef8ec37419a000e76173fee0bc 380 | 2023-07-27 00:32:19.899 - [76] RECEIVED, request_size=495 bytes, height=327562 381 | 2023-07-27 00:32:19.900 - [76] QUEUED, queue_size=1 382 | 2023-07-27 00:32:20.240 - [76] STARTED, queue_wait_time=0.341192 secs 383 | 2023-07-27 00:32:21.477 - [77] RECEIVED, request_size=494 bytes, height=327562 384 | 2023-07-27 00:32:21.477 - [77] QUEUED, queue_size=1 385 | 2023-07-27 00:32:21.734 - [77] STARTED, queue_wait_time=0.256946 secs 386 | 2023-07-27 00:44:00.388 - [78] RECEIVED, request_size=495 bytes, height=327565 387 | 2023-07-27 00:44:00.402 - [78] QUEUED, queue_size=1 388 | 2023-07-27 00:44:01.094 - [78] STARTED, queue_wait_time=0.692536 secs 389 | 2023-07-27 00:44:22.106 - [77] SUCCEEDED, execution_time=720.035 secs, response_size=2999259 bytes, image_hash=bb02f4ad9b3ebaf785d2cf2f207df120eb09125ae2e44e45589a0369b0ed3973 390 | 2023-07-27 00:47:33.794 - [76] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/b1dbf775.jpeg] timed out after 913.043 secs 391 | 2023-07-27 00:59:14.567 - [78] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c06b7c49.jpeg] timed out after 913.047 secs 392 | 2023-07-27 01:07:19.369 - [79] RECEIVED, request_size=495 bytes, height=327576 393 | 2023-07-27 01:07:19.370 - [79] QUEUED, queue_size=1 394 | 2023-07-27 01:07:19.638 - [79] STARTED, queue_wait_time=0.268826 secs 395 | 2023-07-27 01:07:19.726 - [80] RECEIVED, request_size=494 bytes, height=327576 396 | 2023-07-27 01:07:19.727 - [80] QUEUED, queue_size=1 397 | 2023-07-27 01:07:19.804 - [81] RECEIVED, request_size=494 bytes, height=327576 398 | 2023-07-27 01:07:19.804 - [81] QUEUED, queue_size=1 399 | 2023-07-27 01:07:19.977 - [80] STARTED, queue_wait_time=0.248808 secs 400 | 2023-07-27 01:07:20.128 - [82] RECEIVED, request_size=495 bytes, height=327576 401 | 2023-07-27 01:07:20.129 - [82] QUEUED, queue_size=1 402 | 2023-07-27 01:07:20.201 - [81] STARTED, queue_wait_time=0.396719 secs 403 | 2023-07-27 01:07:20.457 - [82] STARTED, queue_wait_time=0.328508 secs 404 | 2023-07-27 01:19:03.895 - [83] RECEIVED, request_size=495 bytes, height=327582 405 | 2023-07-27 01:19:03.898 - [83] QUEUED, queue_size=1 406 | 2023-07-27 01:19:05.148 - [83] STARTED, queue_wait_time=1.249220 secs 407 | 2023-07-27 01:22:22.081 - [80] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/635e8d21.png] timed out after 901.045 secs 408 | 2023-07-27 01:22:22.340 - [81] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5397e783.png] timed out after 901.037 secs 409 | 2023-07-27 01:22:22.451 - [79] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/460becc5.jpeg] timed out after 901.053 secs 410 | 2023-07-27 01:22:22.511 - [82] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/99ee9716.jpeg] timed out after 901.060 secs 411 | 2023-07-27 01:27:45.638 - [83] SUCCEEDED, execution_time=520.042 secs, response_size=8266835 bytes, image_hash=985d86bcaebba69ec9612cccf2e4f4aae41a98d7ef44f99397ac494515274d1c 412 | 2023-07-27 02:05:40.922 - [84] RECEIVED, request_size=495 bytes, height=327601 413 | 2023-07-27 02:05:40.927 - [84] QUEUED, queue_size=1 414 | 2023-07-27 02:05:41.384 - [84] STARTED, queue_wait_time=0.458870 secs 415 | 2023-07-27 02:10:47.643 - [84] SUCCEEDED, execution_time=306.016 secs, response_size=3847939 bytes, image_hash=e0e6b99926ea31ea32728aada07055ef1d22597c0ce212c3713f9d26b9debadd 416 | 2023-07-27 02:52:17.086 - [85] RECEIVED, request_size=495 bytes, height=327621 417 | 2023-07-27 02:52:17.088 - [85] QUEUED, queue_size=1 418 | 2023-07-27 02:52:17.463 - [85] STARTED, queue_wait_time=0.374487 secs 419 | 2023-07-27 02:52:20.206 - [86] RECEIVED, request_size=495 bytes, height=327621 420 | 2023-07-27 02:52:20.207 - [86] QUEUED, queue_size=1 421 | 2023-07-27 02:52:20.398 - [86] STARTED, queue_wait_time=0.191117 secs 422 | 2023-07-27 03:07:29.170 - [85] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/da45755c.jpeg] timed out after 911.057 secs 423 | 2023-07-27 03:07:31.735 - [86] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9abb11fd.jpeg] timed out after 911.048 secs 424 | 2023-07-27 03:15:40.224 - [87] RECEIVED, request_size=495 bytes, height=327635 425 | 2023-07-27 03:15:40.224 - [87] QUEUED, queue_size=1 426 | 2023-07-27 03:15:40.464 - [87] STARTED, queue_wait_time=0.239475 secs 427 | 2023-07-27 03:22:12.671 - [87] SUCCEEDED, execution_time=392.031 secs, response_size=2872166 bytes, image_hash=ab67fc71e25607b77bc897effa32bd5aad65a91242bf408d1fc5f11ce8a3922e 428 | 2023-07-27 03:39:00.159 - [88] RECEIVED, request_size=495 bytes, height=327642 429 | 2023-07-27 03:39:00.161 - [88] QUEUED, queue_size=1 430 | 2023-07-27 03:39:00.542 - [88] STARTED, queue_wait_time=0.381596 secs 431 | 2023-07-27 03:45:30.680 - [88] SUCCEEDED, execution_time=390.026 secs, response_size=1443561 bytes, image_hash=aa29fc1fb2f1328eef19c68ea29824f5182cea291eaace0f4b2243b324e84af9 432 | 2023-07-27 03:50:38.588 - [89] RECEIVED, request_size=495 bytes, height=327643 433 | 2023-07-27 03:50:38.589 - [89] QUEUED, queue_size=1 434 | 2023-07-27 03:50:39.008 - [89] STARTED, queue_wait_time=0.419768 secs 435 | 2023-07-27 03:50:40.564 - [90] RECEIVED, request_size=495 bytes, height=327643 436 | 2023-07-27 03:50:40.565 - [90] QUEUED, queue_size=1 437 | 2023-07-27 03:50:40.820 - [90] STARTED, queue_wait_time=0.255750 secs 438 | 2023-07-27 03:50:42.607 - [91] RECEIVED, request_size=495 bytes, height=327643 439 | 2023-07-27 03:50:42.607 - [91] QUEUED, queue_size=1 440 | 2023-07-27 03:50:42.901 - [91] STARTED, queue_wait_time=0.294437 secs 441 | 2023-07-27 04:02:18.308 - [92] RECEIVED, request_size=494 bytes, height=327645 442 | 2023-07-27 04:02:18.311 - [92] QUEUED, queue_size=1 443 | 2023-07-27 04:02:19.205 - [92] STARTED, queue_wait_time=0.894607 secs 444 | 2023-07-27 04:05:51.482 - [89] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/513eac63.jpeg] timed out after 911.052 secs 445 | 2023-07-27 04:05:52.507 - [90] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/cafd7d35.jpeg] timed out after 911.057 secs 446 | 2023-07-27 04:05:54.386 - [91] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/fd6749f2.jpeg] timed out after 911.075 secs 447 | 2023-07-27 04:07:18.293 - [92] SUCCEEDED, execution_time=299.047 secs, response_size=473241 bytes, image_hash=33c97619a15187de844610770b2357b5fb7b969e7e3baaced4586a5817e5a702 448 | 2023-07-27 04:13:58.526 - [93] RECEIVED, request_size=495 bytes, height=327647 449 | 2023-07-27 04:13:58.527 - [93] QUEUED, queue_size=1 450 | 2023-07-27 04:13:58.541 - [94] RECEIVED, request_size=495 bytes, height=327647 451 | 2023-07-27 04:13:58.541 - [94] QUEUED, queue_size=1 452 | 2023-07-27 04:13:58.874 - [93] STARTED, queue_wait_time=0.347084 secs 453 | 2023-07-27 04:13:59.090 - [94] STARTED, queue_wait_time=0.548990 secs 454 | 2023-07-27 04:14:00.323 - [95] RECEIVED, request_size=495 bytes, height=327647 455 | 2023-07-27 04:14:00.324 - [95] QUEUED, queue_size=1 456 | 2023-07-27 04:14:00.455 - [96] RECEIVED, request_size=495 bytes, height=327647 457 | 2023-07-27 04:14:00.459 - [96] QUEUED, queue_size=1 458 | 2023-07-27 04:14:00.543 - [95] STARTED, queue_wait_time=0.220302 secs 459 | 2023-07-27 04:14:00.744 - [96] STARTED, queue_wait_time=0.285752 secs 460 | 2023-07-27 04:14:01.616 - [97] RECEIVED, request_size=494 bytes, height=327647 461 | 2023-07-27 04:14:01.617 - [97] QUEUED, queue_size=1 462 | 2023-07-27 04:14:01.892 - [97] STARTED, queue_wait_time=0.275047 secs 463 | 2023-07-27 04:29:04.084 - [93] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/4dcda74d.jpeg] timed out after 903.080 secs 464 | 2023-07-27 04:29:04.514 - [94] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/0772a073.jpeg] timed out after 903.052 secs 465 | 2023-07-27 04:29:04.798 - [95] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/193244fd.jpeg] timed out after 903.071 secs 466 | 2023-07-27 04:29:04.982 - [96] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/0a713dc2.jpeg] timed out after 903.089 secs 467 | 2023-07-27 04:29:05.400 - [97] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5bb8833a.png] timed out after 903.094 secs 468 | 2023-07-27 05:12:18.023 - [98] RECEIVED, request_size=495 bytes, height=327667 469 | 2023-07-27 05:12:18.025 - [98] QUEUED, queue_size=1 470 | 2023-07-27 05:12:18.431 - [98] STARTED, queue_wait_time=0.406176 secs 471 | 2023-07-27 05:17:37.803 - [98] SUCCEEDED, execution_time=319.036 secs, response_size=5558738 bytes, image_hash=ff1eecec7f043ce0e337c36c2bf3850684cd86b8f7954e60b2a171cf39965c27 472 | 2023-07-27 05:24:00.869 - [99] RECEIVED, request_size=495 bytes, height=327672 473 | 2023-07-27 05:24:00.870 - [99] QUEUED, queue_size=1 474 | 2023-07-27 05:24:01.149 - [99] STARTED, queue_wait_time=0.279745 secs 475 | 2023-07-27 05:39:04.582 - [99] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/55879279.jpeg] timed out after 903.086 secs 476 | 2023-07-27 05:47:20.393 - [100] RECEIVED, request_size=495 bytes, height=327683 477 | 2023-07-27 05:47:20.394 - [100] QUEUED, queue_size=1 478 | 2023-07-27 05:47:20.605 - [100] STARTED, queue_wait_time=0.211220 secs 479 | 2023-07-27 05:58:58.475 - [101] RECEIVED, request_size=495 bytes, height=327686 480 | 2023-07-27 05:58:58.479 - [101] QUEUED, queue_size=1 481 | 2023-07-27 05:58:58.978 - [101] STARTED, queue_wait_time=0.500288 secs 482 | 2023-07-27 05:59:00.166 - [102] RECEIVED, request_size=495 bytes, height=327686 483 | 2023-07-27 05:59:00.167 - [102] QUEUED, queue_size=1 484 | 2023-07-27 05:59:00.772 - [102] STARTED, queue_wait_time=0.604517 secs 485 | 2023-07-27 06:02:24.081 - [100] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/d1cb1561.jpeg] timed out after 903.094 secs 486 | 2023-07-27 06:14:03.125 - [101] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e1602202.jpeg] timed out after 903.060 secs 487 | 2023-07-27 06:14:04.284 - [102] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/1c243a1e.jpeg] timed out after 903.051 secs 488 | 2023-07-27 06:33:59.441 - [103] RECEIVED, request_size=495 bytes, height=327702 489 | 2023-07-27 06:33:59.445 - [103] QUEUED, queue_size=1 490 | 2023-07-27 06:33:59.690 - [103] STARTED, queue_wait_time=0.248229 secs 491 | 2023-07-27 06:34:00.956 - [104] RECEIVED, request_size=494 bytes, height=327702 492 | 2023-07-27 06:34:00.957 - [104] QUEUED, queue_size=1 493 | 2023-07-27 06:34:01.291 - [104] STARTED, queue_wait_time=0.335103 secs 494 | 2023-07-27 06:49:03.433 - [103] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/0b10c54a.jpeg] timed out after 903.057 secs 495 | 2023-07-27 06:49:04.766 - [104] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/84b763c1.png] timed out after 903.077 secs 496 | 2023-07-27 06:57:20.495 - [105] RECEIVED, request_size=495 bytes, height=327714 497 | 2023-07-27 06:57:20.496 - [105] QUEUED, queue_size=1 498 | 2023-07-27 06:57:20.953 - [105] STARTED, queue_wait_time=0.457401 secs 499 | 2023-07-27 07:00:04.298 - [105] CANCELLED, Dupe detection task [/home/ubuntu/.pastel/dd-server/5a82a712.jpeg] was cancelled after 162.600 secs, process was killed by signal SIGKILL 500 | 2023-07-27 07:32:19.601 - [106] RECEIVED, request_size=495 bytes, height=327730 501 | 2023-07-27 07:32:19.602 - [106] QUEUED, queue_size=1 502 | 2023-07-27 07:32:19.960 - [106] STARTED, queue_wait_time=0.358105 secs 503 | 2023-07-27 07:32:20.527 - [107] RECEIVED, request_size=495 bytes, height=327730 504 | 2023-07-27 07:32:20.528 - [107] QUEUED, queue_size=1 505 | 2023-07-27 07:32:20.787 - [107] STARTED, queue_wait_time=0.259252 secs 506 | 2023-07-27 07:43:59.301 - [108] RECEIVED, request_size=495 bytes, height=327735 507 | 2023-07-27 07:43:59.304 - [108] QUEUED, queue_size=1 508 | 2023-07-27 07:43:59.917 - [108] STARTED, queue_wait_time=0.612982 secs 509 | 2023-07-27 07:47:24.089 - [106] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/26a5a63d.jpeg] timed out after 903.078 secs 510 | 2023-07-27 07:47:24.342 - [107] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/8ce3e963.jpeg] timed out after 903.106 secs 511 | 2023-07-27 07:50:25.324 - [108] SUCCEEDED, execution_time=385.060 secs, response_size=5095999 bytes, image_hash=4cab48568005d0c976901fc97a5cce0b7876f49580ccad86e0675dc26a9f0d17 512 | 2023-07-27 08:07:19.188 - [109] RECEIVED, request_size=495 bytes, height=327744 513 | 2023-07-27 08:07:19.189 - [109] QUEUED, queue_size=1 514 | 2023-07-27 08:07:19.524 - [109] STARTED, queue_wait_time=0.335591 secs 515 | 2023-07-27 08:22:23.062 - [109] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/192ef956.jpeg] timed out after 903.066 secs 516 | 2023-07-27 08:42:18.698 - [110] RECEIVED, request_size=494 bytes, height=327757 517 | 2023-07-27 08:42:18.700 - [110] QUEUED, queue_size=1 518 | 2023-07-27 08:42:18.943 - [110] STARTED, queue_wait_time=0.243795 secs 519 | 2023-07-27 08:44:20.145 - [110] SUCCEEDED, execution_time=121.013 secs, response_size=2999259 bytes, image_hash=bb02f4ad9b3ebaf785d2cf2f207df120eb09125ae2e44e45589a0369b0ed3973 520 | 2023-07-27 09:05:39.544 - [111] RECEIVED, request_size=495 bytes, height=327767 521 | 2023-07-27 09:05:39.546 - [111] QUEUED, queue_size=1 522 | 2023-07-27 09:05:39.606 - [112] RECEIVED, request_size=495 bytes, height=327767 523 | 2023-07-27 09:05:39.607 - [112] QUEUED, queue_size=1 524 | 2023-07-27 09:05:39.899 - [111] STARTED, queue_wait_time=0.353472 secs 525 | 2023-07-27 09:05:39.931 - [113] RECEIVED, request_size=495 bytes, height=327767 526 | 2023-07-27 09:05:39.931 - [113] QUEUED, queue_size=1 527 | 2023-07-27 09:05:40.101 - [112] STARTED, queue_wait_time=0.494276 secs 528 | 2023-07-27 09:05:40.325 - [113] STARTED, queue_wait_time=0.393536 secs 529 | 2023-07-27 09:20:45.717 - [111] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a2900401.jpeg] timed out after 905.110 secs 530 | 2023-07-27 09:20:45.886 - [112] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/94dfad41.jpeg] timed out after 905.075 secs 531 | 2023-07-27 09:20:45.896 - [113] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/24f3a4dd.jpeg] timed out after 905.103 secs 532 | 2023-07-27 09:52:19.763 - [114] RECEIVED, request_size=495 bytes, height=327792 533 | 2023-07-27 09:52:19.776 - [114] QUEUED, queue_size=1 534 | 2023-07-27 09:52:20.025 - [114] STARTED, queue_wait_time=0.249378 secs 535 | 2023-07-27 09:57:40.284 - [114] SUCCEEDED, execution_time=320.066 secs, response_size=3426187 bytes, image_hash=f9a9f88520faa9d586fe17f670a4b18ee184ccf7f7387f94479336997390061f 536 | 2023-07-27 10:04:00.906 - [115] RECEIVED, request_size=495 bytes, height=327796 537 | 2023-07-27 10:04:00.907 - [115] QUEUED, queue_size=1 538 | 2023-07-27 10:04:01.416 - [115] STARTED, queue_wait_time=0.508838 secs 539 | 2023-07-27 10:10:46.685 - [115] SUCCEEDED, execution_time=405.041 secs, response_size=3676206 bytes, image_hash=e0e6b99926ea31ea32728aada07055ef1d22597c0ce212c3713f9d26b9debadd 540 | 2023-07-27 10:15:51.249 - [116] RECEIVED, request_size=495 bytes, height=327799 541 | 2023-07-27 10:15:51.254 - [116] QUEUED, queue_size=1 542 | 2023-07-27 10:15:51.268 - [117] RECEIVED, request_size=494 bytes, height=327799 543 | 2023-07-27 10:15:51.270 - [117] QUEUED, queue_size=1 544 | 2023-07-27 10:15:51.513 - [116] STARTED, queue_wait_time=0.260416 secs 545 | 2023-07-27 10:15:51.774 - [117] STARTED, queue_wait_time=0.504967 secs 546 | 2023-07-27 10:27:22.470 - [118] RECEIVED, request_size=495 bytes, height=327803 547 | 2023-07-27 10:27:22.474 - [118] QUEUED, queue_size=1 548 | 2023-07-27 10:27:23.131 - [118] STARTED, queue_wait_time=0.658841 secs 549 | 2023-07-27 10:30:55.448 - [116] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/481a7f84.jpeg] timed out after 903.072 secs 550 | 2023-07-27 10:30:55.518 - [117] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/de87b151.png] timed out after 903.072 secs 551 | 2023-07-27 10:42:26.754 - [118] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/02d3573f.jpeg] timed out after 903.070 secs 552 | 2023-07-27 11:49:02.671 - [119] RECEIVED, request_size=494 bytes, height=327833 553 | 2023-07-27 11:49:02.675 - [119] QUEUED, queue_size=1 554 | 2023-07-27 11:49:03.027 - [119] STARTED, queue_wait_time=0.352367 secs 555 | 2023-07-27 11:52:54.112 - [119] SUCCEEDED, execution_time=231.021 secs, response_size=630369 bytes, image_hash=d1ce51e32772a4738e73ba8a37def7a7fb910c72a923595ba757e2cff043f2df 556 | 2023-07-28 00:04:07.613 - [120] RECEIVED, request_size=495 bytes, height=328106 557 | 2023-07-28 00:04:07.616 - [120] QUEUED, queue_size=1 558 | 2023-07-28 00:04:07.648 - [121] RECEIVED, request_size=494 bytes, height=328106 559 | 2023-07-28 00:04:07.649 - [121] QUEUED, queue_size=1 560 | 2023-07-28 00:04:07.670 - [122] RECEIVED, request_size=495 bytes, height=328106 561 | 2023-07-28 00:04:07.672 - [122] QUEUED, queue_size=1 562 | 2023-07-28 00:04:08.151 - [123] RECEIVED, request_size=495 bytes, height=328106 563 | 2023-07-28 00:04:08.151 - [123] QUEUED, queue_size=1 564 | 2023-07-28 00:04:08.604 - [120] STARTED, queue_wait_time=0.988738 secs 565 | 2023-07-28 00:04:08.923 - [121] STARTED, queue_wait_time=1.273455 secs 566 | 2023-07-28 00:04:09.178 - [122] STARTED, queue_wait_time=1.504848 secs 567 | 2023-07-28 00:04:09.486 - [123] STARTED, queue_wait_time=1.335199 secs 568 | 2023-07-28 00:04:11.337 - [124] RECEIVED, request_size=494 bytes, height=328106 569 | 2023-07-28 00:04:11.338 - [124] QUEUED, queue_size=1 570 | 2023-07-28 00:04:11.574 - [124] STARTED, queue_wait_time=0.235482 secs 571 | 2023-07-28 00:05:07.692 - [125] RECEIVED, request_size=495 bytes, height=328106 572 | 2023-07-28 00:05:07.693 - [125] QUEUED, queue_size=1 573 | 2023-07-28 00:19:11.689 - [122] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/ee9112c5.jpeg] timed out after 901.096 secs 574 | 2023-07-28 00:19:11.777 - [121] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c92f3bdd.png] timed out after 901.101 secs 575 | 2023-07-28 00:19:11.869 - [120] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/d84b39bd.jpeg] timed out after 901.156 secs 576 | 2023-07-28 00:19:12.173 - [123] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/61f8a434.jpeg] timed out after 901.083 secs 577 | 2023-07-28 00:19:12.245 - [125] STARTED, queue_wait_time=844.551791 secs 578 | 2023-07-28 00:19:13.183 - [124] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/17eac110.png] timed out after 901.114 secs 579 | 2023-07-28 00:27:25.035 - [126] RECEIVED, request_size=494 bytes, height=328110 580 | 2023-07-28 00:27:25.037 - [126] QUEUED, queue_size=1 581 | 2023-07-28 00:27:25.466 - [126] STARTED, queue_wait_time=0.428863 secs 582 | 2023-07-28 00:27:25.493 - [127] RECEIVED, request_size=495 bytes, height=328110 583 | 2023-07-28 00:27:25.494 - [127] QUEUED, queue_size=1 584 | 2023-07-28 00:27:25.764 - [127] STARTED, queue_wait_time=0.269831 secs 585 | 2023-07-28 00:27:26.153 - [128] RECEIVED, request_size=495 bytes, height=328110 586 | 2023-07-28 00:27:26.154 - [128] QUEUED, queue_size=1 587 | 2023-07-28 00:27:26.369 - [128] STARTED, queue_wait_time=0.215672 secs 588 | 2023-07-28 00:27:29.107 - [129] RECEIVED, request_size=494 bytes, height=328110 589 | 2023-07-28 00:27:29.108 - [129] QUEUED, queue_size=1 590 | 2023-07-28 00:27:29.322 - [129] STARTED, queue_wait_time=0.213501 secs 591 | 2023-07-28 00:27:33.805 - [130] RECEIVED, request_size=495 bytes, height=328110 592 | 2023-07-28 00:27:33.806 - [130] QUEUED, queue_size=1 593 | 2023-07-28 00:34:14.353 - [125] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e2b6cde8.jpeg] timed out after 901.117 secs 594 | 2023-07-28 00:34:15.042 - [130] STARTED, queue_wait_time=401.236726 secs 595 | 2023-07-28 00:34:33.974 - [131] RECEIVED, request_size=495 bytes, height=328110 596 | 2023-07-28 00:34:33.974 - [131] QUEUED, queue_size=1 597 | 2023-07-28 00:42:28.166 - [127] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/b43275a4.jpeg] timed out after 901.136 secs 598 | 2023-07-28 00:42:28.367 - [126] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/0daf44ee.png] timed out after 901.097 secs 599 | 2023-07-28 00:42:28.956 - [128] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/feaee290.jpeg] timed out after 901.116 secs 600 | 2023-07-28 00:42:28.968 - [131] STARTED, queue_wait_time=474.993415 secs 601 | 2023-07-28 00:42:31.139 - [129] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/d19c120d.png] timed out after 901.147 secs 602 | 2023-07-28 00:49:16.821 - [130] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/935af2c1.jpeg] timed out after 901.125 secs 603 | 2023-07-28 00:54:38.387 - [131] SUCCEEDED, execution_time=729.132 secs, response_size=4399787 bytes, image_hash=ee9ac1d75785c2d416224ec1b0642609581a6309db3afa5d069caa07bf155595 604 | 2023-07-28 01:02:25.810 - [132] RECEIVED, request_size=495 bytes, height=328116 605 | 2023-07-28 01:02:25.812 - [132] QUEUED, queue_size=1 606 | 2023-07-28 01:02:26.205 - [132] STARTED, queue_wait_time=0.394148 secs 607 | 2023-07-28 01:02:29.091 - [133] RECEIVED, request_size=495 bytes, height=328116 608 | 2023-07-28 01:02:29.091 - [133] QUEUED, queue_size=1 609 | 2023-07-28 01:02:29.346 - [133] STARTED, queue_wait_time=0.255460 secs 610 | 2023-07-28 01:02:30.976 - [134] RECEIVED, request_size=495 bytes, height=328116 611 | 2023-07-28 01:02:30.976 - [134] QUEUED, queue_size=1 612 | 2023-07-28 01:02:31.181 - [134] STARTED, queue_wait_time=0.204476 secs 613 | 2023-07-28 01:17:40.766 - [132] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/3e03098c.jpeg] timed out after 913.075 secs 614 | 2023-07-28 01:17:43.163 - [133] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/08a375e2.jpeg] timed out after 913.058 secs 615 | 2023-07-28 01:17:44.885 - [134] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/3d113666.jpeg] timed out after 913.099 secs 616 | 2023-07-28 01:37:23.886 - [135] RECEIVED, request_size=495 bytes, height=328124 617 | 2023-07-28 01:37:23.894 - [135] QUEUED, queue_size=1 618 | 2023-07-28 01:37:24.230 - [135] STARTED, queue_wait_time=0.336993 secs 619 | 2023-07-28 01:49:08.434 - [136] RECEIVED, request_size=495 bytes, height=328127 620 | 2023-07-28 01:49:08.438 - [136] QUEUED, queue_size=1 621 | 2023-07-28 01:49:08.767 - [136] STARTED, queue_wait_time=0.329883 secs 622 | 2023-07-28 01:52:37.672 - [135] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9a0ae8b3.jpeg] timed out after 913.103 secs 623 | 2023-07-28 02:00:46.093 - [137] RECEIVED, request_size=495 bytes, height=328128 624 | 2023-07-28 02:00:46.095 - [137] QUEUED, queue_size=1 625 | 2023-07-28 02:00:46.684 - [137] STARTED, queue_wait_time=0.576279 secs 626 | 2023-07-28 02:00:48.712 - [138] RECEIVED, request_size=495 bytes, height=328128 627 | 2023-07-28 02:00:48.713 - [138] QUEUED, queue_size=1 628 | 2023-07-28 02:00:49.072 - [138] STARTED, queue_wait_time=0.359648 secs 629 | 2023-07-28 02:00:51.724 - [139] RECEIVED, request_size=494 bytes, height=328128 630 | 2023-07-28 02:00:51.726 - [139] QUEUED, queue_size=1 631 | 2023-07-28 02:00:52.155 - [139] STARTED, queue_wait_time=0.425992 secs 632 | 2023-07-28 02:00:54.339 - [140] RECEIVED, request_size=495 bytes, height=328128 633 | 2023-07-28 02:00:54.340 - [140] QUEUED, queue_size=1 634 | 2023-07-28 02:00:54.931 - [140] STARTED, queue_wait_time=0.591419 secs 635 | 2023-07-28 02:04:22.458 - [136] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/32f2de43.jpeg] timed out after 913.059 secs 636 | 2023-07-28 02:16:00.639 - [137] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c5e8e2ef.jpeg] timed out after 913.058 secs 637 | 2023-07-28 02:16:03.155 - [138] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/68ea9e56.jpeg] timed out after 913.047 secs 638 | 2023-07-28 02:16:05.864 - [139] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/ebf89842.png] timed out after 913.112 secs 639 | 2023-07-28 02:16:08.685 - [140] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/3272fb35.jpeg] timed out after 913.055 secs 640 | 2023-07-28 02:35:43.568 - [141] RECEIVED, request_size=495 bytes, height=328133 641 | 2023-07-28 02:35:43.570 - [141] QUEUED, queue_size=1 642 | 2023-07-28 02:35:43.974 - [141] STARTED, queue_wait_time=0.403794 secs 643 | 2023-07-28 02:35:49.572 - [142] RECEIVED, request_size=494 bytes, height=328133 644 | 2023-07-28 02:35:49.573 - [142] QUEUED, queue_size=1 645 | 2023-07-28 02:35:49.803 - [142] STARTED, queue_wait_time=0.229990 secs 646 | 2023-07-28 02:50:57.944 - [141] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/60067cf9.jpeg] timed out after 913.067 secs 647 | 2023-07-28 02:51:03.451 - [142] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/fff6fec6.png] timed out after 913.059 secs 648 | 2023-07-28 03:10:45.328 - [143] RECEIVED, request_size=494 bytes, height=328139 649 | 2023-07-28 03:10:45.330 - [143] QUEUED, queue_size=1 650 | 2023-07-28 03:10:45.663 - [143] STARTED, queue_wait_time=0.333227 secs 651 | 2023-07-28 03:10:47.356 - [144] RECEIVED, request_size=494 bytes, height=328139 652 | 2023-07-28 03:10:47.357 - [144] QUEUED, queue_size=1 653 | 2023-07-28 03:10:47.597 - [144] STARTED, queue_wait_time=0.239777 secs 654 | 2023-07-28 03:10:48.108 - [145] RECEIVED, request_size=495 bytes, height=328139 655 | 2023-07-28 03:10:48.109 - [145] QUEUED, queue_size=1 656 | 2023-07-28 03:10:48.346 - [145] STARTED, queue_wait_time=0.238197 secs 657 | 2023-07-28 03:26:00.702 - [143] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/09fcf928.png] timed out after 913.042 secs 658 | 2023-07-28 03:26:01.873 - [144] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5fcc4d2e.png] timed out after 913.057 secs 659 | 2023-07-28 03:26:02.560 - [145] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/263ce544.jpeg] timed out after 913.040 secs 660 | 2023-07-28 03:45:45.642 - [146] RECEIVED, request_size=495 bytes, height=328148 661 | 2023-07-28 03:45:45.648 - [146] QUEUED, queue_size=1 662 | 2023-07-28 03:45:45.854 - [147] RECEIVED, request_size=495 bytes, height=328148 663 | 2023-07-28 03:45:45.855 - [147] QUEUED, queue_size=1 664 | 2023-07-28 03:45:45.921 - [146] STARTED, queue_wait_time=0.274223 secs 665 | 2023-07-28 03:45:46.201 - [147] STARTED, queue_wait_time=0.346133 secs 666 | 2023-07-28 04:00:59.592 - [146] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e5ede57b.jpeg] timed out after 913.146 secs 667 | 2023-07-28 04:00:59.737 - [147] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/632a7899.jpeg] timed out after 913.095 secs 668 | 2023-07-28 04:09:08.111 - [148] RECEIVED, request_size=494 bytes, height=328152 669 | 2023-07-28 04:09:08.113 - [148] QUEUED, queue_size=1 670 | 2023-07-28 04:09:08.476 - [148] STARTED, queue_wait_time=0.363484 secs 671 | 2023-07-28 04:16:04.592 - [148] SUCCEEDED, execution_time=416.038 secs, response_size=653761 bytes, image_hash=d1ce51e32772a4738e73ba8a37def7a7fb910c72a923595ba757e2cff043f2df 672 | 2023-07-28 05:30:48.554 - [149] RECEIVED, request_size=495 bytes, height=328171 673 | 2023-07-28 05:30:48.558 - [149] QUEUED, queue_size=1 674 | 2023-07-28 05:30:48.812 - [149] STARTED, queue_wait_time=0.254913 secs 675 | 2023-07-28 05:45:50.385 - [149] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/0984695c.jpeg] timed out after 901.068 secs 676 | 2023-07-28 06:05:48.237 - [150] RECEIVED, request_size=495 bytes, height=328175 677 | 2023-07-28 06:05:48.245 - [150] QUEUED, queue_size=1 678 | 2023-07-28 06:05:48.258 - [151] RECEIVED, request_size=494 bytes, height=328175 679 | 2023-07-28 06:05:48.258 - [151] QUEUED, queue_size=1 680 | 2023-07-28 06:05:48.551 - [150] STARTED, queue_wait_time=0.308033 secs 681 | 2023-07-28 06:05:48.777 - [151] STARTED, queue_wait_time=0.519120 secs 682 | 2023-07-28 06:05:50.920 - [152] RECEIVED, request_size=495 bytes, height=328175 683 | 2023-07-28 06:05:50.923 - [152] QUEUED, queue_size=1 684 | 2023-07-28 06:05:51.141 - [152] STARTED, queue_wait_time=0.214660 secs 685 | 2023-07-28 06:05:51.260 - [153] RECEIVED, request_size=495 bytes, height=328175 686 | 2023-07-28 06:05:51.261 - [153] QUEUED, queue_size=1 687 | 2023-07-28 06:05:51.463 - [154] RECEIVED, request_size=495 bytes, height=328175 688 | 2023-07-28 06:05:51.464 - [154] QUEUED, queue_size=1 689 | 2023-07-28 06:05:51.481 - [153] STARTED, queue_wait_time=0.220841 secs 690 | 2023-07-28 06:05:51.762 - [154] STARTED, queue_wait_time=0.297866 secs 691 | 2023-07-28 06:20:52.269 - [150] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/55f4fa23.jpeg] timed out after 901.083 secs 692 | 2023-07-28 06:20:52.942 - [151] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/45ab9b49.png] timed out after 901.109 secs 693 | 2023-07-28 06:20:53.615 - [153] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5dab9422.jpeg] timed out after 901.042 secs 694 | 2023-07-28 06:20:53.673 - [152] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/ec987385.jpeg] timed out after 901.100 secs 695 | 2023-07-28 06:20:53.855 - [154] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e4988562.jpeg] timed out after 901.040 secs 696 | 2023-07-28 06:29:06.787 - [155] RECEIVED, request_size=494 bytes, height=328181 697 | 2023-07-28 06:29:06.791 - [155] QUEUED, queue_size=1 698 | 2023-07-28 06:29:07.085 - [155] STARTED, queue_wait_time=0.294117 secs 699 | 2023-07-28 06:38:22.370 - [155] SUCCEEDED, execution_time=555.072 secs, response_size=2790116 bytes, image_hash=6442cab9a2e35f9dad5e9384a1814b2a74c4f004629392aac97ce4c8c6a97db5 700 | 2023-07-28 06:40:46.823 - [156] RECEIVED, request_size=495 bytes, height=328185 701 | 2023-07-28 06:40:46.824 - [156] QUEUED, queue_size=1 702 | 2023-07-28 06:40:46.893 - [157] RECEIVED, request_size=495 bytes, height=328185 703 | 2023-07-28 06:40:46.893 - [157] QUEUED, queue_size=1 704 | 2023-07-28 06:40:47.066 - [156] STARTED, queue_wait_time=0.242384 secs 705 | 2023-07-28 06:40:47.343 - [157] STARTED, queue_wait_time=0.449425 secs 706 | 2023-07-28 06:40:48.997 - [158] RECEIVED, request_size=494 bytes, height=328185 707 | 2023-07-28 06:40:48.997 - [158] QUEUED, queue_size=1 708 | 2023-07-28 06:40:49.198 - [158] STARTED, queue_wait_time=0.201450 secs 709 | 2023-07-28 06:56:01.307 - [156] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/25d5c522.jpeg] timed out after 913.074 secs 710 | 2023-07-28 06:56:01.432 - [157] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5d5bb602.jpeg] timed out after 913.043 secs 711 | 2023-07-28 06:56:02.903 - [158] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/f0672673.png] timed out after 913.099 secs 712 | 2023-07-28 07:15:47.070 - [159] RECEIVED, request_size=495 bytes, height=328194 713 | 2023-07-28 07:15:47.073 - [159] QUEUED, queue_size=1 714 | 2023-07-28 07:15:47.299 - [159] STARTED, queue_wait_time=0.226990 secs 715 | 2023-07-28 07:31:00.850 - [159] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/b33e2952.jpeg] timed out after 913.052 secs 716 | 2023-07-28 08:49:12.080 - [160] RECEIVED, request_size=495 bytes, height=328211 717 | 2023-07-28 08:49:12.084 - [160] QUEUED, queue_size=1 718 | 2023-07-28 08:49:12.363 - [160] STARTED, queue_wait_time=0.279720 secs 719 | 2023-07-28 08:49:13.070 - [161] RECEIVED, request_size=495 bytes, height=328211 720 | 2023-07-28 08:49:13.071 - [161] QUEUED, queue_size=1 721 | 2023-07-28 08:49:13.293 - [161] STARTED, queue_wait_time=0.222249 secs 722 | 2023-07-28 08:49:13.542 - [162] RECEIVED, request_size=495 bytes, height=328211 723 | 2023-07-28 08:49:13.542 - [162] QUEUED, queue_size=1 724 | 2023-07-28 08:49:13.762 - [162] STARTED, queue_wait_time=0.219486 secs 725 | 2023-07-28 09:04:26.572 - [160] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/693e5ca2.jpeg] timed out after 913.049 secs 726 | 2023-07-28 09:04:27.120 - [161] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/eced59a6.jpeg] timed out after 913.091 secs 727 | 2023-07-28 09:04:27.385 - [162] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/207cb08d.jpeg] timed out after 913.049 secs 728 | 2023-07-28 09:24:07.542 - [163] RECEIVED, request_size=495 bytes, height=328217 729 | 2023-07-28 09:24:07.546 - [163] QUEUED, queue_size=1 730 | 2023-07-28 09:24:07.993 - [163] STARTED, queue_wait_time=0.447675 secs 731 | 2023-07-28 09:24:12.708 - [164] RECEIVED, request_size=494 bytes, height=328217 732 | 2023-07-28 09:24:12.709 - [164] QUEUED, queue_size=1 733 | 2023-07-28 09:24:12.911 - [164] STARTED, queue_wait_time=0.202627 secs 734 | 2023-07-28 09:24:15.688 - [165] RECEIVED, request_size=495 bytes, height=328217 735 | 2023-07-28 09:24:15.689 - [165] QUEUED, queue_size=1 736 | 2023-07-28 09:24:15.903 - [165] STARTED, queue_wait_time=0.214195 secs 737 | 2023-07-28 09:39:22.136 - [163] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/7ee2d875.jpeg] timed out after 913.098 secs 738 | 2023-07-28 09:39:26.578 - [164] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e4bb4654.png] timed out after 913.061 secs 739 | 2023-07-28 09:39:29.561 - [165] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/76103524.jpeg] timed out after 913.078 secs 740 | 2023-07-28 10:10:45.222 - [166] RECEIVED, request_size=495 bytes, height=328228 741 | 2023-07-28 10:10:45.226 - [166] QUEUED, queue_size=1 742 | 2023-07-28 10:10:45.566 - [166] STARTED, queue_wait_time=0.340942 secs 743 | 2023-07-28 10:25:59.218 - [166] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/64efbc4e.jpeg] timed out after 913.050 secs 744 | 2023-07-28 10:45:52.572 - [167] RECEIVED, request_size=495 bytes, height=328234 745 | 2023-07-28 10:45:52.578 - [167] QUEUED, queue_size=1 746 | 2023-07-28 10:45:52.869 - [167] STARTED, queue_wait_time=0.291175 secs 747 | 2023-07-28 10:45:53.786 - [168] RECEIVED, request_size=495 bytes, height=328234 748 | 2023-07-28 10:45:53.787 - [168] QUEUED, queue_size=1 749 | 2023-07-28 10:45:53.994 - [168] STARTED, queue_wait_time=0.207103 secs 750 | 2023-07-28 10:45:56.661 - [169] RECEIVED, request_size=495 bytes, height=328234 751 | 2023-07-28 10:45:56.662 - [169] QUEUED, queue_size=1 752 | 2023-07-28 10:45:56.858 - [169] STARTED, queue_wait_time=0.196397 secs 753 | 2023-07-28 11:01:07.193 - [167] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/4b1807a1.jpeg] timed out after 913.041 secs 754 | 2023-07-28 11:01:07.763 - [168] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/e38afbd2.jpeg] timed out after 913.057 secs 755 | 2023-07-28 11:01:10.259 - [169] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/82e1af80.jpeg] timed out after 913.031 secs 756 | 2023-07-28 11:20:47.629 - [170] RECEIVED, request_size=495 bytes, height=328239 757 | 2023-07-28 11:20:47.632 - [170] QUEUED, queue_size=1 758 | 2023-07-28 11:20:48.012 - [170] STARTED, queue_wait_time=0.380091 secs 759 | 2023-07-28 11:20:48.441 - [171] RECEIVED, request_size=495 bytes, height=328239 760 | 2023-07-28 11:20:48.442 - [171] QUEUED, queue_size=1 761 | 2023-07-28 11:20:48.528 - [172] RECEIVED, request_size=494 bytes, height=328239 762 | 2023-07-28 11:20:48.529 - [172] QUEUED, queue_size=1 763 | 2023-07-28 11:20:48.682 - [171] STARTED, queue_wait_time=0.239784 secs 764 | 2023-07-28 11:20:48.891 - [172] STARTED, queue_wait_time=0.362902 secs 765 | 2023-07-28 11:20:49.545 - [173] RECEIVED, request_size=494 bytes, height=328239 766 | 2023-07-28 11:20:49.549 - [173] QUEUED, queue_size=1 767 | 2023-07-28 11:20:49.773 - [173] STARTED, queue_wait_time=0.224831 secs 768 | 2023-07-28 11:20:50.371 - [174] RECEIVED, request_size=495 bytes, height=328239 769 | 2023-07-28 11:20:50.371 - [174] QUEUED, queue_size=1 770 | 2023-07-28 11:20:50.612 - [174] STARTED, queue_wait_time=0.241433 secs 771 | 2023-07-28 11:20:50.826 - [175] RECEIVED, request_size=495 bytes, height=328239 772 | 2023-07-28 11:20:50.827 - [175] QUEUED, queue_size=1 773 | 2023-07-28 11:36:02.575 - [170] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/8f45fa9b.jpeg] timed out after 913.052 secs 774 | 2023-07-28 11:36:03.890 - [175] STARTED, queue_wait_time=913.062924 secs 775 | 2023-07-28 11:36:04.217 - [172] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/79d60308.png] timed out after 913.085 secs 776 | 2023-07-28 11:36:04.228 - [171] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/d1936250.jpeg] timed out after 913.059 secs 777 | 2023-07-28 11:36:04.480 - [173] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/bfd7a8e8.png] timed out after 913.066 secs 778 | 2023-07-28 11:36:04.685 - [174] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/d97ed92e.jpeg] timed out after 913.068 secs 779 | 2023-07-28 11:46:28.471 - [175] SUCCEEDED, execution_time=624.090 secs, response_size=5482247 bytes, image_hash=6940d810012d1a815379f80d7da6228948a104f423710bcd7e4050fea0286684 780 | 2023-07-28 12:42:27.238 - [176] RECEIVED, request_size=494 bytes, height=328254 781 | 2023-07-28 12:42:27.242 - [176] QUEUED, queue_size=1 782 | 2023-07-28 12:42:27.441 - [177] RECEIVED, request_size=495 bytes, height=328254 783 | 2023-07-28 12:42:27.442 - [177] QUEUED, queue_size=1 784 | 2023-07-28 12:42:27.551 - [176] STARTED, queue_wait_time=0.310139 secs 785 | 2023-07-28 12:42:27.754 - [177] STARTED, queue_wait_time=0.311970 secs 786 | 2023-07-28 12:42:28.008 - [178] RECEIVED, request_size=495 bytes, height=328254 787 | 2023-07-28 12:42:28.008 - [178] QUEUED, queue_size=1 788 | 2023-07-28 12:42:28.231 - [178] STARTED, queue_wait_time=0.223339 secs 789 | 2023-07-28 12:42:29.013 - [179] RECEIVED, request_size=495 bytes, height=328254 790 | 2023-07-28 12:42:29.015 - [179] QUEUED, queue_size=1 791 | 2023-07-28 12:42:29.266 - [179] STARTED, queue_wait_time=0.252809 secs 792 | 2023-07-28 12:57:29.658 - [177] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9d6e19f7.jpeg] timed out after 901.042 secs 793 | 2023-07-28 12:57:29.825 - [176] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9cc82b5f.png] timed out after 901.040 secs 794 | 2023-07-28 12:57:30.108 - [178] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a69dbc66.jpeg] timed out after 901.032 secs 795 | 2023-07-28 12:57:30.658 - [179] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/ea51752a.jpeg] timed out after 901.029 secs 796 | 2023-07-28 15:14:11.532 - [180] RECEIVED, request_size=495 bytes, height=328281 797 | 2023-07-28 15:14:11.541 - [180] QUEUED, queue_size=1 798 | 2023-07-28 15:14:11.789 - [180] STARTED, queue_wait_time=0.250016 secs 799 | 2023-07-28 15:14:14.037 - [181] RECEIVED, request_size=494 bytes, height=328281 800 | 2023-07-28 15:14:14.038 - [181] QUEUED, queue_size=1 801 | 2023-07-28 15:14:14.252 - [181] STARTED, queue_wait_time=0.215021 secs 802 | 2023-07-28 15:14:14.772 - [182] RECEIVED, request_size=495 bytes, height=328281 803 | 2023-07-28 15:14:14.773 - [182] QUEUED, queue_size=1 804 | 2023-07-28 15:14:14.993 - [182] STARTED, queue_wait_time=0.220779 secs 805 | 2023-07-28 15:25:48.589 - [183] RECEIVED, request_size=495 bytes, height=328284 806 | 2023-07-28 15:25:48.591 - [183] QUEUED, queue_size=1 807 | 2023-07-28 15:25:49.709 - [183] STARTED, queue_wait_time=1.117913 secs 808 | 2023-07-28 15:25:53.616 - [184] RECEIVED, request_size=495 bytes, height=328284 809 | 2023-07-28 15:25:53.621 - [184] QUEUED, queue_size=1 810 | 2023-07-28 15:25:54.480 - [184] STARTED, queue_wait_time=0.860081 secs 811 | 2023-07-28 15:29:15.392 - [180] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a9999e31.jpeg] timed out after 901.030 secs 812 | 2023-07-28 15:29:16.934 - [181] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/38718016.png] timed out after 901.038 secs 813 | 2023-07-28 15:29:16.989 - [182] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/7b09092c.jpeg] timed out after 901.045 secs 814 | 2023-07-28 15:40:51.726 - [183] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/42651e2f.jpeg] timed out after 901.064 secs 815 | 2023-07-28 15:40:56.012 - [184] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a22f4ab4.jpeg] timed out after 901.054 secs 816 | 2023-07-28 17:45:54.875 - [185] RECEIVED, request_size=495 bytes, height=328311 817 | 2023-07-28 17:45:54.879 - [185] QUEUED, queue_size=1 818 | 2023-07-28 17:45:55.121 - [185] STARTED, queue_wait_time=0.242033 secs 819 | 2023-07-28 17:45:55.311 - [186] RECEIVED, request_size=495 bytes, height=328311 820 | 2023-07-28 17:45:55.311 - [186] QUEUED, queue_size=1 821 | 2023-07-28 17:45:55.508 - [186] STARTED, queue_wait_time=0.197257 secs 822 | 2023-07-28 17:45:56.299 - [187] RECEIVED, request_size=495 bytes, height=328311 823 | 2023-07-28 17:45:56.300 - [187] QUEUED, queue_size=1 824 | 2023-07-28 17:45:56.532 - [187] STARTED, queue_wait_time=0.231791 secs 825 | 2023-07-28 17:57:29.982 - [188] RECEIVED, request_size=495 bytes, height=328312 826 | 2023-07-28 17:57:30.007 - [188] QUEUED, queue_size=1 827 | 2023-07-28 17:57:30.897 - [188] STARTED, queue_wait_time=0.890189 secs 828 | 2023-07-28 18:00:57.162 - [185] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/1a01b631.jpeg] timed out after 901.039 secs 829 | 2023-07-28 18:00:57.770 - [186] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/27554aec.jpeg] timed out after 901.038 secs 830 | 2023-07-28 18:00:58.132 - [187] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/68dbfc09.jpeg] timed out after 901.036 secs 831 | 2023-07-28 18:08:37.566 - [188] SUCCEEDED, execution_time=666.064 secs, response_size=8310477 bytes, image_hash=feefb6e0b9aebe52d72f58ecfb8d069d447d9fc74057128a5111b4d472918ff8 832 | 2023-07-28 18:32:34.462 - [189] RECEIVED, request_size=494 bytes, height=328317 833 | 2023-07-28 18:32:34.465 - [189] QUEUED, queue_size=1 834 | 2023-07-28 18:32:34.815 - [189] STARTED, queue_wait_time=0.351010 secs 835 | 2023-07-28 18:44:35.094 - [189] SUCCEEDED, execution_time=720.055 secs, response_size=1991363 bytes, image_hash=eec4ad1ef43029823af778cf28c1dd2128c181ddb293570fabb44c44c0838266 836 | 2023-07-28 20:29:10.823 - [190] RECEIVED, request_size=495 bytes, height=328352 837 | 2023-07-28 20:29:10.827 - [190] QUEUED, queue_size=1 838 | 2023-07-28 20:29:11.087 - [190] STARTED, queue_wait_time=0.260761 secs 839 | 2023-07-28 20:29:11.191 - [191] RECEIVED, request_size=495 bytes, height=328352 840 | 2023-07-28 20:29:11.192 - [191] QUEUED, queue_size=1 841 | 2023-07-28 20:29:11.464 - [192] RECEIVED, request_size=495 bytes, height=328352 842 | 2023-07-28 20:29:11.464 - [192] QUEUED, queue_size=1 843 | 2023-07-28 20:29:11.486 - [191] STARTED, queue_wait_time=0.294168 secs 844 | 2023-07-28 20:29:11.744 - [192] STARTED, queue_wait_time=0.280118 secs 845 | 2023-07-28 20:29:12.701 - [193] RECEIVED, request_size=495 bytes, height=328352 846 | 2023-07-28 20:29:12.702 - [193] QUEUED, queue_size=1 847 | 2023-07-28 20:29:12.935 - [193] STARTED, queue_wait_time=0.233374 secs 848 | 2023-07-28 20:29:13.337 - [194] RECEIVED, request_size=495 bytes, height=328352 849 | 2023-07-28 20:29:13.338 - [194] QUEUED, queue_size=1 850 | 2023-07-28 20:29:13.543 - [194] STARTED, queue_wait_time=0.205178 secs 851 | 2023-07-28 20:44:16.005 - [191] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/4dcae274.jpeg] timed out after 903.061 secs 852 | 2023-07-28 20:44:16.396 - [190] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a1b54b4a.jpeg] timed out after 903.036 secs 853 | 2023-07-28 20:44:16.697 - [192] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c6c945fd.jpeg] timed out after 903.045 secs 854 | 2023-07-28 20:44:16.913 - [193] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/be9d7d0a.jpeg] timed out after 903.038 secs 855 | 2023-07-28 20:44:17.078 - [194] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/ece7fa43.jpeg] timed out after 903.045 secs 856 | 2023-07-28 22:49:12.186 - [195] RECEIVED, request_size=495 bytes, height=328376 857 | 2023-07-28 22:49:12.191 - [195] QUEUED, queue_size=1 858 | 2023-07-28 22:49:12.465 - [195] STARTED, queue_wait_time=0.273743 secs 859 | 2023-07-28 22:49:15.356 - [196] RECEIVED, request_size=495 bytes, height=328376 860 | 2023-07-28 22:49:15.356 - [196] QUEUED, queue_size=1 861 | 2023-07-28 22:49:15.619 - [196] STARTED, queue_wait_time=0.262521 secs 862 | 2023-07-28 23:04:15.967 - [195] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/8d893369.jpeg] timed out after 903.049 secs 863 | 2023-07-28 23:04:19.078 - [196] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/a116c4be.jpeg] timed out after 903.033 secs 864 | 2023-07-28 23:12:30.793 - [197] RECEIVED, request_size=494 bytes, height=328381 865 | 2023-07-28 23:12:30.795 - [197] QUEUED, queue_size=1 866 | 2023-07-28 23:12:31.416 - [197] STARTED, queue_wait_time=0.621014 secs 867 | 2023-07-28 23:12:31.858 - [198] RECEIVED, request_size=495 bytes, height=328381 868 | 2023-07-28 23:12:31.860 - [198] QUEUED, queue_size=1 869 | 2023-07-28 23:12:32.095 - [198] STARTED, queue_wait_time=0.236474 secs 870 | 2023-07-28 23:24:11.778 - [199] RECEIVED, request_size=494 bytes, height=328383 871 | 2023-07-28 23:24:11.784 - [199] QUEUED, queue_size=1 872 | 2023-07-28 23:24:12.687 - [199] STARTED, queue_wait_time=0.903598 secs 873 | 2023-07-28 23:24:13.359 - [200] RECEIVED, request_size=495 bytes, height=328383 874 | 2023-07-28 23:24:13.360 - [200] QUEUED, queue_size=1 875 | 2023-07-28 23:24:14.064 - [200] STARTED, queue_wait_time=0.704243 secs 876 | 2023-07-28 23:24:17.418 - [201] RECEIVED, request_size=495 bytes, height=328383 877 | 2023-07-28 23:24:17.419 - [201] QUEUED, queue_size=1 878 | 2023-07-28 23:24:18.034 - [201] STARTED, queue_wait_time=0.604188 secs 879 | 2023-07-28 23:27:35.846 - [197] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/aa43b2d7.png] timed out after 903.033 secs 880 | 2023-07-28 23:27:36.259 - [198] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/8fa00b86.jpeg] timed out after 903.048 secs 881 | 2023-07-28 23:35:53.136 - [202] RECEIVED, request_size=495 bytes, height=328383 882 | 2023-07-28 23:35:53.140 - [202] QUEUED, queue_size=1 883 | 2023-07-28 23:35:53.229 - [203] RECEIVED, request_size=494 bytes, height=328383 884 | 2023-07-28 23:35:53.230 - [203] QUEUED, queue_size=1 885 | 2023-07-28 23:35:53.558 - [204] RECEIVED, request_size=494 bytes, height=328383 886 | 2023-07-28 23:35:53.559 - [204] QUEUED, queue_size=1 887 | 2023-07-28 23:35:53.779 - [202] STARTED, queue_wait_time=0.635271 secs 888 | 2023-07-28 23:35:54.784 - [203] STARTED, queue_wait_time=1.553463 secs 889 | 2023-07-28 23:39:17.815 - [199] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/ef6e3483.png] timed out after 903.068 secs 890 | 2023-07-28 23:39:18.420 - [200] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/6d8bdbb4.jpeg] timed out after 903.027 secs 891 | 2023-07-28 23:39:18.692 - [204] STARTED, queue_wait_time=205.133680 secs 892 | 2023-07-28 23:39:22.238 - [201] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/ba110ae5.jpeg] timed out after 903.062 secs 893 | 2023-07-28 23:39:55.582 - [205] RECEIVED, request_size=494 bytes, height=328383 894 | 2023-07-28 23:39:55.583 - [205] QUEUED, queue_size=1 895 | 2023-07-28 23:39:55.890 - [205] STARTED, queue_wait_time=0.307086 secs 896 | 2023-07-28 23:39:56.265 - [206] RECEIVED, request_size=495 bytes, height=328383 897 | 2023-07-28 23:39:56.266 - [206] QUEUED, queue_size=1 898 | 2023-07-28 23:39:56.576 - [206] STARTED, queue_wait_time=0.308764 secs 899 | 2023-07-28 23:50:59.197 - [202] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/0c73c6b9.jpeg] timed out after 903.059 secs 900 | 2023-07-28 23:51:00.197 - [203] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/7ca2a80f.png] timed out after 903.034 secs 901 | 2023-07-28 23:54:22.521 - [204] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/75c8ae0f.png] timed out after 903.038 secs 902 | 2023-07-28 23:54:59.625 - [205] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/477a4073.png] timed out after 903.056 secs 903 | 2023-07-28 23:55:00.248 - [206] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/050b016f.jpeg] timed out after 903.058 secs 904 | 2023-07-29 00:10:53.517 - [207] RECEIVED, request_size=495 bytes, height=328387 905 | 2023-07-29 00:10:53.519 - [207] QUEUED, queue_size=1 906 | 2023-07-29 00:10:53.771 - [207] STARTED, queue_wait_time=0.252923 secs 907 | 2023-07-29 00:10:54.894 - [208] RECEIVED, request_size=495 bytes, height=328387 908 | 2023-07-29 00:10:54.894 - [208] QUEUED, queue_size=1 909 | 2023-07-29 00:10:55.129 - [208] STARTED, queue_wait_time=0.234744 secs 910 | 2023-07-29 00:10:55.676 - [209] RECEIVED, request_size=495 bytes, height=328387 911 | 2023-07-29 00:10:55.677 - [209] QUEUED, queue_size=1 912 | 2023-07-29 00:10:55.951 - [209] STARTED, queue_wait_time=0.274283 secs 913 | 2023-07-29 00:11:03.461 - [210] RECEIVED, request_size=495 bytes, height=328387 914 | 2023-07-29 00:11:03.462 - [210] QUEUED, queue_size=1 915 | 2023-07-29 00:11:03.741 - [210] STARTED, queue_wait_time=0.279531 secs 916 | 2023-07-29 00:22:33.290 - [211] RECEIVED, request_size=494 bytes, height=328387 917 | 2023-07-29 00:22:33.295 - [211] QUEUED, queue_size=1 918 | 2023-07-29 00:22:34.286 - [211] STARTED, queue_wait_time=0.992262 secs 919 | 2023-07-29 00:22:41.726 - [212] RECEIVED, request_size=495 bytes, height=328387 920 | 2023-07-29 00:22:41.727 - [212] QUEUED, queue_size=1 921 | 2023-07-29 00:25:58.010 - [207] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/13538dfc.jpeg] timed out after 903.027 secs 922 | 2023-07-29 00:25:59.209 - [212] STARTED, queue_wait_time=197.482007 secs 923 | 2023-07-29 00:26:00.146 - [208] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/fda371fa.jpeg] timed out after 903.047 secs 924 | 2023-07-29 00:26:00.187 - [209] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/b6c32476.jpeg] timed out after 903.054 secs 925 | 2023-07-29 00:26:07.417 - [210] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/9654c5b7.jpeg] timed out after 903.043 secs 926 | 2023-07-29 00:26:42.145 - [213] RECEIVED, request_size=495 bytes, height=328387 927 | 2023-07-29 00:26:42.153 - [213] QUEUED, queue_size=1 928 | 2023-07-29 00:26:42.424 - [213] STARTED, queue_wait_time=0.274960 secs 929 | 2023-07-29 00:26:43.432 - [214] RECEIVED, request_size=495 bytes, height=328387 930 | 2023-07-29 00:26:43.433 - [214] QUEUED, queue_size=1 931 | 2023-07-29 00:26:43.687 - [214] STARTED, queue_wait_time=0.254357 secs 932 | 2023-07-29 00:34:12.597 - [215] RECEIVED, request_size=495 bytes, height=328388 933 | 2023-07-29 00:34:12.607 - [215] QUEUED, queue_size=1 934 | 2023-07-29 00:34:13.758 - [215] STARTED, queue_wait_time=1.145266 secs 935 | 2023-07-29 00:34:13.987 - [216] RECEIVED, request_size=494 bytes, height=328388 936 | 2023-07-29 00:34:13.988 - [216] QUEUED, queue_size=1 937 | 2023-07-29 00:37:40.794 - [211] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/17789efc.png] timed out after 903.041 secs 938 | 2023-07-29 00:37:41.813 - [216] STARTED, queue_wait_time=207.824180 secs 939 | 2023-07-29 00:38:14.512 - [217] RECEIVED, request_size=495 bytes, height=328388 940 | 2023-07-29 00:38:14.514 - [217] QUEUED, queue_size=1 941 | 2023-07-29 00:41:04.241 - [212] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/498f94e2.jpeg] timed out after 903.038 secs 942 | 2023-07-29 00:41:04.783 - [217] STARTED, queue_wait_time=170.267865 secs 943 | 2023-07-29 00:41:46.162 - [213] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/b1281493.jpeg] timed out after 903.056 secs 944 | 2023-07-29 00:41:47.569 - [214] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/7fbf1d33.jpeg] timed out after 903.055 secs 945 | 2023-07-29 00:49:17.560 - [215] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/c44eb073.jpeg] timed out after 903.038 secs 946 | 2023-07-29 00:52:45.515 - [216] TIMEDOUT, Dupe detection task [/home/ubuntu/.pastel/dd-server/5236fa87.png] timed out after 903.026 secs 947 | 2023-07-29 00:55:45.120 - [217] SUCCEEDED, execution_time=880.082 secs, response_size=3674848 bytes, image_hash=d12fb9fcae0634ad6f2a511a1f890d0ef302580788dc986f7e07ad49541c0ce1 948 | 2023-07-29 01:20:57.780 - [218] RECEIVED, request_size=495 bytes, height=328393 949 | 2023-07-29 01:20:57.783 - [218] QUEUED, queue_size=1 950 | 2023-07-29 01:20:57.809 - [219] RECEIVED, request_size=495 bytes, height=328393 951 | 2023-07-29 01:20:57.810 - [219] QUEUED, queue_size=1 952 | 2023-07-29 01:20:57.818 - [220] RECEIVED, request_size=495 bytes, height=328393 953 | 2023-07-29 01:20:57.819 - [220] QUEUED, queue_size=1 954 | 2023-07-29 01:20:58.057 - [218] STARTED, queue_wait_time=0.273547 secs 955 | 2023-07-29 01:20:58.271 - [219] STARTED, queue_wait_time=0.460943 secs 956 | 2023-07-29 01:20:58.501 - [220] STARTED, queue_wait_time=0.682673 secs 957 | 2023-07-29 01:32:30.954 - [221] RECEIVED, request_size=495 bytes, height=328394 958 | --------------------------------------------------------------------------------