├── burp └── incexc │ ├── image_compressed │ ├── std_settings │ ├── windows_program │ ├── audio_compressed │ ├── macos_settings │ ├── generic_exclusions │ ├── generic_excluded_extensions │ ├── video_compressed │ ├── linux_settings │ ├── README.md │ ├── generic_compressed │ └── windows_settings ├── virsh ├── vm_snapshots.sh ├── vm_stats.sh ├── mk_vm.sh ├── vm_move.sh └── offline_rename_opnsense_interfaces.sh ├── LICENSE.TXT ├── README.md ├── ddsplit.sh ├── repairbadblocks.sh ├── emailCheck.sh ├── ssh_jail.sh └── emailCheck.py /burp/incexc/image_compressed: -------------------------------------------------------------------------------- 1 | exclude_comp=gif 2 | exclude_comp=jpg 3 | exclude_comp=jpeg 4 | exclude_comp=png 5 | exclude_comp=webp 6 | exclude_comp=tif 7 | exclude_comp=tiff 8 | -------------------------------------------------------------------------------- /virsh/vm_snapshots.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | vms=$(virsh list --all --name) 4 | for vm in ${vms[@]}; do 5 | echo "VM Name: $vm" 6 | virsh snapshot-list --tree $vm 7 | done 8 | -------------------------------------------------------------------------------- /burp/incexc/std_settings: -------------------------------------------------------------------------------- 1 | . audio_compressed 2 | . image_compressed 3 | . video_compressed 4 | . generic_compressed 5 | . generic_excluded_extensions 6 | . generic_exclusions 7 | 8 | nobackup=.nobackup 9 | atime=0 10 | scan_problem_raises_error=0 11 | -------------------------------------------------------------------------------- /burp/incexc/windows_program: -------------------------------------------------------------------------------- 1 | # Exclusion list that removes most of Windows system and program files 2 | # This list is a complementary to windows_settings 3 | 4 | exclude_regex = ^[A-Z]:/Windows 5 | exclude_regex = ^[A-Z]:/Program Files 6 | exclude_regex = ^[A-Z]:/Program Files (x86) 7 | exclude_regex = ^[A-Z]:/ProgramData 8 | -------------------------------------------------------------------------------- /burp/incexc/audio_compressed: -------------------------------------------------------------------------------- 1 | exclude_comp=aac 2 | exclude_comp=ac3 3 | exclude_comp=aif 4 | exclude_comp=aiff 5 | exclude_comp=asf 6 | exclude_comp=asx 7 | exclude_comp=cdda 8 | exclude_comp=cdr 9 | exclude_comp=flac 10 | exclude_comp=m4a 11 | exclude_comp=mkf 12 | exclude_comp=mp3 13 | exclude_comp=mpg3 14 | exclude_comp=mpga 15 | exclude_comp=oga 16 | exclude_comp=ogg 17 | exclude_comp=wma 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /virsh/vm_stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | printf '%-20s %-4s %-8s\n' "Vm Name" "CPU(s)" "RAM (MiB)" 4 | 5 | mapfile -t vm_array < <( virsh list --all --name ) 6 | 7 | while read -r vm 8 | do 9 | if [ ! -z "$vm" ]; then 10 | MEM_USED=$(($(virsh dominfo $vm | grep "Max memory" | cut -f 7 -d " ") / 1024)) 11 | CPU_USED=$(virsh dominfo $vm | grep "CPU(s)" | cut -f 10 -d " ") 12 | printf "%-40s %4s | %8s\n" "$vm" "$CPU_USED" "$MEM_USED" 13 | MEM_SUM=$((MEM_SUM + MEM_USED)) 14 | CPU_SUM=$((CPU_SUM + CPU_USED)) 15 | fi 16 | done < <( printf '%s\n' "${vm_array[@]}") 17 | 18 | printf -- '-%.0s' {1..36} 19 | printf '\n%-40s %4s | %8s\n' "Totals:" "$CPU_SUM" "$MEM_SUM" 20 | -------------------------------------------------------------------------------- /burp/incexc/macos_settings: -------------------------------------------------------------------------------- 1 | # Generic exclusions 2 | exclude_regex = /dev 3 | exclude_regex = /Network 4 | exclude_regex = /tmp 5 | exclude_regex = /cores 6 | exclude_regex = /afs 7 | exclude_regex = /automount 8 | exclude_regex = /private/Network 9 | exclude_regex = /private/tmp 10 | exclude_regex = /private/var/tmp 11 | exclude_regex = /private/var/folders 12 | exclude_regex = /private/var/run 13 | exclude_regex = /private/var/spool/postfix 14 | exclude_regex = /private/var/automount 15 | exclude_regex = /private/var/db/fseventsd 16 | exclude_regex = /Previous Systems 17 | exclude_regex = .*/lost+found 18 | 19 | exclude_regex = .*\.Trash.* 20 | exclude_regex = /\.vol/.* 21 | exclude_regex = .*/Network Trash Folder 22 | exclude_regex = .*\.fseventsd.* 23 | exclude_regex = *\.Spotlight-.* 24 | exclude_regex = .*Mobile.*Backups/.* 25 | 26 | -------------------------------------------------------------------------------- /burp/incexc/generic_exclusions: -------------------------------------------------------------------------------- 1 | # Microsoft Office lock files (eg */~$somefile.doc) 2 | exclude_regex = .*/~\$[^/]+\.[^/]+ 3 | 4 | # MacOS files 5 | exclude_regex = \.DS_Store 6 | 7 | # odrive sync dir 8 | exclude_regex = \.odrive 9 | 10 | # Generic directories / files 11 | exclude_regex = .*/[Cc]ache?[^/]+\.db 12 | exclude_regex = .*/TemporaryFiles 13 | 14 | # Generic Thumbs files 15 | exclude_regex = .*/([Tt]humb|[Ii]con)s?([Cc]ache)?.*\.db 16 | 17 | # Generic Thumbs directories 18 | exclude_regex = .*/\.?[Tt]humbnails 19 | 20 | # Generic tmp dir 21 | exclude_regex = .*/[Tt]e?mp$ 22 | exclude_regex = .*/[Tt]emporary$ 23 | 24 | # Generic cache dir 25 | exclude_regex = .*/([Ff]ont|[Ff]ile|[Dd]ist|[Nn]ative|[Pp]lay|[Pp]lugin|[Aa]sset|[Aa]ctivities|[Ss]cript|[Gg]pu|[Cc]ode|[Ll]ocal|[Ss]ession|[Ww]eb|JS|CRL)? ?[Cc]ache$ 26 | exclude_regex = .*/[Cc]ache[Ss]torage$ 27 | 28 | # Synology NAS working directory 29 | exclude_regex = .*/.SynologyWorkingDirectory 30 | -------------------------------------------------------------------------------- /burp/incexc/generic_excluded_extensions: -------------------------------------------------------------------------------- 1 | 2 | # Generic excluded extensions 3 | 4 | exclude_ext=back 5 | exclude_ext=bak 6 | exclude_ext=bkp 7 | exclude_ext=cache 8 | exclude_ext=chk 9 | exclude_ext=dmp 10 | exclude_ext=dump 11 | exclude_ext=err 12 | exclude_ext=lock 13 | exclude_ext=lockfile 14 | exclude_ext=log 15 | exclude_ext=log1 16 | exclude_ext=log2 17 | exclude_ext=old 18 | exclude_ext=tmp 19 | exclude_ext=temp 20 | 21 | # Browser not finished downloads 22 | exclude_ext=download 23 | exclude_ext=crdownload 24 | exclude_ext=part 25 | 26 | # Adobe lightroom preview files 27 | exclude_ext=lrprev 28 | 29 | # AutoCAD 30 | exclude_ext=dwl 31 | exclude_ext=dwl2 32 | exclude_ext=atmp 33 | 34 | # Microsoft Access lock file 35 | exclude_ext=laccdb 36 | exclude_ext=swp 37 | 38 | # Microsoft Outlook Exchange sync files 39 | #exclude_ext=ost 40 | 41 | # Microsoft Tracelog files 42 | exclude_ext=etl 43 | 44 | # Python compiled files 45 | exclude_ext=py[cod] 46 | 47 | -------------------------------------------------------------------------------- /burp/incexc/video_compressed: -------------------------------------------------------------------------------- 1 | exclude_comp=264 2 | exclude_comp=3fr 3 | exclude_comp=3g2 4 | exclude_comp=3gp 5 | exclude_comp=3gpp 6 | exclude_comp=asf 7 | exclude_comp=avi 8 | exclude_comp=avchd 9 | exclude_comp=divx 10 | exclude_comp=dv4 11 | exclude_comp=f4v 12 | exclude_comp=flv 13 | exclude_comp=mp4 14 | exclude_comp=mkv 15 | exclude_comp=h263 16 | exclude_comp=h264 17 | exclude_comp=h265 18 | exclude_comp=m4v 19 | exclude_comp=mjpeg 20 | exclude_comp=mkv 21 | exclude_comp=mov 22 | exclude_comp=movie 23 | exclude_comp=mpgv 24 | exclude_comp=mp4 25 | exclude_comp=mpeg 26 | exclude_comp=mpeg2 27 | exclude_comp=mpeg3 28 | exclude_comp=mpeg4 29 | exclude_comp=mpeg 30 | exclude_comp=mpg 31 | exclude_comp=mpv 32 | exclude_comp=ogv 33 | exclude_comp=qt 34 | exclude_comp=qtw 35 | exclude_comp=ram 36 | exclude_comp=rm 37 | exclude_comp=rmi 38 | exclude_comp=rmvb 39 | exclude_comp=swf 40 | exclude_comp=vob 41 | exclude_comp=webm 42 | exclude_comp=wmv 43 | exclude_comp=wmv3 44 | exclude_comp=video 45 | exclude_comp=xvid 46 | -------------------------------------------------------------------------------- /burp/incexc/linux_settings: -------------------------------------------------------------------------------- 1 | # Standard linux system paths 2 | exclude_regex = ^/dev 3 | exclude_regex = .*/lost\+found 4 | exclude_regex = ^/media 5 | exclude_regex = ^/mnt 6 | exclude_regex = ^/proc 7 | exclude_regex = ^/run 8 | exclude_regex = ^/selinux 9 | exclude_regex = ^/sys 10 | exclude_regex = ^/tmp 11 | exclude_regex = ^/var/cache 12 | exclude_regex = ^/var/log 13 | exclude_regex = ^/var/run 14 | exclude_regex = ^/var/tmp 15 | 16 | exclude_fs=tmpfs 17 | 18 | # Linux cache files 19 | exclude_regex = \.cache 20 | exclude_regex = /\.cache/ 21 | 22 | # Generic home exclusions 23 | exclude_regex = ^/home/[^/]+/.debug 24 | exclude_regex = ^/home/[^/]+/.dbus 25 | exclude_regex = ^/home/[^/]+/.gvfs 26 | exclude_regex = ^/home/[^/]+/.local/share/gvfs-metadata 27 | exclude_regex = ^/home/[^/]+/.local/share/Trash 28 | exclude_regex = ^/home/[^/]+/.recently-used 29 | exclude_regex = ^/home/[^/]+/.thumbnails 30 | exclude_regex = ^/home/[^/]+/.xession-errors 31 | exclude_regex = ^/home/[^/]+/.Trash 32 | 33 | # Dropbox, OneDrive, SkyDrive data directories (not excluded by default because of cryptolockers attacks) 34 | #exclude_regex = ^/home/[^/]+/SkyDrive[^/]+ 35 | #exclude_regex = ^/home/[^/]+/Dropbox 36 | #exclude_regex = ^/home/[^/]+/OneDrive 37 | -------------------------------------------------------------------------------- /burp/incexc/README.md: -------------------------------------------------------------------------------- 1 | These are my personal burp incexc rules which: 2 | 3 | - Exclude already compressed files from burp zlib compression 4 | - Exclude a lot of temporary / lock / unuseful file extensions / Outlook offline cache from Exchange 5 | - Exclude some cloud Program cache like SkyDrive / DropBox and others 6 | - Exclude loads of Windows temp/cache/system files 7 | - Exclude browser caches 8 | - Exclude unnecessary Linux paths 9 | - Set standard settings (which you may have to modify to fit your needs) 10 | 11 | The regex are PCRE and validated by https://regexr.com 12 | 13 | In order to use it, simply write the following in the client config file server side: 14 | 15 | - For linux clients 16 | ``` 17 | . incexc/std_settings 18 | . incexc/linux_settings 19 | ``` 20 | 21 | - For windows clients 22 | ``` 23 | . incexc/std_settings 24 | . incexc/windows_settings 25 | ``` 26 | 27 | These rules are designed to exclude most unused system files from backup, but keep system programs and setting files (like Program Files & ProgramData or /sbin & /etc). 28 | It's better to backup too much than not enough :) 29 | 30 | All improvements are welcome. 31 | There may also exist multiple backup exclusion profiles like, whatever your needs are, eg: 32 | 33 | - windows_settings = Generic temp/cache/system path exclusions 34 | - windows_programs = Exclude most system paths (eg Windows / ProgramFiles / ProgramData) 35 | - linux_settings = Genreic path exclusions 36 | - linux_programs = Exclude /bin /sbin /usr/local/bin... 37 | 38 | Missing a MacOS guru that may write specific Mac settings file. 39 | -------------------------------------------------------------------------------- /LICENSE.TXT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2019, Orsiris de Jong. ozy@netpower.fr 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the author nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /burp/incexc/generic_compressed: -------------------------------------------------------------------------------- 1 | 2 | exclude_comp=7z 3 | exclude_comp=ace 4 | exclude_comp=apk 5 | exclude_comp=appx 6 | exclude_comp=appxbundle 7 | exclude_comp=arc 8 | exclude_comp=arj 9 | exclude_comp=b2 10 | exclude_comp=bhx 11 | exclude_comp=bz 12 | exclude_comp=bz2 13 | exclude_comp=cab 14 | exclude_comp=cpio 15 | exclude_comp=deb 16 | exclude_comp=dmg 17 | exclude_comp=jar 18 | exclude_comp=gpg 19 | exclude_comp=gz 20 | exclude_comp=gzip 21 | exclude_comp=hpk 22 | exclude_comp=hqx 23 | exclude_comp=jar 24 | exclude_comp=kgb 25 | exclude_comp=lzh 26 | exclude_comp=lha 27 | exclude_comp=lhz 28 | exclude_comp=lz 29 | exclude_comp=lzx 30 | exclude_comp=lzma 31 | exclude_comp=lzo 32 | exclude_comp=lzx 33 | exclude_comp=msi 34 | exclude_comp=pak 35 | exclude_comp=pit 36 | exclude_comp=rar 37 | exclude_comp=rpm 38 | exclude_comp=rz 39 | exclude_comp=rzip 40 | exclude_comp=s7z 41 | exclude_comp=sfark 42 | exclude_comp=sfx 43 | exclude_comp=sqz 44 | exclude_comp=tbz 45 | exclude_comp=tgz 46 | exclude_comp=txz 47 | exclude_comp=uu 48 | exclude_comp=uue 49 | exclude_comp=wdz 50 | exclude_comp=wim 51 | exclude_comp=xz 52 | exclude_comp=zip 53 | exclude_comp=zoo 54 | 55 | # Office documents are compressed too 56 | exclude_comp=accdb 57 | exclude_comp=accde 58 | exclude_comp=accdr 59 | exclude_comp=accdt 60 | exclude_comp=docm 61 | exclude_comp=docx 62 | exclude_comp=dotm 63 | exclude_comp=dotx 64 | exclude_comp=pptm 65 | exclude_comp=potm 66 | exclude_comp=potx 67 | exclude_comp=ppam 68 | exclude_comp=ppsx 69 | exclude_comp=pptx 70 | exclude_comp=sldx 71 | exclude_comp=sldm 72 | exclude_comp=thmx 73 | exclude_comp=xlsx 74 | exclude_comp=xlsm 75 | exclude_comp=xltx 76 | exclude_comp=xltm 77 | exclude_comp=xlsb 78 | exclude_comp=xlam 79 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # linuxscripts 2 | Collection of useful scripts / tools / files for Linux 3 | 4 | ## Burp incexc rules 5 | A set of regex rules and generic settings for Backup and Restore Program (BURP) from Graham Keeling (http://burp.grke.org) 6 | 7 | ## emailCheck.sh 8 | 9 | Low tech tool to cleanup mailing lists from unwanted emails before making a mass mailing. 10 | Performs various checks on a list of email adresses: 11 | 12 | 1. Converts all addresses to lowercase 13 | 2. Checks address' compliance against RC822 14 | 3. Checks address' domain for known typos and corrects them (please help improve that list) 15 | 4. Checks if email domain has MX records 16 | 5. Checks if email user or domain is test / example / spam, rendering them ambiguous 17 | 18 | Usage: 19 | ``` 20 | emailCheck.sh /path/to/email_list 21 | ``` 22 | 23 | Base script reads one email per line from input file. Script header contains instructions to read multicolumn CSV files. 24 | Warning: Using files comming from windows need prior conversion with dos2unix tool. 25 | 26 | ## ddsplit.sh 27 | 28 | Quick and dirty dd backup, useful to backup unknown file systems on fat32 / vfat devices. 29 | Performs disk backups via dd, compresses and splits into file chunks. 30 | Restores the splitted files to disk. 31 | Works especially well for unknown appliance filesystems (eg pfSense UFS which is not readable with standard Linux tools) 32 | 33 | Usage: 34 | ``` 35 | ddsplit.sh --backup /dev/sdX /mnt/myFile 1G 36 | ddsplit.sh --restore /mnt/ddsplit.1G.main.myFile.gz /dev/sdY 37 | ``` 38 | ## repairbadblocks.sh 39 | 40 | Highly experimental script to force SATA disk firmwares to reallocate a pending sector / bad sector using dd or hdparm. 41 | Use at your very own risk. Might even send your data to an unknown parallel universe where cows rule the world ! You have been warned. 42 | 43 | Usage: 44 | 45 | ``` 46 | repairbadblocks.sh /dev/sdX /tmp 47 | ``` 48 | 49 | ## ssh_jail.sh 50 | Creates a full ssh jail with basic commands like cp, mv, etc 51 | -------------------------------------------------------------------------------- /ddsplit.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROGRAM="ddsplit.sh" # Quick and dirty command to backup / restore disk / data using dd and compress & split backup files 4 | AUTHOR="(L) 2016 by Orsiris de Jong" 5 | CONTACT="http://www.netpower.fr - ozy@netpower.fr" 6 | PROGRAM_VERSION=0.1-stable 7 | PROGRAM_BUILD=2016032302 8 | 9 | # Let dd error impact the whole pipe command 10 | set -o pipefail 11 | 12 | filesPrefix="ddsplit" 13 | filesPrefixLength=${#filesPrefix} 14 | 15 | function Backup { 16 | local dd_cmd_result=0 17 | local splitNumber=0 18 | local filenameSplit=main 19 | 20 | while [ "$dd_cmd_result" == 0 ] 21 | do 22 | cmd="dd if=\"$source\" bs=$splitSize count=1 skip=$splitNumber iflag=fullblock,direct | pigz --fast > \"$FILEPATH/$filesPrefix.$splitSize.$filenameSplit.$FILENAME.gz\"" 23 | echo "$cmd" 24 | eval "$cmd" 25 | dd_cmd_result=$? 26 | splitNumber=$((splitNumber + 1)) 27 | filenameSplit=$splitNumber 28 | done 29 | } 30 | 31 | function Restore { 32 | local splitSize 33 | local splitNumber 34 | local fileToRestore 35 | local filenameSuffix 36 | 37 | if [ "${FILENAME:0:$filesPrefixLength}" != "$filesPrefix" ]; then 38 | echo "Source file does not seem to be a $PROGRAM generated file." 39 | exit 1 40 | fi 41 | 42 | # Remove prefix 43 | filenameSuffix="${FILENAME#$filesPrefix.*}" 44 | # Get splitsize 45 | splitSize="${filenameSuffix%%.*}" 46 | # Remove split size 47 | filenameSuffix="${filenameSuffix#*.}" 48 | # Get split number 49 | splitNumber="${filenameSuffix%%.*}" 50 | # Remove split number 51 | filenameSuffix="${filenameSuffix#*.}" 52 | 53 | fileToRestore="$FILEPATH/$filesPrefix.$splitSize.$splitNumber.$filenameSuffix" 54 | while [ -f "$fileToRestore" ] 55 | do 56 | if [ "$splitNumber" == "main" ]; then 57 | splitNumber=0 58 | fi 59 | cmd="pigz -dc \"$fileToRestore\" | dd of=\"$destination\" bs=$splitSize seek=$splitNumber" 60 | echo "$cmd" 61 | eval "$cmd" 62 | splitNumber=$((splitNumber + 1)) 63 | filenameSplit=$splitNumber 64 | fileToRestore="$FILEPATH/$filesPrefix.$splitSize.$filenameSplit.$filenameSuffix" 65 | done 66 | } 67 | 68 | function CutFileNames { 69 | local filename="${1}" 70 | 71 | FILENAME="${filename##*/}" 72 | FILEPATH="${filename%/*}" 73 | if [ "$FILEPATH" == "" ] || [ "$FILEPATH" == "$FILENAME" ]; then 74 | FILEPATH="." 75 | fi 76 | } 77 | 78 | function Usage { 79 | echo "$PROGRAM - Low tech script to backup / restore with dd into compressed and splitted files" 80 | echo "$AUTHOR" 81 | echo "$CONTACT" 82 | echo "" 83 | echo "ATTENTION: This program may destroy all your data if used wrong. Use at your own risk !" 84 | echo "" 85 | echo "Usage:" 86 | echo "$PROGRAM --backup [source] [destination] [splitsize]" 87 | echo " Produces files called $filesPrefix.splitsize.splitnumber.destination.gz" 88 | echo " splitsize is optional and works just like dd does (eg 1K, 1M, 1G...). Maximum is 1G (default value is 1G)." 89 | echo "$PROGRAM --restore [source] [destination]" 90 | echo " Source needs to be the first split file called $filesPrefix.splitsize.master.somename.gz" 91 | exit 128 92 | } 93 | 94 | command="$1" 95 | source="$2" 96 | destination="$3" 97 | splitSize="${4:-1G}" 98 | 99 | if ([ "$source" == "" ] || [ "$destination" == "" ]); then 100 | Usage 101 | fi 102 | 103 | if [ "$command" == "--backup" ]; then 104 | CutFileNames "$destination" 105 | Backup 106 | fi 107 | 108 | if [ "$command" == "--restore" ]; then 109 | CutFileNames "$source" 110 | Restore 111 | fi 112 | 113 | if [ "$command" == "--version" ] ||[ "$command" == "-v" ]; then 114 | Usage 115 | fi 116 | -------------------------------------------------------------------------------- /virsh/mk_vm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Machine create script 2025070401 4 | 5 | # Use standard or UEFI boot 6 | BOOT=hd 7 | # Boot a kernel or just load a standard cdrom bootfile 8 | BOOT_TYPE=kernel 9 | 10 | # OS (get with osinfo-query os) 11 | # Machine variant (get with /usr/libexec/qemu-kvm -machine help) 12 | 13 | # Example for RHEL 10 14 | OS_VARIANT=rhel10.0 15 | ISO=/public_vm3/iso/AlmaLinux-10.0-x86_64-dvd.iso 16 | KICKSTART=/root/ks.el9-10.cfg 17 | 18 | # RHEL 9 19 | #OS_VARIANT=rhel9.6 20 | #ISO=/opt/AlmaLinux-9.6-x86_64-dvd.iso 21 | #KICKSTART=/root/ks.el9-10.cfg 22 | 23 | # Debian 12 24 | #OS_VARIANT=debian12 25 | #ISO=/data/public_vm/ISO/debian-12.9.0-amd64-DVD-1.iso 26 | 27 | # Windows Server 2022 28 | #OS_VARIANT=win2k22 29 | #OS_VARIANT=win11 30 | #ISO=/data/public_vm/ISO/fr-fr_windows_server_2022_x64_dvd_9f7d1adb.iso 31 | #BOOT=uefi 32 | #BOOT_TYPE=cdrom 33 | #VIRTIO_ISO=/data/public_vm/ISO/virtio-win-0.1.271.iso 34 | #VIDEO="--video virtio --graphics vnc,listen=127.0.0.1,keymap=fr" 35 | #TPM="--tpm emulator" 36 | 37 | # Grommunio based on OpenSUSE 15.6 (needs graphical interface for console menu) 38 | #OS_VARIANT=opensuse15.5 39 | #ISO=/data/public_vm/ISO/grommunio.x86_64-latest.install.iso 40 | #BOOT_TYPE=cdrom 41 | #VIDEO="--video virtio --graphics vnc,listen=127.0.0.1,keymap=fr" 42 | 43 | # Proxmox 44 | #OS_VARIANT=debian12 45 | #ISO=/data/public_vm/ISO/proxmox-mail-gateway_8.1-1.iso 46 | #BOOT_TYPE=cdrom 47 | #VIDEO="--video virtio --graphics vnc,listen=127.0.0.1,keymap=fr" 48 | 49 | # OPNSense 50 | #OS_VARIANT=freebsd14.0 51 | #ISO=/opt/OPNsense-25.1-dvd-amd64.iso 52 | #BOOT=hd 53 | 54 | 55 | TENANT=tenant 56 | VM=haproxy01p.${TENANT}.local 57 | DISKSIZE=30G 58 | DISKPATH=/public_vm3/${TENANT} 59 | #DISKPATH=/var/lib/libvirt/images 60 | DISKFULLPATH="${DISKPATH}/${VM}-disk0.qcow2" 61 | VCPUS=1 62 | RAM=2048 63 | BRIDGE_NAME=br_${TENANT} 64 | #BRIDGE_NAME= 65 | BRIDGE_MTU=1330 66 | 67 | # IO MODE io_uring is fastest on io intesive VMs 68 | # IO MODE native with threads is fast 69 | # IO MODE native has good latency 70 | IO_MODE=,io="native" 71 | # For IO intensive machines, the followng will improve latency at the cost of slighty lower IOPS 72 | # io=threads still reduces performances overall, so io=native,iothread=x is good 73 | #IO_MODE=,io="native,driver.iothread=1,driver.queues=${VCPUS} --iothreads 1" 74 | #IO_MODE=,io="io_uring,driver.queues=${VCPUS} --iothreads 4" 75 | 76 | # Paramètres VM 77 | PRODUCT=vm_elconf 78 | VERSION=5.0 79 | MANUFACTURER=NetPerfect 80 | VENDOR=netperfect_vm 81 | 82 | #IP=192.168.21.1 83 | #NETMASK=255.255.255.0 84 | #GATEWAY=192.168.21.254 85 | #NAMESERVER=192.168.21.254 86 | 87 | NPF_TARGET=generic 88 | #NPF_USER_NAME=user 89 | #NPF_USER_PASSWORD=password 90 | #NPF_ROOT_PASSWORD=rootpassword 91 | 92 | # Host names can only contain the characters 'a-z', 'A-Z', '0-9', '-', or '.', cannot start or end with '-' 93 | NPF_HOSTNAME="${VM}" 94 | if [ "${IP}" != "" ] && [ "${NETMASK}" != "" ]; then 95 | # This is for pre script to pick up 96 | NPF_NETWORK="${IP}:${NETMASK}:${GATEWAY}:${NAMESERVER}" 97 | # This is for anaconda installer to pick up 98 | IP="ip=${IP}::${GATEWAY}:${NETMASK}:${VM}:none nameserver=${NAMESERVER}" 99 | #IP="ip=192.168.151.11::192.168.151.254:255.255.255.0:${VM}:none nameserver=192.168.151.254" 100 | fi 101 | 102 | # If no specific video is asked, consider we're using VNC 103 | if [ -z "${VIDEO}" ]; then 104 | VIDEO="--graphics none" 105 | fi 106 | 107 | BRIDGE="--network bridge=${BRIDGE_NAME},mtu.size=${BRIDGE_MTU}" 108 | #PCI_PASSTHROUGH="--host-device pci_0000_03_00_0 --network none" 109 | 110 | # 440fx machines as well as libvirt < 9.1.0 still need manual watchdog 111 | #WATCHDOG="--watchdog i6300esb,action=reset" 112 | 113 | INST="inst.text inst.lang=en_US inst.keymap=fr" 114 | 115 | ## Prepare commands 116 | if [ "$BOOT_TYPE" == "cdrom" ]; then 117 | BOOT_ARGS="--cdrom ${ISO}" 118 | if [ ${OS_VARIANT:0:3} != "win" ]; then 119 | extra_args="console=tty0 console=ttyS0,115200n8" 120 | fi 121 | else 122 | BOOT_ARGS="--location ${ISO}" 123 | extra_args="console=tty0 console=ttyS0,115200n8 ${INST} ${IP}" 124 | fi 125 | 126 | # Add virtio ISO for windows 127 | if [ ${OS_VARIANT:0:3} == "win" ]; then 128 | BOOT_ARGS="${BOOT_ARGS} --disk device=cdrom,path=${VIRTIO_ISO},bus=sata" 129 | fi 130 | 131 | if [ "${KICKSTART}" != "" ]; then 132 | extra_args="${extra_args} inst.ks=file:/$(basename ${KICKSTART}) inst.nosave=all_ks" 133 | KICKSTART_INJECT="--initrd-inject ${KICKSTART}" 134 | fi 135 | 136 | [ -n "${NPF_TARGET}" ] && extra_args="${extra_args} NPF_TARGET=${NPF_TARGET}" 137 | [ -n "${NPF_USER_NAME}" ] && extra_args="${extra_args} NPF_USER_NAME=${NPF_USER_NAME}" 138 | [ -n "${NPF_USER_PASSWORD}" ] && extra_args="${extra_args} NPF_USER_PASSWORD=${NPF_USER_PASSW0RD}" 139 | [ -n "${NPF_ROOT_PASSWORD}" ] && extra_args="${extra_args} NPF_ROOT_PASSWORD=${NPF_ROOT_PASSWORD}" 140 | [ -n "${NPF_HOSTNAME}" ] && extra_args="${extra_args} NPF_HOSTNAME=${NPF_HOSTNAME}" 141 | [ -n "${NPF_NETWORK}" ] && extra_args="${extra_args} NPF_NETWORK=${NPF_NETWORK}" 142 | 143 | ## Create tenant dir if not exit 144 | [ ! -d "$DISKPATH" ] && mkdir "$DISKPATH" && chown qemu:qemu "$DISKPATH" 145 | 146 | # -o cluster_size=64k 64k is optimal for DB environment (and is default value), should match underlying storage cluster size (recordsize on zfs) 147 | # -o lazy_refcounts: less IO (we mark image as dirty and it will be counted later). DO NOT ENABLE THIS since it may corrupt images and require a repair after a power loss 148 | # -o refcount_bits= : 16 bits as default, 64 bits is default, the more the faster, but will need more memory cache to be configured 149 | disk_cmd="qemu-img create -f qcow2 -o extended_l2=on -o preallocation=metadata -o cluster_size=64k "${DISKFULLPATH}" ${DISKSIZE}" 150 | echo $disk_cmd 151 | $disk_cmd 152 | if [ $? != 0 ]; then 153 | echo "Disk creation failed" 154 | exit 1 155 | fi 156 | 157 | if [ ${OS_VARIANT:0:3} == "win" ] || [ "$BOOT_TYPE" == "cdrom" ]; then 158 | vm_cmd='virt-install --name '${VM}' --ram '${RAM}' --vcpus '${VCPUS}' --cpu host-model --os-variant '${OS_VARIANT}' --disk path='${DISKFULLPATH}',bus=virtio,cache=none'${IO_MODE}' --channel unix,mode=bind,target_type=virtio,name=org.qemu.guest_agent.0 --sound none --boot '${BOOT}' --autostart --sysinfo smbios,bios.vendor='${VENDOR}',system.manufacturer='${MANUFACTURER}',system.version='${VERSION}',system.product='${PRODUCT}' '${BOOT_ARGS}' '${VIDEO}' '${BRIDGE}' '${TPM}' --autoconsole text' 159 | else 160 | vm_cmd='virt-install --name '${VM}' --ram '${RAM}' --vcpus '${VCPUS}' --cpu host-model --os-variant '${OS_VARIANT}' --disk path='${DISKFULLPATH}',bus=virtio,cache=none'${IO_MODE}' --channel unix,mode=bind,target_type=virtio,name=org.qemu.guest_agent.0 --sound none --boot '${BOOT}' --autostart --sysinfo smbios,bios.vendor='${VENDOR}',system.manufacturer='${MANUFACTURER}',system.version='${VERSION}',system.product='${PRODUCT}' '${BOOT_ARGS}' --extra-args "'${extra_args}'" '${KICKSTART_INJECT}' '${VIDEO}' '${BRIDGE}' '${TPM}' --autoconsole text' 161 | fi 162 | echo $vm_cmd 163 | eval "$vm_cmd" 164 | -------------------------------------------------------------------------------- /virsh/vm_move.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Quick and dirty KVM VM local storage move script 4 | # Written by Orsiris de Jong 5 | 6 | SCRIPT_BUILD=2025082301 7 | 8 | 9 | 10 | LOG_FILE="/var/log/$(basename $0).log" 11 | 12 | 13 | SCRIPT_GOOD=true 14 | 15 | # Make sure virsh output is consistent 16 | export LANG=C 17 | 18 | log() { 19 | __log_line="${1}" 20 | __log_level="${2:-INFO}" 21 | 22 | __log_line="${__log_level}: ${__log_line}" 23 | echo "${__log_line}" >> "${LOG_FILE}" 24 | echo "${__log_line}" 25 | 26 | if [ "${__log_level}" = "ERROR" ]; then 27 | SCRIPT_GOOD=false 28 | fi 29 | } 30 | 31 | 32 | move_storage() { 33 | # Filter disk images only 34 | xml_dumped=false 35 | xml_ok=false 36 | disk_pivoted=false 37 | vm_xml="${DST_DIR}/${VM_NAME}.inactive.$(date +"%Y%m%dT%H%M%S").xml" 38 | for disk in $(virsh domblklist "$VM_NAME" --details | grep "file" | grep "disk" | awk '{print $3"="$4}'); do 39 | disk_name="$(echo "${disk}" | awk -F'=' '{print $1}')" 40 | src_disk_path="$(echo "${disk}" | awk -F'=' '{print $2}')" 41 | if [ ! -f "${src_disk_path}" ]; then 42 | log "Source disk ${disk_name} not found in ${src_disk_path}" "ERROR" 43 | break 44 | fi 45 | dst_disk_path="${DST_DIR}/$(basename "${src_disk_path}")" 46 | log "Found disk ${disk_name} in ${src_disk_path}" 47 | if [ "${xml_dumped}" == false ]; then 48 | log "Exporting ${VM_NAME} to ${vm_xml}" 49 | virsh dumpxml --inactive "${VM_NAME}" > "${vm_xml}" 50 | if [ $? != 0 ]; then 51 | log "VM $VM_NAME dump failed. Not trying to migrate it" "ERROR" 52 | break 53 | else 54 | xml_dumped=true 55 | xml_ok=true 56 | fi 57 | log "Undefining $vm_name" 58 | [ "${DRY_RUN}" == true ] || virsh undefine "${VM_NAME}" --keep-nvram 59 | if [ $? != 0 ]; then 60 | log "Undefining $VM_NAME failed" "ERROR" 61 | break 62 | fi 63 | fi 64 | 65 | if [ "${src_disk_path}" == "${dst_disk_path}" ]; then 66 | log "Source and destination are identical. Won't do anything" "ERROR" 67 | continue 68 | fi 69 | 70 | if [ "${VM_IS_OFFLINE}" == true ]; then 71 | log "Cold moving disk ${disk_name} to ${dst_disk_path}" 72 | [ "${DRY_RUN}" == true ] || cp --preserve "${src_disk_path}" "${dst_disk_path}" 73 | else 74 | log "Hot moving disk ${disk_name} to ${dst_disk_path}" 75 | [ "${DRY_RUN}" == true ] || virsh blockcopy "${VM_NAME}" "${disk_name}" --dest="${dst_disk_path}" --wait --pivot --verbose 76 | fi 77 | if [ $? != 0 ]; then 78 | log "Failed to blockcopy $VM_NAME to $DST_DIR/$vm_disk" "ERROR" 79 | disk_pivoted=false 80 | break 81 | else 82 | # Check if disk image is not in use anymore 83 | lsof "${src_disk_path}" > /dev/null 2>&1 84 | if [ $? -eq 0 ]; then 85 | if [ "${DRY_RUN}" == true ]; then 86 | log "Disk ${src_disk_path} is in use by $(lsof "${src_disk_path}")" 87 | else 88 | log "Disk ${src_disk_path} is still in use by $(lsof "${src_disk_path}")" "ERROR" 89 | fi 90 | else 91 | disk_pivoted=true 92 | if [ "${DELETE_SOURCE}" == true ]; then 93 | log "Deleting source disk ${src_disk_path}" 94 | rm "${src_disk_path}" || log "Cannot delete old disk image" "ERROR" 95 | else 96 | old_disk_path="${src_disk_path}.old.$(date +"%Y%m%dT%H%M%S")" 97 | log "Renaming original file to ${old_disk_path}" 98 | mv "${src_disk_path}" "${old_disk_path}" || log "Cannot rename old disk image" "ERROR" 99 | fi 100 | fi 101 | fi 102 | log "Modifying disk path from \"${src_disk_path}\" to \"${dst_disk_path}\"" 103 | sed -i "s#${src_disk_path}#${dst_disk_path}#g" "${vm_xml}" 104 | if [ $? != 0 ]; then 105 | log "Failed to modify XML file $vm_vml" "ERROR" 106 | if [ "${disk_pivoted}" == false ]; then 107 | log "Stopping operation since disks did not pivot yet" 108 | break 109 | else 110 | log "Continuing operations, but xml file is bad" "ERROR" 111 | xml_ok=false 112 | fi 113 | fi 114 | if ! grep "${dst_disk_path}" "$vm_xml" > /dev/null 2>&1; then 115 | log "XML file check did not succeed" "ERROR" 116 | xml_ok=false 117 | fi 118 | done 119 | 120 | if [ "${xml_ok}" == true ]; then 121 | if [ "${DRY_RUN}" == true ]; then 122 | log "Would define VM ${VM_NAME} from ${vm_xml}" 123 | else 124 | log "Redefining VM ${VM_NAME} from ${vm_xml}" 125 | virsh define "$vm_xml" 126 | if [ $? != 0 ]; then 127 | log "Failed to redefine ${VM_NAME}" "ERROR" 128 | fi 129 | fi 130 | else 131 | log "XML file is not okay, cannot redefine ${VM_NAME} from ${vm_xml}" "ERROR" 132 | log "VM ${VM_NAME} is in transient state. Please repair." "ERROR" 133 | fi 134 | } 135 | 136 | # SCRIPT ARGUMENTS 137 | VM_NAME=false 138 | DST_DIR=false 139 | DRY_RUN=false 140 | DELETE_SOURCE=false 141 | 142 | function Usage { 143 | echo "$0" 144 | echo "" 145 | echo "$0 --vm=VM_NAME --dest=/path/to/destination_dir [--delete] [--dry]" 146 | echo "" 147 | echo "--delete Delete source file on succesful copy" 148 | echo "--dry Don't actually move anything" 149 | exit 128 150 | } 151 | 152 | function GetCommandlineArguments { 153 | local isFirstArgument=true 154 | 155 | if [ $# -eq 0 ] 156 | then 157 | Usage 158 | fi 159 | 160 | for i in "${@}"; do 161 | case "$i" in 162 | --dry) 163 | DRY_RUN=true 164 | ;; 165 | --delete) 166 | DELETE_SOURCE=true 167 | ;; 168 | --help|-h|--version|-v) 169 | Usage 170 | ;; 171 | --vm=*) 172 | VM_NAME="${i##*=}" 173 | ;; 174 | --dest=*) 175 | DST_DIR="${i##*=}" 176 | ;; 177 | *) 178 | if [ $isFirstArgument == false ]; then 179 | log "Unknown option '$i'" "CRITICAL" 180 | Usage 181 | fi 182 | ;; 183 | esac 184 | isFirstArgument=false 185 | done 186 | } 187 | 188 | GetCommandlineArguments "${@}" 189 | 190 | 191 | 192 | 193 | [ "${DRY_RUN}" == true ] && log "Running in DRY mode. Nothing will actually be done" "NOTICE" 194 | 195 | if [ "${VM_NAME}" == false ] || [ "${DST_DIR}" == false ]; then 196 | log "Please run $0 vm_name dest_dir [dry_run] [delete_source]" 197 | exit 1 198 | fi 199 | 200 | DST_DIR="$(realpath "${DST_DIR}")" 201 | 202 | [ ! -d "${DST_DIR}" ] && mkdir "${DST_DIR}" && chown qemu:qemu "${DST_DIR}" 203 | if [ ! -w "${DST_DIR}" ]; then 204 | log "Destination dir ${DST_DIR} is not writable" "ERROR" 205 | exit 1 206 | fi 207 | 208 | if ! virsh list --name --all | grep "^${VM_NAME}$" > /dev/null 2>&1; then 209 | log "VM ${VM_NAME} not found via virsh list" "ERROR" 210 | exit 1 211 | fi 212 | 213 | VM_IS_OFFLINE=false 214 | if [ "$(virsh domstate ${VM_NAME})" == "shut off" ]; then 215 | VM_IS_OFFLINE=true 216 | fi 217 | 218 | move_storage "${VM_NAME}" "${DST_DIR}" 219 | 220 | 221 | log "List of transient domains" 222 | virsh list --transient | grep "${VM_NAME}" 223 | virsh list --transient >> "$LOG_FILE" 2>&1 224 | 225 | log "End of line" 226 | 227 | if [ "${SCRIPT_GOOD}" == true ]; then 228 | exit 0 229 | else 230 | exit 1 231 | fi 232 | -------------------------------------------------------------------------------- /repairbadblocks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROGRAM="repairbadsectors" 4 | AUTHOR="Orsiris de Jong & Frederick Fouquet" 5 | VERSION=0.1 6 | PROGRAM_BUILD=2017041101 7 | 8 | function Usage { 9 | echo "Repair bad blocks from hard disk" 10 | echo "CAUTION: Use at your own risk, repair attempts are non destructive but a mechanic disk may totally fail under stress." 11 | echo "Always make nice backups first !" 12 | echo "" 13 | echo "This script has 3 different bad block correction functions:" 14 | echo "dd intelligent repair:" 15 | echo " Makes a list of all bad sectors with [badblock], then read the sectors to a tmp file, zero fill them, and write the data back from the tmp file." 16 | echo "" 17 | echo "hdparm repair:" 18 | echo " Makes a list of all bad sectors with [badblock], then reads them with hdparm, and tries to write them on read failure." 19 | echo "" 20 | echo "dd dumb repair:" 21 | echo " Same as dd intelligent repair, but instead of reading a list of bad sectors, proceeds for all sectors or a sector span." 22 | echo "" 23 | echo "Procedure explanation:" 24 | echo "" 25 | echo "Whenever a block of 4KB (8x512B sectors) is written to a disk, the disk firmware reallocates the sector if the sector is marked bad." 26 | echo "Warning: both dd intelligent repair and hdparm repair may not find bad sectors to repair if they are in \"current pending\" SMART status" 27 | echo "as the badblock utility won't see them." 28 | echo "dd dumb repair will find them, but will take painfully long to cover the whole disk." 29 | echo "dd dumb repair should be used to cover areas of the disk." 30 | echo "" 31 | echo "Usage:" 32 | echo "./repairbadblocks.sh /dev/sdX /tmp/directory" 33 | echo "/dev/sdX is the disk you want to repair" 34 | echo "/tmp/directory is a temporary directory containing sector data and bad blocks lists. Make sure it is a ramdrive or another disk than the one your're trying to repair." 35 | exit 1 36 | } 37 | 38 | #Todo real menu 39 | #Todo spinner 40 | 41 | function isNumeric { 42 | eval "local value=\"${1}\"" # Needed so variable variables can be processed 43 | 44 | local re="^-?[0-9]+([.][0-9]+)?$" 45 | if [[ $value =~ $re ]]; then 46 | echo 1 47 | else 48 | echo 0 49 | fi 50 | } 51 | 52 | function VerComp { 53 | if [ "$1" == "" ] || [ "$2" == "" ]; then 54 | echo "Bogus Vercomp values [$1] and [$2]." 55 | return 1 56 | fi 57 | 58 | if [[ $1 == $2 ]] 59 | then 60 | echo 0 61 | return 62 | fi 63 | 64 | local IFS=. 65 | local i ver1=($1) ver2=($2) 66 | # fill empty fields in ver1 with zeros 67 | for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) 68 | do 69 | ver1[i]=0 70 | done 71 | for ((i=0; i<${#ver1[@]}; i++)) 72 | do 73 | if [[ -z ${ver2[i]} ]] 74 | then 75 | # fill empty fields in ver2 with zeros 76 | ver2[i]=0 77 | fi 78 | if ((10#${ver1[i]} > 10#${ver2[i]})) 79 | then 80 | echo 1 81 | return 82 | fi 83 | if ((10#${ver1[i]} < 10#${ver2[i]})) 84 | then 85 | echo 2 86 | return 87 | fi 88 | done 89 | 90 | echo 0 91 | return 92 | } 93 | 94 | function readBadBlocks { 95 | local drive="${1}" 96 | local tmp_dir="${2}" 97 | 98 | echo "Reading bad blocks from $drive" 99 | badblocks -b 4096 -c 8 -sv -o "$tmp_dir/badblocks.$(basename $drive)" $drive 100 | } 101 | 102 | #Todo function readWriteNonDestructiveBadBlocks 103 | #function readWriteDestructiveBadBlocks 104 | 105 | function hdparmRepair { 106 | local drive="${1}" 107 | local tmp_dir="${2}" 108 | local lbablock="${3}" 109 | 110 | local badblockscount 111 | local counter=0 112 | local block 113 | local sector_begin 114 | local sector_end 115 | local result 116 | 117 | # Todo, what output does badblocks provide ? 118 | # Todo specific lbablock or badblocks 119 | readBadBlocks "$drive" "$tmp_dir" 120 | badblockscount=$(wc -l < "$tmp_dir/badblocks.$(basename $drive)") 121 | 122 | while read $block; do 123 | echo "Reading sector $block ($counter / $badblockscount)" 124 | sector_begin=$((block*8)) 125 | sector_end=$((sector_begin+7)) 126 | for sector in $(seq $sector_begin $sector_end); do 127 | hdparm --read-sector $sector $drive > /dev/null 128 | result=$? 129 | if [ $result -eq 0 ]; then 130 | echo "Sector [$sector] seems okay." 131 | elif [ $result -eq 5 ]; then 132 | echo "Sector [$sector] seems bad. Trying to rewrite it with zeros." 133 | hdparm --write-sector $sector --yes-i-know-what-i-am-doing $drive 134 | elif [ $result -eq 19 ]; then 135 | echo "Missing disk [$drive]." 136 | exit 2 137 | elif [ $result -eq 25 ]; then 138 | echo "Guru meditation failure with guru code [$result]." 139 | exit 3 140 | fi 141 | done 142 | counter=$((counter+1)) 143 | done < "$tmp_dir/badblocks.$(basename $drive)" 144 | echo "Finished repairs. Please do a smart long test on drive [$drive]." 145 | exit 0 146 | } 147 | 148 | function ddIntelligentRepair { 149 | local drive="${1}" 150 | local tmp_dir="${2}" 151 | local lbablock="${3}" 152 | 153 | local badblockscount 154 | # TODO lbablock or readBadBlocks 155 | readBadBlocks "$drive" "$tmp_dir" 156 | badblockscount=$(wc -l < "$tmp_dir/badblocks.$(basename $drive)") 157 | 158 | while read $block; do 159 | echo "Trying to repair block [$block] ($counter / $badblockscount)." 160 | sector=$((block*8)) 161 | dd if=$drive iflag=direct of="$tmp_dir/badblock.$sector.$(basename $drive)" bs=4096 count=1 skip=$sector > /dev/null 162 | dd if=/dev/zero of=$drive oflag=direct bs=4096 count=1 skip=$sector > /dev/null 163 | dd if="$tmp_dir/badblock.$block.$(basename $drive)" of=$drive oflag=direct bs=4096 count=1 skip=$sector > /dev/null 164 | if [ $? == 0 ]; then 165 | rm -f "$tmp_dir/badblock.$block.$(basename $drive)" 166 | else 167 | echo "Failed to dd write block [$block]." 168 | fi 169 | counter=$((counter+1)) 170 | done < "$tmp_dir/badblocks.$(basename $drive)" 171 | echo "Finished repairs. Please do a smart long test on drive [$drive]." 172 | exit 0 173 | } 174 | 175 | 176 | function ddDumbRepair { 177 | local drive="${1}" 178 | local tmp_dir="${2}" 179 | 180 | local block=0 181 | local read_result=0 182 | local continue=true 183 | 184 | local begin_block=0 185 | local end_block=0 186 | 187 | read -r -p "Beginning block number (0) ? " begin_block 188 | read -r -p "Ending block number (end of disk) ?" end_block 189 | 190 | if [ $(isNumeric "$begin_block") -eq 1 ]; then 191 | block=$begin_block 192 | fi 193 | 194 | if [ $(isNumeric "$end_block") -eq 0 ]; then 195 | end_block=0 196 | fi 197 | 198 | while [ $continue == true ]; do 199 | dd if=$drive iflag=direct of="$tmp_dir/badblock.tmp.$(basename $drive)" bs=4096 count=1 skip=$block > /dev/null 2>&1 200 | read_result=$? 201 | dd if=/dev/zero of=$drive oflag=direct bs=4096 count=1 skip=$block > /dev/null 2>&1 202 | dd if="$tmp_dir/badblock.tmp.$(basename $drive)" of=$drive oflag=direct bs=4096 count=1 skip=$block > /dev/null 2>&1 203 | block=$((block+1)) 204 | if [ $((block % 1000)) -eq 0 ]; then 205 | echo "Processed [$block] blocks." 206 | fi 207 | if [ $end_block -ne 0 ] && [ $block -gt $end_block ]; then 208 | continue=false 209 | elif [ $end_block -eq 0 ] && [ $read_result -ne 0 ]; then 210 | continue=false 211 | fi 212 | done 213 | exit 0 214 | } 215 | 216 | function confirmation { 217 | read -r -p "Are you sure to proceed (yes/NO) ?" ack 218 | if [ "$ack" == "yes" ] || [ "$ack" == "YES" ]; then 219 | return 1 220 | else 221 | return 0 222 | fi 223 | } 224 | 225 | function checkEnvironnment { 226 | 227 | if type dd > /dev/null 2>&1; then 228 | DD_PRESENT=true 229 | else 230 | echo "[dd] not found, will not provide dd repair options." 231 | DD_PRESENT=false 232 | fi 233 | 234 | if type badblocks > /dev/null 2>&1; then 235 | BADBLOCKS_PRESENT=true 236 | else 237 | echo "[badblocks] not found, repair time will be *much* longer." 238 | BADBLOCKS_PRESENT=false 239 | fi 240 | 241 | if type hdparm > /dev/null 2>&1; then 242 | hdparm_ver=$(hdparm -V | cut -f2 -d'v') 243 | if [ $(VerComp "$hdparm_ver" "8.0") -lt 2 ]; then 244 | HDPARM_PRESENT=true 245 | else 246 | echo "[hdparm] needs to be >= v8.0 to support repairs. Will not provide hdparm repair option." 247 | HDPARM_PRESENT=false 248 | fi 249 | else 250 | echo "[hdparm] not found, will not provide hdparm repair option." 251 | fi 252 | 253 | if ([ $DD_PRESENT == false ] && ([ $BADBLOCKS_PRESENT == false ] || [ $HDPARM_PRESENT == false ])); then 254 | echo "No required repair tools found. Cannot continue." 255 | exit 1 256 | fi 257 | } 258 | 259 | if [ "$1" != "" ] && [ "$2" != "" ]; then 260 | if [ ! -e "$1" ] || [ ! -w "$2" ]; then 261 | Usage 262 | else 263 | DRIVE="$1" 264 | TMP_DIR="$2" 265 | fi 266 | else 267 | Usage 268 | fi 269 | 270 | checkEnvironnment 271 | 272 | if ([ $DD_PRESENT == true ] && [ $BADBLOCKS_PRESENT == true ]); then 273 | echo "Launch intelligent dd repair" 274 | confirmation 275 | if [ $? -eq 1 ]; then 276 | ddIntelligentRepair "$DRIVE" "$TMP_DIR" 277 | fi 278 | fi 279 | 280 | if ([ $HDPARM_PRESENT == true ] && [ $BADBLOCKS_PRESENT == true ]); then 281 | echo "Launch hdparm repair" 282 | confirmation 283 | if [ $? -eq 1 ]; then 284 | hdparmRepair "$DRIVE" "$TMP_DIR" 285 | fi 286 | fi 287 | 288 | if [ $DD_PRESENT == true ]; then 289 | echo "Launch dumb dd repair (painfully long)" 290 | confirmation 291 | if [ $? -eq 1 ]; then 292 | ddDumbRepair "$DRIVE" "$TMP_DIR" 293 | fi 294 | fi 295 | 296 | echo "No option selected." 297 | exit 0 298 | 299 | -------------------------------------------------------------------------------- /virsh/offline_rename_opnsense_interfaces.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This is a script to modify interfaces inside a qcow2 image of OPNsense 4 | # Basically, when restored to another KVM hypervisor, we must update interface names if network card driver names change 5 | # The key here is the sed line which updates known interface names with new ones 6 | # This has been tested on a lagg interface with two members 7 | # Written by NetInvent SAS in 2025 8 | 9 | SCRIPT_BUILD=2025091601 10 | 11 | OPNSENSE_VM="opnsense01p.domain.local" 12 | VM_XML="/opt/cube/opnsense01p.domain.local.xml" 13 | 14 | # Path to opnsense config.xml file in OPNSense filesystem root 15 | CONF_FILE="/conf/config.xml" 16 | # OPNSense zpool name 17 | ZPOOL="zroot" 18 | # OPNsense zfs dataset mapped to / 19 | ZROOT="zroot/ROOT/default" 20 | # Mountpoint to local filesystem 21 | MOUNT_POINT="/tmp/opnsense_zroot" 22 | # Device name where to mount qcow2 image 23 | DEV=/dev/nbd0 24 | # Path to qcow2 image 25 | DISK_IMAGE="/data/opnsense01p.domain.local-disk0.qcow2" 26 | # Current interface names (can be a space separated list of interfaces 27 | INTERFACES_TO_REPLACE=(ixl[0-9],ixl[0-9]) 28 | INTERFACES_REPLACEMENT=(igb0,igb1,igb2,igb3,igb4) 29 | 30 | # Aribtrary sleep time in seconds for nbd & zpool operations to succeed 31 | SLEEP_TIME=5 32 | 33 | # timeout before forcing VM to go off (seconds) 34 | VM_SHUTDOWN_TIMEOUT=300 35 | 36 | LOG_FILE=/var/log/"$(basename "$0.log")" 37 | 38 | # Make sure virsh output is english 39 | export LANG=C 40 | 41 | # script variables 42 | POST_INSTALL_SCRIPT_GOOD=true 43 | VM_RUNNING=true 44 | 45 | ZFS_BINARY=$(type -p zfs) 46 | ZPOOL_BINARY=$(type -p zpool) 47 | VIRSH_BINARY=$(type -p virsh) 48 | QEMU_NBD_BINARY=$(type -p qemu-nbd) 49 | 50 | 51 | log() { 52 | __log_line="${1}" 53 | __log_level="${2:-INFO}" 54 | 55 | __log_line="$(date):${__log_level}: ${__log_line}" 56 | echo "${__log_line}" >> "${LOG_FILE}" 57 | echo "${__log_line}" 58 | 59 | if [ "${__log_level}" = "ERROR" ]; then 60 | POST_INSTALL_SCRIPT_GOOD=false 61 | fi 62 | } 63 | 64 | log_quit() { 65 | log "${1}" "${2}" 66 | log "Exiting script" 67 | exit 1 68 | } 69 | 70 | CleanUp() { 71 | # Check if zroot is currently mounted 72 | if ${ZPOOL_BINARY} list "${ZPOOL}"; then 73 | while true; do 74 | if mount | grep "${ZROOT}" > /dev/null 2>&1; then 75 | log "Unmounting dataset ${ZROOT}" 76 | ${ZFS_BINARY} umount "${ZROOT}" 2>> "${LOG_FILE}" || log "Cannot unmount ${ZROOT}" "ERROR" 77 | else 78 | log "zfs dataset ${ZROOT} is now unmounted" 79 | break 80 | fi 81 | done 82 | sleep ${SLEEP_TIME} 83 | # very important, we have to make sure zroot is unmounted before setting mountpoint back to / in order to avoid 84 | # overwriting current OS 85 | log "Setting zpool ${ZROOT} mountpoint back to /" 86 | ${ZFS_BINARY} set mountpoint=/ ${ZROOT} 2>> "${LOG_FILE}" || log "Cannot set / as mountpoint on ${ZROOT}" "ERROR" 87 | log "Exporting zpool ${ZPOOL}" 88 | ${ZPOOL_BINARY} export "${ZPOOL}" 2>> "${LOG_FILE}" || log "Cannot export ${ZPOOL}" "ERROR" 89 | while true; do 90 | if ${ZPOOL_BINARY} status | grep "${ZROOT}" > /dev/null 2>&1; then 91 | log "Waiting for zpool export" 92 | else 93 | log "zpool ${ZROOT} is now exported" 94 | break 95 | fi 96 | sleep ${SLEEP_TIME} 97 | done 98 | else 99 | log "zpool ${ZPOOL} is not imported" 100 | fi 101 | 102 | log "Current zpool status" 103 | ${ZPOOL_BINARY} status 104 | ${ZPOOL_BINARY} status >> "${LOG_FILE}" 2>&1 105 | 106 | if cmp -n 512 /dev/nbd0 /dev/zero 2>&1 | grep EOF > /dev/null 2>&1; then 107 | log "NBD device ${DEV} already disconnected" 108 | else 109 | log "Disconnecting NBD device ${DEV}" 110 | ${QEMU_NBD_BINARY} --disconnect "${DEV}" 2>> "${LOG_FILE}" || log "Cannot disconnect ${DEV}" "ERROR" 111 | if [ "${VM_RUNNING}" == true ]; then 112 | log "Restarting vm ${OPNSENSE_VM} since it was running" 113 | ${VIRSH_BINARY} start "${OPNSENSE_VM}" 2>> "${LOG_FILE}" || log "Cannot start ${OPNSENSE_VM}" "ERROR" 114 | fi 115 | fi 116 | 117 | log "End of CleanUp" 118 | } 119 | 120 | TrapQuit() { 121 | local exitcode=0 122 | 123 | CleanUp 124 | 125 | if [ "${POST_INSTALL_SCRIPT_GOOD}" == true ]; then 126 | exitcode=0 127 | else 128 | exitcode=1 129 | fi 130 | exit $exitcode 131 | } 132 | 133 | DestroyVirshSnaphots() { 134 | vm="${1}" 135 | 136 | log "Remove all snapshots from ${vm}" 137 | for snapshot in $(virsh snapshot-list --name "${vm}"); do 138 | virsh snapshot-delete --snapshotname "$snapshot" "${vm}" 2>> "${LOG_FILE}" 139 | if [ $? -ne 0 ]; then 140 | log "Cannot remove snapshot $snapshot from VM ${vm}, will try metadata-only" "ERROR" 141 | virsh snapshot-delete --snapshotname "$snapshot" "${vm}" --metadata 2>> "${LOG_FILE}" 142 | if [ $? -ne 0 ]; then 143 | log "Cannot remove metadata snapshot $snapshot from VM ${vm}" "ERROR" 144 | fi 145 | fi 146 | done 147 | } 148 | 149 | ExportVMxml() { 150 | vm="${1}" 151 | 152 | log "Exporting current VM xml file" 153 | virsh dumpxml --security-info "$vm" > "${VM_XML}" || log "Failed to dump XML for $vm in ${VM_XML}" "ERROR" 154 | } 155 | 156 | log "Starting EL Goat script at $(date)" 157 | [ -z "${BASH_VERSION}" ] && log_quit "This script must be run with bash" "ERROR" 158 | 159 | # Shutdown current opnsense VM if running 160 | if [ "$(${VIRSH_BINARY} domstate "${OPNSENSE_VM}")" == "running" ]; then 161 | log "Machine ${OPNSENSE_VM} running. We need to shut it down for modifications" 162 | ${VIRSH_BINARY} shutdown "${OPNSENSE_VM}" 2>> "${LOG_FILE}" || log "Cannot shutdown ${OPNSENSE_VM}" "ERROR" 163 | timer=0 164 | while [ "$(${VIRSH_BINARY} domstate "${OPNSENSE_VM}")" == "running" ]; do 165 | echo -n . 166 | sleep 1 167 | timer=$((timer+1)) 168 | if [ "${timer}" -gt ${VM_SHUTDOWN_TIMEOUT} ]; then 169 | log "Forcing stop of ${OPNSENSE_VM} after ${timer} seconds" 170 | ${VIRSH_BINARY} destroy "${OPNSENSE_VM}" 2>> "${LOG_FILE}" || log_quit "Cannot destroy ${OPNSENSE_VM}" "ERROR" 171 | else 172 | log "Machine ${OPNSENSE_VM} stopped after ${timer} seconds" 173 | fi 174 | done 175 | else 176 | VM_RUNNING=false 177 | log "Machine ${OPNSENSE_VM} not running, proceeding" 178 | fi 179 | 180 | # First TrapQuit declaration before knowing if we run as daemon or not 181 | trap TrapQuit TERM EXIT HUP QUIT 182 | 183 | # Load qemu-nbd module 184 | if lsmod | grep "^nbd" > /dev/null 2>&1; then 185 | log "nbd kernel module is loaded" 186 | else 187 | log "Trying to load nbd kernel module" 188 | modprobe nbd 2>> "${LOG_FILE}" || log "Cannot load module nbd" "ERROR" 189 | fi 190 | 191 | # Mount opnsense disk only if /dev/nbd0 is empty 192 | if cmp -n 512 /dev/nbd0 /dev/zero 2>&1 | grep EOF > /dev/null 2>&1; then 193 | log "Attaching NDB device ${DEV}" 194 | ${QEMU_NBD_BINARY} --connect "${DEV}" "${DISK_IMAGE}" 2>> "${LOG_FILE}" || log_quit "Cannot mount disk_image ${DISK_IMAGE}" "ERROR" 195 | # Arbitrary sleep time 196 | sleep ${SLEEP_TIME} 197 | else 198 | log "NBD device ${DEV} is already attached" 199 | fi 200 | 201 | if zpool list ${ZPOOL} > /dev/null 2>&1; then 202 | log "zpool ${ZPOOL} is already imported" 203 | else 204 | # Import zpool 205 | log "Import zpool ${ZPOOL}" 206 | ${ZPOOL_BINARY} import -N ${ZPOOL} 2>> "${LOG_FILE}" || log_quit "Cannot import ${ZPOOL}" "ERROR" 207 | fi 208 | 209 | # Create mountpoint if not existing 210 | if [ ! -d "${MOUNT_POINT}" ]; then 211 | log "Creating local mountpoint ${MOUNT_POINT}" 212 | mkdir -p "${MOUNT_POINT}" 2>> "${LOG_FILE}" || log_quit "Cannot create local mountpoint ${MOUNT_POINT}" "ERROR" 213 | fi 214 | 215 | log "Changing zpool mountpoint to ${MOUNT_POINT}" 216 | ${ZFS_BINARY} set mountpoint=${MOUNT_POINT} ${ZROOT} 2>> "${LOG_FILE}" || log_quit "Cannot change mountpoint ${MOUNT_POINT} in ${ZROOT}" "ERROR" 217 | 218 | if mount | grep ${ZROOT} > /dev/null 2>&1; then 219 | log "zpool ${ZROOT} is already mounted" 220 | else 221 | log "Mounting zpool ${ZROOT}" 222 | ${ZFS_BINARY} mount ${ZROOT} 2>> "${LOG_FILE}" || log_quit "Cannot mount ${ZROOT}" "ERROR" 223 | fi 224 | 225 | # Actual interace name replacing 226 | index=0 227 | for interface in "${INTERFACES_TO_REPLACE[@]}"; do 228 | log "Replacing ${interface} with ${INTERFACES_REPLACEMENT[$index]}" "INFO" 229 | sed -i 's#'${interface}'#'${INTERFACES_REPLACEMENT[$index]}'#g' "${MOUNT_POINT}${CONF_FILE}" 2>> "${LOG_FILE}" 230 | if [ $? != 0 ]; then 231 | log_quit "Could not modify members of our mighty lagg0" "ERROR" 232 | else 233 | log "Yihaa. lagg0 now uses igb interfaces" 234 | fi 235 | index=$((index+1)) 236 | done 237 | 238 | log "Changing machine type" 239 | # Optional changes to machine (here, we update from Almalinux 9 to AlmaLinux 10 compat) 240 | sed -i "s#hvm#hvm#g" "${VM_XML}" 2>> "${LOG_FILE}" || log_quit "Cannot update VM type in ${VM_XML}" "ERROR" 241 | 242 | # Remove all snapshots from VM before being able to unregister it 243 | DestroyVirshSnaphots "${OPNSENSE_VM}" 244 | ExportVMxml "${OPNSENSE_VM}" 245 | 246 | if virsh list --all --name | grep "${OPNSENSE_VM}" > /dev/null 2>&1; then 247 | log "Machine ${OPNSENSE_VM} is not defined" 248 | else 249 | log "Undefining ${OPNSENSE_VM}" 250 | ${VIRSH_BINARY} undefine "${OPNSENSE_VM}" 2>> "${LOG_FILE}" || log "Cannot undefine ${OPNSENSE_VM}" "ERROR" 251 | fi 252 | 253 | log "Defining ${OPNSENSE_VM}" 254 | ${VIRSH_BINARY} define "${VM_XML}" 2>> "${LOG_FILE}" || log_quit "Cannot define VM from ${VM_XML}" "ERROR" 255 | 256 | # Once we're done, Trapquit is automatically launched which will launch CleanUp() 257 | -------------------------------------------------------------------------------- /emailCheck.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROGRAM="emailCheck.sh" 4 | AUTHOR="(L) 2014-2025 by Orsiris de Jong" 5 | CONTACT="http://www.netpower.fr/ - ozy@netpower.fr" 6 | PROGRAM_VERSION=0.7.0 7 | PROGRAM_BUILD=2025120501 8 | 9 | ## Email correction script 10 | ## Checks if there are any known domain typos 11 | ## Lowers all characters of email 12 | ## Checks if email format is valid againts RFC822 13 | ## Checks if email domain has valid MX records 14 | ## Checks if email address is ambiguous 15 | 16 | ## Warning: when processing files command windows, use dos2unix first to convert carriage return chars 17 | 18 | 19 | ###################################################################################################### Input file format options 20 | 21 | INPUT_FILE_HAS_HEADER=true 22 | 23 | ## Example: File with only one column per line being an email address 24 | #CSV_EMAIL_IS_FIRST_COLUMN=true 25 | #CSV_INPUT_DELIMITER=$IFS 26 | #CSV_READ='email' 27 | #CSV_WRITE='email' 28 | 29 | ## Example: CSV file with three columns where email is in second column, where output CSV is comma instead of semicolon 30 | #CSV_EMAIL_IS_FIRST_COLUMN=false 31 | #CSV_INPUT_DELIMITER=';' 32 | #CSV_READ='col1 email col3' 33 | #CSV_WRITE='$col1,$email,$col3' 34 | 35 | CSV_EMAIL_IS_FIRST_COLUMN=true 36 | CSV_INPUT_DELIMITER=$IFS # could be ";" o "," 37 | CSV_READ='email' 38 | CSV_WRITE='$email' 39 | 40 | ###################################################################################################### Input file format options 41 | 42 | # Check if internet is working by sending a ping to the following address 43 | inet_addr_to_test=www.google.com 44 | 45 | # Filename prefixes 46 | TMP_PREFIX="tmp" 47 | VALID_PREFIX="valid" 48 | NON_RFC_COMPLIANT_PREFIX="rfc_non_compliant" 49 | MISSING_MX_PREFIX="missing_mx" 50 | AMBIGUOUS_PREFIX="ambiguous" 51 | 52 | 53 | 54 | 55 | 56 | ## NO NEED TO EDIT UNDER THIS LINE 57 | 58 | # Initial counter values 59 | INCORRECT_MAILS=0 60 | INCORRECT_DOMAINS=0 61 | INCORRECT_MX=0 62 | AMBIGUOUS_MAILS=0 63 | VALID_MAILS=0 64 | 65 | # Lowers all characters 66 | function lowercase { 67 | local string="${1}" 68 | 69 | echo "$(echo "$string" | tr '[:upper:]' '[:lower:]')" 70 | } 71 | 72 | # Function checks if argument is valid against RFC822 73 | function checkRFC822 { 74 | local mail="${1}" 75 | local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" 76 | 77 | if [[ $mail =~ $rfc822 ]]; then 78 | return 0 79 | else 80 | return 1 81 | fi 82 | } 83 | 84 | # Function fixes some known typos in email domain parts 85 | function checkDomains { 86 | local mail="${1}" 87 | 88 | declare -a invalid_domains=( .neuf.fr neuf.com neuf.fre neuf.frr neuf neuf.fe neuf.ff 89 | .wanadoo.fr wanado.fr wanado.com wanadoo.fre anadoo.fr wanaoo.fr wanadoo wanadoo.com wanadoo.frr wanadoo wanadoo.fe wanadoo.ff 90 | .orange.fr ornge.fr orage.fr orage.com orange.com orange.frr orange orange.fe orange.ff 91 | .free.fr fre.fr fre.com free.com free.frr free.fe free.ff 92 | .club-internet.fr clubinternet.fr clubinternet.com club-internet.com club-internet clubinternet club_internet.fr club-internet.fe club-internet.ff 93 | .laposte.net laposte.com laposte 94 | .yahoo.fr yaho.fr yaho.com yaho.co.uk yahoo.frr yahoo.ciom yahoo.vom yahoo.c yahoo.cim yahoo.co yaoo.col yahoo.colm yahoo.con 95 | .sfr.fr sfr.fre sfr.frr sfr sfr.fe sfr.ff 96 | .hotmail.fr homail.fr hotail.fr hotmail.fe hotmail.ff homail.com hotail.com hotmail.ciom hotmail.vom hotmail.c hotmail.cim hotmail.co hotmail.col hotmail.colm hotmail.con 97 | .live.com life.fr live.frr live.fe live.ff life.com live.ciom live.vom live.c live.cim live.co live.col live.colm live.con 98 | .outlook.fr outlok.fr outlok.com outlook.ciom outlook.vom outlook.c outlook.cim outlook.co outlook.col outlook.colm outlook.con outhlook.fr outhlook.com 99 | .gmail.com gmail.fr gmal.fr gmail.frr gmail gmail.ciom g.mail gemail.com galml.com gmail.c gmail. gmail.cim gmail.clm gmail.co gmail.col gmail.comp gmail.con gmail.cpm gmail.de gmail.dk gmail.es gmail.org gmail.vom gmaill.com gmal.com gmeil.com gmail.vom gmail.colm 100 | .googlemail.com googlemail.con googlemail.cpm googlemail.de googlemail.fr googlemail.co.uk googlemail.es googlemail.dk googlemail.vom googlemail.c googlemail.cim googlemail.co googlemail.col googlemail.colm googlemail.con 101 | .aliceadsl.fr alice.fr aliseadsl.fr aliceadsl.com aliceadsl.frr aliceadsl aliceadsl.fe aliceadsl.ff 102 | .voila.fr voila.com voila.frr voila voila.fe voila.ff 103 | .skynet.be skynet.bee skynet 104 | .aol.fr aol.ciom aol.vom aol.c aol.cim aol.co aol.col aol.colm aol.con 105 | ) 106 | declare -a valid_domains=( neuf.fr neuf.fr neuf.fr neuf.fr neuf.fr neuf.fr neuf.fr 107 | wanadoo.fr wanadoo.fr wanadoo.fr wanadoo.fr wanadoo.fr wanaoo.fr wanadoo.fr wanadoo.fr wanadoo.fr wanadoo.fr wanadoo.fr wanadoo.fr 108 | orange.fr orange.fr orange.fr orange.fr orange.fr orange.fr orange.fr orange.fr orange.fr 109 | free.fr free.fr free.fr free.fr free.fr free.fr free.fr 110 | club-internet.fr club-internet.fr club-internet.fr club-internet.fr club-internet.fr club-internet.fr club-internet.fr club-internet.fr club-internet.fr 111 | laposte.net laposte.net laposte.net 112 | yahoo.fr yahoo.fr yahoo.com yahoo.co.uk yahoo.fr yahoo.com yahoo.com yahoo.com yahoo.com yahoo.com yahoo.com yahoo.com yahoo.com 113 | sfr.fr sfr.fr sfr.fr sfr.fr sfr.fr sfr.fr 114 | hotmail.fr hotmail.fr hotmail.fr hotmail.fr hotmail.fr hotmail.com hotmail.com hotmail.com hotmail.com hotmail.com hotmail.com hotmail.com hotmail.com hotmail.com hotmail.com 115 | live.com live.fr live.fr live.fr live.fr live.com live.com live.com live.com live.com live.com live.com live.com live.com 116 | outlook.fr outlook.fr outlook.com outlook.com outlook.com outlook.com outlook.com outlook.com outlook.com outlook.com outlook.com outlook.fr outlook.com 117 | gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com gmail.com 118 | googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com googlemail.com 119 | aliceadsl.fr aliceadsl.fr aliceadsl.fr aliceadsl.fr aliceadsl.fr aliceadsl.fr aliceadsl.fr aliceadsl.fr 120 | voila.fr voila.fr voila.fr voila.fr voila.fr voila.fr 121 | skynet.be skynet.be skynet.be 122 | aol.fr aol.com aol.com aol.com aol.com aol.com aol.com aol.com aol.com 123 | ) 124 | 125 | 126 | local count=0 127 | 128 | # Dumb check of the number of elements per table that should match 129 | if [ ${#invalid_domains[@]} -ne ${#valid_domains[@]} ]; then 130 | echo "Bogus domain tables. Cannot continue." 131 | exit 1 132 | fi 133 | 134 | for i in "${invalid_domains[@]}"; do 135 | if [ "$i" == "${mail#*@}" ]; then 136 | mail="${mail%@*}@${valid_domains[$count]}" 137 | fi 138 | count=$((count + 1)) 139 | done 140 | 141 | # Function return 142 | echo "$mail" 143 | } 144 | 145 | # Function checks if MX records exist for the domain of an email address 146 | function checkMXDomains { 147 | local mail="${1}" 148 | 149 | if [ "$(dig "${mail#*@}" mx +short | wc -l)" -ne 0 ]; then 150 | return 0 151 | else 152 | return 1 153 | fi 154 | } 155 | 156 | function checkAmbiguous { 157 | local mail="${1}" 158 | 159 | echo "$mail" | grep -E "test@|example@|exemple@|spam@|noreply@|no-reply@|@test\.|@example\.|@exemple\.|@spam\." > /dev/null 160 | if [ $? -eq 0 ]; then 161 | return 0 162 | else 163 | return 1 164 | fi 165 | } 166 | 167 | function checkEnvironment { 168 | 169 | if ! type dig > /dev/null; then 170 | echo "This script needs dig to resolve MX records." 171 | exit 1 172 | fi 173 | 174 | if ! type tr > /dev/null; then 175 | echo "This script needs tr to transform addresses to lowercase." 176 | exit 1 177 | fi 178 | 179 | echo "Checking for internet access." 180 | if [[ $(uname) == *"CYGWIN"* ]]; then 181 | ping -n 3 $inet_addr_to_test > /dev/null 182 | else 183 | ping -c 3 $inet_addr_to_test > /dev/null 184 | fi 185 | if [ $? != 0 ]; then 186 | echo "This script needs internet to resolve MX records." 187 | exit 1 188 | fi 189 | } 190 | 191 | function usage { 192 | echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD" 193 | echo "$AUTHOR" 194 | echo "$CONTACT" 195 | echo "" 196 | echo "Usage: ./emailCheck.sh /path/to/emailList" 197 | echo "Email list needs to be a list of one email per line, encoded in UTF8 Unix format." 198 | echo "Checks if emails are RFC822 valid, corrects known typos in domain names, checks for valid MX records and checks against known ambiguous mail adresses." 199 | echo "" 200 | echo "Outputs 4 files with suffixes:" 201 | echo "$VALID_PREFIX: All emails that seem valid." 202 | echo "$NON_RFC_COMPLIANT_PREFIX: All emails that aren't RFC822 compliant." 203 | echo "$MISSING_MX_PREFIX: All emails of which domain doesn't have valid MX records." 204 | echo "$AMBIGUOUS_PREFIX: All emails which seem ambiguous." 205 | exit 1 206 | } 207 | 208 | function write_line { 209 | local state="${1}" 210 | local line="${2}" 211 | local curated_file="${3}" 212 | eval "echo \"${state}${CSV_INPUT_DELIMITER}${line}\" >> \"$curated_file\"" 213 | } 214 | 215 | function loop { 216 | local input="${1}" 217 | local curated_file="${2}" 218 | 219 | echo "Checking emails." 220 | 221 | count=0 222 | MISSING_MAILS=0 223 | INCORRECT_MAILS=0 224 | INCORRECT_DOMAINS=0 225 | INCORRECT_MX=0 226 | AMBIGUOUS_MAILS=0 227 | VALID_MAILS=0 228 | 229 | eval "echo \"${STATE}${CSV_INPUT_DELIMITER}${CSV_READ// /$CSV_INPUT_DELIMITER}\" > \"$curated_file\"" 230 | while IFS=$CSV_INPUT_DELIMITER read $CSV_READ; do 231 | if [ $count -eq 0 ] && [ "$INPUT_FILE_HAS_HEADER" = true ] ; then 232 | count=$((count+1)) 233 | continue 234 | fi 235 | 236 | STATE="" 237 | 238 | if [ "${email}" == "" ]; then 239 | write_line "MISSING" "$CSV_WRITE" "$curated_file" 240 | MISSING_MAILS=$((MISSING_MAILS+1)) 241 | continue 242 | fi 243 | 244 | email=$(lowercase "$email") 245 | newemail=$(checkDomains "$email") 246 | ## Ugly hack because incorrect_domains can't be increased directly in function checkDomains 247 | if [ "$newemail" != "$email" ]; then 248 | INCORRECT_DOMAINS=$((INCORRECT_DOMAINS+1)) 249 | email="$newemail" 250 | STATE="FIXED_DOMAIN" 251 | fi 252 | 253 | checkRFC822 "$email" 254 | if [ $? -eq 1 ]; then 255 | INCORRECT_MAILS=$((INCORRECT_MAILS+1)) 256 | write_line "NON_RFC_COMPLIANT" "$CSV_WRITE" "$curated_file" 257 | count=$((count+1)) 258 | continue 259 | fi 260 | 261 | checkMXDomains "$email" 262 | if [ $? -eq 1 ]; then 263 | INCORRECT_MX=$((INCORRECT_MX+1)) 264 | write_line "MISSING_MX" "$CSV_WRITE" "$curated_file" 265 | count=$((count+1)) 266 | continue 267 | fi 268 | 269 | checkAmbiguous "$email" 270 | if [ $? -eq 0 ]; then 271 | AMBIGUOUS_MAILS=$((AMBIGUOUS_MAILS+1)) 272 | write_line "AMBIGUOUS" "$CSV_WRITE" "$curated_file" 273 | count=$((count+1)) 274 | continue 275 | fi 276 | 277 | VALID_MAILS=$((VALID_MAILS+1)) 278 | if [ "${STATE}" == "" ] ; then 279 | STATE="VALID" 280 | fi 281 | write_line "$STATE" "$CSV_WRITE" "$curated_file" 282 | 283 | count=$((count+1)) 284 | if [ $((count % 1000)) -eq 0 ]; then 285 | echo "Time: $SECONDS - $count email addresses processed so far." 286 | fi 287 | done <"$input" 288 | } 289 | 290 | if [ "$1" = "" ]; then 291 | usage 292 | fi 293 | if [ ! -f "$1" ] ; then 294 | echo "No such file: $1" 295 | exit 1 296 | fi 297 | 298 | 299 | checkEnvironment 300 | 301 | input="$1" 302 | input_path="$(dirname "$1")" 303 | input_file="$(basename "$1")" 304 | curated_file="$input_path/curated-$input_file" 305 | 306 | loop "$input" "$curated_file" 307 | if [ ! -f "$curated_file" ]; then 308 | echo "No valid emails found. Check if your file has only email addresses, or configure the read process accordingly to read a multicolumn CSV file in source header." 309 | echo "Also, if your file comes from Windows, convert it using dos2unix first." 310 | fi 311 | 312 | echo "" 313 | echo "$MISSING_MAILS missing emails found." 314 | echo "$INCORRECT_MAILS non rfc822 compliant emails found." 315 | echo "$INCORRECT_DOMAINS emails had incorrect domains and have been corrected." 316 | echo "$INCORRECT_MX emails are missing mx records in their domain." 317 | echo "$AMBIGUOUS_MAILS are ambiguous emails." 318 | echo "$VALID_MAILS emails seem valid." 319 | -------------------------------------------------------------------------------- /burp/incexc/windows_settings: -------------------------------------------------------------------------------- 1 | #### Generic Microsoft excludes 2 | 3 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Temp 4 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/LocalLow 5 | exclude_regex = ^[A-Z]:/Documents and Settings/[^/]+/Cookies 6 | exclude_regex = ^[A-Z]:/Documents and Settings/[^/]+/Recent 7 | exclude_regex = ^[A-Z]:/Documents and Settings/[^/]+/Local Settings/Temp 8 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Microsoft/Windows/Recent 9 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Temp 10 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/History 11 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Application Data 12 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Temporary Internet Files 13 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Internet Explorer 14 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/Temporary Internet Files 15 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/Caches 16 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/History 17 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Terminal Server Client/Cache 18 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/[^/]+/[Tt]e?mp 19 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Microsoft/Windows/Themes/CachedFiles 20 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/[^/]+/Internet Explorer 21 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/[^/]+/Microsoft/[^/]+/[^/]*[Cc]aches? 22 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/[^/]+/Microsoft/[^/]+/InetCookies 23 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/[^/]+/Microsoft Office/[^/]+OfficeFileCache 24 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/[^/]+/Microsoft/Office/Recent 25 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Outlook/RoamCache 26 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Office/[0-9\.]+/WebServiceCache 27 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Office/[0-9\.]+/Lync/Tracing 28 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/LocalLow/Microsoft/CryptnetUrlCache 29 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Downloaded Installations 30 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/GroupPolicy 31 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/AppV 32 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Messenger 33 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/OneNote 34 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Terminal Server Client 35 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/UEV 36 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows Live 37 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows Live Contacts 38 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Application Shortcuts 39 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Notifications 40 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/UsrClass\.dat\.LOG[0-9]+ 41 | exclude_regex = ^[A-Z]:/Users/[^/]+/ntuser\.dat\.LOG[0-9]+ 42 | 43 | # Temp folder for files that are ready to be burned on DVD 44 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/Burn 45 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/CD Burning 46 | 47 | # Generic application cache & temp folders (excludes all aaaCacheaaa or bbbTempbbb dirs) 48 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|LocalLow|Roaming)/[^/]+/[^/]*([Cc]ache|[Tt]emp)[^/]* 49 | 50 | # Various Win 10 caches 51 | exclude_regex = .*/(OfficeFile|SmartLookup|BackstageInAppNav|MruService)Cache 52 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/Caches 53 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/ActionCenterCache/ 54 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Feeds Cache 55 | 56 | # Error reports 57 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/WER 58 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/CrashDumps 59 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Diagnostics 60 | 61 | # Windows 10 Edge 62 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/[^/]*[Cc]ache 63 | exclude_regex = ^[A-Z]:/Users/[^/]+/MicrosoftEdgeBackups/backups 64 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/MicrosoftEdge/SharedCacheContainers 65 | 66 | # Windows 10 Store Application cache and state 67 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Packages/[^/]+/AC 68 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Packages/[^/]+/TempState 69 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Packages/[^/]+/LocalState 70 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Packages/[^/]+/LocalCache 71 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Packages/[^/]+/RoamingState 72 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Packages/[^/]+/AppData/User/Default/CacheStorage 73 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Packages/[^/]+/AppData/CacheStorage 74 | 75 | # Windows 10 various stuff 76 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/Notifications 77 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/Windows/Explorer 78 | 79 | # Windows downloads 80 | exclude_regex = ^[A-Z]:/Users/[^/]+/Downloads 81 | 82 | # Windows update cache 83 | exclude_regex = ^[A-Z]:/Windows/SoftwareDistribution/Download 84 | 85 | # Windows offline files 86 | exclude_regex = ^[A-Z]:/Windows/CSC 87 | 88 | exclude_regex = ^[A-Z]:/Windows/Temp 89 | exclude_regex = ^[A-Z]:/Windows/Downloaded Program Files 90 | exclude_regex = ^[A-Z]:/RECYCLER$ 91 | exclude_regex = ^[A-Z]:/\$recycle\.bin$ 92 | exclude_regex = ^[A-Z]:/System Volume Information$ 93 | # swap file (Windows XP, 7, 8) 94 | exclude_regex = ^[A-Z]:/pagefile\.sys$ 95 | # swap file?? (Windows 8) 96 | exclude_regex = ^[A-Z]:/swapfile\.sys$ 97 | exclude_regex = ^[A-Z]:/hiberfil\.sys$ 98 | # Windows temp installer 99 | exclude_regex = ^[A-Z]:/\$WINDOWS\.~BT$ 100 | # Windows 10 Upgrade previous install 101 | exclude_regex = ^[A-Z]:/Windows\.old$ 102 | exclude_regex = ^[A-Z]:/PerfLogs$ 103 | 104 | 105 | # Windows filesystem directories 106 | exclude_regex = ^[A-Z]:/\$mft$ 107 | exclude_regex = ^[A-Z]:/\$logfile$ 108 | exclude_regex = ^[A-Z]:/\$volume$ 109 | exclude_regex = ^[A-Z]:/\$bitmap$ 110 | exclude_regex = ^[A-Z]:/\$extend$ 111 | exclude_regex = ^[A-Z]:/\$reparse$ 112 | 113 | # Onedrive user executables 114 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Microsoft/OneDrive 115 | 116 | # Unnecessary folder exclusions 117 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Microsoft/Windows/Cookies 118 | exclude_regex = ^[A-Z]:/Users/[^/]+/NetHood 119 | exclude_regex = ^[A-Z]:/Users/[^/]+/PrintHood 120 | exclude_regex = ^[A-Z]:/Users/[^/]+/Cookies 121 | exclude_regex = ^[A-Z]:/Users/[^/]+/Recent 122 | exclude_regex = ^[A-Z]:/Users/[^/]+/SendTo 123 | exclude_regex = ^[A-Z]:/Users/[^/]+/LocalService 124 | exclude_regex = ^[A-Z]:/Users/[^/]+/NetworkService 125 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/LocalLow 126 | exclude_regex = ^[A-Z]:/Users/[^/]+/Tracing 127 | 128 | # Generic system file exlusions 129 | exclude_regex = ^[A-Z]:/*/MSOCache 130 | exclude_regex = ^[A-Z]:/*/Config\.Msi 131 | 132 | #### Applications 133 | 134 | # Office telemetry data 135 | exclude_regex = exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local//Microsoft/Office/OTeleData_.*.etl 136 | 137 | # Chrome 66+ 138 | # Chrome can also store in Roaming instead of Local 139 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/[^/]*[Cc]ache 140 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/ShaderCache 141 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/Session Storage 142 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/Local Storage 143 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/Extensions/Temp 144 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/Service Worker/CacheStorage 145 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/File System 146 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/LOG 147 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/(Default|Profile [0-9]+)/.*/LOG 148 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/SwReporter 149 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Google/Chrome/User Data/PepperFlash 150 | 151 | # Opera 41+ 152 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Opera/Opera/profile/cache4 153 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Opera Software/Opera [^/]+/[^/]*[Cc]ache 154 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Opera Software/Opera [^/]+/Local Storage 155 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Opera Software/Opera [^/]+/Crash Reports 156 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Opera Software/Opera [^/]+/[^/]*[Cc]ache 157 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Opera Software/Opera [^/]+/Sessions 158 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Opera Software/Opera [^/]+/lockfile 159 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Opera Software/Opera [^/]+/LOG 160 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Opera Software/Opera [^/]+/.*/LOG 161 | 162 | # Vivaldi 1.x+ 163 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Vivaldi/Application 164 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Vivaldi/User Data/[^/]+/[^/]*[Cc]ache 165 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Vivaldi/User Data/[^/]+/Local Storage 166 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Vivaldi/User Data/[^/]+/LOG 167 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Vivaldi/User Data/[^/]+/.*/LOG 168 | 169 | # Firefox 44+ 170 | # There might exist a cache or a cache2 directory 171 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Mozilla/Firefox/Profiles/[^/]+/[^/]*[Cc]ache2? 172 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Mozilla/Firefox/Profiles/[^/]+/[Tt]humbnails 173 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Mozilla/Firefox/Crash Reports 174 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Mozilla/Firefox/Profiles/[^/]+/[^/]*[Cc]ache2 175 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Mozilla/Firefox/Profiles/[^/]+/sessionstore\.bak 176 | 177 | # Thunderbird 178 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Thunderbird/Mozilla Thunderbird/updates 179 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Thunderbird/Profiles/[^/]+/[Cc]ache[^/]* 180 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Thunderbird/Crash Reports 181 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Thunderbird/Profiles/[^/]+/crashes 182 | 183 | # Github Desktop 184 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/GitHubDesktop 185 | 186 | # Google Apps Sync 187 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Google/Google Apps Sync/Tracing 188 | 189 | # Adobe Acrobat DC 190 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Adobe/AcroCef/DC/Acrobat/Cache 191 | 192 | # Apple Logs 193 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Apple Computer/Logs 194 | 195 | # Apple iPhone backups :( 196 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Apple Computer/MobileSync/Backup 197 | # iTunes downloaded album artwork 198 | exclude_regex = ^[A-Z]:/Users/[^/]+/Music/iTunes/Album Artwork/Download 199 | 200 | # Java 201 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Sun 202 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(LocalLow|Roaming)/Sun/Java/Deployment/(cache|log|tmp) 203 | 204 | # Cisco Webex 205 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/WebEx/wbxcache 206 | 207 | # Ignite Realtime Spark client logs 208 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Roaming/Spark/logs 209 | 210 | # TeamViewer / SimpleHelp quick suppor 211 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/TeamViewer 212 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/JWrapper-Remote Support 213 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/JWrapper-SimpleHelp Technician 214 | 215 | # Zoom remote tool 216 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/(Local|Roaming)/Zoom 217 | 218 | # Dropbox, OneDrive, SkyDrive data directories (not excluded by default because of cryptolockers attacks) 219 | #exclude_regex = ^[A-Z]:/Users/[^/]+/SkyDrive[^/]+ 220 | #exclude_regex = ^[A-Z]:/Users/[^/]+/Dropbox 221 | #exclude_regex = ^[A-Z]:/Users/[^/]+/OneDrive 222 | 223 | # Dropbox config directory 224 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/Dropbox 225 | 226 | # Owncloud executables 227 | exclude_regex = ^[A-Z]:/Users/[^/]+/AppData/Local/ownCloud/.*\.exe 228 | 229 | #### Burp settings 230 | 231 | # Remove VSS info, also strips NTFS acls (protocol 1 only) 232 | strip_vss=1 233 | # Splitting VSS info would mean that for every file, 3 files exists (header, actual file, footer) 234 | split_vss=0 235 | -------------------------------------------------------------------------------- /ssh_jail.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROGRAM="ssh_jail.sh" # Basic ssh shell jail creation script 4 | AUTHOR="(L) 2014 by Orsiris \"Ozy\" de Jong" 5 | CONTACT="http://www.netpower.fr - ozy@netpower.fr" 6 | PROGRAM_BUILD=2108201403 7 | 8 | ## Creates a SSH chroot jail where a given user can login and execute a minimal set of programs 9 | ## Binaries specified by BINARIES_TO_COPY will be available 10 | ## You may also specifiy /usr/bin and /lib in EXT_DIRS to gain multiple binaries 11 | 12 | ## If you need support for editing programs, consider adding /usr/share/terminfo (CentOS) or /lib/terminfo directories (Debian) to EXT_DIRS 13 | 14 | 15 | # List of binary files to copy to chrooted environment. All dependencies will also be copied. 16 | BINARIES_TO_COPY="/usr/bin/chmod;/usr/bin/chown;/usr/bin/ls;/usr/bin/cat;/usr/bin/ln;/usr/bin/cp;/usr/sbin/ldconfig" # Works on CentOS 7 17 | #BINARIES_TO_COPY="/bin/chmod;/bin/chown;/bin/ls;/bin/cat;/bin/ln;/bin/cp;/bin/mv;/bin/rm;/usr/bin/curl" # Works on Debian 6 18 | 19 | # Directories to copy to chrooted environment 20 | # /etc/php.d is needed to support php modules 21 | # /usr/share/terminfo or /lib/terminfo is needed to support interactive programs like nano or php 22 | # /usr/share/snmp is needed for php-snmp module 23 | # /usr/share/zoneinfo is needed for php-composer 24 | EXT_DIRS="/usr/bin;/usr/lib64;/usr/share/php;/usr/share/terminfo;/usr/include;/etc/php.d;/usr/share/snmp;/usr/share/zoneinfo;/etc/ld.so.conf.d" # Works on CentOS 7 (replace lib by lib64 if needed) 25 | #EXT_DIRS="/bin;/lib;/lib/terminfo" # Works on Debian 6 (replace lib by lib64 if needed) 26 | #EXT_DIRS="/lib/terminfo" 27 | 28 | # Empty directories to create in chrooted environment types (lib and lib64 directories are already included) 29 | DIRS_TO_CREATE="/dev;/etc;/var/tmp;/tmp" 30 | 31 | # Additional files to copy to chrooted environment 32 | # /etc/resolv.conf is needed to enable internet access 33 | # /etc/pki/tls/certs/ca-bundle.crt and /etc/pki/tls/certs/ca-bundle.trust.crt are needed to enable SSL certificate verification 34 | FILES_TO_COPY="/etc/localtime;/etc/passwd;/etc/group;/etc/resolv.conf;/etc/pki/tls/certs/ca-bundle.crt;/etc/pki/tls/certs/ca-bundle.trust.crt;/etc/ld.so.conf" 35 | 36 | # Default group for chrooted users 37 | #GROUP=chroot 38 | GROUP=apache 39 | 40 | # Shell to use in chrooted environment 41 | SHELL=/usr/bin/bash 42 | 43 | # ADD chroot entry to ssh server 44 | ADD_SSH_CHROOT_ENTRY=yes 45 | 46 | # Use alternative chroot method without SSH (login chroot script) 47 | USE_CHROOT_SCRIPT=no 48 | 49 | ## END OF STANDARD CONFIGURATION OPTIONS ########################################################### 50 | 51 | CHROOT_SCRIPT_PATH=/bin/chrootusers 52 | USER_HOME=/home 53 | 54 | # Basic directories present on most linux flavors 55 | BASIC_DIRS="/usr/bin;/usr/lib;/usr/lib64" 56 | # Create symlinks for BASIC_DIRS to root (used on most linux flavors) 57 | CREATE_SYMLINKS="yes" 58 | 59 | SSHD_CONFIG="/etc/ssh/sshd_config" 60 | 61 | # Prevent cp -i alias that stays interactive 62 | alias cp=cp 63 | 64 | function LogDebug 65 | { 66 | if [ "$DEBUG" == "yes" ] 67 | then 68 | echo "$1" 69 | fi 70 | } 71 | 72 | function CheckErr 73 | { 74 | if [ $? != 0 ] 75 | then 76 | exec_error=1 77 | echo "Failed on task: $1" 78 | fi 79 | } 80 | 81 | function Usage 82 | { 83 | echo "$PROGRAM build $PROGRAM_BUILD" 84 | echo $AUTHOR 85 | echo $CONTACT 86 | echo "" 87 | echo "$0 [--alt-chroot-dir=/jail/dir] [--add-program=/path/to/program] [-f]" 88 | echo "Creates user and sets shell jail to its home directory" 89 | echo "--add-program=/some/program Adds a program and its dependencies to an existing chrooted user jail" 90 | echo "--alt-chroot-dir=/another/home By default, chroot is created in /home/. You can specify an alternate root here and chroot will be created in /" 91 | echo " This parameter also triggers a new chrootscript with the alternative home" 92 | echo "-f Forces deletion of existing user and chroot scripts" 93 | exit 128 94 | } 95 | 96 | function CheckEnvironment 97 | { 98 | case $(uname -m) in 99 | 100 | x86_64) 101 | ARCH=x64 102 | LIB="/lib64;/lib" 103 | ;; 104 | i686) 105 | ARCH=x86 106 | LIB=/lib 107 | ;; 108 | *) 109 | ARCH=unknown 110 | ;; 111 | 112 | esac 113 | 114 | if [ "$ARCH" == "unknown" ] 115 | then 116 | echo "Unknown architecture" 117 | exit 1 118 | fi 119 | 120 | if ! type -p bash > /dev/null 2>&1 121 | then 122 | echo "Cannot find bash shell environment." 123 | exit 1 124 | fi 125 | 126 | if ! type -p env > /dev/null 2>&1 127 | then 128 | echo "Cannot find env." 129 | exit 1 130 | else 131 | ENV_BINARY=$(type -p env) 132 | fi 133 | 134 | if ! type -p chroot > /dev/null 2>&1 135 | then 136 | echo "Cannot find chroot executable." 137 | exit 1 138 | else 139 | CHROOT_BINARY=$(type -p chroot) 140 | CHROOT_BINARY_ALT="$CHROOT_BINARY""_alt" 141 | fi 142 | 143 | if ! type -p ldd > /dev/null 2>&1 144 | then 145 | echo "Cannot find ldd executable." 146 | exit 1 147 | fi 148 | 149 | if ! type -p ldconfig > /dev/null 2>&1 150 | then 151 | echo "Cannot find ldconfig executable." 152 | exit 1 153 | fi 154 | 155 | if ! type -p ln > /dev/null 2>&1 156 | then 157 | echo "Cannot find ln executable." 158 | exit 1 159 | else 160 | LN_BINARY=$(type -p ln) 161 | fi 162 | 163 | } 164 | 165 | function AddUserAndGroup 166 | { 167 | echo "Creating group $GROUP" 168 | groupadd "$GROUP" 169 | 170 | echo "Creating user $LOGIN" 171 | # Returns 1 if user doesn't exist 172 | id -nu "$LOGIN" > /dev/null 2>&1 173 | if [ $? != 0 ] || [ "$force" == "1" ] 174 | then 175 | if [ "$USE_CHROOT_SCRIPT" == "yes" ] 176 | then 177 | chroot="-s $CHROOT_SCRIPT_PATH" 178 | else 179 | chroot="" 180 | fi 181 | useradd -c "User chrooted" -d "$CHROOT_DIR/" -g "$GROUP" $chroot "$LOGIN" 182 | CheckErr "Adding user $LOGIN" 183 | else 184 | echo "User $LOGIN already exists." 185 | exit 1 186 | fi 187 | 188 | echo "Please enter password for $LOGIN" 189 | passwd "$LOGIN" > /dev/null 190 | CheckErr "Creating password for $LOGIN" 191 | if ! [ -d "$CHROOT_DIR" ] 192 | then 193 | echo "Creating home directory" 194 | mkdir -p "$CHROOT_DIR/" 195 | CheckErr "Create directory $CHROOT_DIR/" 196 | fi 197 | } 198 | 199 | function AddBinaryPaths 200 | { 201 | OLD_IFS=$IFS 202 | IFS=";" 203 | for binary in $BINARIES_TO_COPY 204 | do 205 | DIRS_TO_CREATE="$DIRS_TO_CREATE;$(dirname $binary)" 206 | done 207 | IFS=$OLD_IFS 208 | LogDebug "List of directories to create: $DIRS_TO_CREATE" 209 | } 210 | 211 | function CreatePaths 212 | { 213 | OLD_IFS=$IFS 214 | IFS=";" 215 | LogDebug "$DIRS_TO_CREATE" 216 | for dir in $DIRS_TO_CREATE 217 | do 218 | if ! [ -d "$CHROOT_DIR$dir" ] 219 | then 220 | LogDebug "Creating $CHROOT_DIR$dir" 221 | mkdir -p "$CHROOT_DIR$dir" 222 | CheckErr "Create directory $CHROOT_DIR$dir" 223 | fi 224 | 225 | chmod 700 "$CHROOT_DIR$dir" 226 | CheckErr "command chmod 700 $CHROOT_DIR$dir" 227 | done 228 | IFS=$OLD_IFS 229 | } 230 | 231 | # Adds binaries and dependancy libs to chroot (list separated by semicolons given as argument) 232 | function AddBinaries 233 | { 234 | OLD_IFS=$IFS 235 | IFS=";" 236 | for binary in $1 237 | do 238 | dependencies="" 239 | if [ ! -d "$CHROOT_DIR$(dirname $binary)" ] 240 | then 241 | mkdir -p "$CHROOT_DIR$(dirname $binary)" 242 | CheckErr "Creating $CHROOT_DIR$(dirname $binary)" 243 | fi 244 | 245 | LogDebug "Copying $binary to $CHROOT_DIR$binary" 246 | cp $binary "$CHROOT_DIR$binary" 247 | CheckErr "Copy $binary to $CHROOT_DIR$binary" 248 | 249 | IFS=$OLD_IFS 250 | # Get all dependant libraries from binary (ldd sometimes gives output at first column and sometimes at third depending on the type of dependency) 251 | dependencies=$(ldd $binary | awk '{print $1}' | grep "^/") 252 | dependencies="$dependencies"$'\n'$(ldd $binary | awk '{print $3}' | grep "^/") 253 | for dependency in $dependencies 254 | do 255 | dependency_dir=$(dirname $dependency) 256 | if [ ! -d "$CHROOT_DIR$dependency_dir" ] 257 | then 258 | mkdir -p "$CHROOT_DIR$dependency_dir" 259 | CheckErr "Creating $CHROOT_DIR$dependency_dir" 260 | fi 261 | 262 | if [ ! -f "$CHROOT_DIR$dependency" ] 263 | then 264 | LogDebug "Copying dependency $dependency to $CHROOT_DIR$dependency_dir/" 265 | cp "$dependency" "$CHROOT_DIR$dependency_dir/" 266 | CheckErr "Copy $dependency to $CHROOT_DIR$dependency_dir/" 267 | fi 268 | done 269 | IFS=";" 270 | done 271 | IFS=$OLD_IFS 272 | } 273 | 274 | function CopyFiles 275 | { 276 | OLD_IFS=$IFS 277 | IFS=";" 278 | for file in $FILES_TO_COPY 279 | do 280 | LogDebug "Copying $file to $CHROOT_DIR$(dirname $file)/" 281 | if ! [ -d "$CHROOT_DIR$(dirname $file)" ] 282 | then 283 | mkdir -p "$CHROOT_DIR$(dirname $file)" 284 | CheckErr "reate $CHROOT_DIR$(dirname $file)" 285 | fi 286 | cp "$file" "$CHROOT_DIR$(dirname $file)/" 287 | CheckErr "Copy $file to $CHROOT_DIR$(dirname $file)/" 288 | done 289 | IFS=$OLD_IFS 290 | } 291 | 292 | function CopyDirs 293 | { 294 | OLD_IFS=$IFS 295 | IFS=";" 296 | for dir in $EXT_DIRS 297 | do 298 | mkdir -p "$CHROOT_DIR$dir" 299 | CheckErr "Creating $CHROOT_DIR$dir" 300 | LogDebug "Copying directory $dir to $CHROOT_DIR/" 301 | cp -R "$dir/" "$CHROOT_DIR$(dirname $dir)" 302 | CheckErr "Copy $dir to $CHROOT_DIR/" 303 | done 304 | IFS=$OLD_IFS 305 | } 306 | 307 | function AllowChrootBinary 308 | { 309 | if [ -f "$CHROOT_BINARY" ] 310 | then 311 | cp "$CHROOT_BINARY" "$CHROOT_BINARY_ALT" 312 | CheckErr "Copy $CHROOT_BINARY to $CHROOT_BINARY_ALT" 313 | # Setuid for allowing execution of chroot binary by normal user 314 | chmod 4755 "$CHROOT_BINARY_ALT" 315 | CheckErr "chmod 4755 $CHROOT_BINARY_ALT" 316 | fi 317 | } 318 | 319 | function AddSSHChrootEntry 320 | { 321 | if [ -f $SSHD_CONFIG ] 322 | then 323 | LogDebug "Adding chroot entry to $SSHD_CONFIG" 324 | echo "" >> $SSHD_CONFIG 325 | echo "Match User $LOGIN" >> $SSHD_CONFIG 326 | echo " ChrootDirectory $CHROOT_DIR" >> $SSHD_CONFIG 327 | echo " AllowTCPForwarding no" >> $SSHD_CONFIG 328 | echo " X11Forwarding no" >> $SSHD_CONFIG 329 | CheckErr "Adding chroot entry to $SSHD_CONFIG" 330 | 331 | echo "Don't forget to reload sshd." 332 | else 333 | echo "Cannot find $SSHD_CONFIG path" 334 | exit 1 335 | fi 336 | } 337 | 338 | function CreateChrootScript 339 | { 340 | if ! [ -f "$CHROOT_SCRIPT_PATH" ] || [ "$force_script" == "1" ] 341 | then 342 | echo "Creating $CHROOT_SCRIPT_PATH" 343 | cat > "$CHROOT_SCRIPT_PATH" << EXTSCRIPT 344 | #!/bin/bash 345 | if [ -d "$USER_HOME/\$USER" ] 346 | then 347 | exec -c "$CHROOT_BINARY_ALT" "$USER_HOME/\$USER" "$ENV_BINARY" -i TERM="\$TERM" HOME="/" $SHELL --login -i 348 | else 349 | echo "No home directory" 350 | exit 1 351 | fi 352 | EXTSCRIPT 353 | CheckErr "Create script $CHROOT_SCRIPT_PATH" 354 | chmod 555 "$CHROOT_SCRIPT_PATH" 355 | CheckErr "chmod 555 $CHROOT_SCRIPT_PATH" 356 | else 357 | echo "$CHROOT_SCRIPT_PATH already exists. Use -f to override." 358 | fi 359 | } 360 | 361 | function AddSymlinks 362 | { 363 | # Add symlinks from BASIC_DIRS to root dir 364 | OLD_IFS=$IFS 365 | IFS=";" 366 | 367 | for link in $BASIC_DIRS 368 | do 369 | LogDebug "Creating symlink $link -> /$(basename $link)" 370 | $LN_BINARY -s "$link" "$CHROOT_DIR/$(basename $link)" 371 | CheckErr "Creating $link symlink" 372 | done 373 | IFS=$OLD_IFS 374 | } 375 | 376 | function AddSpecialFiles 377 | { 378 | if ! [ -c "$CHROOT_DIR/dev/null" ] 379 | then 380 | mknod "$CHROOT_DIR/dev/null" c 1 3 -m 666 381 | CheckErr "Creating /dev/null in jail" 382 | fi 383 | 384 | if ! [ -c "$CHROOT_DIR/dev/console" ] 385 | then 386 | mknod "$CHROOT_DIR/dev/console" c 5 1 -m 622 387 | CheckErr "Creating /dev/console in jail" 388 | fi 389 | 390 | if ! [ -c "$CHROOT_DIR/dev/zero" ] 391 | then 392 | mknod "$CHROOT_DIR/dev/zero" c 1 5 -m 666 393 | CheckErr "Creating /dev/zero in jail" 394 | fi 395 | 396 | if ! [ -c "$CHROOT_DIR/dev/ptmx" ] 397 | then 398 | mknod "$CHROOT_DIR/dev/ptmx" c 5 2 -m 666 399 | CheckErr "Creating /dev/ptmx in jail" 400 | fi 401 | 402 | if ! [ -c "$CHROOT_DIR/dev/tty" ] 403 | then 404 | mknod "$CHROOT_DIR/dev/tty" c 5 1 -m 666 405 | CheckErr "Creating /dev/tty in jail" 406 | fi 407 | 408 | if ! [ -c "$CHROOT_DIR/dev/random" ] 409 | then 410 | mknod "$CHROOT_DIR/dev/random" c 1 8 -m 444 411 | CheckErr "Creating /dev/random in jail" 412 | fi 413 | 414 | if ! [ -c "$CHROOT_DIR/dev/urandom" ] 415 | then 416 | mknod "$CHROOT_DIR/dev/urandom" c 1 9 -m 444 417 | CheckErr "Creating /dev/urandom in jail" 418 | fi 419 | 420 | chown root:tty $CHOOT_DIR/dev/{console,ptmx,tty} 421 | CheckErr "Taking ownership of /dev/console /dev/ptmx and /dev/tty" 422 | } 423 | 424 | function Runldconfig 425 | { 426 | LogDebug "Running ldconfig in chrooted environment" 427 | chroot $CHROOT_DIR /usr/sbin/ldconfig 428 | CheckErr "ldconfig failed" 429 | } 430 | 431 | function SetPermissions 432 | { 433 | LogDebug "Changing owner of $CHROOT_DIR to $LOGIN:$GROUP" 434 | chown -R "$LOGIN:$GROUP" "$CHROOT_DIR/" 435 | CheckErr "chown -R $LOGIN:$GROUP $CHROOT_DIR/" 436 | 437 | chown "root:root" "$CHROOT_DIR" 438 | CheckErr "chown root:root $CHROOT_DIR" 439 | 440 | chmod 755 "$CHROOT_DIR" 441 | CheckErr "chmod 755 $CHROOT_DIR" 442 | } 443 | 444 | if [ "$1" == "" ] 445 | then 446 | Usage 447 | else 448 | LOGIN="$1" 449 | fi 450 | 451 | CheckEnvironment 452 | 453 | force=0 454 | force_script=0 455 | for i in "$@" 456 | do 457 | case $i in 458 | --add-program=*) 459 | ADD_PROGRAM="${i##*=}" 460 | ;; 461 | --alt-chroot-dir=*) 462 | USER_HOME="${i##*=}" 463 | force_script=1 464 | ;; 465 | -f) 466 | force=1 467 | force_script=1 468 | ;; 469 | esac 470 | done 471 | CHROOT_DIR="$USER_HOME/$LOGIN" 472 | 473 | # Add arch dependend lib path to directory list 474 | DIRS_TO_CREATE="$DIRS_TO_CREATE;$CHROOT_DIR" 475 | 476 | if ! [ "$ADD_PROGRAM" == "" ] 477 | then 478 | AddBinaryPaths 479 | CreatePaths 480 | AddBinaries "$ADD_PROGRAM" 481 | SetPermissions 482 | exit 483 | else 484 | # Normal program run 485 | AddUserAndGroup 486 | CreatePaths 487 | CopyDirs 488 | CopyFiles 489 | if [ "$CREATE_SYMLINKS" == "yes" ] 490 | then 491 | AddSymlinks 492 | fi 493 | AddBinaries "$SHELL;$ENV_BINARY;$BINARIES_TO_COPY" 494 | if [ "$ADD_SSH_CHROOT_ENTRY" == "yes" ] 495 | then 496 | AddSSHChrootEntry 497 | fi 498 | if [ "$USE_CHROOT_SCRIPT" == "yes" ] 499 | then 500 | AllowChrootBinary 501 | CreateChrootScript 502 | fi 503 | AddSpecialFiles 504 | Runldconfig 505 | SetPermissions 506 | fi 507 | 508 | if [ "$exec_error" == "1" ] 509 | then 510 | echo "Script finished with errors for user $LOGIN" 511 | else 512 | echo "Created chrooted user $LOGIN" 513 | fi 514 | -------------------------------------------------------------------------------- /emailCheck.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | emailCheck.py 5 | (L) 2014-2025 by Orsiris de Jong 6 | http://www.netpower.fr/ - ozy@netpower.fr 7 | 8 | Email correction script 9 | - Lowers all characters of email 10 | - Checks if email format is valid against RFC822 11 | - Checks if there are any known domain typos 12 | - Checks if email domain has valid MX records (with caching) 13 | - Checks if email address is ambiguous 14 | """ 15 | 16 | import re 17 | import sys 18 | import csv 19 | import subprocess 20 | from pathlib import Path 21 | from functools import lru_cache 22 | 23 | PROGRAM_VERSION = "0.7.0" 24 | PROGRAM_BUILD = "2025120501" 25 | 26 | ###################################################################################################### 27 | # Configuration 28 | ###################################################################################################### 29 | 30 | INPUT_FILE_HAS_HEADER = True 31 | 32 | # CSV configuration 33 | CSV_EMAIL_IS_FIRST_COLUMN = True 34 | CSV_INPUT_DELIMITER = ',' 35 | # We add a "state" column at the start of the file with the email check results 36 | # When multiple email columns exist, we need to fill the state column with multiple states, one per column 37 | STATE_COLUMN_SEPARATOR = '+' # Separator for multiple email states from different email columns 38 | MULTI_EMAIL_SEPARATOR = ';' # Separator for multiple emails in one column 39 | CSV_COLUMNS = ['CT_Num', 'CT_EMail', 'CT_EMail2', 'CT_Fonction', 'CT_Sommeil'] 40 | EMAIL_COLUMNS = ['CT_EMail', 'CT_EMail2'] 41 | 42 | # Internet connectivity check 43 | INET_ADDR_TO_TEST = "www.google.com" 44 | 45 | DNS_CACHE_SIZE = 10000 # Size of the LRU cache for DNS lookups 46 | 47 | ###################################################################################################### 48 | # Output codes 49 | ###################################################################################################### 50 | 51 | EM_VALID = "OK" 52 | EM_MISSING = "VIDE" 53 | EM_NON_RFC = "INVALIDE" 54 | EM_FIXED = "CORRIGE" 55 | EM_NO_MX = "DOMAINE INVALIDE" 56 | EM_AMBIGUOUS = "AMBIGU" 57 | 58 | ###################################################################################################### 59 | # Domain mappings - now using dictionary for O(1) lookup instead of O(n) loop 60 | ###################################################################################################### 61 | 62 | INVALID_TO_VALID_DOMAINS = { 63 | # neuf.fr variants 64 | 'neuf.com': 'neuf.fr', 'neuf.fre': 'neuf.fr', 'neuf.frr': 'neuf.fr', 65 | 'neuf': 'neuf.fr', 'neuf.fe': 'neuf.fr', 'neuf.ff': 'neuf.fr', 66 | 67 | # wanadoo.fr variants 68 | 'wanado.fr': 'wanadoo.fr', 'wanado.com': 'wanadoo.fr', 'wanadoo.fre': 'wanadoo.fr', 69 | 'anadoo.fr': 'wanadoo.fr', 'wanaoo.fr': 'wanadoo.fr', 'wanadoo': 'wanadoo.fr', 70 | 'wanadoo.com': 'wanadoo.fr', 'wanadoo.frr': 'wanadoo.fr', 'wanadoo.fe': 'wanadoo.fr', 71 | 'wanadoo.ff': 'wanadoo.fr', 72 | 73 | # orange.fr variants 74 | 'ornge.fr': 'orange.fr', 'orage.fr': 'orange.fr', 'orage.com': 'orange.fr', 75 | 'orange.com': 'orange.fr', 'orange.frr': 'orange.fr', 'orange': 'orange.fr', 76 | 'orange.fe': 'orange.fr', 'orange.ff': 'orange.fr', 'orange:fr': 'orange.fr', 77 | 78 | # free.fr variants 79 | 'fre.fr': 'free.fr', 'fre.com': 'free.fr', 'free.com': 'free.fr', 80 | 'free.frr': 'free.fr', 'free.fe': 'free.fr', 'free.ff': 'free.fr', 'free:frr': 'free.fr', 81 | 'free.f': 'free.fr', 82 | 83 | # club-internet.fr variants 84 | 'clubinternet.fr': 'club-internet.fr', 'clubinternet.com': 'club-internet.fr', 85 | 'club-internet.com': 'club-internet.fr', 'club-internet': 'club-internet.fr', 86 | 'clubinternet': 'club-internet.fr', 'club_internet.fr': 'club-internet.fr', 87 | 'club-internet.fe': 'club-internet.fr', 'club-internet.ff': 'club-internet.fr', 88 | 89 | # laposte.net variants 90 | 'laposte.com': 'laposte.net', 'laposte': 'laposte.net', 91 | 92 | # yahoo variants 93 | 'yaho.fr': 'yahoo.fr', 'yaho.com': 'yahoo.com', 'yaho.co.uk': 'yahoo.co.uk', 94 | 'yahoo.frr': 'yahoo.fr', 'yahoo.ciom': 'yahoo.com', 'yahoo.vom': 'yahoo.com', 95 | 'yahoo.c': 'yahoo.com', 'yahoo.cim': 'yahoo.com', 'yahoo.co': 'yahoo.com', 96 | 'yaoo.col': 'yahoo.com', 'yahoo.colm': 'yahoo.com', 'yahoo.con': 'yahoo.com', 97 | 98 | # sfr.fr variants 99 | 'sfr.fre': 'sfr.fr', 'sfr.frr': 'sfr.fr', 'sfr': 'sfr.fr', 100 | 'sfr.fe': 'sfr.fr', 'sfr.ff': 'sfr.fr', 101 | 102 | # hotmail variants 103 | 'homail.fr': 'hotmail.fr', 'hotail.fr': 'hotmail.fr', 'hotmail.fe': 'hotmail.fr', 104 | 'hotmail.ff': 'hotmail.fr', 'homail.com': 'hotmail.com', 'hotail.com': 'hotmail.com', 105 | 'hotmail.ciom': 'hotmail.com', 'hotmail.vom': 'hotmail.com', 'hotmail.c': 'hotmail.com', 106 | 'hotmail.cim': 'hotmail.com', 'hotmail.co': 'hotmail.com', 'hotmail.col': 'hotmail.com', 107 | 'hotmail.colm': 'hotmail.com', 'hotmail.con': 'hotmail.com', 108 | 109 | # live.com variants 110 | 'life.fr': 'live.fr', 'live.frr': 'live.fr', 'live.fe': 'live.fr', 111 | 'live.ff': 'live.fr', 'life.com': 'live.com', 'live.ciom': 'live.com', 112 | 'live.vom': 'live.com', 'live.c': 'live.com', 'live.cim': 'live.com', 113 | 'live.co': 'live.com', 'live.col': 'live.com', 'live.colm': 'live.com', 114 | 'live.con': 'live.com', 115 | 116 | # outlook variants 117 | 'outlok.fr': 'outlook.fr', 'outlok.com': 'outlook.com', 'outlook.ciom': 'outlook.com', 118 | 'outlook.vom': 'outlook.com', 'outlook.c': 'outlook.com', 'outlook.cim': 'outlook.com', 119 | 'outlook.co': 'outlook.com', 'outlook.col': 'outlook.com', 'outlook.colm': 'outlook.com', 120 | 'outlook.con': 'outlook.com', 'outhlook.fr': 'outlook.fr', 'outhlook.com': 'outlook.com', 121 | 122 | # gmail.com variants 123 | 'gmail.fr': 'gmail.com', 'gmal.fr': 'gmail.com', 'gmail.frr': 'gmail.com', 124 | 'gmail': 'gmail.com', 'gmail.ciom': 'gmail.com', 'g.mail': 'gmail.com', 125 | 'gemail.com': 'gmail.com', 'galml.com': 'gmail.com', 'gmail.c': 'gmail.com', 126 | 'gmail.': 'gmail.com', 'gmail.cim': 'gmail.com', 'gmail.clm': 'gmail.com', 127 | 'gmail.co': 'gmail.com', 'gmail.col': 'gmail.com', 'gmail.comp': 'gmail.com', 128 | 'gmail.con': 'gmail.com', 'gmail.cpm': 'gmail.com', 'gmail.de': 'gmail.com', 129 | 'gmail.dk': 'gmail.com', 'gmail.es': 'gmail.com', 'gmail.org': 'gmail.com', 130 | 'gmail.vom': 'gmail.com', 'gmaill.com': 'gmail.com', 'gmal.com': 'gmail.com', 131 | 'gmeil.com': 'gmail.com', 'gmail.colm': 'gmail.com', 132 | 133 | # googlemail.com variants 134 | 'googlemail.con': 'googlemail.com', 'googlemail.cpm': 'googlemail.com', 135 | 'googlemail.de': 'googlemail.com', 'googlemail.fr': 'googlemail.com', 136 | 'googlemail.co.uk': 'googlemail.com', 'googlemail.es': 'googlemail.com', 137 | 'googlemail.dk': 'googlemail.com', 'googlemail.vom': 'googlemail.com', 138 | 'googlemail.c': 'googlemail.com', 'googlemail.cim': 'googlemail.com', 139 | 'googlemail.co': 'googlemail.com', 'googlemail.col': 'googlemail.com', 140 | 'googlemail.colm': 'googlemail.com', 141 | 142 | # aliceadsl.fr variants 143 | 'alice.fr': 'aliceadsl.fr', 'aliseadsl.fr': 'aliceadsl.fr', 'aliceadsl.com': 'aliceadsl.fr', 144 | 'aliceadsl.frr': 'aliceadsl.fr', 'aliceadsl': 'aliceadsl.fr', 'aliceadsl.fe': 'aliceadsl.fr', 145 | 'aliceadsl.ff': 'aliceadsl.fr', 146 | 147 | # voila.fr variants 148 | 'voila.com': 'voila.fr', 'voila.frr': 'voila.fr', 'voila': 'voila.fr', 149 | 'voila.fe': 'voila.fr', 'voila.ff': 'voila.fr', 150 | 151 | # skynet.be variants 152 | 'skynet.bee': 'skynet.be', 'skynet': 'skynet.be', 153 | 154 | # aol variants 155 | 'aol.ciom': 'aol.com', 'aol.vom': 'aol.com', 'aol.c': 'aol.com', 156 | 'aol.cim': 'aol.com', 'aol.co': 'aol.com', 'aol.col': 'aol.com', 157 | 'aol.colm': 'aol.com', 'aol.con': 'aol.com', 158 | 159 | # icloud variants 160 | 'iclou.com': 'icloud.com', 'icloud.con': 'icloud.com', 'icloud.cim': 'icloud.com', 161 | 'icloud.co': 'icloud.com', 'icloud.vom': 'icloud.com', 'iclod.com': 'icloud.com', 162 | 163 | # protonmail variants 164 | 'protonmail.ch': 'protonmail.com', 'protonmai.com': 'protonmail.com', 165 | 'protonmail.con': 'protonmail.com', 'protonmail.co': 'protonmail.com', 166 | 167 | # msn variants 168 | 'msn.con': 'msn.com', 'msn.co': 'msn.com', 'msn.vom': 'msn.com', 169 | 170 | # numericable/bouygues variants 171 | 'numericable.com': 'numericable.fr', 'bouygues.com': 'bbox.fr', 172 | 'bouyguestel.fr': 'bbox.fr', 'bouygtel.fr': 'bbox.fr', 173 | 174 | # Common TLD typos (.cmo, .cpm, .con, .net instead of .com, .co.uk, etc.) 175 | 'yahoo.cmo': 'yahoo.com', 'hotmail.cmo': 'hotmail.com', 'gmail.cmo': 'gmail.com', 176 | 177 | # Common reversed/transposed letters 178 | 'gail.com': 'gmail.com', 'gamil.com': 'gmail.com', 'gmial.com': 'gmail.com', 179 | 180 | # Missing dots 181 | 'hotmailcom': 'hotmail.com', 'gmailcom': 'gmail.com', 'yahoocom': 'yahoo.com', 182 | 'outlookcom': 'outlook.com', 'livecom': 'live.com', 183 | 184 | # Common .uk typos 185 | 'yahoo.uk': 'yahoo.co.uk', 'hotmail.uk': 'hotmail.co.uk', 186 | 'gmail.uk': 'gmail.com', # gmail.co.uk doesn't exist 187 | 'outlook.uk': 'outlook.com', # outlook.co.uk redirects to .com 188 | 189 | # Common .ca/.au typos for US services 190 | 'gmail.ca': 'gmail.com', 'gmail.au': 'gmail.com', 191 | 'hotmail.ca': 'hotmail.com', 'hotmail.au': 'hotmail.com', 192 | 193 | # Spaces in domain (will be caught by RFC but good to have) 194 | 'hot mail.com': 'hotmail.com', 'g mail.com': 'gmail.com', 195 | 196 | # Missing hyphen or underscore 197 | 'club internet.fr': 'club-internet.fr', 198 | 199 | # ISP variants (France) 200 | 'nordnet.com': 'nordnet.fr', 'nordnet.net': 'nordnet.fr', 201 | 'cegetel.com': 'cegetel.net', 'cegetel.fr': 'cegetel.net', 202 | 'bbox.com': 'bbox.fr', 203 | 204 | # Common international providers 205 | 'mail.ru': 'mail.ru', # Valid but include for completeness 206 | 'ya.ru': 'yandex.ru', 'yandex.com': 'yandex.ru', 207 | 'web.de': 'web.de', # Valid German provider 208 | 'gmx.com': 'gmx.com', # Valid 209 | 'gmx.de': 'gmx.de', # Valid 210 | 211 | # Gouvermement fr 212 | 'gouv.frr': 'gouv.fr', 'gouv.fe': 'gouv.fr', 'gouv.ff': 'gouv.fr', 213 | 'gouv.f': 'gouv.fr', 214 | } 215 | 216 | # Ambiguous email patterns 217 | AMBIGUOUS_PATTERNS = re.compile( 218 | r'(^|@)(test|example|exemple|spam|noreply|no-reply)(@|\.|$)', 219 | re.IGNORECASE 220 | ) 221 | 222 | # RFC822 email regex (simplified but practical) 223 | RFC822_PATTERN = re.compile( 224 | r"^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*" 225 | r"@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?$" 226 | ) 227 | 228 | SPLIT_PATTERN = re.compile(r'[;,]') 229 | EXTRACT_PATTERN = re.compile(r'[\w.+-]+@[\w-]+\.[\w.-]+') 230 | 231 | ###################################################################################################### 232 | # Functions 233 | ###################################################################################################### 234 | 235 | class EmailStats: 236 | """Track email processing statistics""" 237 | def __init__(self): 238 | self.missing = 0 239 | self.incorrect_mails = 0 240 | self.incorrect_domains = 0 241 | self.incorrect_mx = 0 242 | self.ambiguous = 0 243 | self.valid = 0 244 | 245 | def print_stats(self): 246 | print(f"\n{self.missing} missing emails found.") 247 | print(f"{self.incorrect_mails} non RFC822 compliant emails found.") 248 | print(f"{self.incorrect_domains} emails had incorrect domains and have been corrected.") 249 | print(f"{self.incorrect_mx} emails are missing MX records in their domain.") 250 | print(f"{self.ambiguous} ambiguous emails found.") 251 | print(f"{self.valid} emails seem valid.") 252 | 253 | 254 | def lowercase(s): 255 | """Convert string to lowercase""" 256 | return s.lower() if s else s 257 | 258 | 259 | def check_rfc822(email): 260 | """Check if email is RFC822 compliant""" 261 | if not email: 262 | return False 263 | return RFC822_PATTERN.match(email.lower()) is not None 264 | 265 | 266 | def check_domain(email): 267 | """ 268 | Check and fix known domain typos using dictionary lookup (O(1) instead of O(n)) 269 | Returns: (corrected_email, was_corrected) 270 | """ 271 | if not email or '@' not in email: 272 | return email, False 273 | 274 | username, domain = email.rsplit('@', 1) 275 | domain_lower = domain.lower() 276 | 277 | if domain_lower in INVALID_TO_VALID_DOMAINS: 278 | corrected_domain = INVALID_TO_VALID_DOMAINS[domain_lower] 279 | return f"{username}@{corrected_domain}", True 280 | 281 | return email, False 282 | 283 | 284 | @lru_cache(maxsize=DNS_CACHE_SIZE) 285 | def check_mx_domains(domain): 286 | """ 287 | Check if domain has valid MX records 288 | Uses LRU cache to avoid repeated DNS lookups for the same domain 289 | We also need to make sure we only use root domain for MX checks 290 | """ 291 | 292 | try: 293 | result = subprocess.run( 294 | ['dig', domain, 'mx', '+short'], 295 | capture_output=True, 296 | text=True, 297 | timeout=5 298 | ) 299 | # Check if we got any MX records 300 | return len(result.stdout.strip()) > 0 301 | except (subprocess.TimeoutExpired, FileNotFoundError): 302 | # If dig fails or times out, assume valid to avoid false positives 303 | return True 304 | 305 | 306 | def check_ambiguous(email): 307 | """Check if email appears to be ambiguous/test email""" 308 | if not email: 309 | return False 310 | return AMBIGUOUS_PATTERNS.search(email.lower()) is not None 311 | 312 | 313 | def check_environment(): 314 | """Verify required tools and internet connectivity""" 315 | # Check for dig 316 | try: 317 | subprocess.run(['dig', '-v'], capture_output=True, timeout=2) 318 | except FileNotFoundError: 319 | print("ERROR: This script needs 'dig' to resolve MX records.") 320 | print("Install it using: apt install dnsutils (Debian/Ubuntu) or dnf install bind-utils (RHEL/Fedora)") 321 | sys.exit(1) 322 | 323 | # Check internet connectivity 324 | print("Checking for internet access...") 325 | try: 326 | result = subprocess.run( 327 | ['ping', '-c', '3', INET_ADDR_TO_TEST] if sys.platform != 'win32' 328 | else ['ping', '-n', '3', INET_ADDR_TO_TEST], 329 | capture_output=True, 330 | timeout=10 331 | ) 332 | if result.returncode != 0: 333 | print("ERROR: This script needs internet to resolve MX records.") 334 | sys.exit(1) 335 | except subprocess.TimeoutExpired: 336 | print("ERROR: Internet connectivity check timed out.") 337 | sys.exit(1) 338 | 339 | 340 | def add_to_text(text, addition, separator=STATE_COLUMN_SEPARATOR): 341 | """Helper to add text with newline""" 342 | if text: 343 | return f"{text}{separator}{addition}" 344 | return addition 345 | 346 | 347 | def process_emails(input_file, output_file): 348 | """Process email file and validate/correct emails""" 349 | stats = EmailStats() 350 | 351 | input_path = Path(input_file) 352 | if not input_path.exists(): 353 | print(f"ERROR: No such file: {input_file}") 354 | sys.exit(1) 355 | 356 | print("Checking emails...") 357 | 358 | count = 0 359 | with open(input_file, 'r', encoding='utf-8') as infile, \ 360 | open(output_file, 'w', encoding='utf-8', newline='') as outfile: 361 | 362 | # Read first line to determine if it's a header or data 363 | first_line = infile.readline() 364 | # Remove BOM if present 365 | if first_line.startswith('\ufeff'): 366 | first_line = first_line.lstrip('\ufeff') 367 | 368 | if INPUT_FILE_HAS_HEADER: 369 | # First line is header - use it for fieldnames 370 | fieldnames = first_line.strip().split(CSV_INPUT_DELIMITER) 371 | reader = csv.DictReader(infile, fieldnames=fieldnames, delimiter=CSV_INPUT_DELIMITER) 372 | else: 373 | # First line is data - use configured columns and rewind 374 | infile.seek(0) 375 | reader = csv.DictReader(infile, fieldnames=CSV_COLUMNS, delimiter=CSV_INPUT_DELIMITER) 376 | 377 | # Prepare writer with STATE column 378 | output_columns = ['STATE'] + list(reader.fieldnames) 379 | writer = csv.DictWriter(outfile, fieldnames=output_columns, delimiter=CSV_INPUT_DELIMITER, extrasaction='ignore') 380 | writer.writeheader() 381 | 382 | # Verify EMAIL_COLUMN exists 383 | for column in EMAIL_COLUMNS: 384 | if column not in reader.fieldnames: 385 | print(f"ERROR: Email column '{column}' not found in CSV.") 386 | print(f"Available columns: {', '.join(reader.fieldnames)}") 387 | sys.exit(1) 388 | 389 | for row in reader: 390 | count += 1 391 | row_state = "" 392 | 393 | # we might have multiple email columns to check in a single row 394 | col_state = "" 395 | for email_column in EMAIL_COLUMNS: 396 | # Strip surrounding quotes and spaces 397 | email = row.get(email_column, '').strip("\"' ") 398 | 399 | # Check for missing email 400 | if not email: 401 | stats.missing += 1 402 | col_state = add_to_text(col_state, EM_MISSING) 403 | continue 404 | 405 | # We might have an email column with multiple entries separated by MULTI_EMAIL_SEPARATOR 406 | em_state = "" 407 | new_email_col = "" 408 | continue_checks = True 409 | for em in re.findall(EXTRACT_PATTERN, email): 410 | 411 | # Lowercase the email 412 | em = lowercase(em) 413 | 414 | # Check and fix domain typos 415 | corrected_email, was_corrected = check_domain(em) 416 | if was_corrected: 417 | stats.incorrect_domains += 1 418 | em = corrected_email 419 | em_state = add_to_text(em_state, EM_FIXED, MULTI_EMAIL_SEPARATOR) 420 | 421 | # Check RFC822 compliance 422 | if not check_rfc822(em): 423 | stats.incorrect_mails += 1 424 | em_state = add_to_text(em_state, EM_NON_RFC, MULTI_EMAIL_SEPARATOR) 425 | continue_checks = False 426 | 427 | if continue_checks: 428 | # Check MX records 429 | domain = em.split('@')[1] 430 | if not check_mx_domains(domain): 431 | stats.incorrect_mx += 1 432 | em_state = add_to_text(em_state, EM_NO_MX, MULTI_EMAIL_SEPARATOR) 433 | continue_checks = False 434 | 435 | if continue_checks: 436 | # Check for ambiguous emails 437 | if check_ambiguous(em): 438 | stats.ambiguous += 1 439 | em_state = add_to_text(em_state, EM_AMBIGUOUS, MULTI_EMAIL_SEPARATOR) 440 | continue_checks = False 441 | 442 | # Email is valid 443 | stats.valid += 1 444 | # Set final state for this email addr 445 | if continue_checks: 446 | em_state = add_to_text(em_state, EM_VALID, MULTI_EMAIL_SEPARATOR) 447 | new_email_col = add_to_text(new_email_col, em , MULTI_EMAIL_SEPARATOR) 448 | 449 | col_state = add_to_text(col_state, em_state) 450 | row[email_column] = new_email_col 451 | 452 | row_state = add_to_text(row_state, col_state, STATE_COLUMN_SEPARATOR) 453 | 454 | row['STATE'] = row_state 455 | writer.writerow(row) 456 | # Progress update 457 | if count % 50 == 0: 458 | print(f".", end='', flush=True) 459 | 460 | stats.print_stats() 461 | print(f"\nCurated file saved to: {output_file}") 462 | 463 | # Print cache statistics 464 | cache_info = check_mx_domains.cache_info() 465 | print(f"\nDNS Cache Statistics:") 466 | print(f" Hits: {cache_info.hits}") 467 | print(f" Misses: {cache_info.misses}") 468 | print(f" Cache size: {cache_info.currsize}/{cache_info.maxsize}") 469 | if cache_info.hits + cache_info.misses > 0: 470 | hit_rate = cache_info.hits / (cache_info.hits + cache_info.misses) * 100 471 | print(f" Hit rate: {hit_rate:.1f}%") 472 | 473 | 474 | def usage(): 475 | """Print usage information""" 476 | print(f"emailCheck.py v{PROGRAM_VERSION} (build {PROGRAM_BUILD})") 477 | print("(L) 2014-2025 by Orsiris de Jong") 478 | print("http://www.netpower.fr/ - ozy@netpower.fr") 479 | print() 480 | print("Usage: python emailCheck.py /path/to/emailList") 481 | print("Email list needs to be a CSV file encoded in UTF-8.") 482 | print("Checks if emails are RFC822 valid, corrects known typos in domain names,") 483 | print("checks for valid MX records (with caching), and checks against known ambiguous mail addresses.") 484 | print() 485 | print("Outputs one curated file with a STATE column indicating the validation result.") 486 | sys.exit(1) 487 | 488 | 489 | def main(): 490 | """Main entry point""" 491 | if len(sys.argv) != 2: 492 | usage() 493 | 494 | input_file = sys.argv[1] 495 | 496 | # Check environment 497 | check_environment() 498 | 499 | # Generate output filename 500 | input_path = Path(input_file) 501 | output_file = input_path.parent / f"curated-{input_path.name}" 502 | 503 | # Process emails 504 | print(f"Processing file: {input_file}") 505 | process_emails(input_file, output_file) 506 | 507 | 508 | if __name__ == "__main__": 509 | main() 510 | --------------------------------------------------------------------------------