├── .gitignore ├── slurm-parallel ├── exa1.slurm ├── exa6.slurm ├── exa5.slurm ├── exa4.slurm ├── exa3.slurm └── scriptfile.sh ├── lsd ├── .github ├── weekly-digest.yml └── workflows │ └── blank.yml ├── pscode.sh ├── reniceproc ├── slurm-submit-jobs ├── 00-checkfile-for-01.sh ├── demo.py ├── 01-check-process-broken.slurm ├── test.slurm ├── 02-check-notrun.sh ├── 03-create-split-slurm.sh └── test-xargs.slurm ├── forward.sh ├── changeOwner ├── getjupyter ├── findr ├── killparallel ├── match-data.sh ├── limituser ├── at-schedule-work.sh ├── load-docker-image ├── v2r-deployment.sh ├── cron-mysql.sh ├── save-docker-image ├── download-sra.sh ├── vss ├── README.md ├── download-ebi.sh ├── my-backup-database.sh ├── convertID.R ├── vsc-tunnel-gpu.slrm ├── .vscode └── settings.json ├── download-annovar.sh ├── download-gatk-ftp.sh ├── vsc-tunnel-cpu.slrm ├── geo-lftp-mirror-folder.sh ├── ss-deployment.sh ├── backup-databases.sh ├── my-backup-scripts.sh ├── stat-website-from-apache-log.sh ├── backup-webserver-scripts.sh ├── get-sequence-by-mir-name.py ├── slrm.sh ├── jupyter.slurm ├── download-gsm.py ├── extractOriginalBam2RegionalBam2.sh ├── sortChromosomeAndPosition.py ├── gdc-api-manifest-httr.R ├── generalParallelSlurm ├── getEnsFromEntrezID.py ├── plot-access-by-time.R ├── findMotifByHand.py ├── gdc-download.sh ├── find-gene-miRNA-by-symbol.py ├── jrc ├── find-miRNA-target.py ├── plot-access-by-address.R ├── mapFileUUID2submitterID.py ├── rstudio.slurm ├── crontab-backup-database-and-web.sh ├── ncbi-script ├── get_gene_summary_README ├── geneGoSummary.pl ├── taxidToGeneNames.pl └── geneDocSum.pl ├── jrocker.slurm ├── jukeras ├── generalParallel ├── jrocker ├── cs ├── hust-references.csl ├── ss-bbr.sh └── ss.sh /.gitignore: -------------------------------------------------------------------------------- 1 | nohup.out 2 | -------------------------------------------------------------------------------- /slurm-parallel/exa1.slurm: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lsd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ $# -eq 0 ]] 4 | then 5 | cmd="realpath *" 6 | # echo ${cmd} 7 | eval ${cmd} 8 | exit 1 9 | fi 10 | 11 | cmd="ls|grep -i ${1}|xargs realpath" 12 | # echo ${cmd} 13 | eval ${cmd} 14 | 15 | 16 | -------------------------------------------------------------------------------- /.github/weekly-digest.yml: -------------------------------------------------------------------------------- 1 | # Configuration for weekly-digest - https://github.com/apps/weekly-digest 2 | publishDay: sun 3 | canPublishIssues: true 4 | canPublishPullRequests: true 5 | canPublishContributors: true 6 | canPublishStargazers: true 7 | canPublishCommits: true -------------------------------------------------------------------------------- /pscode.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2021-01-27 10:40:46 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | param=$# 9 | 10 | function fn_usage {} 11 | 12 | function main {} -------------------------------------------------------------------------------- /reniceproc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | param=$# 4 | 5 | [[ ${param} -ne 1 ]] && { 6 | echo "Error: Input process to prioritize." 7 | echo " Example: killparallel gdc-client." 8 | exit 1 9 | } 10 | 11 | name=$1 12 | cmd="ps -fu liucj |grep -iE \"${name}\"|grep -v grep |awk '{print \$2}'|xargs renice -19 {}" 13 | echo $cmd 14 | # eval $cmd -------------------------------------------------------------------------------- /slurm-submit-jobs/00-checkfile-for-01.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mir_dir=/public1/home/pg2204/target_predit/miRmap_02 4 | altutr_out=${mir_dir}/altutr_out 5 | altutr_out_complet=${mir_dir}/altutr_out_complet 6 | 7 | filename=$1 8 | 9 | nline="grep '>' ${filename}|wc -l" 10 | [[ ${nline} -eq 8935410 ]] && mv ${filename} ${altutr_out_complet} 11 | -------------------------------------------------------------------------------- /forward.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2020-05-17 10:46:50 5 | # @DESCRIPTION: 6 | 7 | errout_dir="/home/liuc9/tmp/errout/jrocker" 8 | 9 | for j in $(ls ${errout_dir}/j*er.job.*); do 10 | suffix=$(basename ${j}) 11 | cmd="grep -i 'ssh -N -L' ${j}" 12 | out=$(eval ${cmd}) 13 | echo "${out} & # ${suffix}" 14 | done 15 | -------------------------------------------------------------------------------- /.github/workflows/blank.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Run a one-line script 13 | run: echo Hello, world! 14 | - name: Run a multi-line script 15 | run: | 16 | echo Add other actions to build, 17 | echo test, and deploy your project. 18 | -------------------------------------------------------------------------------- /changeOwner: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | for i in `ls /home`; 4 | do 5 | idi=`id ${i}` 6 | [[ ${idi} =~ 'uid' ]] && { 7 | user=`echo ${idi}|sed -n 's/uid=[[:digit:]]*(\([[:alpha:]]*\)) .*/\1/p'` 8 | grp=`echo ${idi}|sed -n 's/.* gid=[[:digit:]]*(\([[:alpha:]]*\)) .*/\1/p'` 9 | # echo ${user} ${grp} 10 | cmd="chown -R ${user}.${grp} /home/${i} &" 11 | # echo ${cmd} 12 | eval ${cmd} 13 | } 14 | done -------------------------------------------------------------------------------- /slurm-parallel/exa6.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2021-12-30 11:25:37 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | 9 | #SBATCH --job-name=TestJob 10 | #SBATCH --nodes=2 11 | #SBATCH --ntasks=4 12 | #SBATCH --ntasks-per-node=2 13 | #SBATCH --cpus-per-task=1 14 | 15 | hostname & 16 | hostname & 17 | hostname & 18 | hostname & 19 | wait -------------------------------------------------------------------------------- /slurm-submit-jobs/demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # demo.py: 3 | # usage: python demo.py [job-number] 4 | 5 | import sys 6 | import socket 7 | from time import sleep 8 | 9 | 10 | def work(jobnum): 11 | print("Starting job {} on {}.".format(jobnum, socket.gethostname())) 12 | sleep(5) 13 | print("Finished job {}...\n".format(jobnum)) 14 | 15 | 16 | if __name__ == "__main__": 17 | jobnum = sys.argv[1] 18 | work(jobnum) 19 | 20 | -------------------------------------------------------------------------------- /slurm-parallel/exa5.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2021-12-30 11:25:37 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | 9 | #SBATCH --job-name=TestJob 10 | #SBATCH --nodes=3 11 | #SBATCH --ntasks=3 12 | #SBATCH --ntasks-per-node=1 13 | #SBATCH --cpus-per-task=5 14 | 15 | 16 | /home/liuc9/github/useful-scripts/generalParallel /home/liuc9/github/useful-scripts/slurm-parallel/scriptfile.sh 500 -------------------------------------------------------------------------------- /getjupyter: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2023-05-12 21:09:24 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | errout_dir="/home/liuc9/tmp/errout/jrocker" 9 | 10 | for j in $(ls ${errout_dir}/jupyter.*); do 11 | cmd="grep -i 'lab?token' ${j}|grep -v ServerApp|sed -e s/or//|sed 's/ //g'|sed -e s/127.0.0.1/localhost/g" 12 | # echo ${cmd} 13 | out=$(eval ${cmd}) 14 | echo "${out}" 15 | done 16 | -------------------------------------------------------------------------------- /findr: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2020-05-17 10:46:50 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | param=$1 9 | 10 | find ${HOME}/github/autophagy-in-cancer \ 11 | ${HOME}/github/TEP-OV \ 12 | ${HOME}/github/EVAtlas-analysis \ 13 | ${HOME}/github/Deeplatelet \ 14 | ${HOME}/github/learning-codes \ 15 | ${HOME}/github/sqtl \ 16 | -type f \ 17 | -name '*R' | xargs grep -i ${param} -------------------------------------------------------------------------------- /slurm-submit-jobs/01-check-process-broken.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | #SBATCH -p v2_all 5 | #SBATCH -N 2 6 | #SBATCH -n 40 7 | #SBATCH --job-name=check-result 8 | 9 | mir_dir=/public1/home/pg2204/target_predit/miRmap_02 10 | altutr_out=${mir_dir}/altutr_out 11 | altutr_out_complet=${mir_dir}/altutr_out_complet 12 | 13 | 14 | 15 | for i in `ls ${altutr_out}/*res` 16 | do 17 | srun --exclusive --nodes 1 --ntasks 1 bash 00-checkfile-for-01.sh ${i} & 18 | done 19 | 20 | wait 21 | 22 | 23 | -------------------------------------------------------------------------------- /killparallel: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: killparallel.sh 3 | # Author: C.J. Liu 4 | # Mail: chunjie.sam.liu@gmail.com 5 | # Created Time: Thu 19 Jan 2017 04:53:24 PM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | [ "$#" -eq 0 ] && echo "Error: Input process to kill" && echo "Example: killparallel gdc-client " && exit 1 10 | name=$1 11 | cmd="ps aux |grep -iE \"${name}\"|grep -v grep |awk '{print \$2}'|parallel kill -9" 12 | echo $cmd 13 | eval $cmd 14 | 15 | -------------------------------------------------------------------------------- /slurm-parallel/exa4.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2021-12-30 11:25:37 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | 9 | #SBATCH --job-name=TestJob 10 | #SBATCH --nodes=2 11 | #SBATCH --ntasks=4 12 | #SBATCH --ntasks-per-node=2 13 | 14 | sr="srun --exclusive --nodes 1 --ntasks 1" 15 | 16 | 17 | while read line; 18 | do 19 | # echo $line 20 | cmd="${sr} ${line} &" 21 | echo ${cmd} 22 | # srun --exclusive --nodes 1 --ntasks 1 ${line} & 23 | eval ${cmd} 24 | done < scriptfile.sh 25 | 26 | wait 27 | -------------------------------------------------------------------------------- /match-data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | htseq_file='/workspace/liucj/tongji-data-backup/htseq_count800.mod' 4 | dir_root='/workspace/liucj/tongji-data-backup' 5 | dir_target=${dir_root}/cj_tidy_data 6 | 7 | # cat ${htseq_file}|while read line; do 8 | # find ${dir_root} -type d -name $line 9 | # done > ${htseq_file}.find 10 | 11 | cat ${htseq_file}.find|while read line; do 12 | cmd="cp -r ${line} ${dir_target}" 13 | echo ${cmd} 14 | done > /tmp/cj-cp-target-dir.sh 15 | 16 | nohup bash /home/liucj/scripts/generalParallel /tmp/cj-cp-target-dir.sh 20 1>cj-cp-target-dir.sh.log 2>cj-cp-target-dir.sh.err & 17 | -------------------------------------------------------------------------------- /limituser: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | user=${1:-sb} 4 | 5 | iduser=$(id -u ${user}) 6 | 7 | [[ -z ${iduser} ]] && { echo "Error: no user ${user} in the passwd."; exit 1; } 8 | [[ ${user} == "liucj" ]] && { echo "Error: root user"; exit 1; } 9 | [[ ${user} == "root" ]] && { echo "Error: root user"; exit 1; } 10 | 11 | # ps -u ${user} -o pcpu,pid,user,args --no-headers|awk '$1 > 200'|sort -t. -nk1,2 -k4,4 -r 12 | pids=`ps -u ${user} -o pcpu,pid,user,args --no-headers|sort -t. -nk1,2 -k4,4 -r|awk '{print $2}'` 13 | 14 | for p in ${pids[@]} 15 | do 16 | cmd="cpulimit -b -c 1 -l 20 -p ${p}" 17 | eval ${cmd} 18 | done 19 | -------------------------------------------------------------------------------- /slurm-submit-jobs/test.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #SBATCH --partition=pg2_128_pool 4 | #SBATCH --nodes=2 5 | #SBATCH --ntasks=40 6 | #SBATCH --ntasks-per-node=20 7 | #SBATCH --cpus-per-task=1 8 | #SBATCH --time=00:01:00 9 | #SBATCH --job-name=test.slurm 10 | #SBATCH --output=test.slurm-%j.out 11 | #SBATCH --error=test.slurm-%j.err 12 | #SBATCH --mail-user=chunjie-sam-liu@foxmail.com 13 | #SBATCH --mail-type=end 14 | 15 | module load anaconda3/3.5.2 16 | 17 | 18 | for i in $(seq -w 1 40) 19 | do 20 | srun --exclusive --nodes 1 --ntasks 1 python demo.py ${i} & 21 | #python demo.py ${i} & 22 | done 23 | 24 | wait 25 | 26 | -------------------------------------------------------------------------------- /at-schedule-work.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2019-09-17 21:33:21 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | 9 | # test schedule 10 | # echo "/usr/sbin/service apache2 stop" | at -m 21:52 2019-09-17 11 | # echo "/usr/sbin/service apache2 start" | at -m 21:54 2019-09-17 12 | 13 | # schedule one time process for stop apache2 at 06:00 2019-09-18 14 | echo "/usr/sbin/service apache2 stop" | at -m 06:00 2019-09-18 15 | # schedule one time process for start apache2 at 14:00 2019-09-20 16 | echo "/usr/sbin/service apache2 start" | at -m 14:00 2019-09-20 -------------------------------------------------------------------------------- /load-docker-image: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2019-07-12 21:23:43 5 | # @DESCRIPTION: load docker form tar.gz 6 | 7 | 8 | # imgs=(python:2.7 ubuntu:16.04 broadinstitute/gatk:4.0.10.0 broadinstitute/gatk:4.0.8.1 ensemblorg/ensembl-vep:release_94.5 broadinstitute/genomes-in-the-cloud:2.3.1-1512499786 broadinstitute/oncotator:1.9.9.0) 9 | 10 | imgs=(chunjiesamliu/jrocker:latest chunjiesamliu/jukeras:latest) 11 | dir_scripts=/home/liucj/scripts 12 | 13 | for img in ${imgs[@]}; 14 | do 15 | tar_name=`echo ${img}|sed -e 's/\//_/'` 16 | cmd="nohup docker load --input ${dir_scripts}/${tar_name}.tar > ${dir_scripts}/${tar_name}.nohup.out 2>&1 &" 17 | echo ${cmd} 18 | eval ${cmd} 19 | done 20 | 21 | -------------------------------------------------------------------------------- /v2r-deployment.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # thanks to wulabing original repo. 4 | 5 | 6 | # one key 7 | wget -N --no-check-certificate -q -O install.sh "https://raw.githubusercontent.com/chunjie-sam-liu/V2Ray-onekey/master/install.sh" 8 | chmod +x install.sh 9 | bash install.sh 10 | 11 | ## 11 bbr accelerating scripts 12 | ### 2 bbr plus core with and restart vps 13 | 14 | #----------------------------------- 15 | # run install.sh again. 16 | bash install.sh 17 | 18 | ## 11 bbr accelerating scripts 19 | ### 7 bbr plus accelerating 20 | 21 | #----------------------------------- 22 | # run install.sh again. 23 | bash install.sh 24 | 25 | ## 1 install v2ray (Nginx+ws+tls) 26 | ### input domain (subdomain.domain.xxx) 27 | ### connecting port: default 443 28 | ### alterID: 55 -------------------------------------------------------------------------------- /cron-mysql.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: cron-mysql.sh 3 | # Author: C.J. Liu 4 | # Mail: samliu@hust.edu.cn 5 | # Created Time: Fri 23 Feb 2018 10:47:16 AM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | # */30 * * * * /home/liucj/scripts/cron-mysql.sh 10 | #!/bin/bash 11 | PATH=/usr/sbin:/usr/bin:/sbin:/bin 12 | SHELL=/bin/bash 13 | LOG=/home/liucj/scripts/.cron.log 14 | 15 | [[ -f $LOG ]] || touch $LOG 16 | 17 | echo "`date` Check MySQL Status" >> $LOG 18 | 19 | if [[ ! "$(/usr/sbin/service mysql status)" =~ "running" ]] 20 | then 21 | echo "Warnging: MySQL Stop at `date`" >> $LOG 22 | service mysql start 23 | echo "Success: MySQL Restarted" >> $LOG 24 | fi 25 | -------------------------------------------------------------------------------- /save-docker-image: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2019-07-12 21:21:53 5 | # @DESCRIPTION: 6 | 7 | 8 | # imgs=(python:2.7 ubuntu:16.04 broadinstitute/gatk:4.0.10.0 broadinstitute/gatk:4.0.8.1 ensemblorg/ensembl-vep:release_94.5 broadinstitute/genomes-in-the-cloud:2.3.1-1512499786 broadinstitute/oncotator:1.9.9.0) 9 | dir_scripts=/home/liucj/scripts 10 | imgs=(chunjiesamliu/jrocker:latest chunjiesamliu/jukeras:latest) 11 | 12 | for img in ${imgs[@]}; 13 | do 14 | tar_name=`echo ${img}|sed -e 's/\//_/'` 15 | { 16 | docker save --output ${dir_scripts}/${tar_name}.tar ${img} 17 | cs cp 10 ${dir_scripts}/${tar_name}.tar ${dir_scripts} & 18 | cs cp 1 ${dir_scripts}/${tar_name}.tar ${dir_scripts} & 19 | } > ${dir_scripts}/${tar_name}.nohup.out 2>&1 & 20 | done 21 | -------------------------------------------------------------------------------- /download-sra.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: downloadsra.sh 3 | # Author: C.J. Liu 4 | # Mail: samliu@hust.edu.cn 5 | # Created Time: Mon 28 Sep 2015 09:23:24 AM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | #download data from sra through lftp 10 | #input srr id with comma delimited 11 | 12 | 13 | [[ "$#" -eq 2 ]] && destdir=$2 || destdir=$PWD 14 | 15 | if test -z "$1" 16 | then 17 | echo "Error: input srr id list with comma delimited" 18 | exit 1 19 | fi 20 | 21 | sraList=$1 22 | sraList=(${sraList//,/ }) 23 | 24 | for i in ${sraList[@]} 25 | do 26 | dir=${i:0:6} 27 | nohup lftp -e "mirror /sra/sra-instant/reads/ByRun/sra/SRR/${dir}/${i} ${destdir}" ftp-trace.ncbi.nih.gov>${destdir}/${i}.nohup.out& 28 | done 29 | 30 | 31 | -------------------------------------------------------------------------------- /vss: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # The server 1 and 3 can not download dynamically, use vss to transfer 10 bin to 1 and 3. 4 | 5 | dir_bin=/home/liucj/.vscode-server/bin 6 | old_bin_file=${dir_bin}/.old_bin 7 | 8 | [[ ! -s ${old_bin_file} ]] && { 9 | echo "Error: ${old_bin_file} does not exist." 10 | exit 1 11 | } 12 | 13 | # Old bins 14 | readarray old_bins < ${old_bin_file} 15 | 16 | # New bins 17 | new_bins=$(ls ${dir_bin}) 18 | 19 | latest_bin=$(echo ${new_bins[@]} ${old_bins[@]} | tr ' ' '\n' | sort | uniq -u) 20 | 21 | # transfer 22 | cs cp 1 ${dir_bin}/${latest_bin} ${dir_bin} > /dev/null 2>&1 23 | cs cp 2 ${dir_bin}/${latest_bin} ${dir_bin} > /dev/null 2>&1 24 | cs cp 3 ${dir_bin}/${latest_bin} ${dir_bin} > /dev/null 2>&1 25 | cs cp 4 ${dir_bin}/${latest_bin} ${dir_bin} > /dev/null 2>&1 26 | 27 | # save new bin 28 | printf "%s\n" "${new_bins[@]}" > ${dir_bin}/.old_bin -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # useful-scripts 2 | 3 | 4 | This repo is created for my [learning codes](https://github.com/chunjie-sam-liu/learning-codes). Most of scripts are created for bioinformaitc utilities, for example `download-*` scritps are used to download SRA and GSE data although this data can be downloaded through specific softwares like `fastq-dump` now. Afterwrads, some commands that I used often on Linux system, I compiled commands as `shell` or `python` scripts such as `generalParallel` and `killparallel`, and these scripts are used in my daily life. 5 | 6 | `shadowsocks-*`, `jrocker` and `jukeras` are the scripts that aimed for restoration of working environment. `shadowsocks` is the well known software for scientific network. `shadowsocks-*` can be used to deploy the vss system. `jrocker` and `jukeras` are the scripts to use my docker image of `rstudio server` and `jupyter notebook`. 7 | 8 | 9 | # github codespace -------------------------------------------------------------------------------- /download-ebi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2021-05-08 10:34:14 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | 9 | url="ftp.ebi.ac.uk" 10 | remote_dir="/pub/databases/metagenomics/mgnify_genomes/human-gut/v1.0/uhgg_catalogue" 11 | prefix="MGYG-HGUT-0" 12 | 13 | for i in `seq 0 4`; 14 | do 15 | for j in `seq 0 9`; 16 | do 17 | [[ ${i} -eq 4 && ${j} -gt 6 ]] && continue 18 | dir1=${prefix}${i}${j} 19 | for m in `seq 0 9`; 20 | do 21 | for n in `seq 0 9`; 22 | do 23 | [[ ${i}${j}${m}${n} = "0000" ]] && continue 24 | [[ ${i}${j} = "46" && ${m}${n} -gt 44 ]] && continue 25 | dir2=${dir1}${m}${n} 26 | cmd="lftp -e \"get -c ${remote_dir}/${dir1}/${dir2}/genome/${dir2}.fna\" ftp.ebi.ac.uk" 27 | echo ${cmd} 28 | done 29 | done 30 | done 31 | done -------------------------------------------------------------------------------- /my-backup-database.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: backup_scripts.sh 3 | # Author: C.J. Liu 4 | # Mail: samliu@hust.edu.cn 5 | # Created Time: Thu 15 Mar 2018 05:31:25 PM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | backup_time=`date +"%y-%m"` 10 | backup_dir="/home/liucj/tmp/web_backup" 11 | s3_backup=/home/liucj/data/web_backup 12 | databases=('lncRInter' 'lncRNAediting_fly' 'lncRNAediting_human' 'lncRNAediting_mouse' 'lncRNAediting_rhesus' 'bioguoor_miRNA_SNP' 'bioguoor_miRNA_SNP_V2' 'snoric') 13 | 14 | for i in ${databases[@]}; 15 | do 16 | backup_name=$backup_dir/${i}.${backup_time}.sql.gz 17 | cmd="mysqldump -u**** -p**** ${i} > ${backup_name}" 18 | echo $cmd 19 | eval $cmd 20 | 21 | 22 | cmd="copyto.sh 3 ${s3_backup} ${backup_name}" 23 | echo ${cmd} 24 | eval ${cmd} 25 | done 26 | -------------------------------------------------------------------------------- /convertID.R: -------------------------------------------------------------------------------- 1 | library(biomaRt) 2 | 3 | fn_mart <- function(host="useast.ensembl.org") { 4 | .mart <- biomaRt::useMart( 5 | "ENSEMBL_MART_ENSEMBL", 6 | dataset = "hsapiens_gene_ensembl", 7 | host = host 8 | ) 9 | .mart 10 | } 11 | 12 | MART <- fn_mart() 13 | 14 | fn_mart_attr <- function(.mart = MART){ 15 | biomaRt::listAttributes(.mart) 16 | } 17 | 18 | fn_mart_filter <- function(.mart = MART){ 19 | biomaRt::listFilters(.mart) 20 | } 21 | 22 | fn_convertId <- function( 23 | ids, 24 | filters = "ensembl_gene_id", 25 | attrs = c("ensembl_gene_id", "hgnc_symbol", "entrezgene_id", "chromosome_name", "start_position", "end_position", "strand", "gene_biotype", "description"), 26 | mart = MART 27 | ){ 28 | .bm <- biomaRt::getBM( 29 | values = ids, 30 | filters = filters, 31 | attributes = attrs, 32 | mart = mart 33 | ) %>% 34 | tibble::as_tibble() 35 | } 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /vsc-tunnel-gpu.slrm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #SBATCH --job-name=vsc-gpu-tunnel 3 | #SBATCH --ntasks=1 4 | #SBATCH --gres=gpu:1 5 | #SBATCH --partition=gpuq 6 | #SBATCH --mem=80G 7 | #SBATCH --time=120:00:00 8 | #SBATCH --output=/home/%u/tmp/errout/jrocker/vsc-tunnel-gpu.job.%j 9 | # customize --output path as appropriate (to a directory readable only by the user!) 10 | 11 | if [ -f ${HOME}/.bashrc ]; then 12 | source ${HOME}/.bashrc 13 | fi 14 | 15 | source ~/tools/anaconda3/bin/activate 16 | 17 | TYPE=GPU 18 | TEMPDIR=/scr1/users/liuc9/tmp/errout/vsc-tunnel 19 | MACHINENAME=$(uuidgen | cut -d"-" -f1) 20 | 21 | MACHINENAME_TYPE=${TYPE}_${MACHINENAME} 22 | DATADIR=${TEMPDIR}/${MACHINENAME_TYPE} 23 | 24 | echo $MACHINENAME_TYPE 25 | echo $DATADIR 26 | 27 | code tunnel user login --provider github && 28 | code tunnel \ 29 | --no-sleep \ 30 | --name ${MACHINENAME_TYPE} \ 31 | --accept-server-license-terms \ 32 | --cli-data-dir ${DATADIR} 33 | -------------------------------------------------------------------------------- /slurm-submit-jobs/02-check-notrun.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | mir_dir=/public1/home/pg2204/target_predit/miRmap_02 5 | 6 | altutr_out=${mir_dir}/altutr_out 7 | altutr_out_complet=${mir_dir}/altutr_out_complet 8 | 9 | split_dir=/public1/home/pg2204/src/split-not-run 10 | 11 | [[ ! -d ${split_dir} ]] && mkdir -p ${split_dir} 12 | 13 | mirbase=/public1/home/pg2204/data/mirbase.mir 14 | mirdone=/public1/home/pg2204/data/mirbase.mir.done 15 | mirnotrun=/public1/home/pg2204/data/mirbase.mir.notrun 16 | 17 | readarray mirnadone < ${mirdone} 18 | readarray mirnabase < ${mirbase} 19 | 20 | for i in `ls ${altutr_out_complet}/*res` 21 | do 22 | mirna=${i##*/} 23 | mirna=${mirna%%.res} 24 | [[ ! ${mirnadone[@]} =~ ${mirna} ]] && echo ${mirna} >> ${mirdone} 25 | done 26 | 27 | echo ${mirnabase[@]} ${mirnadone[@]} | tr ' ' '\n' | sort | uniq -u > ${mirnotrun} 28 | 29 | cd ${split_dir} 30 | split -l 100 ${mirnotrun} 31 | 32 | wc -l ${mirbase} ${mirdone} ${mirnotrun} 33 | 34 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "workbench.colorCustomizations": { 3 | "activityBar.activeBorder": "#945bc4", 4 | "activityBar.activeBackground": "#65c89b", 5 | "activityBar.background": "#65c89b", 6 | "activityBar.foreground": "#15202b", 7 | "activityBar.inactiveForeground": "#15202b99", 8 | "activityBarBadge.background": "#945bc4", 9 | "activityBarBadge.foreground": "#e7e7e7", 10 | "commandCenter.border": "#15202b99", 11 | "sash.hoverBorder": "#65c89b", 12 | "statusBar.background": "#42b883", 13 | "statusBar.foreground": "#15202b", 14 | "statusBarItem.hoverBackground": "#359268", 15 | "statusBarItem.remoteBackground": "#42b883", 16 | "statusBarItem.remoteForeground": "#15202b", 17 | "titleBar.activeBackground": "#42b883", 18 | "titleBar.activeForeground": "#15202b", 19 | "titleBar.inactiveBackground": "#42b88399", 20 | "titleBar.inactiveForeground": "#15202b99" 21 | }, 22 | "peacock.remoteColor": "#42b883" 23 | } 24 | -------------------------------------------------------------------------------- /download-annovar.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: download-annova.sh 3 | # Author: Chun-Jie Liu 4 | # Mail: chunjie-sam-liu@foxmail.com 5 | # Created Time: Thu 20 Dec 2018 02:18:51 PM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | download='/home/liucj/tools/annovar-201806/annotate_variation.pl -buildver hg38 -downdb -webfrom annovar' 10 | path='/data/liucj/data/refdata/humandb-annovar-201812' 11 | 12 | ref=('refGene' 'knownGene' 'ensGene' 'ljb26_all' 'dbnsfp30a' 'dbnsfp31a_interpro' 'dbnsfp33a' 'dbnsfp35a' 'dbscsnv11' 'intervar_20180118' 'cosmic70' 'esp6500siv2_ea' 'esp6500siv2_aa' 'esp6500siv2_all' 'exac03' 'gnomad_exome' 'gnomad_genome' 'kaviar_20150923' 'hrcr1' '1000g2015aug' 'gme' 'mcap' 'revel' 'avsnp150' 'nci60' 'clinvar_20180603' 'regsnpintron') 13 | 14 | for i in ${ref[*]} 15 | do 16 | cmd="${download} ${i} ${path} &" 17 | echo ${cmd} 18 | eval ${cmd} 19 | done 20 | 21 | -------------------------------------------------------------------------------- /download-gatk-ftp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # download references data from GATK best practice refrence data 4 | 5 | 6 | # broad GATK ftp bundle 7 | #? location: ftp.broadinstitue.org/bundle 8 | #? username: gsapubftp-anonymous 9 | #? password: 10 | 11 | # lftp -u gsapubftp-anonymous,'' \ 12 | # -e "mirror --continue --parallel=10 /bundle/hg38 $PWD; exit" \ 13 | # ftp.broadinstitute.org 14 | 15 | f=$1 16 | [ -z $f ] && echo "Error: Input directory like: /bundle/hg38" && exit 1 17 | 18 | 19 | #? login 20 | host='ftp.broadinstitute.org' 21 | user='gsapubftp-anonymous' 22 | password='' 23 | 24 | #? remote directory 25 | remote_dir=${f} 26 | 27 | #? local direcotry 28 | local_dir='/home/liucj/data/data/refdata/bundle/hg38' 29 | 30 | #? runtime 31 | echo 32 | echo "Starting download ${remote_dir} from ${host} to ${local_dir}" 33 | date 34 | 35 | lftp -u ${user},${password} ${host} < /proc/sys/net/ipv4/tcp_fastopen" >> /etc/rc.local 17 | echo "net.ipv4.tcp_fastopen = 3" >> /etc/sysctl.conf 18 | 19 | # restart shadowsocks 20 | /etc/init.d/shadowsocks restart 21 | 22 | # google scholar ipv6 23 | echo "2404:6800:4008:c06::be scholar.google.com" >> /etc/hosts 24 | echo "2404:6800:4008:c06::be scholar.google.com.hk" >> /etc/hosts 25 | echo "2404:6800:4008:c06::be scholar.google.com.tw" >> /etc/hosts 26 | 27 | # restart shadowsocks 28 | /etc/init.d/shadowsocks restart 29 | 30 | # run bbr 31 | wget --no-check-certificate https://raw.githubusercontent.com/chunjie-sam-liu/useful-scripts/master/ss-bbr.sh 32 | chmod +x ss-bbr.sh 33 | bash ss-bbr.sh 34 | 35 | 36 | -------------------------------------------------------------------------------- /backup-databases.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: backup_databases.sh 3 | # Author: C.J. Liu 4 | # Mail: samliu@hust.edu.cn 5 | # Created Time: Thu 15 Mar 2018 04:27:45 PM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | # Directory 10 | web=/home/liucj/web 11 | web_backup=/home/liucj/tmp/web_backup 12 | s3_backup=/home/liucj/data/web_backup 13 | 14 | # For logs 15 | scpd=/home/liucj/scripts 16 | backup_time=`date +"%y-%m"` 17 | log=${scpd}/.webbackup_${backup_time}.log 18 | 19 | zipweb () { 20 | # $1 is the web name 21 | dest=${web_backup}/${1}_$backup_time.tar.gz 22 | src=${web}/$1 23 | 24 | echo "Notice: Backup ${1} to ${dest}!" >> ${log} 25 | cmd="tar -zcvf ${dest} ${src} >> ${log}" 26 | echo ${cmd} >> $log 27 | eval ${cmd} 28 | 29 | echo "Notice: Backup ${dest} to server 3" >> ${log} 30 | cmd="copyto.sh 3 ${s3_backup} ${dest} >> ${log}" 31 | echo ${cmd} >> $log 32 | eval ${cmd} 33 | } 34 | 35 | for w in `ls ${web}` 36 | do 37 | zipweb ${w} 38 | done 39 | -------------------------------------------------------------------------------- /my-backup-scripts.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: backup_databases.sh 3 | # Author: C.J. Liu 4 | # Mail: samliu@hust.edu.cn 5 | # Created Time: Thu 15 Mar 2018 04:27:45 PM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | # Directory 10 | web=/home/liucj/web 11 | web_backup=/home/liucj/tmp/web_backup 12 | s3_backup=/home/liucj/data/web_backup 13 | 14 | # For logs 15 | scpd=/home/liucj/scripts 16 | backup_time=`date +"%y-%m"` 17 | log=${scpd}/.webbackup_${backup_time}.log 18 | 19 | zipweb () { 20 | # $1 is the web name 21 | dest=${web_backup}/${1}_$backup_time.tar.gz 22 | src=${web}/$1 23 | 24 | echo "Notice: Backup ${1} to ${dest}!" >> ${log} 25 | cmd="tar -zcvf ${dest} ${src} >> ${log}" 26 | echo ${cmd} >> $log 27 | eval ${cmd} 28 | 29 | echo "Notice: Backup ${dest} to server 3" >> ${log} 30 | cmd="copyto.sh 3 ${s3_backup} ${dest} >> ${log}" 31 | echo ${cmd} >> $log 32 | eval ${cmd} 33 | } 34 | 35 | for w in `ls ${web}` 36 | do 37 | zipweb ${w} 38 | done 39 | -------------------------------------------------------------------------------- /slurm-parallel/exa3.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2021-12-30 11:25:37 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | 9 | #SBATCH --job-name=TestJob 10 | #SBATCH --nodes=2 11 | #SBATCH --ntasks=4 12 | #SBATCH --ntasks-per-node=2 13 | 14 | srun --exclusive --nodes 1 --ntasks 1 sleep 10 & 15 | srun --exclusive --nodes 1 --ntasks 1 sleep 20 & 16 | srun --exclusive --nodes 1 --ntasks 1 sleep 10 & 17 | srun --exclusive --nodes 1 --ntasks 1 sleep 30 & 18 | srun --exclusive --nodes 1 --ntasks 1 sleep 20 & 19 | srun --exclusive --nodes 1 --ntasks 1 sleep 10 & 20 | srun --exclusive --nodes 1 --ntasks 1 sleep 30 & 21 | srun --exclusive --nodes 1 --ntasks 1 sleep 20 & 22 | srun --exclusive --nodes 1 --ntasks 1 sleep 10 & 23 | srun --exclusive --nodes 1 --ntasks 1 sleep 20 & 24 | srun --exclusive --nodes 1 --ntasks 1 sleep 10 & 25 | srun --exclusive --nodes 1 --ntasks 1 sleep 30 & 26 | srun --exclusive --nodes 1 --ntasks 1 sleep 20 & 27 | srun --exclusive --nodes 1 --ntasks 1 sleep 10 & 28 | srun --exclusive --nodes 1 --ntasks 1 sleep 30 & 29 | srun --exclusive --nodes 1 --ntasks 1 sleep 20 & 30 | wait 31 | -------------------------------------------------------------------------------- /stat-website-from-apache-log.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: guo.sh 3 | # Author: Chun-Jie Liu 4 | # Mail: chunjie-sam-liu@foxmail.com 5 | # 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | # ! access log files 10 | 11 | keyword=$1 12 | 13 | # ! Test input keywords 14 | [[ -z ${keyword} ]] && echo "Notice: Input the website name." && echo "Notice: such as gscalite or snoric." && exit 15 | 16 | # ! apache2 log directory 17 | log_dir="/var/log/apache2/" 18 | logs=`find ${log_dir} -name "access*"` 19 | archive_dir="/home/liucj/tmp/stat-web-access" 20 | 21 | # ! output filename 22 | today=`date +'%Y-%m-%d'` 23 | filename="${archive_dir}/${keyword}-${today}.log" 24 | [[ -f ${filename} ]] && true > ${filename} 25 | 26 | for log in ${logs} 27 | do 28 | zgrep -i ${keyword} ${log} | awk -F ':' '{print $1}' | awk -F ' - - \\[' '{print $1,$2}' >> ${filename} 29 | done 30 | sed -i '/bin/Id' ${filename} 31 | 32 | # ? plot website access by time 33 | Rscript plot-access-by-time.R ${filename} & 34 | 35 | # ? plot website access by location 36 | Rscript plot-access-by-address.R ${filename} & -------------------------------------------------------------------------------- /backup-webserver-scripts.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: backup_scripts.sh 3 | # Author: Chun-Jie Liu 4 | # Mail: chunjie-sam-liu@foxmail.com 5 | # Created Time: 2018 Jul 10 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | web=/home/backup/web 10 | cp /etc/apache2/sites-available/000-default.conf ${web} 11 | conf=${web}/000-default.conf 12 | # The scripts backup based on the apapche sites available 13 | # Supported by the main web framework Django and Flask. 14 | # Parse 000-default.conf file to extract the directory of 15 | # the source scripts directory. 16 | 17 | web_dir=$(grep -v "#" ${conf}|grep -v 'liut'|sed -n -e '/python-path/p' |awk -F "=" '{print $2}') 18 | liut="/home/liut/EVmiRNA" 19 | 20 | web_dir="${web_dir} ${liut}" 21 | 22 | for i in ${web_dir[@]} 23 | do 24 | d=${i%:*} 25 | tmp=${d#/home/} 26 | user=${tmp%%/*} 27 | webname=`basename ${d}` 28 | gz=${user}-${webname}-`date +%Y-%m-%d`.tar.gz 29 | cmd="tar -zcvf ${web}/${gz} ${i}" 30 | echo "Start zip ${user}'s ${webname}" 31 | eval ${cmd} 32 | echo "End zip ${webname}" 33 | done 34 | 35 | echo "Done!" -------------------------------------------------------------------------------- /get-sequence-by-mir-name.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding:utf-8 -*- 3 | ################################################ 4 | #File Name: get_sequence_by_mir_name.py 5 | #Author: C.J. Liu 6 | #Mail: samliu@hust.edu.cn 7 | #Created Time: Thu 24 Mar 2016 04:17:56 PM CST 8 | ################################################ 9 | 10 | import sys,os 11 | from Bio import SeqIO 12 | ROOT = os.path.dirname(os.path.abspath(__file__)) 13 | 14 | 15 | def run(mirnafile): 16 | fasta_file = ROOT + "/mature.fa" # Input fasta file 17 | wanted_file = mirnafile # Input interesting sequence IDs, one per line 18 | result_file = mirnafile + ".fasta" # Output fasta file 19 | 20 | wanted = set() 21 | with open(wanted_file) as f: 22 | for line in f: 23 | line = line.strip() 24 | if line != "": 25 | wanted.add(line) 26 | 27 | fasta_sequences = SeqIO.parse(open(fasta_file),'fasta') 28 | with open(result_file, "w") as f: 29 | for seq in fasta_sequences: 30 | # print seq 31 | if seq.id in wanted: 32 | SeqIO.write([seq], f, "fasta") 33 | 34 | def main(): 35 | run(sys.argv[1]) 36 | if __name__ == "__main__": 37 | main() 38 | # print ROOT 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /slrm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2022-02-20 18:22:36 5 | # @DESCRIPTION: 6 | 7 | params=$# 8 | script=${1} 9 | 10 | 11 | 12 | function usage { 13 | Description="Notice: The script aimed at running shell command in general parallel." 14 | Usage="Uage: generalParallel shell_script.sh ntasks(default:20) nodes(default:5) ntasks_per_node(default:1)" 15 | ErrorNo="Error: Number of arguments must be 1 or 2" 16 | ErrorScript="Error: suffix of script file must be sh." 17 | if [ "$params" -lt 1 ]; then 18 | echo $ErrorNo 19 | echo $Description 20 | echo $Usage 21 | exit 1 22 | fi 23 | if [ "${script#*.}" != "sh" ]; then 24 | echo $ErrorScript 25 | echo $Description 26 | echo $Usage 27 | exit 1 28 | fi 29 | } 30 | 31 | 32 | function run { 33 | sbatch << EOF 34 | #!/usr/bin/env bash 35 | #SBATCH --signal=USR2 36 | #SBATCH --ntasks=1 37 | #SBATCH --cpus-per-task=20 38 | #SBATCH --mem=30G 39 | #SBATCH --time=720:00:00 40 | #SBATCH --output=${HOME}/tmp/errout/${name}.%j.out 41 | #SBATCH --error=${HOME}/tmp/errout/${name}.%j.err 42 | #SBATCH --mail-user=chunjie.sam.liu@gmail.com 43 | 44 | bash ${script} 45 | 46 | EOF 47 | } 48 | 49 | usage 50 | name=`basename ${script%%.sh}` 51 | run -------------------------------------------------------------------------------- /slurm-submit-jobs/03-create-split-slurm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | src_dir=/public1/home/pg2204/src 4 | split_dir=${src_dir}/split-not-run 5 | split_slurm_dir=${src_dir}/split-slurm-submit-jobs 6 | 7 | [[ -d ${split_slurm_dir} ]] && rm -rf ${split_slurm_dir} 8 | 9 | mkdir -p ${split_slurm_dir} 10 | 11 | for i in `ls ${split_dir}/x*` 12 | do 13 | # create x.slurm 14 | filename=`basename $i` 15 | slurmfile=${split_slurm_dir}/${filename}.slurm 16 | echo $slurmfile 17 | cat < ${slurmfile} 18 | #!/usr/bin/env bash 19 | #SBATCH -p v2_all 20 | #SBATCH -N 5 21 | #SBATCH -n 100 22 | #SBATCH --ntasks-per-node=20 23 | #SBATCH --job-name=${filename} 24 | #SBATCH -o ${slurmfile}.log 25 | 26 | source /public1/home/pg2204/soft/pypy-2.7-v7.1.1-cg2/pypy.sh 27 | export PYTHONPATH=/public1/home/pg2204/tools/DendroPy-4.4.0/src:\$PYTHONPATH 28 | export PYTHONPATH=/public1/home/pg2204/tools/miRmap-1.1/src:\$PYTHONPATH 29 | 30 | while read mirna; 31 | do 32 | srun --exclusive --nodes 1 --ntasks 1 pypy /public1/home/pg2204/tools/miRmap-1.1/scripts/run_mirmap.py \${mirna} /public1/home/pg2204/data/truncate_altutr_03.key /public1/home/pg2204/data/muture.fa.hsa.json /public1/home/pg2204/data/truncate_altutr_03.json /public1/home/pg2204/target_predit/miRmap_02/altutr_out/\${mirna}.res & 33 | done < ${i} 34 | wait 35 | EOF 36 | done 37 | -------------------------------------------------------------------------------- /jupyter.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #SBATCH --signal=USR2 3 | #SBATCH --ntasks=1 4 | #SBATCH --cpus-per-task=50 5 | #SBATCH --mem=300G 6 | #SBATCH --time=1080:00:00 7 | #SBATCH --output=/home/%u/tmp/errout/jrocker/jupyter.job.%j 8 | # customize --output path as appropriate (to a directory readable only by the user!) 9 | 10 | source ~/tools/anaconda3/bin/activate 11 | USER=$(id -un) 12 | PASSWORD=$(openssl rand -base64 15) 13 | 14 | readonly PORT=$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') 15 | # PORT=22222 16 | cat 1>&2 <" |grep -v "#" |grep -v "GEO Accession viewer"|grep -v ^$ >result/%s' %(gsm, gsm)) 43 | os.system('rm result/*.tmp') 44 | 45 | 46 | 47 | def run(f): 48 | with open(f, 'r') as foo: 49 | for gsm in foo: 50 | gsm = gsm.rstrip() 51 | geturl(gsm) 52 | gethtml(gsm) 53 | print('Done') 54 | 55 | 56 | 57 | 58 | 59 | if __name__ =="__main__": 60 | run(sys.argv[1]) 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /extractOriginalBam2RegionalBam2.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: extractOriginalBam2RegionalBam2.sh 3 | # Author: C.J. Liu 4 | # Mail: samliu@hust.edu.cn 5 | # Created Time: Tue 06 Dec 2016 03:34:19 PM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | index='/data/hg38/genomeBuild/hg38.fasta' 10 | tumorDir='/WXS_RAW/tumor' 11 | 12 | [ $# -eq 0 ] && echo "Error: Input regional file" && exit 1 13 | 14 | region=$1 15 | 16 | [ ! -f "$region" ] && echo "Error: $region does not exist or not regional file" && exit 1 17 | 18 | allBams=(`find $tumorDir -name "*bam" -type f `) 19 | 20 | outputDir=`readlink -f $region` 21 | outputDir=`dirname $outputDir` 22 | 23 | extractedBams=(`ls $outputDir`) 24 | 25 | unExtractedBams=() 26 | 27 | for bam in ${allBams[@]} 28 | do 29 | name=${bam%%.*} 30 | bname=`basename $name` 31 | [[ "${extractedBams[@]}" =~ "${bname}" ]] || unExtractedBams+=($bam) 32 | done 33 | echo ${#unExtractedBams[@]} 34 | 35 | # Make fifo file. 36 | tmp_fifo='/tmp/$$.fifo' 37 | mkfifo $tmp_fifo 38 | exec 6<>$tmp_fifo 39 | rm -rf $tmp_fifo 40 | 41 | for (( i=0; i<30; i++ )) 42 | do 43 | echo "" 44 | done >&6 45 | 46 | for bam in "${unExtractedBams[@]}" 47 | do 48 | read -u6 49 | { 50 | name=`basename $bam` 51 | name=${name%.bam}.extracted.bam 52 | cmd="samtools view -b -L $region $bam -o ${outputDir}/${name}" 53 | echo "Notice - Running the command : $cmd" 54 | eval $cmd 55 | echo "Success - $bam was extracted to ${outputDir}/${name}!" 56 | echo "" >&6 57 | }& 58 | done 59 | wait 60 | exec 6>&- 61 | 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /slurm-submit-jobs/test-xargs.slurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #SBATCH --partition=pg2_128_pool 4 | #SBATCH --nodes=2 5 | #SBATCH --cpus-per-task=1 6 | #SBATCH --mem-per-cpu=4G 7 | #SBATCH --job-name=test.slurm 8 | #SBATCH --output=test.slurm-%j.out 9 | #SBATCH --error=test.slurm-%j.err 10 | #SBATCH --mail-user=chunjie-sam-liu@foxmail.com 11 | #SBATCH --mail-type=end 12 | 13 | # The key here to running multiple jobs is using the "cpus-per-task" directive and then setting the amount of memory per CPU (i.e. per task) with #SBATCH --mem-per-cpu 4G rather than asking for an overall amount of memory per node. Failure to do this may result in strange and sub-optimal behaviour! 14 | 15 | # This two should not be set 16 | # The slurm knows the $SLURM_TASKS_PER_NODE to arrange the task to run on the specific node. 17 | #--ntasks=40 18 | #--ntasks-per-node=20 19 | #--time=00:01:00 20 | 21 | module load anaconda3/3.5.2 22 | module load intel/18.0.2 23 | module load mpi/intel/18.0.2 24 | 25 | echo $SLURM_TASKS_PER_NODE 26 | 27 | # calculate how many tasks can we run simultaneously (i.e. how many cores are available) 28 | NR_TASKS=$(echo $SLURM_TASKS_PER_NODE | sed 's/\([0-9]\+\)(x\([0-9]\+\))/\1*\2/' | bc) 29 | 30 | echo $NR_TASKS 31 | 32 | # This will produce errors, some srun can not add to the steps. 33 | 34 | seq -w 1 300 | xargs -I {} --max-procs=$NR_TASKS bash -c "srun --exclusive --nodes 1 --ntasks 1 python demo.py {}" 35 | 36 | # demo.py should provide exit code 0. 37 | # exit code 1 will throw errors. 38 | # log into specific node 39 | # srun --pty --nodelist=d0806 --partition=v3_64_single bash 40 | 41 | # srun --pty --nodelist=m-2-01 bash 42 | 43 | # Show the limits associated with account. 44 | # sacctmgr show assoc -------------------------------------------------------------------------------- /sortChromosomeAndPosition.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding:utf-8 -*- 3 | ################################################ 4 | #File Name: sortChromosomeAndPosition.py 5 | #Author: C.J. Liu 6 | #Mail: samliu@hust.edu.cn 7 | #Created Time: Mon 16 May 2016 03:50:08 PM CST 8 | ################################################ 9 | 10 | import os ,sys 11 | 12 | def usage(): 13 | if len(sys.argv) != 2 and len(sys.argv) != 4 : 14 | print ("Description:") 15 | print ("\tSort file by chromosome and position as chr1, chr2...") 16 | print ("Usage:") 17 | print ("\tpython %prog inputFile chromField positionField (default chrom and position is first two colunm)") 18 | print ("Example:") 19 | print ("\tpython %prog hello.world 1 2") 20 | sys.exit(1) 21 | 22 | 23 | def sortFile(f, chrom = 1, position = 2): 24 | ofile = open(f + ".sortByChrom", 'a') 25 | sortOrder = ["chr" + str(i) for i in range(1,23)] + ["chrX","chrY", "chrM"] 26 | for c in sortOrder: 27 | posList = list() 28 | lineList = list() 29 | with open(f, 'r') as foo: 30 | for line in foo: 31 | line = line.rstrip() 32 | arr = line.split("\t") 33 | if arr[chrom - 1] != c:continue 34 | posList.append(int(arr[position - 1])) 35 | lineList.append(line) 36 | posListSort = sorted(range(len(posList)), key = lambda k : posList[k]) 37 | for index in posListSort: 38 | ofile.write(lineList[index] + os.linesep) 39 | 40 | 41 | 42 | 43 | def run(): 44 | usage() 45 | if len(sys.argv) == 2: 46 | sortFile(sys.argv[1]) 47 | else: 48 | sortFile(sys.argv[1], chrom = int(sys.argv[2]),position = int(sys.argv[3])) 49 | 50 | def main(): 51 | run() 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /gdc-api-manifest-httr.R: -------------------------------------------------------------------------------- 1 | 2 | # Library ----------------------------------------------------------------- 3 | 4 | library(magrittr) 5 | 6 | 7 | 8 | # GDC API ----------------------------------------------------------------- 9 | 10 | 11 | files_endpoint = "https://api.gdc.cancer.gov/files" 12 | 13 | filter_json = '{ 14 | "filters": { 15 | "op": "and", 16 | "content": [{ 17 | "content": { 18 | "field": "cases.project.program.name", 19 | "value": ["TCGA"] 20 | }, 21 | "op": "in" 22 | }, { 23 | "op": "in", 24 | "content": { 25 | "field": "files.access", 26 | "value": ["open"] 27 | } 28 | }, { 29 | "content": { 30 | "field": "files.data_category", 31 | "value": ["transcriptome profiling"] 32 | }, 33 | "op": "in" 34 | }, { 35 | "op": "in", 36 | "content": { 37 | "field": "files.data_format", 38 | "value": ["txt"] 39 | } 40 | }, { 41 | "op": "in", 42 | "content": { 43 | "field": "files.data_type", 44 | "value": ["miRNA Expression Quantification"] 45 | } 46 | }, { 47 | "content": { 48 | "field": "files.experimental_strategy", 49 | "value": ["miRNA-Seq"] 50 | }, 51 | "op": "in" 52 | }] 53 | }, 54 | "format":"JSON", 55 | "size":20000, 56 | "return_type":"manifest" 57 | }' 58 | 59 | response <- httr::POST(url = files_endpoint, body = jsonlite::fromJSON(filter_json), encode = 'json') 60 | 61 | res_table <- httr::content(x = response, as = 'parsed', type = 'text/tab-separated-values') 62 | 63 | 64 | readr::write_tsv(x = manifest_file, path = '/home/liucj/data/refdata/tcga-somatic-mutation-and-mirna-expression-grch38/mirna-expression/gdc_manifest_mirna_expression.txt') 65 | -------------------------------------------------------------------------------- /generalParallelSlurm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2021-12-30 12:44:36 5 | # @DESCRIPTION: 6 | 7 | # Number of input parameters 8 | 9 | 10 | params=$# 11 | script=${1} 12 | 13 | # Set default threads as 20 14 | ntasks=${2:-10} 15 | nodes=${3:-2} 16 | ntasks_per_node=${4:-5} 17 | 18 | 19 | function usage { 20 | Description="Notice: The script aimed at running shell command in general parallel." 21 | Usage="Uage: generalParallel shell_script.sh ntasks(default:20) nodes(default:5) ntasks_per_node(default:1)" 22 | ErrorNo="Error: Number of arguments must be 1 or 2" 23 | ErrorScript="Error: suffix of script file must be sh." 24 | if [ "$params" -lt 1 ]; then 25 | echo $ErrorNo 26 | echo $Description 27 | echo $Usage 28 | exit 1 29 | fi 30 | if [ "${script#*.}" != "sh" ]; then 31 | echo $ErrorScript 32 | echo $Description 33 | echo $Usage 34 | exit 1 35 | fi 36 | if [ $ntasks -gt 20 ]; then 37 | echo "Warnings: Your THREADS is $threads" 38 | echo "Warnings: Be careful! Your THREADS EXCEED 20!!!" 39 | fi 40 | } 41 | 42 | function run { 43 | sbatch << EOF 44 | #!/usr/bin/env bash 45 | #SBATCH --job-name=${name} 46 | #SBATCH --nodes=${nodes} 47 | #SBATCH --ntasks=${ntasks} 48 | #SBATCH --ntasks-per-node=${ntasks_per_node} 49 | #SBATCH --cpus-per-task=1 50 | #SBATCH --mem-per-cpu=5G 51 | #SBATCH --time=120:00:00 52 | #SBATCH --output=${HOME}/tmp/errout/${name}.%j.out 53 | #SBATCH --error=${HOME}/tmp/errout/${name}.%j.err 54 | #SBATCH --mail-user=chunjie.sam.liu@gmail.com 55 | 56 | while read line; 57 | do 58 | cmd="srun --exclusive --nodes 1 --ntasks 1 \${line} &" 59 | # echo \${cmd} 60 | eval \${cmd} 61 | done < $script 62 | wait 63 | EOF 64 | } 65 | 66 | usage 67 | name=`basename ${script%%.sh}` 68 | echo "Notice - Your are running $script" 69 | echo "Notice - Total number jobs `wc -l $script`" 70 | run 71 | -------------------------------------------------------------------------------- /getEnsFromEntrezID.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding:utf-8 -*- 3 | ################################################ 4 | #File Name: getEnsFromEntrezID.py 5 | #Author: C.J. Liu 6 | #Mail: samliu@hust.edu.cn 7 | #Created Time: Fri 08 Apr 2016 04:08:25 PM CST 8 | ################################################ 9 | 10 | import os,sys 11 | import pickle 12 | import pprint 13 | root = os.path.dirname(os.path.realpath(__file__)) 14 | 15 | def storeEntrezID(): 16 | entrez2Ens = dict() 17 | with open(root + os.sep + "Homo_sapiens.gene_info_20160408") as foo: 18 | for line in foo: 19 | 20 | if line.startswith("#"): continue 21 | arr = line.split("\t") 22 | dbref = dict(item.split(":") for item in arr[5].split("|") if len(item.split(":")) == 2) 23 | try: 24 | ens = dbref["Ensembl"] 25 | except: 26 | ens = "-" 27 | entrez2Ens[arr[1]] = ens 28 | pprint.pprint(entrez2Ens) 29 | # pickle.dump(entrez2Ens, open("entrez2Ens.pickle","wb")) 30 | 31 | def idConvertion(f): 32 | entrezEnsIdList = pickle.load(open(root + os.sep + "entrez2Ens.pickle", "rb")) 33 | ensEntrezIdList = {v:k for k, v in entrezEnsIdList.items()} 34 | with open(f, 'r') as foo: 35 | for line in foo: 36 | if line.startswith("#"): continue 37 | line = line.rstrip() 38 | if line.startswith("ENSG"): 39 | try: 40 | print line,"\t",ensEntrezIdList[line] 41 | except: 42 | print line,"\t","-" 43 | else: 44 | try: 45 | print line,"\t",entrezEnsIdList[line] 46 | except: 47 | print line,"\t","-" 48 | 49 | def run(): 50 | # storeEntrezID() 51 | idConvertion(sys.argv[1]) 52 | 53 | def help(): 54 | if len(sys.argv) != 2: 55 | print "Description:" 56 | print "\tEntrez gene ID and Ensembl gene ID convertion" 57 | print "\tInput one file with one column entrez or ensembl ID" 58 | print "Usage:" 59 | print "\tpython %s entrezIDList.txt" %sys.argv[0] 60 | sys.exit(1) 61 | 62 | def main(): 63 | help() 64 | run() 65 | 66 | if __name__ == "__main__": 67 | main() 68 | 69 | 70 | -------------------------------------------------------------------------------- /plot-access-by-time.R: -------------------------------------------------------------------------------- 1 | 2 | # libs -------------------------------------------------------------------- 3 | 4 | library(magrittr) 5 | library(ggplot2) 6 | library(lubridate) 7 | 8 | # load log ---------------------------------------------------------------- 9 | 10 | args <- commandArgs(TRUE) 11 | filename <- "/home/liucj/tmp/stat-web-access/gscalite-2018-07-09.log" 12 | filename <- args[1] 13 | 14 | filename %>% 15 | readr::read_delim(delim = " ", col_names = FALSE ) %>% 16 | dplyr::rename(ip = X1, date = X2) %>% 17 | dplyr::mutate(date = lubridate::dmy(date)) %>% 18 | dplyr::distinct() %>% 19 | dplyr::arrange(date) -> 20 | log 21 | 22 | span <- months(1) 23 | span_seq <- "1 month" 24 | 25 | # count by time span ------------------------------------------------------ 26 | 27 | span_count <- function(.x, .y = log, .s = span) { 28 | .x_1 <- .x + .s 29 | sum(.y$date >= .x & .y$date < .x_1) 30 | } 31 | 32 | seq(log$date[1], tail(log$date, 1), by = span_seq) %>% 33 | tibble::as_tibble() %>% 34 | dplyr::rename(date = value) %>% 35 | dplyr::mutate(n = purrr::map_int(.x = date, .f = span_count, .y = log, .s = span)) %>% 36 | dplyr::mutate(month = format.Date(date, "%Y-%m")) %>% 37 | range_time 38 | 39 | range_time %>% 40 | ggplot(aes(date, y = n)) + 41 | geom_point() + 42 | geom_line() + 43 | scale_x_date(date_breaks = span_seq, labels = scales::date_format("%Y-%m-%d")) + 44 | theme_minimal() + 45 | labs( 46 | x = "Months", 47 | y = "Access count" 48 | ) + 49 | theme( 50 | panel.grid = element_blank(), 51 | axis.text.x = element_text(angle = 90, hjust = 1, vjust = -0.5), 52 | axis.text = element_text(size = 22), 53 | axis.title = element_text(size = 24), 54 | axis.line = element_line(), 55 | axis.ticks.y = element_line() 56 | ) -> 57 | plot_time_seria 58 | 59 | ggsave( 60 | filename = paste(sub(pattern = ".log", "", filename),'time-seria.pdf', sep = "-"), 61 | plot = plot_time_seria, 62 | device = "pdf", 63 | width = 5, 64 | height = 5 65 | ) 66 | -------------------------------------------------------------------------------- /findMotifByHand.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding:utf-8 -*- 3 | ################################################ 4 | #File Name: find_motif.py 5 | #Author: C.J. Liu 6 | #Mail: samliu@hust.edu.cn 7 | #Created Time: Sun 09 Oct 2016 05:25:28 PM CST 8 | ################################################ 9 | 10 | import os,sys 11 | import re 12 | from itertools import product 13 | from Bio import SeqIO 14 | import numpy as np 15 | ROOT = os.path.dirname(os.path.abspath(__file__)) 16 | 17 | def help(): 18 | helpMessage=''' Description: 19 | Find motif through nt number. It relies on the exact match, not caculating the possibility or confidence coeffient like MEME, it's used for tuning motif results produced by MEME and speeding up handpick. 20 | It's based on the Cartesian product with 4 alphabets. The nt number is not recommended over 10nt. It's very slow when nt number is over 10 which will loop for one million times. 21 | Example: 22 | python find_motif.py seq.fasta motifLength 23 | ''' 24 | 25 | if len(sys.argv) < 3: 26 | print(helpMessage) 27 | sys.exit(1) 28 | 29 | def motifList(motifLength): 30 | motifLength = int(motifLength) 31 | alpha = ["A", "G", "C", "T"] 32 | searchList = list(product(alpha, repeat=motifLength)) 33 | searchList = [''.join(i) for i in searchList] 34 | return searchList 35 | 36 | def run(fastaFile,motifLength): 37 | # print(motifList(motifLength)) 38 | searchList = motifList(motifLength) 39 | for motif in searchList: 40 | searchedID = list() 41 | prog = re.compile(motif, re.I) 42 | for record in SeqIO.parse(fastaFile, 'fasta'): 43 | id = record.id 44 | seq = str(record.seq) 45 | if prog.search(seq): 46 | searchedID.append(id) 47 | if len(searchedID) < 5: continue 48 | print(motif,str(len(searchedID)),','.join(searchedID), sep="\t") 49 | 50 | def main(): 51 | help() 52 | run(sys.argv[1], sys.argv[2]) 53 | 54 | if __name__ == "__main__": 55 | main() 56 | -------------------------------------------------------------------------------- /gdc-download.sh: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: gdc_download.sh 3 | # Author: C.J. Liu 4 | # Mail: samliu@hust.edu.cn 5 | # Created Time: Mon 21 Nov 2016 01:32:17 PM CST 6 | ######################################################################### 7 | #!/bin/bash 8 | 9 | #$1 is Manifest file 10 | #$2 Total number of download files 11 | #open a fifo file 12 | 13 | # Update token regularly 14 | token="/extraspace/TCGA/WXS_RAW/tokens/gdc-user-token.txt" 15 | 16 | [[ -f "$token" ]] || echo "Need gdc token or update token" 17 | 18 | function usage { 19 | if [ "$param" -lt 1 ]; then 20 | echo "Usage:" 21 | echo " bash gdc_download.sh manifest.csv parallel_number(default 20)" 22 | exit 1 23 | fi 24 | 25 | if [ ! -f $manifest ]; then 26 | echo "Error: First argument must be manifest file" 27 | echo "Usage:" 28 | echo " bash gdc_download.sh manifest.csv parallel_number(default 20)" 29 | exit 1 30 | fi 31 | } 32 | 33 | manifest=$1 34 | [[ ! -z $2 ]] && thread=$2 || thread=20 35 | param=$# 36 | 37 | 38 | # Usage 39 | usage 40 | 41 | # Make fifo 42 | tmp_fifofile='/tmp/$$.fifo' 43 | mkfifo $tmp_fifofile 44 | exec 6<>$tmp_fifofile 45 | rm -rf $tmp_fifofile 46 | 47 | for (( i=0; i < $thread; i++ )) 48 | do 49 | echo "" 50 | done>&6 51 | 52 | # All file id 53 | fileID=() 54 | while read line;do name=(${line//"\t"/ }); fileID+=(${name[0]}); done < $manifest 55 | 56 | # echo ${#fileID[@]} 57 | 58 | # All task table with 59 | Total=${#fileID[@]} 60 | for (( i=0; i < $Total; i++ )); 61 | do 62 | read -u6 63 | { 64 | # echo ${fileID[i]} 65 | # sleep 5 66 | # Download data 20 parallel 67 | gdc-client download ${fileID[i]} -t $token 68 | echo "">&6 69 | }& 70 | 71 | done 72 | wait 73 | exec 6>&- 74 | 75 | #md5sum check download files. 76 | Root=$PWD 77 | 78 | [[ -d ${Root}/md5check ]] && echo "${Root}/md5check already exists" || mkdir -p ${Root}/md5check 79 | 80 | ln -s ${Root}/*/*bam ${Root}/md5check/. 81 | awk '{print $3,$2}' $manifest > ${Root}/md5check/md5checklist.txt 82 | 83 | cd ${Root}/md5check/ 84 | cat ${Root}/md5check/md5checklist.txt | parallel --pipe -N50 md5sum -c > ${Root}/md5check/md5checklist.txt.result & 85 | exit 0 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /find-gene-miRNA-by-symbol.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding:utf-8 -*- 3 | ################################################ 4 | #File Name: find_gene_miRNA_by_symbol.py 5 | #Author: C.J. Liu 6 | #Mail: samliu@hust.edu.cn 7 | #Created Time: Wed 30 Mar 2016 02:25:50 PM CST 8 | ################################################ 9 | 10 | import sys, os 11 | import pickle 12 | import pprint 13 | import re 14 | 15 | def help(): 16 | if len(sys.argv) != 2: 17 | print "Description:" 18 | print "\tFind gene regulated by miRNAs" 19 | print "Input:" 20 | print "\tPlease input gene list with comma or semicolon seperated" 21 | print "\tOr input a file containing gene line by line" 22 | print "Example:" 23 | print "\t1. python %s NR3C1" % sys.argv[0] 24 | print "\t2. python %s genelistfile.txt" %sys.argv[0] 25 | sys.exit(1) 26 | 27 | def target2mirna(): 28 | miRNA2targetDict = dict() 29 | with open(os.path.dirname(os.path.realpath(__file__))+"/Total_miRNA2target.20160314.txt", "r") as foo: 30 | for line in foo: 31 | arr = line.rstrip().split("\t") 32 | if len(arr) <2 : 33 | continue 34 | try: 35 | miRNA2targetDict[arr[1]].append(arr[0]) 36 | except: 37 | miRNA2targetDict.setdefault(arr[1],[arr[0]]) 38 | pickle.dump(miRNA2targetDict, open("target2miRNA.dict.pickle","wb")) 39 | # pprint.pprint(miRNA2targetDict) 40 | 41 | def run(f): 42 | miRNA2targetDict = pickle.load(open(os.path.dirname(os.path.realpath(__file__))+"/target2miRNA.dict.pickle","rb")) 43 | print "Search\tGene\tmiRNA" 44 | if os.path.isfile(f): 45 | with open(f, 'r') as foo : 46 | for line in foo: 47 | line = line.rstrip() 48 | genes = [i for i,j in miRNA2targetDict.items() if re.search(line,i,re.IGNORECASE)] 49 | if len(genes) ==0: 50 | print line,"\t","-","\t","-" 51 | else: 52 | for i in genes: 53 | print line,"\t",i,"\t","\t".join(miRNA2targetDict[i]) 54 | else: 55 | inputMiRNAList = re.split(",|;",f) 56 | for line in inputMiRNAList: 57 | line = line.rstrip() 58 | genes = [i for i,j in miRNA2targetDict.items() if re.search(line,i,re.IGNORECASE)] 59 | if len(genes) ==0: 60 | print line,"\t","-","\t","-" 61 | else: 62 | for i in genes: 63 | print line,"\t",i,"\t","\t".join(miRNA2targetDict[i]) 64 | 65 | def main(): 66 | help() 67 | # target2mirna() 68 | run(sys.argv[1]) 69 | 70 | if __name__ == "__main__": 71 | main() 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /jrc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2019-07-12 00:18:08 5 | # @DESCRIPTION: jrocker connect through jump Guo4 6 | 7 | # IP LIST 8 | ipfile=${HOME}/.ssh/known_iplist 9 | 10 | [[ ! -s ${ipfile} ]] && { 11 | echo "Error: ${ipfile} does not exist." 12 | exit 1 13 | } 14 | 15 | # check the mode of known_iplist 16 | [[ $(stat -f "%OLp" ${ipfile}) -ne 600 ]] && { 17 | echo "Warning: The mode of file ${ipfile} is not 700. chmod 600 for ${ipfile}." 18 | chmod 600 ${ipfile} 19 | } 20 | 21 | # load ipmaps 22 | declare -a ipmaps 23 | declare -a servernames 24 | readarray known_ips < ${ipfile} 25 | 26 | param=$# 27 | server=$1 28 | 29 | remote_port=${2:-8686} 30 | local_port=${3:-8686} 31 | 32 | 33 | function fn_errorinfo { 34 | echo "Usage:" 35 | echo " jrc [server] [remote_port|local_port]" 36 | echo "For example1:" 37 | echo " jrc 1 ." 38 | echo " example2:" 39 | echo " jrc 1 8686" 40 | echo " example3:" 41 | echo " jrc 1 8686 9696." 42 | } 43 | 44 | function fn_usage { 45 | [[ ${param} -lt 1 || ${param} -gt 3 ]] && { 46 | echo "Error: Parameters should be in 1, 2, 3, ." 47 | echo "****************************************************" 48 | fn_errorinfo 49 | exit 1 50 | } 51 | 52 | [[ ! (${server} -eq 1 || ${server} -eq 10) ]] && { 53 | echo "Error: Server must be 1 or 10." 54 | echo "****************************************************" 55 | fn_errorinfo 56 | exit 1 57 | } 58 | } 59 | 60 | function fn_load_iplist { 61 | for line in ${known_ips[@]}; 62 | do 63 | line=${line//\~/ } 64 | line=(${line}) 65 | # save ips 66 | ipmaps[${line[1]}]=${line[0]} 67 | # save server names 68 | servernames[${line[1]}]=${line[2]} 69 | done 70 | } 71 | 72 | function fn_parse_ip { 73 | local ip_str="$1" 74 | line=${ip_str//\`/ } 75 | echo $line 76 | } 77 | 78 | function fn_connect { 79 | cmd="sshpass -p '${loginfo[3]}' ssh -p ${jump_proxy[2]} -N -f -L localhost:${local_port}:${loginfo[1]}:${remote_port} ${jump_proxy[0]}@${jump_proxy[1]}" 80 | echo ${cmd} 81 | echo "****************************************************" 82 | eval ${cmd} 83 | exit 0 84 | } 85 | 86 | function main { 87 | fn_usage 88 | # load ip list 89 | fn_load_iplist 90 | jump_proxy=($(fn_parse_ip ${ipmaps[4]})) 91 | loginfo=($(fn_parse_ip ${ipmaps[$server]})) 92 | 93 | fn_connect 94 | } 95 | 96 | main -------------------------------------------------------------------------------- /find-miRNA-target.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding:utf-8 -*- 3 | ################################################ 4 | #File Name: find_miRNA_target.py 5 | #Author: C.J. Liu 6 | #Mail: samliu@hust.edu.cn 7 | #Created Time: Thu 24 Mar 2016 01:39:06 PM CST 8 | ################################################ 9 | 10 | import sys, os 11 | import pickle 12 | import pprint 13 | import re 14 | 15 | def miRNA2target(): 16 | miRNA2targetDict = dict() 17 | with open(os.path.dirname(os.path.realpath(__file__))+"/Total_miRNA2target.20160314.txt", "r") as foo: 18 | for line in foo: 19 | arr = line.rstrip().split("\t") 20 | if len(arr) <2 : 21 | continue 22 | try: 23 | miRNA2targetDict[arr[0]].append(arr[1]) 24 | except: 25 | miRNA2targetDict.setdefault(arr[0],[arr[1]]) 26 | pickle.dump(miRNA2targetDict, open("miRNA2target.dict.pickle","wb")) 27 | # pprint.pprint(miRNA2targetDict) 28 | 29 | def run(f): 30 | miRNA2targetDict = pickle.load(open(os.path.dirname(os.path.realpath(__file__))+"/miRNA2target.dict.pickle","rb")) 31 | print "Search\tmiRNA\tTarget" 32 | if os.path.isfile(f): 33 | with open(f, 'r') as foo : 34 | for line in foo: 35 | if not line.startswith("hsa-"): continue 36 | line = line.rstrip() 37 | miRNAs = [i for i,j in miRNA2targetDict.items() if re.search(line+"\D",i,re.IGNORECASE)] 38 | miRNAs.extend([i for i,j in miRNA2targetDict.items() if re.search(line+"$",i,re.IGNORECASE)]) 39 | if len(miRNAs) ==0: 40 | print line,"\t","-","\t","-" 41 | else: 42 | for i in miRNAs: 43 | print line,"\t",i,"\t","\t".join(miRNA2targetDict[i]) 44 | else: 45 | inputMiRNAList = re.split(",|;",f) 46 | for line in inputMiRNAList: 47 | if not line.startswith("hsa-"): continue 48 | line = line.rstrip() 49 | miRNAs = [i for i,j in miRNA2targetDict.items() if re.search(line+"\D",i,re.IGNORECASE)] 50 | miRNAs.extend([i for i,j in miRNA2targetDict.items() if re.search(line+"$",i,re.IGNORECASE)]) 51 | if len(miRNAs) ==0: 52 | print line,"\t","-","\t","-" 53 | else: 54 | for i in miRNAs: 55 | print line,"\t",i,"\t","\t".join(miRNA2targetDict[i]) 56 | 57 | def help(): 58 | if len(sys.argv) != 2: 59 | print "Description:" 60 | print "\tFind miRNA targets" 61 | print "Input:" 62 | print "\tPlease input miRNA list with comma or semicolon seperated" 63 | print "\tOr input a file containing miRNA line by line" 64 | print "Example:" 65 | print "\t1. python %s hsa-mir-1,hsa-mir-2" % sys.argv[0] 66 | print "\t2. python %s miRNAfile.txt" %sys.argv[0] 67 | sys.exit(1) 68 | def main(): 69 | help() 70 | run(sys.argv[1]) 71 | # miRNA2target() 72 | 73 | if __name__ == "__main__": 74 | main() 75 | -------------------------------------------------------------------------------- /plot-access-by-address.R: -------------------------------------------------------------------------------- 1 | 2 | # libs -------------------------------------------------------------------- 3 | 4 | library(magrittr) 5 | library(ggplot2) 6 | 7 | 8 | # load log ---------------------------------------------------------------- 9 | 10 | args <- commandArgs(TRUE) 11 | filename <- "/home/liucj/tmp/stat-web-access/gscalite-2018-07-09.log" 12 | filename <- args[1] 13 | 14 | filename %>% 15 | readr::read_delim(delim = " ", col_names = FALSE ) %>% 16 | dplyr::rename(ip = X1, date = X2) %>% 17 | dplyr::mutate(date = lubridate::dmy(date)) %>% 18 | dplyr::distinct() %>% 19 | dplyr::arrange(date) -> 20 | log 21 | 22 | # loc pie ----------------------------------------------------------------- 23 | 24 | ip2loc <- function(ip) { 25 | url = "http://ip.taobao.com/service/getIpInfo.php?ip=" 26 | api <- glue::glue({"{url}{ip}"}) 27 | tryCatch( 28 | expr = rjson::fromJSON(file = api)$data$country, 29 | error = function(e) NULL, 30 | warning = function(w) NULL 31 | ) -> .c 32 | .c 33 | } 34 | 35 | # multidplyr 36 | 37 | log %>% 38 | dplyr::mutate(loc = purrr::map(.x = ip, .f = ip2loc)) %>% 39 | dplyr::filter(purrr::map_lgl(.x = loc, .f = Negate(is.null))) %>% 40 | tidyr::unnest() -> 41 | log_ip 42 | 43 | 44 | log_ip %>% 45 | dplyr::group_by(loc) %>% 46 | dplyr::filter(loc != "XX") %>% 47 | dplyr::summarise(m = sum(n())) %>% 48 | dplyr::arrange(dplyr::desc(m)) %>% 49 | dplyr::pull(loc) %>% 50 | head(5) -> top6 51 | 52 | log_ip %>% 53 | dplyr::mutate(loc = ifelse(loc %in% top6, loc, "其他")) %>% 54 | dplyr::group_by(loc) %>% 55 | dplyr::summarise(m = sum(n())) %>% 56 | dplyr::arrange(dplyr::desc(m)) -> 57 | log_ip_p 58 | 59 | log_ip_p$loc -> lev 60 | 61 | log_ip_p %>% 62 | dplyr::mutate(loc = factor(loc, levels = lev)) %>% 63 | dplyr::mutate(pos = (cumsum(c(0, m)) + c(m / 2, .01))[1:nrow(.)]) %>% 64 | ggplot(aes(x = 1, y = m, fill = loc)) + 65 | geom_col(position = position_stack(reverse = TRUE), show.legend = FALSE) + 66 | coord_polar('y', start = 0) + 67 | ggrepel::geom_text_repel( 68 | aes( 69 | x = 1.45, 70 | y = pos, 71 | label = loc 72 | ), 73 | nudge_x = 0.3, 74 | segment.size = .3 75 | ) + 76 | scale_fill_brewer(palette = "Set2", name = "Country") + 77 | theme_minimal() + 78 | theme( 79 | axis.title = element_blank(), 80 | axis.text = element_blank(), 81 | axis.ticks = element_blank(), 82 | panel.border = element_blank(), 83 | panel.grid = element_blank() 84 | ) + 85 | labs( 86 | title = glue::glue("Unique region: {length(unique(log_ip$loc))}") 87 | ) -> plot_loc 88 | 89 | ggsave( 90 | filename = paste(sub(pattern = ".log", "", filename),'loc-pie.pdf', sep = "-"), 91 | plot = plot_loc, 92 | device = "pdf", 93 | width = 5, 94 | height = 5 95 | ) 96 | -------------------------------------------------------------------------------- /mapFileUUID2submitterID.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding:utf-8 -*- 3 | ################################################ 4 | #File Name: mapFileUUID2submitterID.py 5 | #Author: C.J. Liu 6 | #Mail: samliu@hust.edu.cn 7 | #Created Time: Tue 15 Nov 2016 03:53:10 PM CST 8 | ################################################ 9 | 10 | 11 | 12 | import os,sys 13 | import requests 14 | import json 15 | import argparse 16 | 17 | def usage(): 18 | description = ''' 19 | Problem: 20 | I choose project BRCA on GDC(https://gdc-portal.nci.nih.gov/projects/t) and forward to WXS(1,050 cases, 2,175 bam files). Download manifest file from the summary tab. In fact the manifest only contains the file_id, file_name and md5. I can't tell sample information like TCGA barcode. On the Files tab in the search result of GDC-Portal, we can download sample information in form of json in some condition. However, I can't get sample information from the BRCA-WXS json file. Use GDC-API files endpoint. 21 | Task: 22 | Map file_id to sample sample id through GDC-API (https://gdc-docs.nci.nih.gov/API/Users_Guide/Search_and_Retrieval/#filters-specifying-the-quer 23 | ''' 24 | usage = """%(prog)s -i """ 25 | parser = argparse.ArgumentParser(description = description, usage = usage) 26 | 27 | parser.add_argument("-i", "--input", dest = 'manifest', type = str, help = "Input manifest file downloaded from gdc-portal or file contains first column as file_id", required = True) 28 | parser.add_argument('-v','--version',action='version', version='%(prog)s 1.0') 29 | 30 | args = parser.parse_args() 31 | return args 32 | 33 | def loadManifest(manifest): 34 | file_ids = list() 35 | with open(manifest, 'r') as foo: 36 | for line in foo: 37 | arr = line.rstrip().split("\t") 38 | if len(arr[0]) != 36: continue 39 | file_ids.append(arr[0]) 40 | return file_ids 41 | 42 | def makeParams(file_ids): 43 | params ={ 44 | "filters":{ 45 | "op":"in", 46 | "content":{ 47 | "field":"files.file_id", 48 | "value":file_ids 49 | } 50 | }, 51 | "format":"TSV", 52 | "fields":"file_id,file_name,cases.case_id,cases.submitter_id,cases.samples.sample_id,cases.samples.submitter_id,cases.samples.portions.analytes.aliquots.aliquot_id,cases.samples.portions.analytes.aliquots.submitter_id,cases.samples.sample_type,cases.samples.tissue_type,data_category,data_type",#The must be no space after comma 53 | "size":len(file_ids) 54 | } 55 | return params 56 | 57 | def gdcAPI(file_ids, manifest): 58 | output = manifest + '.map2submitterID' 59 | outputh = open(output, 'w') 60 | files_endpt = "https://api.gdc.cancer.gov/files" 61 | params = makeParams(file_ids) 62 | response = requests.post(files_endpt, json = params) 63 | outputh.write(response.text) 64 | outputh.close() 65 | # print(response.text) 66 | 67 | def run(manifest): 68 | file_ids = loadManifest(manifest) 69 | gdcAPI(file_ids, manifest) 70 | 71 | def main(): 72 | args = usage() 73 | run(args.manifest) 74 | 75 | if __name__ == '__main__': 76 | main() 77 | 78 | -------------------------------------------------------------------------------- /rstudio.slurm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=08:00:00 3 | #SBATCH --signal=USR2 4 | #SBATCH --ntasks=1 5 | #SBATCH --cpus-per-task=2 6 | #SBATCH --mem=8192 7 | #SBATCH --output=/home/%u/rstudio-server.job.%j 8 | # customize --output path as appropriate (to a directory readable only by the user!) 9 | 10 | # Create temporary directory to be populated with directories to bind-mount in the container 11 | # where writable file systems are necessary. Adjust path as appropriate for your computing environment. 12 | workdir=$(python -c 'import tempfile; print(tempfile.mkdtemp())') 13 | 14 | mkdir -p -m 700 ${workdir}/run ${workdir}/tmp ${workdir}/var/lib/rstudio-server 15 | cat > ${workdir}/database.conf < ${workdir}/rsession.sh <&2 <&2 -------------------------------------------------------------------------------- /crontab-backup-database-and-web.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # @AUTHOR: Chun-Jie Liu 3 | # @CONTACT: chunjie.sam.liu.at.gmail.com 4 | # @DATE: 2019-09-26 10:32:59 5 | # @DESCRIPTION: 6 | 7 | mnt_dir=/mnt/database-backup 8 | backup_time=`date +'%Y-%m-%d'` 9 | 10 | # log files 11 | logs=${mnt_dir}/backup-${backup_time}.logs 12 | exec &>> ${logs} 13 | 14 | echo "${backup_time} Backup start." 15 | # root path 16 | root_dir=${mnt_dir}/backup-${backup_time} 17 | [[ -d ${root_dir} ]] && { 18 | echo "Backup data and scripts to ${root_dir}." 19 | } || { 20 | echo "Path ${root_dir} does not exists, create a new directory." 21 | mkdir ${root_dir} 22 | echo "Backup data and scripts to ${root_dir}." 23 | } 24 | 25 | # database path 26 | database_dir=${root_dir}/database 27 | [[ -d ${database_dir} ]] && { 28 | echo "Backup data to ${database_dir}." 29 | } || { 30 | echo "Path ${database_dir} does not exists, create a new directory." 31 | mkdir ${database_dir} 32 | echo "Backup data to ${database_dir}." 33 | } 34 | 35 | # MongoDB 36 | { 37 | echo "Start backup MongoDB." 38 | mongo_dir=${database_dir}/mongo-all-data-${backup_time}.mongo 39 | mongodump --username 🙃 --password '🙃' --port 🙃 -o ${mongo_dir} 40 | tar -czf ${mongo_dir}.tar.gz ${mongo_dir} 41 | rm -rf ${mongo_dir} 42 | } 43 | 44 | # MySQL 45 | { 46 | echo "Start backup MySQL." 47 | mysql_gz=mysql-all-data-${backup_time}.sql.gz 48 | mysqldump -u🙃 -p🙃 -A --lock-tables=false | gzip > ${database_dir}/${mysql_gz} 49 | } 50 | 51 | 52 | # Scripts 53 | 54 | web_dir=${root_dir}/web 55 | [[ -d ${web_dir} ]] && { 56 | echo "Backup web scripts to ${web_dir}." 57 | } || { 58 | echo "Path ${web_dir} does not exists, create a new directory." 59 | mkdir ${web_dir} 60 | echo "Backup web scripts to ${web_dir}." 61 | } 62 | 63 | echo "Backup apache2 configure file." 64 | cp /etc/apache2/sites-available/000-default.conf ${web_dir} 65 | conf=${web_dir}/000-default.conf 66 | 67 | # apache configure file scripts path. 68 | apache_conf=$(grep -v "#" ${conf}|grep -v 'liut'|sed -n -e '/python-path/p' |awk -F "=" '{print $2}') 69 | 70 | # shiny apps path. 71 | shiny_apps=$(find /srv/shiny-server -type l|xargs readlink|grep -v "shiny-server") 72 | 73 | # other database scripts. 74 | outside_conf="/home/liucj/web/miRNASNP2 /home/liucj/web/segdt /home/liucj/web/lncRInter /home/liut/EVmiRNA /home/liut/miR_path /home/liut/DODP /home/liut/comid1.0" 75 | 76 | total_web_dir="${apache_conf} ${shiny_apps} ${outside_conf}" 77 | 78 | for i in ${total_web_dir[@]} 79 | do 80 | d=${i%:*} 81 | tmp=${d#/home/} 82 | user=${tmp%%/*} 83 | webname=`basename ${d}` 84 | gz=${user}-${webname}-${backup_time}.tar.gz 85 | cmd="tar -zcvf ${web_dir}/${gz} ${i}" 86 | 87 | { 88 | echo "Start backup ${user}'s ${webname} from ${i} to ${web_dir}/${gz}" 89 | eval ${cmd} 90 | echo "End zip ${webname}" 91 | } 92 | done 93 | 94 | 95 | # remove old backup 96 | 97 | backup_logs=(`ls -rt ${mnt_dir}/*.logs`) 98 | 99 | [[ ${#backup_logs[@]} -le 2 ]] && { 100 | echo "Keep at least two backup." 101 | exit 1 102 | } || { 103 | echo "Remove old backups." 104 | unset 'backup_logs[${#backup_logs[@]}-1]' 105 | unset 'backup_logs[${#backup_logs[@]}-1]' 106 | for i in ${backup_logs[@]}; 107 | do 108 | cmd="rm -rf ${i%.logs}*" 109 | echo ${cmd} 110 | eval ${cmd} 111 | done 112 | } 113 | -------------------------------------------------------------------------------- /ncbi-script/get_gene_summary_README: -------------------------------------------------------------------------------- 1 | README: Gene_tools Last modified: February 3, 2010 2 | 3 | As files are added, notification will be sent via the gene-announce list. 4 | 5 | You may subscribe to the gene-announce list here: 6 | http://www.ncbi.nlm.nih.gov/mailman/listinfo/gene-announce 7 | 8 | SCOPE: 9 | Documentation for tools of interest to those processing information 10 | in Entrez Gene. 11 | 12 | 13 | Table of Contents: 14 | I. taxidToGeneNames.pl 15 | Sample script using E-utilities to extract name information from 16 | Entrez Gene for a species 17 | 18 | II. gene2xml 19 | program to process Entrez Gene's binary ASN.1 files into XML 20 | 21 | III. geneDocSum.pl 22 | Sample script using E-utilities to extract GeneIDs and other fields 23 | from Entrez Gene using a query 24 | 25 | ========================================================================== 26 | I. taxidToGeneNames.pl 27 | --------------------------------------------------------------------------- 28 | A representative perl script, using ESearch and ESummary, to extract 29 | GeneIDs, names and names for a species (i.e. by Taxonomy's id). Usage notes 30 | provide when no arguments are supplied are: 31 | 32 | Usage: taxidToGeneNames.pl [option] -t taxonomyId -o xml|tab 33 | Options: -h Display this usage help information 34 | -v Verbose 35 | -o output options 36 | xml - XML 37 | tab - tab-delimited 38 | 39 | Output is written to STDOUT. 40 | 41 | Sample execution statement: 42 | 43 | taxidToGeneNames.pl -t 9615 -o xml > 9615_genes 44 | 45 | 46 | ========================================================================== 47 | II. gene2xml 48 | --------------------------------------------------------------------------- 49 | gene2xml is a standalone program that converts Entrez Gene ASN.1 into XML. 50 | It also interconverts different formats of Entrez Gene ASN.1. It is 51 | available for multiple platforms. 52 | 53 | You may download this program and view its documentation from: 54 | 55 | ftp://ftp.ncbi.nlm.nih.gov/asn1-converters/by_program/gene2xml 56 | 57 | 58 | ========================================================================== 59 | III. geneDocSum.pl 60 | --------------------------------------------------------------------------- 61 | A representative perl script, using ESearch and ESummary, to extract 62 | GeneIDs and other fields from the Document Summary (DocSum). Usage notes 63 | provided when no arguments are supplied are: 64 | 65 | Usage: ./geneDocSum.pl [options] -q query -o xml|tab 66 | Options: -h Display this usage help information 67 | -v Verbose 68 | -q Query to run against Entrez Gene, e.g. "has summary[prop]" 69 | -o Output options 70 | xml - XML 71 | tab - tab-delimited 72 | -t Tag from eutils xml to extract, e.g. "Summary" 73 | - is case sensitive 74 | - may be specified multiple times to extract multiple 75 | tags & values 76 | - used only with "-o tab" option 77 | - to see all available xml tags in the DocSum, run first 78 | with "-o xml" option 79 | 80 | 81 | Output is written to STDOUT. 82 | 83 | Sample execution statement: 84 | 85 | geneDocSum.pl -q "has_summary[prop] AND chimpanzee[orgn]" -o tab -t Name -t Summary 86 | 87 | 88 | -------------------------------------------------------------------------------- /jrocker.slurm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --signal=USR2 3 | #SBATCH --ntasks=1 4 | #SBATCH --cpus-per-task=30 5 | #SBATCH --mem=80G 6 | #SBATCH --time=1080:00:00 7 | #SBATCH --output=/home/%u/tmp/errout/jrocker/jrocker.job.%j 8 | # customize --output path as appropriate (to a directory readable only by the user!) 9 | 10 | # Create temporary directory to be populated with directories to bind-mount in the container 11 | # where writable file systems are necessary. Adjust path as appropriate for your computing environment. 12 | workdir=$(python -c 'import tempfile;from pathlib import Path; print(tempfile.mkdtemp(prefix="jrocker_", dir=str(Path.home()/"tmp/tmp")))') 13 | 14 | mkdir -p -m 700 ${workdir}/run ${workdir}/tmp ${workdir}/var/lib/rstudio-server 15 | cat >${workdir}/database.conf <${workdir}/rsession.sh <&2 <&2 79 | -------------------------------------------------------------------------------- /jukeras: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | param=$# 4 | serv=$1 5 | variable=$2 6 | port=${variable:-10000} 7 | 8 | arr[1]='up' 9 | arr[2]='dn' 10 | arr[3]='ss' 11 | arr[4]='ex' 12 | arr[5]='imgs' 13 | arr[6]='rmi' 14 | arr[7]='pull' 15 | 16 | 17 | function usage { 18 | if [ "$param" -gt 2 -o "${param}" -lt 1 ] 19 | then 20 | echo "Usage:" 21 | echo " jukeras [up|dn|ss|ex|imgs|rmi] [port|image-id]" 22 | echo "Details:" 23 | echo " up -> startup a container for jukeras, default port 10000." 24 | echo " dn -> shutdown a container or all container of jukeras." 25 | echo " ex -> exec jukeras container." 26 | echo " ss -> status of jukeras container." 27 | echo " imgs -> list images of chunjiesamliu/jukeras" 28 | echo " rmi -> remove image of chunjiesamliu/jukeras" 29 | echo " pull -> pull latest image of chunjiesamliu/jukeras" 30 | echo "For example:" 31 | echo " jukeras up 10000" 32 | exit 1 33 | fi 34 | 35 | if [[ ! "${arr[@]}" =~ "$serv" ]] 36 | then 37 | echo "Error:" 38 | echo " Input must be one of 'up', 'dn', 'ss', 'ex', 'imgs' and 'rmi' to start jukeras" 39 | echo "Example:" 40 | echo " jukeras ss" 41 | exit 1 42 | fi 43 | } 44 | 45 | function up { 46 | cs=(`docker ps -a|grep -o Jukeras.*$`) 47 | [[ ${cs[@]} =~ "Jukeras-${port}" ]] && echo "Error: Jukeras-${port} exists." && exit 1 48 | 49 | [[ ${port} -lt 10000 ]] && echo "Error: New port should be larger than 10000." && exit 1 50 | 51 | docker run -d -p ${port}:8888 \ 52 | -v /home/liucj/:/home/liucj/jupyter-workspace/ \ 53 | -v /data/liucj/:/home/liucj/jupyter-workspace/data/ \ 54 | -v /project/liucj/:/home/liucj/jupyter-workspace/project/ \ 55 | --user root -w /home/liucj/jupyter-workspace \ 56 | -e NB_USER=liucj -e NB_UID=$EUID \ 57 | -e NB_GID=$EUID -e NB_GROUP=liucj \ 58 | -e CHOWN_HOME=yes -e CHOWN_HOME_OPTS='-R' \ 59 | -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes \ 60 | --name Jukeras-${port} chunjiesamliu/jukeras start-notebook.sh \ 61 | --NotebookApp.password='🙃' 62 | 63 | echo "Start Jukeras successfully with port ${port}." 64 | } 65 | 66 | function dn { 67 | cs=(`docker ps -a|grep -o Jukeras.*$`) 68 | if [ ${param} -eq 2 ] 69 | then 70 | c="Jukeras-${port}" 71 | if [[ ${cs[@]} =~ ${port} ]] 72 | then 73 | docker container stop ${c} 74 | echo "Container ${c} is stopped" 75 | docker container rm ${c} 76 | echo "Container ${c} is removed" 77 | else 78 | echo "Error: Container ${c} not in the docker ps -a" 79 | fi 80 | else 81 | echo "Notice: You are going to stop all Jukeras container ${cs[@]}." 82 | echo "Input (yes/no):" 83 | read choice 84 | case ${choice} in 85 | yes) 86 | echo "All Jukeras container stopped." 87 | for c in ${cs[@]}; 88 | do 89 | docker container stop ${c} 90 | echo "Container ${c} is stopped" 91 | docker container rm ${c} 92 | echo "Container ${c} is removed" 93 | done 94 | ;; 95 | no) 96 | echo "No Jukeras container stopped." 97 | ;; 98 | *) 99 | echo "No Jukeras container stopped." 100 | ;; 101 | esac 102 | fi 103 | } 104 | 105 | function ss { 106 | info=`docker ps|grep Jukeras` 107 | [[ -z ${info} ]] && echo "Notice: no Jukeras container." || docker ps|grep Jukeras 108 | } 109 | 110 | function ex { 111 | cs=(`docker ps -a|grep -o Jukeras.*$`) 112 | c="Jukeras-${port}" 113 | [[ ${param} -ne 2 ]] && echo "Error: Input port for container." && ss && exit 1 114 | 115 | if [[ ${param} -eq 2 && ${cs[@]} =~ ${port} ]] 116 | then 117 | docker exec -it -u liucj:root -w ~/jupyter-workspace ${c} bash 118 | else 119 | echo "Notice: Input valid Jukeras port." 120 | docker ps -a|grep Jukeras 121 | fi 122 | } 123 | 124 | function imgs { 125 | docker image ls chunjiesamliu/jukeras 126 | } 127 | 128 | function rmi { 129 | if [ ${param} -eq 2 ] 130 | then 131 | docker image rm ${variable} 132 | else 133 | echo "Notice: Input valid Jukeras image." 134 | exit 1 135 | fi 136 | 137 | } 138 | 139 | 140 | function pull { 141 | docker pull chunjiesamliu/jukeras:latest 142 | } 143 | 144 | usage 145 | [[ ${serv} = 'up' ]] && up || usage 146 | [[ ${serv} = 'dn' ]] && dn || usage 147 | [[ ${serv} = 'ss' ]] && ss || usage 148 | [[ ${serv} = 'ex' ]] && ex || usage 149 | [[ ${serv} = 'rmi' ]] && rmi || usage 150 | [[ ${serv} = 'imgs' ]] && imgs || usage 151 | [[ ${serv} = 'pull' ]] && pull || usage 152 | -------------------------------------------------------------------------------- /generalParallel: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ######################################################################### 3 | # File Name: generalParallel 4 | # Author: C.J. Liu 5 | # Mail: samliu@hust.edu.cn 6 | # Created Time: Tue 06 Dec 2016 04:39:49 PM CST 7 | # Description: Run shell commands in parallel from a command file 8 | ######################################################################### 9 | 10 | # Default values 11 | DEFAULT_THREADS=20 12 | MAX_RECOMMENDED_THREADS=50 13 | 14 | # Parse arguments 15 | command_file="$1" 16 | num_threads="${2:-$DEFAULT_THREADS}" 17 | 18 | function show_usage() { 19 | cat < [num_threads] 21 | 22 | Description: 23 | This script runs shell commands in parallel by reading them from a file. 24 | Each line in the command file should contain one shell command. 25 | Empty lines and lines starting with '#' are ignored. 26 | 27 | Arguments: 28 | command_file File containing commands to execute (one per line) 29 | num_threads Number of parallel threads (default: $DEFAULT_THREADS) 30 | 31 | Examples: 32 | generalParallel commands.txt 33 | generalParallel my_tasks.sh 10 34 | generalParallel job_list.txt 30 35 | EOF 36 | } 37 | 38 | function validate_arguments() { 39 | # Check number of arguments 40 | if [ $# -lt 1 ] || [ $# -gt 2 ]; then 41 | echo "Error: Invalid number of arguments" 42 | show_usage 43 | exit 1 44 | fi 45 | 46 | # Check if command file exists 47 | if [ ! -f "$command_file" ]; then 48 | echo "Error: Command file '$command_file' not found" 49 | exit 1 50 | fi 51 | 52 | # Check if command file is readable 53 | if [ ! -r "$command_file" ]; then 54 | echo "Error: Command file '$command_file' is not readable" 55 | exit 1 56 | fi 57 | 58 | # Validate thread count 59 | if ! [[ "$num_threads" =~ ^[0-9]+$ ]] || [ "$num_threads" -lt 1 ]; then 60 | echo "Error: Number of threads must be a positive integer" 61 | exit 1 62 | fi 63 | 64 | # Warning for high thread count 65 | if [ "$num_threads" -gt "$MAX_RECOMMENDED_THREADS" ]; then 66 | echo "Warning: Thread count ($num_threads) is quite high. Consider using fewer threads to avoid system overload." 67 | fi 68 | } 69 | 70 | function setup_job_control() { 71 | # Create temporary directory and FIFO for job control 72 | local tmp_dir="${HOME}/tmp" 73 | mkdir -p "$tmp_dir" 74 | 75 | job_pipe="${tmp_dir}/parallel_$$.fifo" 76 | mkfifo "$job_pipe" 77 | exec 6<>"$job_pipe" 78 | rm -f "$job_pipe" 79 | 80 | # Initialize job slots (semaphore pattern) 81 | for ((i = 0; i < num_threads; i++)); do 82 | echo "slot" >&6 83 | done 84 | } 85 | 86 | function cleanup() { 87 | exec 6>&- 2>/dev/null 88 | rm -f "$job_pipe" 2>/dev/null 89 | } 90 | 91 | function run_parallel_jobs() { 92 | local job_count=0 93 | local line_number=0 94 | 95 | echo "Processing commands from: $command_file" 96 | echo "Using $num_threads parallel threads" 97 | echo "Started at: $(date)" 98 | echo "----------------------------------------" 99 | 100 | while IFS= read -r command || [ -n "$command" ]; do 101 | ((line_number++)) 102 | 103 | # Skip empty lines and comments 104 | if [[ -z "$command" || "$command" =~ ^[[:space:]]*# ]]; then 105 | continue 106 | fi 107 | 108 | # Wait for an available job slot 109 | read -u6 110 | 111 | # Run command in background 112 | { 113 | # Remove trailing & if present 114 | command="${command%&}" 115 | command="${command## }" # Remove leading spaces 116 | 117 | echo "[$(date +%T)] Line $line_number: Starting: $command" 118 | 119 | if eval "$command"; then 120 | echo "[$(date +%T)] Line $line_number: ✓ Completed: $command" 121 | else 122 | echo "[$(date +%T)] Line $line_number: ✗ Failed: $command" 123 | fi 124 | 125 | # Release job slot 126 | echo "slot" >&6 127 | } & 128 | 129 | ((job_count++)) 130 | done <"$command_file" 131 | 132 | # Wait for all background jobs to complete 133 | wait 134 | 135 | echo "----------------------------------------" 136 | echo "All jobs completed at: $(date)" 137 | echo "Total commands processed: $job_count" 138 | } 139 | 140 | # Main execution 141 | trap cleanup EXIT 142 | 143 | # Validate input 144 | validate_arguments "$@" 145 | 146 | # Setup parallel job control 147 | setup_job_control 148 | 149 | # Run the parallel jobs 150 | run_parallel_jobs 151 | 152 | echo "Parallel execution finished successfully" 153 | -------------------------------------------------------------------------------- /jrocker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | param=$# 4 | serv=$1 5 | variable=$2 6 | port=${variable:-8686} 7 | 8 | 9 | arr[1]='up' 10 | arr[2]='dn' 11 | arr[3]='ss' 12 | arr[4]='ex' 13 | arr[5]='imgs' 14 | arr[6]='rmi' 15 | arr[7]='pull' 16 | 17 | function usage { 18 | if [ "$param" -gt 2 -o "${param}" -lt 1 ] 19 | then 20 | echo "Usage:" 21 | echo " jrocker [up|dn|ss|ex|imgs|rmi] [port|image-id]" 22 | echo "Details:" 23 | echo " up -> startup a container for jrocker, default port 8686." 24 | echo " dn -> shutdown a container or all container of jrocker." 25 | echo " ex -> exec jrocker container." 26 | echo " ss -> status of jrocker container." 27 | echo " imgs -> list images of chunjiesamliu/jrocker" 28 | echo " rmi -> remove image of chunjiesamliu/jrocker" 29 | echo " pull -> pull latest image of chunjiesamliu/jrocker" 30 | echo "For example:" 31 | echo " jrocker up 9191" 32 | exit 1 33 | fi 34 | 35 | if [[ ! "${arr[@]}" =~ "$serv" ]] 36 | then 37 | echo "Error:" 38 | echo " Input must be one of 'up', 'dn', 'ss', 'ex', 'imgs' and 'rmi' to start jrocker" 39 | echo "Example:" 40 | echo " jrocker ss" 41 | exit 1 42 | fi 43 | } 44 | 45 | function cpf { 46 | d=$1 47 | for i in `find ${HOME} -mindepth 1 -prune -name '.*'`; 48 | do 49 | name=`basename ${i}` 50 | [[ ${name} =~ '.rstudio' ]] && continue 51 | [[ ${name} =~ '.vscode' ]] && continue 52 | cmd="cp -rf ${i} ${d}/${name}" 53 | eval ${cmd} 54 | done 55 | } 56 | 57 | function lnf { 58 | d=$1 59 | for i in `find ${HOME} -mindepth 1 -prune -type f -name '.*'`; 60 | do 61 | name=`basename ${i}` 62 | [[ ${name} =~ '.rstudio' ]] && continue 63 | [[ ${name} =~ '.vscode' ]] && continue 64 | cmd="ln ${i} ${d}/${name}" 65 | eval ${cmd} 66 | done 67 | } 68 | 69 | function up { 70 | 71 | cs=(`docker ps|grep -o ${USER}-rstudio.*$`) 72 | [[ ${cs[@]} =~ "${USER}-rstudio-${port}" ]] && echo "Error: ${USER}-rstudio-${port} exists." && exit 1 73 | 74 | [[ ${port} -lt 8686 ]] && echo "Error: New port should be larger than 8686." && exit 1 75 | 76 | jd=${HOME}/.${USER}-rstudio/${USER}-rstudio-${port} 77 | 78 | [[ ! -d ${jd} ]] \ 79 | && mkdir -p ${jd} \ 80 | && lnf ${jd} \ 81 | && ln -s /workspace/${USER} ${jd}/workspace \ 82 | && ln -s /project/${USER} ${jd}/project \ 83 | && ln -s /data/${USER} ${jd}/data 84 | 85 | 86 | docker run -d -p ${port}:8787 \ 87 | -m 1000g \ 88 | -v ${jd}:${HOME}/ \ 89 | -v /workspace/${USER}/:${HOME}/workspace/ \ 90 | -v /project/${USER}/:${HOME}/project/ \ 91 | -v /data/${USER}/:${HOME}/data/ \ 92 | -v ${HOME}/R/:${HOME}/R/ \ 93 | -v ${HOME}/scripts/:${HOME}/scripts/ \ 94 | -v ${HOME}/tools/:${HOME}/tools/ \ 95 | -v ${HOME}/github/:${HOME}/github/ \ 96 | -v ${HOME}/tmp/:${HOME}/tmp/ \ 97 | -e USER=${USER} -e PASSWORD=🙃 \ 98 | -e USERID=$UID -e GROUPID=$GID -e ROOT=TRUE \ 99 | --name ${USER}-rstudio-${port} chunjiesamliu/jrocker:latest 100 | 101 | echo "Start ${USER}-rstudio successfully with port ${port}." 102 | } 103 | 104 | function dn { 105 | cs=(`docker ps|grep -o ${USER}-rstudio.*$`) 106 | if [ ${param} -eq 2 ] 107 | then 108 | c="${USER}-rstudio-${port}" 109 | if [[ ${cs[@]} =~ ${port} ]] 110 | then 111 | docker container stop ${c} 112 | echo "Container ${c} is stopped" 113 | docker container rm ${c} 114 | echo "Container ${c} is removed" 115 | else 116 | echo "Error: Container ${c} not in the docker ps" 117 | fi 118 | else 119 | echo "Notice: You are going to stop all ${USER}-rstudio container ${cs[@]}." 120 | echo "Input (yes/no):" 121 | read choice 122 | case ${choice} in 123 | yes) 124 | echo "All jrocker container stopped." 125 | for c in ${cs[@]}; 126 | do 127 | docker container stop ${c} 128 | echo "Container ${c} is stopped" 129 | docker container rm ${c} 130 | echo "Container ${c} is removed" 131 | done 132 | ;; 133 | no) 134 | echo "No ${USER}-rstudio container stopped." 135 | ;; 136 | *) 137 | echo "No ${USER}-rstudio container stopped." 138 | ;; 139 | esac 140 | fi 141 | } 142 | 143 | function ss { 144 | info=`docker ps|grep ${USER}-rstudio` 145 | [[ -z ${info} ]] && echo "Notice: no ${USER}-rstudio container." || docker ps|grep ${USER}-rstudio 146 | } 147 | 148 | function ex { 149 | cs=(`docker ps|grep -o ${USER}-rstudio.*$`) 150 | c="${USER}-rstudio-${port}" 151 | [[ ${param} -ne 2 ]] && echo "Error: Input port for container." && ss && exit 1 152 | 153 | if [[ ${param} -eq 2 && ${cs[@]} =~ ${port} ]] 154 | then 155 | docker exec -it -u $UID:$GID -w ~ ${c} bash 156 | else 157 | echo "Notice: Input valid jrocker port." 158 | docker ps|grep ${USER}-rstudio 159 | fi 160 | } 161 | 162 | function imgs { 163 | docker image ls chunjiesamliu/jrocker 164 | } 165 | 166 | function rmi { 167 | if [ ${param} -eq 2 ] 168 | then 169 | docker image rm ${variable} 170 | else 171 | echo "Notice: Input valid jrocker image." 172 | exit 1 173 | fi 174 | 175 | } 176 | 177 | function pull { 178 | docker pull chunjiesamliu/jrocker:latest 179 | } 180 | 181 | usage 182 | [[ ${serv} = 'up' ]] && up || usage 183 | [[ ${serv} = 'dn' ]] && dn || usage 184 | [[ ${serv} = 'ss' ]] && ss || usage 185 | [[ ${serv} = 'ex' ]] && ex || usage 186 | [[ ${serv} = 'rmi' ]] && rmi || usage 187 | [[ ${serv} = 'imgs' ]] && imgs || usage 188 | [[ ${serv} = 'pull' ]] && pull || usage 189 | -------------------------------------------------------------------------------- /cs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # The scripts is for connect, copyto and download for multiple servers. 4 | # The known_iplist should be provided with username@ip@port@password@index@servername. 5 | # The known_iplist should be put in ~/.ssh and status code change to 600. 6 | 7 | 8 | # IP LIST 9 | ipfile=${HOME}/.ssh/known_iplist 10 | 11 | [[ ! -s ${ipfile} ]] && { 12 | echo "Error: ${ipfile} does not exist." 13 | exit 1 14 | } 15 | 16 | # check the mode of known_iplist 17 | [[ $(stat -c "%a" ${ipfile}) -ne 600 ]] && { 18 | echo "Warning: The mode of file ${ipfile} is not 700. chmod 600 for ${ipfile}." 19 | chmod 600 ${ipfile} 20 | } 21 | 22 | 23 | # load ipmaps 24 | declare -a ipmaps 25 | declare -a servernames 26 | readarray known_ips < ${ipfile} 27 | 28 | # number of input parameters 29 | param=$# 30 | 31 | # set up functions 32 | # in [login|copyto|download] 33 | func=$1 34 | funcs[1]='login' 35 | funcs[2]='copyto' 36 | funcs[3]='download' 37 | funcs[4]='lg' 38 | funcs[5]='cp' 39 | funcs[6]='dl' 40 | 41 | # interface to server 42 | server=$2 43 | # source file 44 | src=$3 45 | # destination directory 46 | dest=${4:-${PWD}} 47 | 48 | 49 | function fn_errorinfo { 50 | echo "Usage:" 51 | echo " cs [login|copyto|download] [server]" 52 | echo "Details:" 53 | echo " login|lg -> login, log into server." 54 | echo " copyto|cp -> copyto, copyto files to server." 55 | echo " download|dl -> downlaod, download from server." 56 | echo "For example1:" 57 | echo " cs login 1." 58 | echo " example2:" 59 | echo " cs copyto 1 source destination." 60 | echo " example3:" 61 | echo " cs download 1 source destination." 62 | echo "Notice: The source and destination should be absolute path. Source is a file and destination is directory." 63 | } 64 | 65 | function fn_inarray { 66 | # $1 is element 67 | # $2 is ${array[@]} 68 | local e match="$1" 69 | shift 70 | for e; do [[ "$e" == "$match" ]] && return 0; done 71 | return 1 72 | } 73 | 74 | function fn_load_iplist { 75 | for line in ${known_ips[@]}; 76 | do 77 | line=${line//\~/ } 78 | line=(${line}) 79 | # save ips 80 | ipmaps[${line[1]}]=${line[0]} 81 | # save server names 82 | servernames[${line[1]}]=${line[2]} 83 | done 84 | } 85 | 86 | function fn_parse_ip { 87 | local ip_str="$1" 88 | line=${ip_str//\`/ } 89 | echo $line 90 | } 91 | 92 | function fn_usage { 93 | # The parameters should be in 2 3 4 94 | [[ ! (${param} -eq 2 || ${param} -eq 4) ]] && { 95 | echo "Error: Parameters should be in 2, 4." 96 | echo "****************************************************" 97 | fn_errorinfo 98 | exit 1 99 | } 100 | 101 | # func should be in the funcs list 102 | ! fn_inarray ${func} ${funcs[@]} && { 103 | echo "Error: Function '${func}' not in connect server." 104 | echo "****************************************************" 105 | fn_errorinfo 106 | exit 1 107 | } 108 | 109 | # server should be valid. 110 | ! fn_inarray ${server} ${!ipmaps[@]} && { 111 | echo "Error: Server '${server}' should be integer and in the '${!ipmaps[@]}'." 112 | echo "****************************************************" 113 | fn_errorinfo 114 | exit 1 115 | } 116 | 117 | } 118 | 119 | function fn_login { 120 | # func == login, param should be equal to 2 121 | # copyto and download should be four parameters. 122 | 123 | [[ ${param} -ne 2 ]] && { 124 | echo "Error: login should be two parameters." 125 | echo "****************************************************" 126 | fn_errorinfo 127 | exit 1 128 | } 129 | 130 | cmd="sshpass -p '${loginfo[3]}' ssh -p ${loginfo[2]} ${loginfo[0]}@${loginfo[1]}" 131 | echo "Notice: log into '${servernames[${server}]}'." 132 | echo ${cmd} 133 | echo "****************************************************" 134 | eval ${cmd} 135 | exit 0 136 | } 137 | 138 | function fn_copyto { 139 | # for copyto, src file should exist. 140 | [[ ! -e ${src} ]] && { 141 | echo "Error: file '${src}' not exits." 142 | echo "****************************************************" 143 | fn_errorinfo 144 | exit 1 145 | } 146 | 147 | cmd="sshpass -p '${loginfo[3]}' rsync -rvz -e 'ssh -p ${loginfo[2]}' --progress '${src}' ${loginfo[0]}@${loginfo[1]}:${dest} " 148 | echo "Notice: copy file '${src}' to '${dest}' on server '${servernames[${server}]}'." 149 | echo ${cmd} 150 | echo "****************************************************" 151 | eval ${cmd} 152 | exit 0 153 | } 154 | 155 | function fn_download { 156 | # for download, dest directory should exist. 157 | [[ ! -d ${dest} ]] && { 158 | echo "Error: directory '${dest}' not exists." 159 | echo "****************************************************" 160 | fn_errorinfo 161 | exit 1 162 | } 163 | 164 | cmd="sshpass -p '${loginfo[3]}' rsync -rvz -e 'ssh -p ${loginfo[2]}' --progress ${loginfo[0]}@${loginfo[1]}:${src} ${dest}" 165 | echo "Notice: download file '${src}' to '${dest}' from server '${servernames[${server}]}'." 166 | echo ${cmd} 167 | echo "****************************************************" 168 | eval ${cmd} 169 | exit 0 170 | } 171 | 172 | function fn_run { 173 | # Run 174 | [[ ${func} = 'login' ]] && fn_login || fn_usage 175 | [[ ${func} = 'copyto' ]] && fn_copyto || fn_usage 176 | [[ ${func} = 'download' ]] && fn_download || fn_usage 177 | 178 | # Alias 179 | [[ ${func} = 'lg' ]] && fn_login || fn_usage 180 | [[ ${func} = 'cp' ]] && fn_copyto || fn_usage 181 | [[ ${func} = 'dl' ]] && fn_download || fn_usage 182 | } 183 | 184 | function main { 185 | # load ip list 186 | fn_load_iplist 187 | 188 | # Usage 189 | fn_usage 190 | 191 | # parse text ip 192 | loginfo=($(fn_parse_ip ${ipmaps[$server]})) 193 | 194 | # main run function 195 | fn_run 196 | } 197 | 198 | main -------------------------------------------------------------------------------- /hust-references.csl: -------------------------------------------------------------------------------- 1 | 2 | 212 | -------------------------------------------------------------------------------- /ncbi-script/geneGoSummary.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # =========================================================================== 3 | # 4 | # PUBLIC DOMAIN NOTICE 5 | # National Center for Biotechnology Information 6 | # 7 | # This software/database is a "United States Government Work" under the 8 | # terms of the United States Copyright Act. It was written as part of 9 | # the author's official duties as a United States Government employee and 10 | # thus cannot be copyrighted. This software/database is freely available 11 | # to the public for use. The National Library of Medicine and the U.S. 12 | # Government have not placed any restriction on its use or reproduction. 13 | # 14 | # Although all reasonable efforts have been taken to ensure the accuracy 15 | # and reliability of the software and data, the NLM and the U.S. 16 | # Government do not and cannot warrant the performance or results that 17 | # may be obtained by using this software or data. The NLM and the U.S. 18 | # Government disclaim all warranties, express or implied, including 19 | # warranties of performance, merchantability or fitness for any particular 20 | # purpose. 21 | # 22 | # Please cite the author in any work or product based on this material. 23 | # 24 | # =========================================================================== 25 | # 26 | # Author: Craig Wallin 27 | # 28 | # File Description: 29 | # 30 | # Accepts chromosome and genomic location range as input data. 31 | # Sample input: 32 | # chr12:1000000-2000000 33 | # Gets document summary (DocSum) information from Entrez Gene via e-utils. 34 | # Gets GO data from gene2go FTP file (must be downloaded and decompressed 35 | # first). 36 | # Outputs a summary of GO data for genes in the specified range, in 37 | # tab-delimited format. 38 | 39 | use strict; 40 | 41 | # --------------------------------------------------------------------------- 42 | # 43 | # use libraries 44 | 45 | use LWP::Simple; # Define library for the 'get' function. 46 | 47 | # --------------------------------------------------------------------------- 48 | # 49 | # Global variables 50 | 51 | my $arg_tax_id = 9606; # human by default 52 | my $arg_verbose = 0; 53 | my $arg_input_data_file = "-"; # STDIN by default 54 | my $arg_go_file = ""; 55 | 56 | my $beginTime = time(); 57 | 58 | my %gene_go_terms = (); 59 | 60 | # --------------------------------------------------------------------------- 61 | # 62 | # Main program 63 | # 64 | 65 | &processCommandLineParameters; 66 | 67 | &processInputData; 68 | 69 | &end; 70 | 71 | 72 | ################################################################################ 73 | # 74 | sub processCommandLineParameters { 75 | 76 | while ( my $term = shift @ARGV ) { 77 | if ($term eq '-h') { 78 | &showUsageAndExit; 79 | } elsif ($term eq '-v') { 80 | $arg_verbose = 1; 81 | } elsif ($term eq '-i') { 82 | $arg_input_data_file = shift @ARGV; 83 | } elsif ($term eq '-g') { 84 | $arg_go_file = shift @ARGV; 85 | } elsif ($term eq '-t') { 86 | $arg_tax_id = shift @ARGV; 87 | } else { 88 | # Ignore extra input 89 | } 90 | } 91 | 92 | if ($arg_input_data_file eq "") {&showUsageAndExit} 93 | if ($arg_go_file eq "") {&showUsageAndExit} 94 | 95 | die "gene2go file does not exist: $arg_go_file\n" unless -e $arg_go_file; 96 | } 97 | 98 | ################################################################################ 99 | 100 | sub showUsageAndExit { 101 | 102 | my $usage = qq/ 103 | Usage: $0 [options] 104 | Options: -h Display this usage help information 105 | -v Verbose 106 | -i Input file (or - for stdin) with ranges 107 | -g gene2go file 108 | -t tax id 109 | /; 110 | 111 | print STDERR "$usage\n"; 112 | 113 | &end 114 | } 115 | 116 | 117 | ################################################################################ 118 | 119 | sub printHeader { 120 | 121 | print STDOUT join "\t", ( "Genomic coordinates", "Enzyme", "Receptor", "Hormone", "Structural" ); 122 | print STDOUT "\n"; 123 | } 124 | 125 | 126 | ################################################################################ 127 | # This populates %gene_go_terms 128 | 129 | sub readGoFile { 130 | 131 | print STDERR "Reading gene2go file...\n"; 132 | 133 | open IN, "<$arg_go_file" or die "Failed to open: $arg_go_file\n"; 134 | 135 | while ( ) { 136 | 137 | chomp(); 138 | next if m/^#/; # skip header 139 | 140 | my ( $tax_id, $GeneID, $GO_ID, $Evidence, $Qualifier, $GO_term, $PubMed, $Category ) = split /\t/; 141 | 142 | next unless $tax_id == $arg_tax_id; 143 | next if $Qualifier =~ m/^NOT/; 144 | 145 | # Accumulate terms for this gene. 146 | 147 | my $gene_terms = $gene_go_terms{ $GeneID }; 148 | $gene_terms .= "enzyme " if $GO_term =~ m/enzyme/i; 149 | $gene_terms .= "receptor " if $GO_term =~ m/receptor/i; 150 | $gene_terms .= "hormone " if $GO_term =~ m/hormone/i; 151 | $gene_terms .= "structural " if $GO_term =~ m/structural/i; 152 | $gene_go_terms{ $GeneID } = $gene_terms; 153 | } 154 | 155 | print STDERR "Done reading gene2go file, read data for " . scalar(keys(%gene_go_terms)) . " genes.\n"; 156 | 157 | close IN; 158 | } 159 | 160 | ################################################################################ 161 | 162 | sub processInputData { 163 | 164 | readGoFile(); 165 | 166 | my $maxResults = 15000; 167 | 168 | my $esearch_result; 169 | my $esummary_result; 170 | 171 | &printHeader; 172 | 173 | my @input_records = (); 174 | 175 | open IN, "<$arg_input_data_file" or die "Failed to open input: $arg_input_data_file\n"; 176 | 177 | while ( my $line = ) { 178 | 179 | chomp($line); 180 | 181 | push @input_records, $line; 182 | } 183 | 184 | my $total_records = scalar @input_records; 185 | my $record_count = 0; 186 | 187 | while ( my $line = shift @input_records ) { 188 | 189 | $record_count++; 190 | if ( $record_count % 100 == 0 ) { 191 | print STDERR "Read $record_count of $total_records records\n"; 192 | } 193 | 194 | $line =~ m/^chr(\w+):(\d+)-(\d+)$/ or die "Could not parse input line: $line\n"; 195 | my ( $chr, $start, $stop ) = ( $1, $2, $3 ); 196 | 197 | my $location_str = "chr$chr:$start\-$stop"; 198 | 199 | my $terms="txid9606 AND $chr \[chr\] AND $start : $stop \[chrpos\]"; 200 | 201 | my $request = "db=gene&retmax=$maxResults&term=$terms"; 202 | 203 | #print "$request\n"; 204 | 205 | $esearch_result = &Eutil ("esearch", $request); 206 | $esearch_result =~ m/(\d+)<\/Count>/ 207 | or die "$esearch_result did not contain expected ,\n for request $request\n"; 208 | 209 | # Summarize counts as the number of genes in the range with the 210 | # desired keywords. 211 | 212 | my $num_enzyme = 0; 213 | my $num_receptor = 0; 214 | my $num_hormone = 0; 215 | my $num_structural = 0; 216 | 217 | while ($esearch_result =~ m/(\d+)<\/Id>/g) { 218 | 219 | my $GeneID = $1; # extract a geneId 220 | 221 | my $gene_terms = $gene_go_terms{ $GeneID }; 222 | 223 | next unless $gene_terms; 224 | 225 | $num_enzyme++ if $gene_terms =~ m/enzyme/; 226 | $num_receptor++ if $gene_terms =~ m/receptor/; 227 | $num_hormone++ if $gene_terms =~ m/hormone/; 228 | $num_structural++ if $gene_terms =~ m/structural/; 229 | } 230 | 231 | print STDOUT join "\t", ( $location_str, $num_enzyme, $num_receptor, $num_hormone, $num_structural ); 232 | print STDOUT "\n"; 233 | } 234 | 235 | close IN; 236 | } 237 | 238 | ################################################################################ 239 | # Subroutine to handle all eutil calls. 240 | 241 | # Create a BEGIN block to provide effect of static data for sub 242 | BEGIN { 243 | 244 | use Time::HiRes qw( usleep gettimeofday tv_interval ); 245 | 246 | # static data 247 | my $lastEutilTime = [gettimeofday]; # init once 248 | 249 | # storing local constants here too. 250 | my $eutilBaseUrl = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils"; 251 | 252 | # most frequent usage allowed for eutils is 3 requests per second 253 | my $minUtilPeriod_microsec = 333333; # microseconds 254 | 255 | sub Eutil { 256 | 257 | my ($eutil, $querystringData) = @_; 258 | 259 | my $elapsed_sec = tv_interval( $lastEutilTime ); 260 | my $delay_microsec = $minUtilPeriod_microsec - ( $elapsed_sec * 1000000 ); 261 | if ($delay_microsec < 0) {$delay_microsec = 0} 262 | usleep ($delay_microsec); 263 | $lastEutilTime = [gettimeofday]; # save for next time 264 | 265 | my $eutilUrl = "$eutilBaseUrl/$eutil.fcgi?$querystringData"; 266 | if ($arg_verbose) {print STDERR "\neutilUrl: $eutilUrl\n";} 267 | 268 | my $result = get($eutilUrl); # get result of the eutil for return 269 | if ((not defined $result) or ($result eq "")) 270 | { 271 | $result = ""; # Simplify error testing on return 272 | print STDERR "$eutil failed for request: $querystringData\n\n"; 273 | } 274 | 275 | if ($arg_verbose) { 276 | print STDERR "\neutil result: $result\n"; 277 | my $elapsedTime = tv_interval( $lastEutilTime ); 278 | print STDERR "$eutil took $elapsedTime seconds\n"; 279 | } 280 | 281 | $result; # for return 282 | } 283 | } 284 | 285 | ################################################################################ 286 | 287 | sub end { 288 | 289 | if ($arg_verbose) { 290 | my $elapsedTime = time() - $beginTime; 291 | print STDERR "\nElapsed time: $elapsedTime seconds\n"; 292 | } 293 | exit; 294 | } 295 | 296 | -------------------------------------------------------------------------------- /ncbi-script/taxidToGeneNames.pl: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/perl -w 2 | # =========================================================================== 3 | # 4 | # PUBLIC DOMAIN NOTICE 5 | # National Center for Biotechnology Information 6 | # 7 | # This software/database is a "United States Government Work" under the 8 | # terms of the United States Copyright Act. It was written as part of 9 | # the author's official duties as a United States Government employee and 10 | # thus cannot be copyrighted. This software/database is freely available 11 | # to the public for use. The National Library of Medicine and the U.S. 12 | # Government have not placed any restriction on its use or reproduction. 13 | # 14 | # Although all reasonable efforts have been taken to ensure the accuracy 15 | # and reliability of the software and data, the NLM and the U.S. 16 | # Government do not and cannot warrant the performance or results that 17 | # may be obtained by using this software or data. The NLM and the U.S. 18 | # Government disclaim all warranties, express or implied, including 19 | # warranties of performance, merchantability or fitness for any particular 20 | # purpose. 21 | # 22 | # Please cite the author in any work or product based on this material. 23 | # 24 | # =========================================================================== 25 | # 26 | # Author: Zev Hochberg 27 | # 28 | # File Description: 29 | # Accepts tax_id as an argument 30 | # Gets name information from gene via e-utils 31 | # 32 | 33 | use strict; 34 | 35 | # --------------------------------------------------------------------------- 36 | # 37 | use LWP::Simple; # Define library for the 'get' function. 38 | 39 | # --------------------------------------------------------------------------- 40 | # 41 | # Global variables 42 | 43 | my $verbose = 0; 44 | my $taxId; 45 | my $dopt; 46 | my $beginTime = time(); 47 | 48 | my $startTag = ""; 49 | my $stopTag = ""; 50 | 51 | # --------------------------------------------------------------------------- 52 | # 53 | # Main program 54 | # 55 | 56 | &processCommandLineParameters; 57 | 58 | &verifyInput; 59 | 60 | &printHeader if $dopt eq 'tab'; 61 | 62 | &process; 63 | 64 | &end; 65 | 66 | 67 | ################################################################################ 68 | # 69 | sub processCommandLineParameters { 70 | 71 | $taxId = ""; 72 | $dopt = ""; 73 | 74 | while(defined ($_= shift @ARGV)) { 75 | if ($_ eq '-h') { 76 | &showUsageAndExit; 77 | } elsif ($_ eq '-v') { 78 | $verbose = 1; 79 | } elsif ($_ eq '-t') { 80 | $taxId = shift @ARGV; 81 | } elsif ($_ eq '-o') { 82 | $dopt = shift @ARGV; 83 | } else { 84 | # Ignore extra input 85 | } 86 | } 87 | if ($taxId eq "") {&showUsageAndExit} 88 | if ($dopt eq "") {&showUsageAndExit} 89 | } 90 | 91 | ################################################################################ 92 | # 93 | sub verifyInput { 94 | unless (($taxId =~ m/^\d*$/) and ($taxId != 0)) { 95 | print STDERR "taxonomyId (-t) must be numeric and > 0\n"; 96 | &showUsageAndExit; 97 | } 98 | unless (($dopt eq 'xml') or ($dopt eq 'tab')) { 99 | print STDERR "output (-o) is required and must equal 'xml' or 'tab'\n"; 100 | &showUsageAndExit; 101 | } 102 | } 103 | 104 | ################################################################################ 105 | # 106 | sub showUsageAndExit { 107 | my $usage = qq/ 108 | Usage: $0 [option] -t taxonomyId -o xml|tab 109 | Options: -h Display this usage help information 110 | -v Verbose 111 | -o output options 112 | xml - XML 113 | tab - tab-delimited 114 | /; 115 | 116 | print STDERR "$usage\n\n"; 117 | 118 | &end 119 | } 120 | 121 | 122 | ################################################################################ 123 | # 124 | sub printHeader { 125 | print STDOUT "geneId\tname\tdescription\n"; 126 | } 127 | 128 | 129 | ################################################################################ 130 | # 131 | sub process { 132 | my $geneStart = 0; 133 | my $maxGenes = 250; 134 | my $totalGenes; 135 | my $haveTotal = 0; 136 | 137 | my $geneId = ""; 138 | my $geneCount = 0; 139 | 140 | my $qs; 141 | my $esearch_result; 142 | my $esummary_result; 143 | 144 | my $first = 1; 145 | 146 | # Main loop: get up to maxGenes Gene ID's for requested taxId from Gene 147 | $totalGenes = $geneStart+1; # to get started 148 | 149 | GeneLoop: 150 | for (; $geneStart<$totalGenes; $geneStart+=$maxGenes) { 151 | if ($verbose) {print STDOUT "Processing genes $geneStart to ", $geneStart+$maxGenes-1, "\n";} 152 | 153 | #defining the query 154 | #this option looks for GeneIDs with the $taxId value of interest that are 155 | #encoded on the mitochondrion. Note use of the field restriction [properties] 156 | #and the boolean AND 157 | #$qs = "db=gene&retstart=$geneStart&retmax=$maxGenes&term=" . $taxId . "[taxid]+AND+source_mitochondrion[properties]"; 158 | 159 | #this option looks for GeneIDs with the $taxId value of interest that are 160 | #NOT encoded on the mitochondrion and do not have RefSeqs of the type model. 161 | #Note use of the boolean NOT and field restriction 162 | $qs = "db=gene&retstart=$geneStart&retmax=$maxGenes&term=" . $taxId . "[taxid]+NOT+source_mitochondrion[properties]+NOT+srcdb_refseq_model[properties]"; 163 | 164 | $esearch_result = &Eutil ("esearch", $qs); 165 | if ($esearch_result =~ m/(\d+)<\/Count>/) { 166 | if ($haveTotal) { 167 | if ($totalGenes != $1) { 168 | die "esearch reported new total genes: was $totalGenes; now $1\nFor request $qs\n"; 169 | } 170 | } else { 171 | $totalGenes = $1; # extract total 172 | $haveTotal = 1; 173 | } 174 | } else { 175 | die "$esearch_result did not contain expected ,\n for request $qs\n"; 176 | } 177 | 178 | 179 | # Build querystring for GENE search 180 | $qs = "db=gene&id="; 181 | while ($esearch_result =~ m/(\d+)<\/Id>/g) { 182 | $geneCount++; 183 | $geneId = $1; # extract a geneId 184 | $qs .= "$geneId,"; 185 | } 186 | chop($qs); 187 | 188 | # Get Gene summary 189 | $esummary_result = &Eutil ("esummary", $qs); 190 | 191 | # Extract and output information for all genes 192 | if ($dopt eq 'tab'){ 193 | &extractAndOutput ($esummary_result); 194 | } else { 195 | &extractAndOutputXml ($esummary_result, $first); 196 | } 197 | 198 | $first = 0; 199 | 200 | if ($verbose) { 201 | print STDOUT "\ngeneCount: $geneCount\n"; 202 | } 203 | } 204 | 205 | # Close xml output 206 | if ($dopt eq 'xml') {print STDOUT $stopTag} 207 | } 208 | 209 | ################################################################################ 210 | # 211 | sub extractAndOutput { 212 | my $xml = $_[0]; 213 | 214 | my $match; 215 | 216 | my $geneId; 217 | my $name; 218 | my $description; 219 | 220 | while ($xml =~ m/(.*?)<\/DocSum>/gs) { 221 | $match = $1; 222 | 223 | $geneId = ""; 224 | $name = ""; 225 | $description = ""; 226 | 227 | if ($match =~ /(\d+)<\/Id>/) {$geneId = $1} 228 | if ($match =~ m/(.+)<\/Item>/) {$name = $1} 229 | if ($match =~ m/(.+)<\/Item>/) {$description = $1} 230 | 231 | print STDOUT "$geneId\t$name\t$description\n"; 232 | } 233 | } 234 | 235 | ################################################################################ 236 | # 237 | sub extractAndOutputXml { 238 | my $xml = shift; 239 | my $first = shift; 240 | 241 | # Strip leading tags, 242 | # from all but first set 243 | 244 | # Strip closing tag from all 245 | 246 | if ($first) { 247 | my $start = 0; 248 | my $stop = index ($xml, $stopTag); 249 | print STDOUT substr ($xml, $start, $stop-$start); 250 | } else { 251 | my $start = index ($xml, $startTag) + length($startTag); 252 | my $stop = index ($xml, $stopTag); 253 | print STDOUT substr ($xml, $start, $stop-$start); 254 | } 255 | } 256 | 257 | ################################################################################ 258 | # 259 | # Subroutine to handle all eutil calls 260 | # 261 | # Create a BEGIN block to provide effect of static data for sub 262 | BEGIN { 263 | # static data 264 | my $lastEutilTime = 0; # init to avoid delay on first Eutil 265 | 266 | # storing local constants here too. 267 | my $eutilBaseUrl = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils"; 268 | my $minUtilPeriod = 3; # seconds 269 | 270 | sub Eutil { 271 | my ($eutil, $querystringData) = @_; 272 | 273 | my $elapsed = time() - $lastEutilTime; 274 | my $delay = $minUtilPeriod - $elapsed; 275 | if ($delay < 0) {$delay = 0} 276 | sleep ($delay); 277 | $lastEutilTime = time(); # save for next time 278 | 279 | my $eutilUrl = "$eutilBaseUrl/$eutil.fcgi?$querystringData"; 280 | if ($verbose) {print STDOUT "\neutilUrl: $eutilUrl\n";} 281 | 282 | my $result = get($eutilUrl); # get result of the eutil for return 283 | if ((not defined $result) or ($result eq "")) 284 | { 285 | $result = ""; # Simplify error testing on return 286 | print STDERR "$eutil failed for request: $querystringData\n\n"; 287 | } 288 | 289 | if ($verbose) { 290 | print STDOUT "\neutil result: $result\n"; 291 | my $elapsedTime = time() - $lastEutilTime; 292 | print STDOUT "$eutil took $elapsedTime seconds\n"; 293 | } 294 | 295 | $result; # for return 296 | } 297 | } 298 | 299 | ################################################################################ 300 | # 301 | sub end { 302 | if ($verbose) { 303 | my $elapsedTime = time() - $beginTime; 304 | print STDOUT "\nElapsed time: $elapsedTime seconds\n"; 305 | } 306 | exit; 307 | } 308 | -------------------------------------------------------------------------------- /ncbi-script/geneDocSum.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # =========================================================================== 3 | # 4 | # PUBLIC DOMAIN NOTICE 5 | # National Center for Biotechnology Information 6 | # 7 | # This software/database is a "United States Government Work" under the 8 | # terms of the United States Copyright Act. It was written as part of 9 | # the author's official duties as a United States Government employee and 10 | # thus cannot be copyrighted. This software/database is freely available 11 | # to the public for use. The National Library of Medicine and the U.S. 12 | # Government have not placed any restriction on its use or reproduction. 13 | # 14 | # Although all reasonable efforts have been taken to ensure the accuracy 15 | # and reliability of the software and data, the NLM and the U.S. 16 | # Government do not and cannot warrant the performance or results that 17 | # may be obtained by using this software or data. The NLM and the U.S. 18 | # Government disclaim all warranties, express or implied, including 19 | # warranties of performance, merchantability or fitness for any particular 20 | # purpose. 21 | # 22 | # Please cite the author in any work or product based on this material. 23 | # 24 | # =========================================================================== 25 | # 26 | # Author: Craig Wallin 27 | # 28 | # File Description: 29 | # 30 | # Accepts query as an argument 31 | # Gets document summary (DocSum) information from Entrez Gene via e-utils 32 | # Outputs either XML or a specified set of fields in tab-delimited format 33 | # 34 | 35 | use strict; 36 | 37 | # --------------------------------------------------------------------------- 38 | # 39 | use LWP::Simple; # Define library for the 'get' function. 40 | 41 | # --------------------------------------------------------------------------- 42 | # 43 | # Global variables 44 | 45 | my $verbose = 0; 46 | my $dopt; 47 | my $query = ""; 48 | my $beginTime = time(); 49 | 50 | my $startTag = ""; 51 | my $stopTag = "\n"; 52 | 53 | my @tags_to_find = ("geneId"); # start with geneId, add user's options to it 54 | 55 | # --------------------------------------------------------------------------- 56 | # 57 | # Main program 58 | # 59 | 60 | &processCommandLineParameters; 61 | 62 | &verifyInput; 63 | 64 | &printHeader if $dopt eq 'tab'; 65 | 66 | &process; 67 | 68 | &end; 69 | 70 | 71 | ################################################################################ 72 | # 73 | sub processCommandLineParameters { 74 | 75 | $dopt = ""; 76 | 77 | while(defined ($_= shift @ARGV)) { 78 | if ($_ eq '-h') { 79 | &showUsageAndExit; 80 | } elsif ($_ eq '-v') { 81 | $verbose = 1; 82 | } elsif ($_ eq '-q') { 83 | $query = shift @ARGV; 84 | } elsif ($_ eq '-t') { 85 | my $tag = shift @ARGV; 86 | push @tags_to_find, $tag; 87 | } elsif ($_ eq '-o') { 88 | $dopt = shift @ARGV; 89 | } else { 90 | # Ignore extra input 91 | } 92 | } 93 | if ($query eq "") {&showUsageAndExit} 94 | if ($dopt eq "") {&showUsageAndExit} 95 | } 96 | 97 | ################################################################################ 98 | # 99 | sub verifyInput { 100 | unless ($query) { 101 | print STDERR "query (-q) is required\n"; 102 | &showUsageAndExit; 103 | } 104 | unless (($dopt eq 'xml') or ($dopt eq 'tab')) { 105 | print STDERR "output (-o) is required and must equal 'xml' or 'tab'\n"; 106 | &showUsageAndExit; 107 | } 108 | } 109 | 110 | ################################################################################ 111 | # 112 | sub showUsageAndExit { 113 | my $usage = qq/ 114 | Usage: $0 [options] -q query -o xml|tab 115 | Options: -h Display this usage help information 116 | -v Verbose 117 | -q Query to run against Entrez Gene, e.g. "has summary[prop]" 118 | -o Output options 119 | xml - XML 120 | tab - tab-delimited 121 | -t Tag from eutils xml to extract, e.g. "Summary" 122 | - is case sensitive 123 | - may be specified multiple times to extract multiple 124 | tags & values 125 | - used only with "-o tab" option 126 | - to see all available xml tags in the DocSum, run first 127 | with "-o xml" option 128 | /; 129 | 130 | print STDERR "$usage\n\n"; 131 | 132 | &end 133 | } 134 | 135 | 136 | ################################################################################ 137 | # 138 | sub printHeader { 139 | print STDOUT join( "\t", @tags_to_find ) . "\n"; 140 | } 141 | 142 | 143 | ################################################################################ 144 | # 145 | sub process { 146 | my $geneStart = 0; 147 | my $maxGenes = 250; 148 | my $totalGenes; 149 | my $haveTotal = 0; 150 | 151 | my $geneId = ""; 152 | my $geneCount = 0; 153 | 154 | my $qs; 155 | my $esearch_result; 156 | my $esummary_result; 157 | 158 | my $first = 1; 159 | 160 | # Main loop: get up to maxGenes Gene ID's for requested query from Gene 161 | $totalGenes = $geneStart+1; # to get started 162 | 163 | GeneLoop: 164 | for (; $geneStart<$totalGenes; $geneStart+=$maxGenes) { 165 | if ($verbose) {print STDOUT "Processing genes $geneStart to ", $geneStart+$maxGenes-1, "\n";} 166 | 167 | #this option looks for GeneIDs with the $query value of interest 168 | $qs = "db=gene&retstart=$geneStart&retmax=$maxGenes&term=$query"; 169 | 170 | $esearch_result = &Eutil ("esearch", $qs); 171 | if ($esearch_result =~ m/(\d+)<\/Count>/) { 172 | if ($haveTotal) { 173 | if ($totalGenes != $1) { 174 | die "esearch reported new total genes: was $totalGenes; now $1\nFor request $qs\n"; 175 | } 176 | } else { 177 | $totalGenes = $1; # extract total 178 | $haveTotal = 1; 179 | } 180 | } else { 181 | die "$esearch_result did not contain expected ,\n for request $qs\n"; 182 | } 183 | 184 | # Build querystring for GENE search 185 | $qs = "db=gene&id="; 186 | while ($esearch_result =~ m/(\d+)<\/Id>/g) { 187 | $geneCount++; 188 | $geneId = $1; # extract a geneId 189 | $qs .= "$geneId,"; 190 | } 191 | chop($qs); # remove last comma 192 | $qs .= "&filter=asis"; 193 | 194 | # Get Gene summary 195 | $esummary_result = &Eutil ("esummary", $qs); 196 | 197 | # Extract and output information for all genes 198 | if ($dopt eq 'tab'){ 199 | &extractAndOutput ($esummary_result); 200 | } else { 201 | &extractAndOutputXml ($esummary_result, $first); 202 | } 203 | 204 | $first = 0; 205 | 206 | if ($verbose) { 207 | print STDOUT "\ngeneCount: $geneCount\n"; 208 | } 209 | } 210 | 211 | # Close xml output 212 | if ($dopt eq 'xml') {print STDOUT "$stopTag\n"} 213 | } 214 | 215 | ################################################################################ 216 | # 217 | sub extractAndOutput { 218 | my $xml = $_[0]; 219 | 220 | my $match; 221 | 222 | my $geneId; 223 | my $name; 224 | my $description; 225 | 226 | while ($xml =~ m/(.*?)<\/DocumentSummary>/gs) { 227 | $geneId = $1; 228 | $match = $2; 229 | my %values = (); 230 | $values{ geneId } = $geneId; 231 | 232 | foreach my $tag ( @tags_to_find ) { 233 | 234 | while ( $match =~ m|<$tag>(.*?)|gs ) { 235 | my $val = $1; 236 | next if $val eq ""; 237 | if ( exists $values{ $tag } ) { 238 | $values{ $tag } .= "|$val"; # multiple values concatenated with pipes 239 | } else { 240 | $values{ $tag } = $val; # first value seen for this tag and gene so far 241 | } 242 | } 243 | } 244 | 245 | my $out = ""; 246 | my $val; 247 | foreach my $tag ( @tags_to_find ) { 248 | 249 | $val = $values{ $tag } || "-"; 250 | $out .= "$val\t"; 251 | } 252 | chop($out); # remove last tab 253 | print "$out\n"; 254 | 255 | } 256 | } 257 | 258 | ################################################################################ 259 | # 260 | sub extractAndOutputXml { 261 | my $xml = shift; 262 | my $first = shift; 263 | 264 | # Strip leading tags, 265 | # from all but first set 266 | 267 | # Strip closing tag from all 268 | 269 | if ($first) { 270 | my $start = 0; 271 | my $stop = index ($xml, $stopTag); 272 | print STDOUT substr ($xml, $start, $stop-$start); 273 | } else { 274 | my $start = index ($xml, $startTag) + length($startTag); 275 | my $stop = index ($xml, $stopTag); 276 | print STDOUT substr ($xml, $start, $stop-$start); 277 | } 278 | } 279 | 280 | ################################################################################ 281 | # 282 | # Subroutine to handle all eutil calls 283 | # 284 | # Create a BEGIN block to provide effect of static data for sub 285 | BEGIN { 286 | # static data 287 | my $lastEutilTime = 0; # init to avoid delay on first Eutil 288 | 289 | # storing local constants here too. 290 | my $eutilBaseUrl = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils"; 291 | my $minUtilPeriod = 3; # seconds 292 | 293 | sub Eutil { 294 | my ($eutil, $querystringData) = @_; 295 | 296 | my $elapsed = time() - $lastEutilTime; 297 | my $delay = $minUtilPeriod - $elapsed; 298 | if ($delay < 0) {$delay = 0} 299 | sleep ($delay); 300 | $lastEutilTime = time(); # save for next time 301 | 302 | my $eutilUrl = "$eutilBaseUrl/$eutil.fcgi?$querystringData"; 303 | if ($verbose) {print STDOUT "\neutilUrl: $eutilUrl\n";} 304 | 305 | my $result = get($eutilUrl); # get result of the eutil for return 306 | if ((not defined $result) or ($result eq "")) 307 | { 308 | $result = ""; # Simplify error testing on return 309 | print STDERR "$eutil failed for request: $querystringData\n\n"; 310 | } 311 | 312 | if ($verbose) { 313 | print STDOUT "\neutil result: $result\n"; 314 | my $elapsedTime = time() - $lastEutilTime; 315 | print STDOUT "$eutil took $elapsedTime seconds\n"; 316 | } 317 | 318 | $result; # for return 319 | } 320 | } 321 | 322 | ################################################################################ 323 | # 324 | sub end { 325 | if ($verbose) { 326 | my $elapsedTime = time() - $beginTime; 327 | print STDOUT "\nElapsed time: $elapsedTime seconds\n"; 328 | } 329 | exit; 330 | } 331 | 332 | -------------------------------------------------------------------------------- /ss-bbr.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Auto install latest kernel for TCP BBR 4 | # 5 | # System Required: CentOS 6+, Debian7+, Ubuntu12+ 6 | # 7 | # Copyright (C) 2016-2018 Teddysun 8 | # 9 | # URL: https://teddysun.com/489.html 10 | # 11 | 12 | red='\033[0;31m' 13 | green='\033[0;32m' 14 | yellow='\033[0;33m' 15 | plain='\033[0m' 16 | 17 | cur_dir=$(pwd) 18 | 19 | [[ $EUID -ne 0 ]] && echo -e "${red}Error:${plain} This script must be run as root!" && exit 1 20 | 21 | [[ -d "/proc/vz" ]] && echo -e "${red}Error:${plain} Your VPS is based on OpenVZ, which is not supported." && exit 1 22 | 23 | if [ -f /etc/redhat-release ]; then 24 | release="centos" 25 | elif cat /etc/issue | grep -Eqi "debian"; then 26 | release="debian" 27 | elif cat /etc/issue | grep -Eqi "ubuntu"; then 28 | release="ubuntu" 29 | elif cat /etc/issue | grep -Eqi "centos|red hat|redhat"; then 30 | release="centos" 31 | elif cat /proc/version | grep -Eqi "debian"; then 32 | release="debian" 33 | elif cat /proc/version | grep -Eqi "ubuntu"; then 34 | release="ubuntu" 35 | elif cat /proc/version | grep -Eqi "centos|red hat|redhat"; then 36 | release="centos" 37 | else 38 | release="" 39 | fi 40 | 41 | is_digit(){ 42 | local input=${1} 43 | if [[ "$input" =~ ^[0-9]+$ ]]; then 44 | return 0 45 | else 46 | return 1 47 | fi 48 | } 49 | 50 | is_64bit(){ 51 | if [ $(getconf WORD_BIT) = '32' ] && [ $(getconf LONG_BIT) = '64' ]; then 52 | return 0 53 | else 54 | return 1 55 | fi 56 | } 57 | 58 | get_valid_valname(){ 59 | local val=${1} 60 | local new_val=$(eval echo $val | sed 's/[-.]/_/g') 61 | echo ${new_val} 62 | } 63 | 64 | get_hint(){ 65 | local val=${1} 66 | local new_val=$(get_valid_valname $val) 67 | eval echo "\$hint_${new_val}" 68 | } 69 | 70 | #Display Memu 71 | display_menu(){ 72 | local soft=${1} 73 | local default=${2} 74 | eval local arr=(\${${soft}_arr[@]}) 75 | local default_prompt 76 | if [[ "$default" != "" ]]; then 77 | if [[ "$default" == "last" ]]; then 78 | default=${#arr[@]} 79 | fi 80 | default_prompt="(default ${arr[$default-1]})" 81 | fi 82 | local pick 83 | local hint 84 | local vname 85 | local prompt="which ${soft} you'd select ${default_prompt}: " 86 | 87 | while : 88 | do 89 | echo -e "\n------------ ${soft} setting ------------\n" 90 | for ((i=1;i<=${#arr[@]};i++ )); do 91 | vname="$(get_valid_valname ${arr[$i-1]})" 92 | hint="$(get_hint $vname)" 93 | [[ "$hint" == "" ]] && hint="${arr[$i-1]}" 94 | echo -e "${green}${i}${plain}) $hint" 95 | done 96 | echo 97 | read -p "${prompt}" pick 98 | if [[ "$pick" == "" && "$default" != "" ]]; then 99 | pick=${default} 100 | break 101 | fi 102 | 103 | if ! is_digit "$pick"; then 104 | prompt="Input error, please input a number" 105 | continue 106 | fi 107 | 108 | if [[ "$pick" -lt 1 || "$pick" -gt ${#arr[@]} ]]; then 109 | prompt="Input error, please input a number between 1 and ${#arr[@]}: " 110 | continue 111 | fi 112 | 113 | break 114 | done 115 | 116 | eval ${soft}=${arr[$pick-1]} 117 | vname="$(get_valid_valname ${arr[$pick-1]})" 118 | hint="$(get_hint $vname)" 119 | [[ "$hint" == "" ]] && hint="${arr[$pick-1]}" 120 | echo -e "\nyour selection: $hint\n" 121 | } 122 | 123 | version_ge(){ 124 | test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1" 125 | } 126 | 127 | get_latest_version() { 128 | latest_version=($(wget -qO- https://kernel.ubuntu.com/~kernel-ppa/mainline/ | awk -F'\"v' '/v[4-9]./{print $2}' | cut -d/ -f1 | grep -v - | sort -V)) 129 | 130 | [ ${#latest_version[@]} -eq 0 ] && echo -e "${red}Error:${plain} Get latest kernel version failed." && exit 1 131 | 132 | kernel_arr=() 133 | for i in ${latest_version[@]}; do 134 | if version_ge $i 4.14; then 135 | kernel_arr+=($i); 136 | fi 137 | done 138 | 139 | display_menu kernel last 140 | 141 | if [[ `getconf WORD_BIT` == "32" && `getconf LONG_BIT` == "64" ]]; then 142 | deb_name=$(wget -qO- https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/ | grep "linux-image" | grep "generic" | awk -F'\">' '/amd64.deb/{print $2}' | cut -d'<' -f1 | head -1) 143 | deb_kernel_url="https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/${deb_name}" 144 | deb_kernel_name="linux-image-${kernel}-amd64.deb" 145 | modules_deb_name=$(wget -qO- https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/ | grep "linux-modules" | grep "generic" | awk -F'\">' '/amd64.deb/{print $2}' | cut -d'<' -f1 | head -1) 146 | deb_kernel_modules_url="https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/${modules_deb_name}" 147 | deb_kernel_modules_name="linux-modules-${kernel}-amd64.deb" 148 | else 149 | deb_name=$(wget -qO- https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/ | grep "linux-image" | grep "generic" | awk -F'\">' '/i386.deb/{print $2}' | cut -d'<' -f1 | head -1) 150 | deb_kernel_url="https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/${deb_name}" 151 | deb_kernel_name="linux-image-${kernel}-i386.deb" 152 | modules_deb_name=$(wget -qO- https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/ | grep "linux-modules" | grep "generic" | awk -F'\">' '/i386.deb/{print $2}' | cut -d'<' -f1 | head -1) 153 | deb_kernel_modules_url="https://kernel.ubuntu.com/~kernel-ppa/mainline/v${kernel}/${modules_deb_name}" 154 | deb_kernel_modules_name="linux-modules-${kernel}-i386.deb" 155 | fi 156 | 157 | [ -z ${deb_name} ] && echo -e "${red}Error:${plain} Getting Linux kernel binary package name failed, maybe kernel build failed. Please choose other one and try again." && exit 1 158 | } 159 | 160 | get_opsy() { 161 | [ -f /etc/redhat-release ] && awk '{print ($1,$3~/^[0-9]/?$3:$4)}' /etc/redhat-release && return 162 | [ -f /etc/os-release ] && awk -F'[= "]' '/PRETTY_NAME/{print $3,$4,$5}' /etc/os-release && return 163 | [ -f /etc/lsb-release ] && awk -F'[="]+' '/DESCRIPTION/{print $2}' /etc/lsb-release && return 164 | } 165 | 166 | opsy=$( get_opsy ) 167 | arch=$( uname -m ) 168 | lbit=$( getconf LONG_BIT ) 169 | kern=$( uname -r ) 170 | 171 | get_char() { 172 | SAVEDSTTY=`stty -g` 173 | stty -echo 174 | stty cbreak 175 | dd if=/dev/tty bs=1 count=1 2> /dev/null 176 | stty -raw 177 | stty echo 178 | stty $SAVEDSTTY 179 | } 180 | 181 | getversion() { 182 | if [[ -s /etc/redhat-release ]]; then 183 | grep -oE "[0-9.]+" /etc/redhat-release 184 | else 185 | grep -oE "[0-9.]+" /etc/issue 186 | fi 187 | } 188 | 189 | centosversion() { 190 | if [ x"${release}" == x"centos" ]; then 191 | local code=$1 192 | local version="$(getversion)" 193 | local main_ver=${version%%.*} 194 | if [ "$main_ver" == "$code" ]; then 195 | return 0 196 | else 197 | return 1 198 | fi 199 | else 200 | return 1 201 | fi 202 | } 203 | 204 | check_bbr_status() { 205 | local param=$(sysctl net.ipv4.tcp_congestion_control | awk '{print $3}') 206 | if [[ x"${param}" == x"bbr" ]]; then 207 | return 0 208 | else 209 | return 1 210 | fi 211 | } 212 | 213 | check_kernel_version() { 214 | local kernel_version=$(uname -r | cut -d- -f1) 215 | if version_ge ${kernel_version} 4.9; then 216 | return 0 217 | else 218 | return 1 219 | fi 220 | } 221 | 222 | install_elrepo() { 223 | 224 | if centosversion 5; then 225 | echo -e "${red}Error:${plain} not supported CentOS 5." 226 | exit 1 227 | fi 228 | 229 | rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org 230 | 231 | if centosversion 6; then 232 | rpm -Uvh https://www.elrepo.org/elrepo-release-6-8.el6.elrepo.noarch.rpm 233 | elif centosversion 7; then 234 | rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm 235 | fi 236 | 237 | if [ ! -f /etc/yum.repos.d/elrepo.repo ]; then 238 | echo -e "${red}Error:${plain} Install elrepo failed, please check it." 239 | exit 1 240 | fi 241 | } 242 | 243 | sysctl_config() { 244 | sed -i '/net.core.default_qdisc/d' /etc/sysctl.conf 245 | sed -i '/net.ipv4.tcp_congestion_control/d' /etc/sysctl.conf 246 | echo "net.core.default_qdisc = fq" >> /etc/sysctl.conf 247 | echo "net.ipv4.tcp_congestion_control = bbr" >> /etc/sysctl.conf 248 | sysctl -p >/dev/null 2>&1 249 | } 250 | 251 | install_config() { 252 | if [[ x"${release}" == x"centos" ]]; then 253 | if centosversion 6; then 254 | if [ ! -f "/boot/grub/grub.conf" ]; then 255 | echo -e "${red}Error:${plain} /boot/grub/grub.conf not found, please check it." 256 | exit 1 257 | fi 258 | sed -i 's/^default=.*/default=0/g' /boot/grub/grub.conf 259 | elif centosversion 7; then 260 | if [ ! -f "/boot/grub2/grub.cfg" ]; then 261 | echo -e "${red}Error:${plain} /boot/grub2/grub.cfg not found, please check it." 262 | exit 1 263 | fi 264 | grub2-set-default 0 265 | fi 266 | elif [[ x"${release}" == x"debian" || x"${release}" == x"ubuntu" ]]; then 267 | /usr/sbin/update-grub 268 | fi 269 | } 270 | 271 | reboot_os() { 272 | echo 273 | echo -e "${green}Info:${plain} The system needs to reboot." 274 | read -p "Do you want to restart system? [y/n]" is_reboot 275 | if [[ ${is_reboot} == "y" || ${is_reboot} == "Y" ]]; then 276 | reboot 277 | else 278 | echo -e "${green}Info:${plain} Reboot has been canceled..." 279 | exit 0 280 | fi 281 | } 282 | 283 | install_bbr() { 284 | check_bbr_status 285 | if [ $? -eq 0 ]; then 286 | echo 287 | echo -e "${green}Info:${plain} TCP BBR has already been installed. nothing to do..." 288 | exit 0 289 | fi 290 | check_kernel_version 291 | if [ $? -eq 0 ]; then 292 | echo 293 | echo -e "${green}Info:${plain} Your kernel version is greater than 4.9, directly setting TCP BBR..." 294 | sysctl_config 295 | echo -e "${green}Info:${plain} Setting TCP BBR completed..." 296 | exit 0 297 | fi 298 | 299 | if [[ x"${release}" == x"centos" ]]; then 300 | install_elrepo 301 | [ ! "$(command -v yum-config-manager)" ] && yum install -y yum-utils > /dev/null 2>&1 302 | [ x"$(yum-config-manager elrepo-kernel | grep -w enabled | awk '{print $3}')" != x"True" ] && yum-config-manager --enable elrepo-kernel > /dev/null 2>&1 303 | if centosversion 6; then 304 | if is_64bit; then 305 | rpm_kernel_name="kernel-ml-4.18.20-1.el6.elrepo.x86_64.rpm" 306 | rpm_kernel_devel_name="kernel-ml-devel-4.18.20-1.el6.elrepo.x86_64.rpm" 307 | rpm_kernel_url_1="http://repos.lax.quadranet.com/elrepo/archive/kernel/el6/x86_64/RPMS/" 308 | else 309 | rpm_kernel_name="kernel-ml-4.18.20-1.el6.elrepo.i686.rpm" 310 | rpm_kernel_devel_name="kernel-ml-devel-4.18.20-1.el6.elrepo.i686.rpm" 311 | rpm_kernel_url_1="http://repos.lax.quadranet.com/elrepo/archive/kernel/el6/i386/RPMS/" 312 | fi 313 | rpm_kernel_url_2="https://dl.lamp.sh/files/" 314 | wget -c -t3 -T60 -O ${rpm_kernel_name} ${rpm_kernel_url_1}${rpm_kernel_name} 315 | if [ $? -ne 0 ]; then 316 | rm -rf ${rpm_kernel_name} 317 | wget -c -t3 -T60 -O ${rpm_kernel_name} ${rpm_kernel_url_2}${rpm_kernel_name} 318 | fi 319 | wget -c -t3 -T60 -O ${rpm_kernel_devel_name} ${rpm_kernel_url_1}${rpm_kernel_devel_name} 320 | if [ $? -ne 0 ]; then 321 | rm -rf ${rpm_kernel_devel_name} 322 | wget -c -t3 -T60 -O ${rpm_kernel_devel_name} ${rpm_kernel_url_2}${rpm_kernel_devel_name} 323 | fi 324 | if [ -f "${rpm_kernel_name}" ]; then 325 | rpm -ivh ${rpm_kernel_name} 326 | else 327 | echo -e "${red}Error:${plain} Download ${rpm_kernel_name} failed, please check it." 328 | exit 1 329 | fi 330 | if [ -f "${rpm_kernel_devel_name}" ]; then 331 | rpm -ivh ${rpm_kernel_devel_name} 332 | else 333 | echo -e "${red}Error:${plain} Download ${rpm_kernel_devel_name} failed, please check it." 334 | exit 1 335 | fi 336 | rm -f ${rpm_kernel_name} ${rpm_kernel_devel_name} 337 | elif centosversion 7; then 338 | yum -y install kernel-ml kernel-ml-devel 339 | if [ $? -ne 0 ]; then 340 | echo -e "${red}Error:${plain} Install latest kernel failed, please check it." 341 | exit 1 342 | fi 343 | fi 344 | elif [[ x"${release}" == x"debian" || x"${release}" == x"ubuntu" ]]; then 345 | [[ ! -e "/usr/bin/wget" ]] && apt-get -y update && apt-get -y install wget 346 | echo -e "${green}Info:${plain} Getting latest kernel version..." 347 | get_latest_version 348 | if [ -n ${modules_deb_name} ]; then 349 | wget -c -t3 -T60 -O ${deb_kernel_modules_name} ${deb_kernel_modules_url} 350 | if [ $? -ne 0 ]; then 351 | echo -e "${red}Error:${plain} Download ${deb_kernel_modules_name} failed, please check it." 352 | exit 1 353 | fi 354 | fi 355 | wget -c -t3 -T60 -O ${deb_kernel_name} ${deb_kernel_url} 356 | if [ $? -ne 0 ]; then 357 | echo -e "${red}Error:${plain} Download ${deb_kernel_name} failed, please check it." 358 | exit 1 359 | fi 360 | [ -f ${deb_kernel_modules_name} ] && dpkg -i ${deb_kernel_modules_name} 361 | dpkg -i ${deb_kernel_name} 362 | rm -f ${deb_kernel_name} ${deb_kernel_modules_name} 363 | else 364 | echo -e "${red}Error:${plain} OS is not be supported, please change to CentOS/Debian/Ubuntu and try again." 365 | exit 1 366 | fi 367 | 368 | install_config 369 | sysctl_config 370 | reboot_os 371 | } 372 | 373 | 374 | clear 375 | echo "---------- System Information ----------" 376 | echo " OS : $opsy" 377 | echo " Arch : $arch ($lbit Bit)" 378 | echo " Kernel : $kern" 379 | echo "----------------------------------------" 380 | echo " Auto install latest kernel for TCP BBR" 381 | echo 382 | echo " URL: https://teddysun.com/489.html" 383 | echo "----------------------------------------" 384 | echo 385 | echo "Press any key to start...or Press Ctrl+C to cancel" 386 | char=`get_char` 387 | 388 | install_bbr 2>&1 | tee ${cur_dir}/install_bbr.log 389 | -------------------------------------------------------------------------------- /ss.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin 3 | export PATH 4 | #=================================================================# 5 | # System Required: CentOS 6+, Debian 7+, Ubuntu 12+ # 6 | # Description: One click Install Shadowsocks-Python server # 7 | # Author: Teddysun # 8 | # Thanks: @clowwindy # 9 | # Intro: https://teddysun.com/342.html # 10 | #=================================================================# 11 | 12 | clear 13 | echo 14 | echo "#############################################################" 15 | echo "# One click Install Shadowsocks-Python server #" 16 | echo "# Intro: https://teddysun.com/342.html #" 17 | echo "# Author: Teddysun #" 18 | echo "# Github: https://github.com/shadowsocks/shadowsocks #" 19 | echo "#############################################################" 20 | echo 21 | 22 | libsodium_file="libsodium-1.0.16" 23 | libsodium_url="https://github.com/jedisct1/libsodium/releases/download/1.0.16/libsodium-1.0.16.tar.gz" 24 | 25 | # Current folder 26 | cur_dir=`pwd` 27 | # Stream Ciphers 28 | ciphers=( 29 | aes-256-gcm 30 | aes-192-gcm 31 | aes-128-gcm 32 | aes-256-ctr 33 | aes-192-ctr 34 | aes-128-ctr 35 | aes-256-cfb 36 | aes-192-cfb 37 | aes-128-cfb 38 | camellia-128-cfb 39 | camellia-192-cfb 40 | camellia-256-cfb 41 | chacha20-ietf-poly1305 42 | chacha20-ietf 43 | chacha20 44 | rc4-md5 45 | ) 46 | # Color 47 | red='\033[0;31m' 48 | green='\033[0;32m' 49 | yellow='\033[0;33m' 50 | plain='\033[0m' 51 | 52 | # Make sure only root can run our script 53 | [[ $EUID -ne 0 ]] && echo -e "[${red}Error${plain}] This script must be run as root!" && exit 1 54 | 55 | # Disable selinux 56 | disable_selinux(){ 57 | if [ -s /etc/selinux/config ] && grep 'SELINUX=enforcing' /etc/selinux/config; then 58 | sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config 59 | setenforce 0 60 | fi 61 | } 62 | 63 | #Check system 64 | check_sys(){ 65 | local checkType=$1 66 | local value=$2 67 | 68 | local release='' 69 | local systemPackage='' 70 | 71 | if [[ -f /etc/redhat-release ]]; then 72 | release="centos" 73 | systemPackage="yum" 74 | elif grep -Eqi "debian" /etc/issue; then 75 | release="debian" 76 | systemPackage="apt" 77 | elif grep -Eqi "ubuntu" /etc/issue; then 78 | release="ubuntu" 79 | systemPackage="apt" 80 | elif grep -Eqi "centos|red hat|redhat" /etc/issue; then 81 | release="centos" 82 | systemPackage="yum" 83 | elif grep -Eqi "debian" /proc/version; then 84 | release="debian" 85 | systemPackage="apt" 86 | elif grep -Eqi "ubuntu" /proc/version; then 87 | release="ubuntu" 88 | systemPackage="apt" 89 | elif grep -Eqi "centos|red hat|redhat" /proc/version; then 90 | release="centos" 91 | systemPackage="yum" 92 | fi 93 | 94 | if [[ "${checkType}" == "sysRelease" ]]; then 95 | if [ "${value}" == "${release}" ]; then 96 | return 0 97 | else 98 | return 1 99 | fi 100 | elif [[ "${checkType}" == "packageManager" ]]; then 101 | if [ "${value}" == "${systemPackage}" ]; then 102 | return 0 103 | else 104 | return 1 105 | fi 106 | fi 107 | } 108 | 109 | # Get version 110 | getversion(){ 111 | if [[ -s /etc/redhat-release ]]; then 112 | grep -oE "[0-9.]+" /etc/redhat-release 113 | else 114 | grep -oE "[0-9.]+" /etc/issue 115 | fi 116 | } 117 | 118 | # CentOS version 119 | centosversion(){ 120 | if check_sys sysRelease centos; then 121 | local code=$1 122 | local version="$(getversion)" 123 | local main_ver=${version%%.*} 124 | if [ "$main_ver" == "$code" ]; then 125 | return 0 126 | else 127 | return 1 128 | fi 129 | else 130 | return 1 131 | fi 132 | } 133 | 134 | # Get public IP address 135 | get_ip(){ 136 | local IP=$( ip addr | egrep -o '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | egrep -v "^192\.168|^172\.1[6-9]\.|^172\.2[0-9]\.|^172\.3[0-2]\.|^10\.|^127\.|^255\.|^0\." | head -n 1 ) 137 | [ -z ${IP} ] && IP=$( wget -qO- -t1 -T2 ipv4.icanhazip.com ) 138 | [ -z ${IP} ] && IP=$( wget -qO- -t1 -T2 ipinfo.io/ip ) 139 | [ ! -z ${IP} ] && echo ${IP} || echo 140 | } 141 | 142 | get_char(){ 143 | SAVEDSTTY=`stty -g` 144 | stty -echo 145 | stty cbreak 146 | dd if=/dev/tty bs=1 count=1 2> /dev/null 147 | stty -raw 148 | stty echo 149 | stty $SAVEDSTTY 150 | } 151 | 152 | # Pre-installation settings 153 | pre_install(){ 154 | if check_sys packageManager yum || check_sys packageManager apt; then 155 | # Not support CentOS 5 156 | if centosversion 5; then 157 | echo -e "$[{red}Error${plain}] Not supported CentOS 5, please change to CentOS 6+/Debian 7+/Ubuntu 12+ and try again." 158 | exit 1 159 | fi 160 | else 161 | echo -e "[${red}Error${plain}] Your OS is not supported. please change OS to CentOS/Debian/Ubuntu and try again." 162 | exit 1 163 | fi 164 | # Set shadowsocks config password 165 | echo "Please enter password for shadowsocks-python" 166 | read -p "(Default password: teddysun.com):" shadowsockspwd 167 | [ -z "${shadowsockspwd}" ] && shadowsockspwd="teddysun.com" 168 | echo 169 | echo "---------------------------" 170 | echo "password = ${shadowsockspwd}" 171 | echo "---------------------------" 172 | echo 173 | # Set shadowsocks config port 174 | while true 175 | do 176 | dport=$(shuf -i 9000-19999 -n 1) 177 | echo "Please enter a port for shadowsocks-python [1-65535]" 178 | read -p "(Default port: ${dport}):" shadowsocksport 179 | [ -z "$shadowsocksport" ] && shadowsocksport=${dport} 180 | expr ${shadowsocksport} + 1 &>/dev/null 181 | if [ $? -eq 0 ]; then 182 | if [ ${shadowsocksport} -ge 1 ] && [ ${shadowsocksport} -le 65535 ] && [ ${shadowsocksport:0:1} != 0 ]; then 183 | echo 184 | echo "---------------------------" 185 | echo "port = ${shadowsocksport}" 186 | echo "---------------------------" 187 | echo 188 | break 189 | fi 190 | fi 191 | echo -e "[${red}Error${plain}] Please enter a correct number [1-65535]" 192 | done 193 | 194 | # Set shadowsocks config stream ciphers 195 | while true 196 | do 197 | echo -e "Please select stream cipher for shadowsocks-python:" 198 | for ((i=1;i<=${#ciphers[@]};i++ )); do 199 | hint="${ciphers[$i-1]}" 200 | echo -e "${green}${i}${plain}) ${hint}" 201 | done 202 | read -p "Which cipher you'd select(Default: ${ciphers[0]}):" pick 203 | [ -z "$pick" ] && pick=1 204 | expr ${pick} + 1 &>/dev/null 205 | if [ $? -ne 0 ]; then 206 | echo -e "[${red}Error${plain}] Please enter a number" 207 | continue 208 | fi 209 | if [[ "$pick" -lt 1 || "$pick" -gt ${#ciphers[@]} ]]; then 210 | echo -e "[${red}Error${plain}] Please enter a number between 1 and ${#ciphers[@]}" 211 | continue 212 | fi 213 | shadowsockscipher=${ciphers[$pick-1]} 214 | echo 215 | echo "---------------------------" 216 | echo "cipher = ${shadowsockscipher}" 217 | echo "---------------------------" 218 | echo 219 | break 220 | done 221 | 222 | echo 223 | echo "Press any key to start...or Press Ctrl+C to cancel" 224 | char=`get_char` 225 | # Install necessary dependencies 226 | if check_sys packageManager yum; then 227 | yum install -y python python-devel python-setuptools openssl openssl-devel curl wget unzip gcc automake autoconf make libtool 228 | elif check_sys packageManager apt; then 229 | apt-get -y update 230 | apt-get -y install python python-dev python-setuptools openssl libssl-dev curl wget unzip gcc automake autoconf make libtool 231 | fi 232 | cd ${cur_dir} 233 | } 234 | 235 | # Download files 236 | download_files(){ 237 | # Download libsodium file 238 | if ! wget --no-check-certificate -O ${libsodium_file}.tar.gz ${libsodium_url}; then 239 | echo -e "[${red}Error${plain}] Failed to download ${libsodium_file}.tar.gz!" 240 | exit 1 241 | fi 242 | # Download Shadowsocks file 243 | if ! wget --no-check-certificate -O shadowsocks-master.zip https://github.com/shadowsocks/shadowsocks/archive/master.zip; then 244 | echo -e "[${red}Error${plain}] Failed to download shadowsocks python file!" 245 | exit 1 246 | fi 247 | # Download Shadowsocks init script 248 | if check_sys packageManager yum; then 249 | if ! wget --no-check-certificate https://raw.githubusercontent.com/teddysun/shadowsocks_install/master/shadowsocks -O /etc/init.d/shadowsocks; then 250 | echo -e "[${red}Error${plain}] Failed to download shadowsocks chkconfig file!" 251 | exit 1 252 | fi 253 | elif check_sys packageManager apt; then 254 | if ! wget --no-check-certificate https://raw.githubusercontent.com/teddysun/shadowsocks_install/master/shadowsocks-debian -O /etc/init.d/shadowsocks; then 255 | echo -e "[${red}Error${plain}] Failed to download shadowsocks chkconfig file!" 256 | exit 1 257 | fi 258 | fi 259 | } 260 | 261 | # Config shadowsocks 262 | config_shadowsocks(){ 263 | cat > /etc/shadowsocks.json<<-EOF 264 | { 265 | "server":"0.0.0.0", 266 | "server_port":${shadowsocksport}, 267 | "local_address":"127.0.0.1", 268 | "local_port":1080, 269 | "password":"${shadowsockspwd}", 270 | "timeout":300, 271 | "method":"${shadowsockscipher}", 272 | "fast_open":false 273 | } 274 | EOF 275 | } 276 | 277 | # Firewall set 278 | firewall_set(){ 279 | echo -e "[${green}Info${plain}] firewall set start..." 280 | if centosversion 6; then 281 | /etc/init.d/iptables status > /dev/null 2>&1 282 | if [ $? -eq 0 ]; then 283 | iptables -L -n | grep -i ${shadowsocksport} > /dev/null 2>&1 284 | if [ $? -ne 0 ]; then 285 | iptables -I INPUT -m state --state NEW -m tcp -p tcp --dport ${shadowsocksport} -j ACCEPT 286 | iptables -I INPUT -m state --state NEW -m udp -p udp --dport ${shadowsocksport} -j ACCEPT 287 | /etc/init.d/iptables save 288 | /etc/init.d/iptables restart 289 | else 290 | echo -e "[${green}Info${plain}] port ${shadowsocksport} has already been set up." 291 | fi 292 | else 293 | echo -e "[${yellow}Warning${plain}] iptables looks like shutdown or not installed, please manually set it if necessary." 294 | fi 295 | elif centosversion 7; then 296 | systemctl status firewalld > /dev/null 2>&1 297 | if [ $? -eq 0 ]; then 298 | firewall-cmd --permanent --zone=public --add-port=${shadowsocksport}/tcp 299 | firewall-cmd --permanent --zone=public --add-port=${shadowsocksport}/udp 300 | firewall-cmd --reload 301 | else 302 | echo -e "[${yellow}Warning${plain}] firewalld looks like not running or not installed, please enable port ${shadowsocksport} manually if necessary." 303 | fi 304 | fi 305 | echo -e "[${green}Info${plain}] firewall set completed..." 306 | } 307 | 308 | # Install Shadowsocks 309 | install(){ 310 | # Install libsodium 311 | if [ ! -f /usr/lib/libsodium.a ]; then 312 | cd ${cur_dir} 313 | tar zxf ${libsodium_file}.tar.gz 314 | cd ${libsodium_file} 315 | ./configure --prefix=/usr && make && make install 316 | if [ $? -ne 0 ]; then 317 | echo -e "[${red}Error${plain}] libsodium install failed!" 318 | install_cleanup 319 | exit 1 320 | fi 321 | fi 322 | 323 | ldconfig 324 | # Install Shadowsocks 325 | cd ${cur_dir} 326 | unzip -q shadowsocks-master.zip 327 | if [ $? -ne 0 ];then 328 | echo -e "[${red}Error${plain}] unzip shadowsocks-master.zip failed! please check unzip command." 329 | install_cleanup 330 | exit 1 331 | fi 332 | 333 | cd ${cur_dir}/shadowsocks-master 334 | python setup.py install --record /usr/local/shadowsocks_install.log 335 | 336 | if [ -f /usr/bin/ssserver ] || [ -f /usr/local/bin/ssserver ]; then 337 | chmod +x /etc/init.d/shadowsocks 338 | if check_sys packageManager yum; then 339 | chkconfig --add shadowsocks 340 | chkconfig shadowsocks on 341 | elif check_sys packageManager apt; then 342 | update-rc.d -f shadowsocks defaults 343 | fi 344 | /etc/init.d/shadowsocks start 345 | else 346 | echo 347 | echo -e "[${red}Error${plain}] Shadowsocks install failed! please visit https://teddysun.com/342.html and contact." 348 | install_cleanup 349 | exit 1 350 | fi 351 | 352 | clear 353 | echo 354 | echo -e "Congratulations, Shadowsocks-python server install completed!" 355 | echo -e "Your Server IP : \033[41;37m $(get_ip) \033[0m" 356 | echo -e "Your Server Port : \033[41;37m ${shadowsocksport} \033[0m" 357 | echo -e "Your Password : \033[41;37m ${shadowsockspwd} \033[0m" 358 | echo -e "Your Encryption Method: \033[41;37m ${shadowsockscipher} \033[0m" 359 | echo 360 | echo "Welcome to visit:https://teddysun.com/342.html" 361 | echo "Enjoy it!" 362 | echo 363 | } 364 | 365 | # Install cleanup 366 | install_cleanup(){ 367 | cd ${cur_dir} 368 | rm -rf shadowsocks-master.zip shadowsocks-master ${libsodium_file}.tar.gz ${libsodium_file} 369 | } 370 | 371 | # Uninstall Shadowsocks 372 | uninstall_shadowsocks(){ 373 | printf "Are you sure uninstall Shadowsocks? (y/n) " 374 | printf "\n" 375 | read -p "(Default: n):" answer 376 | [ -z ${answer} ] && answer="n" 377 | if [ "${answer}" == "y" ] || [ "${answer}" == "Y" ]; then 378 | ps -ef | grep -v grep | grep -i "ssserver" > /dev/null 2>&1 379 | if [ $? -eq 0 ]; then 380 | /etc/init.d/shadowsocks stop 381 | fi 382 | if check_sys packageManager yum; then 383 | chkconfig --del shadowsocks 384 | elif check_sys packageManager apt; then 385 | update-rc.d -f shadowsocks remove 386 | fi 387 | # delete config file 388 | rm -f /etc/shadowsocks.json 389 | rm -f /var/run/shadowsocks.pid 390 | rm -f /etc/init.d/shadowsocks 391 | rm -f /var/log/shadowsocks.log 392 | if [ -f /usr/local/shadowsocks_install.log ]; then 393 | cat /usr/local/shadowsocks_install.log | xargs rm -rf 394 | fi 395 | echo "Shadowsocks uninstall success!" 396 | else 397 | echo 398 | echo "uninstall cancelled, nothing to do..." 399 | echo 400 | fi 401 | } 402 | 403 | # Install Shadowsocks-python 404 | install_shadowsocks(){ 405 | disable_selinux 406 | pre_install 407 | download_files 408 | config_shadowsocks 409 | if check_sys packageManager yum; then 410 | firewall_set 411 | fi 412 | install 413 | install_cleanup 414 | } 415 | 416 | # Initialization step 417 | action=$1 418 | [ -z $1 ] && action=install 419 | case "$action" in 420 | install|uninstall) 421 | ${action}_shadowsocks 422 | ;; 423 | *) 424 | echo "Arguments error! [${action}]" 425 | echo "Usage: `basename $0` [install|uninstall]" 426 | ;; 427 | esac -------------------------------------------------------------------------------- /slurm-parallel/scriptfile.sh: -------------------------------------------------------------------------------- 1 | sleep 10 2 | sleep 20 3 | sleep 10 4 | sleep 30 5 | sleep 20 6 | sleep 10 7 | sleep 30 8 | sleep 20 9 | sleep 10 10 | sleep 20 11 | sleep 10 12 | sleep 30 13 | sleep 20 14 | sleep 10 15 | sleep 30 16 | sleep 20 17 | sleep 10 18 | sleep 20 19 | sleep 10 20 | sleep 30 21 | sleep 20 22 | sleep 10 23 | sleep 30 24 | sleep 20 25 | sleep 10 26 | sleep 20 27 | sleep 10 28 | sleep 30 29 | sleep 20 30 | sleep 10 31 | sleep 30 32 | sleep 20 33 | sleep 10 34 | sleep 20 35 | sleep 10 36 | sleep 30 37 | sleep 20 38 | sleep 10 39 | sleep 30 40 | sleep 20 41 | sleep 10 42 | sleep 20 43 | sleep 10 44 | sleep 30 45 | sleep 20 46 | sleep 10 47 | sleep 30 48 | sleep 20 49 | sleep 10 50 | sleep 20 51 | sleep 10 52 | sleep 30 53 | sleep 20 54 | sleep 10 55 | sleep 30 56 | sleep 20 57 | sleep 10 58 | sleep 20 59 | sleep 10 60 | sleep 30 61 | sleep 20 62 | sleep 10 63 | sleep 30 64 | sleep 20 65 | sleep 10 66 | sleep 20 67 | sleep 10 68 | sleep 30 69 | sleep 20 70 | sleep 10 71 | sleep 30 72 | sleep 20 73 | sleep 10 74 | sleep 20 75 | sleep 10 76 | sleep 30 77 | sleep 20 78 | sleep 10 79 | sleep 30 80 | sleep 20 81 | sleep 10 82 | sleep 20 83 | sleep 10 84 | sleep 30 85 | sleep 20 86 | sleep 10 87 | sleep 30 88 | sleep 20 89 | sleep 10 90 | sleep 20 91 | sleep 10 92 | sleep 30 93 | sleep 20 94 | sleep 10 95 | sleep 30 96 | sleep 20 97 | sleep 10 98 | sleep 20 99 | sleep 10 100 | sleep 30 101 | sleep 20 102 | sleep 10 103 | sleep 30 104 | sleep 20 105 | sleep 10 106 | sleep 20 107 | sleep 10 108 | sleep 30 109 | sleep 20 110 | sleep 10 111 | sleep 30 112 | sleep 20 113 | sleep 10 114 | sleep 20 115 | sleep 10 116 | sleep 30 117 | sleep 20 118 | sleep 10 119 | sleep 30 120 | sleep 20 121 | sleep 10 122 | sleep 20 123 | sleep 10 124 | sleep 30 125 | sleep 20 126 | sleep 10 127 | sleep 30 128 | sleep 20 129 | sleep 10 130 | sleep 20 131 | sleep 10 132 | sleep 30 133 | sleep 20 134 | sleep 10 135 | sleep 30 136 | sleep 20 137 | sleep 10 138 | sleep 20 139 | sleep 10 140 | sleep 30 141 | sleep 20 142 | sleep 10 143 | sleep 30 144 | sleep 20 145 | sleep 10 146 | sleep 20 147 | sleep 10 148 | sleep 30 149 | sleep 20 150 | sleep 10 151 | sleep 30 152 | sleep 20 153 | sleep 10 154 | sleep 20 155 | sleep 10 156 | sleep 30 157 | sleep 20 158 | sleep 10 159 | sleep 30 160 | sleep 20 161 | sleep 10 162 | sleep 20 163 | sleep 10 164 | sleep 30 165 | sleep 20 166 | sleep 10 167 | sleep 30 168 | sleep 20 169 | sleep 10 170 | sleep 20 171 | sleep 10 172 | sleep 30 173 | sleep 20 174 | sleep 10 175 | sleep 30 176 | sleep 20 177 | sleep 10 178 | sleep 20 179 | sleep 10 180 | sleep 30 181 | sleep 20 182 | sleep 10 183 | sleep 30 184 | sleep 20 185 | sleep 10 186 | sleep 20 187 | sleep 10 188 | sleep 30 189 | sleep 20 190 | sleep 10 191 | sleep 30 192 | sleep 20 193 | sleep 10 194 | sleep 20 195 | sleep 10 196 | sleep 30 197 | sleep 20 198 | sleep 10 199 | sleep 30 200 | sleep 20 201 | sleep 10 202 | sleep 20 203 | sleep 10 204 | sleep 30 205 | sleep 20 206 | sleep 10 207 | sleep 30 208 | sleep 20 209 | sleep 10 210 | sleep 20 211 | sleep 10 212 | sleep 30 213 | sleep 20 214 | sleep 10 215 | sleep 30 216 | sleep 20 217 | sleep 10 218 | sleep 20 219 | sleep 10 220 | sleep 30 221 | sleep 20 222 | sleep 10 223 | sleep 30 224 | sleep 20 225 | sleep 10 226 | sleep 20 227 | sleep 10 228 | sleep 30 229 | sleep 20 230 | sleep 10 231 | sleep 30 232 | sleep 20 233 | sleep 10 234 | sleep 20 235 | sleep 10 236 | sleep 30 237 | sleep 20 238 | sleep 10 239 | sleep 30 240 | sleep 20 241 | sleep 10 242 | sleep 20 243 | sleep 10 244 | sleep 30 245 | sleep 20 246 | sleep 10 247 | sleep 30 248 | sleep 20 249 | sleep 10 250 | sleep 20 251 | sleep 10 252 | sleep 30 253 | sleep 20 254 | sleep 10 255 | sleep 30 256 | sleep 20 257 | sleep 10 258 | sleep 20 259 | sleep 10 260 | sleep 30 261 | sleep 20 262 | sleep 10 263 | sleep 30 264 | sleep 20 265 | sleep 10 266 | sleep 20 267 | sleep 10 268 | sleep 30 269 | sleep 20 270 | sleep 10 271 | sleep 30 272 | sleep 20 273 | sleep 10 274 | sleep 20 275 | sleep 10 276 | sleep 30 277 | sleep 20 278 | sleep 10 279 | sleep 30 280 | sleep 20 281 | sleep 10 282 | sleep 20 283 | sleep 10 284 | sleep 30 285 | sleep 20 286 | sleep 10 287 | sleep 30 288 | sleep 20 289 | sleep 10 290 | sleep 20 291 | sleep 10 292 | sleep 30 293 | sleep 20 294 | sleep 10 295 | sleep 30 296 | sleep 20 297 | sleep 10 298 | sleep 20 299 | sleep 10 300 | sleep 30 301 | sleep 20 302 | sleep 10 303 | sleep 30 304 | sleep 20 305 | sleep 10 306 | sleep 20 307 | sleep 10 308 | sleep 30 309 | sleep 20 310 | sleep 10 311 | sleep 30 312 | sleep 20 313 | sleep 10 314 | sleep 20 315 | sleep 10 316 | sleep 30 317 | sleep 20 318 | sleep 10 319 | sleep 30 320 | sleep 20 321 | sleep 10 322 | sleep 20 323 | sleep 10 324 | sleep 30 325 | sleep 20 326 | sleep 10 327 | sleep 30 328 | sleep 20 329 | sleep 10 330 | sleep 20 331 | sleep 10 332 | sleep 30 333 | sleep 20 334 | sleep 10 335 | sleep 30 336 | sleep 20 337 | sleep 10 338 | sleep 20 339 | sleep 10 340 | sleep 30 341 | sleep 20 342 | sleep 10 343 | sleep 30 344 | sleep 20 345 | sleep 10 346 | sleep 20 347 | sleep 10 348 | sleep 30 349 | sleep 20 350 | sleep 10 351 | sleep 30 352 | sleep 20 353 | sleep 10 354 | sleep 20 355 | sleep 10 356 | sleep 30 357 | sleep 20 358 | sleep 10 359 | sleep 30 360 | sleep 20 361 | sleep 10 362 | sleep 20 363 | sleep 10 364 | sleep 30 365 | sleep 20 366 | sleep 10 367 | sleep 30 368 | sleep 20 369 | sleep 10 370 | sleep 20 371 | sleep 10 372 | sleep 30 373 | sleep 20 374 | sleep 10 375 | sleep 30 376 | sleep 20 377 | sleep 10 378 | sleep 20 379 | sleep 10 380 | sleep 30 381 | sleep 20 382 | sleep 10 383 | sleep 30 384 | sleep 20 385 | sleep 10 386 | sleep 20 387 | sleep 10 388 | sleep 30 389 | sleep 20 390 | sleep 10 391 | sleep 30 392 | sleep 20 393 | sleep 10 394 | sleep 20 395 | sleep 10 396 | sleep 30 397 | sleep 20 398 | sleep 10 399 | sleep 30 400 | sleep 20 401 | sleep 10 402 | sleep 20 403 | sleep 10 404 | sleep 30 405 | sleep 20 406 | sleep 10 407 | sleep 30 408 | sleep 20 409 | sleep 10 410 | sleep 20 411 | sleep 10 412 | sleep 30 413 | sleep 20 414 | sleep 10 415 | sleep 30 416 | sleep 20 417 | sleep 10 418 | sleep 20 419 | sleep 10 420 | sleep 30 421 | sleep 20 422 | sleep 10 423 | sleep 30 424 | sleep 20 425 | sleep 10 426 | sleep 20 427 | sleep 10 428 | sleep 30 429 | sleep 20 430 | sleep 10 431 | sleep 30 432 | sleep 20 433 | sleep 10 434 | sleep 20 435 | sleep 10 436 | sleep 30 437 | sleep 20 438 | sleep 10 439 | sleep 30 440 | sleep 20 441 | sleep 10 442 | sleep 20 443 | sleep 10 444 | sleep 30 445 | sleep 20 446 | sleep 10 447 | sleep 30 448 | sleep 20 449 | sleep 10 450 | sleep 20 451 | sleep 10 452 | sleep 30 453 | sleep 20 454 | sleep 10 455 | sleep 30 456 | sleep 20 457 | sleep 10 458 | sleep 20 459 | sleep 10 460 | sleep 30 461 | sleep 20 462 | sleep 10 463 | sleep 30 464 | sleep 20 465 | sleep 10 466 | sleep 20 467 | sleep 10 468 | sleep 30 469 | sleep 20 470 | sleep 10 471 | sleep 30 472 | sleep 20 473 | sleep 10 474 | sleep 20 475 | sleep 10 476 | sleep 30 477 | sleep 20 478 | sleep 10 479 | sleep 30 480 | sleep 20 481 | sleep 10 482 | sleep 20 483 | sleep 10 484 | sleep 30 485 | sleep 20 486 | sleep 10 487 | sleep 30 488 | sleep 20 489 | sleep 10 490 | sleep 20 491 | sleep 10 492 | sleep 30 493 | sleep 20 494 | sleep 10 495 | sleep 30 496 | sleep 20 497 | sleep 10 498 | sleep 20 499 | sleep 10 500 | sleep 30 501 | sleep 20 502 | sleep 10 503 | sleep 30 504 | sleep 20 505 | sleep 10 506 | sleep 20 507 | sleep 10 508 | sleep 30 509 | sleep 20 510 | sleep 10 511 | sleep 30 512 | sleep 20 513 | sleep 10 514 | sleep 20 515 | sleep 10 516 | sleep 30 517 | sleep 20 518 | sleep 10 519 | sleep 30 520 | sleep 20 521 | sleep 10 522 | sleep 20 523 | sleep 10 524 | sleep 30 525 | sleep 20 526 | sleep 10 527 | sleep 30 528 | sleep 20 529 | sleep 10 530 | sleep 20 531 | sleep 10 532 | sleep 30 533 | sleep 20 534 | sleep 10 535 | sleep 30 536 | sleep 20 537 | sleep 10 538 | sleep 20 539 | sleep 10 540 | sleep 30 541 | sleep 20 542 | sleep 10 543 | sleep 30 544 | sleep 20 545 | sleep 10 546 | sleep 20 547 | sleep 10 548 | sleep 30 549 | sleep 20 550 | sleep 10 551 | sleep 30 552 | sleep 20 553 | sleep 10 554 | sleep 20 555 | sleep 10 556 | sleep 30 557 | sleep 20 558 | sleep 10 559 | sleep 30 560 | sleep 20 561 | sleep 10 562 | sleep 20 563 | sleep 10 564 | sleep 30 565 | sleep 20 566 | sleep 10 567 | sleep 30 568 | sleep 20 569 | sleep 10 570 | sleep 20 571 | sleep 10 572 | sleep 30 573 | sleep 20 574 | sleep 10 575 | sleep 30 576 | sleep 20 577 | sleep 10 578 | sleep 20 579 | sleep 10 580 | sleep 30 581 | sleep 20 582 | sleep 10 583 | sleep 30 584 | sleep 20 585 | sleep 10 586 | sleep 20 587 | sleep 10 588 | sleep 30 589 | sleep 20 590 | sleep 10 591 | sleep 30 592 | sleep 20 593 | sleep 10 594 | sleep 20 595 | sleep 10 596 | sleep 30 597 | sleep 20 598 | sleep 10 599 | sleep 30 600 | sleep 20 601 | sleep 10 602 | sleep 20 603 | sleep 10 604 | sleep 30 605 | sleep 20 606 | sleep 10 607 | sleep 30 608 | sleep 20 609 | sleep 10 610 | sleep 20 611 | sleep 10 612 | sleep 30 613 | sleep 20 614 | sleep 10 615 | sleep 30 616 | sleep 20 617 | sleep 10 618 | sleep 20 619 | sleep 10 620 | sleep 30 621 | sleep 20 622 | sleep 10 623 | sleep 30 624 | sleep 20 625 | sleep 10 626 | sleep 20 627 | sleep 10 628 | sleep 30 629 | sleep 20 630 | sleep 10 631 | sleep 30 632 | sleep 20 633 | sleep 10 634 | sleep 20 635 | sleep 10 636 | sleep 30 637 | sleep 20 638 | sleep 10 639 | sleep 30 640 | sleep 20 641 | sleep 10 642 | sleep 20 643 | sleep 10 644 | sleep 30 645 | sleep 20 646 | sleep 10 647 | sleep 30 648 | sleep 20 649 | sleep 10 650 | sleep 20 651 | sleep 10 652 | sleep 30 653 | sleep 20 654 | sleep 10 655 | sleep 30 656 | sleep 20 657 | sleep 10 658 | sleep 20 659 | sleep 10 660 | sleep 30 661 | sleep 20 662 | sleep 10 663 | sleep 30 664 | sleep 20 665 | sleep 10 666 | sleep 20 667 | sleep 10 668 | sleep 30 669 | sleep 20 670 | sleep 10 671 | sleep 30 672 | sleep 20 673 | sleep 10 674 | sleep 20 675 | sleep 10 676 | sleep 30 677 | sleep 20 678 | sleep 10 679 | sleep 30 680 | sleep 20 681 | sleep 10 682 | sleep 20 683 | sleep 10 684 | sleep 30 685 | sleep 20 686 | sleep 10 687 | sleep 30 688 | sleep 20 689 | sleep 10 690 | sleep 20 691 | sleep 10 692 | sleep 30 693 | sleep 20 694 | sleep 10 695 | sleep 30 696 | sleep 20 697 | sleep 10 698 | sleep 20 699 | sleep 10 700 | sleep 30 701 | sleep 20 702 | sleep 10 703 | sleep 30 704 | sleep 20 705 | sleep 10 706 | sleep 20 707 | sleep 10 708 | sleep 30 709 | sleep 20 710 | sleep 10 711 | sleep 30 712 | sleep 20 713 | sleep 10 714 | sleep 20 715 | sleep 10 716 | sleep 30 717 | sleep 20 718 | sleep 10 719 | sleep 30 720 | sleep 20 721 | sleep 10 722 | sleep 20 723 | sleep 10 724 | sleep 30 725 | sleep 20 726 | sleep 10 727 | sleep 30 728 | sleep 20 729 | sleep 10 730 | sleep 20 731 | sleep 10 732 | sleep 30 733 | sleep 20 734 | sleep 10 735 | sleep 30 736 | sleep 20 737 | sleep 10 738 | sleep 20 739 | sleep 10 740 | sleep 30 741 | sleep 20 742 | sleep 10 743 | sleep 30 744 | sleep 20 745 | sleep 10 746 | sleep 20 747 | sleep 10 748 | sleep 30 749 | sleep 20 750 | sleep 10 751 | sleep 30 752 | sleep 20 753 | sleep 10 754 | sleep 20 755 | sleep 10 756 | sleep 30 757 | sleep 20 758 | sleep 10 759 | sleep 30 760 | sleep 20 761 | sleep 10 762 | sleep 20 763 | sleep 10 764 | sleep 30 765 | sleep 20 766 | sleep 10 767 | sleep 30 768 | sleep 20 769 | sleep 10 770 | sleep 20 771 | sleep 10 772 | sleep 30 773 | sleep 20 774 | sleep 10 775 | sleep 30 776 | sleep 20 777 | sleep 10 778 | sleep 20 779 | sleep 10 780 | sleep 30 781 | sleep 20 782 | sleep 10 783 | sleep 30 784 | sleep 20 785 | sleep 10 786 | sleep 20 787 | sleep 10 788 | sleep 30 789 | sleep 20 790 | sleep 10 791 | sleep 30 792 | sleep 20 793 | sleep 10 794 | sleep 20 795 | sleep 10 796 | sleep 30 797 | sleep 20 798 | sleep 10 799 | sleep 30 800 | sleep 20 801 | sleep 10 802 | sleep 20 803 | sleep 10 804 | sleep 30 805 | sleep 20 806 | sleep 10 807 | sleep 30 808 | sleep 20 809 | sleep 10 810 | sleep 20 811 | sleep 10 812 | sleep 30 813 | sleep 20 814 | sleep 10 815 | sleep 30 816 | sleep 20 817 | sleep 10 818 | sleep 20 819 | sleep 10 820 | sleep 30 821 | sleep 20 822 | sleep 10 823 | sleep 30 824 | sleep 20 825 | sleep 10 826 | sleep 20 827 | sleep 10 828 | sleep 30 829 | sleep 20 830 | sleep 10 831 | sleep 30 832 | sleep 20 833 | sleep 10 834 | sleep 20 835 | sleep 10 836 | sleep 30 837 | sleep 20 838 | sleep 10 839 | sleep 30 840 | sleep 20 841 | sleep 10 842 | sleep 20 843 | sleep 10 844 | sleep 30 845 | sleep 20 846 | sleep 10 847 | sleep 30 848 | sleep 20 849 | sleep 10 850 | sleep 20 851 | sleep 10 852 | sleep 30 853 | sleep 20 854 | sleep 10 855 | sleep 30 856 | sleep 20 857 | sleep 10 858 | sleep 20 859 | sleep 10 860 | sleep 30 861 | sleep 20 862 | sleep 10 863 | sleep 30 864 | sleep 20 865 | sleep 10 866 | sleep 20 867 | sleep 10 868 | sleep 30 869 | sleep 20 870 | sleep 10 871 | sleep 30 872 | sleep 20 873 | sleep 10 874 | sleep 20 875 | sleep 10 876 | sleep 30 877 | sleep 20 878 | sleep 10 879 | sleep 30 880 | sleep 20 881 | sleep 10 882 | sleep 20 883 | sleep 10 884 | sleep 30 885 | sleep 20 886 | sleep 10 887 | sleep 30 888 | sleep 20 889 | sleep 10 890 | sleep 20 891 | sleep 10 892 | sleep 30 893 | sleep 20 894 | sleep 10 895 | sleep 30 896 | sleep 20 897 | sleep 10 898 | sleep 20 899 | sleep 10 900 | sleep 30 901 | sleep 20 902 | sleep 10 903 | sleep 30 904 | sleep 20 905 | sleep 10 906 | sleep 20 907 | sleep 10 908 | sleep 30 909 | sleep 20 910 | sleep 10 911 | sleep 30 912 | sleep 20 913 | sleep 10 914 | sleep 20 915 | sleep 10 916 | sleep 30 917 | sleep 20 918 | sleep 10 919 | sleep 30 920 | sleep 20 921 | sleep 10 922 | sleep 20 923 | sleep 10 924 | sleep 30 925 | sleep 20 926 | sleep 10 927 | sleep 30 928 | sleep 20 929 | sleep 10 930 | sleep 20 931 | sleep 10 932 | sleep 30 933 | sleep 20 934 | sleep 10 935 | sleep 30 936 | sleep 20 937 | sleep 10 938 | sleep 20 939 | sleep 10 940 | sleep 30 941 | sleep 20 942 | sleep 10 943 | sleep 30 944 | sleep 20 945 | sleep 10 946 | sleep 20 947 | sleep 10 948 | sleep 30 949 | sleep 20 950 | sleep 10 951 | sleep 30 952 | sleep 20 953 | sleep 10 954 | sleep 20 955 | sleep 10 956 | sleep 30 957 | sleep 20 958 | sleep 10 959 | sleep 30 960 | sleep 20 961 | sleep 10 962 | sleep 20 963 | sleep 10 964 | sleep 30 965 | sleep 20 966 | sleep 10 967 | sleep 30 968 | sleep 20 969 | sleep 10 970 | sleep 20 971 | sleep 10 972 | sleep 30 973 | sleep 20 974 | sleep 10 975 | sleep 30 976 | sleep 20 977 | sleep 10 978 | sleep 20 979 | sleep 10 980 | sleep 30 981 | sleep 20 982 | sleep 10 983 | sleep 30 984 | sleep 20 985 | sleep 10 986 | sleep 20 987 | sleep 10 988 | sleep 30 989 | sleep 20 990 | sleep 10 991 | sleep 30 992 | sleep 20 993 | sleep 10 994 | sleep 20 995 | sleep 10 996 | sleep 30 997 | sleep 20 998 | sleep 10 999 | sleep 30 1000 | sleep 20 1001 | sleep 10 1002 | sleep 20 1003 | sleep 10 1004 | sleep 30 1005 | sleep 20 1006 | sleep 10 1007 | sleep 30 1008 | sleep 20 1009 | sleep 10 1010 | sleep 20 1011 | sleep 10 1012 | sleep 30 1013 | sleep 20 1014 | sleep 10 1015 | sleep 30 1016 | sleep 20 1017 | sleep 10 1018 | sleep 20 1019 | sleep 10 1020 | sleep 30 1021 | sleep 20 1022 | sleep 10 1023 | sleep 30 1024 | sleep 20 1025 | sleep 10 1026 | sleep 20 1027 | sleep 10 1028 | sleep 30 1029 | sleep 20 1030 | sleep 10 1031 | sleep 30 1032 | sleep 20 1033 | sleep 10 1034 | sleep 20 1035 | sleep 10 1036 | sleep 30 1037 | sleep 20 1038 | sleep 10 1039 | sleep 30 1040 | sleep 20 1041 | sleep 10 1042 | sleep 20 1043 | sleep 10 1044 | sleep 30 1045 | sleep 20 1046 | sleep 10 1047 | sleep 30 1048 | sleep 20 1049 | sleep 10 1050 | sleep 20 1051 | sleep 10 1052 | sleep 30 1053 | sleep 20 1054 | sleep 10 1055 | sleep 30 1056 | sleep 20 1057 | sleep 10 1058 | sleep 20 1059 | sleep 10 1060 | sleep 30 1061 | sleep 20 1062 | sleep 10 1063 | sleep 30 1064 | sleep 20 1065 | sleep 10 1066 | sleep 20 1067 | sleep 10 1068 | sleep 30 1069 | sleep 20 1070 | sleep 10 1071 | sleep 30 1072 | sleep 20 1073 | sleep 10 1074 | sleep 20 1075 | sleep 10 1076 | sleep 30 1077 | sleep 20 1078 | sleep 10 1079 | sleep 30 1080 | sleep 20 1081 | sleep 10 1082 | sleep 20 1083 | sleep 10 1084 | sleep 30 1085 | sleep 20 1086 | sleep 10 1087 | sleep 30 1088 | sleep 20 1089 | sleep 10 1090 | sleep 20 1091 | sleep 10 1092 | sleep 30 1093 | sleep 20 1094 | sleep 10 1095 | sleep 30 1096 | sleep 20 1097 | sleep 10 1098 | sleep 20 1099 | sleep 10 1100 | sleep 30 1101 | sleep 20 1102 | sleep 10 1103 | sleep 30 1104 | sleep 20 1105 | sleep 10 1106 | sleep 20 1107 | sleep 10 1108 | sleep 30 1109 | sleep 20 1110 | sleep 10 1111 | sleep 30 1112 | sleep 20 1113 | sleep 10 1114 | sleep 20 1115 | sleep 10 1116 | sleep 30 1117 | sleep 20 1118 | sleep 10 1119 | sleep 30 1120 | sleep 20 1121 | sleep 10 1122 | sleep 20 1123 | sleep 10 1124 | sleep 30 1125 | sleep 20 1126 | sleep 10 1127 | sleep 30 1128 | sleep 20 1129 | sleep 10 1130 | sleep 20 1131 | sleep 10 1132 | sleep 30 1133 | sleep 20 1134 | sleep 10 1135 | sleep 30 1136 | sleep 20 1137 | sleep 10 1138 | sleep 20 1139 | sleep 10 1140 | sleep 30 1141 | sleep 20 1142 | sleep 10 1143 | sleep 30 1144 | sleep 20 1145 | sleep 10 1146 | sleep 20 1147 | sleep 10 1148 | sleep 30 1149 | sleep 20 1150 | sleep 10 1151 | sleep 30 1152 | sleep 20 1153 | sleep 10 1154 | sleep 20 1155 | sleep 10 1156 | sleep 30 1157 | sleep 20 1158 | sleep 10 1159 | sleep 30 1160 | sleep 20 1161 | sleep 10 1162 | sleep 20 1163 | sleep 10 1164 | sleep 30 1165 | sleep 20 1166 | sleep 10 1167 | sleep 30 1168 | sleep 20 1169 | sleep 10 1170 | sleep 20 1171 | sleep 10 1172 | sleep 30 1173 | sleep 20 1174 | sleep 10 1175 | sleep 30 1176 | sleep 20 1177 | sleep 10 1178 | sleep 20 1179 | sleep 10 1180 | sleep 30 1181 | sleep 20 1182 | sleep 10 1183 | sleep 30 1184 | sleep 20 1185 | sleep 10 1186 | sleep 20 1187 | sleep 10 1188 | sleep 30 1189 | sleep 20 1190 | sleep 10 1191 | sleep 30 1192 | sleep 20 1193 | sleep 10 1194 | sleep 20 1195 | sleep 10 1196 | sleep 30 1197 | sleep 20 1198 | sleep 10 1199 | sleep 30 1200 | sleep 20 1201 | sleep 10 1202 | sleep 20 1203 | sleep 10 1204 | sleep 30 1205 | sleep 20 1206 | sleep 10 1207 | sleep 30 1208 | sleep 20 1209 | sleep 10 1210 | sleep 20 1211 | sleep 10 1212 | sleep 30 1213 | sleep 20 1214 | sleep 10 1215 | sleep 30 1216 | sleep 20 1217 | sleep 10 1218 | sleep 20 1219 | sleep 10 1220 | sleep 30 1221 | sleep 20 1222 | sleep 10 1223 | sleep 30 1224 | sleep 20 1225 | sleep 10 1226 | sleep 20 1227 | sleep 10 1228 | sleep 30 1229 | sleep 20 1230 | sleep 10 1231 | sleep 30 1232 | sleep 20 1233 | sleep 10 1234 | sleep 20 1235 | sleep 10 1236 | sleep 30 1237 | sleep 20 1238 | sleep 10 1239 | sleep 30 1240 | sleep 20 1241 | sleep 10 1242 | sleep 20 1243 | sleep 10 1244 | sleep 30 1245 | sleep 20 1246 | sleep 10 1247 | sleep 30 1248 | sleep 20 1249 | sleep 10 1250 | sleep 20 1251 | sleep 10 1252 | sleep 30 1253 | sleep 20 1254 | sleep 10 1255 | sleep 30 1256 | sleep 20 1257 | sleep 10 1258 | sleep 20 1259 | sleep 10 1260 | sleep 30 1261 | sleep 20 1262 | sleep 10 1263 | sleep 30 1264 | sleep 20 1265 | sleep 10 1266 | sleep 20 1267 | sleep 10 1268 | sleep 30 1269 | sleep 20 1270 | sleep 10 1271 | sleep 30 1272 | sleep 20 1273 | sleep 10 1274 | sleep 20 1275 | sleep 10 1276 | sleep 30 1277 | sleep 20 1278 | sleep 10 1279 | sleep 30 1280 | sleep 20 1281 | sleep 10 1282 | sleep 20 1283 | sleep 10 1284 | sleep 30 1285 | sleep 20 1286 | sleep 10 1287 | sleep 30 1288 | sleep 20 1289 | sleep 10 1290 | sleep 20 1291 | sleep 10 1292 | sleep 30 1293 | sleep 20 1294 | sleep 10 1295 | sleep 30 1296 | sleep 20 1297 | sleep 10 1298 | sleep 20 1299 | sleep 10 1300 | sleep 30 1301 | sleep 20 1302 | sleep 10 1303 | sleep 30 1304 | sleep 20 1305 | sleep 10 1306 | sleep 20 1307 | sleep 10 1308 | sleep 30 1309 | sleep 20 1310 | sleep 10 1311 | sleep 30 1312 | sleep 20 1313 | sleep 10 1314 | sleep 20 1315 | sleep 10 1316 | sleep 30 1317 | sleep 20 1318 | sleep 10 1319 | sleep 30 1320 | sleep 20 1321 | sleep 10 1322 | sleep 20 1323 | sleep 10 1324 | sleep 30 1325 | sleep 20 1326 | sleep 10 1327 | sleep 30 1328 | sleep 20 1329 | sleep 10 1330 | sleep 20 1331 | sleep 10 1332 | sleep 30 1333 | sleep 20 1334 | sleep 10 1335 | sleep 30 1336 | sleep 20 1337 | sleep 10 1338 | sleep 20 1339 | sleep 10 1340 | sleep 30 1341 | sleep 20 1342 | sleep 10 1343 | sleep 30 1344 | sleep 20 1345 | sleep 10 1346 | sleep 20 1347 | sleep 10 1348 | sleep 30 1349 | sleep 20 1350 | sleep 10 1351 | sleep 30 1352 | sleep 20 1353 | sleep 10 1354 | sleep 20 1355 | sleep 10 1356 | sleep 30 1357 | sleep 20 1358 | sleep 10 1359 | sleep 30 1360 | sleep 20 1361 | sleep 10 1362 | sleep 20 1363 | sleep 10 1364 | sleep 30 1365 | sleep 20 1366 | sleep 10 1367 | sleep 30 1368 | sleep 20 1369 | sleep 10 1370 | sleep 20 1371 | sleep 10 1372 | sleep 30 1373 | sleep 20 1374 | sleep 10 1375 | sleep 30 1376 | sleep 20 1377 | sleep 10 1378 | sleep 20 1379 | sleep 10 1380 | sleep 30 1381 | sleep 20 1382 | sleep 10 1383 | sleep 30 1384 | sleep 20 1385 | sleep 10 1386 | sleep 20 1387 | sleep 10 1388 | sleep 30 1389 | sleep 20 1390 | sleep 10 1391 | sleep 30 1392 | sleep 20 1393 | sleep 10 1394 | sleep 20 1395 | sleep 10 1396 | sleep 30 1397 | sleep 20 1398 | sleep 10 1399 | sleep 30 1400 | sleep 20 1401 | sleep 10 1402 | sleep 20 1403 | sleep 10 1404 | sleep 30 1405 | sleep 20 1406 | sleep 10 1407 | sleep 30 1408 | sleep 20 1409 | sleep 10 1410 | sleep 20 1411 | sleep 10 1412 | sleep 30 1413 | sleep 20 1414 | sleep 10 1415 | sleep 30 1416 | sleep 20 1417 | sleep 10 1418 | sleep 20 1419 | sleep 10 1420 | sleep 30 1421 | sleep 20 1422 | sleep 10 1423 | sleep 30 1424 | sleep 20 1425 | sleep 10 1426 | sleep 20 1427 | sleep 10 1428 | sleep 30 1429 | sleep 20 1430 | sleep 10 1431 | sleep 30 1432 | sleep 20 1433 | sleep 10 1434 | sleep 20 1435 | sleep 10 1436 | sleep 30 1437 | sleep 20 1438 | sleep 10 1439 | sleep 30 1440 | sleep 20 1441 | sleep 10 1442 | sleep 20 1443 | sleep 10 1444 | sleep 30 1445 | sleep 20 1446 | sleep 10 1447 | sleep 30 1448 | sleep 20 1449 | sleep 10 1450 | sleep 20 1451 | sleep 10 1452 | sleep 30 1453 | sleep 20 1454 | sleep 10 1455 | sleep 30 1456 | sleep 20 1457 | sleep 10 1458 | sleep 20 1459 | sleep 10 1460 | sleep 30 1461 | sleep 20 1462 | sleep 10 1463 | sleep 30 1464 | sleep 20 1465 | sleep 10 1466 | sleep 20 1467 | sleep 10 1468 | sleep 30 1469 | sleep 20 1470 | sleep 10 1471 | sleep 30 1472 | sleep 20 1473 | sleep 10 1474 | sleep 20 1475 | sleep 10 1476 | sleep 30 1477 | sleep 20 1478 | sleep 10 1479 | sleep 30 1480 | sleep 20 1481 | sleep 10 1482 | sleep 20 1483 | sleep 10 1484 | sleep 30 1485 | sleep 20 1486 | sleep 10 1487 | sleep 30 1488 | sleep 20 1489 | sleep 10 1490 | sleep 20 1491 | sleep 10 1492 | sleep 30 1493 | sleep 20 1494 | sleep 10 1495 | sleep 30 1496 | sleep 20 1497 | sleep 10 1498 | sleep 20 1499 | sleep 10 1500 | sleep 30 1501 | sleep 20 1502 | sleep 10 1503 | sleep 30 1504 | sleep 20 1505 | sleep 10 1506 | sleep 20 1507 | sleep 10 1508 | sleep 30 1509 | sleep 20 1510 | sleep 10 1511 | sleep 30 1512 | sleep 20 1513 | sleep 10 1514 | sleep 20 1515 | sleep 10 1516 | sleep 30 1517 | sleep 20 1518 | sleep 10 1519 | sleep 30 1520 | sleep 20 1521 | sleep 10 1522 | sleep 20 1523 | sleep 10 1524 | sleep 30 1525 | sleep 20 1526 | sleep 10 1527 | sleep 30 1528 | sleep 20 1529 | sleep 10 1530 | sleep 20 1531 | sleep 10 1532 | sleep 30 1533 | sleep 20 1534 | sleep 10 1535 | sleep 30 1536 | sleep 20 1537 | sleep 10 1538 | sleep 20 1539 | sleep 10 1540 | sleep 30 1541 | sleep 20 1542 | sleep 10 1543 | sleep 30 1544 | sleep 20 1545 | sleep 10 1546 | sleep 20 1547 | sleep 10 1548 | sleep 30 1549 | sleep 20 1550 | sleep 10 1551 | sleep 30 1552 | sleep 20 1553 | sleep 10 1554 | sleep 20 1555 | sleep 10 1556 | sleep 30 1557 | sleep 20 1558 | sleep 10 1559 | sleep 30 1560 | sleep 20 1561 | sleep 10 1562 | sleep 20 1563 | sleep 10 1564 | sleep 30 1565 | sleep 20 1566 | sleep 10 1567 | sleep 30 1568 | sleep 20 1569 | sleep 10 1570 | sleep 20 1571 | sleep 10 1572 | sleep 30 1573 | sleep 20 1574 | sleep 10 1575 | sleep 30 1576 | sleep 20 1577 | sleep 10 1578 | sleep 20 1579 | sleep 10 1580 | sleep 30 1581 | sleep 20 1582 | sleep 10 1583 | sleep 30 1584 | sleep 20 1585 | sleep 10 1586 | sleep 20 1587 | sleep 10 1588 | sleep 30 1589 | sleep 20 1590 | sleep 10 1591 | sleep 30 1592 | sleep 20 1593 | sleep 10 1594 | sleep 20 1595 | sleep 10 1596 | sleep 30 1597 | sleep 20 1598 | sleep 10 1599 | sleep 30 1600 | sleep 20 1601 | sleep 10 1602 | sleep 20 1603 | sleep 10 1604 | sleep 30 1605 | sleep 20 1606 | sleep 10 1607 | sleep 30 1608 | sleep 20 1609 | sleep 10 1610 | sleep 20 1611 | sleep 10 1612 | sleep 30 1613 | sleep 20 1614 | sleep 10 1615 | sleep 30 1616 | sleep 20 1617 | sleep 10 1618 | sleep 20 1619 | sleep 10 1620 | sleep 30 1621 | sleep 20 1622 | sleep 10 1623 | sleep 30 1624 | sleep 20 1625 | sleep 10 1626 | sleep 20 1627 | sleep 10 1628 | sleep 30 1629 | sleep 20 1630 | sleep 10 1631 | sleep 30 1632 | sleep 20 1633 | sleep 10 1634 | sleep 20 1635 | sleep 10 1636 | sleep 30 1637 | sleep 20 1638 | sleep 10 1639 | sleep 30 1640 | sleep 20 1641 | sleep 10 1642 | sleep 20 1643 | sleep 10 1644 | sleep 30 1645 | sleep 20 1646 | sleep 10 1647 | sleep 30 1648 | sleep 20 1649 | sleep 10 1650 | sleep 20 1651 | sleep 10 1652 | sleep 30 1653 | sleep 20 1654 | sleep 10 1655 | sleep 30 1656 | sleep 20 1657 | sleep 10 1658 | sleep 20 1659 | sleep 10 1660 | sleep 30 1661 | sleep 20 1662 | sleep 10 1663 | sleep 30 1664 | sleep 20 1665 | sleep 10 1666 | sleep 20 1667 | sleep 10 1668 | sleep 30 1669 | sleep 20 1670 | sleep 10 1671 | sleep 30 1672 | sleep 20 1673 | sleep 10 1674 | sleep 20 1675 | sleep 10 1676 | sleep 30 1677 | sleep 20 1678 | sleep 10 1679 | sleep 30 1680 | sleep 20 1681 | sleep 10 1682 | sleep 20 1683 | sleep 10 1684 | sleep 30 1685 | sleep 20 1686 | sleep 10 1687 | sleep 30 1688 | sleep 20 1689 | sleep 10 1690 | sleep 20 1691 | sleep 10 1692 | sleep 30 1693 | sleep 20 1694 | sleep 10 1695 | sleep 30 1696 | sleep 20 1697 | sleep 10 1698 | sleep 20 1699 | sleep 10 1700 | sleep 30 1701 | sleep 20 1702 | sleep 10 1703 | sleep 30 1704 | sleep 20 1705 | sleep 10 1706 | sleep 20 1707 | sleep 10 1708 | sleep 30 1709 | sleep 20 1710 | sleep 10 1711 | sleep 30 1712 | sleep 20 1713 | sleep 10 1714 | sleep 20 1715 | sleep 10 1716 | sleep 30 1717 | sleep 20 1718 | sleep 10 1719 | sleep 30 1720 | sleep 20 1721 | sleep 10 1722 | sleep 20 1723 | sleep 10 1724 | sleep 30 1725 | sleep 20 1726 | sleep 10 1727 | sleep 30 1728 | sleep 20 1729 | sleep 10 1730 | sleep 20 1731 | sleep 10 1732 | sleep 30 1733 | sleep 20 1734 | sleep 10 1735 | sleep 30 1736 | sleep 20 1737 | sleep 10 1738 | sleep 20 1739 | sleep 10 1740 | sleep 30 1741 | sleep 20 1742 | sleep 10 1743 | sleep 30 1744 | sleep 20 1745 | sleep 10 1746 | sleep 20 1747 | sleep 10 1748 | sleep 30 1749 | sleep 20 1750 | sleep 10 1751 | sleep 30 1752 | sleep 20 1753 | sleep 10 1754 | sleep 20 1755 | sleep 10 1756 | sleep 30 1757 | sleep 20 1758 | sleep 10 1759 | sleep 30 1760 | sleep 20 1761 | sleep 10 1762 | sleep 20 1763 | sleep 10 1764 | sleep 30 1765 | sleep 20 1766 | sleep 10 1767 | sleep 30 1768 | sleep 20 1769 | sleep 10 1770 | sleep 20 1771 | sleep 10 1772 | sleep 30 1773 | sleep 20 1774 | sleep 10 1775 | sleep 30 1776 | sleep 20 1777 | sleep 10 1778 | sleep 20 1779 | sleep 10 1780 | sleep 30 1781 | sleep 20 1782 | sleep 10 1783 | sleep 30 1784 | sleep 20 1785 | sleep 10 1786 | sleep 20 1787 | sleep 10 1788 | sleep 30 1789 | sleep 20 1790 | sleep 10 1791 | sleep 30 1792 | sleep 20 --------------------------------------------------------------------------------