├── README.md ├── awk └── jstack_hotcode.awk ├── ksh ├── find-exp.sh ├── jardiff.sh └── javatop.sh ├── logo ├── buddha.txt └── cnm.txt ├── pac └── gfw.pac └── taobao ├── bushosts ├── check_hsf.py ├── consignhosts ├── hp ├── hsfqps ├── ip2app ├── jssh ├── lchosts ├── timeouthosts ├── xscp └── xssh /README.md: -------------------------------------------------------------------------------- 1 | unixtools 2 | ========= 3 | 4 | 我在unixtools下积累的常用工具 5 | PS，这里所有的工具都是给淘宝环境下使用，如果你不能理解脚本中的含义，请勿自己使用，谢谢呐。 6 | 7 | Java线程过高问题排查 8 | ``` 9 | curl -sLk "https://raw.github.com/oldmanpushcart/unixtools/master/javatop.sh" | ksh -s 5 `pgrep -u admin java` 10 | ``` 11 | 12 | Java Dump脚本 13 | ``` 14 | /opt/taobao/java/bin/jmap -dump:live,format=b,file=heap-`date +%Y%m%d%H%M`.bin `pgrep -u admin java` 15 | ``` 16 | -------------------------------------------------------------------------------- /awk/jstack_hotcode.awk: -------------------------------------------------------------------------------- 1 | #!/bin/awk 2 | 3 | # write by : oldmanpushcart@gmail.com 4 | # date : 2014-12-20 5 | # version : 0.0.1 6 | 7 | /tid=0x[a-z0-9]+ nid=0x[a-z0-9]+/,/^$/{ 8 | 9 | // head line 10 | if($0~/tid=0x/&&$0~/nid=0x/){ 11 | 12 | // take tid 13 | if(match($0,/tid=0x[a-z0-9]+/)){ 14 | tid=substr($0,RSTART,RLENGTH) 15 | } 16 | 17 | // take nid 18 | if(match($0,/nid=0x[a-z0-9]+/)){ 19 | nid=substr($0,RSTART,RLENGTH) 20 | } 21 | 22 | } 23 | 24 | // state line 25 | if($0~/java\.lang\.Thread\.State: /) { 26 | if(match($0,/NEW|RUNNABLE|BLOCKED|WAITING|TIMED_WAITING|TERMINATED/)){ 27 | state=substr($0,RSTART,RLENGTH) 28 | } 29 | } 30 | 31 | // code line 32 | if($0~/^\tat /&&match($0,/\([^)]+\)/)){ 33 | 34 | # take code 35 | code=substr($0,RSTART,RLENGTH) 36 | gsub(" ","_",code) 37 | 38 | # take method 39 | method=substr($0,5,index($0,"(")-5) 40 | 41 | # output 42 | r[NR]=sprintf("%s\t%s\t%s\t%s\t%s",tid,nid,state,method,code) 43 | 44 | # sum 45 | sum_method[method]++ 46 | sum_code[code]++ 47 | 48 | # format output, max col length 49 | m[1]=max(m[1],length(tid)) 50 | m[2]=max(m[2],length(nid)) 51 | m[3]=max(m[3],length(state)) 52 | m[4]=max(m[4],length(method)) 53 | m[5]=max(m[5],length(code)) 54 | 55 | } 56 | } 57 | 58 | function max(a,b){ 59 | return a>b?a:b 60 | } 61 | 62 | END{ 63 | for(i in r){ 64 | split(r[i],col,"\t") 65 | printf("%-"m[1]"s\t%-"m[2]"s\t%-"m[3]"s\t%-"m[4]"s\t%d\t%-"m[5]"s\t%d\n", 66 | col[1],col[2],col[3],col[4],sum_method[col[4]],col[5],sum_code[col[5]]) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /ksh/find-exp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | 3 | # write by : oldmanpushcart@gmail.com 4 | # date : 2014-09-24 5 | # version : 0.02 6 | 7 | find ${1} -type f -name "${2}" | while read f;do 8 | grep -Eo '[A-Za-z]+Exception' ${f}|while read exp;do 9 | printf "%s\t%s\n" ${f} ${exp} 10 | done 11 | done|sort|uniq -c 12 | -------------------------------------------------------------------------------- /ksh/jardiff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | 3 | # write by : oldmanpushcart@gmail.com 4 | # date : 2014-01-16 5 | # version : 0.0.7 6 | 7 | typeset FIND_DIR=${1} 8 | typeset JARDIFF_TMP_FILE=/tmp/jardiff.$$ 9 | 10 | # the main function 11 | function _main 12 | { 13 | 14 | find ${FIND_DIR} -type f -iname *.jar\ 15 | |while read f;do\ 16 | unzip -v ${f}\ 17 | |awk '$7>0&&$8~/class$/{print f"\t"$7"\t"$8}' f=${f};\ 18 | done > ${JARDIFF_TMP_FILE} 19 | 20 | cat ${JARDIFF_TMP_FILE}\ 21 | |cut -f3|sort|uniq -c|awk '$1>1{print $2}'|while read f;do 22 | typeset local CRC_32_C=$(awk -v "f=${f}" '$3!=f{print $2}' ${JARDIFF_TMP_FILE}|sort|uniq|wc -l) 23 | [[ $CRC_32_C -gt 1 ]] && fgrep ${f} ${JARDIFF_TMP_FILE} 24 | done 25 | 26 | _finish 27 | } 28 | 29 | # the finish function need clean something 30 | function _finish 31 | { 32 | rm -rf ${JARDIFF_TMP_FILE} 33 | exit 0; 34 | } 35 | 36 | trap "_finish" SIGINT 37 | _main "$@" 38 | -------------------------------------------------------------------------------- /ksh/javatop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | 3 | # write by : oldmanpushcart@gmail.com 4 | # date : 2014-01-16 5 | # version : 0.07 6 | 7 | typeset top=${1:-10} 8 | typeset pid=${2:-$(pgrep -u $USER java)} 9 | typeset tmp_file=/tmp/java_${pid}_$$.trace 10 | 11 | # fix for alibaba-inc.com 12 | export JAVA_HOME=/opt/taobao/java 13 | 14 | $JAVA_HOME/bin/jstack $pid > $tmp_file 15 | ps H -eo user,pid,ppid,tid,time,%cpu --sort=%cpu --no-headers\ 16 | | tail -$top\ 17 | | awk -v "pid=$pid" '$2==pid{print $4"\t"$6}'\ 18 | | while read line; 19 | do 20 | typeset nid=$(echo "$line"|awk '{printf("0x%x",$1)}') 21 | typeset cpu=$(echo "$line"|awk '{print $2}') 22 | awk -v "cpu=$cpu" '/nid='"$nid"'/,/^$/{print $0"\t"(isF++?"":"cpu="cpu"%");}' $tmp_file 23 | done 24 | 25 | rm -f $tmp_file 26 | -------------------------------------------------------------------------------- /logo/buddha.txt: -------------------------------------------------------------------------------- 1 | 2 | _oo0oo_ 3 | o88888880 4 | 88" . "88 5 | (| -_- |) 6 | 0\ = /0 7 | ____/`---'\____ 8 | ' \\| + |// ` 9 | / \\||| : ||// \ 10 | / _|||| -:- ||||- \ 11 | | |\\\ - ///| | 12 | | \_| ''\---/'' | | 13 | \ .-\__ `-` __/-. / 14 | ---`. .' /--.--\ `. .' ___ 15 | ""'< `.___\_<|>_/___.' >'"" 16 | ' : `_ \`.;`\_/';.'/_' : ' 17 | | | `_\ __ ' | ' __ /_' | | 18 | ======`-.____`-.__\__"__/___'-`____.-'====== 19 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 20 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 21 | 22 | -------------------------------------------------------------------------------- /logo/cnm.txt: -------------------------------------------------------------------------------- 1 | ┏┓ ┏┓ 2 | ┏┛┻━━━┛┻┓ 3 | ┃ ┃ 4 | ┃ ━ ┃ 5 | ┃ ┳┛ ┗┳ ┃ 6 | ┃ ┃ 7 | ┃ ┻ ┃ 8 | ┃ ┃ 9 | ┗━┓ ┏━┛ 10 | ┃ ┃ 11 | ┃ ┃ 12 | ┃ ┗━━━┓ 13 | ┃ ┣┓ 14 | ┃ ┏┛ 15 | ┗┓┓┏━┳┓┏┛ 16 | ┃┫┫ ┃┫┫ 17 | ┗┻┛ ┗┻┛ 18 | -------------------------------------------------------------------------------- /pac/gfw.pac: -------------------------------------------------------------------------------- 1 | /** 2 | * PAC 文件的入口函数，通过该函数指定访问目标地址的连接方式。 3 | * 4 | * @param {string} url 将要访问的网址（如＂http://www.domain.com：8080/simple.htm＂）。 5 | * @param {string} host 由网址取得的主机名称（如当访问＂http://www.domain.com：8080/simple.htm＂时为＂www.domain.com＂） 6 | * @returns {string} 返回一个包含一个或多个访问规则的字符串，多个规则用＂;＂分隔。一个访问规则的字符串可以为以下方式： 7 | * 1. "DIRECT". (表示直接连接) 8 | * 2. "PROXY host:port". (表示使用 HTTP 代理，eg: "PROXY domain.com:7070") 9 | * 3. "SOCKS host:port". (表示使用 SOCKS 代理，eg: "SOCKS localhost:7070") 10 | * 4. "SOCKS5 host:port". (表示使用 SOCKS5 代理，eg: "SOCKS5 127.0.0.1:7070") 11 | */ 12 | function FindProxyForURL(url, host){ 13 | 14 | var RESULT_SOCKS5 = "SOCKS5 localhost:3658"; 15 | 16 | var proxy_list = [ 17 | "google.com", 18 | "google.com.hk", 19 | "googleapis.com", 20 | "facebook.com", 21 | "twitter.com", 22 | "youtube.com", 23 | "gstatic.com", 24 | "ytimg.com", 25 | "python.com", 26 | ]; 27 | 28 | for( index=0;index 1: 20 | t2_temp = re.findall('(\d{4}-\d{2}-\d{2} \d{2}\:\d{2}\:\d{2})', lastline[0]) 21 | if len(t2_temp) > 0: 22 | t2 = t2_temp[0] 23 | if t1 == 'x': 24 | t1 = t2 25 | if t1 != t2: 26 | try: 27 | del lastline[0] 28 | except IndexError: 29 | pass 30 | break 31 | continue 32 | f.close() 33 | return lastline 34 | 35 | 36 | def grep_info(log): 37 | p = re.compile('^(HSF-Consumer|HSF-ProviderDetail|HSF-Provider-Timeout|HSF-Consumer-Timeout)\01.*') 38 | l = {} 39 | k = {} 40 | 41 | for i in log: 42 | match = p.match(i) 43 | if match: 44 | i = match.group().split('\01') 45 | t = i[6] 46 | if t not in l: 47 | l[t] = {} 48 | cat = i[0] 49 | if not l[t].has_key(cat): 50 | l[t][cat] = [] 51 | k = {'key':i[2], 'rt_sum':i[5], 'qps_sum':i[4]} 52 | l[t][cat].append(k) 53 | return l 54 | 55 | def init_dict(): 56 | x = {} 57 | o = {} 58 | categorys = ['HSF-ProviderDetail', 'HSF-Consumer', 'HSF-Provider-Timeout', 'HSF-Consumer-Timeout'] 59 | for cat in categorys: 60 | x[cat] = {} 61 | o[cat] = {} 62 | x[cat]['total'] = {'qps_sum':0, 'rt_sum':0} 63 | o[cat]['total'] = {'qps':0, 'rt':0} 64 | return (x, o) 65 | 66 | 67 | def cat_calc(info, d): 68 | x = d[0] 69 | o = d[1] 70 | def oo(x):return ('%.2f' % x).rstrip('0').rstrip('.') 71 | 72 | for i in info[1]: 73 | for j in info[1][i]: 74 | key = j['key'] 75 | if not x[i].has_key(key): 76 | x[i][key] = {'qps_sum':0, 'rt_sum':0} 77 | o[i][key] = {'qps':0, 'rt':0} 78 | x[i][key]['qps_sum'] += float(j['qps_sum']) 79 | x[i][key]['rt_sum'] += float(j['rt_sum']) 80 | x[i]['total']['qps_sum'] += float(j['qps_sum']) 81 | x[i]['total']['rt_sum'] += float(j['rt_sum']) 82 | for j in x[i]: 83 | key_qps = float(x[i][j]['qps_sum']) / 120 84 | key_qps = oo(key_qps) 85 | key_rt = float(x[i][j]['rt_sum']) / x[i][j]['qps_sum'] 86 | key_rt = oo(key_rt) 87 | o[i][j] = {'qps':key_qps, 'rt':key_rt} 88 | title = info[0] + '\t' + os.uname()[1] + '\t' + 'thsf' + '\t' 89 | P_qps = float(o['HSF-ProviderDetail']['total']['qps']) 90 | C_qps = float(o['HSF-Consumer']['total']['qps']) 91 | P_rt = float(o['HSF-ProviderDetail']['total']['rt']) 92 | C_rt = float(o['HSF-Consumer']['total']['rt']) 93 | P_Fail_Rate = 0 94 | C_Fail_Rate = 0 95 | if P_qps != 0: 96 | P_Fail_Rate = float(x['HSF-Provider-Timeout']['total']['qps_sum']) / float(x['HSF-ProviderDetail']['total']['qps_sum']) * 100 97 | if C_qps != 0: 98 | C_Fail_Rate = float(x['HSF-Consumer-Timeout']['total']['qps_sum']) / float(x['HSF-Consumer']['total']['qps_sum']) * 100 99 | 100 | detail = {'Detail':o} 101 | del detail['Detail']['HSF-Consumer']['total'] 102 | del detail['Detail']['HSF-Consumer-Timeout']['total'] 103 | del detail['Detail']['HSF-Provider-Timeout']['total'] 104 | del detail['Detail']['HSF-ProviderDetail']['total'] 105 | output = {'title':title, 'P_qps':P_qps, 'P_rt':P_rt, 'C_qps':C_qps, 'C_rt':C_rt, 'P_Fail_Rate':P_Fail_Rate, 'C_Fail_Rate':C_Fail_Rate, 'Detail':detail} 106 | 107 | return output 108 | 109 | 110 | def printinfo(output, options): 111 | print '%(title)sP_qps=%(P_qps)0.2f\tP_rt=%(P_rt)0.2f\tC_qps=%(C_qps)0.2f\tC_rt=%(C_rt)0.2f\tP_Fail_Rate=%(P_Fail_Rate)0.2f\tC_Fail_Rate=%(C_Fail_Rate)0.2f' % output 112 | if options.detail is True: 113 | def _rjust(x, y):return "%s" % x.rjust(y) 114 | def _ljust(x, y):return "%s" % x.ljust(y) 115 | for i, j in output['Detail'].values()[0].items(): 116 | for k in j.items(): 117 | print _ljust(i + '/' + k[0], 50), 'QPS:', _rjust(k[1]['qps'], 8), 'RT:', _rjust(k[1]['rt'], 8) 118 | 119 | def main(): 120 | parser = OptionParser() 121 | # parser = OptionParser(usage="usage: %prog [options]",version="%prog 1.0") 122 | parser.add_option("-a", "--all", action="store_true", dest="all", default=False, help="HSF log output mode, -a output today HSF log ,default ouput lastline HSF log") 123 | parser.add_option("-u", "--user", action="store", dest="user", default="admin", help="HSF log USER,log path: /home/[user]/logs/monitor/monitor-app-org.eclipse.osgi.internal.baseadaptor.DefaultClassLoader.log") 124 | parser.add_option("-n", "--filedate", action="store", dest="filedate", default="0", help="analyse n days HSF log, use -a -n [ 1 ~ ], 1:yesterday, 2:the day before yesterday") 125 | parser.add_option("-d", "--detail", action="store_true", dest="detail", default=False, help="service detail info") 126 | parser.add_option("-g", "--g", action="store_true", dest="snmpd_g", default=False, help="only for snmpd use") 127 | 128 | (options, args) = parser.parse_args() 129 | 130 | d = '' 131 | if int(options.filedate) > 0: 132 | d = "." + str(datetime.date.today() - datetime.timedelta(days=int(options.filedate))) 133 | 134 | hsf_log = "/home/%s/logs/monitor/monitor-app-org.eclipse.osgi.internal.baseadaptor.DefaultClassLoader.log%s" % (options.user, d) 135 | 136 | if os.path.isfile(hsf_log): 137 | if options.all is False: 138 | l = grep_info(last(hsf_log)) 139 | 140 | if not l: 141 | currenttime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) 142 | l = {currenttime: []} 143 | 144 | lasttime = int(time.mktime(time.strptime(l.keys()[0], '%Y-%m-%d %H:%M:%S'))) 145 | currenttime = int(time.time()) 146 | if (currenttime - lasttime) > 123: 147 | currenttime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) 148 | title = str(999) + '/' + str(currenttime) + '/' + os.uname()[1] + '\t' + 'thsf' + '\t' 149 | output = {'title':title, 'P_qps':0, 'P_rt':0, 'C_qps':0, 'C_rt':0, 'P_Fail_Rate':0, 'C_Fail_Rate':0} 150 | printinfo(output) 151 | sys.exit() 152 | 153 | else: 154 | f = open(hsf_log, 'rb') 155 | t = f.readline().strip().split('\02') 156 | f.close() 157 | l = grep_info(t) 158 | else: 159 | print "ERROR: %s file is not exist!" % hsf_log 160 | sys.exit() 161 | 162 | for a in sorted(l.items(), key=lambda l:l[0]): 163 | printinfo(cat_calc(a, init_dict()), options) 164 | # 165 | if __name__ == '__main__': 166 | main() 167 | 168 | -------------------------------------------------------------------------------- /taobao/consignhosts: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | # author: dukun@taobao.com 3 | # date: 2011-06-25 4 | 5 | typeset GROUP="consign" 6 | 7 | armory -g "$GROUP"\ 8 | |awk 'BEGIN{ORS=""}{if($0~/^\-/&&1!=NR)o=o"\n";if($1~/(nodename|dns_ip|state|rack|site|nodoegroup|vmparent|nodegroup)/)o=o$2"\t"}END{print o"\n"}'\ 9 | |awk '{if(NF == 6)printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\n",$1,$2,$3,$4,$5,$6,"-");else print $0;}' \ 10 | |awk '$6~/'^"$GROUP"'.*host$/' 11 | -------------------------------------------------------------------------------- /taobao/hp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldmanpushcart/shell/55dbf5298b891ff9729710a4d716204f8e095573/taobao/hp -------------------------------------------------------------------------------- /taobao/hsfqps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oldmanpushcart/shell/55dbf5298b891ff9729710a4d716204f8e095573/taobao/hsfqps -------------------------------------------------------------------------------- /taobao/ip2app: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | 3 | while read ip; 4 | do 5 | armory -i $ip |sed 's/host$//g'|awk '$1~/^nodegroup$/{print ip"\t"$2}' ip="$ip" 6 | done 7 | -------------------------------------------------------------------------------- /taobao/jssh: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | 3 | 4 | # define the job file 5 | typeset _JOB_FILE=/tmp/_job_jssh.$$ 6 | 7 | # add a job into job arrays 8 | # args 9 | # $1 : the function for job run 10 | function job 11 | { 12 | typeset local pid=$(exec sh -c 'echo $PPID') 13 | typeset local token=$pid"-"$(date +"%Y%m%d%H%M%S") 14 | echo $token >> $_JOB_FILE 15 | $1 "$@" 16 | echo $token >> $_JOB_FILE 17 | } 18 | 19 | # waiting for jobs done. 20 | function wait_for_job_done 21 | { 22 | while [[ True ]]; 23 | do 24 | typeset local jc=$(cat $_JOB_FILE) 25 | typeset local wc=$(echo "$jc"|sort|uniq -c|awk '$1==1'|wc -l) 26 | #typeset local total=$(echo "$jc"|sort|uniq|wc -l) 27 | if [[ $wc == 0 ]];then 28 | break; 29 | fi 30 | sleep 1 31 | done 32 | } 33 | 34 | 35 | function show_help 36 | { 37 | print " 38 | 例子: 39 | jssh -h 40 | echo 172.24.24.92|jssh -c \"uptime\" 41 | echo 172.24.24.92|jssh -f \"./test.bat\" 42 | 参数: 43 | -h: 44 | 输出帮助说明! 45 | -c: 46 | 远程执行命令 47 | -f: 48 | 远程执行脚本文件 49 | -p: 50 | 并发执行线程数，默认1 51 | " 52 | } 53 | 54 | # 主程序开始 55 | function main 56 | { 57 | typeset local IS_COMMAND=false 58 | typeset local IS_BATFILE=false 59 | typeset local PARALLEL=1 60 | 61 | # 解析命令行传来的参数 62 | while getopts hp:c:f: OPTION 63 | do 64 | case "$OPTION" in 65 | f) 66 | IS_BATFILE=true 67 | BATFILE_PATH="$OPTARG" 68 | ;; 69 | c) 70 | IS_COMMAND=true 71 | COMMAND="$OPTARG" 72 | ;; 73 | p) 74 | PARALLEL="$OPTARG" 75 | ;; 76 | h|\?) 77 | show_help 78 | exit 79 | ;; 80 | esac 81 | done 82 | 83 | if [[ $IS_COMMAND != true && $IS_BATFILE != true ]];then 84 | echo "ERROR! the arguments (-c/-f) was missed." > /dev/stderr 85 | show_help 86 | exit -1 87 | fi 88 | 89 | rm -f $_JOB_FILE 90 | touch $_JOB_FILE 91 | 92 | typeset local count=0; 93 | while read ip; 94 | do 95 | job do_xssh $IS_COMMAND $IS_BATFILE "$ip" "$COMMAND" "$BATFILE_PATH" & 96 | count=$((count+1)) 97 | if [[ $count -ge $PARALLEL ]];then 98 | count=0 99 | wait_for_job_done 100 | fi 101 | done 102 | 103 | sleep 1 104 | wait_for_job_done 105 | finish 106 | } 107 | 108 | # do the xssh 109 | # $1 : name of function 110 | # $2 : IS_COMMAND 111 | # $3 : IS_BATFILE 112 | # $4 : IP 113 | # $5 : COMMAND 114 | # $6 : BATFILE_PATH 115 | function do_xssh 116 | { 117 | typeset local IS_COMMAND=$2 118 | typeset local IS_BATFILE=$3 119 | typeset local ip="$4" 120 | typeset local COMMAND="$5" 121 | typeset local BATFILE_PATH="$6" 122 | 123 | typeset local content="" 124 | typeset local re_try=0 125 | while [[ True ]]; 126 | do 127 | if [[ $IS_COMMAND = true ]];then 128 | typeset local command_content="echo -e '========== $ip ==========';"$COMMAND 129 | content=$(echo "$command_content"|xssh -i "$ip") 130 | elif [[ $IS_BATFILE = true ]];then 131 | typeset local batfile_content="echo -e '========== $ip ==========';"$(cat "$BATFILE_PATH") 132 | content=$(echo "$batfile_content"|xssh -i "$ip") 133 | else 134 | content="ERROR for get content!" 135 | fi 136 | if [[ -z $content && $re_try -lt 5 ]];then 137 | re_try=$(($re_try+1)) 138 | continue 139 | fi 140 | break 141 | done 142 | echo "$content" 143 | } 144 | 145 | function finish 146 | { 147 | cat $_JOB_FILE|sort|uniq -c|awk '$1==1{print $2}'|awk -F "-" '{print $1}'|while read stop_pid; 148 | do 149 | kill -9 $stop_pid 150 | done 151 | rm -f $_JOB_FILE 152 | echo "FINISH..." 153 | exit 0 154 | } 155 | 156 | # 捕捉CTRL+C信号 157 | trap "finish" SIGINT 158 | main "$@" 159 | -------------------------------------------------------------------------------- /taobao/lchosts: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | # author: dukun@taobao.com 3 | # date: 2011-06-25 4 | 5 | typeset GROUP="logisticscenter" 6 | 7 | armory -g "$GROUP"\ 8 | |awk 'BEGIN{ORS=""}{if($0~/^\-/&&1!=NR)o=o"\n";if($1~/(nodename|dns_ip|state|rack|site|nodoegroup|vmparent|nodegroup)/)o=o$2"\t"}END{print o"\n"}'\ 9 | |awk '{if(NF == 6)printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\n",$1,$2,$3,$4,$5,$6,"-");else print $0;}' \ 10 | |awk '$6~/'^"$GROUP"'.*host$/' 11 | -------------------------------------------------------------------------------- /taobao/timeouthosts: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | # author: dukun@taobao.com 3 | # date: 2011-06-25 4 | 5 | typeset GROUP="logisticstimeout" 6 | 7 | armory -g "$GROUP"\ 8 | |awk 'BEGIN{ORS=""}{if($0~/^\-/&&1!=NR)o=o"\n";if($1~/(nodename|dns_ip|state|rack|site|nodoegroup|vmparent|nodegroup)/)o=o$2"\t"}END{print o"\n"}'\ 9 | |awk '{if(NF == 6)printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\n",$1,$2,$3,$4,$5,$6,"-");else print $0;}' \ 10 | |awk '$6~/'^"$GROUP"'.*host$/' 11 | -------------------------------------------------------------------------------- /taobao/xscp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # auth : dukun@taobao.com 4 | # xscp write by python 5 | # usage : 6 | # xscp -r /tmp 127.0.0.1:22:/tmp 7 | # xscp /tmp/a.txt 127.0.0.1:22:/tmp/b.txt 8 | # xscp -r -t 5 /tmp 127.0.0.1:22:/tmp 9 | # xscp -r --timeout=5 /tmp 127.0.0.1:22:/tmp 10 | 11 | 12 | import sys 13 | import getopt 14 | import StringIO 15 | import socket 16 | import ConfigParser 17 | import os 18 | import string 19 | 20 | # error code exception 21 | class XsshErrorCodeException(Exception): 22 | message = None 23 | def __init__(self, message): 24 | self.message = message 25 | def __str__(self): 26 | return repr(self.message) 27 | 28 | # The Location class 29 | class Location: 30 | isLocal=None 31 | ip=None 32 | port=None 33 | path=None 34 | def __init__(self,pathExp): 35 | self.isLocal = pathExp.find(":") < 0 36 | if self.isLocal: 37 | # prase local path 38 | self.path=pathExp 39 | else: 40 | # parse remote path 41 | remoteArgs = pathExp.split(":")[0:] 42 | if len(remoteArgs) == 2: 43 | self.ip = remoteArgs[0] 44 | self.port = 22 45 | self.path = remoteArgs[1] 46 | elif len(remoteArgs) == 3: 47 | self.ip = remoteArgs[0] 48 | self.port = remoteArgs[1] 49 | self.path = remoteArgs[2] 50 | else: 51 | raise XsshErrorCodeException("The remote location is illegal.") 52 | 53 | 54 | # usage for xssh 55 | def usage(): 56 | print """ 57 | Show The Help! 58 | by dukun@taobao.com 59 | """ 60 | 61 | 62 | def parse_from_args(): 63 | 64 | # define the args 65 | isRecursion=False 66 | timeout=10 67 | fromLoc=None 68 | toLoc=None 69 | 70 | # parse option 71 | options,args = getopt.getopt(sys.argv[1:], "hrt:", ["help","timeout="]) 72 | for key, value in options: 73 | if key in ["-h", "--help"]: 74 | usage() 75 | sys.exit(0) 76 | elif key in ["-r"]: 77 | isRecursion = True 78 | elif key in ["-t","--timeout"]: 79 | timeout = value 80 | 81 | # parse from,to 82 | def remove_option_filter(e): 83 | return e.find("-")!=0 84 | removeOptionArgs = filter(remove_option_filter,sys.argv[1:]) 85 | fromLoc = removeOptionArgs[0] 86 | toLoc = removeOptionArgs[1] 87 | 88 | return isRecursion, timeout, fromLoc, toLoc 89 | 90 | def parse_from_xssh_file(): 91 | # read the xssh file from $HOME/.xssh 92 | cf = ConfigParser.ConfigParser() 93 | cf.read( os.path.expandvars('$HOME/.xssh')) 94 | return cf.get("xssh", "username"), cf.get("xssh", "password") 95 | 96 | def execute_on_scp(_username, _password, ip, _port, _timeout, fromLoc, toLoc): 97 | 98 | # connect to ssh server 99 | ssh = paramiko.SSHClient() 100 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 101 | ssh.connect(ip, port=_port, username=_username, password=_password, timeout=_timeout) 102 | chan = ssh.get_transport().open_session() 103 | 104 | # execute command 105 | chan.exec_command(cmd) 106 | 107 | # To capture Data. Need to read the entire buffer to caputure output 108 | contents_buf = StringIO.StringIO() 109 | error_buf = StringIO.StringIO() 110 | while not chan.exit_status_ready(): 111 | if chan.recv_ready(): 112 | data = chan.recv(1024) 113 | while data: 114 | contents_buf.write(data) 115 | data = chan.recv(1024) 116 | if chan.recv_stderr_ready(): 117 | error_buff = chan.recv_stderr(1024) 118 | while error_buff: 119 | error_buf.write(error_buff) 120 | error_buff = chan.recv_stderr(1024) 121 | 122 | return chan.recv_exit_status(), contents_buf.getvalue(), error_buf.getvalue() 123 | 124 | # the main 125 | if __name__ == "__main__": 126 | isRecursion, timeout, fromLocExp, toLocExp = parse_from_args() 127 | fromLoc = Location(fromLocExp) 128 | toLoc = Location(toLocExp) 129 | 130 | print toLoc.path 131 | -------------------------------------------------------------------------------- /taobao/xssh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # auth : dukun@taobao.com 4 | # xssh write by python 5 | 6 | 7 | import sys 8 | import getopt 9 | import StringIO 10 | import socket 11 | import ConfigParser 12 | import os 13 | import paramiko 14 | import string 15 | import logging 16 | 17 | # error code exception 18 | class XsshErrorCodeException(Exception): 19 | def __init__(self, value): 20 | self.value = value 21 | def __str__(self): 22 | return repr(self.value) 23 | 24 | # usage for xssh 25 | def usage(): 26 | print """ 27 | author : oldmanpushcart@gmail.com 28 | usage : 29 | echo uptime | xssh -i 127.0.0.1 -p 22 30 | echo uptime | xssh --ip 127.0.0.1 --port 22 31 | xssh -h 32 | xssh --help 33 | """ 34 | 35 | 36 | def parse_from_args(): 37 | ip = None 38 | port = "22" 39 | timeout = "10" 40 | options, args = getopt.getopt(sys.argv[1:], "Dhp:i:t:", ["help", "ip=", "port=", "timeout=", "debug"]) 41 | for key, value in options: 42 | if key in ["-h", "--help"]: 43 | usage() 44 | sys.exit(0) 45 | if key in ["-i", "--ip"]: 46 | ip = value 47 | if key in ["-p", "--port"]: 48 | port = value 49 | if key in ["-t", "--timeout"]: 50 | timeout = value 51 | if key in ["-D", "--debug"]: 52 | logging.basicConfig(filename=os.path.join(os.getcwd(), 'xssh.log'), level=logging.DEBUG) 53 | if ip == None: 54 | raise XsshErrorCodeException("the --ip/-i is required.") 55 | return ip, string.atoi(port), string.atof(timeout) 56 | 57 | 58 | def parse_from_xssh_file(): 59 | # read the xssh file from $HOME/.xssh 60 | cf = ConfigParser.ConfigParser() 61 | cf.read(os.path.expandvars('$HOME/.xssh')) 62 | return cf.get("xssh", "username"), cf.get("xssh", "password") 63 | 64 | def read_command_from_stdin(): 65 | cmd_buf = StringIO.StringIO() 66 | cmd_buf.writelines(sys.stdin.readlines()) 67 | return cmd_buf.getvalue() 68 | 69 | def execute_on_ssh(_username, _password, ip, _port, _timeout, cmd): 70 | 71 | # connect to ssh server 72 | byte_len = 1024 73 | ssh = paramiko.SSHClient() 74 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 75 | ssh.connect(ip, port=_port, username=_username, password=_password, timeout=_timeout) 76 | logging.debug("ssh connected.") 77 | 78 | try: 79 | session = ssh.get_transport().open_session() 80 | logging.debug("ssh session opened.") 81 | 82 | try: 83 | # execute command 84 | session.exec_command(cmd) 85 | logging.debug("session command exec done.") 86 | 87 | # To capture Data. Need to read the entire buffer to caputure output 88 | contents_buf = StringIO.StringIO() 89 | error_buf = StringIO.StringIO() 90 | while not session.exit_status_ready(): 91 | if session.recv_ready(): 92 | data = session.recv(byte_len) 93 | logging.debug("recv ready...data.lenght=%s" %(len(data))) 94 | while data: 95 | contents_buf.write(data) 96 | data = session.recv(byte_len) 97 | if session.recv_stderr_ready(): 98 | data = session.recv_stderr(byte_len) 99 | logging.debug("recv err ready...data.lenght=%s" %(len(data))) 100 | while data: 101 | error_buf.write(data) 102 | data = session.recv_stderr(byte_len) 103 | #logging.debug("waiting for exit_status_ready...") 104 | finally: 105 | session.close() 106 | logging.debug("ssh session closed.") 107 | 108 | logging.debug("exec_command finished. exit_status=%s" %(session.recv_exit_status())) 109 | return session.recv_exit_status(), contents_buf.getvalue(), error_buf.getvalue() 110 | finally: 111 | ssh.close() 112 | logging.debug("ssh closed.") 113 | 114 | 115 | # the main 116 | if __name__ == "__main__": 117 | 118 | status = 0 119 | try: 120 | ip, port, timeout = parse_from_args() 121 | username, password = parse_from_xssh_file() 122 | command = read_command_from_stdin() 123 | status, contents, errors = execute_on_ssh(username, password, ip, port, timeout, command) 124 | sys.stderr.write(errors) 125 | sys.stdout.write(contents) 126 | except getopt.GetoptError, ge: 127 | print >> sys.stderr, "Error! %s" %(ge) 128 | sys.exit(1) 129 | except XsshErrorCodeException, ece: 130 | print >> sys.stderr, "Error! %s" %(ece) 131 | sys.exit(1) 132 | except paramiko.AuthenticationException: 133 | print >> sys.stderr, "Error! username/password error, login failed." 134 | sys.exit(1) 135 | except socket.error: 136 | print >> sys.stderr, "Error! can't connect to remote host." 137 | sys.exit(1) 138 | except KeyboardInterrupt: 139 | pass 140 | --------------------------------------------------------------------------------