├── .gitignore ├── _config.yml ├── LICENSE ├── APPS ├── auto_retweet.sh ├── tikarecv.sh ├── comike_search.sh ├── gathertw.md └── sec2hour_anl.sh ├── CONFIG └── COMMON.SHLIB.SAMPLE ├── UTL ├── SRC │ └── sleep.c ├── urlencode ├── tarize ├── unescj.sh └── mktemp ├── BIN ├── getbtwid.sh ├── deldmtw.sh ├── bretwer.sh ├── twfollow.sh ├── twunfollow.sh ├── previous.ver │ └── deldmtw.sh ├── twfing.sh ├── twfer.sh ├── deltweet.sh └── retweet.sh └── README.ja.md /.gitignore: -------------------------------------------------------------------------------- 1 | CONFIG/COMMON.SHLIB -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-hacker -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /APPS/auto_retweet.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # 引数で与えられたキーワードにマッチするツイートをリツイートする 6 | # 7 | ###################################################################### 8 | 9 | 10 | # === このシステム(kotoriotoko)のホームディレクトリー等 ============== 11 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 12 | Dir_mime="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d"; pwd)" 13 | Dir_retweeted="$Dir_mime/${0##*/}.already" 14 | 15 | PATH="$Homedir/UTL:$Homedir/TOOL:/usr/bin/:/bin:/usr/local/bin:$PATH" 16 | 17 | 18 | # === 一時ファイルのプレフィックス =================================== 19 | Tmp="/tmp/${0##*/}.$(date +%Y%m%d%H%M%S).$$" 20 | 21 | 22 | # === 既にツイート済のIDの格納されているファイルの存在確認 =========== 23 | mkdir -p "$Dir_retweeted" 24 | File_retweetedids="$Dir_retweeted/$(printf '%s' "$@" | tr '/ \t' '___').id.txt" 25 | touch "$File_retweetedids" 26 | 27 | 28 | # === 該当するツイートIDをリツイートし、リツイート済として登録 ======= 29 | "$Homedir/BIN/twsrch.sh" "$@" | 30 | sed -n '/^- http/{s/^.*\///;p;}' | 31 | # 1:API検索でヒットした全ツイートのID # 32 | sort | 33 | join -1 1 -2 1 -v 2 "$File_retweetedids" - | 34 | # 1:ヒットしたもののうち未リツイートのID # 35 | while read id; do # 36 | "$Homedir/BIN/retweet.sh" "$id" >/dev/null 2>&1 # 37 | [ $? -eq 0 ] && echo "$id" # 38 | done | 39 | up3 key=1 "$File_retweetedids" - > $Tmp-updatedtweetids 40 | [ $? -eq 0 ] && mv $Tmp-updatedtweetids "$File_retweetedids" 41 | 42 | 43 | # === 終了 =========================================================== 44 | [ -n "$Tmp" ] && rm -rf "$Tmp"* 45 | exit 0 46 | -------------------------------------------------------------------------------- /CONFIG/COMMON.SHLIB.SAMPLE: -------------------------------------------------------------------------------- 1 | ###################################################################### 2 | # 3 | # COMMON.SHLIB -- The Common Configuration File 4 | # 5 | ###################################################################### 6 | 7 | 8 | ###################################################################### 9 | # Files and Directories 10 | ###################################################################### 11 | 12 | #Homedir= # Home directory of this app (unnecessary to set normally) 13 | 14 | 15 | ###################################################################### 16 | # Web access config 17 | ###################################################################### 18 | 19 | # === If you cannot use any kotoriotoko commands at all due to old === 20 | # root certificate, enable the following two variables. 21 | # (But it makes the commands insecurer) 22 | #no_cert_curl='-k' 23 | #no_cert_wget='--no-check-certificate' 24 | 25 | # === If curl command on your environment does not support =========== 26 | # "--compress" option, comment out the following line. 27 | # (It has been reported the following OS's curl doesn't surpport. 28 | # OpenWrt, LEDE) 29 | curl_comp_opt='--compressed' 30 | 31 | 32 | ###################################################################### 33 | # Configurations For Authorizing Kotoriotoko 34 | ###################################################################### 35 | 36 | readonly KOTORIOTOKO_apikey='vzKr0rF0EzsSppDXdzh1ZUz4M' 37 | readonly KOTORIOTOKO_apisec='pEP4LccLOd5i5OYR2XoDPo5WMUhVtb7wqenlBAgIoF6TiHS4YB' 38 | 39 | 40 | ###################################################################### 41 | # My account info 42 | ###################################################################### 43 | 44 | # === There are 2 ways to get the followin parameters ================ 45 | # (1) Use "BIN/getstarted.sh" command (recommended) 46 | # (2) Make your own app on https://apps.twitter.com/ and get them 47 | readonly MY_scname='Set Your Twitter Login Name (without "@")' 48 | readonly MY_apikey='Set Consumer Key (API Key)' 49 | readonly MY_apisec='Set Consumer Secret (API Secret)' 50 | readonly MY_atoken='Set Your Access Token' 51 | readonly MY_atksec='Set Your Access Token Secret' 52 | 53 | # === If you want to use "BIN/btw*.sh", run "BIN/getbtwid.sh" ======== 54 | # to get the following parameter 55 | #readonly MY_bearer='Your Bearer Token' 56 | -------------------------------------------------------------------------------- /APPS/tikarecv.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # ※ Raspberry Pi専用 6 | # 7 | # 引数で与えられたキーワードにマッチするツイート文に含まれる 8 | # 「チカ」という文字列の数だけLチカする。 9 | # 10 | # [例] 11 | # 1) Twitterで予め次のツイートをしておく。 12 | # "#kotoriotoko リッチー大佐 喰らえ! チカチカチカチカ" 13 | # "#kotoriotoko リッチー大佐 もう一度喰らえ! チカチカチカ" 14 | # 2) リッチー大佐は"#kotoriotoko"というキーワードと共に自分の名前で 15 | # このプログラムを次のように実行する。 16 | # ./tikarecv '#kotoriotoko' 'リッチー大佐' 17 | # するとリッチー大佐のRaspberry PiのLEDが7回Lチカする。 18 | # 19 | # [必要なもの] 20 | # ・Raspberry Pi 21 | # ・https://projects.drogon.net/raspberry-pi/wiringpi/the-gpio-utility/ 22 | # をインストールしておく。 23 | # ・LEDをGIPO#5に装着しておく。 24 | # 25 | ###################################################################### 26 | 27 | # === このシステム(kotoriotoko)のホームディレクトリー等 ============== 28 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 29 | Dir_mine="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d"; pwd)" 30 | Dir_retweeted="$Dir_mine/${0##*/}.already" 31 | 32 | PATH="$Homedir/UTL:$Homedir/TOOL:/usr/bin/:/bin:/usr/local/bin:$PATH" 33 | 34 | 35 | # === Lチカアクションを登録 ========================================== 36 | # ・第一引数にLチカさせる回数（デフォルトは1回） 37 | l_tika() { 38 | gpio -g mode 18 out 39 | 40 | n=$(printf '%s' "${1:-}" | tr -Cd 0123456789) 41 | [ -n "$n" ] || n=1 42 | 43 | i=0 44 | while [ $i -lt $n ]; do 45 | gpio -g write 18 1 46 | sleep 0.20 47 | gpio -g write 18 0 48 | sleep 0.20 49 | i=$((i+1)) 50 | done 51 | } 52 | 53 | 54 | # === 一時ファイルのプレフィックス =================================== 55 | Tmp="/tmp/${0##*/}.$(date +%Y%m%d%H%M%S).$$" 56 | 57 | 58 | # === 既に処理済のツイートIDの格納されているファイルの存在確認 ======= 59 | mkdir -p "$Dir_retweeted" 60 | File_retweetedids="$Dir_retweeted/$(printf '%s' "$@" | tr '/ \t' '___').id.txt" 61 | touch "$File_retweetedids" 62 | 63 | 64 | # === 検索で該当するツイートIDを見つけ、Lチカし、処理済として登録 ==== 65 | "$Homedir/BIN/twsrch.sh" "$@" | 66 | awk 'NR%5==3{ # 67 | # ツイートの中から"チカ"の数を数えて加算する # 68 | sub( /^../ ,"" ,$0); # 先頭の2文字削除 # 69 | gsub(/チカ/,"\006",$0); # "チカ"を記号"\006"に変更 # 70 | gsub(/[^\006]/,"" ,$0); # "\006"以外の文字を全削除 # 71 | n = length($0); # 文字列長が"チカ"の個数 # 72 | } # 73 | NR%5==0{ # 74 | # ツイートIDを抽出する # 75 | sub( /^.*\//,"" ,$0); # URL内のツイートID以外削除 # 76 | id = $0; # 77 | print id, n; # IDと回数を並べて出力 # 78 | }' | 79 | # 1:API検索でヒットした全ツイートのID 2:その内の"チカ"の数 # 80 | sort | 81 | join -1 1 -2 1 -v 2 "$File_retweetedids" - | 82 | # 1:ヒットしたもののうち未リツイートのID 2:その内の"チカ"の数 83 | while read id n; do # 84 | l_tika "$n" # 85 | echo "$id" # 86 | done | 87 | up3 key=1 "$File_retweetedids" - > $Tmp-updatedtweetids 88 | [ $? -eq 0 ] && mv $Tmp-updatedtweetids "$File_retweetedids" 89 | 90 | 91 | # === 終了 =========================================================== 92 | [ -n "$Tmp" ] && rm -rf "$Tmp"* 93 | exit 0 94 | -------------------------------------------------------------------------------- /UTL/SRC/sleep.c: -------------------------------------------------------------------------------- 1 | /*#################################################################### 2 | # 3 | # SLEEP - Sleep Command Which Supported Non-Integer Numbers 4 | # 5 | # USAGE : sleep seconds 6 | # Args : seconds ... The number of second to sleep for. You can 7 | # give not only an integer number but also a 8 | # non-integer number here. 9 | # Retuen : Return 0 only when succeeded to sleep 10 | # 11 | # How to compile : cc -o __CMDNAME__ __SRCNAME__ 12 | # 13 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-03-19 14 | # 15 | # This is a public-domain software (CC0). It means that all of the 16 | # people can use this for any purposes with no restrictions at all. 17 | # By the way, We are fed up with the side effects which are brought 18 | # about by the major licenses. 19 | # 20 | # The latest version is distributed at the following page. 21 | # https://github.com/ShellShoccar-jpn/misc-tools 22 | # 23 | ####################################################################*/ 24 | 25 | 26 | /*#################################################################### 27 | # Initial Configuration 28 | ####################################################################*/ 29 | 30 | /*=== Initial Setting ==============================================*/ 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | char* gpszCmdname; 38 | 39 | /*=== Define the functions for printing usage and error ============*/ 40 | void print_usage_and_exit(void) { 41 | fprintf(stderr, 42 | "USAGE : %s seconds\n" 43 | "Args : seconds ... The number of second to sleep for. You can\n" 44 | " give not only an integer number but also a\n" 45 | " non-integer number here.\n" 46 | "Retuen : Return 0 only when succeeded to sleep\n" 47 | "Version : 2020-03-19 12:18:14 JST\n" 48 | " (POSIX C language)\n" 49 | "\n" 50 | "Shell-Shoccar Japan (@shellshoccarjpn), No rights reserved.\n" 51 | "This is public domain software. (CC0)\n" 52 | "\n" 53 | "The latest version is distributed at the following page.\n" 54 | "https://github.com/ShellShoccar-jpn/misc-tools\n" 55 | ,gpszCmdname); 56 | exit(1); 57 | } 58 | void error_exit(int iErrno, const char* szFormat, ...) { 59 | va_list va; 60 | va_start(va, szFormat); 61 | fprintf(stderr,"%s: ",gpszCmdname); 62 | vfprintf(stderr,szFormat,va); 63 | va_end(va); 64 | exit(iErrno); 65 | } 66 | 67 | 68 | /*#################################################################### 69 | # Main 70 | ####################################################################*/ 71 | 72 | int main(int argc, char *argv[]) { 73 | 74 | /*=== Initial Setting ============================================*/ 75 | struct timespec tspcSleeping_time; 76 | double dNum; 77 | int i,iRet; 78 | 79 | gpszCmdname = argv[0]; 80 | for (i=0; *(gpszCmdname+i)!='\0'; i++) { 81 | if (*(gpszCmdname+i)=='/') {gpszCmdname=gpszCmdname+i+1;} 82 | } 83 | 84 | /*=== Parse options ==============================================*/ 85 | if (argc != 2 ) {print_usage_and_exit();} 86 | if (sscanf(argv[1], "%lf", &dNum) != 1) {print_usage_and_exit();} 87 | if (dNum > INT_MAX ) {print_usage_and_exit();} 88 | 89 | /*=== Sleep ======================================================*/ 90 | if (dNum <= 0 ) {exit(0); } 91 | tspcSleeping_time.tv_sec = (time_t)dNum; 92 | tspcSleeping_time.tv_nsec = (dNum - tspcSleeping_time.tv_sec) * 1000000000; 93 | 94 | iRet = nanosleep(&tspcSleeping_time, NULL); 95 | if (iRet != 0) {error_exit(iRet,"Error happend while nanosleeping\n");} 96 | 97 | /*=== Finish =====================================================*/ 98 | return 0; 99 | } 100 | -------------------------------------------------------------------------------- /APPS/comike_search.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh -u 2 | 3 | ###################################################################### 4 | # 5 | # comike_search.sh 6 | # コミケ関連のツイートを連続的に収集する 7 | # 8 | # [書式] 9 | # comike_search.sh <取得間隔(秒)> <プロセス間の間隔(秒)> <番号> 10 | # 11 | # [例] 12 | # comike_search.sh 20 2 3 13 | # とすると、時刻における秒の桁が 6-7,26-27,46-47 の時に各1回 14 | # Twitter APIに検索クエリーを送る。 15 | # 従って、 16 | # comike_search.sh 20 2 0 & comike_search.sh 20 2 1 & \ 17 | # comike_search.sh 20 2 2 & comike_search.sh 20 2 3 & \ 18 | # comike_search.sh 20 2 4 & comike_search.sh 20 2 5 & \ 19 | # comike_search.sh 20 2 6 & comike_search.sh 20 2 7 & \ 20 | # comike_search.sh 20 2 8 & comike_search.sh 20 2 9 & 21 | # のようにコマンド実行すると、10個の並列起動されたプロセスを使い、 22 | # 2秒毎にTwitter APIにアクセスする。 23 | # (各プロセスは、20秒以内に1周すれば取りこぼさずに結果を取得できる) 24 | # 尚、これらを停止させる時は、 25 | # jobs -l | awk '{print $2}' | xargs kill 26 | # と打ち込めばよい。（Bourneシェルの場合） 27 | # 28 | ###################################################################### 29 | 30 | 31 | ###################################################################### 32 | # 検索条件設定 33 | ###################################################################### 34 | 35 | geocode='35.630554,139.797358,1km' # 検索エリア（BigSight中心部から半径1km以内） 36 | query='' # 検索ワード 37 | #query='C89 OR コミケ OR コミケット OR コミックマーケット OR 冬コミ OR comiket' 38 | 39 | count=100 # 1度のクエリーで取得する最大ツイート数(100まで設定可) 40 | 41 | 42 | ###################################################################### 43 | # 初期設定 44 | ###################################################################### 45 | 46 | # === このシステム(kotoriotoko)のホームディレクトリー ================ 47 | Homedir=`case "$0" in */*) d="${0%/*}/";; *) d='./';; esac 48 | cd "$d" >/dev/null; echo "$PWD" ` 49 | 50 | # === 各種ディレクトリー設定 ========================================= 51 | Dir_RAW_BASE="$Homedir/${0##*/}.data/RESULT/RAW" 52 | Dir_RES_BASE="$Homedir/${0##*/}.data/RESULT/RES" 53 | File_lastid="$Homedir/${0##*/}.data/LASTID.txt" 54 | 55 | 56 | ###################################################################### 57 | # 引数取得 58 | ###################################################################### 59 | 60 | case $# in 3) :;; *) echo '*** 3 argument required, exit' 1>&2; exit 1;; esac 61 | interval=${1:-} 62 | printf '%s\n' "$interval" | grep -q '^[0-9]\{1,\}$' || { 63 | echo '*** Invalid 1st argument (interval time)' 1>&2 64 | exit 1 65 | } 66 | unit=${2:-} 67 | printf '%s\n' "$unit" | grep -q '^[0-9]\{1,\}$' || { 68 | echo '*** Invalid 2nd argument (unit)' 1>&2 69 | exit 1 70 | } 71 | number=${3:-} 72 | printf '%s\n' "$number" | grep -q '^[0-9]\{1,\}$' || { 73 | echo '*** Invalid 3rd argument (number)' 1>&2 74 | exit 1 75 | } 76 | 77 | 78 | ###################################################################### 79 | # メインループ 80 | ###################################################################### 81 | 82 | datetime0='' 83 | while :; do 84 | 85 | datetime=$(date '+%Y %m %d %H %M %S') 86 | datetime=${datetime%[0-9][0-9][0-9][0-9][0-9][0-9]} 87 | [ "$datetime" = "$datetime0" ] && { sleep 0.1; continue; } 88 | # 89 | datetime0=$datetime 90 | sec=${datetime##* } 91 | awk -v sec=$sec -v interval=$interval -v unit=$unit -v number=$number ' 92 | BEGIN { 93 | start = (int(sec/interval))*interval + unit*number; 94 | end = start + unit; 95 | exit ((sec >= start) && (sec < end)) ? 0 : 1; 96 | } 97 | ' || { sleep 0.1; continue; } 98 | 99 | [ -f "$File_lastid" ] && since_id=$(cat "$File_lastid") 100 | case "${since_id:-}" in '') since_id=1;; esac 101 | 102 | set -- $datetime 103 | File_RAW="$Dir_RAW_BASE/$1$2$3/$4/$4$5$6.$$.json" 104 | File_RES="$Dir_RES_BASE/$1$2$3/$4/$4$5$6.$$.txt" 105 | mkdir -p "${File_RAW%/*}" 106 | mkdir -p "${File_RES%/*}" 107 | "$Homedir/../BIN/btwsrch.sh" "--rawout=$File_RAW" \ 108 | "--timeout=$((interval-2))" \ 109 | -s "$since_id" \ 110 | -n "$count" \ 111 | -g "$geocode" \ 112 | "$query" > "$File_RES" 113 | # 114 | id=$(cat "$File_RES" | 115 | tail -n +5 | 116 | head -n 1 | 117 | sed 's/.*\///' | 118 | grep '^[0-9]\{1,\}$') 119 | [ -n "$id" ] && echo "$id" > "$File_lastid" 120 | [ -s "$File_RAW" ] || rm "$File_RAW" 121 | [ -s "$File_RES" ] || rm "$File_RES" 122 | sleep $unit 123 | 124 | done 125 | -------------------------------------------------------------------------------- /UTL/urlencode: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # URLENCODE - URL Encoder on the Basis of RFC 3986 6 | # 7 | # USAGE: urlencode [-r|--raw] ... 8 | # -r ...... RAW MODE : when this option is set, all of " " are 9 | # replaced with "%20" instead of "+". 10 | # --raw ... same as the "-r" option 11 | # 12 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-05-06 13 | # 14 | # This is a public-domain software (CC0). It means that all of the 15 | # people can use this for any purposes with no restrictions at all. 16 | # By the way, We are fed up with the side effects which are brought 17 | # about by the major licenses. 18 | # 19 | # The latest version is distributed at the following page. 20 | # https://github.com/ShellShoccar-jpn/misc-tools 21 | # 22 | ###################################################################### 23 | 24 | 25 | ###################################################################### 26 | # Initial Configuration 27 | ###################################################################### 28 | 29 | # === Initialize shell environment =================================== 30 | set -eu 31 | umask 0022 32 | export LC_ALL=C 33 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 34 | case $PATH in :*) PATH=${PATH#?};; esac 35 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 36 | 37 | # === Define the functions for printing usage ======================== 38 | print_usage_and_exit () { 39 | cat <<-USAGE 1>&2 40 | Usage : ${0##*/} [-r|--raw] ... 41 | Args : ...... Text file for URL encoding 42 | Options : -r, --raw ... RAW MODE : 43 | " " will not be converted into "+" but "%20" 44 | Version : 2020-05-06 22:42:19 JST 45 | (POSIX Bourne Shell/POSIX commands) 46 | USAGE 47 | exit 1 48 | } 49 | 50 | 51 | ###################################################################### 52 | # Parse Options 53 | ###################################################################### 54 | 55 | # === Print help message if required ================================= 56 | case "${1:-}" in 57 | --help|--version|-h) print_usage_and_exit;; 58 | esac 59 | 60 | # === Other options ================================================== 61 | instead_of_spc='+'; 62 | case $# in [!0]*) 63 | for arg in ${1+"$@"}; do 64 | case "$arg" in 65 | -r|--raw) instead_of_spc='%20'; shift;break;; 66 | --) shift;break;; 67 | *) : ;; 68 | esac 69 | done 70 | ;; 71 | esac 72 | 73 | 74 | ###################################################################### 75 | # Main 76 | ###################################################################### 77 | 78 | (cat ${1+"$@"}; echo '') | 79 | awk ' # 80 | BEGIN { # 81 | # --- prepare # 82 | OFS = ""; # 83 | ORS = ""; # 84 | # --- prepare encoding # 85 | for(i= 0;i<256;i++){c2p[sprintf("%c",i)]=sprintf("%%%02X",i);} # 86 | c2p[" "]="'"$instead_of_spc"'"; # 87 | for(i=48;i< 58;i++){c2p[sprintf("%c",i)]=sprintf("%c",i); } # 88 | for(i=65;i< 91;i++){c2p[sprintf("%c",i)]=sprintf("%c",i); } # 89 | for(i=97;i<123;i++){c2p[sprintf("%c",i)]=sprintf("%c",i); } # 90 | c2p["-"]="-"; c2p["."]="."; c2p["_"]="_"; c2p["~"]="~"; # 91 | # --- encode # 92 | while (getline line) { # 93 | for (i=1; i<=length(line); i++) { # 94 | print c2p[substr(line,i,1)]; # 95 | } # 96 | print "\n"; # 97 | } # 98 | }' | 99 | awk ' # 100 | BEGIN{ # 101 | ORS=""; # 102 | OFS=""; # 103 | getline line; # 104 | print line; # 105 | while (getline line) { # 106 | print "\n",line; # 107 | } # 108 | } # 109 | ' 110 | -------------------------------------------------------------------------------- /UTL/tarize: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # TARIZE : Apply "tar.gz" to the Specified Directories 6 | # 7 | # This command converts a file or a directory into a "tar.gz" archive 8 | # files. However, this requires some non-POSIX commands. That is one 9 | # of the following patterns. 10 | # (a) tar command 11 | # (b) pax command that supports the "-z" option 12 | # (c) pax command that does not support the "-z" option + gzip command 13 | # 14 | # === Usage === 15 | # Usage : tarize file_or_dir#1 [file_or_dir#2 [...]] 16 | # Args : file_or_dir ... the target file to be converted 17 | # 18 | # 19 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2021-10-27 20 | # 21 | # This is a public-domain software (CC0). It means that all of the 22 | # people can use this for any purposes with no restrictions at all. 23 | # By the way, We are fed up with the side effects which are brought 24 | # about by the major licenses. 25 | # 26 | ###################################################################### 27 | 28 | 29 | ###################################################################### 30 | # Initial configuration 31 | ###################################################################### 32 | 33 | # === Initialize shell environment =================================== 34 | set -u 35 | umask 0022 36 | export LC_ALL=C 37 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 38 | case $PATH in :*) PATH=${PATH#?};; esac 39 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 40 | 41 | # === Usage printing function ======================================== 42 | print_usage_and_exit () { 43 | cat <<-USAGE 1>&2 44 | Usage : ${0##*/} file_or_dir#1 [file_or_dir#2 [...]] 45 | Args : file_or_dir ... the target file to be converted 46 | Version : 2021-10-27 18:09:32 JST 47 | (POSIX Bourne Shell/POSIX commands/archiver commands) 48 | USAGE 49 | exit 1 50 | } 51 | error_exit() { 52 | ${2+:} false && echo "${0##*/}: $2" 1>&2 53 | exit ${1:-0} 54 | } 55 | 56 | # === Make sure the dependent commands exist ========================= 57 | if tar zcf - /dev/null >/dev/null 2>&1; then 58 | type=1 59 | elif pax -wz /dev/null >/dev/null 2>&1; then 60 | type=2 61 | elif pax -w /dev/null >/dev/null 2>&1 && type gzip >/dev/null 2>&1; then 62 | type=3 63 | else 64 | error_exit 1 'Dependent command(s) not found. See the header comment of me.' 65 | fi 66 | 67 | 68 | ###################################################################### 69 | # Parse Arguments 70 | ###################################################################### 71 | 72 | case $# in 0) print_usage_and_exit;; esac 73 | 74 | 75 | ###################################################################### 76 | # Main 77 | ###################################################################### 78 | 79 | for Dir_target in "$@"; do 80 | case "$Dir_target" in 81 | /|..|../|.|./) error_exit 1 "$Dir_target"': Invalid file or dir' 82 | ;; 83 | /*) Dir_base=${Dir_target%/*} 84 | case "$Dir_base" in '') Dir_base='/';; esac 85 | s=${Dir_target%/} 86 | file_trg=${s##*/} 87 | case "$file_trg" in '') 88 | error_exit 1 "$Dir_target"': Invalid file or dir' 89 | ;; esac 90 | ;; 91 | ../*|./*) Dir_base=$(cd "${Dir_target%/*}" && pwd) 92 | case "$Dir_base" in '') 93 | error_exit 1 "$Dir_target"': Invalid file or dir' 94 | ;; esac 95 | s=${Dir_target%/} 96 | file_trg=${s##*/} 97 | case "$file_trg" in '') 98 | error_exit 1 "$Dir_target"': Invalid file or dir' 99 | ;; esac 100 | ;; 101 | *) Dir_base=$(cd "./${Dir_target%/*}" && pwd) 102 | case "$Dir_base" in '') 103 | error_exit 1 "$Dir_target"': Invalid file or dir' 104 | ;; esac 105 | s=${Dir_target%/} 106 | file_trg=${s##*/} 107 | ;; 108 | esac 109 | case "$file_trg" in *.gz) 110 | echo "${0##*/}: $Dir_target: It is already gzipped, skip it." 1>&2 111 | continue 112 | ;; esac 113 | cd "$Dir_base" || error_exit 1 "$Dir_target"': Invalid file or dir' 114 | case $type in 115 | 1) if ! tar zcpf ${file_trg}.tar.gz ${file_trg}; then 116 | echo "${0##*/}: $Dir_target: Failed to compress, skip it." 1>&2 117 | cd -; continue 118 | fi 119 | ;; 120 | 2) if ! pax -wz ${file_trg} > ${file_trg}.tar.gz; then 121 | echo "${0##*/}: $Dir_target: Failed to compress, skip it." 1>&2 122 | cd -; continue 123 | fi 124 | ;; 125 | 3) if ! pax -w ${file_trg} > ${file_trg}.tar; then 126 | echo "${0##*/}: $Dir_target: Failed to compress, skip it." 1>&2 127 | continue 128 | elif ! gzip ${file_trg}.tar ; then 129 | echo "${0##*/}: $Dir_target: Failed to compress, skip it." 1>&2 130 | cd -; continue 131 | fi 132 | ;; 133 | esac 134 | printf '%s.tar.gz\n' "$Dir_target" 1>&2 135 | rm -rf "${file_trg}" 136 | cd - >/dev/null 2>&1 137 | 138 | done 139 | 140 | 141 | ###################################################################### 142 | # Finish 143 | ###################################################################### 144 | 145 | exit 0 146 | -------------------------------------------------------------------------------- /BIN/getbtwid.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # GETBTWID.SH : Get Your Bearer Token 6 | # (It is requiered by btw*.sh commands, which are to get 7 | # tweets in shorter interval) 8 | # 9 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-05-06 10 | # 11 | # This is a public-domain software (CC0). It means that all of the 12 | # people can use this for any purposes with no restrictions at all. 13 | # By the way, We are fed up with the side effects which are brought 14 | # about by the major licenses. 15 | # 16 | ###################################################################### 17 | 18 | 19 | ###################################################################### 20 | # Initial Configuration 21 | ###################################################################### 22 | 23 | # === Initialize shell environment =================================== 24 | set -u 25 | umask 0022 26 | export LC_ALL=C 27 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 28 | case $PATH in :*) PATH=${PATH#?};; esac 29 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 30 | 31 | # === Define the functions for printing usage and error message ====== 32 | print_usage_and_exit () { 33 | cat <<-USAGE 1>&2 34 | Usage : ${0##*/} 35 | REQUIREMENT : You have to fill the following variables on CONFIG.SHLIB 36 | before execute this command. 37 | * MY_apikey 38 | * MY_apisec 39 | Version : 2020-05-06 22:42:19 JST 40 | USAGE 41 | exit 1 42 | } 43 | error_exit() { 44 | ${2+:} false && echo "${0##*/}: $2" 1>&2 45 | exit $1 46 | } 47 | 48 | # === Detect home directory of this app. and define more ============= 49 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 50 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 51 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 52 | 53 | # === Confirm that the required commands exist ======================= 54 | # --- 1.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 1 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if any arguments are given ================ 69 | case "$#" in [!0]*) print_usage_and_exit;; esac 70 | 71 | 72 | ###################################################################### 73 | # Collect IDs to get bearer token 74 | ###################################################################### 75 | 76 | # === Confirm MY_apikey exists ======================================= 77 | case "${MY_apikey:-}" in '') error_exit 1 'MY_apikey is not set';; esac 78 | 79 | # === Confirm MY_apisec exists ======================================= 80 | case "${MY_apisec:-}" in '') error_exit 1 'MY_apisec is not set';; esac 81 | 82 | # === Warn if the IDs are same as the ones which kotoriotoko has ===== 83 | [ "${MY_apikey:-}" = "${KOTORIOTOKO_apikey:-}" ] && 84 | [ "${MY_apisec:-}" = "${KOTORIOTOKO_apisec:-}" ] && { 85 | cat <<-'WARNING_MESSAGE' 86 | ********************************************************************** 87 | WARNING 88 | ********************************************************************** 89 | ${MY_apikey} and ${MY_apisec} which are written in your COMMON.SHLIB 90 | are both same with ${KOTORIOTOKO_apikey} and ${KOTORIOTOKO_apisec}. 91 | 92 | The bearer token which will be generated soon is almost USELESS 93 | because of a lot of user use the same one and scramble the access 94 | limit of the token. 95 | 96 | You strongly should get a pair of app-key for your personal use at 97 | "Twitter Apps" (https://apps.twitter.com/). 98 | And set it into ${MY_apikey} and ${MY_apisec}. 99 | ---------------------------------------------------------------------- 100 | 101 | WARNING_MESSAGE 102 | sleep 5 103 | } 104 | 105 | 106 | ###################################################################### 107 | # Main Routine 108 | ###################################################################### 109 | 110 | # === Set parameters of Twitter API endpoint ========================= 111 | # (1)endpoint 112 | readonly API_endpt='https://api.twitter.com/oauth2/token' 113 | # (2)Content-Type header 114 | readonly HDR_ctype='Content-Type: application/x-www-form-urlencoded;charset=UTF-8' 115 | # (3)grant_type (string for requesting with POST method) 116 | readonly POS_gtype='grant_type=client_credentials' 117 | 118 | # === Generate the auth header to get the token ====================== 119 | readonly HDR_auth="$(printf '%s' "$MY_apikey:$MY_apisec" | 120 | base64 -w 0 | 121 | grep ^ | 122 | sed 's/^/Authorization: Basic /' )" 123 | 124 | # === Access to the endpoint ========================================= 125 | if [ -n "${CMD_WGET:-}" ]; then 126 | apires=$("$CMD_WGET" ${no_cert_wget:-} -q -O - \ 127 | --header="$HDR_auth" \ 128 | --header="$HDR_ctype" \ 129 | --post-data="$POS_gtype" \ 130 | "$API_endpt" | 131 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 132 | grep ^ | sed 's/\\/\\\\/g' # 133 | else # 134 | cat # 135 | fi ) 136 | elif [ -n "${CMD_CURL:-}" ]; then 137 | apires=$("$CMD_CURL" ${no_cert_curl:-} -s ${curl_comp_opt:-} \ 138 | -H "$HDR_auth" \ 139 | -H "$HDR_ctype" \ 140 | -d "$POS_gtype" \ 141 | "$API_endpt" | 142 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 143 | grep ^ | sed 's/\\/\\\\/g' # 144 | else # 145 | cat # 146 | fi ) 147 | fi 148 | 149 | # === Exit immediately if it failed to access ======================== 150 | case $? in [!0]*) error_exit 1 'Failed to access API';; esac 151 | 152 | # === Print a message for success ==================================== 153 | echo "$apires" | 154 | parsrj.sh | 155 | awk '$1=="$.access_token"{bearer =$2;} # 156 | END { # 157 | if (bearer!="") { # 158 | print "readonly MY_bearer=\047" $2 "\047"; # 159 | print ""; # 160 | print "Write the variable into COMMON.SHLIB."; # 161 | print "And you can use btw*.sh commands."; # 162 | } else { # 163 | exit 1; # 164 | } # 165 | } ' 166 | 167 | # === Print a error message if some error occured ==================== 168 | case $? in [!0]*) 169 | err=$(echo "$apires" | 170 | parsrj.sh 2>/dev/null | 171 | awk 'BEGIN {errcode=-1; } # 172 | $1~/\.code$/ {errcode=$2; } # 173 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 174 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 175 | END {print errcode, errmsg; }') 176 | [ -z "${err#* }" ] || { error_exit 1 "API error(${err%% *}): ${err#* }"; } 177 | error_exit 1 "API returned an unknown message: $apires" 178 | ;; esac 179 | 180 | 181 | ###################################################################### 182 | # Finish 183 | ###################################################################### 184 | 185 | exit 0 186 | -------------------------------------------------------------------------------- /APPS/gathertw.md: -------------------------------------------------------------------------------- 1 | # 擬似リアルタイムTwitter検索・収集コマンド“**gathertw.sh**”使い方 2 | 3 | 4 | ## 概要 5 | 6 | このコマンドは、Twitterクライアント「恐怖！小鳥男」のbtwsrch.shを連続で呼び出し、目的の検索クエリーに該当するツイート（公開ツイートのみ）を収集するコマンドだ。 7 | 8 | btwsrch.shを単独で呼び出すと該当するツイートの最新100件が取得されて終わりだが、これをつかえば過去に遡って集められる。ただしTwitter APIの制約で10日前までのものしか集められんがな。 9 | 10 | さらにこいつのすごいところは、過去のツイートを集め終えれば最初に戻ってその過去の時点までの範囲で検索・収集を続行できるのだ。しかもその様子は画面でモニターすることもできるから擬似的なリアルタイム検索ができるというわけだ。 11 | 12 | 13 | ## 事前準備 14 | 15 | ### その1 独自のアプリケーションキー 16 | 17 | このコマンドを使いたくば、まず[Twitterアプリケーション登録](https://apps.twitter.com/)サイトへ行って独自アプリケーションを作っておくことだな。小鳥男の設定ファイルCONFIG/COMMON.SHLIBにはそこで発行されたConsumer Key (API Key)、Consumer Secret (API Secret)を設定しておかねば使い物にならん。 18 | 19 | なぜかだと？btwsrch.shを含むbtw*.shコマンド皆、お前たちのユーザーアカウントではなくアプリケーションに対して設定されたアクセス制限を受ける。小鳥男デフォルトのアプリケーションは他に誰が使っているかわからんからな。そいつらと共にアクセス頻度の制限を奪い合って、結局使い物にならんからだ。 20 | 21 | 既に自分でアプリケーションを作っているなら構わんが、無いなら作っておけ。そして発行されたキーを設定ファイルCONFIG/COMMON.SHLIBに書き込め。 22 | 23 | ### その2 ベアラートークンの取得・設定 24 | 25 | btwsrch.shを含むbtw*.shコマンドを使うには、小鳥男の設定ファイルCONFIG/COMMON.SHLIBにTwitterアプリケーションのベアラートークン文字列を設定せねばならん。 26 | 27 | それを取得したければgetbtwid.shコマンドを実行するのだ。次のように打ち込めば目的のベアラートークン文字列が発行される。 28 | 29 | ```sh: 30 | $ cd <小鳥男のインストールディレクトリー>/BIN 31 | $ ./getbtwid.sh 32 | readonly MY_bearer='1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012' 33 | 34 | Write the variable into COMMON.SHLIB. 35 | And you can use btw* commands. 36 | $ 37 | ``` 38 | 39 | これを、COMMON.SHLIB ファイルに追加せよ。 `readonly MY_bearer=` という文字列ごと COMMON.SHLIB に貼り付けるのだ。 40 | 41 | 42 | ## 書式 43 | 44 | コマンドに対するオプションと、それに続いて検索キーワード（複数指定可）を記述する。 45 | 46 | ``` 47 | gathertw.sh [オプション] [検索キーワード ...] 48 | ``` 49 | 50 | オプション（後述の検索キーワード系）も検索キーワード（クエリー）も省略した場合はUsageが表示される。 51 | 52 | ### 検索キーワード（クエリー） 53 | 54 | 検索キーワード（クエリー）はTwitter APIの規則に準じる。 `POSIX UNIX` のように2単語を指定すれば、“POSIX”と“UNIX”の両方を含む検索（AND検索）になるし、 `POSIX OR UNIX` のように2単語目を“OR”にした3単語を指定すれば“POSIX”か“UNIX”のどちらかを含む検索（OR検索）になる。 55 | 56 | なお、英字は通常大文字小文字が区別されないし、 `シェルショッカー日本支部` のようにして長い単語を与えれば“シェルショッカー”と“日本支部”に分けられたうえでAND検索されたりする。 57 | 58 | このあたりの規則はTwitterの検索コマンドの仕様に準じる。[Twitterの検索機能の使い方 (検索コマンド一覧)](https://syncer.jp/twitter-how-to-use-search)あたりがよくまとまっているな。 59 | 60 | ちなみに、 `min_retweets:1000` というキーワードをつければリツイート数が1000以上のものに絞り込まれたり、 `:)` をつければポジティブな内容のツイートに絞り込まれるなど、高度な検索条件が付けられるらしいぞ。 61 | 62 | ### オプション一覧(1) ― 検索キーワード（クエリー）系 63 | 64 | オプションは2種類に大別され、ここでは検索キーワード（クエリー）に準ずるものを記す。 65 | 66 | ``` 67 | -g <緯度,経度,半径> : 68 | --geocode=<緯度,経度,半径>: 69 | 検索条件として、<緯度>、<経度>を中心とした<半径>で指定された円の中でツイートされた 70 | ツイートだけに絞り込む。（緯度、経度の順番に注意） 71 | 従って、このオプションを指定すると位置情報無しのツイートは全て検索対象外になる。 72 | <例> -g 35.630554,139.797358,1km 73 | 74 | -l <言語> : 75 | --lang=<言語>: 76 | 検索条件として、<言語>に該当するツイートだけに絞り込む。 77 | <例> -l ja 78 | 79 | -o <言語> : 80 | --locale=<言語>: 81 | 検索キーワード（クエリー）の指定言語を指定する。 82 | 日本語ならjaと指定するか、省略して自動認識させる方が無難。 83 | <例> -o ja 84 | ``` 85 | 86 | ### オプション一覧(2) ― 動作指示系 87 | 88 | 2種類に大別されるオプションのうち、ここではこのコマンドの動作を指示するためのものを記す。 89 | 90 | ``` 91 | -c,--continuously: 92 | 検索条件で指定された最古のツイートまで検索し終わると、取得済みの最新ツイートの 93 | 日時の次に訪れる午前0時（UTC）を起点としてそこから過去方向へ、 94 | 取得済み最新最新ツイートの日時まで再検索する。 95 | （1つもツイートが取得できていない場合は現在日時を起点とする） 96 | このオプションを指定すれば、擬似的なリアルタイム検索ができる。 97 | <過去ツイートの検索が要らない場合> 98 | 一度本コマンドを実行し（注1）、1個以上ツイートが獲得できたら[CTRL]+[C]を押して 99 | すぐ中断する。その後、再び実行すれば（ただし-Mオプションなしで） 100 | さっきの検索で見つかった日時より新しいツイートのみが検索されるので 101 | 擬似的なリアルタイム検索が始まる。 102 | (注1) 初回実行時、再実行時ともに-dオプションで同じディレクトリーを指定すること。 103 | 104 | -m,--monitoring: 105 | 検索条件で指定された最新のツイートを常に監視する。 106 | つまり、2秒間隔で最新のツイートを収集し、もし（API仕様による）収集限界の100を 107 | 超えた場合は取り切れなかった分を収集せずに、次に来る最新のツイートを 108 | 収集しようとする。（監視モード） 109 | 110 | -r <最大リトライ回数> : 111 | --retry=<最大リトライ回数>: 112 | 検索結果が0だった場合に，同じ条件での検索を再度試みる回数を指定する。 113 | デフォルトは4（-cオプションが有効な場合は1）。 114 | これは、Twitter APIの性質により、一見検索できなかったようにみえて、 115 | 再試行するとツイートが取れる場合があるために用意されたオプションである。 116 | 117 | -d <ディレクトリー名> : 118 | --datadir=<ディレクトリー名>: 119 | 収集したデータを格納するディレクトリーを指定する。 120 | ディレクトリーは既存あってもなくてもいい。（なければ新規作成する） 121 | <デフォルト> -d "gathertw.sh.data" 122 | 123 | -u : 124 | --until=: 125 | 指定した日付を含み、その時点から過去のツイートを検索する。 126 | 尚、ここで指定した日付はUTCに基づくものであるため、日本時間では午前9時が境界になる。 127 | また、-cオプション指定時は、条件に基づく過去の検索が終わると、 128 | 2周目以降の再実行時にはこのオプションの日付が翌日に変更されたのと同等の動作をする。 129 | 130 | -M <ツイートID> : 131 | --maxid=<ツイートID>: 132 | 指定した<ツイートID>を含み、その時点から過去のツイートを検索する。 133 | また、-cオプション指定時は、条件に基づく過去の検索が終わると、 134 | 2周目以降の再実行時にはこのオプションの指定は解除される。 135 | 136 | -S <ツイートID> : 137 | --sinceid=<ツイートID>: 138 | 検索をしていて、指定した<ツイートID>の時点まで遡ったら検索を終了する。 139 | なお、指定した<ツイートID>は含まない。（それより未来のツイートのみ検索される） 140 | また、-cオプション指定時は、条件に基づく過去の検索が終わると、 141 | 2周目以降の再実行時には、データ格納ディレクトリー内にある最新ツイートのIDが 142 | このオプションに指定されたのと同等の動作をする。 143 | 144 | -s : 145 | --sincedt=: 146 | 検索をしていて、指定した日時（または）の時点まで 147 | 遡ったら検索を終了する。 148 | なお、検索結果に指定日時以前のツイートが含まれていたらその時点で過去への検索は 149 | 終了するが、今検索されて取得してしまったツイートに関しては、 150 | 指定日時より過去であっても格納される。 151 | 152 | -p,-p1,-p2,-p3 : 153 | --peek,--peek1,--peek2,--peek3: 154 | 検索して取得されたツイートをリアルタイムに標準出力に送る（画面表示する）。 155 | 数字はどれくらい詳細に表示するかのレベルである。 156 | -p,-p1,--peek1 ... ツイート日時とツイート者、本文 157 | -p2,--peek2 ...... ＋リツイート数＆いいね数、URL 158 | -p3,--peek3 ...... ＋Twitterクライアント名、位置情報（あれば） 159 | 160 | --noraw,--nores,--noanl: 161 | ツイートのデータファイルを生成しない。 162 | norawはRAWディレクトリー（JSON生データ）を、 163 | noresはRESディレクトリー（JSONパース済データ）を、生成しない。 164 | noanlはANLディレクトリー（JSONパース済分析用データ）を、生成しない。 165 | （LAST_TWID.txtとSINCE_TWID.txtは生成される） 166 | -pオプションで、単にツイート結果を画面に表示したいだけで、 167 | ディスク領域を消費したくない場合に便利。 168 | ``` 169 | 170 | 171 | ## 例 172 | 173 | ### その1 174 | 175 | 過去の検索可能な（=10日前までの）“POSIX”と“UNIX”の両方を含むツイートを検索し、 `HOGE` ディレクトリーに収集する。（一度収集したら検索終了） 176 | 177 | ``` 178 | gathertw.sh -d HOGE POSIX OR UNIX 179 | ``` 180 | 181 | ### その2 182 | 183 | 過去の検索可能な（=10日前までの）“POSIX”または“UNIX”を含むツイートを検索し、 `HOGE` ディレクトリーに収集する。その要素は画面にモニターし（最小レベル）、過去の検索できるところまで遡り終えたら現在に戻って検索するという動作を無限に続ける。 184 | 185 | ``` 186 | gathertw.sh -d HOGE -p -c POSIX OR UNIX 187 | ``` 188 | 189 | ### その3 190 | 191 | 今日が2016/05/09だとして、今年のゴールデンウィークに関するツイートを集めたいとする。 192 | 193 | まず、検索できなくなってしまう前にゴールデンウィーク序盤（UTCの2016/04/30 00:00:00から過去）のツイートを最初に収集してしまい、それを取り終えたら今度はUTCの2016/05/01 00:00:00から24時間前まで、……、UTCの2016/05/09 00:00:00から24時間前まで、それが終わったら現在日時からのツイートをエンドレスで集める（収集先ディレクトリーは `GW2016` ）。ただし、“仕事”を含むツイートは除外したい。 194 | 195 | ``` 196 | gathertw.sh -d GW2016 --until=2016-04-30 ゴールデンウィーク -仕事 197 | ``` 198 | 199 | もし、 `-仕事` を検索キーワードとして最初に指定したい場合、オプションと間違えられないようにするにはオプションの後に “ `--` ”を記述すれば、それ以降の単語は必ず検索キーワード文字列であると見なされる。 200 | 201 | ``` 202 | gathertw.sh -d GW2016 --until=2016-04-30 -- -仕事 ゴールデンウィーク 203 | ``` 204 | 205 | 206 | ## 収集データ構造 207 | 208 | ### データディレクトリー構成 209 | 210 | `-d`オプションで指定されたディレクトリーの中には、次のようなディレクトリー構造にてデータが収集される。 211 | 212 | ``` 213 | . 214 | | 215 | |-- NUM_OF_TWEETS.txt ............ 収集した全ツイート数 216 | |-- LAST_TWID.txt ................ 収集した全ツイートのIDの中で最新のもの（*1） 217 | |-- SINCE_TWID.txt ............... 収集した全ツイートのIDの中で最古のもの 218 | | *1 連続収集モードの際、このID以前ものもは検索しないように 219 | | するために、本コマンドによって参照される） 220 | | 221 | |-- RES/ ......................... 収集データファイル（JSONパース済）置き場 222 | | | 223 | | |-- / .................... 該当日付の収集ツイート格納ディレクトリー 224 | | | | 225 | | | |-- / ...................... 該当時間の収集ツイート格納ディレクトリー 226 | | | | | 227 | | | | |-- .txt ........... 該当時分秒の収集ツイート格納ファイル 228 | | | | |-- : 229 | | | | | : 230 | | | | 231 | | | |-- / 232 | | | | : 233 | | | : 234 | | | 235 | | |-- / 236 | | | : 237 | | : 238 | | 239 | |-- ANL/ ......................... 分析用収集データファイル（JSONパース済）置き場 240 | | （ディレクトリー構造はRESと同じ） 241 | | 242 | `-- RAW/ ......................... 収集データファイル（JSON形式の生データ）置き場 243 | | 244 | |-- / .................... 検索結果JSONの先頭ツイートが該当日付であるデータ置き場 245 | | | 246 | | |-- / ...................... 検索結果JSONの先頭ツイートが該当時間であるデータ置き場 247 | | | | 248 | | | |-- .json . 検索結果JSONの先頭ツイートが該当日時であるデータファイル 249 | | | |-- : 250 | | | | : 251 | | | 252 | | |-- / 253 | | | : 254 | | : 255 | | 256 | |-- / 257 | | : 258 | : 259 | ``` 260 | 261 | ### RESデータファイル構造 262 | 263 | ``` 264 | 2016/05/17 12:34:56 265 | - リッチ―大佐 (@col_richie) 266 | - 世界制覇だ！ by col_richie 267 | - ret:12 fav:345 268 | - Bando-shi, Ibaraki (36.0,139.9) 269 | - Twitter for iPhone (http://twitter.com/download/iphone) 270 | - https://twitter.com/c_richie/status/123456789012345678 271 | ``` 272 | 273 | 1. ツイートされた日時（タイムゾーンは収集したホストのもの） 274 | 2. ユーザー名と、ユーザーID（スクリーンネーム）。認証済みアカウントの場合は行末に`[v]`が付く。 275 | 3. ツイート本文。公式リツイートの場合は1文字先頭に`RT`が付き、さらに1文字字下げされる。 276 | 4. 収集時点でのリツイート数といいね数、自分がリツイートやいいねを既にしていれば`RET`や`FAV`と表示される。 277 | 5. 場所の名前と緯度経度、多くのツイートにはこの情報がなく、その場合は`-`と表示される。 278 | 6. Twitterクライアント名とその公式ページURL 279 | 7. そのツイートのURL 280 | 281 | ### ANLデータファイル構造 282 | 283 | RESデータは1ツイートが7行で表現されており、各種UNIXコマンドでフィルタリングするには向いていないため1行に直したものがこのデータ形式だ。 284 | 285 | ``` 286 | 2016/05/17-12:34:56 リッチ―大佐 @col_richie - - 世界制覇だ！_by_col\_richie 12 \ 287 | 345 Bando-shi,_Ibaraki 36.0,139.9 Twitter_for_iPhone http://twitter.com/downloa \ 288 | d/iphone https://twitter.com/c_richie/status/123456789012345678 289 | ``` 290 | 291 | * RESデータの各行が1行に並んでいる。ただし、いくつかの元の行は更に分離されるため、最終的には次に示す13列のデータになる。 292 | 1. ツイートされた日時（YYYY/MM/DD-hh:mm:ss、タイムゾーンは収集したホストのもの） 293 | 2. ユーザー名 294 | 3. ユーザーID（スクリーンネーム） 295 | 4. 認証済みアカウントなら"v"、そうでないなら"-" 296 | 5. 公式リツイートなら"RT"、そうでないなら"-" 297 | 6. ツイート本文 298 | 7. 収集時点でのリツイート数 299 | 8. 収集時点でのいいね数 300 | 9. ツイート場所の名前（ない場合は"-"） 301 | 10. 緯度,経度（ない場合は"-"） 302 | 11. Twitterクライアント名 303 | 12. Twitterクライアントの公式ページURL 304 | 13. そのツイートのURL 305 | * 次に記す各列は、元の半角空白が"\_"エスケープされている。（"\_"はバックスラッシュ付になる） 306 | * 2列目（ユーザー名） 307 | * 6列目（ツイート本文） 308 | * 9列目（ツイート場所の名前） 309 | * 11列目（Twitterクライアント名） 310 | -------------------------------------------------------------------------------- /README.ja.md: -------------------------------------------------------------------------------- 1 | # シェルスクリプト製Twitter怪人「恐怖！小鳥男」 2 | 3 | ## お前は何者だ？ 4 | 5 | POSIX原理主義に基づき、シェルスクリプトで組まれたTwitterクライアント怪人だ。 6 | 7 | * 投稿系 8 | * 投稿（画像・動画、位置情報も添付可能） 9 | * リツイート 10 | * それらの取消 11 | * お気に入り登録、取消 12 | * ツイート閲覧系 13 | * タイムライン表示 14 | * 検索 15 | * ストリーミング検索 16 | * ユーザー系 17 | * フォロー 18 | * フォロー解除 19 | * フォロー先表示 20 | * フォロワー表示 21 | * 詳細表示 22 | * ダイレクトメッセージ系 23 | * 送信（最大10000字、画像・動画、位置情報も添付可能） 24 | * 削除 25 | * 一覧表示 26 | * 詳細表示 27 | * その他 28 | * トレンドキーワード表示 29 | 30 | と、一通り、いやそれ以上のことができる。 31 | 32 | POSIX原理主義集団「秘密結社シェルショッカー」は、これをもってTwitterにはびこる人間どもを洗脳し、世界征服を果たすのだ。ワッハッハ！ 33 | 34 | ## 動かすのに必要なものその１（ホスト） 35 | 36 | * WebアクセスできるUNIXホスト（これがなくては話にならん） 37 | * 一部の符号化演算コマンドとして次のうちのどちらか 38 | * [OpenSSL](https://www.openssl.org/) 39 | * [LibreSSL](http://www.libressl.org/) 40 | * HTTPアクセスコマンドとして次のうちのどちらか 41 | * [cURL](http://curl.haxx.se/) 42 | * [GNU Wget](http://www.gnu.org/software/wget/) 43 | 44 | レンタルサーバーなら、大抵のところが要件を満たしているはずだ。 45 | 46 | ### ※なに、「それのどこがPOSIX原理主義なのか」だと？ 47 | 48 | HTTPアクセスコマンドのみならず、2014年にHeartbleedという**巨大なセキュリティーホールをもたらしたOpenSSLにまで手を出して、もはやそれのどこがPOSIX原理主義なのか、**と言いたいわけだなお前たちは。えぇい、黙れ黙れ！ 49 | 50 | 確かにしょっちゅう脆弱性の見つかるソフトウェアに依存しているようではメンテナンス地獄から解放されず、POSIX原理主義の素晴らしさは発揮できん。だが、cURLコマンドやsendmailコマンドに手を出す時に言ったはずだ。 51 | 52 | > 「同等の機能を備えた代替品がすぐに見つかる」、「すぐに乗り換えられる」という状態を担保すべく、利用するにしても基本的な部分だけにとどめるのだ。 53 | 54 | と。幸いにしてOpenSSLは、Heartbleed事件の反省から、LibreSSLという同等機能の別実装が登場し、しかも"openssl"という名前まで一緒のコマンドまで用意されているのだ。つまり、**片方の実装に何か問題が起こったらもう片方の実装に簡単に乗り換えられることを担保しておる。** 55 | 56 | それにだ！今回の侵略で利用する**Twitter API 1.1やOAuth1.0aがそもそも10年、20年の長きに渡って使えるとも思えん。** だから、Twitterに関してはこれくらいの担保があれば十分なのだよ！ 57 | 58 | ## 動かすのに必要なものその２（Twitterアカウント） 59 | 60 | さてホストの用意の次は、アカウント登録だ。Twitterに投稿もするのだからTwitterアカウントが必要なのは言うまでも無かろう。持っていないのなら[ユーザー登録](https://twitter.com/signup)をするのだ。 61 | 62 | そして、そのアカウントに紐づいたアクセスキー（4種類） 63 | 64 | * Consumer Key (API Key) 65 | * Consumer Secret (API Secret) 66 | * Access Token 67 | * Access Token Secret 68 | 69 | を手に入れ、小鳥男に設定してやらねばならん。そうしなければ小鳥男はお前のアカウントとしてツイートをすることができんからだ。 70 | 71 | これらアクセスキーを手に入れる方法は2通りある。次のどちらかを選んで実行し、発行されたアクセスキーをメモしておけ。 72 | 73 | ### 方法1. getstarted.shコマンドを使う（簡単） 74 | 75 | こっちは簡単だ。後で説明する getstarted.sh コマンドを実行し、指示に従うだけだ。ただ、途中でWebブラウザーを使ってこの怪人がお前のアカウントを操作することを承認する作業が必要になるがな。 76 | 77 | コマンドが正常終了すれば4種のアクセスキーが発行され、小鳥男の設定ファイルへの書き足しまで済まされる（できない場合は書き方が表示される）。至れり尽くせりだろう。 78 | 79 | こっちの方法で済ませたいのなら小鳥男のプログラムをダウンロードするまで何もせんでよい。さっさと次へ進むがよい。 80 | 81 | ### 方法2. Twitterアプリケーション登録をする（b*.shコマンドも使いたい場合） 82 | 83 | 1つ目の方法だと、すべてのツイートメッセージは"Kotoriotoko (production model)"というアプリがしたという扱いなる。また、アクセス頻度制限を緩和するb*.shコマンドも使えない（実質役に立たない）。それがいやなら自分で[Twitterアプリケーション登録](https://developer.twitter.com/en/portal/projects-and-apps)をして自力でアクセスキーを発行することだな。 84 | 85 | ただ2015年あたりから審査が厳しくなりおった。実在証明のためにSMSを受信できる携帯電話が必要だから気を付けることだ。我々のような秘密結社を阻止しようということかもしれんが……、小癪な真似をしおって。そんなもの我々には無力だ！ 86 | 87 | ## ハウ・トゥ・インストール 88 | 89 | では、怪人を起動するまでの方法（インストール）を教えてやろう。 90 | 91 | ### 0) cURL,Wgetや*SSLがなければインストール 92 | 93 | 先程言ったように、この小鳥男（プログラム）を動かすには、cURLまたはWget、そしてOpenSSLまたはLibreSSLが必要だ。もし、無いならインストールし、コマンドにパスを通しておくのだ。パッケージだろうとソースからmakeだろうと構わんし、設定もデフォルトのままで構わん。 94 | 95 | ### 1) GitHubからgit clone 96 | 97 | そうしたらまずは、このリポジトリーをgit cloneせよ。「まずは」というかそれでインストールはおしまいだ！あとは設定のみ。 98 | 99 | ```sh: 100 | $ cd <適当なインストールディレクトリー> 101 | $ git clone https://github.com/ShellShoccar-jpn/kotoriotoko.git 102 | ``` 103 | 104 | もしgitコマンドが使えない場合は、このリポジトリーの[zipファイル](https://github.com/ShellShoccar-jpn/kotoriotoko/archive/master.zip)をダウンロード、展開し、その中の"BIN"と"TOOL"と"UTL"ディレクトリー内の各ファイルに実行パーミッションを与えるのだ。 105 | 106 | ```sh: 107 | $ cd <適当なインストールディレクトリー> 108 | $ wget https://github.com/ShellShoccar-jpn/kotoriotoko/archive/master.zip 109 | $ unzip master.zip 110 | $ chmod +x kotoriotoko/BIN/* kotoriotoko/TOOL/* kotoriotoko/UTL/* kotoriotoko/APPS/*.sh 111 | ``` 112 | 113 | ### 2) 設定ファイルにアカウント情報を書き込む 114 | 115 | 残りは設定作業だけだ。インストール前のTwitterアカウント発行で、「方法1」「方法2」のどちらを選択したかですべきことは変わる。 116 | 117 | 「方法1」を選択したのなら次のようにしてBINディレクトリーの中のgetstared.shを実行して、指示に従え。"Enjoy now!"とメッセージが出るまで進めば完了だ。 118 | 119 | ```sh: 120 | $ cd <小鳥男のインストールディレクトリー>/BIN 121 | $ ./getstarted.sh 122 | ``` 123 | 124 | 「方法2」を選択していたり、getstarted.shで"Enjoy now!"ではなく"Almost Finish preparing"と表示されたのなら次の作業をしろ。 125 | 126 | まず"CONFIG"ディレクトリーの中にある"COMMON.SHLIB.SAMPLE"というファイルを同じ場所に"COMMON.SHLIB"という名前でコピーし、 127 | 128 | ```sh: 129 | $ cd <小鳥男のインストールディレクトリー>/CONFIG 130 | $ cp COMMON.SHLIB.SAMPLE COMMON.SHLIB 131 | $ vi COMMON.SHLIB 132 | ``` 133 | 134 | この中身に、自分のアカウント名（ログインID）＋用意（またはgetstarted.shで表示された）したTwitterの4つのアクセスキー（計5つ）を書き込むのだ。 135 | 136 | ```text: 137 | : 138 | : 139 | ###################################################################### 140 | # My account info 141 | ###################################################################### 142 | 143 | readonly MY_scname='hogehoge' 144 | readonly MY_apikey='1234567890123456789012345' 145 | readonly MY_apisec='12345678901234567890123456789012345678901234567890' 146 | readonly MY_atoken='1234567890-123456789012345678901234567890123456789' 147 | readonly MY_atksec='123456789012345678901234567890123456789012345' 148 | : 149 | : 150 | ``` 151 | 152 | 5つのコードは、それぞれだいたいこれくらいの長さになっているはずだ。あまりにも違うようならもう一度確認しておくがいい。 153 | 154 | ## 追加の設定（b*.shコマンドを使いたい場合） 155 | 156 | 例えばテレビ番組で「特定のハッシュタグを使ってツイートしてください」などといい、物凄い密度でツイートが集まることがある。そういったツイートを全部かき集めて分析し、侵略計画を策略するのに役立てたい者もお前たちの中にはいるかもしれん。しかし、例えば検索コマンドなどは通常の認証方法では最大で5秒あたり1回の頻度（厳密には15分に180回まで）というアクセス制限がある。しかも、1回あたり最大で100ツイートしか取れん。これでは、ツイートをかき集めるには心もとないな。 157 | 158 | そこで、このアクセス頻度制限が緩和される b*.sh というコマンド群を用意した。例えば btwsrch.sh なら、今まで最大で5秒あたり1回の頻度（厳密には15分に180回まで）でしか検索できなかったものが、2秒あたり1回の頻度で検索できるのだ。だが、これらを使うにはもう一つ追加の設定作業が必要だ。 159 | 160 | ### 1) 「方法2」でアクセスキーを作り直す 161 | 162 | 小鳥男の設定ファイルCONFIG/COMMON.SHLIBにはそこで発行されたConsumer Key (API Key)、Consumer Secret (API Secret)を設定しておかねば使い物にならん。なぜかだと？btwsrch.shを含むb*.shコマンド皆、お前たちのユーザーアカウントではなくアプリケーションに対して設定されたアクセス制限を受ける。小鳥男デフォルトのアプリケーションは他に誰が使っているかわからんからな。そいつらと共にアクセス頻度の制限を奪い合って、結局使い物にならんからだ。 163 | 164 | だからアクセスキーの用意のところで「方法1」を選択していたのなら、「方法2」でアクセスキーの発行・設定をやり直してこい。 165 | 166 | ### 2) ベアラートークンを取得し、設定する 167 | 168 | btwsrch.shを含むb*.shコマンドを使うには、小鳥男の設定ファイルCONFIG/COMMON.SHLIBにTwitterアプリケーションのベアラートークン文字列を設定せねばならん。 169 | 170 | それを取得したければgetbtwid.shコマンドを実行するのだ。次のように打ち込めば目的のベアラートークン文字列が発行される。 171 | 172 | ```sh: 173 | $ cd <小鳥男のインストールディレクトリー>/BIN 174 | $ ./getbtwid.sh 175 | readonly MY_bearer='1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012' 176 | 177 | Write the variable into COMMON.SHLIB. 178 | And you can use btw* commands. 179 | $ 180 | ``` 181 | 182 | これを、COMMON.SHLIB ファイルに追加せよ。 `readonly MY_bearer=` という文字列ごと COMMON.SHLIB に貼り付けるのだ。 183 | 184 | 185 | ## つかいかた 186 | 187 | まずは、インストールされたTwitterクライアント怪人のディレクトリー構成を見よ！これを見れば、もうだいたいの使い方はわかるだろう。BINディレクトリーの中にあるのが、怪人を動かしてTwitter民を侵略するためのプログラムだ。 188 | 189 | ### ファイル・ディレクトリー構成 190 | 191 | ``` 192 | . 193 | |-- README.md ................ このファイル 194 | | 195 | |-- BIN/ ..................... Twitterコマンド群（普段使うのはこの中） 196 | | | 197 | | |-- getstarted.sh ........ アクセスキーを取得する（インストール時に実行する） 198 | | | 199 | | |-- tweet.sh ............. 与えられた文字列でツイートする 200 | | |-- ltweet.sh ............ 1行1ツイートと見なして連続ツイートする 201 | | |-- retweet.sh ........... 指定されたツイートをリツイート 202 | | |-- deltweet.sh .......... 指定されたツイート・リツイートを削除 203 | | |-- twmediup.sh .......... 指定されたメディアファイル（画像等）をアップロード 204 | | | （tweet.shで画像付ツイートをする時のサブコマンドでもある） 205 | | |-- twvideoup.sh ......... 指定された動画メディアファイル（MP4）をアップロード 206 | | | （twmediup.shでMP4動画を指定された時のサブコマンドでもある） 207 | | | 208 | | |-- twview.sh ............ 指定されたツイートの詳細を表示する 209 | | |-- twtl.sh .............. 指定されたユーザーのタイムラインを表示する 210 | | |-- twsrch.sh ............ 指定された文字列で検索する 211 | | |-- retwers.sh ........... 指定されたツイートをリツイートしたユーザー一覧を表示する 212 | | | 213 | | |-- twfav.sh ............. 指定されたツイートをお気に入り登録する 214 | | |-- twunfav.sh ........... 指定されたツイートのお気に入りを解除する 215 | | |-- favtws.sh ............ 指定されたユーザーのお気に入りツイートを表示する 216 | | | 217 | | |-- twfollow.sh .......... 指定されたユーザーをフォローする 218 | | |-- twunfollow.sh ........ 指定されたユーザーのフォローをやめる 219 | | |-- twfer.sh ............. 指定されたユーザーのフォロワーを表示する 220 | | |-- twfing.sh ............ 指定されたユーザーのフォローユーザーを表示する 221 | | |-- twusers.sh ........... 指定されたユーザーの情報を表示する 222 | | | 223 | | |-- getbtwid.sh .......... 高頻度APIアクセスコマンド（btw*.sh）用のIDを取得する 224 | | |-- btwsrch.sh ........... 指定された文字列で検索する（ベアラートークンモード*1） 225 | | |-- btwtl.sh ............. 指定ユーザーのタイムラインを表示（ベアラートークンモード*2） 226 | | |-- bretwer.sh ........... 指定ツイートのretweet者一覧を表示（ベアラートークンモード*3） 227 | | | *1 通常5秒あたり1回までの制限頻度が2秒あたり1回まで短縮可能 228 | | | *2 通常5秒あたり1回までの制限頻度が3秒あたり1回まで短縮可能 229 | | | *3 通常1分あたり1回までの制限頻度が15秒あたり1回まで短縮可能 230 | | | 231 | | |-- stwsrch.sh ........... 指定された文字列で検索する（ストリームモード*4） 232 | | | *4 アクセス制限無しの究極の検索コマンド（ただし英数字のみ） 233 | | | 234 | | |-- twplsrch.sh .......... 指定された文字列でTwitter上の位置情報を検索する 235 | | |-- twtrends.sh .......... 指定された場所のトレンドキーワードを表示する 236 | | | 237 | | |-- dmtweet.sh ........... 指定された相手に与えられた文字列でダイレクトメッセージ送信 238 | | |-- ldmtweet.sh .......... 指定された相手に1行1メッセージのダイレクトメッセージを連続送信 239 | | |-- deldmtw.sh ........... 指定されたダイレクトメッセージを削除する 240 | | |-- dmtwview.sh .......... 指定されたダイレクトメッセージを表示する 241 | | `-- dmtwlist.sh .......... 送受信済のダイレクトメッセージを一覧表示する 242 | | 243 | | 244 | |-- CONFIG/ .................. 設定ファイル置き場 245 | | | 246 | | |-- COMMON.SHLIB ......... 共通設定ファイル 247 | | | ・Twitter APIのキーやアクセストークン等を書き込む 248 | | | ・下の ".SAMPLE" ファイルをコピーして作る 249 | | `-- COMMON.SHLIB.SAMPLE .. 共通設定ファイルのテンプレート 250 | | 251 | | 252 | |-- TOOL/ .................... シェルスクリプト開発用コマンドセット "Open usp Tukubai" の一部 253 | | | ・だたしここに置いてあるのは、POSIX環境に移植したクローン 254 | | | 255 | | |-- calclock ............. 日常日時―UNIX時間変換コマンド 256 | | | ・Twitter APIからUTC日時で返す日時をJST日時等へ変換するのに利用 257 | | | ・Twitter APIが要求する、現在日時のUNIX時間表現の生成に利用 258 | | `-- self ................. 列抽出コマンド 259 | | ・self 1 3 5 は、awk '{print $1,$3,$5}'と同じ 260 | | ・可読性のために利用 261 | | 262 | |-- UTL/ ..................... その他、自作シェルスクリプト製コマンド置き場 263 | | | 264 | | |-- urlencode ............ URLエンコーダー 265 | | | ・Twitter APIが要求するURLエンコード済文字列の生成に利用 266 | | |-- parsrj.sh ............ JSONをシェルスクリプト向けに正規化するコマンド"PARSeR-Json" 267 | | | ・Twitter APIが返すJSONの解読に利用 268 | | |-- unescj.sh ............ エスケープ表現されたJSON中のユニコード文字を元に戻すコマンド 269 | | | ・日本語ツイートがこれで読めるようになる 270 | | | ・Twitter APIが返すJSONの解読に利用 271 | | `-- mime-make ............ MIMEマルチパート作成作成コマンド 272 | | ・画像ファイルをTwitterサーバーにアップロードする時に利用 273 | | 274 | `-- APPS/ .................... 小鳥男のコマンド（BINの中）を応用した各種サンプルシェルスクリプト 275 | | 276 | `-- gathertw.sh .......... 指定キーワードを含むツイートを収集するコマンド 277 | ・REST API反復呼出しによるツイートの連続自動収集を行う 278 | ・リアルタイム検索も可能（疑似的に） 279 | ・しかもStreaming APIでは不可能な日本語キーワード指定可 280 | ・しかもStreaming APIでありがちな輻輳時データ間引きもなし 281 | ・[詳しい使い方はここ](APPS/gathertw.md) 282 | ``` 283 | 284 | 詳細な使い方は省略するが、各Twitterコマンドの書式が知りたくば"--help"オプションを付けて実行してみるがいい。それでだいたいわかるはずだ。 285 | 286 | なぁに大丈夫だ、間違っても大したことないからいろいろ試してみるがいい。大変なことになるとしたら、周到な準備無しに殺人予告や爆破予告をツイートするくらいなものだ。まぁ我々組織としてはそういう使い方は大いに歓迎だがな、フハハハ……。 287 | 288 | さぁお前たちもこの怪人を操作してTwitter民たちを炎上させ、世界征服を目指すのだ、ゆけぃ！ 289 | 290 | 291 | # ライセンス 292 | 293 | 何、ライセンスだと？愚問だな。似るなり焼くなり売るなり改造するなり、好きにするがいい。我々のような秘密結社だろうが何だろうが誰でもだ。 294 | この意味がわからぬか？完全なるPUBLIC DOMAIN (CC0)という意味だ。 295 | 296 | ただ、一つだけ憶えておくことだ。使い出した瞬間、それはお前たちは我らに洗脳されたということだ。ワッハッハ！ -------------------------------------------------------------------------------- /BIN/deldmtw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # DMDELTW.SH : Delete A Direct Message 6 | # 7 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-09-27 8 | # 9 | # This is a public-domain software (CC0). It means that all of the 10 | # people can use this for any purposes with no restrictions at all. 11 | # By the way, We are fed up with the side effects which are brought 12 | # about by the major licenses. 13 | # 14 | ###################################################################### 15 | 16 | 17 | ###################################################################### 18 | # Initial Configuration 19 | ###################################################################### 20 | 21 | # === Initialize shell environment =================================== 22 | set -u 23 | umask 0022 24 | export LC_ALL=C 25 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 26 | case $PATH in :*) PATH=${PATH#?};; esac 27 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 28 | 29 | # === Define the functions for printing usage and error message ====== 30 | print_usage_and_exit () { 31 | cat <<-USAGE 1>&2 32 | Usage : ${0##*/} [tweet_id...] 33 | Version : 2020-09-27 23:09:39 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 5 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 5 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | tweetid='' 75 | rawoutputfile='' 76 | timeout='' 77 | 78 | # === Read options =================================================== 79 | while :; do 80 | case "${1:-}" in 81 | --rawout=*) # for debug 82 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 83 | rawoutputfile=$s 84 | shift 85 | ;; 86 | --timeout=*) # for debug 87 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 88 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 89 | error_exit 5 'Invalid --timeout option' 90 | } 91 | timeout=$s 92 | shift 93 | ;; 94 | --|-) break 95 | ;; 96 | --*|-*) error_exit 5 'Invalid option' 97 | ;; 98 | *) break 99 | ;; 100 | esac 101 | done 102 | 103 | 104 | ###################################################################### 105 | # Main Routine 106 | ###################################################################### 107 | 108 | # === Set parameters of Twitter API endpoint (common) ================ 109 | # (1)endpoint 110 | API_endpt='https://api.twitter.com/1.1/direct_messages/events/destroy.json' 111 | readonly API_endpt 112 | # (2)method for the endpoint 113 | readonly API_methd='DELETE' 114 | 115 | # === BEGIN: Tweet-ID Loop =========================================== 116 | num_of_tweets=0; num_of_success=0 117 | while read tweetid; do num_of_tweets=$((num_of_tweets+1)) 118 | 119 | # === Validate the Tweet-ID ========================================== 120 | printf '%s\n' "$tweetid" | grep -Eq '^[0-9]+$' || { 121 | echo "${0##*/}: $tweetid: Invalid tweet-ID" 1>&2; continue 122 | } 123 | 124 | # === Set parameters of Twitter API endpoint (indivisual) ============ 125 | # (1)parameters 126 | API_param=$(cat <<-PARAM | 127 | id=$tweetid 128 | PARAM 129 | grep -v '^[A-Za-z0-9_]\{1,\}=$') 130 | 131 | # === Pack the parameters for the API ================================ 132 | # --- 1.URL-encode only the right side of "=" 133 | # (note: This string is also used to generate OAuth 1.0 signature) 134 | apip_enc=$(printf '%s\n' "${API_param}" | 135 | grep -v '^$' | 136 | urlencode -r | 137 | sed 's/%3[Dd]/=/' ) 138 | # --- 2.joint all lines with "&" (note: string for giving to the API) 139 | apip_del=$(printf '%s' "${apip_enc}" | 140 | tr '\n' '&' | 141 | grep ^ | 142 | sed 's/^./?&/' ) 143 | 144 | # === Generate the signature string of OAuth 1.0 ===================== 145 | # --- 1.a random string 146 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 147 | # --- 2.the current UNIX time 148 | nowutime=$(date '+%Y%m%d%H%M%S' | 149 | calclock 1 | 150 | self 2 ) 151 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 152 | # (note: This string is also used for an HTTP header) 153 | oa_param=$(cat <<-OAUTHPARAM 154 | oauth_version=1.0 155 | oauth_signature_method=HMAC-SHA1 156 | oauth_consumer_key=${MY_apikey} 157 | oauth_token=${MY_atoken} 158 | oauth_timestamp=${nowutime} 159 | oauth_nonce=${randmstr} 160 | OAUTHPARAM 161 | ) 162 | # --- 4.generate pre-string of the signature 163 | # (note: the API parameters and OAuth 1.0 parameters 164 | # are formed a line like a CGI parameter of GET method) 165 | sig_param=$(cat <<-OAUTHPARAM | 166 | ${oa_param} 167 | ${apip_enc} 168 | OAUTHPARAM 169 | grep -v '^ *$' | 170 | sort -k 1,1 -t '=' | 171 | tr '\n' '&' | 172 | grep ^ | 173 | sed 's/&$//' ) 174 | # --- 5.generate the signature string 175 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 176 | # and the above No.4 string respectively at first. and transfer to 177 | # HMAC-SHA1 with the key string which made of the access-keys) 178 | sig_strin=$(cat <<-KEY_AND_DATA | 179 | ${MY_apisec} 180 | ${MY_atksec} 181 | ${API_methd} 182 | ${API_endpt} 183 | ${sig_param} 184 | KEY_AND_DATA 185 | urlencode -r | 186 | tr '\n' ' ' | 187 | grep ^ | 188 | sed 's/ *$//' | 189 | # 1:API-key 2:APIsec 3:method # 190 | # 4:API-endpoint 5:API-parameter # 191 | while read key sec mth ept par; do # 192 | printf '%s&%s&%s' $mth $ept $par | # 193 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 194 | "$CMD_OSSL" enc -e -base64 # 195 | done ) 196 | 197 | # === Access to the endpoint ========================================= 198 | # --- 1.connect and get a response 199 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 200 | "${oa_param}" \ 201 | "${sig_strin}" \ 202 | "${API_param}" | 203 | urlencode -r | 204 | sed 's/%3[Dd]/=/' | 205 | sort -k 1,1 -t '=' | 206 | tr '\n' ',' | 207 | grep ^ | 208 | sed 's/^,*//' | 209 | sed 's/,*$//' | 210 | sed 's/^/Authorization: OAuth /' | 211 | while read -r oa_hdr; do # 212 | if [ -n "${CMD_WGET:-}" ]; then # 213 | [ -n "$timeout" ] && { # 214 | timeout="--connect-timeout=$timeout" # 215 | } # 216 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 217 | --method=DELETE --header="$oa_hdr" \ 218 | $timeout \ 219 | "$API_endpt$apip_del" # 220 | elif [ -n "${CMD_CURL:-}" ]; then # 221 | [ -n "$timeout" ] && { # 222 | timeout="--connect-timeout $timeout" # 223 | } # 224 | "$CMD_CURL" ${no_cert_curl:-} -s -X DELETE \ 225 | $timeout ${curl_comp_opt:-} \ 226 | -H "$oa_hdr" \ 227 | "$API_endpt$apip_del" # 228 | fi # 229 | done | 230 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 231 | grep ^ | sed 's/\\/\\\\/g' # 232 | else # 233 | cat # 234 | fi ) 235 | # --- 2.exit immediately if it failed to access 236 | case $? in [!0]*) error_exit 4 'Failed to access API';; esac 237 | 238 | # === Print error message if some error occured ====================== 239 | case "$apires" in 240 | '') :;; 241 | *) err=$(echo "$apires" | 242 | parsrj.sh 2>/dev/null | 243 | awk 'BEGIN {errcode=-1; } # 244 | $1~/\.code$/ {errcode=$2; } # 245 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 246 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 247 | END {print errcode, errmsg; }') 248 | [ -z "${err#* }" ] || { error_exit 4 "API error(${err%% *}): ${err#* }"; } 249 | error_exit 4 "API returned an unknown message: $apires" 250 | ;; 251 | esac 252 | 253 | # === END: Tweet-ID Loop ============================================= 254 | num_of_success=$((num_of_success+1)) 255 | done <&2 33 | Usage : ${0##*/} [options] [tweet_id...] 34 | Options : -n |--count= 35 | --rawout= 36 | --timeout= 37 | Version : 2020-10-01 22:33:22 JST 38 | USAGE 39 | check_my_bearer_token_and_print || exit $? 40 | exit 1 41 | } 42 | check_my_bearer_token_and_print() { 43 | case "${MY_bearer:-}" in '') 44 | echo '*** Bearer token is missing (you must set it into $MY_bearer)' 2>&1 45 | return 255 46 | ;; 47 | esac 48 | return 0 49 | } 50 | error_exit() { 51 | ${2+:} false && echo "${0##*/}: $2" 1>&2 52 | exit $1 53 | } 54 | 55 | # === Detect home directory of this app. and define more ============= 56 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 57 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 58 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 59 | 60 | # === Confirm that the required commands exist ======================= 61 | # --- 1..cURL or Wget 62 | if type curl >/dev/null 2>&1; then 63 | CMD_CURL='curl' 64 | elif type wget >/dev/null 2>&1; then 65 | CMD_WGET='wget' 66 | else 67 | error_exit 5 'No HTTP-GET/POST command found.' 68 | fi 69 | 70 | 71 | ###################################################################### 72 | # Argument Parsing 73 | ###################################################################### 74 | 75 | # === Print usage and exit if one of the help options is set ========= 76 | case "$# ${1:-}" in 77 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 78 | esac 79 | 80 | # === Initialize parameters ========================================== 81 | tweetid='' 82 | count='' 83 | rawoutputfile='' 84 | timeout='' 85 | 86 | # === Initialize parameters ========================================== 87 | while :; do 88 | case "${1:-}" in 89 | --count=*) count=$(printf '%s' "${1#--count=}" | tr -d '\n') 90 | shift 91 | ;; 92 | -n) case $# in 1) error_exit 5 'Invalid -n option';; esac 93 | count=$(printf '%s' "$2" | tr -d '\n') 94 | shift 2 95 | ;; 96 | --rawout=*) # for debug 97 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 98 | rawoutputfile=$s 99 | shift 100 | ;; 101 | --timeout=*) # for debug 102 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 103 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 104 | error_exit 5 'Invalid --timeout option' 105 | } 106 | timeout=$s 107 | shift 108 | ;; 109 | --|-) break 110 | ;; 111 | --*|-*) error_exit 5 'Invalid option' 112 | ;; 113 | *) break 114 | ;; 115 | esac 116 | done 117 | printf '%s\n' "$count" | grep -q '^[0-9]*$' || { 118 | error_exit 5 'Invalid -n option' 119 | } 120 | 121 | 122 | ###################################################################### 123 | # Main Routine 124 | ###################################################################### 125 | 126 | # === Set parameters of Twitter API endpoint (common) ================ 127 | # (1)method for the endpoint 128 | readonly API_methd='GET' 129 | # (2)parameters 130 | API_param=$(cat <<-PARAM | 131 | count=${count} 132 | PARAM 133 | grep -v '^[A-Za-z0-9_]\{1,\}=$') 134 | readonly API_param 135 | 136 | # === BEGIN: Tweet-ID Loop =========================================== 137 | num_of_tweets=0; num_of_success=0 138 | while read tweetid; do num_of_tweets=$((num_of_tweets+1)) 139 | 140 | # === Validate the Tweet-ID ========================================== 141 | printf '%s\n' "$tweetid" | grep -Eq '^[0-9]+$' || { 142 | echo "${0##*/}: $tweetid: Invalid tweet-ID" 1>&2; continue 143 | } 144 | 145 | # === Set parameters of Twitter API endpoint (indivisual) ============ 146 | # (1)endpoint 147 | API_endpt="https://api.twitter.com/1.1/statuses/retweets/$tweetid.json" 148 | 149 | # === Pack the parameters for the API ================================ 150 | # --- 1.URL-encode only the right side of "=" 151 | apip_enc=$(printf '%s\n' "${API_param}" | 152 | grep -v '^$' | 153 | urlencode -r | 154 | sed 's/%3[Dd]/=/' ) 155 | # --- 2.joint all lines with "&" (note: string for giving to the API) 156 | apip_get=$(printf '%s' "${apip_enc}" | 157 | tr '\n' '&' | 158 | grep ^ | 159 | sed 's/^./?&/' ) 160 | 161 | # === Check whether my bearer token is available or not ============== 162 | check_my_bearer_token_and_print || exit $? 163 | 164 | # === Access to the endpoint ========================================= 165 | # --- 1.connect and get a response 166 | apires=$(echo "Authorization: Bearer $MY_bearer" | 167 | while read -r oa_hdr; do # 168 | if [ -n "${CMD_WGET:-}" ]; then # 169 | [ -n "$timeout" ] && { # 170 | timeout="--connect-timeout=$timeout" # 171 | } # 172 | if type gunzip >/dev/null 2>&1; then # 173 | comp='--header=Accept-Encoding: gzip' # 174 | else # 175 | comp='' # 176 | fi # 177 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 178 | --header="$oa_hdr" \ 179 | $timeout "$comp" \ 180 | "$API_endpt$apip_get" | # 181 | if [ -n "$comp" ]; then gunzip; else cat; fi # 182 | elif [ -n "${CMD_CURL:-}" ]; then # 183 | [ -n "$timeout" ] && { # 184 | timeout="--connect-timeout $timeout" # 185 | } # 186 | "$CMD_CURL" ${no_cert_curl:-} -s \ 187 | $timeout ${curl_comp_opt:-} \ 188 | -H "$oa_hdr" \ 189 | "$API_endpt$apip_get" # 190 | fi # 191 | done | 192 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 193 | grep ^ | sed 's/\\/\\\\/g' # 194 | else # 195 | cat # 196 | fi ) 197 | # --- 2.exit immediately if it failed to access 198 | case $? in [!0]*) error_exit 4 'Failed to access API';; esac 199 | 200 | # === Parse the response ============================================= 201 | # --- 1.extract the required parameters from the response (written in JSON) # 202 | echo "$apires" | 203 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 204 | parsrj.sh 2>/dev/null | 205 | unescj.sh -n 2>/dev/null | 206 | tr -d '\000\034' | 207 | sed 's/&/'$(printf '\034')'/g' | 208 | sed 's/<//g' | tr '\034' '&' | 209 | sed 's/^\$\[\([0-9]\{1,\}\)\]\.user\.\([^ .]*\)/ \1 \2/' | 210 | grep '^ ' | 211 | awk ' # 212 | BEGIN {init_param(2); } # 213 | $2=="id" {id= substr($0,length($1 $2)+4);print_tw(); next;} # 214 | $2=="name" {nm= substr($0,length($1 $2)+4);print_tw(); next;} # 215 | $2=="screen_name" {sn= substr($0,length($1 $2)+4);print_tw(); next;} # 216 | $2=="verified" {vf=(substr($0,length($1 $2)+4)=="true")?"[v]":""; # 217 | next;} # 218 | function init_param(lv) {if (lv<2) {return;} # 219 | id=""; nm=""; sn=""; vf=""; } # 220 | function print_tw( stat){if (id=="") {return;} # 221 | if (nm=="") {return;} # 222 | if (sn=="") {return;} # 223 | printf("%-19s %s (@%s)%s\n",id,nm,sn,vf); # 224 | init_param(2); }' | 225 | # --- 2.regard as an error if no line was outputed # 226 | awk '{print;} END{exit 1-(NR>0);}' 227 | 228 | # === Print error message if some error occured ====================== 229 | case $? in [!0]*) 230 | err=$(echo "$apires" | 231 | parsrj.sh 2>/dev/null | 232 | awk 'BEGIN {errcode=-1; } # 233 | $1~/\.code$/ {errcode=$2; } # 234 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 235 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 236 | END {print errcode, errmsg; }') 237 | [ -z "${err#* }" ] || { error_exit 4 "API error(${err%% *}): ${err#* }"; } 238 | ;; esac 239 | 240 | # === END: Tweet-ID Loop ============================================= 241 | num_of_success=$((num_of_success+1)) 242 | done <s which has been converted from "\ux000a"s. 17 | # 18 | # === Usage === 19 | # Usage: unescj.sh [-n] [JSONPath-value_textfile] 20 | # 21 | # 22 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-06-27 23 | # 24 | # This is a public-domain software (CC0). It means that all of the 25 | # people can use this for any purposes with no restrictions at all. 26 | # By the way, We are fed up with the side effects which are brought 27 | # about by the major licenses. 28 | # 29 | ###################################################################### 30 | 31 | 32 | ###################################################################### 33 | # Initial configuration 34 | ###################################################################### 35 | 36 | # === Initialize shell environment =================================== 37 | set -eu 38 | umask 0022 39 | export LC_ALL=C 40 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 41 | case $PATH in :*) PATH=${PATH#?};; esac 42 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 43 | 44 | # === Define the functions for printing usage and error message ====== 45 | print_usage_and_exit () { 46 | cat <<-USAGE 1>&2 47 | Usage : ${0##*/} [-n] [JSONPath-value_textfile] 48 | Version : 2020-06-27 02:14:39 JST 49 | (POSIX Bourne Shell/POSIX commands) 50 | USAGE 51 | exit 1 52 | } 53 | error_exit() { 54 | ${2+:} false && echo "${0##*/}: $2" 1>&2 55 | exit $1 56 | } 57 | 58 | 59 | ###################################################################### 60 | # Prepare for the Main Routine 61 | ###################################################################### 62 | 63 | # === Define some chrs. to escape some special chrs. temporarily ===== 64 | BS=$( printf '\010' ) # Back Space 65 | TAB=$(printf '\011' ) # Tab 66 | LFs=$(printf '\\\n_');LFs=${LFs%_} # Line Feed (for sed command) 67 | FF=$( printf '\014' ) # New Pafe (Form Feed) 68 | CR=$( printf '\015' ) # Carridge Return 69 | ACK=$(printf '\006' ) # Escape chr. for "\\" 70 | 71 | # === Get the options and the filepath =============================== 72 | # --- initialize option parameters ----------------------------------- 73 | optn=0 74 | file='' 75 | # 76 | # --- get them ------------------------------------------------------- 77 | case "$#" in [!0]*) case "$1" in '-n') optn=1;shift;; esac;; esac 78 | case $# in 79 | 0) : ;; 80 | 1) file=$1 ;; 81 | *) print_usage_and_exit;; 82 | esac 83 | 84 | # === Validate the arguments ========================================= 85 | if [ "_$file" = '_' ] || 86 | [ "_$file" = '_-' ] || 87 | [ "_$file" = '_/dev/stdin' ] || 88 | [ "_$file" = '_/dev/fd/0' ] || 89 | [ "_$file" = '_/proc/self/fd/0' ] ; then 90 | file='' 91 | elif [ -f "$file" ] || 92 | [ -c "$file" ] || 93 | [ -p "$file" ] ; then 94 | [ -r "$file" ] || error_exit 1 'Cannot open the file: '"$file" 95 | else 96 | print_usage_and_exit 97 | fi 98 | case "$file" in ''|-|/*|./*|../*) :;; *) file="./$file";; esac 99 | 100 | 101 | ###################################################################### 102 | # Main Routine (Convert and Generate) 103 | ###################################################################### 104 | 105 | # === Open the data source =================================================== # 106 | grep '' ${file:+"$file"} | 107 | # # 108 | # === Escape "\\" to ACK temporarily ========================================= # 109 | sed 's/\\\\/'"$ACK"'/g' | 110 | # # 111 | # === Mark the original <0x0A> with <0x0A>+"\N" after it ===================== # 112 | sed 's/$/'"$LFs"'\\N/' | 113 | # # 114 | # === Insert <0x0A> into the behind of "\uXXXX" ============================== # 115 | sed 's/\(\\u[0-9A-Fa-f]\{4\}\)/'"$LFs"'\1/g' | 116 | # # 117 | # === Unescape "\uXXXX" into UTF-8 =========================================== # 118 | # (But the following ones are transfer the following strings # 119 | # \u000a -> \n, \u000d -> \r, \u005c -> \\, \u0000 -> \0, \u0006 -> \A) # 120 | awk ' # 121 | BEGIN { # 122 | OFS=""; ORS=""; # 123 | for(i=255;i>0;i--) { # 124 | s=sprintf("%c",i); # 125 | bhex2chr[sprintf("%02x",i)]=s; # 126 | bhex2int[sprintf("%02x",i)]=i; # (a) # 127 | } # 128 | bhex2chr["00"]="\\0" ; # 129 | bhex2chr["06"]="\\A" ; # 130 | bhex2chr["0a"]="\\n" ; # Both (a) and (b) are also the # 131 | bhex2chr["0d"]="\\r" ; # transferring table from a 2 bytes # 132 | bhex2chr["5c"]="\\\\"; # of hex number to a decimal one. # 133 | #for(i=65535;i>=0;i--) { # (b) # (a) is to use 256 keys twice. (b) # 134 | # whex2int[sprintf("%02x",i)]=i; # : # is to use 65536 keys once. And (a) # 135 | #} # : # was a litter faster than (b). # 136 | j=0; # 137 | while (getline l) { # 138 | if (l=="\\N") {print "\n"; continue; } # 139 | if (match(l,/^\\u00[0-7][0-9a-fA-F]/)) { # 140 | print bhex2chr[tolower(substr(l,5,2))], substr(l,7); # 141 | continue; # 142 | } # 143 | if (match(l,/^\\u0[0-7][0-9a-fA-F][0-9a-fA-F]/)) { # 144 | #i=whex2int[tolower(substr(l,3,4))]; # <-(a) V(b) # 145 | i=bhex2int[tolower(substr(l,3,2))]*256+bhex2int[tolower(substr(l,5,2))]; # 146 | printf("%c%c",192+int(i/64),128+i%64); # 147 | print substr(l,7); # 148 | continue; # 149 | } # 150 | if (match(l,/^\\u[Dd][89AaBb][0-9a-fA-F][0-9a-fA-F]$/)) { # 151 | # Decode high-surrogate part # 152 | j = bhex2int["0" tolower(substr(l,4,1))]*262144 + \ 153 | bhex2int[ tolower(substr(l,5,2))]* 1024 - \ 154 | 2031616; # 155 | continue; # 156 | } # 157 | if (match(l,/^\\u[Dd][C-Fc-f][0-9a-fA-F][0-9a-fA-F]/ )) { # 158 | # Decode low-surrogate part # 159 | j += bhex2int["0" tolower(substr(l,4,1))]*256 + \ 160 | bhex2int[tolower(substr(l,5,2))] - \ 161 | 3072; # 162 | i1=240+int(j/262144); j%=262144; # 163 | i2=128+int(j/ 4096); j%= 4096; # 164 | i3=128+int(j/ 64); j%= 64; # 165 | i4=128+ j ; j = 0; # 166 | printf("%c%c%c%c",i1,i2,i3,i4); # 167 | print substr(l,7); # 168 | continue; # 169 | } # 170 | if (match(l,/^\\u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]/)) { # 171 | #i=whex2int[tolower(substr(l,3,4))]; # <-(a) V(b) # 172 | i=bhex2int[tolower(substr(l,3,2))]*256+bhex2int[tolower(substr(l,5,2))]; # 173 | printf("%c%c%c",224+int(i/4096),128+int((i%4096)/64),128+i%64); # 174 | print substr(l,7); # 175 | continue; # 176 | } # 177 | print l; # 178 | } # 179 | }' | 180 | # === Unsscape escaped strings except "\n", "\0" and "\\" ==================== # 181 | sed 's/\\"/"/g' | 182 | sed 's/\\\//\//g' | 183 | sed 's/\\b/'"$BS"'/g' | 184 | sed 's/\\f/'"$FF"'/g' | 185 | sed 's/\\r/'"$CR"'/g' | 186 | sed 's/\\t/'"$TAB"'/g' | 187 | # # 188 | # === Also unescape "\0", "\r", "\n", "\\" when "-n" option is not given ===== # 189 | case "$optn" in # 190 | 0) sed 's/\\0//g' | # - "\0" should be deleted # 191 | sed 's/\\r/'"$CR"'/g' | # without conv to <0x00> # 192 | sed 's/\\n/'"$LFs"'/g' | # # 193 | sed 's/'"$ACK"'/\\\\/g' | # - Unescaoe escaped "\\"s # 194 | sed 's/\([^\\]\(\\\\\)*\)\\A/\1'"$ACK"'/g' | # and then restore "\A"s # 195 | sed 's/\([^\\]\(\\\\\)*\)\\A/\1'"$ACK"'/g' | # to s # 196 | sed 's/^\(\(\\\\\)*\)\\A/\1'"$ACK"'/g' | # : # 197 | sed 's/\\\\/\\/g' ;; # - Unescape "\\"s into "\"s# 198 | *) sed 's/'"$ACK"'/\\\\/g' | # 199 | sed 's/\([^\\]\(\\\\\)*\)\\A/\1'"$ACK"'/g' | # 200 | sed 's/\([^\\]\(\\\\\)*\)\\A/\1'"$ACK"'/g' | # 201 | sed 's/^\(\(\\\\\)*\)\\A/\1'"$ACK"'/g' ;; # 202 | esac 203 | -------------------------------------------------------------------------------- /UTL/mktemp: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # MKTEMP - The "mktemp" Command also Works on Just a POSIX Environment 6 | # 7 | # USAGE: mktemp [--by-myself] [options] [template [...]] 8 | # 9 | # --by-myself ..... (write as the 1st argument when use) 10 | # Not use the "built-in" mktemp command 11 | # and always do mktemp by myself but it is 12 | # inferior to the built-in in performance 13 | # -d .............. Makes a directory instead of a file 14 | # -p ........ Defines the temporary directory instead of 15 | # $TMPDIR 16 | # -q .............. Quiet even if some error occurred 17 | # -u .............. Makes a file but remove it immediately 18 | # -t .............. Adds the temporary directory as a prefix 19 | # in front of the filepath 20 | # --suffix= . Adds the suffix behind the filename 21 | # 22 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-05-06 23 | # 24 | # This is a public-domain software (CC0). It means that all of the 25 | # people can use this for any purposes with no restrictions at all. 26 | # By the way, We are fed up with the side effects which are brought 27 | # about by the major licenses. 28 | # 29 | # The latest version is distributed at the following page. 30 | # https://github.com/ShellShoccar-jpn/misc-tools 31 | # 32 | ###################################################################### 33 | 34 | 35 | ###################################################################### 36 | # Initial Configuration 37 | ###################################################################### 38 | 39 | # === Initialize shell environment =================================== 40 | set -u 41 | umask 0022 42 | export LC_ALL=C 43 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 44 | case $PATH in :*) PATH=${PATH#?};; esac 45 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 46 | 47 | # === Define the functions for printing usage and error message ====== 48 | print_usage_and_exit () { 49 | cat <<-USAGE 1>&2 50 | Usage : ${0##*/} [--by-myself] [options] [template [...]] 51 | Arg&Opts: Almost compatible with GNU mktemp 52 | The following options are available and compatible with it 53 | -d, -p , -q, -u, -t, --suffix= 54 | But --by-myself is the only original option, it prevent 55 | from use the built-in same name command even if available 56 | Version : 2020-05-06 22:42:19 JST 57 | (POSIX Bourne Shell/POSIX commands) 58 | USAGE 59 | exit 1 60 | } 61 | error_exit() { 62 | ${2+:} false && echo "${0##*/}: $2" 1>&2 63 | exit $1 64 | } 65 | 66 | # === Exec the built-in mktemp command if OK and exists ============== 67 | by_myself=0 68 | case "${1:-}" in 69 | '--by-myself') 70 | shift; by_myself=1 71 | ;; 72 | *) export mydir=$(d=${0%/*}/;[ "_$d" = "_$0/" ]||cd "$d";echo "$(pwd)") 73 | path0=${PATH:-} 74 | PATH=$(printf '%s\n' "$path0" | 75 | tr ':' '\n' | 76 | awk '$0!=ENVIRON["mydir"]{print;}' | 77 | tr '\n' ':' | 78 | grep -v '^:$' | 79 | sed 's/:$//' ) 80 | CMD_builtin=$(command -v mktemp 2>/dev/null || :) 81 | case "$CMD_builtin" in '') by_myself=1;; esac 82 | PATH=$path0 83 | unset mydir 84 | ;; 85 | esac 86 | case $by_myself in 0) exec "$CMD_builtin" ${1+"$@"}; exit 1;; esac 87 | 88 | # === Investigate the temporary directory if set ===================== 89 | Dir_tmp=$(set | grep ^TMPDIR= | sed 's/^[^=]\{1,\}=//'); 90 | case "$Dir_tmp" in 91 | '') Dir_tmp='/tmp' ;; 92 | /) Dir_tmp='/.' ;; 93 | *) Dir_tmp=${Dir_tmp%/};; 94 | esac 95 | 96 | # === Misc definitions =============================================== 97 | chrs='0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_' 98 | max_retry_when_failed=10 99 | LFs=$(printf '\\\n_');LFs=${LFs%_} 100 | 101 | 102 | ###################################################################### 103 | # Parse Arguments 104 | ###################################################################### 105 | 106 | # === Get the options and the filepath =============================== 107 | # --- initialize option parameters ----------------------------------- 108 | optd=0 109 | optu=0 110 | optq=0 111 | optt=0 112 | opts='' 113 | # 114 | # --- get them ------------------------------------------------------- 115 | optmode='' 116 | while [ $# -gt 0 ]; do 117 | case $# in 0) break;; esac 118 | case "$optmode" in 119 | '') case "$1" in 120 | -[duqtp]*) s=$(printf '%s\n' "${1#-}" | 121 | awk '{d = "_"; u = "_"; q = "_"; # 122 | t = "_"; p = "_"; err = 0; # 123 | for (i=1;i<=length($0);i++) { # 124 | s = substr($0,i,1); # 125 | if (s == "d") { d = "d"; } # 126 | else if (s == "u") { u = "u"; } # 127 | else if (s == "q") { q = "q"; } # 128 | else if (s == "t") { t = "t"; } # 129 | else if (s == "p") { p = p ; } # 130 | else { err = 1 ; } # 131 | } # 132 | p = (substr($0,i-1)=="p") ? "p" : "_"; # 133 | printf("%s%s%s%s%s%s",d,u,q,t,p,err); }') 134 | case "$s" in *1*) print_usage_and_exit;; esac 135 | case "$s" in *d*) optd=1 ;; esac 136 | case "$s" in *u*) optu=1 ;; esac 137 | case "$s" in *q*) optq=1 ;; esac 138 | case "$s" in *t*) optt=1 ;; esac 139 | case "$s" in *p*) optmode='p' ;; esac 140 | shift; continue 141 | ;; 142 | --directory) optd=1 ; shift; continue;; 143 | --dry-run) optu=1 ; shift; continue;; 144 | --quiet) optq=1 ; shift; continue;; 145 | --tmpdir) shift; continue;; 146 | --tmpdir=*) optmode='p' 147 | s=${1#--tmpdir=} ;; 148 | --suffix=*) optmode='s' 149 | s=${1#--suffix=} ;; 150 | -*) print_usage_and_exit ;; 151 | esac ;; 152 | *) s=$1 ;; 153 | esac 154 | case "$optmode" in 155 | p) [ -d "$s" ] || error_exit 1 'Invalid path by -p,--tmpdir option' 156 | Dir_tmp=${s%/}; [ -n "$Dir_tmp" ] || Dir_tmp='/.' 157 | optmode=''; shift; continue ;; 158 | s) { printf '%s' "$s" | grep -q '/'; } && { 159 | error_exit 1 'Invalid suffix option' 160 | } 161 | opts=$s 162 | optmode=''; shift; continue ;; 163 | esac 164 | break 165 | done 166 | 167 | 168 | ###################################################################### 169 | # Prepare for the Main Routine 170 | ###################################################################### 171 | 172 | # ===== FUNC: random string generator ================================ 173 | # arg : $1=length 174 | # ret : 0 175 | # stdout: generated random string 176 | random_string () { 177 | # calculate the number of words which required 178 | nw=$(echo "${1}*l(${#chrs})/11.09+1" | # 11.09=ln(65536) 179 | bc -l | 180 | sed 's/\..*$//' ) 181 | 182 | # make a random hexadecimal digit 183 | if [ -c /dev/urandom ]; then 184 | hstr=$(dd if=/dev/urandom bs=2 count=$nw 2>/dev/null | 185 | od -A n -t x2 -v | 186 | tr 'abcdef ' 'ABCDEF\n' | 187 | tr -Cd 0123456789ABCDEF ) 188 | else 189 | hstr=$( (ps -Ao pid,etime,pcpu,vsz; date) | 190 | od -t d4 -A n -v | 191 | sed 's/[^0-9]\{1,\}/'"$LFs"'/g' | 192 | grep '[0-9]' | 193 | tail -n 42 | 194 | sed 's/.*\(.\{8\}\)$/\1/g' | 195 | awk 'BEGIN{a=-2147483648;} # 196 | {a+=$1; } # 197 | END { # 198 | srand(a); # 199 | for(i=0;i<'$nw';i++){ # 200 | printf("%02X",int(rand()*65536)); # 201 | } # 202 | }' ) 203 | fi 204 | 205 | # make a random string from the hexadecimal digit 206 | echo "obase=${#chrs};ibase=16;$hstr" | 207 | bc | 208 | tr -d '\\\n' | 209 | tr ' ' '\n' | 210 | awk 'BEGIN {for(i=1;i<'$1';i++){print 0;}} # 211 | /[0-9]/{print; }' | 212 | awk 'BEGIN {ORS=""; # 213 | s="'"$chrs"'"; # 214 | for(i=0;i "$Path_target") 2>/dev/null || { 280 | [ -f "$Path_target" ] && { n=$((n-1)); continue; } 281 | n=-1; break; 282 | } 283 | ;; 284 | 1) umask 077; mkdir "$Path_target" 2>/dev/null || { 285 | [ -d "$Path_target" ] && { n=$((n-1)); continue; } 286 | n=-1; break; 287 | } 288 | ;; 289 | esac 290 | 291 | break 292 | done 293 | 294 | # --- Print error message when failed to make a file --------------- 295 | case "${optq}${n}" in 296 | '0-1') printf '%s\n' "${0##*/}: failed on $Dir_trg${Path_target##*/}" 1>&2 297 | err=1 298 | continue 299 | ;; 300 | '1-1') err=1 301 | continue 302 | ;; 303 | esac 304 | 305 | # --- Print the path of the file ----------------------------------- 306 | printf '%s%s\n' "$Dir_trg" "${Path_target##*/}" 307 | 308 | # --- Remove the file if -u,--dry-run option is set ---------------- 309 | case $optu in 1) rm -rf "$Path_target";; esac 310 | 311 | done 312 | 313 | 314 | ###################################################################### 315 | # Finish 316 | ###################################################################### 317 | 318 | exit $err 319 | -------------------------------------------------------------------------------- /APPS/sec2hour_anl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # SEC2HOUR_ANL.SH : Concatenate the Files In Seconds into In Hours 6 | # 7 | # This command is a converter. It reads the files the command "gathertw.sh" 8 | # generated. Then it outputs concatinated files by the following rule. 9 | # 10 | # ----------------------------------------------- 11 | # [before] DIR_BY_GATHERTW.SH/ 12 | # | 13 | # +-- ANL/ 14 | # | | 15 | # | +-- yyyymmdd/ 16 | # | | | 17 | # | | +-- 00/ --+-- 00mmss.txt 18 | # | | | +-- 00mmss.txt 19 | # | | | : 20 | # | | | 21 | # | | +-- 01/ --+-- 01mmss.txt 22 | # | | | +-- 01mmss.txt 23 | # | | | : 24 | # | | : 25 | # | | +-- 23/ --+-- 23mmss.txt 26 | # | | +-- 23mmss.txt 27 | # | | : 28 | # | | 29 | # | +-- yyyymmdd/ 30 | # : : 31 | # 32 | # [after] DIR_BY_GATHERTW.SH/ 33 | # | 34 | # +-- ANL/ 35 | # | | 36 | # | +-- yyyymmdd/ --+-- 00.txt 37 | # | | +-- 00.txt 38 | # | | + : 39 | # | | + : 40 | # | | +-- 23.txt 41 | # | | 42 | # | +-- yyyymmdd/ 43 | # : : : 44 | # ----------------------------------------------- 45 | # 46 | # The rule means that all "hhmmss.txt" files in the same directory 47 | # "yyyymmdd/hh" will be concatenated to a file "yyyymmdd/hh.txt". This 48 | # conversion helps you save the number of inodes consumed in your disk. 49 | # 50 | # This command can be applies ONLY to "ANL"-format directories, 51 | # not "RES" directories. If you want to save the number of inodes 52 | # on "RES"-format directory, use "tarize" command instead. It is 53 | # in "UTL" directory. 54 | # 55 | # See the help message for more detail. 56 | # 57 | # 58 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2021-11-08 59 | # 60 | # This is a public-domain software (CC0). It means that all of the 61 | # people can use this for any purposes with no restrictions at all. 62 | # By the way, We are fed up with the side effects which are brought 63 | # about by the major licenses. 64 | # 65 | ###################################################################### 66 | 67 | 68 | ###################################################################### 69 | # Initial Configuration 70 | ###################################################################### 71 | 72 | # === Initialize shell environment =================================== 73 | set -u 74 | umask 0022 75 | export LC_ALL=C 76 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 77 | case $PATH in :*) PATH=${PATH#?};; esac 78 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 79 | 80 | # === Define the functions for printing usage and exiting ============ 81 | print_usage_and_exit () { 82 | cat <<-USAGE 1>&2 83 | Usage : ${0##*/} [options] datadir_by_gathertw [...] 84 | Options : -i|--impatient .. Abort the directory immediately when 85 | finding out one of date directory is 86 | already done 87 | * The combination of "-i" and "-n" is 88 | convenient for a daily batch. 89 | -k|--keep ....... Keep the original files 90 | -o|--oldest ..... Convert from the oldest date (def.) 91 | -n|--newest ..... Convert from the newest date 92 | -p|--parallel ... Executable parallelly 93 | * The temporary directory 94 | "datadir_by_gathertw.tmp" will be left 95 | even after this command ends. So, you 96 | have to remove the directory yourself. 97 | Version : 2021-11-08 23:23:29 JST 98 | USAGE 99 | exit 1 100 | } 101 | exit_trap() { 102 | set -- ${1:-} $? # $? is set as $1 if no argument given 103 | trap '' EXIT HUP INT QUIT PIPE ALRM TERM 104 | [ -d "${Dir_now:-}" ] && rm -rf "$Dir_now" 105 | case $parallel in 0) [ -d "${Dir_tmp:-}" ] && rm -rf "$Dir_tmp";; esac 106 | trap '-' EXIT HUP INT QUIT PIPE ALRM TERM 107 | exit $1 108 | } 109 | error_exit() { 110 | ${2+:} false && echo "${0##*/}: $2" 1>&2 111 | exit $1 112 | } 113 | 114 | 115 | ###################################################################### 116 | # Argument Parsing 117 | ###################################################################### 118 | 119 | # === Set the default parameters ===================================== 120 | keep=0 121 | fromold=1 122 | parallel=0 123 | impatient=0 124 | 125 | # === Read and validate options ====================================== 126 | while [ $# -gt 0 ]; do 127 | case "${1:-}" in 128 | --) shift 129 | break 130 | ;; 131 | -) break 132 | ;; 133 | -*) s=$(echo "_${1#?}" | 134 | sed 's/^_//; s/./& /g' | 135 | tr ' ' '\n' | 136 | sed '$d' | 137 | awk 'BEGIN {o[0]=""; o[1]=""; o[2]=""; o[3]=""; # 138 | o[4]=""; o[5]=""; o[6]=""; } # 139 | $1=="h"{o[1]="h"; next; } # 140 | $1=="i"{o[2]="i"; next; } # 141 | $1=="k"{o[3]="k"; next; } # 142 | $1=="n"{o[4]="n"; o[5]=""; next; } # 143 | $1=="o"{o[5]="o"; o[4]=""; next; } # 144 | $1=="p"{o[6]="p"; next; } # 145 | {o[0]="-"; next; } # 146 | END {print o[0],o[1],o[2],o[3],o[4],o[5],o[6];}') 147 | case $s in *-*) error_exit 1 'Invalid option';; esac 148 | case $s in *h*) print_usage_and_exit ;; esac 149 | case $s in *i*) impatient=1 ;; esac 150 | case $s in *k*) keep=1 ;; esac 151 | case $s in *n*) fromold=0 ;; esac 152 | case $s in *o*) fromold=1 ;; esac 153 | case $s in *p*) parallel=1 ;; esac 154 | case $s in *y*) yyyy=1 ;; esac 155 | shift 156 | ;; 157 | --impatient) impatient=1 158 | shift 159 | ;; 160 | --keep) keep=1 161 | shift 162 | ;; 163 | --newest) fromold=0 164 | shift 165 | ;; 166 | --oldest) fromold=1 167 | shift 168 | ;; 169 | --parallel) parallel=1 170 | shift 171 | ;; 172 | --help) print_usage_and_exit 173 | ;; 174 | --*) error_exit 1 'Invalid option' 175 | ;; 176 | *) break 177 | ;; 178 | esac 179 | done 180 | case $# in 0) print_usage_and_exit;; esac 181 | 182 | 183 | ###################################################################### 184 | # Main Routine 185 | ###################################################################### 186 | 187 | # === Memorize the current directory path ============================ 188 | Dir_base=$(pwd) 189 | 190 | # === BEGINNING OF LOOP : Get and validate the directory name ======== 191 | for arg in "$@"; do 192 | # 193 | # --- Define the directories 194 | case $arg in 195 | /) s='/' ;; 196 | /*|./*|../*) s=${arg%/} ;; 197 | *) s="./${arg%/}";; 198 | esac 199 | s=$(cd "$s" 2>/dev/null && pwd) 200 | case "$s" in '') 201 | echo "$arg: Cannt access to this directory. Skip it." 202 | continue 203 | esac 204 | Dir_src="${s}/ANL" 205 | Dir_tmp="${s}.tmp" 206 | Dir_dst="${s}/ANL" 207 | 208 | # === Prepare ======================================================== 209 | # 210 | # --- Validate the source directory 211 | find "$Dir_src" -type d \( -name '[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' \ 212 | -o -name '[0-9][0-9][0-9][0-9]' \) | 213 | grep -Eq '/ANL/([0-9]{8}|[0-9]{4}/[0-9]{4})$' 214 | case $? in [!0]*) 215 | echo "$arg: Not the directory generated by gathertw or not found. Skip it." 216 | continue 217 | ;; 218 | esac 219 | # 220 | # --- Skip the directory if it is being processed on parallel mode 221 | case $parallel in 222 | 0) [ -e "$Dir_tmp" ] && { 223 | echo "$arg: Could be being processed in parallel mode. Skip it." 224 | continue 225 | } 226 | ;; 227 | esac 228 | # 229 | # --- Mkdir 230 | mkdir -p "$Dir_tmp" || { echo "$Dir_tmp: Cannot mkdir. Skip $arg."; continue; } 231 | trap 'exit_trap' EXIT HUP INT QUIT PIPE ALRM TERM 232 | mkdir -p "$Dir_dst" || { echo "$Dir_dst: Cannot mkdir. Skip $arg."; continue; } 233 | 234 | # === Concatinate second files into a hour file ====================== 235 | 236 | cd "$Dir_src" || { echo "$arg: Cannot access. Skip."; continue; } 237 | 238 | echo [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9] [0-9][0-9][0-9][0-9]/[0-9][0-9][0-9][0-9] | 239 | tr ' ' '\n' | 240 | awk '/^[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]$/ {print $0,$0; next;} # 241 | /^[0-9][0-9][0-9][0-9]\/[0-9][0-9][0-9][0-9]$/{s=$0;sub(/\//,"_",s); # 242 | print $0, s; next; }' | 243 | case $fromold in 0) sort -k 1r,1;; *) sort -k 1,1;; esac | 244 | while read ymd_src ymd_tmp; do 245 | trap 'exit_trap' HUP INT QUIT PIPE ALRM TERM # to avoid an old bash bug 246 | 247 | # --- Try to get the right to process the directory of the date 248 | Dir_now="$Dir_tmp/$ymd_tmp" 249 | mkdir "$Dir_now" 2>/dev/null || { 250 | Dir_now='' 251 | case $parallel in 252 | 0) error_exit 1 'The temporary directory is something wrong.' ;; 253 | *) echo "Skip \"${arg%/}/ANL/$ymd_src\" (Processing it by another)" 254 | continue ;; 255 | esac 256 | } 257 | 258 | # --- Concatinate the files 259 | cd "$Dir_src/$ymd_src" || { 260 | echo "Skip \"${arg%/}/ANL/$ymd_src\" (cannot access to it)" 261 | continue 262 | } 263 | echo "$(date '+%Y/%m/%d-%H:%M:%S'): Now converting \"${arg%/}/ANL/$ymd_src\"" 264 | for hh in [0-9][0-9]; do 265 | case ${#hh} in 266 | 2) : 267 | ;; 268 | *) rm -rf "$Dir_now"; Dir_now=''; 269 | case $impatient in 270 | 0) echo "Skip \"${arg%/}/ANL/$ymd_src\" (already done)" 271 | continue 2 272 | ;; 273 | *) echo "Abort \"${arg%/}\" (\"$ymd_src\" is already done)" 274 | break 2 275 | ;; 276 | esac 277 | ;; 278 | esac 279 | cd "$Dir_src/$ymd_src/$hh" || continue 280 | echo "$(date '+%Y/%m/%d-%H:%M:%S'): processing \"${ymd_src}/${hh}\"" 281 | find . -name '*.txt' | 282 | xargs awk '$NF~/^https:/{s=$NF;sub(/^.*\//,"",s);print s,$0}' | 283 | sort -bk 1n,1 | 284 | sed 's/^[^[:blank:]]* //' > "$Dir_now/${hh}.txt" 285 | done 286 | 287 | # --- Move the concatinated data file to the destination directory 288 | if [ -d "$Dir_dst/$ymd_src" ]; then 289 | s=$(cd "$Dir_now" 2>/dev/null && echo [0-9][0-9].txt) 290 | if mv "$Dir_now"/[0-9][0-9].txt "$Dir_dst/$ymd_src"; then 291 | rm -rf "$Dir_now"; Dir_now='' 292 | case $keep in 0) 293 | for s in $s; do 294 | case $s in [0-9][0-9].txt) :;; *) break;; esac 295 | rm -rf "$Dir_dst/$ymd_src/${s%.txt}" 296 | done 297 | esac 298 | else 299 | rm -rf "$Dir_now"; Dir_now='' 300 | s="${arg%/}/ANL/$ymd_src" 301 | echo "Skip \"$s\" (cannot write the concatinated files there)" 302 | continue 303 | fi 304 | elif [ -e "$Dir_dst/$ymd_src" ]; then 305 | if mv "$Dir_dst/$ymd_src" "$Dir_dst/${ymd_src}.bak"; then 306 | if mv "$Dir_now" "$Dir_dst/$ymd_src"; then 307 | Dir_now='' 308 | case $keep in 0) rm -rf "$Dir_dst/${ymd_src}.bak";; esac 309 | else 310 | mv -f "$Dir_dst/${ymd_src}.bak" "$Dir_dst" 311 | rm -rf "$Dir_now"; Dir_now='' 312 | echo "Skip \"${arg%/}/ANL/$ymd_src\" (cannot replace the directory)" 313 | continue 314 | fi 315 | else 316 | rm -rf "$Dir_now"; Dir_now='' 317 | s="${arg%/}/ANL/$ymd_src" 318 | echo "Skip \"$s\" (cannot evacuate the source directory)" 319 | continue 320 | fi 321 | else 322 | case "$ymd_src" in */*) mkdir -p "$Dir_dst/${ymd_src%/*}";; esac 323 | if mv "$Dir_now" "$Dir_dst/$ymd_src"; then 324 | Dir_now='' 325 | else 326 | rm -rf "$Dir_now"; Dir_now='' 327 | s="${arg%/}/ANL/$ymd_src" 328 | echo "Skip \"$s\" (cannot move the directory of the date)" 329 | continue 330 | fi 331 | fi 332 | done 333 | 334 | # === END OF LOOP : Put away the temporary directory and go back to the top 335 | case $parallel in 0) 336 | [ -d "${Dir_tmp:-}" ] && rm -rf "$Dir_tmp" && Dir_tmp='' 337 | ;; esac 338 | cd "$Dir_base" || error_exit 1 'Cannot "cd" to the original directory' 339 | done 340 | 341 | 342 | ###################################################################### 343 | # Finish 344 | ###################################################################### 345 | 346 | exit 0 347 | -------------------------------------------------------------------------------- /BIN/twfollow.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # TWFOLLOW.SH : Follow A User 6 | # 7 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-05-06 8 | # 9 | # This is a public-domain software (CC0). It means that all of the 10 | # people can use this for any purposes with no restrictions at all. 11 | # By the way, We are fed up with the side effects which are brought 12 | # about by the major licenses. 13 | # 14 | ###################################################################### 15 | 16 | 17 | ###################################################################### 18 | # Initial Configuration 19 | ###################################################################### 20 | 21 | # === Initialize shell environment =================================== 22 | set -u 23 | umask 0022 24 | export LC_ALL=C 25 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 26 | case $PATH in :*) PATH=${PATH#?};; esac 27 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 28 | 29 | # === Define the functions for printing usage and error message ====== 30 | print_usage_and_exit () { 31 | cat <<-USAGE 1>&2 32 | Usage : ${0##*/} 33 | Version : 2020-05-06 22:42:19 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 1 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 1 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | scname='' 75 | rawoutputfile='' 76 | timeout='' 77 | 78 | # === Read options =================================================== 79 | while :; do 80 | case "${1:-}" in 81 | --rawout=*) # for debug 82 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 83 | rawoutputfile=$s 84 | shift 85 | ;; 86 | --timeout=*) # for debug 87 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 88 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 89 | error_exit 1 'Invalid --timeout option' 90 | } 91 | timeout=$s 92 | shift 93 | ;; 94 | --|-) break 95 | ;; 96 | --*|-*) error_exit 1 'Invalid option' 97 | ;; 98 | *) break 99 | ;; 100 | esac 101 | done 102 | 103 | # === Get the loginname (screen name) ================================ 104 | case $# in 105 | 1) scname=$(printf '%s' "${1#@}" | tr -d '\n');; 106 | *) print_usage_and_exit ;; 107 | esac 108 | printf '%s\n' "$scname" | grep -Eq '^[A-Za-z0-9_]+$' || { 109 | print_usage_and_exit 110 | } 111 | 112 | 113 | ###################################################################### 114 | # Main Routine 115 | ###################################################################### 116 | 117 | # === Set parameters of Twitter API endpoint ========================= 118 | # (1)endpoint 119 | readonly API_endpt='https://api.twitter.com/1.1/friendships/create.json' 120 | readonly API_methd='POST' 121 | # (2)parameters 122 | API_param=$(cat <<-PARAM | 123 | screen_name=$scname 124 | PARAM 125 | grep -v '^[A-Za-z0-9_]\{1,\}=$') 126 | readonly API_param 127 | 128 | # === Pack the parameters for the API ================================ 129 | # --- 1.URL-encode only the right side of "=" 130 | # (note: This string is also used to generate OAuth 1.0 signature) 131 | apip_enc=$(printf '%s\n' "${API_param}" | 132 | grep -v '^$' | 133 | urlencode -r | 134 | sed 's/%3[Dd]/=/' ) 135 | # --- 2.joint all lines with "&" (note: string for giving to the API) 136 | apip_pos=$(printf '%s' "${apip_enc}" | 137 | tr '\n' '&' ) 138 | 139 | # === Generate the signature string of OAuth 1.0 ===================== 140 | # --- 1.a random string 141 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 142 | # --- 2.the current UNIX time 143 | nowutime=$(date '+%Y%m%d%H%M%S' | 144 | calclock 1 | 145 | self 2 ) 146 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 147 | # (note: This string is also used for an HTTP header) 148 | oa_param=$(cat <<-OAUTHPARAM 149 | oauth_version=1.0 150 | oauth_signature_method=HMAC-SHA1 151 | oauth_consumer_key=${MY_apikey} 152 | oauth_token=${MY_atoken} 153 | oauth_timestamp=${nowutime} 154 | oauth_nonce=${randmstr} 155 | OAUTHPARAM 156 | ) 157 | # --- 4.generate pre-string of the signature 158 | # (note: the API parameters and OAuth 1.0 parameters 159 | # are formed a line like a CGI parameter of GET method) 160 | sig_param=$(cat <<-OAUTHPARAM | 161 | ${oa_param} 162 | ${apip_enc} 163 | OAUTHPARAM 164 | grep -v '^ *$' | 165 | sort -k 1,1 -t '=' | 166 | tr '\n' '&' | 167 | grep ^ | 168 | sed 's/&$//' ) 169 | # --- 5.generate the signature string 170 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 171 | # and the above No.4 string respectively at first. and transfer to 172 | # HMAC-SHA1 with the key string which made of the access-keys) 173 | sig_strin=$(cat <<-KEY_AND_DATA | 174 | ${MY_apisec} 175 | ${MY_atksec} 176 | ${API_methd} 177 | ${API_endpt} 178 | ${sig_param} 179 | KEY_AND_DATA 180 | urlencode -r | 181 | tr '\n' ' ' | 182 | grep ^ | 183 | sed 's/ *$//' | 184 | # 1:API-key 2:APIsec 3:method # 185 | # 4:API-endpoint 5:API-parameter # 186 | while read key sec mth ept par; do # 187 | printf '%s&%s&%s' $mth $ept $par | # 188 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 189 | "$CMD_OSSL" enc -e -base64 # 190 | done ) 191 | 192 | # === Access to the endpoint ========================================= 193 | # --- 1.connect and get a response 194 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 195 | "${oa_param}" \ 196 | "${sig_strin}" \ 197 | "${API_param}" | 198 | urlencode -r | 199 | sed 's/%3[Dd]/=/' | 200 | sort -k 1,1 -t '=' | 201 | tr '\n' ',' | 202 | grep ^ | 203 | sed 's/^,*//' | 204 | sed 's/,*$//' | 205 | sed 's/^/Authorization: OAuth /' | 206 | while read -r oa_hdr; do # 207 | if [ -n "${CMD_WGET:-}" ]; then # 208 | [ -n "$timeout" ] && { # 209 | timeout="--connect-timeout=$timeout" # 210 | } # 211 | if type gunzip >/dev/null 2>&1; then # 212 | comp='--header=Accept-Encoding: gzip' # 213 | else # 214 | comp='' # 215 | fi # 216 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 217 | --header="$oa_hdr" \ 218 | --post-data="$apip_pos" \ 219 | $timeout "$comp" \ 220 | "$API_endpt" | # 221 | if [ -n "$comp" ]; then gunzip; else cat; fi # 222 | elif [ -n "${CMD_CURL:-}" ]; then # 223 | [ -n "$timeout" ] && { # 224 | timeout="--connect-timeout $timeout" # 225 | } # 226 | "$CMD_CURL" ${no_cert_curl:-} -s \ 227 | $timeout ${curl_comp_opt:-} \ 228 | -H "$oa_hdr" \ 229 | -d "$apip_pos" \ 230 | "$API_endpt" # 231 | fi # 232 | done | 233 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 234 | grep ^ | sed 's/\\/\\\\/g' # 235 | else # 236 | cat # 237 | fi ) 238 | # --- 2.exit immediately if it failed to access 239 | case $? in [!0]*) error_exit 1 'Failed to access API';; esac 240 | 241 | # === Parse the response ============================================= 242 | # --- 1.extract the required parameters from the response (written in JSON) 243 | echo "$apires" | 244 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 245 | parsrj.sh 2>/dev/null | 246 | awk 'BEGIN {fid=0; fca=0; } # 247 | $1~/^\$\.created_at$/{fca=1; sca=substr($0,index($0," ")+1);} # 248 | $1~/^\$\.id$/ {fid=1; sid=$2; } # 249 | END {if(fid*fca) {print sca,sid} }' | 250 | # 1:DayOfWeek 2:NameOfMonth 3:day 4:HH:MM:SS 5:delta(local-UTC) # 251 | # 6:year 7:ID # 252 | # --- 2.convert date string into "YYYY/MM/DD hh:mm:ss" # 253 | awk 'BEGIN { # 254 | m["Jan"]="01"; m["Feb"]="02"; m["Mar"]="03"; m["Apr"]="04"; # 255 | m["May"]="05"; m["Jun"]="06"; m["Jul"]="07"; m["Aug"]="08"; # 256 | m["Sep"]="09"; m["Oct"]="10"; m["Nov"]="11"; m["Dec"]="12";} # 257 | /^[A-Z]/ { # 258 | t=$4; # 259 | gsub(/:/,"",t); # 260 | d=substr($5,1,1) (substr($5,2,2)*3600+substr($5,4)*60); # 261 | d*=1; # 262 | printf("%04d%02d%02d%s %s %s\n",$6,m[$2],$3,t,d,$7); }' | 263 | # 1:YYYYMMDDHHMMSS 2:delta(local-UTC) 3:ID # 264 | TZ=UTC+0 calclock 1 | 265 | # 1:YYYYMMDDHHMMSS 2:UNIX-time 3:delta(local-UTC) 4:ID # 266 | awk '{print $2-$3,$4;}' | 267 | # 1::UNIX-time(adjusted) 2:ID # 268 | calclock -r 1 | 269 | # 1::UNIX-time(adjusted) 2:localtime 3:ID # 270 | self 2 3 | 271 | # 1:localtime 2:ID # 272 | awk 'BEGIN {fmt="at=%04d/%02d/%02d %02d:%02d:%02d\nid=%s\n"; } # 273 | {gsub(/[0-9][0-9]/,"& ",$1);sub(/ /,"",$1);split($1,t); # 274 | printf(fmt,t[1],t[2],t[3],t[4],t[5],t[6],$2); }' | 275 | # --- 3.regard as an error if no line was outputed # 276 | awk '{print;} END{exit 1-(NR>0);}' 277 | 278 | # === Print error message if some error occured ====================== 279 | case $? in [!0]*) 280 | err=$(echo "$apires" | 281 | parsrj.sh 2>/dev/null | 282 | awk 'BEGIN {errcode=-1; } # 283 | $1~/\.code$/ {errcode=$2; } # 284 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 285 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 286 | END {print errcode, errmsg; }') 287 | [ -z "${err#* }" ] || { error_exit 1 "API error(${err%% *}): ${err#* }"; } 288 | error_exit 1 "API returned an unknown message: $apires" 289 | ;; esac 290 | 291 | 292 | ###################################################################### 293 | # Finish 294 | ###################################################################### 295 | 296 | exit 0 297 | -------------------------------------------------------------------------------- /BIN/twunfollow.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # TWUNFOLLOW.SH : Finish Following A User 6 | # 7 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-05-06 8 | # 9 | # This is a public-domain software (CC0). It means that all of the 10 | # people can use this for any purposes with no restrictions at all. 11 | # By the way, We are fed up with the side effects which are brought 12 | # about by the major licenses. 13 | # 14 | ###################################################################### 15 | 16 | 17 | ###################################################################### 18 | # Initial Configuration 19 | ###################################################################### 20 | 21 | # === Initialize shell environment =================================== 22 | set -u 23 | umask 0022 24 | export LC_ALL=C 25 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 26 | case $PATH in :*) PATH=${PATH#?};; esac 27 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 28 | 29 | # === Define the functions for printing usage and error message ====== 30 | print_usage_and_exit () { 31 | cat <<-USAGE 1>&2 32 | Usage : ${0##*/} 33 | Version : 2020-05-06 22:42:19 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 1 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 1 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | scname='' 75 | rawoutputfile='' 76 | timeout='' 77 | 78 | # === Read options =================================================== 79 | while :; do 80 | case "${1:-}" in 81 | --rawout=*) # for debug 82 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 83 | rawoutputfile=$s 84 | shift 85 | ;; 86 | --timeout=*) # for debug 87 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 88 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 89 | error_exit 1 'Invalid --timeout option' 90 | } 91 | timeout=$s 92 | shift 93 | ;; 94 | --|-) break 95 | ;; 96 | --*|-*) error_exit 1 'Invalid option' 97 | ;; 98 | *) break 99 | ;; 100 | esac 101 | done 102 | 103 | # === Get the loginname (screen name) ================================ 104 | case $# in 105 | 1) scname=$(printf '%s' "${1#@}" | tr -d '\n');; 106 | *) print_usage_and_exit ;; 107 | esac 108 | printf '%s\n' "$scname" | grep -Eq '^[A-Za-z0-9_]+$' || { 109 | print_usage_and_exit 110 | } 111 | 112 | 113 | ###################################################################### 114 | # Main Routine 115 | ###################################################################### 116 | 117 | # === Set parameters of Twitter API endpoint ========================= 118 | # (1)endpoint 119 | readonly API_endpt='https://api.twitter.com/1.1/friendships/destroy.json' 120 | readonly API_methd='POST' 121 | # (2)parameters 122 | API_param=$(cat <<-PARAM | 123 | screen_name=$scname 124 | PARAM 125 | grep -v '^[A-Za-z0-9_]\{1,\}=$') 126 | readonly API_param 127 | 128 | # === Pack the parameters for the API ================================ 129 | # --- 1.URL-encode only the right side of "=" 130 | # (note: This string is also used to generate OAuth 1.0 signature) 131 | apip_enc=$(printf '%s\n' "${API_param}" | 132 | grep -v '^$' | 133 | urlencode -r | 134 | sed 's/%3[Dd]/=/' ) 135 | # --- 2.joint all lines with "&" (note: string for giving to the API) 136 | apip_pos=$(printf '%s' "${apip_enc}" | 137 | tr '\n' '&' ) 138 | 139 | # === Generate the signature string of OAuth 1.0 ===================== 140 | # --- 1.a random string 141 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 142 | # --- 2.the current UNIX time 143 | nowutime=$(date '+%Y%m%d%H%M%S' | 144 | calclock 1 | 145 | self 2 ) 146 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 147 | # (note: This string is also used for an HTTP header) 148 | oa_param=$(cat <<-OAUTHPARAM 149 | oauth_version=1.0 150 | oauth_signature_method=HMAC-SHA1 151 | oauth_consumer_key=${MY_apikey} 152 | oauth_token=${MY_atoken} 153 | oauth_timestamp=${nowutime} 154 | oauth_nonce=${randmstr} 155 | OAUTHPARAM 156 | ) 157 | # --- 4.generate pre-string of the signature 158 | # (note: the API parameters and OAuth 1.0 parameters 159 | # are formed a line like a CGI parameter of GET method) 160 | sig_param=$(cat <<-OAUTHPARAM | 161 | ${oa_param} 162 | ${apip_enc} 163 | OAUTHPARAM 164 | grep -v '^ *$' | 165 | sort -k 1,1 -t '=' | 166 | tr '\n' '&' | 167 | grep ^ | 168 | sed 's/&$//' ) 169 | # --- 5.generate the signature string 170 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 171 | # and the above No.4 string respectively at first. and transfer to 172 | # HMAC-SHA1 with the key string which made of the access-keys) 173 | sig_strin=$(cat <<-KEY_AND_DATA | 174 | ${MY_apisec} 175 | ${MY_atksec} 176 | ${API_methd} 177 | ${API_endpt} 178 | ${sig_param} 179 | KEY_AND_DATA 180 | urlencode -r | 181 | tr '\n' ' ' | 182 | grep ^ | 183 | sed 's/ *$//' | 184 | # 1:API-key 2:APIsec 3:method # 185 | # 4:API-endpoint 5:API-parameter # 186 | while read key sec mth ept par; do # 187 | printf '%s&%s&%s' $mth $ept $par | # 188 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 189 | "$CMD_OSSL" enc -e -base64 # 190 | done ) 191 | 192 | # === Access to the endpoint ========================================= 193 | # --- 1.connect and get a response 194 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 195 | "${oa_param}" \ 196 | "${sig_strin}" \ 197 | "${API_param}" | 198 | urlencode -r | 199 | sed 's/%3[Dd]/=/' | 200 | sort -k 1,1 -t '=' | 201 | tr '\n' ',' | 202 | grep ^ | 203 | sed 's/^,*//' | 204 | sed 's/,*$//' | 205 | sed 's/^/Authorization: OAuth /' | 206 | while read -r oa_hdr; do # 207 | if [ -n "${CMD_WGET:-}" ]; then # 208 | [ -n "$timeout" ] && { # 209 | timeout="--connect-timeout=$timeout" # 210 | } # 211 | if type gunzip >/dev/null 2>&1; then # 212 | comp='--header=Accept-Encoding: gzip' # 213 | else # 214 | comp='' # 215 | fi # 216 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 217 | --header="$oa_hdr" \ 218 | --post-data="$apip_pos" \ 219 | $timeout "$comp" \ 220 | "$API_endpt" | # 221 | if [ -n "$comp" ]; then gunzip; else cat; fi # 222 | elif [ -n "${CMD_CURL:-}" ]; then # 223 | [ -n "$timeout" ] && { # 224 | timeout="--connect-timeout $timeout" # 225 | } # 226 | "$CMD_CURL" ${no_cert_curl:-} -s \ 227 | $timeout ${curl_comp_opt:-} \ 228 | -H "$oa_hdr" \ 229 | -d "$apip_pos" \ 230 | "$API_endpt" # 231 | fi # 232 | done | 233 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 234 | grep ^ | sed 's/\\/\\\\/g' # 235 | else # 236 | cat # 237 | fi ) 238 | # --- 2.exit immediately if it failed to access 239 | case $? in [!0]*) error_exit 1 'Failed to access API';; esac 240 | 241 | # === Parse the response ============================================= 242 | # --- 1.extract the required parameters from the response (written in JSON) 243 | echo "$apires" | 244 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 245 | parsrj.sh 2>/dev/null | 246 | awk 'BEGIN {fid=0; fca=0; } # 247 | $1~/^\$\.created_at$/{fca=1; sca=substr($0,index($0," ")+1);} # 248 | $1~/^\$\.id$/ {fid=1; sid=$2; } # 249 | END {if(fid*fca) {print sca,sid} }' | 250 | # 1:DayOfWeek 2:NameOfMonth 3:day 4:HH:MM:SS 5:delta(local-UTC) # 251 | # 6:year 7:ID # 252 | # --- 2.convert date string into "YYYY/MM/DD hh:mm:ss" # 253 | awk 'BEGIN { # 254 | m["Jan"]="01"; m["Feb"]="02"; m["Mar"]="03"; m["Apr"]="04"; # 255 | m["May"]="05"; m["Jun"]="06"; m["Jul"]="07"; m["Aug"]="08"; # 256 | m["Sep"]="09"; m["Oct"]="10"; m["Nov"]="11"; m["Dec"]="12";} # 257 | /^[A-Z]/ { # 258 | t=$4; # 259 | gsub(/:/,"",t); # 260 | d=substr($5,1,1) (substr($5,2,2)*3600+substr($5,4)*60); # 261 | d*=1; # 262 | printf("%04d%02d%02d%s %s %s\n",$6,m[$2],$3,t,d,$7); }' | 263 | # 1:YYYYMMDDHHMMSS 2:delta(local-UTC) 3:ID # 264 | TZ=UTC+0 calclock 1 | 265 | # 1:YYYYMMDDHHMMSS 2:UNIX-time 3:delta(local-UTC) 4:ID # 266 | awk '{print $2-$3,$4;}' | 267 | # 1::UNIX-time(adjusted) 2:ID # 268 | calclock -r 1 | 269 | # 1::UNIX-time(adjusted) 2:localtime 3:ID # 270 | self 2 3 | 271 | # 1:localtime 2:ID # 272 | awk 'BEGIN {fmt="at=%04d/%02d/%02d %02d:%02d:%02d\nid=%s\n"; } # 273 | {gsub(/[0-9][0-9]/,"& ",$1);sub(/ /,"",$1);split($1,t); # 274 | printf(fmt,t[1],t[2],t[3],t[4],t[5],t[6],$2); }' | 275 | # --- 3.regard as an error if no line was outputed # 276 | awk '{print;} END{exit 1-(NR>0);}' 277 | 278 | # === Print error message if some error occured ====================== 279 | case $? in [!0]*) 280 | err=$(echo "$apires" | 281 | parsrj.sh 2>/dev/null | 282 | awk 'BEGIN {errcode=-1; } # 283 | $1~/\.code$/ {errcode=$2; } # 284 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 285 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 286 | END {print errcode, errmsg; }') 287 | [ -z "${err#* }" ] || { error_exit 1 "API error(${err%% *}): ${err#* }"; } 288 | error_exit 1 "API returned an unknown message: $apires" 289 | ;; esac 290 | 291 | 292 | ###################################################################### 293 | # Finish 294 | ###################################################################### 295 | 296 | exit 0 -------------------------------------------------------------------------------- /BIN/previous.ver/deldmtw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # DMDELTW.SH : Delete A Direct Message 6 | # 7 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2017-07-18 8 | # 9 | # This is a public-domain software (CC0). It means that all of the 10 | # people can use this for any purposes with no restrictions at all. 11 | # By the way, We are fed up with the side effects which are brought 12 | # about by the major licenses. 13 | # 14 | ###################################################################### 15 | 16 | 17 | ###################################################################### 18 | # Initial Configuration 19 | ###################################################################### 20 | 21 | # === Initialize shell environment =================================== 22 | set -u 23 | umask 0022 24 | export LC_ALL=C 25 | type command >/dev/null 2>&1 && type getconf >/dev/null 2>&1 && 26 | export PATH="$(command -p getconf PATH)${PATH+:}${PATH-}" 27 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 28 | 29 | # === Define the functions for printing usage and error message ====== 30 | print_usage_and_exit () { 31 | cat <<-USAGE 1>&2 32 | Usage : ${0##*/} 33 | Version : 2017-07-18 02:39:39 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 1 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 1 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | tweetid='' 75 | rawoutputfile='' 76 | timeout='' 77 | 78 | # === Read options =================================================== 79 | while :; do 80 | case "${1:-}" in 81 | --rawout=*) # for debug 82 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 83 | rawoutputfile=$s 84 | shift 85 | ;; 86 | --timeout=*) # for debug 87 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 88 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 89 | error_exit 1 'Invalid --timeout option' 90 | } 91 | timeout=$s 92 | shift 93 | ;; 94 | --|-) break 95 | ;; 96 | --*|-*) error_exit 1 'Invalid option' 97 | ;; 98 | *) break 99 | ;; 100 | esac 101 | done 102 | 103 | # === Get a tweet ID to delete ======================================= 104 | case $# in 105 | 1) tweetid=$(printf '%s' "$1" | tr -d '\n');; 106 | *) print_usage_and_exit ;; 107 | esac 108 | printf '%s\n' "$tweetid" | grep -Eq '^[0-9]+$' || { 109 | print_usage_and_exit 110 | } 111 | 112 | 113 | ###################################################################### 114 | # Main Routine 115 | ###################################################################### 116 | 117 | # === Set parameters of Twitter API endpoint ========================= 118 | # (1)endpoint 119 | readonly API_endpt="https://api.twitter.com/1.1/direct_messages/destroy.json" 120 | readonly API_methd='POST' 121 | # (2)parameters 122 | API_param=$(cat <<-PARAM | 123 | id=$tweetid 124 | PARAM 125 | grep -v '^[A-Za-z0-9_]\{1,\}=$') 126 | readonly API_param 127 | 128 | # === Pack the parameters for the API ================================ 129 | # --- 1.URL-encode only the right side of "=" 130 | # (note: This string is also used to generate OAuth 1.0 signature) 131 | apip_enc=$(printf '%s\n' "${API_param}" | 132 | grep -v '^$' | 133 | urlencode -r | 134 | sed 's/%3[Dd]/=/' ) 135 | # --- 2.joint all lines with "&" (note: string for giving to the API) 136 | apip_pos=$(printf '%s' "${apip_enc}" | 137 | tr '\n' '&' ) 138 | 139 | # === Generate the signature string of OAuth 1.0 ===================== 140 | # --- 1.a random string 141 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 142 | # --- 2.the current UNIX time 143 | nowutime=$(date '+%Y%m%d%H%M%S' | 144 | calclock 1 | 145 | self 2 ) 146 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 147 | # (note: This string is also used for an HTTP header) 148 | oa_param=$(cat <<-OAUTHPARAM 149 | oauth_version=1.0 150 | oauth_signature_method=HMAC-SHA1 151 | oauth_consumer_key=${MY_apikey} 152 | oauth_token=${MY_atoken} 153 | oauth_timestamp=${nowutime} 154 | oauth_nonce=${randmstr} 155 | OAUTHPARAM 156 | ) 157 | # --- 4.generate pre-string of the signature 158 | # (note: the API parameters and OAuth 1.0 parameters 159 | # are formed a line like a CGI parameter of GET method) 160 | sig_param=$(cat <<-OAUTHPARAM | 161 | ${oa_param} 162 | ${apip_enc} 163 | OAUTHPARAM 164 | grep -v '^ *$' | 165 | sort -k 1,1 -t '=' | 166 | tr '\n' '&' | 167 | grep ^ | 168 | sed 's/&$//' ) 169 | # --- 5.generate the signature string 170 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 171 | # and the above No.4 string respectively at first. and transfer to 172 | # HMAC-SHA1 with the key string which made of the access-keys) 173 | sig_strin=$(cat <<-KEY_AND_DATA | 174 | ${MY_apisec} 175 | ${MY_atksec} 176 | ${API_methd} 177 | ${API_endpt} 178 | ${sig_param} 179 | KEY_AND_DATA 180 | urlencode -r | 181 | tr '\n' ' ' | 182 | grep ^ | 183 | sed 's/ *$//' | 184 | # 1:API-key 2:APIsec 3:method # 185 | # 4:API-endpoint 5:API-parameter # 186 | while read key sec mth ept par; do # 187 | printf '%s&%s&%s' $mth $ept $par | # 188 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 189 | "$CMD_OSSL" enc -e -base64 # 190 | done ) 191 | 192 | # === Access to the endpoint ========================================= 193 | # --- 1.connect and get a response 194 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 195 | "${oa_param}" \ 196 | "${sig_strin}" \ 197 | "${API_param}" | 198 | urlencode -r | 199 | sed 's/%3[Dd]/=/' | 200 | sort -k 1,1 -t '=' | 201 | tr '\n' ',' | 202 | grep ^ | 203 | sed 's/^,*//' | 204 | sed 's/,*$//' | 205 | sed 's/^/Authorization: OAuth /' | 206 | while read -r oa_hdr; do # 207 | if [ -n "${CMD_WGET:-}" ]; then # 208 | [ -n "$timeout" ] && { # 209 | timeout="--connect-timeout=$timeout" # 210 | } # 211 | if type gunzip >/dev/null 2>&1; then # 212 | comp='--header=Accept-Encoding: gzip' # 213 | else # 214 | comp='' # 215 | fi # 216 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 217 | --header="$oa_hdr" \ 218 | --post-data="$apip_pos" \ 219 | $timeout "$comp" \ 220 | "$API_endpt" | # 221 | if [ -n "$comp" ]; then gunzip; else cat; fi # 222 | elif [ -n "${CMD_CURL:-}" ]; then # 223 | [ -n "$timeout" ] && { # 224 | timeout="--connect-timeout $timeout" # 225 | } # 226 | "$CMD_CURL" ${no_cert_curl:-} -s \ 227 | $timeout ${curl_comp_opt:-} \ 228 | -H "$oa_hdr" \ 229 | -d "$apip_pos" \ 230 | "$API_endpt" # 231 | fi # 232 | done | 233 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 234 | grep ^ | sed 's/\\/\\\\/g' # 235 | else # 236 | cat # 237 | fi ) 238 | # --- 2.exit immediately if it failed to access 239 | case $? in [!0]*) error_exit 1 'Failed to access API';; esac 240 | 241 | # === Parse the response ============================================= 242 | # --- 1.extract the required parameters from the response (written in JSON) 243 | echo "$apires" | 244 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 245 | parsrj.sh 2>/dev/null | 246 | awk 'BEGIN {fid=0; fca=0; } # 247 | $1~/^\$\.created_at$/{fca=1; sca=substr($0,index($0," ")+1);} # 248 | $1~/^\$\.id$/ {fid=1; sid=$2; } # 249 | END {if(fid*fca) {print sca,sid} }' | 250 | # 1:DayOfWeek 2:NameOfMonth 3:day 4:HH:MM:SS 5:delta(local-UTC) # 251 | # 6:year 7:ID # 252 | # --- 2.convert date string into "YYYY/MM/DD hh:mm:ss" # 253 | awk 'BEGIN { # 254 | m["Jan"]="01"; m["Feb"]="02"; m["Mar"]="03"; m["Apr"]="04"; # 255 | m["May"]="05"; m["Jun"]="06"; m["Jul"]="07"; m["Aug"]="08"; # 256 | m["Sep"]="09"; m["Oct"]="10"; m["Nov"]="11"; m["Dec"]="12";} # 257 | /^[A-Z]/ { # 258 | t=$4; # 259 | gsub(/:/,"",t); # 260 | d=substr($5,1,1) (substr($5,2,2)*3600+substr($5,4)*60); # 261 | d*=1; # 262 | printf("%04d%02d%02d%s %s %s\n",$6,m[$2],$3,t,d,$7); }' | 263 | # 1:YYYYMMDDHHMMSS 2:delta(local-UTC) 3:ID # 264 | TZ=UTC+0 calclock 1 | 265 | # 1:YYYYMMDDHHMMSS 2:UNIX-time 3:delta(local-UTC) 4:ID # 266 | awk '{print $2-$3,$4;}' | 267 | # 1::UNIX-time(adjusted) 2:ID # 268 | calclock -r 1 | 269 | # 1::UNIX-time(adjusted) 2:localtime 3:ID # 270 | self 2 3 | 271 | # 1:localtime 2:ID # 272 | awk 'BEGIN {fmt="at=%04d/%02d/%02d %02d:%02d:%02d\nid=%s\n"; } # 273 | {gsub(/[0-9][0-9]/,"& ",$1);sub(/ /,"",$1);split($1,t); # 274 | printf(fmt,t[1],t[2],t[3],t[4],t[5],t[6],$2); }' | 275 | # --- 3.regard as an error if no line was outputed # 276 | awk '{print;} END{exit 1-(NR>0);}' 277 | 278 | # === Print error message if some error occured ====================== 279 | case $? in [!0]*) 280 | err=$(echo "$apires" | 281 | parsrj.sh 2>/dev/null | 282 | awk 'BEGIN {errcode=-1; } # 283 | $1~/\.code$/ {errcode=$2; } # 284 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 285 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 286 | END {print errcode, errmsg; }') 287 | [ -z "${err#* }" ] || { error_exit 1 "API error(${err%% *}): ${err#* }"; } 288 | error_exit 1 "API returned an unknown message: $apires" 289 | ;; esac 290 | 291 | 292 | ###################################################################### 293 | # Finish 294 | ###################################################################### 295 | 296 | exit 0 297 | -------------------------------------------------------------------------------- /BIN/twfing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # TWFING.SH : List Following Users Of A Person 6 | # 7 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-10-01 8 | # 9 | # This is a public-domain software (CC0). It means that all of the 10 | # people can use this for any purposes with no restrictions at all. 11 | # By the way, We are fed up with the side effects which are brought 12 | # about by the major licenses. 13 | # 14 | ###################################################################### 15 | 16 | 17 | ###################################################################### 18 | # Initial Configuration 19 | ###################################################################### 20 | 21 | # === Initialize shell environment =================================== 22 | set -u 23 | umask 0022 24 | export LC_ALL=C 25 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 26 | case $PATH in :*) PATH=${PATH#?};; esac 27 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 28 | 29 | # === Define the functions for printing usage and error message ====== 30 | print_usage_and_exit () { 31 | cat <<-USAGE 1>&2 32 | Usage : ${0##*/} [-n |--count=] [loginname] 33 | Version : 2020-10-01 22:33:22 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 1 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 1 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | scname='' 75 | count='' 76 | rawoutputfile='' 77 | timeout='' 78 | 79 | # === Read options =================================================== 80 | while :; do 81 | case "${1:-}" in 82 | --count=*) count=$(printf '%s' "${1#--count=}" | tr -d '\n') 83 | shift 84 | ;; 85 | -n) case $# in 1) error_exit 1 'Invalid -n option';; esac 86 | count=$(printf '%s' "$2" | tr -d '\n') 87 | shift 2 88 | ;; 89 | --rawout=*) # for debug 90 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 91 | rawoutputfile=$s 92 | shift 93 | ;; 94 | --timeout=*) # for debug 95 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 96 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 97 | error_exit 1 'Invalid --timeout option' 98 | } 99 | timeout=$s 100 | shift 101 | ;; 102 | --|-) : 103 | ;; 104 | --*|-*) error_exit 1 'Invalid option' 105 | ;; 106 | *) break 107 | ;; 108 | esac 109 | done 110 | printf '%s\n' "$count" | grep -q '^[0-9]*$' || { 111 | error_exit 1 'Invalid -n option' 112 | } 113 | 114 | # === Get the loginname (screen name) ================================ 115 | case $# in 116 | 0) scname=$MY_scname ;; 117 | 1) scname=$(printf '%s' "${1#@}" | tr -d '\n');; 118 | *) print_usage_and_exit ;; 119 | esac 120 | printf '%s\n' "$scname" | grep -Eq '^[A-Za-z0-9_]+$' || { 121 | print_usage_and_exit 122 | } 123 | 124 | 125 | ###################################################################### 126 | # Main Routine 127 | ###################################################################### 128 | 129 | # === Set parameters of Twitter API endpoint ========================= 130 | # (1)endpoint 131 | API_endpt='https://api.twitter.com/1.1/friends/list.json' 132 | API_methd='GET' 133 | # (2)parameters 134 | API_param=$(cat <<-PARAM | 135 | count=${count} 136 | screen_name=@${scname} 137 | PARAM 138 | grep -v '^[A-Za-z0-9_]\{1,\}=$') 139 | readonly API_param 140 | 141 | # === Pack the parameters for the API ================================ 142 | # --- 1.URL-encode only the right side of "=" 143 | # (note: This string is also used to generate OAuth 1.0 signature) 144 | apip_enc=$(printf '%s\n' "${API_param}" | 145 | grep -v '^$' | 146 | urlencode -r | 147 | sed 's/%3[Dd]/=/' ) 148 | # --- 2.joint all lines with "&" (note: string for giving to the API) 149 | apip_get=$(printf '%s' "${apip_enc}" | 150 | tr '\n' '&' | 151 | grep ^ | 152 | sed 's/^./?&/' ) 153 | 154 | # === Generate the signature string of OAuth 1.0 ===================== 155 | # --- 1.a random string 156 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 157 | # --- 2.the current UNIX time 158 | nowutime=$(date '+%Y%m%d%H%M%S' | 159 | calclock 1 | 160 | self 2 ) 161 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 162 | # (note: This string is also used for an HTTP header) 163 | oa_param=$(cat <<-OAUTHPARAM 164 | oauth_version=1.0 165 | oauth_signature_method=HMAC-SHA1 166 | oauth_consumer_key=${MY_apikey} 167 | oauth_token=${MY_atoken} 168 | oauth_timestamp=${nowutime} 169 | oauth_nonce=${randmstr} 170 | OAUTHPARAM 171 | ) 172 | # --- 4.generate pre-string of the signature 173 | # (note: the API parameters and OAuth 1.0 parameters 174 | # are formed a line like a CGI parameter of GET method) 175 | sig_param=$(cat <<-OAUTHPARAM | 176 | ${oa_param} 177 | ${apip_enc} 178 | OAUTHPARAM 179 | grep -v '^ *$' | 180 | sort -k 1,1 -t '=' | 181 | tr '\n' '&' | 182 | grep ^ | 183 | sed 's/&$//' ) 184 | # --- 5.generate the signature string 185 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 186 | # and the above No.4 string respectively at first. and transfer to 187 | # HMAC-SHA1 with the key string which made of the access-keys) 188 | sig_strin=$(cat <<-KEY_AND_DATA | 189 | ${MY_apisec} 190 | ${MY_atksec} 191 | ${API_methd} 192 | ${API_endpt} 193 | ${sig_param} 194 | KEY_AND_DATA 195 | urlencode -r | 196 | tr '\n' ' ' | 197 | grep ^ | 198 | sed 's/ *$//' | 199 | # 1:API-key 2:APIsec 3:method # 200 | # 4:API-endpoint 5:API-parameter # 201 | while read key sec mth ept par; do # 202 | printf '%s&%s&%s' $mth $ept $par | # 203 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 204 | "$CMD_OSSL" enc -e -base64 # 205 | done ) 206 | 207 | # === Access to the endpoint ========================================= 208 | # --- 1.connect and get a response 209 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 210 | "${oa_param}" \ 211 | "${sig_strin}" \ 212 | "${API_param}" | 213 | urlencode -r | 214 | sed 's/%3[Dd]/=/' | 215 | sort -k 1,1 -t '=' | 216 | tr '\n' ',' | 217 | grep ^ | 218 | sed 's/^,*//' | 219 | sed 's/,*$//' | 220 | sed 's/^/Authorization: OAuth /' | 221 | while read -r oa_hdr; do # 222 | if [ -n "${CMD_WGET:-}" ]; then # 223 | [ -n "$timeout" ] && { # 224 | timeout="--connect-timeout=$timeout" # 225 | } # 226 | if type gunzip >/dev/null 2>&1; then # 227 | comp='--header=Accept-Encoding: gzip' # 228 | else # 229 | comp='' # 230 | fi # 231 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 232 | --header="$oa_hdr" \ 233 | $timeout "$comp" \ 234 | "$API_endpt$apip_get" | # 235 | if [ -n "$comp" ]; then gunzip; else cat; fi # 236 | elif [ -n "${CMD_CURL:-}" ]; then # 237 | [ -n "$timeout" ] && { # 238 | timeout="--connect-timeout $timeout" # 239 | } # 240 | "$CMD_CURL" ${no_cert_curl:-} -s \ 241 | $timeout ${curl_comp_opt:-} \ 242 | -H "$oa_hdr" \ 243 | "$API_endpt$apip_get" # 244 | fi # 245 | done | 246 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 247 | grep ^ | sed 's/\\/\\\\/g' # 248 | else # 249 | cat # 250 | fi ) 251 | # --- 2.exit immediately if it failed to access 252 | case $? in [!0]*) error_exit 1 'Failed to access API';; esac 253 | 254 | # === Parse the response ============================================= 255 | # --- 1.extract the required parameters from the response (written in JSON) # 256 | echo "$apires" | 257 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 258 | parsrj.sh 2>/dev/null | 259 | unescj.sh -n 2>/dev/null | 260 | tr -d '\000\034' | 261 | sed 's/&/'$(printf '\034')'/g' | 262 | sed 's/<//g' | tr '\034' '&' | 263 | sed 's/^\$\.users\[\([0-9]\{1,\}\)\]\./\1 /' | 264 | grep -v '^\$' | 265 | awk ' # 266 | BEGIN {init_param(2); } # 267 | $2=="id" {id= substr($0,length($1 $2)+3);print_tw(); next;} # 268 | $2=="name" {nm= substr($0,length($1 $2)+3);print_tw(); next;} # 269 | $2=="screen_name" {sn= substr($0,length($1 $2)+3);print_tw(); next;} # 270 | $2=="verified" {vf=(substr($0,length($1 $2)+3)=="true")?"[v]":""; # 271 | next;} # 272 | function init_param(lv) {if (lv<2) {return;} # 273 | id=""; nm=""; sn=""; fl=""; vf=""; } # 274 | function print_tw() { # 275 | if (id=="") {return;} # 276 | if (nm=="") {return;} # 277 | if (sn=="") {return;} # 278 | printf("-=> %-19s %s (@%s)%s\n",id,nm,sn,vf); # 279 | init_param(2); }' | 280 | # --- 2.regard as an error if no line was outputed # 281 | awk '{print;} END{exit 1-(NR>0);}' 282 | 283 | # === Print error message if some error occured ====================== 284 | case $? in [!0]*) 285 | err=$(echo "$apires" | 286 | parsrj.sh 2>/dev/null | 287 | awk 'BEGIN {errcode=-1; } # 288 | $1~/\.code$/ {errcode=$2; } # 289 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 290 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 291 | END {print errcode, errmsg; }') 292 | [ -z "${err#* }" ] || { error_exit 1 "API error(${err%% *}): ${err#* }"; } 293 | ;; esac 294 | 295 | 296 | ###################################################################### 297 | # Finish 298 | ###################################################################### 299 | 300 | exit 0 301 | -------------------------------------------------------------------------------- /BIN/twfer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # TWFER.SH : List Followers Of A Person 6 | # 7 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-10-01 8 | # 9 | # This is a public-domain software (CC0). It means that all of the 10 | # people can use this for any purposes with no restrictions at all. 11 | # By the way, We are fed up with the side effects which are brought 12 | # about by the major licenses. 13 | # 14 | ###################################################################### 15 | 16 | 17 | ###################################################################### 18 | # Initial Configuration 19 | ###################################################################### 20 | 21 | # === Initialize shell environment =================================== 22 | set -u 23 | umask 0022 24 | export LC_ALL=C 25 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 26 | case $PATH in :*) PATH=${PATH#?};; esac 27 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 28 | 29 | # === Define the functions for printing usage and error message ====== 30 | print_usage_and_exit () { 31 | cat <<-USAGE 1>&2 32 | Usage : ${0##*/} [-n |--count=] [loginname] 33 | Version : 2020-10-01 22:33:22 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 1 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 1 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | scname='' 75 | count='' 76 | rawoutputfile='' 77 | timeout='' 78 | 79 | # === Read options =================================================== 80 | while :; do 81 | case "${1:-}" in 82 | --count=*) count=$(printf '%s' "${1#--count=}" | tr -d '\n') 83 | shift 84 | ;; 85 | -n) case $# in 1) error_exit 1 'Invalid -n option';; esac 86 | count=$(printf '%s' "$2" | tr -d '\n') 87 | shift 2 88 | ;; 89 | --rawout=*) # for debug 90 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 91 | rawoutputfile=$s 92 | shift 93 | ;; 94 | --timeout=*) # for debug 95 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 96 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 97 | error_exit 1 'Invalid --timeout option' 98 | } 99 | timeout=$s 100 | shift 101 | ;; 102 | --|-) break 103 | ;; 104 | --*|-*) error_exit 1 'Invalid option' 105 | ;; 106 | *) break 107 | ;; 108 | esac 109 | done 110 | printf '%s\n' "$count" | grep -q '^[0-9]*$' || { 111 | error_exit 1 'Invalid -n option' 112 | } 113 | 114 | # === Get the loginname (screen name) ================================ 115 | case $# in 116 | 0) scname=$MY_scname ;; 117 | 1) scname=$(printf '%s' "${1#@}" | tr -d '\n');; 118 | *) print_usage_and_exit ;; 119 | esac 120 | printf '%s\n' "$scname" | grep -Eq '^[A-Za-z0-9_]+$' || { 121 | print_usage_and_exit 122 | } 123 | 124 | 125 | ###################################################################### 126 | # Main Routine 127 | ###################################################################### 128 | 129 | # === Set parameters of Twitter API endpoint ========================= 130 | # (1)endpoint 131 | API_endpt='https://api.twitter.com/1.1/followers/list.json' 132 | API_methd='GET' 133 | # (2)parameters 134 | API_param=$(cat <<-PARAM | 135 | count=${count} 136 | screen_name=@${scname} 137 | PARAM 138 | grep -v '^[A-Za-z0-9_]\{1,\}=$') 139 | readonly API_param 140 | 141 | # === Pack the parameters for the API ================================ 142 | # --- 1.URL-encode only the right side of "=" 143 | # (note: This string is also used to generate OAuth 1.0 signature) 144 | apip_enc=$(printf '%s\n' "${API_param}" | 145 | grep -v '^$' | 146 | urlencode -r | 147 | sed 's/%3[Dd]/=/' ) 148 | # --- 2.joint all lines with "&" (note: string for giving to the API) 149 | apip_get=$(printf '%s' "${apip_enc}" | 150 | tr '\n' '&' | 151 | grep ^ | 152 | sed 's/^./?&/' ) 153 | 154 | # === Generate the signature string of OAuth 1.0 ===================== 155 | # --- 1.a random string 156 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 157 | # --- 2.the current UNIX time 158 | nowutime=$(date '+%Y%m%d%H%M%S' | 159 | calclock 1 | 160 | self 2 ) 161 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 162 | # (note: This string is also used for an HTTP header) 163 | oa_param=$(cat <<-OAUTHPARAM 164 | oauth_version=1.0 165 | oauth_signature_method=HMAC-SHA1 166 | oauth_consumer_key=${MY_apikey} 167 | oauth_token=${MY_atoken} 168 | oauth_timestamp=${nowutime} 169 | oauth_nonce=${randmstr} 170 | OAUTHPARAM 171 | ) 172 | # --- 4.generate pre-string of the signature 173 | # (note: the API parameters and OAuth 1.0 parameters 174 | # are formed a line like a CGI parameter of GET method) 175 | sig_param=$(cat <<-OAUTHPARAM | 176 | ${oa_param} 177 | ${apip_enc} 178 | OAUTHPARAM 179 | grep -v '^ *$' | 180 | sort -k 1,1 -t '=' | 181 | tr '\n' '&' | 182 | grep ^ | 183 | sed 's/&$//' ) 184 | # --- 5.generate the signature string 185 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 186 | # and the above No.4 string respectively at first. and transfer to 187 | # HMAC-SHA1 with the key string which made of the access-keys) 188 | sig_strin=$(cat <<-KEY_AND_DATA | 189 | ${MY_apisec} 190 | ${MY_atksec} 191 | ${API_methd} 192 | ${API_endpt} 193 | ${sig_param} 194 | KEY_AND_DATA 195 | urlencode -r | 196 | tr '\n' ' ' | 197 | grep ^ | 198 | sed 's/ *$//' | 199 | # 1:API-key 2:APIsec 3:method # 200 | # 4:API-endpoint 5:API-parameter # 201 | while read key sec mth ept par; do # 202 | printf '%s&%s&%s' $mth $ept $par | # 203 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 204 | "$CMD_OSSL" enc -e -base64 # 205 | done ) 206 | 207 | # === Access to the endpoint ========================================= 208 | # --- 1.connect and get a response 209 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 210 | "${oa_param}" \ 211 | "${sig_strin}" \ 212 | "${API_param}" | 213 | urlencode -r | 214 | sed 's/%3[Dd]/=/' | 215 | sort -k 1,1 -t '=' | 216 | tr '\n' ',' | 217 | grep ^ | 218 | sed 's/^,*//' | 219 | sed 's/,*$//' | 220 | sed 's/^/Authorization: OAuth /' | 221 | while read -r oa_hdr; do # 222 | if [ -n "${CMD_WGET:-}" ]; then # 223 | [ -n "$timeout" ] && { # 224 | timeout="--connect-timeout=$timeout" # 225 | } # 226 | if type gunzip >/dev/null 2>&1; then # 227 | comp='--header=Accept-Encoding: gzip' # 228 | else # 229 | comp='' # 230 | fi # 231 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 232 | --header="$oa_hdr" \ 233 | $timeout "$comp" \ 234 | "$API_endpt$apip_get" | # 235 | if [ -n "$comp" ]; then gunzip; else cat; fi # 236 | elif [ -n "${CMD_CURL:-}" ]; then # 237 | [ -n "$timeout" ] && { # 238 | timeout="--connect-timeout $timeout" # 239 | } # 240 | "$CMD_CURL" ${no_cert_curl:-} -s \ 241 | $timeout ${curl_comp_opt:-} \ 242 | -H "$oa_hdr" \ 243 | "$API_endpt$apip_get" # 244 | fi # 245 | done | 246 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 247 | grep ^ | sed 's/\\/\\\\/g' # 248 | else # 249 | cat # 250 | fi ) 251 | # --- 2.exit immediately if it failed to access 252 | case $? in [!0]*) error_exit 1 'Failed to access API';; esac 253 | 254 | # === Parse the response ============================================= 255 | # --- 1.extract the required parameters from the response (written in JSON) # 256 | echo "$apires" | 257 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 258 | parsrj.sh 2>/dev/null | 259 | unescj.sh -n 2>/dev/null | 260 | tr -d '\000\034' | 261 | sed 's/&/'$(printf '\034')'/g' | 262 | sed 's/<//g' | tr '\034' '&' | 263 | sed 's/^\$\.users\[\([0-9]\{1,\}\)\]\./\1 /' | 264 | grep -v '^\$' | 265 | awk ' # 266 | BEGIN {init_param(2); } # 267 | $2=="id" {id= substr($0,length($1 $2)+3);print_tw(); next;} # 268 | $2=="name" {nm= substr($0,length($1 $2)+3);print_tw(); next;} # 269 | $2=="screen_name" {sn= substr($0,length($1 $2)+3);print_tw(); next;} # 270 | $2=="following" {fl= substr($0,length($1 $2)+3);print_tw(); next;} # 271 | $2=="verified" {vf=(substr($0,length($1 $2)+3)=="true")?"[v]":""; # 272 | next;} # 273 | function init_param(lv) {if (lv<2) {return;} # 274 | id=""; nm=""; sn=""; fl=""; vf=""; } # 275 | function print_tw( stat){ # 276 | if (id=="") {return;} # 277 | if (nm=="") {return;} # 278 | if (sn=="") {return;} # 279 | if (fl=="") {return;} # 280 | stat = (fl=="true") ? "<=>" : "<=="; # 281 | printf("%s %-19s %s (@%s)%s\n",stat,id,nm,sn,vf); # 282 | init_param(2); }' | 283 | # --- 2.regard as an error if no line was outputed # 284 | awk '{print;} END{exit 1-(NR>0);}' 285 | 286 | # === Print error message if some error occured ====================== 287 | case $? in [!0]*) 288 | err=$(echo "$apires" | 289 | parsrj.sh 2>/dev/null | 290 | awk 'BEGIN {errcode=-1; } # 291 | $1~/\.code$/ {errcode=$2; } # 292 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 293 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 294 | END {print errcode, errmsg; }') 295 | [ -z "${err#* }" ] || { error_exit 1 "API error(${err%% *}): ${err#* }"; } 296 | ;; esac 297 | 298 | 299 | ###################################################################### 300 | # Finish 301 | ###################################################################### 302 | 303 | exit 0 304 | -------------------------------------------------------------------------------- /BIN/deltweet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ###################################################################### 4 | # 5 | # DELTWEET.SH : Delete A Tweet 6 | # 7 | # Written by Shell-Shoccar Japan (@shellshoccarjpn) on 2020-09-27 8 | # 9 | # This is a public-domain software (CC0). It means that all of the 10 | # people can use this for any purposes with no restrictions at all. 11 | # By the way, We are fed up with the side effects which are brought 12 | # about by the major licenses. 13 | # 14 | ###################################################################### 15 | 16 | 17 | ###################################################################### 18 | # Initial Configuration 19 | ###################################################################### 20 | 21 | # === Initialize shell environment =================================== 22 | set -u 23 | umask 0022 24 | export LC_ALL=C 25 | export PATH="$(command -p getconf PATH 2>/dev/null)${PATH+:}${PATH-}" 26 | case $PATH in :*) PATH=${PATH#?};; esac 27 | export UNIX_STD=2003 # to make HP-UX conform to POSIX 28 | 29 | # === Define the functions for printing usage and error message ====== 30 | print_usage_and_exit () { 31 | cat <<-USAGE 1>&2 32 | Usage : ${0##*/} [tweet_id...] 33 | Version : 2020-09-27 23:21:48 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 5 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 5 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | tweetid='' 75 | rawoutputfile='' 76 | timeout='' 77 | 78 | # === Read options =================================================== 79 | while :; do 80 | case "${1:-}" in 81 | --rawout=*) # for debug 82 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 83 | rawoutputfile=$s 84 | shift 85 | ;; 86 | --timeout=*) # for debug 87 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 88 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 89 | error_exit 5 'Invalid --timeout option' 90 | } 91 | timeout=$s 92 | shift 93 | ;; 94 | --|-) break 95 | ;; 96 | --*|-*) error_exit 5 'Invalid option' 97 | ;; 98 | *) break 99 | ;; 100 | esac 101 | done 102 | 103 | 104 | ###################################################################### 105 | # Main Routine 106 | ###################################################################### 107 | 108 | # === Set parameters of Twitter API endpoint (common) ================ 109 | # (1)method for the endpoint 110 | readonly API_methd='POST' 111 | # (2)parameters 112 | readonly API_param='' 113 | 114 | # === BEGIN: Tweet-ID Loop =========================================== 115 | num_of_tweets=0; num_of_success=0 116 | while read tweetid; do num_of_tweets=$((num_of_tweets+1)) 117 | 118 | # === Validate the Tweet-ID ========================================== 119 | printf '%s\n' "$tweetid" | grep -Eq '^[0-9]+$' || { 120 | echo "${0##*/}: $tweetid: Invalid tweet-ID" 1>&2; continue 121 | } 122 | 123 | # === Set parameters of Twitter API endpoint (indivisual) ============ 124 | # (1)endpoint 125 | API_endpt="https://api.twitter.com/1.1/statuses/destroy/$tweetid.json" 126 | 127 | # === Pack the parameters for the API ================================ 128 | # --- 1.URL-encode only the right side of "=" 129 | # (note: This string is also used to generate OAuth 1.0 signature) 130 | apip_enc=$(printf '%s\n' "${API_param}" | 131 | grep -v '^$' | 132 | urlencode -r | 133 | sed 's/%3[Dd]/=/' ) 134 | # --- 2.joint all lines with "&" (note: string for giving to the API) 135 | apip_pos=$(printf '%s' "${apip_enc}" | 136 | tr '\n' '&' ) 137 | 138 | # === Generate the signature string of OAuth 1.0 ===================== 139 | # --- 1.a random string 140 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 141 | # --- 2.the current UNIX time 142 | nowutime=$(date '+%Y%m%d%H%M%S' | 143 | calclock 1 | 144 | self 2 ) 145 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 146 | # (note: This string is also used for an HTTP header) 147 | oa_param=$(cat <<-OAUTHPARAM 148 | oauth_version=1.0 149 | oauth_signature_method=HMAC-SHA1 150 | oauth_consumer_key=${MY_apikey} 151 | oauth_token=${MY_atoken} 152 | oauth_timestamp=${nowutime} 153 | oauth_nonce=${randmstr} 154 | OAUTHPARAM 155 | ) 156 | # --- 4.generate pre-string of the signature 157 | # (note: the API parameters and OAuth 1.0 parameters 158 | # are formed a line like a CGI parameter of GET method) 159 | sig_param=$(cat <<-OAUTHPARAM | 160 | ${oa_param} 161 | ${apip_enc} 162 | OAUTHPARAM 163 | grep -v '^ *$' | 164 | sort -k 1,1 -t '=' | 165 | tr '\n' '&' | 166 | grep ^ | 167 | sed 's/&$//' ) 168 | # --- 5.generate the signature string 169 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 170 | # and the above No.4 string respectively at first. and transfer to 171 | # HMAC-SHA1 with the key string which made of the access-keys) 172 | sig_strin=$(cat <<-KEY_AND_DATA | 173 | ${MY_apisec} 174 | ${MY_atksec} 175 | ${API_methd} 176 | ${API_endpt} 177 | ${sig_param} 178 | KEY_AND_DATA 179 | urlencode -r | 180 | tr '\n' ' ' | 181 | grep ^ | 182 | sed 's/ *$//' | 183 | # 1:API-key 2:APIsec 3:method # 184 | # 4:API-endpoint 5:API-parameter # 185 | while read key sec mth ept par; do # 186 | printf '%s&%s&%s' $mth $ept $par | # 187 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 188 | "$CMD_OSSL" enc -e -base64 # 189 | done ) 190 | 191 | # === Access to the endpoint ========================================= 192 | # --- 1.connect and get a response 193 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 194 | "${oa_param}" \ 195 | "${sig_strin}" \ 196 | "${API_param}" | 197 | urlencode -r | 198 | sed 's/%3[Dd]/=/' | 199 | sort -k 1,1 -t '=' | 200 | tr '\n' ',' | 201 | grep ^ | 202 | sed 's/^,*//' | 203 | sed 's/,*$//' | 204 | sed 's/^/Authorization: OAuth /' | 205 | while read -r oa_hdr; do # 206 | if [ -n "${CMD_WGET:-}" ]; then # 207 | [ -n "$timeout" ] && { # 208 | timeout="--connect-timeout=$timeout" # 209 | } # 210 | if type gunzip >/dev/null 2>&1; then # 211 | comp='--header=Accept-Encoding: gzip' # 212 | else # 213 | comp='' # 214 | fi # 215 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 216 | --header="$oa_hdr" \ 217 | --post-data="$apip_pos" \ 218 | $timeout "$comp" \ 219 | "$API_endpt" | # 220 | if [ -n "$comp" ]; then gunzip; else cat; fi # 221 | elif [ -n "${CMD_CURL:-}" ]; then # 222 | [ -n "$timeout" ] && { # 223 | timeout="--connect-timeout $timeout" # 224 | } # 225 | "$CMD_CURL" ${no_cert_curl:-} -s \ 226 | $timeout ${curl_comp_opt:-} \ 227 | -H "$oa_hdr" \ 228 | -d "$apip_pos" \ 229 | "$API_endpt" # 230 | fi # 231 | done | 232 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 233 | grep ^ | sed 's/\\/\\\\/g' # 234 | else # 235 | cat # 236 | fi ) 237 | # --- 2.exit immediately if it failed to access 238 | case $? in [!0]*) error_exit 5 'Failed to access API';; esac 239 | 240 | # === Parse the response ============================================= 241 | # --- 1.extract the required parameters from the response (written in JSON) 242 | echo "$apires" | 243 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 244 | parsrj.sh 2>/dev/null | 245 | awk 'BEGIN {fid=0; fca=0; } # 246 | $1~/^\$\.created_at$/{fca=1; sca=substr($0,index($0," ")+1);} # 247 | $1~/^\$\.id$/ {fid=1; sid=$2; } # 248 | END {if(fid*fca) {print sca,sid} }' | 249 | # 1:DayOfWeek 2:NameOfMonth 3:day 4:HH:MM:SS 5:delta(local-UTC) # 250 | # 6:year 7:ID # 251 | # --- 2.convert date string into "YYYY/MM/DD hh:mm:ss" # 252 | awk 'BEGIN { # 253 | m["Jan"]="01"; m["Feb"]="02"; m["Mar"]="03"; m["Apr"]="04"; # 254 | m["May"]="05"; m["Jun"]="06"; m["Jul"]="07"; m["Aug"]="08"; # 255 | m["Sep"]="09"; m["Oct"]="10"; m["Nov"]="11"; m["Dec"]="12";} # 256 | /^[A-Z]/ { # 257 | t=$4; # 258 | gsub(/:/,"",t); # 259 | d=substr($5,1,1) (substr($5,2,2)*3600+substr($5,4)*60); # 260 | d*=1; # 261 | printf("%04d%02d%02d%s %s %s\n",$6,m[$2],$3,t,d,$7); }' | 262 | # 1:YYYYMMDDHHMMSS 2:delta(local-UTC) 3:ID # 263 | TZ=UTC+0 calclock 1 | 264 | # 1:YYYYMMDDHHMMSS 2:UNIX-time 3:delta(local-UTC) 4:ID # 265 | awk '{print $2-$3,$4;}' | 266 | # 1::UNIX-time(adjusted) 2:ID # 267 | calclock -r 1 | 268 | # 1::UNIX-time(adjusted) 2:localtime 3:ID # 269 | self 2 3 | 270 | # 1:localtime 2:ID # 271 | awk 'BEGIN {fmt="at=%04d/%02d/%02d %02d:%02d:%02d\nid=%s\n"; } # 272 | {gsub(/[0-9][0-9]/,"& ",$1);sub(/ /,"",$1);split($1,t); # 273 | printf(fmt,t[1],t[2],t[3],t[4],t[5],t[6],$2); }' | 274 | # --- 3.regard as an error if no line was outputed # 275 | awk '{print;} END{exit 1-(NR>0);}' 276 | 277 | # === Print error message if some error occured ====================== 278 | case $? in [!0]*) 279 | err=$(echo "$apires" | 280 | parsrj.sh 2>/dev/null | 281 | awk 'BEGIN {errcode=-1; } # 282 | $1~/\.code$/ {errcode=$2; } # 283 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 284 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 285 | END {print errcode, errmsg; }') 286 | [ -z "${err#* }" ] || { error_exit 4 "API error(${err%% *}): ${err#* }"; } 287 | error_exit 4 "API returned an unknown message: $apires" 288 | ;; esac 289 | 290 | # === END: Tweet-ID Loop ============================================= 291 | num_of_success=$((num_of_success+1)) 292 | done <&2 32 | Usage : ${0##*/} [tweet_id...] 33 | Version : 2020-09-28 00:27:57 JST 34 | USAGE 35 | exit 1 36 | } 37 | error_exit() { 38 | ${2+:} false && echo "${0##*/}: $2" 1>&2 39 | exit $1 40 | } 41 | 42 | # === Detect home directory of this app. and define more ============= 43 | Homedir="$(d=${0%/*}/; [ "_$d" = "_$0/" ] && d='./'; cd "$d.."; pwd)" 44 | PATH="$Homedir/UTL:$Homedir/TOOL:$PATH" # for additional command 45 | . "$Homedir/CONFIG/COMMON.SHLIB" # account infomation 46 | 47 | # === Confirm that the required commands exist ======================= 48 | # --- 1.OpenSSL or LibreSSL 49 | if type openssl >/dev/null 2>&1; then 50 | CMD_OSSL='openssl' 51 | else 52 | error_exit 5 'OpenSSL command is not found.' 53 | fi 54 | # --- 2.cURL or Wget 55 | if type curl >/dev/null 2>&1; then 56 | CMD_CURL='curl' 57 | elif type wget >/dev/null 2>&1; then 58 | CMD_WGET='wget' 59 | else 60 | error_exit 5 'No HTTP-GET/POST command found.' 61 | fi 62 | 63 | 64 | ###################################################################### 65 | # Argument Parsing 66 | ###################################################################### 67 | 68 | # === Print usage and exit if one of the help options is set ========= 69 | case "$# ${1:-}" in 70 | '1 -h'|'1 --help'|'1 --version') print_usage_and_exit;; 71 | esac 72 | 73 | # === Initialize parameters ========================================== 74 | tweetid='' 75 | rawoutputfile='' 76 | timeout='' 77 | 78 | # === Read options =================================================== 79 | while :; do 80 | case "${1:-}" in 81 | --rawout=*) # for debug 82 | s=$(printf '%s' "${1#--rawout=}" | tr -d '\n') 83 | rawoutputfile=$s 84 | shift 85 | ;; 86 | --timeout=*) # for debug 87 | s=$(printf '%s' "${1#--timeout=}" | tr -d '\n') 88 | printf '%s\n' "$s" | grep -q '^[0-9]\{1,\}$' || { 89 | error_exit 5 'Invalid --timeout option' 90 | } 91 | timeout=$s 92 | shift 93 | ;; 94 | --|-) break 95 | ;; 96 | --*|-*) error_exit 5 'Invalid option' 97 | ;; 98 | *) break 99 | ;; 100 | esac 101 | done 102 | 103 | 104 | ###################################################################### 105 | # Main Routine 106 | ###################################################################### 107 | 108 | # === Set parameters of Twitter API endpoint (common) ================ 109 | # (1)method for the endpoint 110 | readonly API_methd='POST' 111 | # (2)parameters 112 | readonly API_param='' 113 | 114 | # === BEGIN: Tweet-ID Loop =========================================== 115 | num_of_tweets=0; num_of_success=0 116 | while read tweetid; do num_of_tweets=$((num_of_tweets+1)) 117 | 118 | # === Validate the Tweet-ID ========================================== 119 | printf '%s\n' "$tweetid" | grep -Eq '^[0-9]+$' || { 120 | echo "${0##*/}: $tweetid: Invalid tweet-ID" 1>&2; continue 121 | } 122 | 123 | # === Set parameters of Twitter API endpoint (indivisual) ============ 124 | # (1)endpoint 125 | API_endpt="https://api.twitter.com/1.1/statuses/retweet/$tweetid.json" 126 | 127 | # === Pack the parameters for the API ================================ 128 | # --- 1.URL-encode only the right side of "=" 129 | # (note: This string is also used to generate OAuth 1.0 signature) 130 | apip_enc=$(printf '%s\n' "${API_param}" | 131 | grep -v '^$' | 132 | urlencode -r | 133 | sed 's/%3[Dd]/=/' ) 134 | # --- 2.joint all lines with "&" (note: string for giving to the API) 135 | apip_pos=$(printf '%s' "${apip_enc}" | 136 | tr '\n' '&' ) 137 | 138 | # === Generate the signature string of OAuth 1.0 ===================== 139 | # --- 1.a random string 140 | randmstr=$("$CMD_OSSL" rand 8 | "$CMD_OSSL" md5 | sed 's/.*\(.\{16\}\)$/\1/') 141 | # --- 2.the current UNIX time 142 | nowutime=$(date '+%Y%m%d%H%M%S' | 143 | calclock 1 | 144 | self 2 ) 145 | # --- 3.OAuth 1.0 parameters (generated with 1 and 2) 146 | # (note: This string is also used for an HTTP header) 147 | oa_param=$(cat <<-OAUTHPARAM 148 | oauth_version=1.0 149 | oauth_signature_method=HMAC-SHA1 150 | oauth_consumer_key=${MY_apikey} 151 | oauth_token=${MY_atoken} 152 | oauth_timestamp=${nowutime} 153 | oauth_nonce=${randmstr} 154 | OAUTHPARAM 155 | ) 156 | # --- 4.generate pre-string of the signature 157 | # (note: the API parameters and OAuth 1.0 parameters 158 | # are formed a line like a CGI parameter of GET method) 159 | sig_param=$(cat <<-OAUTHPARAM | 160 | ${oa_param} 161 | ${apip_enc} 162 | OAUTHPARAM 163 | grep -v '^ *$' | 164 | sort -k 1,1 -t '=' | 165 | tr '\n' '&' | 166 | grep ^ | 167 | sed 's/&$//' ) 168 | # --- 5.generate the signature string 169 | # (note: URL-encode API-access-method -- GET or POST --, the endpoint, 170 | # and the above No.4 string respectively at first. and transfer to 171 | # HMAC-SHA1 with the key string which made of the access-keys) 172 | sig_strin=$(cat <<-KEY_AND_DATA | 173 | ${MY_apisec} 174 | ${MY_atksec} 175 | ${API_methd} 176 | ${API_endpt} 177 | ${sig_param} 178 | KEY_AND_DATA 179 | urlencode -r | 180 | tr '\n' ' ' | 181 | grep ^ | 182 | sed 's/ *$//' | 183 | # 1:API-key 2:APIsec 3:method # 184 | # 4:API-endpoint 5:API-parameter # 185 | while read key sec mth ept par; do # 186 | printf '%s&%s&%s' $mth $ept $par | # 187 | "$CMD_OSSL" dgst -sha1 -hmac "$key&$sec" -binary | # 188 | "$CMD_OSSL" enc -e -base64 # 189 | done ) 190 | 191 | # === Access to the endpoint ========================================= 192 | # --- 1.connect and get a response 193 | apires=$(printf '%s\noauth_signature=%s\n%s\n' \ 194 | "${oa_param}" \ 195 | "${sig_strin}" \ 196 | "${API_param}" | 197 | urlencode -r | 198 | sed 's/%3[Dd]/=/' | 199 | sort -k 1,1 -t '=' | 200 | tr '\n' ',' | 201 | grep ^ | 202 | sed 's/^,*//' | 203 | sed 's/,*$//' | 204 | sed 's/^/Authorization: OAuth /' | 205 | while read -r oa_hdr; do # 206 | if [ -n "${CMD_WGET:-}" ]; then # 207 | [ -n "$timeout" ] && { # 208 | timeout="--connect-timeout=$timeout" # 209 | } # 210 | if type gunzip >/dev/null 2>&1; then # 211 | comp='--header=Accept-Encoding: gzip' # 212 | else # 213 | comp='' # 214 | fi # 215 | "$CMD_WGET" ${no_cert_wget:-} -q -O - \ 216 | --header="$oa_hdr" \ 217 | --post-data="$apip_pos" \ 218 | $timeout "$comp" \ 219 | "$API_endpt" | # 220 | if [ -n "$comp" ]; then gunzip; else cat; fi # 221 | elif [ -n "${CMD_CURL:-}" ]; then # 222 | [ -n "$timeout" ] && { # 223 | timeout="--connect-timeout $timeout" # 224 | } # 225 | "$CMD_CURL" ${no_cert_curl:-} -s \ 226 | $timeout ${curl_comp_opt:-} \ 227 | -H "$oa_hdr" \ 228 | -d "$apip_pos" \ 229 | "$API_endpt" # 230 | fi # 231 | done | 232 | if [ $(echo '1\n1' | tr '\n' '_') = '1_1_' ]; then # 233 | grep ^ | sed 's/\\/\\\\/g' # 234 | else # 235 | cat # 236 | fi ) 237 | # --- 2.exit immediately if it failed to access 238 | case $? in [!0]*) error_exit 4 'Failed to access API';; esac 239 | 240 | # === Parse the response ============================================= 241 | # --- 1.extract the required parameters from the response (written in JSON) 242 | echo "$apires" | 243 | if [ -n "$rawoutputfile" ]; then tee "$rawoutputfile"; else cat; fi | 244 | parsrj.sh 2>/dev/null | 245 | awk 'BEGIN {fid=0; fca=0; } # 246 | $1~/^\$\.created_at$/{fca=1; sca=substr($0,index($0," ")+1);} # 247 | $1~/^\$\.id$/ {fid=1; sid=$2; } # 248 | END {if(fid*fca) {print sca,sid} }' | 249 | # 1:DayOfWeek 2:NameOfMonth 3:day 4:HH:MM:SS 5:delta(local-UTC) # 250 | # 6:year 7:ID # 251 | # --- 2.convert date string into "YYYY/MM/DD hh:mm:ss" # 252 | awk 'BEGIN { # 253 | m["Jan"]="01"; m["Feb"]="02"; m["Mar"]="03"; m["Apr"]="04"; # 254 | m["May"]="05"; m["Jun"]="06"; m["Jul"]="07"; m["Aug"]="08"; # 255 | m["Sep"]="09"; m["Oct"]="10"; m["Nov"]="11"; m["Dec"]="12";} # 256 | /^[A-Z]/ { # 257 | t=$4; # 258 | gsub(/:/,"",t); # 259 | d=substr($5,1,1) (substr($5,2,2)*3600+substr($5,4)*60); # 260 | d*=1; # 261 | printf("%04d%02d%02d%s %s %s\n",$6,m[$2],$3,t,d,$7); }' | 262 | # 1:YYYYMMDDHHMMSS 2:delta(local-UTC) 3:ID # 263 | TZ=UTC+0 calclock 1 | 264 | # 1:YYYYMMDDHHMMSS 2:UNIX-time 3:delta(local-UTC) 4:ID # 265 | awk '{print $2-$3,$4;}' | 266 | # 1::UNIX-time(adjusted) 2:ID # 267 | calclock -r 1 | 268 | # 1::UNIX-time(adjusted) 2:localtime 3:ID # 269 | self 2 3 | 270 | # 1:localtime 2:ID # 271 | awk 'BEGIN {fmt="at=%04d/%02d/%02d %02d:%02d:%02d\nid=%s\n"; } # 272 | {gsub(/[0-9][0-9]/,"& ",$1);sub(/ /,"",$1);split($1,t); # 273 | printf(fmt,t[1],t[2],t[3],t[4],t[5],t[6],$2); }' | 274 | # --- 3.regard as an error if no line was outputed # 275 | awk '{print;} END{exit 1-(NR>0);}' 276 | 277 | # === Print error message if some error occured ====================== 278 | case $? in [!0]*) 279 | err=$(echo "$apires" | 280 | parsrj.sh 2>/dev/null | 281 | awk 'BEGIN {errcode=-1; } # 282 | $1~/\.code$/ {errcode=$2; } # 283 | $1~/\.message$/{errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 284 | $1~/\.error$/ {errmsg =$0;sub(/^.[^ ]* /,"",errmsg);} # 285 | END {print errcode, errmsg; }') 286 | [ -z "${err#* }" ] || { error_exit 4 "API error(${err%% *}): ${err#* }"; } 287 | error_exit 4 "API returned an unknown message: $apires" 288 | ;; esac 289 | 290 | # === END: Tweet-ID Loop ============================================= 291 | num_of_success=$((num_of_success+1)) 292 | done <