├── Makefile
├── test.sh
├── detail_report.sh
├── merge_all_thread.py
├── report.sh
├── README.md
└── ProcessGroupMonitor.cpp


/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: clean
2 | 
3 | PGM: clean
4 | 	g++ -std=c++11 ProcessGroupMonitor.cpp -g -o PGM 
5 | 
6 | clean:
7 | 	rm -f PGM
8 | 


--------------------------------------------------------------------------------
/test.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | echo "Task start ..."
 4 | 
 5 | for ((i=1; i<=1000000; i ++))
 6 | do
 7 | 		echo $i
 8 | done
 9 | 
10 | echo "Task end ..."
11 | 


--------------------------------------------------------------------------------
/detail_report.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | function run1(){
 4 | awk 'BEGIN{tb=-1;tp=-1;tn=-1;mm=m;mc=0;r=0;tct=0}{if($1 == "cmd" && r>1) print $0 ; if($1=="###"){ r+=1;if(r==1){ tb=0+$4; tp=tb;} }; if(r<2){if($1=="---"){ tn = 0+$4 ;} else if ($1 == "MEM") {if( $2 > mm ) mm=$2 ; } else if ( $1 == "CPU") {t=tn-tp; tct+=t*(0+$2);tp=tn; if($2>mc)mc=$2;} } } END{printf("MEM_max %s KB\nCPU_max %s\nreal_time %s seconds\ncpu_time %s seconds\n",mm,mc,tn-tb,tct/100);}' $1
 5 | }
 6 | 
 7 | for x in $*
 8 | do
 9 |         echo "----------------------------"
10 |         echo "log_file: $x"
11 |         run1 $x
12 |         echo "----------------------------"
13 |         echo ""
14 | done
15 | 
16 | 


--------------------------------------------------------------------------------
/merge_all_thread.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | import numpy as np
 3 | 
 4 | data = pd.read_csv('tmp_pglog_all.csv',sep='\t',header=0)
 5 | data.columns = ['tip','pid','time','cpu','memory']
 6 | 
 7 | data['time'] = data['time']-data['time'].min()
 8 | data['time'] = data['time'] / 60
 9 | data['time'] = data['time'].astype(int)
10 | data = data.groupby(['time','pid'])[['cpu','memory']].max().reset_index()
11 | data.to_csv('report_pglog_all.csv',sep='\t',header=True)
12 | 
13 | data = data.groupby('time')[['cpu','memory']].sum().reset_index()
14 | print(f'CPU max:  {data["cpu"].max()}',flush=True) 
15 | print(f'MEM max:  {data["memory"].max()}',flush=True) 
16 | data.to_csv('report_pglog_all_as_one.csv',sep='\t',header=True)
17 | 


--------------------------------------------------------------------------------
/report.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | FD=`realpath $0`
 4 | FD=`dirname $FD`
 5 | 
 6 | mem=`tail -n6 pglog_* |grep MEM_max | awk 'BEGIN{x=0;}{if($2>x)x=$2;}END{print x;}'`
 7 | cpu=`tail -n6 pglog_* |grep CPU_max | awk 'BEGIN{x=0;}{if($2>x)x=$2;}END{print x;}'`
 8 | t=`tail -n6 pglog_* |grep TIME_total | awk 'BEGIN{x=0;}{if($2>x)x=$2;}END{print x;}'`
 9 | ct=`tail -n6 pglog_* |grep -E "CPU_max|TIME_total" | awk 'BEGIN{CT=0;c=0;}{if($1=="CPU_max"){c=0+$2;}else{t=0+$2; ct=c*t; CT=CT+ct;  c=0; }} END { printf("%d\n", CT ); }'`
10 | 
11 | echo "---------report in single process mode---------------"
12 | echo "-----    Total report begin =========="
13 | echo "-----    MEM_max               $mem   KB."
14 | echo "-----    CPU_max               $cpu%  threads."
15 | echo "-----    TIME                  $t     seconds."
16 | echo "-----    sum(CPU_max*TIME)*100 $ct    "
17 | echo "-----    Total report end =========="
18 | echo "-----------------------------------------------------"
19 | 
20 | 
21 | echo "---------report in multi process mode---------------"
22 | grep -Inr DataFrame pglog_* >tmp_pglog_all.csv
23 | python3 $FD/merge_all_thread.py
24 | echo "----------------------------------------------------"
25 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # ProcessGroupMonitor
  2 | 
  3 | A simple tool to monitor a process group and record its CPU, MEM and time usage.
  4 | 
  5 | ## INSTALL
  6 | 
  7 |     > make          # to make PGM
  8 |     > make clean    # to remove PGM
  9 | 
 10 | ## USAGE 
 11 | 
 12 | ### Monitor a running process group 
 13 | 
 14 | ./PGM pgid [pgid_num_to_monitor]
 15 | 
 16 | For example, if pgid is 100:
 17 | 
 18 | > ./PGM pgid 100
 19 | 
 20 | Note that you can use the following command to find the pgid for XXX
 21 | 
 22 | > ps -A -o stat,pid,pgid,cmd | grep XXX
 23 | 
 24 | ### Start a new command and monitor it 
 25 | 
 26 | ./PGM file_to_run  args_list
 27 | 
 28 | For example, if you want to run 
 29 | 
 30 | > ./test.sh 10  # this test.sh needs 1 parameter 
 31 | 
 32 | and monitor its CPU, MEM and time usage, then you can simply run:
 33 | 
 34 | > ./PGM ./test.sh 10
 35 | 
 36 | ### Output
 37 | 
 38 | #### PGM will generate a log file for each process. 
 39 | 
 40 | * the name of log file : cmd + pid + start_time .
 41 | 
 42 | * any character in cmd that is neither a digit nor
 43 |   an alphabet will become a '_' like:
 44 | 
 45 | > __share_app_bwa_0_7_12_bwa_index_chr19_standard_contig__206046_1538103905
 46 | 
 47 | #### To get a detail and accurate data for all subcommand
 48 | 
 49 | ```
 50 | ./detail_report.sh pglog_*
 51 | ```
 52 | 
 53 | The report looks like
 54 | 
 55 | ```
 56 | ----------------------------
 57 | log_file: pglog_xxx
 58 | cmd 14994 classify --hap paternal.unique.filter.mer --hap maternal.unique.filter.mer --thread 30 --read rel3-nanopore-wgs.fastq --format fastq
 59 | MEM_max 21823048 KB
 60 | CPU_max 2782
 61 | real_time 5435 seconds
 62 | cpu_time 130997 seconds
 63 | ----------------------------
 64 | 
 65 | ----------------------------
 66 | log_file: pglog_14994_awk__print__2___83594_1590736593
 67 | cmd 14994 awk {print $2}
 68 | MEM_max 9936 KB
 69 | CPU_max 45.8
 70 | real_time 7 seconds
 71 | cpu_time 3.185 seconds
 72 | ----------------------------
 73 | ```
 74 | 
 75 | #### To get the overview of a complete process, use 
 76 | 
 77 | > report.sh
 78 | 
 79 | You will get a report like 
 80 | 
 81 | ```
 82 | Total report begin ==========
 83 | MEM_max   327149736    KB.
 84 | CPU_max   1443    threads.
 85 | TIME      327149736    seconds.
 86 | Total report end ==========
 87 | ```
 88 | Or if your want check a specific command :
 89 | 
 90 | > tail -n 6 your_log_file_name 
 91 | 
 92 | You will get a report like 
 93 | 
 94 |              ###     Final report    1538103946      ###
 95 |              cmd  /share/app/bwa-0.7.12/bwa index chr19_standard.contig
 96 |              pid     206046
 97 |              CPU_max 98.5
 98 |              MEM_max 126148
 99 |              TIME    41 seconds
100 | 
101 | 
102 | If you want more information other than just a simple report, then you can read the whole log file and get information for each snapshot.
103 | 
104 | ### Snapshot frequency
105 |             
106 | The default value is 5 seconds per snapshot.
107 | 
108 | If a different frequency is desired, then you can try to modify the codes of sleep function.
109 | 


--------------------------------------------------------------------------------
/ProcessGroupMonitor.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <sstream>
  3 | #include <fstream>
  4 | #include <map>
  5 | #include <set>
  6 | #include <cassert>
  7 | 
  8 | #include <unistd.h>
  9 | #include <sys/wait.h>
 10 | #include <string.h>
 11 | #include <errno.h>
 12 | 
 13 | struct AProc
 14 | {
 15 |     bool touched ;
 16 |     private:
 17 | 
 18 |     int pid ;
 19 | 
 20 |     long start_time ;
 21 | 
 22 |     float CPU_max ;
 23 | 
 24 |     float CPU_curr ;
 25 | 
 26 |     long Mem_max ;
 27 | 
 28 |     long Mem_curr ;
 29 | 
 30 |     long curr_time ;
 31 | 
 32 |     std::string cmd ;
 33 | 
 34 |     std::string cmd_full ;
 35 | 
 36 |     std::ofstream * ofs ;
 37 | 
 38 |     std::string Photo()
 39 |     {
 40 |         std::ostringstream ost ;
 41 |         ost<<"---\tTouch report\t"<<curr_time<<"\t---\n";
 42 |         ost<<"pid\t"<<pid<<'\n';
 43 |         ost<<"CPU\t"<<CPU_curr<<'\n';
 44 |         ost<<"MEM\t"<<Mem_curr<<'\n';
 45 |         ost<<"DataFrame>>\t"<<pid<<'\t'<<curr_time<<'\t'<<CPU_curr<<'\t'<<Mem_curr;
 46 |         return ost.str();
 47 |     }
 48 | 
 49 |     std::string Final()
 50 |     {
 51 |         std::ostringstream ost ;
 52 |         ost<<"###\tFinal report\t"<<curr_time<<"\t###\n";
 53 |         ost<<"cmd "<<cmd_full<<'\n';
 54 |         ost<<"pid\t"<<pid<<'\n';
 55 |         ost<<"CPU_max\t"<<CPU_max<<'\n';
 56 |         ost<<"MEM_max\t"<<Mem_max<<'\n';
 57 |         ost<<"TIME_total\t"<<curr_time-start_time<<" seconds";
 58 |         return ost.str();
 59 |     }
 60 |     std::string InitReport()
 61 |     {
 62 |         std::ostringstream ost ;
 63 |         ost<<"###\tInit report\t"<<curr_time<<"\t###\n";
 64 |         ost<<"cmd "<<cmd_full<<'\n';
 65 |         ost<<"pid\t"<<pid<<'\n';
 66 |         ost<<"TIME\t"<<start_time<<" seconds";
 67 |         return ost.str();
 68 |     }
 69 |     public:
 70 |     void End()
 71 |     {
 72 |         std::cerr<<" End of "<<pid<<" cmd "<<cmd<<std::endl;
 73 |         (*ofs)<<Final();
 74 |         (*ofs)<<std::endl;
 75 |         delete ofs ;
 76 |         ofs = NULL ;
 77 |     }
 78 |     void Init(int p , const std::string & c, const std::string & c1 ,long time_steamp )
 79 |     {
 80 |         pid = p ;
 81 |         start_time = time_steamp ;
 82 |         curr_time = time_steamp ;
 83 |         cmd = c ;
 84 |         cmd_full = c1 ;
 85 |         CPU_max = 0 ;
 86 |         Mem_max = 0 ;
 87 |         touched = false ;
 88 |         std::string path ;
 89 |         for(char c : cmd_full)
 90 |         {
 91 |             if( std::isalpha(c) || std::isdigit(c) )
 92 |                 path += c;
 93 |             else
 94 |                 path += '_';
 95 |         }
 96 |         if(path.size() > 150 )
 97 |             path= path.substr(0,150);
 98 |         std::string file_name =std::string("pglog_")+ path +"_"+ std::to_string(pid)+"_"+std::to_string(curr_time);
 99 |         std::cerr<<" start of "<<pid<<" cmd "<<cmd<<" into file "<<file_name<<std::endl;
100 |         ofs = new std::ofstream(file_name);
101 |         (*ofs)<<InitReport()<<std::endl;
102 |     }
103 | 
104 |     void Touch(float CPU , long MEM ,long MEM_Max, long time_steamp )
105 |     {
106 |         assert( MEM <= MEM_Max );
107 |         CPU_curr = CPU ;
108 |         if ( CPU > CPU_max )
109 |             CPU_max = CPU ;
110 |         Mem_curr = MEM ;
111 |         if ( Mem_max < MEM_Max )
112 |             Mem_max = MEM ;
113 |         curr_time = time_steamp ;
114 |         touched  = true ;
115 |         (*ofs)<<Photo()<<std::endl;
116 |     }
117 | 
118 |     void UnTouch()
119 |     {
120 |         touched = false ;
121 |     }
122 | };
123 | 
124 | bool GetProcInfo(int pid, float & CPU , long & MEM , long & MEM_Max)
125 | {
126 |     std::string cmd = std::string("ps aux | grep ")+std::to_string(pid)+" | grep -v grep";
127 |     //root      2112  5.2  1.9 2927056 169380 ?      Sl   Sep25 148:36 /usr/bin/kwin_x11
128 |     FILE * pp ;
129 |     if( (pp = popen(cmd.c_str(), "r")) == NULL )
130 |     {
131 |         printf("popen() error!/n");
132 |         exit(1);
133 |     }
134 |     bool ret = false ;
135 |     char buf[10240];
136 |     while(fgets(buf, sizeof buf, pp))
137 |     {
138 |         char tmp[10240];
139 |         char status[10];
140 |         int pid1 ,tmp1;
141 |         float  mem ;
142 |         sscanf(buf,"%s %d %f %f %lu %d %s %s ",
143 |                 tmp,&pid1,&CPU,&mem ,&MEM,
144 |                 &tmp1,tmp,status);
145 |         if( pid1 != pid  )
146 |             continue ;
147 |         if( status[0] == 'Z' || status[0] == 'z')
148 |             continue;
149 |         MEM_Max = MEM ;
150 |         ret = true ;
151 |         break;
152 |     }
153 |     pclose(pp);
154 |     return ret;
155 | };
156 | 
157 | struct PGGroup
158 | {
159 |     std::map< int , AProc >  datas ;
160 | 
161 |     long curr_time ;
162 |     void TouchStart( long time )
163 |     {
164 |         curr_time = time ;
165 |         for( auto & pair : datas )
166 |         {
167 |             pair.second.UnTouch();
168 |         }
169 |     };
170 | 
171 |     bool Touch( int pid , std::string cmd , std::string cmd_full )
172 |     {
173 |         if( datas.find( pid ) == datas.end () )
174 |         {
175 |             datas[pid].Init(pid,cmd,cmd_full,curr_time);
176 |         }
177 |         long MEM , MEM_Max ;
178 |         float CPU ;
179 |         if( GetProcInfo(pid , CPU , MEM , MEM_Max) )
180 |         {
181 |             datas[pid].Touch(CPU ,MEM,MEM_Max , curr_time );
182 |             return true ;
183 |         }
184 |         return false;
185 |     }
186 | 
187 |     void TouchEnd()
188 |     {
189 |         std::set<int> dels;
190 |         for( auto pair : datas )
191 |         {
192 |             if( ! pair.second.touched )
193 |             {
194 |                 pair.second.End();
195 |                 dels.insert(pair.first);
196 |             }
197 |         }
198 |         for( int x :dels )
199 |             datas.erase(x);
200 |     }
201 | } pgdata;
202 | 
203 | void monitor_pgid(int pgid , int pid_self)
204 | {
205 |     int tick = 0 ;
206 |     std::ostringstream ost;
207 |     //Ssl    1009   1009 /usr/sbin/ModemManager
208 |     ost<<"ps -A -o stat,pid,pgid,cmd | grep "<<pgid<<" | grep -v ps | grep -v grep ";
209 |     std::string cmd = ost.str();
210 |     do
211 |     {
212 |         char buf[10240];
213 |         FILE *pp = NULL ;
214 |         long time_steamp = time(0);
215 |         pgdata.TouchStart(time_steamp);
216 | 
217 |         if( (pp = popen(cmd.c_str(), "r")) == NULL )
218 |         {
219 |             printf("popen() error!/n");
220 |             exit(1);
221 |         }
222 |         while(fgets(buf, sizeof buf, pp))
223 |         {
224 |             char exec_file[10240];
225 |             char status[10];
226 |             int pid1 , pgid1 ;
227 |             sscanf(buf,"%s %d %d %s",status,&pid1, &pgid1,exec_file);
228 |             if( pid1 == pid_self )
229 |                 continue ;
230 |             if ( pgid1 != pgid )
231 |                 continue ;
232 |             if( status[0] == 'Z' || status[0] == 'z')
233 |                 continue;
234 |             std::string pgid_str = std::to_string(pgid1);
235 |             std::string cmd_buffer1 = std::string(buf);
236 |             std::string cmd = std::string (exec_file);
237 |             std::string cmd_full = cmd_buffer1.substr(cmd_buffer1.rfind(pgid_str)+1);
238 |             if ( cmd_full[cmd_full.size()-1] == '\n' ) cmd_full[cmd_full.size()-1] = ' ';
239 |             if( pgdata.Touch(pid1,cmd,cmd_full) )
240 |             {
241 |                 std::cerr<<" touch pid "<<pid1<<" cmd [ "<<cmd_full<<" ] succ !!! !!! "<<std::endl;
242 |                 tick = 0 ;
243 |             }
244 |         }
245 |         tick ++ ;
246 |         pgdata.TouchEnd();
247 |         pclose(pp);
248 |         sleep(5);
249 |     } while( tick < 3 );
250 | }
251 | 
252 | int main( int argc , char **argv )
253 | {
254 |     if( argc < 2 )
255 |     {
256 |         std::cerr<<"Usage   :   "<< std::endl;
257 |         std::cerr<<"    to monitor exsit pgid , use "<<argv[0]<<" pgid pgid_num_to_monitor "<< std::endl;
258 |         std::cerr<<"        example     :   "<<argv[0]<<" pgid 100"<< std::endl;
259 |         std::cerr<<"    to start a new command and monitor it , use "<<argv[0]<<" file_to_run args_list"<< std::endl;
260 |         std::cerr<<"        example     :   "<<argv[0]<<" ./test.sh 10 "<< std::endl;
261 |         return 1 ;
262 |     }
263 |     if( std::string(argv[1]) == "pgid" )
264 |     {
265 |         if( argc != 3 )
266 |         {
267 |             std::cerr<<"Usage for pgid  :   "<<argv[0]<<" pgid pgid_num_to_monitor "<< std::endl;
268 |             std::cerr<<"        example     :   "<<argv[0]<<" pgid 100"<< std::endl;
269 |             return 1 ;
270 |         }
271 |         int pgid_to_monitor ;
272 |         try 
273 |         {
274 |             pgid_to_monitor = std::stoi(std::string(argv[2]));
275 |         }
276 |         catch ( ... )
277 |         {
278 |             std::cerr<<"ERROR   :   "<<argv[2]<<" is not a valid pgid "<<std::endl;
279 |             std::cerr<<"Usage for pgid  :   "<<argv[0]<<" pgid pgid_num_to_monitor "<< std::endl;
280 |             std::cerr<<"        example     :   "<<argv[0]<<" pgid 100"<< std::endl;
281 |             return 1 ;
282 |         }
283 |         std::cerr<<"Info    :   pgid to be moniterd is -- "<<pgid_to_monitor<<std::endl;
284 |         pid_t pid = getpid();
285 |         monitor_pgid(pgid_to_monitor,pid);
286 |         return 0 ;
287 |     }
288 | 
289 |     std::cerr<<"Info    :   command to be moniterd is -- ";
290 |     for( int i = 1 ; i < argc ; i++ )
291 |         std::cerr<<" "<<argv[i];
292 |     std::cerr<<std::endl;
293 |     pid_t pid = getpid();
294 |     pid_t pgid = getpgid(pid);
295 |     if( pid != pgid )
296 |     {
297 |         setpgid(pid,pid);
298 |         std::cerr<<"Info    :   old pgid is -- "<<pgid<<std::endl;
299 |         std::cerr<<"Info    :   set pgid to -- "<<pid<<std::endl;
300 |     }
301 |     else
302 |         std::cerr<<"Info    :   curr pgid is -- "<<pid<<std::endl;
303 |     char ** argv_new = new char * [argc-1];
304 |     for( int i = 0 ; i < argc -1 ; i ++ )
305 |     {
306 |         argv_new[i] =argv[i+1];
307 |     }
308 |     argv_new[argc-1] = NULL;
309 |     pid_t child_pid = fork();
310 | 
311 |     if( child_pid < 0 )
312 |     {
313 |         std::cerr<<"Error   :   fork failed !!!"<<strerror(errno)<<std::endl;
314 |         return 1 ;
315 |     }
316 |     if (child_pid > 0) {    // Monitor process 
317 |         monitor_pgid(pid,pid);
318 |         //delete []argv_new ;
319 |         wait(NULL);
320 |         return 0 ;
321 |     }
322 |     else { // Job process
323 |        execv(argv[1] , argv_new);
324 |        std::cerr<<"Error   :   execv failed !!!"<<strerror(errno)<<std::endl;
325 |        return 1 ;
326 |     }
327 |     return 0 ;
328 | }
329 | 
330 | 


--------------------------------------------------------------------------------