├── ExampleScreenShot.png ├── queue_status_dav_files.png ├── README.md └── casper_resource_status.pl /ExampleScreenShot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NCAR/casper_resource_status/main/ExampleScreenShot.png -------------------------------------------------------------------------------- /queue_status_dav_files.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NCAR/casper_resource_status/main/queue_status_dav_files.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # casper_resource_status 2 | This Perl script generates an html table showing the current node and batch queue usage on NCAR's Casper cluster. 3 | The script is executed every 5 minutes via cron by user 'csgteam'. 4 | 5 | On GLADE, the script's source is in the file /glade/u/home/csgteam/scripts/queue_status_dav/casper_resource_status.pl 6 | 7 | The output html table is written to /glade/u/home/csgteam/scripts/queue_status_dav/nodes_table.html 8 | and embedded in the CISL Resource Status page, https://www2.cisl.ucar.edu/user-support/cisl-resource-status 9 | 10 | # Usage 11 | For typical production execution: 12 | /glade/u/home/csgteam/scripts/queue_status_dav/casper_resource_status.pl 13 | 14 | There are two optional parameters, "use_qstat_cache" and "test_mode", that were added to aid in 15 | development and debugging. 16 | /glade/u/home/csgteam/scripts/queue_status_dav/casper_resource_status.pl -test_mode 17 | Dumps print output to terminal and writes output files to the local working directory. 18 | There is a known side effect using this where an empty output log file is generated. 19 | 20 | /glade/u/home/csgteam/scripts/queue_status_dav/casper_resource_status.pl -use_qstat_cache 21 | makes use of existing output *.out files (see bbelow) to reduce queries 22 | against the PBS database. This also turns on test mode. 23 | 24 | # Output files 25 | In addition to "queues_table_ch.html" 8 temporary output files are re-generated in each execution. 26 | ![alt text](https://github.com/mickcoady/casper_resource_status/blob/main/queue_status_dav_files.png "output files") 27 | 28 | 29 | # Example output html table 30 | ![alt text](https://github.com/mickcoady/casper_resource_status/blob/main/ExampleScreenShot.png "Example table") 31 | 32 | 33 | -------------------------------------------------------------------------------- /casper_resource_status.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use Switch; 4 | #use warnings; 5 | 6 | 7 | my $targetdir = " "; 8 | my $logfilename = " "; 9 | 10 | my $tmpdatestamp=`date "+%m%d%y_%R"`; 11 | chomp $tmpdatestamp; 12 | 13 | # 14 | # Check for optional arguments - only "-use_qstat_cache" and "-test_mode" are valid 15 | # 16 | my $nArgs = scalar @ARGV; 17 | print "\nnumber of input arguments = $nArgs (@ARGV) \n"; 18 | 19 | my $testing_mode = 0; 20 | my $use_qstat_cache = 0; 21 | 22 | use Getopt::Long; 23 | GetOptions( 'use_qstat_cache' => \$use_qstat_cache, 24 | 'test_mode' => \$testing_mode); 25 | print ("after GetOptions use_qstat_cache = $use_qstat_cache testing_mode = $testing_mode \n\n"); 26 | 27 | if ( $nArgs > 0 ) { 28 | if ($use_qstat_cache == 1) { 29 | $testing_mode = 1; 30 | print "Will use existing qstat cache files. \n"; 31 | } 32 | if ($testing_mode == 1) { 33 | print "Will run in test mode. \n"; 34 | } 35 | if (($use_qstat_cache+$testing_mode) == 0) { 36 | print "\nInvalid option(s). Only valid options are '-use_qstat_cache'and '-test_mode'.\n"; 37 | exit; 38 | } 39 | } 40 | 41 | # 42 | # Set up output file names and directories 43 | # 44 | if ($testing_mode == 1) { 45 | use Cwd qw(cwd); 46 | $targetdir = cwd; 47 | $logfilename = $targetdir . "/" . $tmpdatestamp . ".log"; 48 | } else { 49 | $targetdir = "/glade/u/home/csgteam/scripts/queue_status_dav"; 50 | $logfilename = "/glade/scratch/csgteam/ca_queue_status_logs/" . $tmpdatestamp . ".log"; 51 | } 52 | 53 | # 54 | # if running in "production" mode send print diagnostics output to $logfilename, otherwise 55 | # just echo to terminal 56 | # 57 | print "\noutput target directory: $targetdir \n"; 58 | print "output log file: $logfilename \n\n\n"; 59 | open(my $LOG, '>>', $logfilename) or die "Could not open file '$logfilename' $! \n"; 60 | select $LOG; 61 | if ($testing_mode == 1) { 62 | select STDOUT; # send print output to terminal 63 | } 64 | 65 | my $timeout = 60; # seconds to wait for "qstat" and "pbsnodes" commands to return 66 | #$timeout = 1; # Short timeout for testing purposes only 67 | 68 | # 69 | # STATUSFILE is a simple text file version of the output html file that will be accessed 70 | # by users with a command script "show_status". 71 | # 72 | open HTMLFILE, ">$targetdir/nodes_table.html" or die "Could not open file HTMLFILE $!"; 73 | open STATUSFILE, ">$targetdir/tmp_show_status.out" or die "Could not open file STATUSFILE $!"; 74 | 75 | my $qstat_out_len = 0; 76 | my $nodestate_out_len = 0; 77 | 78 | ## my @all_share_queue_reservations; 79 | ## my @share_queue_reservations; 80 | my @all_reservations; 81 | my @reservations; 82 | my %q; 83 | 84 | my $color = "#000000"; 85 | my $PBSbin = "/opt/pbs/bin"; 86 | my $qstatcmd = "/glade/u/apps/ch/opt/usr/bin/qstat"; 87 | 88 | my $nNodes_Free = 0; 89 | my $nNodes_PartialUsed = 0; 90 | my $nNodes_100Used = 0; 91 | my $nNodes_offline = 0; 92 | 93 | # 94 | # generate the files with PBS qstat and pbsnodes command output. These files will be parsed repeatedly 95 | # 96 | eval { 97 | local $SIG{ALRM} = sub { die "alarm\n" }; 98 | alarm $timeout; 99 | 100 | # throttle the demand on the PBS server 101 | my $sleepcmd = "sleep 10"; 102 | if ($testing_mode == 1) { 103 | $sleepcmd = "sleep 1"; 104 | } 105 | 106 | ## Not using this function yet. May opt to use it at some point in the future to 107 | ## programmatically determine all queue names. 108 | ## my @queuenames = `$PBSbin/qstat -Q \@casper | grep "Exe\*" | grep -v Type`; 109 | ## print "queuenames = \n", @queuenames, "\n"; 110 | 111 | 112 | if ($use_qstat_cache == 0) { 113 | `timeout -s SIGKILL $timeout $qstatcmd \@casper | grep -vi "job id" | grep ".casper-pbs" > "$targetdir/qstat.out"`; 114 | 115 | `timeout -s SIGKILL $timeout $qstatcmd \@casper -n -w | grep " R "> "$targetdir/qstat-wn.out"`; 116 | 117 | `ssh casper timeout -s SIGKILL $timeout $PBSbin/pbsnodes -a | grep "state = " | grep -v comment | grep -v last_state_change_time | sort | uniq -c > "$targetdir/nodestate.out"`; 118 | 119 | `ssh casper timeout -s SIGKILL $timeout $PBSbin/pbsnodes -a > "$targetdir/pbsnodes-a.out"`; 120 | 121 | `ssh casper timeout -s SIGKILL $timeout $PBSbin/pbsnodes -aSj -D "," > "$targetdir/pbsnodes-aSj.out"`; 122 | 123 | `ssh casper timeout -s SIGKILL $timeout $PBSbin/pbsnodes -l > "$targetdir/pbsnodes-l.out"`; 124 | 125 | `ssh casper timeout -s SIGKILL $timeout $PBSbin/pbs_rstat | grep '[RS][0-9][0-9]' | grep " RN " | awk '{print \$2}' > "$targetdir/pbs_reservations.out"`; 126 | } 127 | 128 | alarm 0; 129 | }; 130 | 131 | $nNodes_offline = `cat "$targetdir/pbsnodes-l.out" | wc -l`; 132 | chomp $nNodes_offline; 133 | 134 | $qstat_out_len = `wc -l $targetdir/qstat.out`; 135 | $nodestate_out_len = `grep state $targetdir/nodestate.out | wc -l`; 136 | 137 | 138 | if ($@ | ($nodestate_out_len == 0) | ($qstat_out_len == 0)) { 139 | # qstat and/or pbsnodes command timed out or did not return anything useful 140 | if ( ($nodestate_out_len == 0) | ($qstat_out_len == 0) ) { $@ = "alarm\n";} 141 | die unless $@ eq "alarm\n"; # propagate unexpected errors 142 | 143 | my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); 144 | $year += 1900; $mon++; 145 | my $datestamp = sprintf('%02d:%02d %02d/%02d/%04d',$hour,$min,$mon,$mday,$year); 146 | 147 | print HTMLFILE qq{ 148 | 149 | 157 | 158 | 159 | 160 | 161 | 162 | 167 | 168 | 169 |
163 | The Casper job scheduling system was not responding as of $datestamp.
164 | If the problem persists, users will receive email updates through our Notifier service.
165 | qstat_out_len = $qstat_out_len nodestate_out_len = $nodestate_out_len
166 |
170 | }; 171 | close (HTMLFILE); 172 | 173 | print STATUSFILE " \n"; 174 | print STATUSFILE " The Casper job scheduling system was not responding as of $datestamp. \n"; 175 | print STATUSFILE " If the problem persists, users will receive email updates through our Notifier service. \n"; 176 | print STATUSFILE " \n"; 177 | close (STATUSFILE); 178 | 179 | print "\nqstat and/or pbsnodes commands timed out or did not return anything useful\n"; 180 | print "processing was aborted. \n\n"; 181 | 182 | } else { # PBS commands did not time out so proceed 183 | 184 | my $nNodes_runningjobs = `cat "$targetdir/qstat-wn.out" | grep " R " | awk '{print \$12 }' | tr + '\n' | cut -d "/" -f 1 | sort | uniq | wc -l`; 185 | chomp $nNodes_runningjobs; 186 | 187 | $nNodes_Free = `cat "$targetdir/pbsnodes-aSj.out" | grep -c " free 0 0"`; 188 | chomp $nNodes_Free; 189 | 190 | $nNodes_PartialUsed = `cat "$targetdir/pbsnodes-aSj.out" | grep " free " | grep -v "free 0 0" | wc -l`; 191 | chomp $nNodes_PartialUsed; 192 | 193 | $nNodes_100Used = `cat "$targetdir/pbsnodes-aSj.out" | grep -c job-busy`; 194 | chomp $nNodes_100Used; 195 | 196 | my $tot_Casper_nodes = `ssh casper $PBSbin/pbsnodes -a | grep Mom | wc -l`; 197 | chomp $tot_Casper_nodes; 198 | 199 | print "number of PBS nodes = $tot_Casper_nodes \n"; 200 | print "nNodes_Free = $nNodes_Free \n"; 201 | print "nNodes_PartialUsed = $nNodes_PartialUsed \n"; 202 | print "nNodes_100Used = $nNodes_100Used \n"; 203 | print "nNodes_runningjobs = $nNodes_runningjobs\n\n"; 204 | 205 | 206 | my @nodes_jobs = `cat "$targetdir/nodestate.out" | grep "job-busy" | grep -v down | grep -v offline | awk '{ print \$1 }'`; 207 | 208 | my $nNodes_jobs = 0; 209 | foreach my $nnodes_job (@nodes_jobs) { 210 | $nNodes_jobs += $nnodes_job; 211 | } 212 | 213 | @reservations = `cat "$targetdir/pbs_reservations.out"`; 214 | print scalar @reservations; print " running reservations found \n"; 215 | print @reservations, "\n"; 216 | 217 | my $nNodes_reserved = 0; 218 | my $nJobs_reservations = 0; 219 | my $nResNodes_jobs = 0; 220 | 221 | my @reservedNodes_list = `ssh casper $PBSbin/pbs_rstat -F | grep resv_nodes | awk '{print \$3}' | sed 's/(//g' | awk '{sub (/+/, "\\n"); print}' | cut -d ":" -f1`; 222 | my $num_reservations = scalar @reservedNodes_list; 223 | print "number of reservations found = $num_reservations \n"; 224 | print "reservedNodes_list = \n @reservedNodes_list \n"; 225 | 226 | foreach my $res (@reservations) { 227 | chomp $res; 228 | my $nNodes_reservation = `ssh casper timeout -s SIGKILL $timeout $PBSbin/pbs_rstat -F $res | grep nodect | awk '{ print \$3 }'`; 229 | chomp $nNodes_reservation; 230 | print "number of nodes in reservation $res: $nNodes_reservation \n"; 231 | 232 | $nNodes_reserved += $nNodes_reservation; 233 | 234 | my $num_running_jobs = `cat "$targetdir/qstat.out" | grep $res | grep " R " | wc -l`; 235 | chomp $num_running_jobs; 236 | $nJobs_reservations += $num_running_jobs; 237 | print "number of running jobs in reservation $res: $num_running_jobs \n"; 238 | 239 | my @jobids; 240 | my $nresnodes = `grep $res "$targetdir/qstat-wn.out" | awk '{ print \$6 }'`; 241 | chomp $nresnodes; 242 | if ($nresnodes > 0) { 243 | print "reservation $res found in qstat-wn.out nresnodes = $nresnodes \n"; 244 | @jobids = `grep $res "$targetdir/qstat-wn.out" | awk '{ print \$1 }' | cut -d '.' -f 1`; 245 | print " jobids = @jobids"; 246 | } 247 | $nResNodes_jobs += $nresnodes; 248 | } 249 | 250 | print "\nTotal number of reserved nodes: $nNodes_reserved \n"; 251 | print "Total number of reserved nodes in use = $nResNodes_jobs \n"; 252 | print "Total number of jobs running in reservations = $nJobs_reservations \n\n"; 253 | 254 | 255 | print "According to pbsnodes command: \n"; 256 | print "number of PBS nodes = $tot_Casper_nodes \n"; 257 | print "number of nodes free = $nNodes_Free \n"; 258 | print "number of nodes busy = $nNodes_jobs \n"; 259 | print "number of nodes offline = $nNodes_offline \n"; 260 | print "number of nodes reserved = $nNodes_reserved \n"; 261 | my $tmp_nodecount = $nNodes_Free + $nNodes_jobs + $nNodes_offline + $nNodes_reserved; 262 | print "total number of nodes accounted for by pbsnodes = $tmp_nodecount\n\n"; 263 | 264 | 265 | my $run_htc = `cat "$targetdir/qstat.out" | grep " R htc" | wc -l`; chomp $run_htc; 266 | my $run_vis = `cat "$targetdir/qstat.out" | grep " R vis" | wc -l`; chomp $run_vis; 267 | my $run_rda = `cat "$targetdir/qstat.out" | grep " R rda" | wc -l`; chomp $run_rda; 268 | my $run_gpgpu = `cat "$targetdir/qstat.out" | grep " R gpgpu" | wc -l`; chomp $run_gpgpu; 269 | my $run_lrgmem = `cat "$targetdir/qstat.out" | grep " R largemem" | wc -l`; chomp $run_lrgmem; 270 | my $run_mixgpu = `cat "$targetdir/qstat.out" | grep " R mixgpu" | wc -l`; chomp $run_mixgpu; 271 | 272 | my $tot_jobs_running = 0; 273 | print "jobs running in htc queue = $run_htc\n"; 274 | print "jobs running in vis queue = $run_vis\n"; 275 | print "jobs running in gpgpu queue = $run_gpgpu\n"; 276 | print "jobs running in rda queue = $run_rda\n"; 277 | print "jobs running in largemem = $run_lrgmem\n"; 278 | print "jobs running in mixgpu = $run_mixgpu\n"; 279 | print "jobs running in reservations = $nJobs_reservations\n"; 280 | 281 | 282 | $tot_jobs_running = $run_htc + $run_vis + $run_rda + $run_gpgpu + $run_lrgmem + $run_mixgpu + $nJobs_reservations; 283 | print "number of batch jobs running = $tot_jobs_running \n\n"; 284 | 285 | 286 | # output the html 287 | # the loop through the queues also counts the number of users ($q{$queue}[3]) for each queue. 288 | # the "if" portion of those statements avoids displaying "0" for empty queues. 289 | 290 | print HTMLFILE q{ 291 | 292 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | }; 311 | print STATUSFILE " \n"; 312 | print STATUSFILE " Queue Jobs Jobs Jobs Users \n"; 313 | print STATUSFILE " Running Queued Held \n"; 314 | 315 | # forces a particular order on the queues in the table. 316 | # inserts "-" in place of blanks/zeros. 317 | my @queues = qw(htc vis gpgpu rda largemem mixgpu reservations); 318 | 319 | foreach my $queue (@queues) { 320 | 321 | $q{$queue}[0] = '-' unless ($q{$queue}[0]); # number of jobs running in $queue 322 | $q{$queue}[1] = '-' unless ($q{$queue}[1]); # number of nodes in running jobs in $queue 323 | $q{$queue}[2] = '-' unless ($q{$queue}[2]); # number of jobs queued in $queue 324 | $q{$queue}[3] = '-' unless ($q{$queue}[3]); # number of jobs held in $queue 325 | $q{$queue}[4] = '-' unless ($q{$queue}[4]); # number of unique users with jobs in $queue 326 | 327 | if ($queue =~ m/htc|vis|gpgpu|rda|largemem|mixgpu/) { 328 | my $queue_name = $queue; 329 | 330 | $q{$queue}[0] = `cat "$targetdir/qstat.out" | grep $queue_name | grep " R " | wc -l`; 331 | $q{$queue}[2] = `cat "$targetdir/qstat.out" | grep $queue_name | grep " Q " | wc -l`; 332 | $q{$queue}[3] = `cat "$targetdir/qstat.out" | grep $queue_name | grep " H " | wc -l`; 333 | $q{$queue}[4] = `cat "$targetdir/qstat.out" | grep $queue_name | awk '{ print \$3 }' | sort | uniq | wc -l`; # number of unique users 334 | print "queue: $queue number of unique users = $q{$queue}[4]"; 335 | 336 | my $queue_node_count = int(`cat "$targetdir/qstat-wn.out" | grep $queue_name | grep " R " | awk '{print \$12}' | cut -d '/' -f 1 | sort | uniq -c | wc -l`); 337 | 338 | chomp $queue_node_count; 339 | print " unique node count = $queue_node_count\n"; 340 | $q{$queue}[1] = $queue_node_count; 341 | 342 | } # for htc, vis, gpgpu, largemem, rda, mixgpu queues 343 | 344 | else { 345 | my $queue_name = "reservations"; 346 | my $nResJobs_Q = 0; 347 | my $nUnique_Users = 0; 348 | 349 | foreach my $res (@reservations) { 350 | chomp $res; 351 | 352 | my $njobs_queued = `cat "$targetdir/qstat.out" | grep $res | grep " Q " | wc -l`; 353 | chomp $njobs_queued; 354 | $nResJobs_Q += $njobs_queued; 355 | 356 | my $nusers = `cat "$targetdir/qstat.out" | grep $res | awk '{ print \$2 }' | sort | uniq | wc -l`; # this reservations number of unique users 357 | chomp $nusers; 358 | $nUnique_Users += $nusers; 359 | } 360 | 361 | $q{$queue}[0] = $nJobs_reservations; 362 | $q{$queue}[1] = $nResNodes_jobs; 363 | $q{$queue}[2] = $nResJobs_Q; 364 | $q{$queue}[3] = 0; # For now assuming that no reservation jobs are on hold. 365 | $q{$queue}[4] = $nUnique_Users; 366 | print "reservation: $queue number of unique users = $q{$queue}[4]"; 367 | 368 | } # for reservations 369 | print "\n"; 370 | 371 | } # foreach queue - htc | vis| gpgpu | rda | mixgpu | largemem | reservations 372 | 373 | 374 | # determine if any queues are completely empty of any jobs - those entries will skipped. 375 | my $numRows = 18; 376 | foreach my $queue (@queues) { 377 | if ( ($q{$queue}[0] + $q{$queue}[2] + $q{$queue}[3]) == 0 ) { # count number of jobs running, queued or held 378 | $numRows--; 379 | print "no jobs found for queue $queue - will not be reported \n"; 380 | } 381 | else { 382 | my $tmp_njobs = $q{$queue}[0] + $q{$queue}[2] + $q{$queue}[3]; 383 | print "queue $queue has a total number of $tmp_njobs jobs\n"; 384 | } 385 | } 386 | 387 | print "\ntable will have $numRows rows\n"; 388 | print HTMLFILE qq{ 389 | 390 | 393 | 394 | }; 395 | print "Green light written to table\n\n"; 396 | 397 | 398 | foreach my $queue (@queues) { 399 | if ( ($q{$queue}[0] + $q{$queue}[2] + $q{$queue}[3]) > 0 ) { 400 | print HTMLFILE qq{ 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | }; 409 | printf STATUSFILE "%12s %9d %8d %8d %8d %8d \n", $queue, $q{$queue}[0], $q{$queue}[2], $q{$queue}[3], $q{$queue}[4]; 410 | } 411 | } 412 | 413 | my $total_jobs = 0; 414 | my $total_nodes = 0; 415 | my $total_queued = 0; 416 | my $total_held = 0; 417 | my $total_users = 0; 418 | 419 | foreach my $queue (@queues) { 420 | $total_jobs += $q{$queue}[0]; 421 | $total_nodes += $q{$queue}[1]; 422 | $total_queued += $q{$queue}[2]; 423 | $total_held += $q{$queue}[3]; 424 | $total_users += $q{$queue}[4]; 425 | } 426 | 427 | print HTMLFILE qq{ 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | }; 436 | printf STATUSFILE "---------------------------------------------------------- \n"; 437 | printf STATUSFILE " Totals "; 438 | printf STATUSFILE "%9d %8d %8d %8d %8d \n\n", $total_jobs, $total_queued, $total_held, $total_users; 439 | 440 | 441 | # Add the "Node Activity" table. Assuming that both the number of free nodes and the number 442 | # of nodes allocated for the share queue but not in use will always be greater than zero. 443 | print HTMLFILE qq{ 444 | 445 | 446 | 447 | 448 | 449 | 450 | 451 | 452 | 453 | 454 | }; 455 | 456 | printf STATUSFILE "Node Activity: \n"; 457 | printf STATUSFILE " All 36 CPUs in Use %5d \n", $nNodes_100Used; 458 | printf STATUSFILE " 1-35 CPUs in Use %5d \n", $nNodes_PartialUsed; 459 | printf STATUSFILE " 0 CPUs in Use %5d \n", $nNodes_Free; 460 | printf STATUSFILE " Reserved / In Use %5d /%2d \n", $nNodes_reserved, $nResNodes_jobs; 461 | printf STATUSFILE " Down/Offline %5d \n\n", $nNodes_offline; 462 | 463 | 464 | my $Total_Node_Count = $nNodes_Free + $nNodes_PartialUsed + $nNodes_100Used + $nNodes_offline + $nNodes_reserved; 465 | print "Total_Node_Count = $Total_Node_Count \n"; 466 | # if ($Total_Node_Count != $tot_Casper_nodes) { 467 | # $Total_Node_Count = $tot_Casper_nodes; 468 | # } 469 | 470 | 471 | print HTMLFILE qq{ 472 | 473 | 474 | 475 | 476 | 477 | 478 | 479 | 480 | 481 | 482 | 483 | 484 | 485 | 486 | 487 | 488 | 489 | 490 | 491 | 492 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | 501 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | 509 | 510 | 511 | 512 | 513 | }; 514 | 515 | 516 | my $datestamp=`date "+%l:%M %P %Z %a %b %e %Y"`; # used for display in HTML table 517 | print HTMLFILE qq{ 518 | 519 | 520 | 521 | 522 |
SystemQueueJobs
Running		Jobs
Queued		Jobs
Held		Users
Casper
391 | 392 |
$queue$q{$queue}[0] $q{$queue}[2] $q{$queue}[3] $q{$queue}[4]
Totals $total_jobs $total_queued $total_held $total_users
Node Activity
All 36 CPUs in Use $nNodes_100Used
1-35 CPUs in Use $nNodes_PartialUsed
0 CPUs in Use $nNodes_Free
Reserved / In Use $nNodes_reserved / $nResNodes_jobs
Down/Offline $nNodes_offline
Total Nodes $Total_Node_Count
Updated $datestamp
523 | }; 524 | #printf STATUSFILE "Updated %s \n\n", $datestamp; 525 | 526 | close (HTMLFILE); 527 | close (STATUSFILE); 528 | select STDOUT; 529 | 530 | my $cmd = "cp $targetdir/tmp_show_status.out $targetdir/show_status.out"; 531 | my $noopt = `$cmd`; 532 | 533 | if ($testing_mode == 0) { 534 | my $cmd = "rm -f $logfilename"; 535 | # my $noopt = `$cmd`; 536 | } 537 | 538 | } 539 | --------------------------------------------------------------------------------