├── OSSMETADATA
├── conf
    └── rpm.spec
├── root
    └── apps
    │   └── apache
    │       ├── conf.d
    │           └── admin_gc_viz.conf
    │       └── htdocs
    │           └── AdminGCViz
    │               ├── remote-data-collection
    │                   ├── prepend_epoch.py
    │                   ├── facet_events_by_type.sh
    │                   ├── gc_events_by_type.sh
    │                   ├── facet_events_by_country.sh
    │                   ├── gcdotlog_extract_sizes.pl
    │                   ├── gcdotlog_one_event_per_line.pl
    │                   ├── parse-proc-pid-maps.py
    │                   ├── vms_facet_info_transform.py
    │                   ├── gcdotlog_convert_relative_into_absolute_time.py
    │                   ├── gcdotlog_extract_time.pl
    │                   ├── process_vms_object_cache_stats.py
    │                   └── collect_remote_data.sh
    │               ├── gc_event_types
    │               ├── index
    │               ├── vmsgcvizutils.py
    │               ├── BUGS
    │               ├── README
    │               ├── visualize-facets.py
    │               ├── generate
    │               └── visualize-gc.py
├── LICENSE
├── README
└── INSTALL


/OSSMETADATA:
--------------------------------------------------------------------------------
1 | osslifecycle=archived
2 | 


--------------------------------------------------------------------------------
/conf/rpm.spec:
--------------------------------------------------------------------------------
 1 | Name:      @name@
 2 | Summary:   Netflix GC Visualization
 3 | Version:   @version@
 4 | Release:   @release@
 5 | License:   NFLX
 6 | Packager:  Engineering Tools
 7 | Vendor:    Netflix, Inc.
 8 | Group:     Netflix Base
 9 | AutoReqProv: no
10 | Requires: nflx-python-matplotlib, nflx-python-numpy, nflx-python-scipy
11 | 
12 | %description
13 | GC Visualization
14 | ----------
15 | @build.metadata@
16 | 
17 | %install
18 | cat $0 > %{_topdir}/install.txt
19 | mkdir -p $RPM_BUILD_ROOT
20 | mv ${RPM_BUILD_DIR}/* $RPM_BUILD_ROOT
21 | 
22 | %clean
23 | rm -rf $RPM_BUILD_ROOT/*
24 | 
25 | %files
26 | %defattr(-,root,root)
27 | /
28 | 
29 | %post
30 | 


--------------------------------------------------------------------------------
/root/apps/apache/conf.d/admin_gc_viz.conf:
--------------------------------------------------------------------------------
 1 | 
 2 | # configuration for GC visualization admin app
 3 | # mounted under /AdminGCViz and /AdminGCVizImages
 4 | 
 5 | ScriptAlias /AdminGCViz/ "/apps/apache/htdocs/AdminGCViz/"
 6 | Alias /AdminGCVizImages/ "/mnt/logs/gc-reports/"
 7 | 
 8 | <LocationMatch "^/AdminGCViz">
 9 |   Order deny,allow
10 |   Deny from all
11 |   Allow from 127.0.0.1/32
12 | </LocationMatch>
13 | 
14 | # send requests to /AdminGCViz [with and without trailing /] to /index
15 | RewriteRule ^/AdminGCViz/?$    /AdminGCViz/index    [R]
16 | # Force apache to handle the rest [this catches AdminGCVizImages too]
17 | RewriteRule ^/AdminGCViz       -                    [L]
18 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | /*
 2 |  *
 3 |  *  Copyright 2013 Netflix, Inc.
 4 |  *
 5 |  *     Licensed under the Apache License, Version 2.0 (the "License");
 6 |  *     you may not use this file except in compliance with the License.
 7 |  *     You may obtain a copy of the License at
 8 |  *
 9 |  *         http://www.apache.org/licenses/LICENSE-2.0
10 |  *
11 |  *     Unless required by applicable law or agreed to in writing, software
12 |  *     distributed under the License is distributed on an "AS IS" BASIS,
13 |  *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  *     See the License for the specific language governing permissions and
15 |  *     limitations under the License.
16 |  *
17 |  */
18 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/prepend_epoch.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python2.7
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/prepend_epoch.py#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | import fileinput
 9 | import sys
10 | 
11 | # Ugh.
12 | sys.path.insert(0, "/apps/apache/htdocs/AdminGCViz")
13 | import vmsgcvizutils   
14 | 
15 | def mayne():
16 |     print "secs_since_epoch"
17 |     for line in fileinput.input('-'):
18 |         line = line.rstrip('\r\n')
19 |         print "%s" % (vmsgcvizutils.timestamp_to_epoch(line),)
20 | 
21 | if __name__ == "__main__":
22 |     mayne()
23 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/gc_event_types:
--------------------------------------------------------------------------------
 1 | ParNew                            (stop-the-world)
 2 | CMS-initial-mark                  (stop-the-world)
 3 | CMS-concurrent-mark               (concurrent includes yields to other theads)
 4 | CMS-concurrent-abortable-preclean (concurrent)
 5 | CMS-concurrent-preclean           (concurrent)
 6 | CMS-remark                        (stop the world)
 7 | CMS-concurrent-sweep              (concurrent)
 8 | CMS-concurrent-reset              (concurrent?)
 9 | concurrent mode failure           (stop the world)
10 | promotion failed                  (stop the world)
11 | Full GC                           (stop the world)
12 | 
13 | markers
14 | CMS-concurrent-mark-start
15 | CMS-concurrent-preclean-start
16 | CMS-concurrent-sweep-start
17 | CMS-concurrent-reset-start
18 | CMS-concurrent-abortable-preclean-start
19 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/facet_events_by_type.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/facet_events_by_type.sh#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | if [ -z "$1" ]; then
 9 |     echo Usage: $0 output-dir
10 |     exit 1
11 | fi
12 | 
13 | OUTPUTDIR=$1
14 | OUTDIR=${OUTPUTDIR}/facet-events-by-cache
15 | INFILE=${OUTPUTDIR}/vms-cache-refresh-facet-info.csv
16 | mkdir -p ${OUTDIR}
17 | EVENT_TYPES=`awk -F, '{print $4}' ${INFILE} | sort -u | grep -v cache`
18 | echo event_Types is ${EVENT_TYPES}
19 | for e in ${EVENT_TYPES};
20 | do
21 |     echo egrep -e ${e} -e seconds_since_epoch ${INFILE} \> ${OUTDIR}/${e}
22 |     egrep -e ${e} -e seconds_since_epoch ${INFILE} > ${OUTDIR}/${e}
23 | done
24 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gc_events_by_type.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gc_events_by_type.sh#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | if [ -z "$1" ]; then
 9 |     echo Usage: $0 output-dir
10 |     exit 1
11 | fi
12 | 
13 | OUTPUTDIR=$1
14 | OUTDIR=${OUTPUTDIR}/gc-events-duration-by-event
15 | INFILE=${OUTPUTDIR}/gc-events-duration-in-seconds-only
16 | mkdir -p ${OUTDIR}
17 | EVENT_TYPES=`awk -F, '{print $4}' ${INFILE} | sort -u | grep -v gc_event_type`
18 | echo event_Types is ${EVENT_TYPES}
19 | for e in ${EVENT_TYPES};
20 | do
21 |     echo egrep -e ${e} -e secs_since_epoch ${INFILE} \> ${OUTDIR}/${e}
22 |     egrep -e ${e} -e secs_since_epoch ${INFILE} > ${OUTDIR}/${e}
23 | done
24 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/facet_events_by_country.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/facet_events_by_country.sh#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | if [ -z "$1" ]; then
 9 |     echo Usage: $0 output-dir
10 |     exit 1
11 | fi
12 | 
13 | OUTPUTDIR=$1
14 | OUTDIR=${OUTPUTDIR}/facet-events-by-country
15 | INFILE=${OUTPUTDIR}/vms-cache-refresh-facet-info.csv
16 | mkdir -p ${OUTDIR}
17 | EVENT_TYPES=`awk -F, '{print $3}' ${INFILE} | sort -u | grep -v country`
18 | echo event_Types is ${EVENT_TYPES}
19 | for e in ${EVENT_TYPES};
20 | do
21 |     echo egrep -e ,${e}, -e seconds_since_epoch ${INFILE} \> ${OUTDIR}/${e}
22 |     egrep -e ,${e}, -e seconds_since_epoch ${INFILE} > ${OUTDIR}/${e}
23 | done
24 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_extract_sizes.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl -w
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_extract_sizes.pl#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | use strict;
 9 | 
10 | my $num_args = $#ARGV + 1;
11 | 
12 | if($num_args != 1) {
13 |     die "Usage: $0 output-dir";
14 | }
15 | 
16 | my $outputdir = $ARGV[0];
17 | my $fname = "${outputdir}/gcdotlog_extract_sizes_rejected_lines"; 
18 | open(REJECTS, ">$fname") or die "cannot open $fname for writing";
19 | 
20 | print "datetimestamp,secs_since_jvm_boot,young_begin_k,young_end_k,young_total_k,whole_heap_begin_k,whole_heap_end_k,whole_heap_total_k\n";
21 | while(<STDIN>) {
22 |     my $line = $_;
23 |     chomp($line);
24 | 
25 |     if(0) {
26 |         # make all the lines elsif to make moving blocks easier
27 |     }
28 |     elsif($line =~ m/->.*->.*->/) {
29 |         # reject lines with three arrows
30 |         print REJECTS "$line\n";
31 |     }
32 |     elsif($line =~ m/^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{3}[+]0000): ([0-9]+[.][0-9]{3}): .* ([0-9]+)K->([0-9]+)K\(([0-9]+)K\).* ([0-9]+)K->([0-9]+)K\(([0-9]+)K\)/) {
33 |         print "$1,$2,$3,$4,$5,$6,$7,$8\n";
34 |     }
35 |     else {
36 |         print REJECTS "$line\n";
37 |     }
38 | }
39 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/index:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/index#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Author: mooreb $
 6 | # $Change: 1838706 $
 7 | 
 8 | cd `dirname $0`
 9 | 
10 | prog=`basename $0`
11 | 
12 | NFENV=/etc/profile.d/netflix_environment.sh
13 | if [ -f ${NFENV} ]; then
14 |     . ${NFENV}
15 |     NETFLIX_VMS_EVENTS=checked
16 | else
17 |     NFENV=""
18 | fi
19 | 
20 | cat <<EndOfHeader
21 | Content-Type: text/html
22 | 
23 | <html>
24 | <head>
25 |  <title>AdminGCViz</title>
26 | </head>
27 | <body>
28 | EndOfHeader
29 | 
30 | cat <<EndOfGenerate
31 | Generate a GC visualization report now
32 | <form action="generate" method="POST">
33 |   <input type="checkbox" name="jmap_histo_live" value="on" checked> jmap -histo:live<br>
34 |   <input type="checkbox" name="vms_refresh_events" value="on" ${NETFLIX_VMS_EVENTS}> parse catalina logs looking for netflix VMS events<br>
35 |   <input type="submit" value="Generate GCViz report">
36 | </form>
37 | EndOfGenerate
38 | 
39 | echo Look at previous reports
40 | echo "<ul>"
41 | for f in `find /mnt/logs/gc-reports -type f -name "*.png" | LANG=C sort -rn`;
42 | do
43 |     u=`echo ${f} | sed 's|/mnt/logs/gc-reports/||'`
44 |     echo "<li> <a href=\"/AdminGCVizImages/${u}\">${f}</a>"
45 | done
46 | echo "</ul>"
47 | 
48 | 
49 | cat <<EndOfFooter
50 | </body>
51 | </html>
52 | EndOfFooter
53 | 


--------------------------------------------------------------------------------
/README:
--------------------------------------------------------------------------------
 1 | # Deprecation Note
 2 | 
 3 | Gcviz is no longer maintained and should be considered archived.
 4 | 
 5 | Note that gcviz code is not intended to be deployed on untrusted networks, or without external authentication and authorization. Specifically, it  has functionality that can allow execution of system commands by remote users.
 6 | 
 7 | # Original README
 8 | 
 9 | This is gcviz, a set of programs that help generate visualizations
10 | from gc.log, a log file that the HotSpot, a Java Virual Machine,
11 | writes when configured with the following flags:
12 |   -verbose:gc
13 |   -verbose:sizes
14 |   -Xloggc:/apps/tomcat/logs/gc.log
15 |   -XX:+PrintGCDetails
16 |   -XX:+PrintGCDateStamps
17 |   -XX:+PrintTenuringDistribution
18 | 
19 | gcviz is intended to be used as a webapp when installed on the same
20 | host as tomcat, or any other Java web container. The gcviz program
21 | itself is served by apache httpd inside netflix, but could be served
22 | by any webserver that supports CGI. gcviz implicitly assumes to
23 | be running in a linux environment.
24 | 
25 | By default gcviz is available at:
26 |   http://127.0.0.1:8080/AdminGCViz/index
27 | 
28 | Internally, gcviz is a bundle of four sorts of things:
29 |   * python programs that require matplotlib, numpy, pylab, etc.
30 |   * cgi scripts that invoke these python programs
31 |   * some minor assistive perl scripts
32 |   * very minor rpm infrastructure to package the previous things.
33 | 
34 | I wrote gcviz to address challenges we face inside Netflix. If you
35 | feel that any changes you might propose could be helpful for Netflix
36 | or for the community at large, please write.
37 | 
38 | Brian Moore
39 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_one_event_per_line.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl -w
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_one_event_per_line.pl#3 $
 4 | # $DateTime: 2013/11/05 12:09:10 $
 5 | # $Change: 2024106 $
 6 | # $Author: mooreb $
 7 | 
 8 | use strict;
 9 | 
10 | sub flush_previous_record {
11 |     my ($previous_record) = @_;
12 |     if ("" ne "$previous_record") {
13 |         print "$previous_record\n";
14 |     }
15 | }
16 | 
17 | my $previous_record = "";
18 | while(<STDIN>) {
19 |     my $line = $_;
20 |     chomp($line);
21 |     # -XX:+PrintGCDateStamps
22 |     if($line =~ m/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}/) {
23 |         # starting a new record
24 |         flush_previous_record($previous_record);
25 |         $previous_record = $line;
26 |     }
27 |     elsif($line =~ m/^[0-9]+[.][0-9]+: /) {
28 |         # starting a new record
29 |         flush_previous_record($previous_record);
30 |         $previous_record = $line;
31 |     }
32 |     elsif($line =~ m/(^.*[0-9]+[.][0-9]+ secs])([0-9]+[.][0-9]+: .*$)/) {
33 |         # two records are conflated 
34 |         $previous_record = $previous_record . $1;
35 |         flush_previous_record($previous_record);
36 |         $previous_record = $2;
37 |     }
38 |     elsif($line =~ m/(^.*)([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.*)/) {
39 |         # two records (with -XX:+PrintGCDateStamps) are conflated 
40 |         $previous_record = $previous_record . $1;
41 |         flush_previous_record($previous_record);
42 |         $previous_record = $2;
43 |     }
44 |     else {
45 |         # append to the previous record
46 |         $previous_record = $previous_record . $line;
47 |     }
48 | }
49 | flush_previous_record($previous_record);
50 | 
51 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/parse-proc-pid-maps.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python2.7
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/parse-proc-pid-maps.py#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | import sys
 9 | 
10 | def bytesToHumanReadable(n):
11 |     kay = 1024
12 |     meg = kay*1024
13 |     gig = meg*1024
14 |     if ((0 <= n) and (n < kay)):
15 |         return "%d bytes" % n
16 |     elif ((kay <= n) and (n < meg)):
17 |         return "%.2fkb" % ((n+0.0)/kay)
18 |     elif ((meg <= n) and (n < gig)):
19 |         return "%.2fmb" % ((n+0.0)/meg)
20 |     else:
21 |         return "%.2fgb" % ((n+0.0)/gig)
22 | 
23 | def readfile(filename):
24 |     segments = []
25 |     numSegments=0L
26 |     totalBytes=0L
27 |     infile = open(filename, 'r')
28 |     line = infile.readline()
29 |     while line:
30 |         line = line.rstrip()
31 |         fields = line.split(None, 5)
32 |         if 6 == len(fields):
33 |             (memRange, perms, offset, dev, inode, pathname) = fields
34 |         elif 5 == len(fields):
35 |             (memRange, perms, offset, dev, inode) = fields
36 |             pathname = ''
37 |         else:
38 |             raise Exception('cannot unpack %s' % (line,))
39 |         (begin,end) = memRange.split('-')
40 |         t = long(end, 16) - long(begin, 16)
41 |         totalBytes = totalBytes + t
42 |         numSegments = numSegments + 1
43 |         x = (t, memRange, pathname)
44 |         segments.append(x)
45 |         line = infile.readline()
46 |     print filename
47 |     print "\tnum segments = %s" % (numSegments,)
48 |     print "\ttotal bytes = %s (%s)" % (totalBytes, bytesToHumanReadable(totalBytes))
49 |     for i in sorted(segments, key=lambda x: x[0], reverse=True):
50 |         print "\t\t%12s bytes (%s) %33s %s" % (i[0], bytesToHumanReadable(i[0]), i[1], i[2])
51 | 
52 | readfile(sys.argv[1])
53 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/vms_facet_info_transform.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python2.7
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/vms_facet_info_transform.py#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | import fileinput
 9 | import re
10 | 
11 | # Ugh.
12 | sys.path.insert(0, "/apps/apache/htdocs/AdminGCViz")
13 | import vmsgcvizutils   
14 | 
15 | # Input:
16 | # 2012-03-30 00:47:11,771 country(GF) cache(VideoImages) numItems(24189) totalTime(1397531) timeToCopyToDisc(5550) timeToFill(1391981)
17 | # ...
18 | #
19 | # Output
20 | # seconds_since_epoch,datetimestamp,country,cache,numitems,totalTime,timeToCopyToDisc,timeToFill
21 | # 1333068431.771,2012-03-30T00:47:11.771+0000,GF,VideoImages,24189,1397531,5550,1391981
22 | # ...
23 | 
24 | facetPattern = re.compile('^([0-9]{4}-[0-9]{2}-[0-9]{2}) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]{3}) country\(([A-Z]{2})\) cache\(([^)]+)\) numItems\(([0-9]+)\) totalTime\(([0-9]+)\) timeToCopyToDisc\(([0-9]+)\) timeToFill\(([0-9]+)\)$')
25 | 
26 | print 'seconds_since_epoch,datetimestamp,country,cache,numitems,totalTime,timeToCopyToDisc,timeToFill'
27 | for line in fileinput.input('-'):
28 |     line = line.rstrip('\r\n')
29 |     found = facetPattern.search(line)
30 |     if found:
31 |         ymd = found.group(1)
32 |         hms = found.group(2)
33 |         milliseconds = found.group(3)
34 |         iso8601Timestamp = '%sT%s.%s+0000' % (ymd,hms,milliseconds)
35 |         secsSinceEpoch = vmsgcvizutils.timestamp_to_epoch(iso8601Timestamp)
36 |         country = found.group(4)
37 |         cache = found.group(5)
38 |         numItems = found.group(6)
39 |         totalTime = found.group(7)
40 |         timeToCopyToDisc = found.group(8)
41 |         timeToFill = found.group(9)
42 |         print '%s,%s,%s,%s,%s,%s,%s,%s' % (secsSinceEpoch, iso8601Timestamp, country, cache, numItems, totalTime, timeToCopyToDisc, timeToFill)
43 |     else:
44 |         sys.stderr.write(line)
45 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/vmsgcvizutils.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python2.7
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/vmsgcvizutils.py#3 $
 4 | # $DateTime: 2013/11/12 19:42:41 $
 5 | # $Change: 2030932 $
 6 | # $Author: mooreb $
 7 | 
 8 | import calendar
 9 | import math
10 | import time
11 | import re
12 | 
13 | def timestamp_to_epoch(timeStringISO8601):
14 |     # Potential BUG: +0000 is hardcoded. I'd like to use %z but
15 |     # cannot, as it's not supported by time.strptime and there's no
16 |     # easy workaround: http://wiki.python.org/moin/WorkingWithTime
17 |     secondFractionPattern = re.compile('([.][0-9]+)[+]0000')
18 |     match = secondFractionPattern.search(timeStringISO8601)
19 |     if match:
20 |         fraction = match.group(1)
21 |     else:
22 |         fraction = ""
23 |     timeStringISO8601 = timeStringISO8601.replace(fraction, '')
24 |     # Potential BUG: +0000 is hardcoded. I'd like to use %z but
25 |     # cannot, as it's not supported by time.strptime and there's no
26 |     # easy workaround: http://wiki.python.org/moin/WorkingWithTime
27 |     bootTimeTuple = time.strptime(timeStringISO8601, "%Y-%m-%dT%H:%M:%S+0000")
28 |     bootTimeSecondsSinceEpoch = "%s" % calendar.timegm(bootTimeTuple)
29 |     return (bootTimeSecondsSinceEpoch + fraction)
30 | 
31 | def convertTimeStamp(absoluteBaselineTime, secondsAfterBaseline):
32 |     offsetTime = absoluteBaselineTime + secondsAfterBaseline
33 |     offsetTimeTuple = time.gmtime(offsetTime)
34 |     # Potential BUG: +0000 is hardcoded. I'd like to use %z but
35 |     # cannot, as it's not supported by time.strptime and there's no
36 |     # easy workaround: http://wiki.python.org/moin/WorkingWithTime
37 |     offsetTimeString = time.strftime("%Y-%m-%dT%H:%M:%S+0000", offsetTimeTuple)
38 |     fractionSecondsString = '%.3f' % (secondsAfterBaseline - math.floor(secondsAfterBaseline))
39 |     retval = offsetTimeString.replace('+', fractionSecondsString[1:] + '+')
40 |     return retval
41 | 
42 | 
43 | def envFileAsDictionary(fname):
44 |     retval = {}
45 |     fp = open(fname, 'r')
46 |     for line in fp:
47 |         line = line.rstrip('\r\n')
48 |         try:
49 |             (k, v) = line.split('=', 1)
50 |         except ValueError:
51 |             continue
52 |         retval[k] = v
53 |     return retval
54 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/BUGS:
--------------------------------------------------------------------------------
 1 | This is more of a todo list than things that are deeply wrong, but I want to make the fringes that I know about public.
 2 | 
 3 | * all of the BUGs and Potential BUGs in all of the sources
 4 | * visualize-cluster does not work
 5 | * compute throughput, allocation rate from gc data (do this with PrintGCStats?)
 6 | * sar data (cpu, etc) needs to be visualized
 7 | 
 8 | 
 9 | 
10 | * netflix internal: need to visualize facet data
11 | * netflix internal: need to ensure that clients output vms cache refresh event overall and facet-level timing
12 |     * gps
13 |     * api
14 |     * merchweb
15 |     * ecweb (curently no facet-level timing for demand-fill)
16 |     * accountweb (curently no facet-level timing for demand-fill)
17 |     * ...
18 | * netflix-internal: delta fail needs to be added to catalina parsing (right now all lines are plotted as begin/end overall)
19 | * netflix-internal, maybe outside too: truncated gc logs (when ec2rotate logs purges stuff older than 7 days creates "unknown" gc events.
20 | * netflix-internal: have if -z checks for status-properties-1, -2. delete empty files. don't do both if we get the output for one. Consider getting URL from discovery/entrypoints.
21 | * netflix-internal: the squirreled away vms-gc-reports location is <appname>/<date>/<data> and doesn't have instance id in the path... This could be a problem for visualize-cluster.
22 | * netflix-internal: Consider grabbing some number of recent ttime and threaddump files from /apps/tomcat/logs/cores
23 |     drwxrwsr-x 2 root         nac   94208 Apr 17 23:44 .
24 |     -rw-r--r-- 1 merchwebprod nac  168050 Apr 17 23:44 ttime.20120417.234401.1586.txt
25 |     lrwxrwxrwx 1 merchwebprod nac      59 Apr 17 23:44 latest -> /apps/tomcat/logs/cores/threaddump.20120417.234401.1586.txt
26 |     -rw-r--r-- 1 merchwebprod nac 1117242 Apr 17 23:44 threaddump.20120417.234401.1586.txt
27 |     -rw-r--r-- 1 merchwebprod nac  168251 Apr 17 23:34 ttime.20120417.233401.1586.txt
28 |     -rw-r--r-- 1 merchwebprod nac 1119200 Apr 17 23:34 threaddump.20120417.233401.1586.txt
29 |     -rw-r--r-- 1 merchwebprod nac  168371 Apr 17 23:24 ttime.20120417.232401.1586.txt
30 |     -rw-r--r-- 1 merchwebprod nac 1120525 Apr 17 23:24 threaddump.20120417.232401.1586.txt
31 |     -rw-r--r-- 1 merchwebprod nac  168490 Apr 17 23:15 ttime.20120417.231401.1586.txt
32 | * netflix-internal: visualize objectCache lines in ${OUTPUTDIR}/vms-object-cache-stats
33 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_convert_relative_into_absolute_time.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python2.7
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_convert_relative_into_absolute_time.py#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | import calendar
 9 | import fileinput
10 | import math
11 | import os
12 | import sys
13 | import time
14 | import re
15 | 
16 | # Ugh.
17 | sys.path.insert(0, "/apps/apache/htdocs/AdminGCViz")
18 | import vmsgcvizutils   
19 | 
20 | def mayne():
21 |     TMPFILE = '/tmp/gcdotlog-relative-time-tmpfile'
22 |     numArgs = len(sys.argv) - 1
23 |     if(2 != numArgs):
24 |         print "Usage: %s {iso8601 combined date and time representations} outputdir" % (sys.argv[0],)
25 |         sys.exit(1);
26 | 
27 |     bootTimeStringISO8601 = sys.argv[1]
28 |     bootTimeSecondsSinceEpochString = vmsgcvizutils.timestamp_to_epoch(bootTimeStringISO8601)
29 |     bootTimeSecondsSinceEpoch = float(bootTimeSecondsSinceEpochString)
30 |     outputDir = sys.argv[2]
31 |     bootTimeEpochFile = open(outputDir + '/jvm_boottime.epoch', 'w')
32 |     bootTimeEpochFile.write("%s\n" % bootTimeSecondsSinceEpochString)
33 |     bootTimeEpochFile.close()
34 |     
35 |     tmpFile = open(TMPFILE, 'w')
36 |     lineStartsWithFloatingPointNumberPattern = re.compile("^([0-9]+[.][0-9]+): ")
37 |     lastSecsSinceBoot = 0.0;
38 |     for line in fileinput.input('-'):
39 |         line = line.rstrip('\r\n')
40 |         found = lineStartsWithFloatingPointNumberPattern.search(line)
41 |         if found:
42 |             secsSinceBootString = found.group(1)
43 |             secsSinceBoot = float(secsSinceBootString)
44 |             if secsSinceBoot < lastSecsSinceBoot:
45 |                 # now we need to truncate the output file, since we have
46 |                 # seen a restart; this is not the most recent JVM boot
47 |                 tmpFile.close()
48 |                 tmpFile = open(TMPFILE, 'w')
49 |             lastSecsSinceBoot = secsSinceBoot
50 |             timeStamp = vmsgcvizutils.convertTimeStamp(bootTimeSecondsSinceEpoch, secsSinceBoot)
51 |             tmpFile.write("%s: %s\n" % (timeStamp, line))
52 |         else:
53 |             tmpFile.write("%s\n" % (line,))
54 | 
55 |     tmpFile.close()
56 |     for line in fileinput.input(TMPFILE):
57 |         line = line.rstrip('\r\n')
58 |         print "%s" % (line,)
59 | 
60 |     os.unlink(TMPFILE)
61 | 
62 | if __name__ == "__main__":
63 |     mayne()
64 | 


--------------------------------------------------------------------------------
/INSTALL:
--------------------------------------------------------------------------------
 1 | This document will attempt to describe installation of gcviz.
 2 | 
 3 | This process is not entirely straightforward and assumes familiarity
 4 | with your OS and webserver.
 5 | 
 6 | The rest of this document assumes a *nix-style system and apache httpd.
 7 | 
 8 | (1) Find the folder: root/apps/apache/htdocs/AdminGCViz
 9 | 
10 | (2) Copy this folder into your htdocs root. Assure the permissions are something like:
11 | 
12 | # ls -l /apps/apache/htdocs/AdminGCViz/
13 | total 60
14 | -rw-r--r-- 1 youruser yourgroup  2299 Mar 10 00:12 BUGS
15 | -rw-r--r-- 1 youruser yourgroup   733 Mar 10 00:12 gc_event_types
16 | -rwxr-xr-x 1 youruser yourgroup  5353 Mar 10 00:12 generate
17 | -rwxr-xr-x 1 youruser yourgroup  1253 Mar 10 00:12 index
18 | -rw-r--r-- 1 youruser yourgroup  2866 Mar 10 00:12 README
19 | drwxr-xr-x 2 youruser yourgroup  4096 Mar 14 22:22 remote-data-collection
20 | -rwxr-xr-x 1 youruser yourgroup  1670 Mar 10 00:12 visualize-cluster.py
21 | -rwxr-xr-x 1 youruser yourgroup  3478 Mar 10 00:12 visualize-facets.py
22 | -rwxr-xr-x 1 youruser yourgroup 10463 Mar 10 00:12 visualize-gc.py
23 | -rwxr-xr-x 1 youruser yourgroup  4101 Mar 10 00:12 visualize-instance.sh
24 | -rwxr-xr-x 1 youruser yourgroup  2071 Mar 10 00:12 vmsgcvizutils.py
25 | 
26 |  ls -l /apps/apache/htdocs/AdminGCViz/remote-data-collection/
27 | total 52
28 | -rwxr-xr-x 1 youruser yourgroup 7776 Mar 10 00:12 collect_remote_data.sh
29 | -rwxr-xr-x 1 youruser yourgroup  707 Mar 10 00:12 facet_events_by_country.sh
30 | -rwxr-xr-x 1 youruser yourgroup  696 Mar 10 00:12 facet_events_by_type.sh
31 | -rwxr-xr-x 1 youruser yourgroup 2209 Mar 10 00:12 gcdotlog_convert_relative_into_absolute_time.py
32 | -rwxr-xr-x 1 youruser yourgroup 1177 Mar 10 00:12 gcdotlog_extract_sizes.pl
33 | -rwxr-xr-x 1 youruser yourgroup 2524 Mar 10 00:12 gcdotlog_extract_time.pl
34 | -rwxr-xr-x 1 youruser yourgroup 1554 Mar 10 00:12 gcdotlog_one_event_per_line.pl
35 | -rwxr-xr-x 1 youruser yourgroup  703 Mar 10 00:12 gc_events_by_type.sh
36 | -rwxr-xr-x 1 youruser yourgroup 1691 Mar 10 00:12 parse-proc-pid-maps.py
37 | -rwxr-xr-x 1 youruser yourgroup  556 Mar 10 00:12 prepend_epoch.py
38 | -rwxr-xr-x 1 youruser yourgroup 2778 Mar 10 00:12 process_vms_object_cache_stats.py
39 | -rwxr-xr-x 1 youruser yourgroup 1792 Mar 10 00:12 vms_facet_info_transform.py
40 | 
41 | (3) Find the config file: root/apps/apache/conf.d/admin_gc_viz.conf
42 | 
43 | (4) Use the config file from (3) as a guide to configure your local
44 | apache. It is intended to be Include'able (in the apache sense) but
45 | please don't blindly include it without understanding what it does and
46 | how it interacts with your other apache configuration.
47 | 
48 | ADDITIONAL NOTES
49 |   * you may need to ensure that the shebang line is correct for your environment. The shebang lines assume a netflix environment.
50 |   * the included rpm .spec file: conf/rpm.spec may be helpful to you; YMMV
51 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_extract_time.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl -w
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/gcdotlog_extract_time.pl#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | use strict;
 9 | 
10 | sub classify_gc_event_type {
11 |     my ($line) = @_;
12 | 
13 |     if(0) {
14 |         # make all the lines "elsif" so that they are interchangable
15 |     }
16 |     elsif($line =~ m/Full GC/) {
17 |         return "FullGC";
18 |     }
19 |     elsif($line =~ m/\(concurrent mode failure\)/) {
20 |         return "concurrent-mode-failure";
21 |     }
22 |     elsif($line =~ m/\(promotion failed\)/) {
23 |         return "promotion-failed";
24 |     }
25 |     elsif($line =~ m/ParNew/) {
26 |         return "ParNew";
27 |     }
28 |     elsif($line =~ m/CMS-initial-mark/) {
29 |         return "CMS-initial-mark";
30 |     }
31 |     elsif($line =~ m/CMS-concurrent-mark/) {
32 |         return "CMS-concurrent-mark";
33 |     }
34 |     elsif($line =~ m/CMS-concurrent-abortable-preclean/) {
35 |         return "CMS-concurrent-abortable-preclean";
36 |     }
37 |     elsif($line =~ m/CMS-concurrent-preclean/) {
38 |         return "CMS-concurrent-preclean";
39 |     }
40 |     elsif($line =~ m/CMS-remark/) {
41 |         return "CMS-remark";
42 |     }
43 |     elsif($line =~ m/CMS-concurrent-sweep/) {
44 |         return "CMS-concurrent-sweep";
45 |     }
46 |     elsif($line =~ m/CMS-concurrent-reset/) {
47 |         return "CMS-concurrent-reset";
48 |     }
49 |     elsif($line =~ /PSYoungGen/) {
50 |         return "ParallelScavengeYoungGen";
51 |     }
52 |     elsif($line =~ /DefNew/) {
53 |         return "DefNew";
54 |     }
55 |     else {
56 |         return "unknown";
57 |     }
58 | }
59 | 
60 | sub mayne {
61 |     my $num_args = $#ARGV + 1;
62 | 
63 |     if($num_args != 1) {
64 |         die "Usage: $0 output-dir";
65 |     }
66 | 
67 |     my $outputdir = $ARGV[0];
68 |     my $fname = "${outputdir}/gcdotlog_extract_time_rejected_lines"; 
69 |     open(FP, ">$fname") or die "cannot open $fname";
70 | 
71 |     print "datetimestamp,secs_since_jvm_boot,gc_event_type,gc_event_duration_in_seconds\n";
72 |     while(<STDIN>) {
73 |         my $line = $_;
74 |         chomp($line);
75 |         # -XX:+PrintGCDateStamps
76 |         if($line =~ m/^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{3}[+]0000): ([0-9]+[.][0-9]+): .*real=([0-9][0-9]*[.][0-9][0-9]*) secs\]\s*$/) {
77 |             my $datestamp = $1;
78 |             my $secs_since_jvm_boot = $2;
79 |             my $gctime_in_seconds = $3;
80 |             my $gc_event_type = classify_gc_event_type($line);
81 |             print "${datestamp},${secs_since_jvm_boot},${gc_event_type},${gctime_in_seconds}\n";
82 |         }
83 |         else {
84 |             print FP "$line\n";
85 |         }
86 |     }
87 |     close(FP);
88 | }
89 | 
90 | mayne()
91 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/README:
--------------------------------------------------------------------------------
 1 | * This program is split into thee conceptual parts:
 2 |   * a top-level "driver" (visualize-instance.sh and visualize-cluster.py)
 3 |   * a remote data collection component (remote-data-collection/collect_remote_data.sh)
 4 |   * a visualization component (visualize-gc.py)
 5 | 
 6 | It's difficult to get a cross-platform visualization component, so I
 7 | opted for python's matplotlib, which is cross platform and has a
 8 | single-click installer available for windows, macintosh and linux.
 9 | 
10 | * To visialize the data you'lll need python's matplotlib. One
11 |   simple-to-install (and free) distribution that contains this is EPD:
12 |     http://www.enthought.com/repo/free/
13 |   This will (attempt) to patch your .profile equivalent to place itself first on your PATH.
14 | 
15 | * A note about how GC events are parsed. This software does not, at
16 |   the time of this writing, use PrintGCFixup. Instead it uses the
17 |   -XX:+PrintGCDateStamps datetime stamps (if available, secs since vm
18 |   boot if not) as an anchor, and treats each of those things as an
19 |   event.
20 | 
21 | In this context:
22 | 2012-04-04T19:07:40.395+0000: 510958.888: [GC [1 CMS-initial-mark: 18431999K(18432000K)] 18939679K(29491200K), 0.5050890 secs] [Times: user=0.50 sys=0.00, real=0.50 secs]
23 | 2012-04-04T19:07:40.903+0000: 510959.397: [CMS-concurrent-mark-start]
24 | 2012-04-04T19:07:56.564+0000: 510975.058: [CMS-concurrent-mark: 15.410/15.661 secs] [Times: user=49.94 sys=1.89, real=15.66 secs]
25 | 2012-04-04T19:07:56.565+0000: 510975.058: [CMS-concurrent-preclean-start]
26 | 2012-04-04T19:08:23.054+0000: 511001.548: [Full GC 511001.549: [CMS2012-04-04T19:08:48.906+0000: 511027.400: [CMS-concurrent-preclean: 51.957/52.341 secs] [Times: user=76.72 sys=0.15, real=52.34 secs]
27 |  (concurrent mode failure): 18431999K->16174249K(18432000K), 106.0788490 secs] 29491199K->16174249K(29491200K), [CMS Perm : 69005K->69005K(115372K)], 106.0801410 secs] [Times: user=106.01 sys=0.00, real=106.06 secs]
28 | 2012-04-04T19:10:09.150+0000: 511107.644: [GC [1 CMS-initial-mark: 16174249K(18432000K)] 16363184K(29491200K), 0.0263250 secs] [Times: user=0.02 sys=0.00, real=0.03 secs]
29 | 
30 | GC events of this form:
31 | 2012-04-04T19:08:23.054+0000: 511001.548: [Full GC 511001.549: [CMS2012-04-04T19:08:48.906+0000: 511027.400: [CMS-concurrent-preclean: 51.957/52.341 secs] [Times: user=76.72 sys=0.15, real=52.34 secs]
32 |  (concurrent mode failure): 18431999K->16174249K(18432000K), 106.0788490 secs] 29491199K->16174249K(29491200K), [CMS Perm : 69005K->69005K(115372K)], 106.0801410 secs] [Times: user=106.01 sys=0.00, real=106.06 secs]
33 | 
34 | are represented as a Full GC requiring 106 seconds rather than the
35 | CMS-preclean part of ~52 seconds. I'm not sure if ~106 or 106-52 is
36 | the stop-the-world part, but at that long of a pause, I'm not entirely
37 | convinced that it matters. Bill Jackson votes that 106-52 is the stop
38 | the world part. He's probably right. I'm surprised that the preclean
39 | wasn't aborted.
40 | 
41 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/process_vms_object_cache_stats.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python2.7
 2 | 
 3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/process_vms_object_cache_stats.py#2 $
 4 | # $DateTime: 2013/05/15 18:34:23 $
 5 | # $Change: 1838706 $
 6 | # $Author: mooreb $
 7 | 
 8 | import fileinput
 9 | import os
10 | import re
11 | import sys
12 | 
13 | # Ugh.
14 | sys.path.insert(0, "/apps/apache/htdocs/AdminGCViz")
15 | import vmsgcvizutils   
16 | 
17 | numArgs = len(sys.argv) - 1
18 | if(2 != numArgs):
19 |     print "Usage: %s object-cache-stats-file outputdir" % (sys.argv[0],)
20 |     sys.exit(1);
21 | 
22 | objectCacheStatsFile = sys.argv[1]
23 | baseOutputDir = sys.argv[2]
24 | outputDir = baseOutputDir + os.path.sep + 'vms-object-cache-stats-by-cache'
25 | os.mkdir(outputDir)
26 | outputFiles = {}
27 | rejects = open(objectCacheStatsFile + '.rejects', 'w')
28 | 
29 | timestampPattern = re.compile('^([0-9]{4}-[0-9]{2}-[0-9]{2}) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]{3}) INFO (main|vms-timer-refresh) VMClientCacheManager - Processed Countries')
30 | objectCachePattern = re.compile('objectCache\(([^)]*)\) references\(([^)]*)\) size\(([^)]*)\) ratio\(([^)]*)\) prevsize\(([^)]*)\) additions\(([^)]*)\) transfers\(([^)]*)\) hits\(([^)]*)\) orphans\(([^)]*)\)')
31 | 
32 | header="secsSinceEpoch,iso8601Timestamp,references,size,ratio,prevsize,additions,transfers,hits,orphans\n"
33 | 
34 | iso8601Timestamp = "1970-01-01T00:00:00.000+0000"
35 | secsSinceEpoch = "0.000"
36 | for line in fileinput.input(objectCacheStatsFile):
37 |     line = line.rstrip('\r\n')
38 |     foundTimestamp = timestampPattern.search(line)
39 |     if foundTimestamp:
40 |         ymd = foundTimestamp.group(1)
41 |         hms = foundTimestamp.group(2)
42 |         milliseconds = foundTimestamp.group(3)
43 |         iso8601Timestamp = '%sT%s.%s+0000' % (ymd,hms,milliseconds)
44 |         secsSinceEpoch = vmsgcvizutils.timestamp_to_epoch(iso8601Timestamp)
45 |         continue
46 | 
47 |     foundObjectCache = objectCachePattern.search(line)
48 |     if foundObjectCache:
49 |         cacheName  = foundObjectCache.group(1)
50 |         references = foundObjectCache.group(2)
51 |         size       = foundObjectCache.group(3)
52 |         ratio      = foundObjectCache.group(4)
53 |         prevsize   = foundObjectCache.group(5)
54 |         additions  = foundObjectCache.group(6)
55 |         transfers  = foundObjectCache.group(7)
56 |         hits       = foundObjectCache.group(8)
57 |         orphans    = foundObjectCache.group(9)
58 |         l = "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n" % (secsSinceEpoch,iso8601Timestamp,references,size,ratio,prevsize,additions,transfers,hits,orphans)
59 |         f = outputFiles.get(cacheName)
60 |         if f:
61 |             f.write(l)
62 |         else:
63 |             f = open(outputDir + os.path.sep + cacheName, 'w')
64 |             f.write(header)
65 |             f.write(l)
66 |             outputFiles[cacheName] = f
67 |         continue
68 |     else:
69 |         rejects.write("%s\n" % (line,))
70 | 
71 | rejects.close()
72 | for fp in outputFiles.values():
73 |     fp.close()
74 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/visualize-facets.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python2.7
  2 | 
  3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/visualize-facets.py#2 $
  4 | # $DateTime: 2013/05/15 18:34:23 $
  5 | # $Change: 1838706 $
  6 | # $Author: mooreb $
  7 | 
  8 | import numpy as np
  9 | import matplotlib.pyplot as plt
 10 | import matplotlib.mlab as mlab
 11 | import matplotlib.ticker as ticker
 12 | import matplotlib.dates as mdates
 13 | import matplotlib.lines as lines
 14 | import pylab
 15 | import sys
 16 | import os
 17 | import vmsgcvizutils
 18 | from mpl_toolkits.mplot3d import Axes3D
 19 | 
 20 | numArgs = len(sys.argv) - 1
 21 | if(2 != numArgs):
 22 |     print "Usage: %s now:iso8601timestamp vms-gc-report-directory" % (sys.argv[0],)
 23 |     sys.exit(1);
 24 | 
 25 | now = sys.argv[1]
 26 | vmsGCReportDirectory = sys.argv[2]
 27 | 
 28 | # seconds_since_epoch,datetimestamp,country,cache,numitems,totalTime,timeToCopyToDisc,timeToFill
 29 | # 1333073736.242,2012-03-30T02:15:36.242+0000,BS,VideoEDFulfillmentData,23242,1352,757,595
 30 | fnameFullPath = vmsGCReportDirectory + os.path.sep + 'vms-cache-refresh-facet-info.csv'
 31 | recordset = mlab.csv2rec(fnameFullPath)
 32 | 
 33 | countries = recordset.country
 34 | countriesDict = {}
 35 | countriesList = []
 36 | countryNum = 0
 37 | for c in countries:
 38 |     cPrime = countriesDict.get(c)
 39 |     if cPrime:
 40 |         countriesList.append(cPrime)
 41 |     else:
 42 |         countryNum = countryNum + 1
 43 |         countriesDict[c] = countryNum
 44 |         countriesList.append(countryNum)
 45 | 
 46 | caches = recordset.cache
 47 | cachesDict = {}
 48 | cachesList = []
 49 | cacheNum = 0
 50 | for c in caches:
 51 |     cPrime = cachesDict.get(c)
 52 |     if cPrime:
 53 |         cachesList.append(cPrime)
 54 |     else:
 55 |         cacheNum = cacheNum + 1
 56 |         cachesDict[c] = cacheNum
 57 |         cachesList.append(cacheNum)
 58 | 
 59 | allTimeSpentInAllFacets = 0
 60 | perFacetRecordSets = []
 61 | facetEventByCacheDir = vmsGCReportDirectory + os.path.sep + 'facet-events-by-cache'
 62 | dirList=os.listdir(facetEventByCacheDir)
 63 | for fname in dirList:
 64 |     fnameFullPath = facetEventByCacheDir + os.path.sep + fname
 65 |     r = mlab.csv2rec(fnameFullPath)
 66 |     allTimeSpentInFacet = sum(r.totaltime)
 67 |     allTimeSpentInAllFacets = allTimeSpentInAllFacets + allTimeSpentInFacet
 68 |     timeToCopyFacetToDisc = sum(r.timetocopytodisc)
 69 |     timeToFillFacet = sum(r.timetofill)
 70 |     d = {'totaltime' : allTimeSpentInFacet, 
 71 |          'copytime'  : timeToCopyFacetToDisc,
 72 |          'filltime'  : timeToFillFacet,
 73 |          'facetName' : fname, 
 74 |          'recordset' : r}
 75 |     perFacetRecordSets.append(d)
 76 | perFacetRecordSets.sort(reverse=True, key=lambda d: d['totaltime']) # sort by all time spent in facet
 77 | 
 78 | facetReportFileName = vmsGCReportDirectory + os.path.sep + 'facet-report.txt'
 79 | facetReportFP = open(facetReportFileName, 'w')
 80 | for r in perFacetRecordSets:
 81 |     timeThisFacet = r['totaltime']
 82 |     s = '%10s milliseconds spent in %25s (%5.2f%%) {copy: %5.2f%%; fill: %5.2f%%}' % (
 83 |         timeThisFacet, 
 84 |         r['facetName'], 
 85 |         ((timeThisFacet*100.0)/allTimeSpentInAllFacets), 
 86 |         (r['copytime']*100.0/allTimeSpentInAllFacets), 
 87 |         (r['filltime']*100.0/allTimeSpentInAllFacets),
 88 |         )
 89 |     facetReportFP.write("%s\n" % (s,))
 90 |     print s
 91 | facetReportFP.close()
 92 | 
 93 | # BUG
 94 | sys.exit(0)
 95 | 
 96 | fig = plt.figure()
 97 | ax = fig.gca(projection='3d')
 98 | ax.plot(countriesList, recordset.totaltime/1000, zs=cachesList, zdir='z', marker='o')
 99 | 
100 | ax.set_xlabel('country')
101 | ax.set_ylabel('time to fill this country/facet (seconds)')
102 | ax.set_zlabel('facet')
103 | ax.set_title('total time to fill each facet for each country')
104 | 
105 | # BUG: save all generated figures.
106 | 
107 | plt.show()
108 | 
109 | # sort by total time descending:
110 | #   sort -t , -k 6 -rn vms-cache-refresh-facet-info.csv | more
111 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/generate:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/generate#3 $
  4 | # $DateTime: 2013/05/15 18:34:23 $
  5 | # $Author: mooreb $
  6 | # $Change: 1838706 $
  7 | 
  8 | if [[ "POST" != "${REQUEST_METHOD}" ]] ; then
  9 |     cat <<END_OF_405
 10 | Status: 405 Method Not Allowed
 11 | Content-type: text/html
 12 | 
 13 | <HTML>
 14 | <HEAD><TITLE>405 Method Not Allowed</TITLE></HEAD>
 15 | <BODY>
 16 |   <H1>Error</H1>
 17 |   <P>generate cannot be called with anything but a POST</P>
 18 | </BODY>
 19 | </HTML>
 20 | END_OF_405
 21 |         exit 0;
 22 | fi
 23 | 
 24 | ###############################################################################################################################################
 25 | ## 
 26 | ## lifted from http://oinkzwurgl.org/bash_cgi thank you Philippe 
 27 | ## 
 28 | # (internal) routine to store POST data
 29 | function cgi_get_POST_vars()
 30 | {
 31 |     # check content type
 32 |     # FIXME: not sure if we could handle uploads with this..
 33 |     [ "${CONTENT_TYPE}" != "application/x-www-form-urlencoded" ] && \
 34 |     echo "bash.cgi warning: you should probably use MIME type "\
 35 |          "application/x-www-form-urlencoded!" 1>&2
 36 |     # save POST variables (only first time this is called)
 37 |     [ -z "$QUERY_STRING_POST" \
 38 |       -a "$REQUEST_METHOD" = "POST" -a ! -z "$CONTENT_LENGTH" ] && \
 39 |         read -n $CONTENT_LENGTH QUERY_STRING_POST
 40 |     # prevent shell execution
 41 |     local t
 42 |     t=${QUERY_STRING_POST//%60//} # %60 = `
 43 |     t=${t//\`//}
 44 |     t=${t//\$(//}
 45 |     QUERY_STRING_POST=${t}
 46 |     return
 47 | }
 48 | 
 49 | # (internal) routine to decode urlencoded strings
 50 | function cgi_decodevar()
 51 | {
 52 |     [ $# -ne 1 ] && return
 53 |     local v t h
 54 |     # replace all + with whitespace and append %%
 55 |     t="${1//+/ }%%"
 56 |     while [ ${#t} -gt 0 -a "${t}" != "%" ]; do
 57 |     v="${v}${t%%\%*}" # digest up to the first %
 58 |     t="${t#*%}"       # remove digested part
 59 |     # decode if there is anything to decode and if not at end of string
 60 |     if [ ${#t} -gt 0 -a "${t}" != "%" ]; then
 61 |         h=${t:0:2} # save first two chars
 62 |         t="${t:2}" # remove these
 63 |         v="${v}"`echo -e \\\\x${h}` # convert hex to special char
 64 |     fi
 65 |     done
 66 |     # return decoded string
 67 |     echo "${v}"
 68 |     return
 69 | }
 70 | 
 71 | # routine to get variables from http requests
 72 | # usage: cgi_getvars method varname1 [.. varnameN]
 73 | # method is either GET or POST or BOTH
 74 | # the magic varible name ALL gets everything
 75 | function cgi_getvars()
 76 | {
 77 |     [ $# -lt 2 ] && return
 78 |     local q p k v s
 79 |     # prevent shell execution
 80 |     t=${QUERY_STRING//%60//} # %60 = `
 81 |     t=${t//\`//}
 82 |     t=${t//\$(//}
 83 |     QUERY_STRING=${t}
 84 |     # get query
 85 |     case $1 in
 86 |     GET)
 87 |         [ ! -z "${QUERY_STRING}" ] && q="${QUERY_STRING}&"
 88 |         ;;
 89 |     POST)
 90 |         cgi_get_POST_vars
 91 |         [ ! -z "${QUERY_STRING_POST}" ] && q="${QUERY_STRING_POST}&"
 92 |         ;;
 93 |     BOTH)
 94 |         [ ! -z "${QUERY_STRING}" ] && q="${QUERY_STRING}&"
 95 |         cgi_get_POST_vars
 96 |         [ ! -z "${QUERY_STRING_POST}" ] && q="${q}${QUERY_STRING_POST}&"
 97 |         ;;
 98 |     esac
 99 |     shift
100 |     s=" $* "
101 |     # parse the query data
102 |     while [ ! -z "$q" ]; do
103 |     p="${q%%&*}"  # get first part of query string
104 |     k="${p%%=*}"  # get the key (variable name) from it
105 |     v="${p#*=}"   # get the value from it
106 |     q="${q#$p&*}" # strip first part from query string
107 |     # decode and evaluate var if requested
108 |     [ "$1" = "ALL" -o "${s/ $k /}" != "$s" ] && \
109 |         eval "$k=\"`cgi_decodevar \"$v\"`\""
110 |     done
111 |     return
112 | }
113 | 
114 | cgi_getvars POST ALL
115 | 
116 | ##
117 | ##
118 | ##
119 | ###############################################################################################################################################
120 | 
121 | #cat <<END_OF_EARLY_RETURN
122 | #Status: 200 OK
123 | #Content-type: text/html
124 | #
125 | #<HTML>
126 | #<HEAD><TITLE>200 OK: early return</TITLE></HEAD>
127 | #<BODY>
128 | #jmap_histo_live is ${jmap_histo_live}
129 | #vms_refresh_events is ${vms_refresh_events}
130 | #</BODY>
131 | #</HTML>
132 | #END_OF_EARLY_RETURN
133 | #exit 0
134 | 
135 | if [ -z "$jmap_histo_live" ]; then
136 |     jmap_histo_live="nojmap_histo_live"
137 | else
138 |     jmap_histo_live="jmap_histo_live"
139 | fi
140 | 
141 | if [ -z "$vms_refresh_events" ]; then
142 |     vms_refresh_events="novms_refresh_events"
143 | else
144 |     vms_refresh_events="vms_refresh_events"
145 | fi
146 | 
147 | 
148 | NFENV=/etc/profile.d/netflix_environment.sh
149 | if [ -f ${NFENV} ]; then
150 |     . ${NFENV}
151 | else
152 |     NFENV=""
153 | fi
154 | 
155 | cd `dirname $0`
156 | 
157 | prog=`basename $0`
158 | 
159 | cat <<EndOfHeader
160 | Content-Type: text/html
161 | 
162 | <html>
163 | <head>
164 |  <title>AdminGCViz</title>
165 | </head>
166 | <body>
167 | EndOfHeader
168 | 
169 | echo "<pre>"
170 | NOW=`date +%Y-%m-%dT%T`
171 | bash /apps/apache/htdocs/AdminGCViz/remote-data-collection/collect_remote_data.sh ${NOW} ${jmap_histo_live} ${vms_refresh_events}
172 | /usr/bin/python2.7 /apps/apache/htdocs/AdminGCViz/visualize-gc.py ${NOW} /mnt/logs/gc-reports/${NOW}
173 | 
174 | 
175 | if [ -n "${NFENV}" ]; then
176 |     GCREPORT=/mnt/logs/gc-reports/${NOW}.tar.gz 
177 |     S3LOCATION=s3://netflix.bulkdata.${NETFLIX_ENVIRONMENT}/gc-report-${NETFLIX_APP}-${NOW}.tar.gz
178 |     echo bundling up results on ${GCREPORT}
179 |     tar -C /mnt/logs/gc-reports -zcf ${GCREPORT} ${NOW}
180 |     echo copying gc report to ${S3LOCATION}
181 |     s3cp ${GCREPORT} ${S3LOCATION}
182 | fi
183 | 
184 | echo
185 | echo java -version is:
186 | cat /mnt/logs/gc-reports/${NOW}/java-version
187 | echo
188 | 
189 | echo
190 | echo java commandline is:
191 | cat /mnt/logs/gc-reports/${NOW}/cmdline
192 | echo
193 | 
194 | echo starting to render gc events and heap size images:
195 | echo "</pre>"
196 | 
197 | for fullpath in `ls /mnt/logs/gc-reports/${NOW}/*.png`;
198 | do
199 |     filename=`basename ${fullpath}`
200 |     echo "<img src=\"/AdminGCVizImages/${NOW}/${filename}\">"
201 | done
202 | 
203 | cat <<EndOfFooter
204 | </body>
205 | </html>
206 | EndOfFooter
207 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/collect_remote_data.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/sh
  2 | 
  3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/remote-data-collection/collect_remote_data.sh#5 $
  4 | # $DateTime: 2013/05/15 18:34:23 $
  5 | # $Change: 1838706 $
  6 | # $Author: mooreb $
  7 | 
  8 | # This script must be run on the instance where the gc.log and catalina.out must be analyzed.
  9 | 
 10 | NFENV=/etc/profile.d/netflix_environment.sh
 11 | if [ -f ${NFENV} ]; then
 12 |     . ${NFENV}
 13 | else
 14 |     NFENV=""
 15 | fi
 16 | 
 17 | # NOW=`date +%Y-%m-%dT%T`
 18 | if [ -z "$1" ]; then
 19 |     echo Usage: $0 {iso8601 combined date and time representation} [ options ... ]
 20 |     echo "  where options are: [no]jmap_histo_live [no]vms_refresh_events ... (absence implies the option is on)"
 21 |     exit 1
 22 | fi
 23 | NOW=$1
 24 | 
 25 | # start processing optional arguments
 26 | if [ -z "$2" ]; then
 27 |     jmap_histo_live=true
 28 |     vms_refresh_events=true
 29 | else
 30 |     shift
 31 |     while [ $# -gt 0 ];
 32 |     do
 33 |         case "$1" in 
 34 |             nojmap_histo_live)
 35 |                 jmap_histo_live=false
 36 |                 ;;
 37 |             jmap_histo_live)
 38 |                 jmap_histo_live=true
 39 |                 ;;
 40 |             novms_refresh_events)
 41 |                 vms_refresh_events=false
 42 |                 ;;
 43 |             vms_refresh_events)
 44 |                 vms_refresh_events=true
 45 |                 ;;
 46 |         esac
 47 |         shift
 48 |     done
 49 | fi
 50 | 
 51 | 
 52 | # Potential BUG: this assumes tomcat. It might be better to use jps, but the corresponding jps expression:
 53 | #   JAVAPID=`jps | grep Bootstrap | awk '{print $1}'`
 54 | # doesn't seem much (any?) better
 55 | JAVAPID=`ps -elfww | grep java | grep Bootstrap | grep -v grep | awk '{print $4}'`
 56 | if [ -z "${JAVAPID}" ]; then
 57 |     echo CANNOT DETERMINE JAVAPID. EXITING.
 58 |     exit 1
 59 | fi
 60 | 
 61 | JAVACMDLINE=`cat /proc/${JAVAPID}/cmdline`
 62 | # This does not work due to crontabs or other programs (or users!) being able to touch files in /proc
 63 | # BOOTTIME=`ls --full-time /proc/${JAVAPID}/cmdline | awk '{print $6,$7,$8}' | sed 's/[.][0-9][0-9]*//' | sed 's/ /T/' | sed 's/ //'`
 64 | BOOTTIME_LOCAL=`ps h -p ${JAVAPID} -o lstart`
 65 | BOOTTIME=`date --date="${BOOTTIME_LOCAL}" "+%Y-%m-%dT%T+0000"`
 66 | 
 67 | RDCS=/apps/apache/htdocs/AdminGCViz/remote-data-collection
 68 | OUTPUTROOT=/mnt/logs/gc-reports
 69 | OUTPUTDIR=${OUTPUTROOT}/${NOW}
 70 | 
 71 | echo making output directories
 72 | mkdir -p ${OUTPUTDIR}/raw-gc-data
 73 | mkdir -p ${OUTPUTDIR}/raw-sar-data
 74 | 
 75 | if [ "true" = "$jmap_histo_live" ]; then
 76 |     echo 'getting jmap -histo:live logs'
 77 |     mkdir -p /apps/tomcat/logs/jmap-histos
 78 |     jmap -histo:live ${JAVAPID} > /apps/tomcat/logs/jmap-histos/jmap-histo-live-`date +%Y-%m-%d--%H-%M-%S` 2>&1
 79 |     cp -ar /apps/tomcat/logs/jmap-histos ${OUTPUTDIR}
 80 | fi
 81 | 
 82 | echo copying gc logs
 83 | cp -a /apps/tomcat/logs/archive/*gc.log* /apps/tomcat/logs/gc.log ${OUTPUTDIR}/raw-gc-data
 84 | for f in `ls ${OUTPUTDIR}/raw-gc-data`
 85 | do
 86 |     g=${OUTPUTDIR}/raw-gc-data/${f}
 87 |     if [ "$(tail -c 1 ${g})" != "" ]; then
 88 |         echo >> ${g}
 89 |     fi
 90 | done
 91 | 
 92 | echo processing gc logs
 93 | cat ${OUTPUTDIR}/raw-gc-data/*tomcat_gc.log* ${OUTPUTDIR}/raw-gc-data/gc.log | tr -d '\000' | perl -w ${RDCS}/gcdotlog_one_event_per_line.pl > ${OUTPUTDIR}/gc-events-one-per-line
 94 | cat ${OUTPUTDIR}/gc-events-one-per-line | python ${RDCS}/gcdotlog_convert_relative_into_absolute_time.py ${BOOTTIME} ${OUTPUTDIR} > ${OUTPUTDIR}/gc-events-one-per-line-in-absolute-time
 95 | cat ${OUTPUTDIR}/gc-events-one-per-line-in-absolute-time | perl -w ${RDCS}/gcdotlog_extract_time.pl ${OUTPUTDIR} > ${OUTPUTDIR}/gc-events-duration-in-seconds-only
 96 | cat ${OUTPUTDIR}/gc-events-duration-in-seconds-only |grep -v secs_since_jvm_boot | awk -F, '{print $1}' | python ${RDCS}/prepend_epoch.py > /tmp/epochs
 97 | paste --delim=, /tmp/epochs ${OUTPUTDIR}/gc-events-duration-in-seconds-only > /tmp/foo
 98 | mv /tmp/foo ${OUTPUTDIR}/gc-events-duration-in-seconds-only
 99 | bash ${RDCS}/gc_events_by_type.sh ${OUTPUTDIR}
100 | cat ${OUTPUTDIR}/gc-events-one-per-line-in-absolute-time | perl -w ${RDCS}/gcdotlog_extract_sizes.pl ${OUTPUTDIR} > ${OUTPUTDIR}/gc-sizes.csv
101 | cat ${OUTPUTDIR}/gc-sizes.csv |grep -v secs_since_jvm_boot | awk -F, '{print $1}' | python ${RDCS}/prepend_epoch.py > /tmp/epochs
102 | paste --delim=, /tmp/epochs ${OUTPUTDIR}/gc-sizes.csv > /tmp/foo
103 | mv /tmp/foo ${OUTPUTDIR}/gc-sizes.csv
104 | rm /tmp/epochs
105 | 
106 | if [ -n "${NFENV}" ] && [ "true" = "$vms_refresh_events" ]; then
107 |     echo looking for vms refresh events
108 |     grep --no-filename vms-refreshexecutor /apps/tomcat/logs/archive/*catalina.out* /apps/tomcat/logs/catalina.out  | egrep -e RefreshStat: -e 'Refreshing caches for country' > ${OUTPUTDIR}/vms-cache-refresh-events
109 |     grep -o 'CacheRefresh:[0-9][0-9]*' ${OUTPUTDIR}/vms-cache-refresh-events  | awk -F : '{print $2}' > ${OUTPUTDIR}/vms-cache-refresh-events-milliseconds
110 |     grep --no-filename CacheManagerStats /apps/tomcat/logs/archive/*catalina.out* /apps/tomcat/logs/catalina.out  | grep RefreshStat: | grep OVERALL > ${OUTPUTDIR}/vms-cache-refresh-overall-events
111 |     grep -o 'CacheRefresh:[0-9][0-9]*.*duration:[0-9][0-9]*' ${OUTPUTDIR}/vms-cache-refresh-overall-events | awk '{print $1,$7}' | awk -F : '{print $2,$3}' | awk '{print $1,$3}' > ${OUTPUTDIR}/vms-cache-refresh-overall-events-milliseconds
112 | 
113 |     echo looking for vms facet info
114 |     grep --no-filename BaseManager /apps/tomcat/logs/archive/*catalina.out* /apps/tomcat/logs/catalina.out | grep CacheRefreshInfo: > ${OUTPUTDIR}/vms-cache-refresh-facet-info
115 |     awk '{print $1,$2,$8,$9,$11,$12,$13,$14}' ${OUTPUTDIR}/vms-cache-refresh-facet-info | python ${RDCS}/vms_facet_info_transform.py > ${OUTPUTDIR}/vms-cache-refresh-facet-info.csv 2> ${OUTPUTDIR}/vms-cache-refresh-facet-info-rejected-lines
116 |     bash ${RDCS}/facet_events_by_type.sh ${OUTPUTDIR}
117 |     bash ${RDCS}/facet_events_by_country.sh ${OUTPUTDIR}
118 | 
119 |     echo looking for vms objectCache stats
120 |     egrep --no-filename -r -e VMClientCacheManager -e objectCache /apps/tomcat/logs/archive/*catalina.out* /apps/tomcat/logs/catalina.out | egrep -e Processed -e objectCache > ${OUTPUTDIR}/vms-object-cache-stats
121 |     python ${RDCS}/process_vms_object_cache_stats.py ${OUTPUTDIR}/vms-object-cache-stats ${OUTPUTDIR}
122 | fi
123 | 
124 | echo grabbing environment
125 | cat /proc/${JAVAPID}/cmdline | tr '\000' '\n' >  ${OUTPUTDIR}/cmdline
126 | echo ${BOOTTIME} > ${OUTPUTDIR}/jvm_boottime
127 | env | LANG=C sort > ${OUTPUTDIR}/env
128 | md5sum /apps/tomcat/webapps/ROOT/WEB-INF/lib/* | LANG=C sort -k2 > ${OUTPUTDIR}/web-inf-lib
129 | /proc/${JAVAPID}/exe -version > ${OUTPUTDIR}/java-version 2>&1
130 | cat /proc/${JAVAPID}/maps > ${OUTPUTDIR}/maps
131 | python ${RDCS}/parse-proc-pid-maps.py ${OUTPUTDIR}/maps > ${OUTPUTDIR}/maps-parsed
132 | cat /proc/meminfo > ${OUTPUTDIR}/meminfo
133 | cat /proc/cpuinfo > ${OUTPUTDIR}/cpuinfo
134 | free -m > ${OUTPUTDIR}/free-m
135 | 
136 | if [ -n "${NFENV}" ]; then
137 |     echo grabbing netflix-specific environment
138 |     find /apps/tomcat/webapps/ROOT/WEB-INF/classes -name "*.properties" -exec grep videometadata /dev/null {} \; > ${OUTPUTDIR}/videometadata-properties 2> /dev/null
139 |     rpm -q ${NETFLIX_APP} > ${OUTPUTDIR}/netflix-rpm-version
140 |     rpm -qa | LANG=C sort > ${OUTPUTDIR}/netflix-rpm-all-packages-versions
141 | fi
142 | 
143 | echo collecting sar data
144 | for f in `ls /var/log/sa/ | grep ^sa | grep -v sar`; do sar -f /var/log/sa/${f} > ${OUTPUTDIR}/raw-sar-data/sar-cpu-${f}; done
145 | for f in `ls /var/log/sa/ | grep ^sa | grep -v sar`; do sar -n ALL -f /var/log/sa/${f} > ${OUTPUTDIR}/raw-sar-data/sar-network-${f}; done
146 | 
147 | # BUG: fix this implicit agreement between collect_remote_data.sh and visualize-instance.sh
148 | # echo bundling up results
149 | # tar -C ${OUTPUTROOT} -zcf ${OUTPUTDIR}.tar.gz ${NOW}
150 | 


--------------------------------------------------------------------------------
/root/apps/apache/htdocs/AdminGCViz/visualize-gc.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python2.7
  2 | 
  3 | # $Id: //depot/cloud/rpms/nflx-webadmin-gcviz/root/apps/apache/htdocs/AdminGCViz/visualize-gc.py#6 $
  4 | # $DateTime: 2013/11/12 19:42:41 $
  5 | # $Change: 2030932 $
  6 | # $Author: mooreb $
  7 | 
  8 | import numpy as np
  9 | import matplotlib.pyplot as plt
 10 | import matplotlib.mlab as mlab
 11 | import matplotlib.ticker as ticker
 12 | import matplotlib.dates as mdates
 13 | import matplotlib.lines as lines
 14 | import pylab
 15 | import sys
 16 | import os
 17 | import vmsgcvizutils
 18 | 
 19 | def isNetflixInternal():
 20 |     try:
 21 |         open("/etc/profile.d/netflix_environment.sh", "r")
 22 |         return True
 23 |     except IOError:
 24 |         return False
 25 | 
 26 | numArgs = len(sys.argv) - 1
 27 | if(2 != numArgs):
 28 |     print "Usage: %s now:iso8601timestamp vms-gc-report-directory" % (sys.argv[0],)
 29 |     sys.exit(1);
 30 | 
 31 | now = sys.argv[1]
 32 | vmsGCReportDirectory = sys.argv[2]
 33 | gcEventDirectory = vmsGCReportDirectory + os.path.sep + 'gc-events-duration-by-event'
 34 | 
 35 | event_to_symbol_and_color = {
 36 | "FullGC"                            : ("mD", 15),  #(stop the world)
 37 | "concurrent-mode-failure"           : ("rh", 10),  #(stop the world)
 38 | "promotion-failed"                  : ("rH", 10),  #(stop the world)
 39 | "ParNew"                            : ("ro",  5),  #(stop-the-world)
 40 | "CMS-initial-mark"                  : ("r^",  5),  #(stop-the-world)
 41 | "CMS-remark"                        : ("rs",  5),  #(stop the world)
 42 | "CMS-concurrent-mark"               : ("g,",  1),  #(concurrent includes yields to other theads)
 43 | "CMS-concurrent-abortable-preclean" : ("g,",  1),  #(concurrent)
 44 | "CMS-concurrent-preclean"           : ("g,",  1),  #(concurrent)
 45 | "CMS-concurrent-sweep"              : ("g,",  1),  #(concurrent)
 46 | "CMS-concurrent-reset"              : ("g,",  1),  #(concurrent?)
 47 | "ParallelScavengeYoungGen"          : ("ro",  5),  #(stop-the-world)
 48 | "DefNew"                            : ("ro",  5),  #(stop-the-world)
 49 | "unknown"                           : ("r*",  15), #???
 50 | }
 51 | 
 52 | # These markers are present in gc.log but not accounted for above as they have no duration.
 53 | # They might be interesting to visualize but we currently minimize the CMS events anyway.
 54 | #   CMS-concurrent-mark-start
 55 | #   CMS-concurrent-preclean-start
 56 | #   CMS-concurrent-sweep-start
 57 | #   CMS-concurrent-reset-start
 58 | #   CMS-concurrent-abortable-preclean-start
 59 | 
 60 | ## Read environmental information
 61 | def getSmallEnvDict():
 62 |     try:
 63 |         fullEnvDict = vmsgcvizutils.envFileAsDictionary(vmsGCReportDirectory + os.path.sep + 'env')
 64 |     except:
 65 |         fullEnvDict = {}
 66 |     smallEnvDict = {
 67 |         'ec2PublicHostname' : fullEnvDict.get('EC2_PUBLIC_HOSTNAME', 'no-public-hostname'),
 68 |         'instanceID'        : fullEnvDict.get('EC2_INSTANCE_ID', 'no-instance-id'),
 69 |         'instanceType'      : fullEnvDict.get('EC2_INSTANCE_TYPE', 'no-instance-type'),
 70 |         'appname'           : fullEnvDict.get('NETFLIX_APP', 'unknown-app'),
 71 |         'asg'               : fullEnvDict.get('NETFLIX_AUTO_SCALE_GROUP', 'unknown-asg'),
 72 |         'env'               : fullEnvDict.get('NETFLIX_ENVIRONMENT', 'unknown-env'),
 73 |     }
 74 |     return smallEnvDict
 75 |     
 76 | 
 77 | ## Read GC event records, one file per type.
 78 | # The gc event records have this format:
 79 | #   secs_since_epoch,datetimestamp,secs_since_jvm_boot,gc_event_type,gc_event_duration_in_seconds
 80 | # for example:
 81 | #   1333055023.424,2012-03-29T21:03:43.424+0000,11.272,ParNew,0.19
 82 | # in numpy-speak:
 83 | #   dtype=[('secs_since_epoch', '<f8'), ('datetimestamp', '|O4'), ('secs_since_jvm_boot', '<f8'), ('gc_event_type', '|S33'), ('gc_event_duration_in_seconds', '<f8')])
 84 | 
 85 | dirList=os.listdir(gcEventDirectory)
 86 | recordsets = []
 87 | maxGCEventDuration = 0.0
 88 | maxSTWGCEventDuration = 0.0
 89 | for fname in dirList:
 90 |     fnameFullPath = gcEventDirectory + os.path.sep + fname
 91 |     print 'Reading %s' % (fnameFullPath,)
 92 |     (color, markersize) = event_to_symbol_and_color[fname]
 93 |     recordset = mlab.csv2rec(fnameFullPath)
 94 |     thisRecordsetMax = max(recordset.gc_event_duration_in_seconds)
 95 |     maxGCEventDuration = max(maxGCEventDuration, thisRecordsetMax)
 96 |     if color.startswith('r') or color.startswith('m'):
 97 |         maxSTWGCEventDuration = max(maxSTWGCEventDuration, thisRecordsetMax)
 98 |     mpldatenums = mdates.epoch2num(recordset.secs_since_epoch)
 99 |     tuple = (fname, recordset, mpldatenums, color, markersize)
100 |     recordsets.append(tuple)
101 | 
102 | ## Plot the GC event records
103 | # example plots in:
104 | #   http://matplotlib.sourceforge.net/gallery.html
105 | #
106 | # this one is particularly good:
107 | #   http://matplotlib.sourceforge.net/examples/pylab_examples/usetex_demo.html
108 | # BUG: add multiple plots. example in:
109 | #   http://matplotlib.sourceforge.net/examples/pylab_examples/anscombe.html
110 | 
111 | fig = plt.figure()
112 | # 1x1 grid = 1 row, 1 column. 2x3 grid = 2 rows, 3 columns. The third number starts from 1 and increments row-first. See documentation of subplot() for more info.
113 | ax = fig.add_subplot(111)
114 | locator = mdates.AutoDateLocator()
115 | formatter = mdates.DateFormatter('%Y-%m-%d %H:%M:%SZ')
116 | ax.xaxis.set_major_locator(locator)
117 | ax.xaxis.set_major_formatter(formatter)
118 | 
119 | for (dataname, dataset, mpldatenums, color, markersize) in recordsets:
120 |     ax.plot(mpldatenums, dataset.gc_event_duration_in_seconds, color, label=dataname, ms=markersize, lw=markersize)
121 | 
122 | # draw the most recent jvm boot time line
123 | fp = open(vmsGCReportDirectory + os.path.sep + 'jvm_boottime.epoch')
124 | jvmBootEpoch = fp.readline()
125 | jvmBootEpoch = jvmBootEpoch.rstrip('\r\n')
126 | jvmBootDays = mdates.epoch2num(long(jvmBootEpoch))
127 | jvmBootLine = lines.Line2D([jvmBootDays,jvmBootDays], [0,maxGCEventDuration], label='jvm boot time', linewidth=2)
128 | ax.add_line(jvmBootLine)
129 | fp.close()
130 | fp = open(vmsGCReportDirectory + os.path.sep + 'jvm_boottime')
131 | jvmBootTimestamp = fp.readline()
132 | jvmBootTimestamp = jvmBootTimestamp.rstrip('\r\n')
133 | fp.close()
134 | 
135 | # draw the vms cache refresh event lines if we're inside netflix
136 | def try_to_draw_vms_cache_refresh_lines():
137 |     if(isNetflixInternal()):
138 |         try:
139 |             fp = open(vmsGCReportDirectory + os.path.sep + 'vms-cache-refresh-overall-events-milliseconds')
140 |         except IOError:
141 |             return
142 |         for line in fp:
143 |             line = line.rstrip('\r\n')
144 |             try:
145 |                 (finish_time_ms_str, duration_ms_str) = line.split()
146 |             except ValueError:
147 |                 continue
148 |             finish_time_ms = long(finish_time_ms_str)
149 |             duration_ms = long(duration_ms_str)
150 |             start_time_ms = finish_time_ms - duration_ms
151 |             start_time_secs = start_time_ms/1000.0
152 |             start_time_days = mdates.epoch2num(start_time_secs)
153 |             start_time_line = lines.Line2D([start_time_days,start_time_days], [0,maxGCEventDuration], color='r')
154 |             ax.add_line(start_time_line)
155 |             finish_time_secs = finish_time_ms/1000.0
156 |             finish_time_days = mdates.epoch2num(finish_time_secs)
157 |             finish_time_line = lines.Line2D([finish_time_days,finish_time_days], [0,maxGCEventDuration], color='c')
158 |             ax.add_line(finish_time_line)
159 |         fp.close()
160 |         # draw some fake lines just to get them into the legend
161 |         fake_vms_start_line = lines.Line2D([jvmBootDays,0], [jvmBootDays,0], label='VMS cache refresh start', color='r')
162 |         fake_vms_end_line = lines.Line2D([jvmBootDays,0], [jvmBootDays,0], label='VMS cache refresh end', color='c')
163 |         ax.add_line(fake_vms_start_line)
164 |         ax.add_line(fake_vms_end_line)
165 | 
166 | try_to_draw_vms_cache_refresh_lines()
167 | 
168 | # various chart options
169 | smallEnvDict = getSmallEnvDict()
170 | ax.set_title('gc events over time for %s %s %s %s %s %s' % (smallEnvDict['appname'], smallEnvDict['env'], smallEnvDict['ec2PublicHostname'], smallEnvDict['instanceID'], smallEnvDict['instanceType'], smallEnvDict['asg']))
171 | ax.grid(True)
172 | ax.set_xlabel('gc event start timestamp')
173 | ax.set_ylabel('gc event duration (seconds)')
174 | fig.autofmt_xdate()
175 | plt.ylim([0,maxSTWGCEventDuration])
176 | fig.set_size_inches(21,12)
177 | pylab.figlegend(*ax.get_legend_handles_labels(), loc='lower center', ncol=5)
178 | 
179 | # BUG: add sar charts, including but not limited to cpu, network
180 | # BUG: plot secs since jvm start
181 | # BUG: visualize the facet data collected in vms-cache-refresh-facet-info.csv. Maybe leave this up to visualize-instance.
182 | 
183 | savedImageBaseFileName = vmsGCReportDirectory + os.path.sep + smallEnvDict['appname'] + '-gc-events-' + now
184 | savedIamgeFileName = savedImageBaseFileName + '.png'
185 | plt.savefig(savedIamgeFileName)
186 | print "output on %s" % (savedIamgeFileName,)
187 | savedIamgeFileName = savedImageBaseFileName + '.pdf'
188 | plt.savefig(savedIamgeFileName)
189 | print "output on %s" % (savedIamgeFileName,)
190 | 
191 | # Potential BUG: do we want to issue the plt.show()
192 | plt.show()
193 | 
194 | 
195 | heapSizesFileName = vmsGCReportDirectory + os.path.sep + 'gc-sizes.csv'
196 | heapSizesRecordSet = mlab.csv2rec(heapSizesFileName)
197 | heapSizesFig = plt.figure()
198 | heapSizesAx = heapSizesFig.add_subplot(111)
199 | heapSizesLocator = mdates.AutoDateLocator()
200 | heapSizesFormatter = mdates.DateFormatter('%Y-%m-%d %H:%M:%SZ')
201 | heapSizesAx.xaxis.set_major_locator(heapSizesLocator)
202 | heapSizesAx.xaxis.set_major_formatter(heapSizesFormatter)
203 | 
204 | ONE_K   = 1024.0
205 | ONE_MEG = 1024.0*1024.0
206 | ONE_GIG = 1024.0*1024.0*1024.0
207 | 
208 | whole_heap_end_k_max = heapSizesRecordSet.whole_heap_end_k.max()
209 | whole_heap_end_bytes_max = whole_heap_end_k_max* 1024.0
210 | 
211 | if (whole_heap_end_bytes_max < ONE_MEG):
212 |     heap_end_units = "kilobytes"
213 |     whole_heap_end = heapSizesRecordSet.whole_heap_end_k
214 | elif ((ONE_MEG <= whole_heap_end_bytes_max) and (whole_heap_end_bytes_max < ONE_GIG)):
215 |     heap_end_units = "megabytes"
216 |     whole_heap_end = heapSizesRecordSet.whole_heap_end_k / ONE_K
217 | elif (ONE_GIG <= whole_heap_end_bytes_max):
218 |     heap_end_units = "gigabytes"
219 |     whole_heap_end = heapSizesRecordSet.whole_heap_end_k / ONE_MEG
220 | else:
221 |     raise Exception("did not expect to get here")
222 | 
223 | heapSizesAx.plot(mdates.epoch2num(heapSizesRecordSet.secs_since_epoch), whole_heap_end, 'o')
224 | heapSizesFig.autofmt_xdate()
225 | heapSizesAx.set_title('heap size over time for %s %s %s %s %s %s' % (smallEnvDict['appname'], smallEnvDict['env'], smallEnvDict['ec2PublicHostname'], smallEnvDict['instanceID'], smallEnvDict['instanceType'], smallEnvDict['asg']))
226 | heapSizesAx.grid(True)
227 | heapSizesAx.set_xlabel('time')
228 | heapSizesAx.set_ylabel('Total Heap Size After Collection (%s)' % (heap_end_units,))
229 | 
230 | heapSizesFig.set_size_inches(21,12)
231 | heapSizesSavedImageBaseName = vmsGCReportDirectory + os.path.sep + smallEnvDict['appname'] + '-heap-size-' + now
232 | fname = heapSizesSavedImageBaseName + '.png'
233 | plt.savefig(fname)
234 | print 'output on ' + fname
235 | fname = heapSizesSavedImageBaseName + '.pdf'
236 | plt.savefig(fname)
237 | print 'output on ' + fname
238 | plt.show()
239 | 


--------------------------------------------------------------------------------