├── .gitignore
├── config.yaml.example
├── stylesheets
    └── main.css
├── appengine_config.py
├── test.json
├── app.yaml
├── generate_random_data.py
├── index.html
└── phone_home.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | index.yaml
3 | config.yaml
4 | boto


--------------------------------------------------------------------------------
/config.yaml.example:
--------------------------------------------------------------------------------
1 | amazon_access_key: XXX
2 | amazon_secret_key: YYY


--------------------------------------------------------------------------------
/stylesheets/main.css:
--------------------------------------------------------------------------------
1 | body {
2 |   font-family: Verdana, Helvetica, sans-serif;
3 |   background-color: #DDDDDD;
4 | }


--------------------------------------------------------------------------------
/appengine_config.py:
--------------------------------------------------------------------------------
1 | # appengine_config.py
2 | from google.appengine.ext import vendor
3 | 
4 | # Add any libraries install in the "lib" folder.
5 | vendor.add('lib')
6 | 
7 | 


--------------------------------------------------------------------------------
/test.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "instanceId" : "i-48b5bb65",
 3 |   "billingProducts" : null,
 4 |   "imageId" : "ami-e2dc278a",
 5 |   "architecture" : "x86_64",
 6 |   "pendingTime" : "2014-08-20T17:22:21Z",
 7 |   "instanceType" : "t1.micro",
 8 |   "accountId" : "555219204010",
 9 |   "kernelId" : "aki-0b4aa462",
10 |   "ramdiskId" : null,
11 |   "region" : "us-east-1",
12 |   "version" : "2010-08-31",
13 |   "availabilityZone" : "us-east-1b",
14 |   "privateIp" : "10.178.172.95",
15 |   "devpayProductCodes" : null
16 | }


--------------------------------------------------------------------------------
/app.yaml:
--------------------------------------------------------------------------------
 1 | runtime: python27
 2 | api_version: 1
 3 | threadsafe: true
 4 | 
 5 | handlers:
 6 | 
 7 | # - url: /admin/.*
 8 | #   script: $PYTHON_LIB/google/appengine/ext/admin
 9 | #   login: admin
10 | 
11 | - url: /stylesheets
12 |   static_dir: stylesheets
13 | 
14 | - url: /.*
15 |   script: phone_home.application
16 | 
17 | # [START libraries]
18 | libraries:
19 | - name: webapp2
20 |   version: latest
21 | - name: jinja2
22 |   version: latest
23 | # - name: ssl
24 | #   version: "latest"
25 | # - name: boto3
26 | #   version: "latest"
27 | # [END libraries]
28 | 


--------------------------------------------------------------------------------
/generate_random_data.py:
--------------------------------------------------------------------------------
 1 | # DO NOT RUN THIS IN PRODUCTION! It's for testing only
 2 | 
 3 | import os
 4 | import pprint
 5 | import random
 6 | import time
 7 | import datetime
 8 | import string
 9 | import hashlib
10 | 
11 | from google.appengine.api import memcache
12 | from google.appengine.api import mail
13 | from google.appengine.api import urlfetch
14 | from google.appengine.ext import db
15 | 
16 | #pprint.pprint(os.environ.copy())
17 | 
18 | from phone_home import AMILaunch, get_parent
19 | 
20 | def randstring(n):
21 |     return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(n))
22 | 
23 | 
24 | def random_date(start_time_string, end_time_string, format_string, random_number):
25 |     """
26 |     Get a time at a proportion of a range of two formatted times.
27 |     start and end should be strings specifying times formated in the
28 |     given format (strftime-style), giving an interval [start, end].
29 |     prop specifies how a proportion of the interval to be taken after
30 |     start.  The returned time will be in the specified format.
31 |     """
32 |     dt_start = datetime.datetime.strptime(start_time_string, format_string)
33 |     dt_end = datetime.datetime.strptime(end_time_string, format_string)
34 | 
35 |     start_time = time.mktime(dt_start.timetuple()) + dt_start.microsecond / 1000000.0
36 |     end_time = time.mktime(dt_end.timetuple()) + dt_end.microsecond / 1000000.0
37 | 
38 |     random_time = start_time + random_number * (end_time - start_time)
39 | 
40 |     ret = datetime.datetime.fromtimestamp(random_time)#.strftime(format_string)
41 |     # print(ret.__class__.__name__)
42 |     return ret
43 | 
44 | 
45 | 
46 | for i in range(200):
47 |     ami_launch = AMILaunch(parent=get_parent())
48 |     account_id = randstring(12)
49 |     ami_launch.is_bioc_account = bool(random.getrandbits(1))
50 |     if ami_launch.is_bioc_account:
51 |         account_id = "some consistent string"
52 |     ami_launch.account_hash = hashlib.md5(account_id.encode()).hexdigest()
53 |     ami_launch.ami_id = "ami-" + randstring(8)
54 |     ami_launch.instance_type = randstring(5)
55 |     ami_launch.region = randstring(6)
56 |     ami_launch.availability_zone = randstring(9)
57 |     ami_launch.ami_name = randstring(20)
58 |     ami_launch.ami_description = randstring(30)
59 |     ami_launch.bioc_version = randstring(3)
60 |     #print(ami_launch)
61 |     ami_launch.date = random_date("2000/01/01 00:00:00.000000", "2049/12/31 23:59:59.999999", '%Y/%m/%d %H:%M:%S.%f', random.random())
62 |     #print(ami_launch.date)
63 |     ami_launch.put()
64 | 


--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | {% autoescape true %}
 3 | <html>
 4 | <head>
 5 |   <title>Bioconductor AMI Phone Home</title>
 6 |   <link type="text/css" rel="stylesheet" href="/stylesheets/main.css" />
 7 | 
 8 | <!-- DataTables CSS -->
 9 | <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.2/css/jquery.dataTables.css">
10 | 
11 | <!-- jQuery -->
12 | <script type="text/javascript" charset="utf8" src="//code.jquery.com/jquery-1.10.2.min.js"></script>
13 | 
14 | <!-- DataTables -->
15 | <script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.2/js/jquery.dataTables.js"></script>
16 | 
17 | <script type="text/javascript">
18 | $(document).ready( function () {
19 | 
20 |   var cursor = null;
21 |   $.fn.dataTableExt.afnFiltering.push(function(oSettings, aData, iDataIndex) {
22 |         var checked = $('#checkbox').is(':checked');
23 |         if (checked)
24 |           return aData[6] != "True";
25 |         else
26 |           return true
27 |     });
28 | 
29 |     var oTable = $('#hit_table').dataTable({
30 |       "order": [[7, "desc"]],
31 |       "info": false,
32 |       "pagingType": "simple",
33 |       "searching": false,
34 |       "bLengthChange": false,
35 |       "processing": true,
36 |       "serverSide": true,
37 |       "serverMethod": "POST",
38 |       "ajax": {
39 |         "url": "/table-data",
40 |         "data": function(d) {
41 |             console.log("outgoing (to server): set cursor to " + cursor);
42 |             d.next_cursor = cursor;
43 |         },
44 |         "dataSrc": function(data) {
45 |             console.log("data is ");
46 |             console.log(data);
47 |             console.log("from server, got cursor " + data.next_cursor);
48 |             cursor = data.next_cursor;
49 |             console.log("local variable cursor is " + cursor);
50 |             console.log("cursor is " + data.next_cursor);
51 |             return data.data; // IMPORTANT! This must be here.
52 |         }
53 |       }
54 |     });
55 | 
56 |     $('#checkbox').on("click", function(e) {
57 |         console.log('click');
58 |         oTable.fnDraw();
59 |         console.log("num is " + oTable.fnSettings().fnRecordsDisplay());
60 |         var numRows = oTable.fnSettings().fnRecordsDisplay();
61 |         $('#num_entries').html(numRows);
62 |     });
63 | 
64 | } );
65 | </script>
66 | 
67 | </head>
68 |   <body>
69 |     <h1>ami phone home</h1>
70 | 
71 |     <table id="hit_table" class="display">
72 |       <thead>
73 |         <tr>
74 |         <th>ami-id</th>
75 |         <th>bioc version</th>
76 |         <th>ami name</th>
77 |         <th>instance type</th>
78 |         <th>region</th>
79 |         <th>availability zone</th>
80 |         <th>is bioc account</th>
81 |         <th>date</th>
82 |         <th>account hash</th>
83 |       </tr>
84 |       </thead>
85 |       <tbody>
86 | 
87 |       </tbody>
88 |     </table>
89 | 
90 | 
91 |     <hr>
92 | 
93 | 
94 |   </body>
95 | </html>
96 | {% endautoescape %}
97 | 


--------------------------------------------------------------------------------
/phone_home.py:
--------------------------------------------------------------------------------
  1 | # [START imports]
  2 | import os
  3 | import urllib
  4 | import hashlib
  5 | import sys
  6 | 
  7 | from google.appengine.api import users
  8 | from google.appengine.ext import ndb
  9 | from google.appengine.datastore.datastore_query import Cursor
 10 | from webapp2_extras import json
 11 | 
 12 | import jinja2
 13 | import webapp2
 14 | from webapp2_extras import json
 15 | import yaml
 16 | import urllib2
 17 | import boto.ec2
 18 | 
 19 | JINJA_ENVIRONMENT = jinja2.Environment(
 20 |     loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
 21 |     extensions=['jinja2.ext.autoescape'],
 22 |     autoescape=True)
 23 | # [END imports]
 24 | 
 25 | def get_parent():
 26 |     return ndb.Key('phone-home Parent', 'phone-home Parent name')
 27 | 
 28 | def get_bioc_version(ami_id):
 29 |     response = urllib2.urlopen('https://raw.githubusercontent.com/Bioconductor/bioconductor.org/master/config.yaml')
 30 |     obj = yaml.load(response)
 31 |     for k,v in obj['ami_ids'].iteritems():
 32 |         if v == ami_id:
 33 |             return k.replace("bioc", "").replace("_", ".")
 34 |     return None
 35 | 
 36 | def get_ami_info(ami_id):
 37 |     configfile = os.path.join(os.path.dirname(__file__), "config.yaml")
 38 |     stream = open(configfile, 'r')
 39 |     config = yaml.load(stream)
 40 |     conn = boto.ec2.connect_to_region("us-east-1",
 41 |         aws_access_key_id=config['amazon_access_key'],
 42 |         aws_secret_access_key=config['amazon_secret_key'],
 43 |         validate_certs=False)
 44 |     try:
 45 |         img = conn.get_all_images([ami_id])
 46 |         return({'name': img[0].name, 'description': img[0].description})
 47 |     except:
 48 |         return None
 49 | 
 50 | class AMILaunch(ndb.Model):
 51 |     """Models a launch of a BioC AMI"""
 52 |     is_bioc_account = ndb.BooleanProperty()
 53 |     account_hash = ndb.StringProperty()
 54 |     is_aws_ip = ndb.BooleanProperty()
 55 |     ami_id = ndb.StringProperty()
 56 |     bioc_version = ndb.StringProperty()
 57 |     ami_name = ndb.StringProperty()
 58 |     ami_description = ndb.StringProperty()
 59 |     instance_type = ndb.StringProperty()
 60 |     region = ndb.StringProperty()
 61 |     date = ndb.DateTimeProperty(auto_now_add=True)
 62 |     # date = ndb.DateTimeProperty() # only needed when populating synthetic data
 63 |     availability_zone = ndb.StringProperty()
 64 | 
 65 | class FrontPage(webapp2.RequestHandler):
 66 |     def get(self):
 67 |         return webapp2.redirect('/phone-home')
 68 | 
 69 | class TableData(webapp2.RequestHandler):
 70 |     def post(self):
 71 |         in_obj = self.request.POST
 72 | 
 73 |         out_obj = {}
 74 |         out_obj['draw'] = int(in_obj['draw'])
 75 | 
 76 |         self.response.content_type = 'application/json'
 77 | 
 78 |         count =  99999999 # FIXME if real number of records exceeds this, increment this!!
 79 |         out_obj['recordsTotal'] = count
 80 |         out_obj['recordsFiltered'] = count # OK?
 81 | 
 82 |         start = in_obj['start'] # starts at 0
 83 |         pagesize = int(in_obj['length']) # number of rows to return
 84 | 
 85 |         cursor = None
 86 |         if len(in_obj['next_cursor']):
 87 |             cursor = Cursor(urlsafe=in_obj['next_cursor'])
 88 |         print("cursor is")
 89 |         if cursor:
 90 |             print(cursor.urlsafe())
 91 |         else:
 92 |             print("(none)")
 93 | 
 94 | 
 95 |         query = AMILaunch.query(ancestor=get_parent()).order(-AMILaunch.date)
 96 |         rows, next_cursor, more = query.fetch_page(pagesize, start_cursor=cursor)
 97 | 
 98 |         if next_cursor:
 99 |             print("urlsafe is")
100 |             print(next_cursor.urlsafe())
101 | 
102 |             out_obj['next_cursor'] = next_cursor.urlsafe()
103 |         else:
104 |             out_obj['next_cursor'] = None
105 |             print("urlsafe is None")
106 | 
107 |         data = []
108 | 
109 |         for row in rows:
110 |             data.append([row.ami_id, row.bioc_version, row.ami_name,
111 |                          row.instance_type, row.region, row.availability_zone,
112 |                          row.is_bioc_account, str(row.date), row.account_hash])
113 | 
114 |         out_obj['data'] = data
115 |         self.response.write(json.encode(out_obj))
116 | 
117 | class AWSHandler(webapp2.RequestHandler):
118 |     def get(self):
119 |         template = JINJA_ENVIRONMENT.get_template('index.html')
120 |         self.response.write(template.render())
121 | 
122 |     def post(self):
123 |         # remote_addr shows up as "::1" when calling from localhost
124 |         # or is it when using the simulated version of the
125 |         # GAE environment? not sure.
126 |         # print("remote ip is %s" % self.request.remote_addr)
127 |         raw = self.request.body
128 |         try:
129 |             obj = json.decode(raw)
130 |         except ValueError:
131 |             self.response.out.write("invalid json")
132 |             return
133 | 
134 | 
135 |         ami_launch = AMILaunch(parent=get_parent())
136 |         ami_launch.is_bioc_account = obj['accountId'] == "555219204010"
137 |         if not ami_launch.is_bioc_account:
138 |             ami_launch.account_hash = hashlib.md5(obj['accountId'].encode()).hexdigest()
139 |         ami_launch.ami_id = obj['imageId']
140 |         ami_launch.instance_type = obj['instanceType']
141 |         ami_launch.region = obj['region']
142 |         ami_launch.availability_zone = obj['availabilityZone']
143 | 
144 |         is_aws_ip("foo")
145 | 
146 |         ami_launch.bioc_version = get_bioc_version(obj['imageId'])
147 | 
148 |         ami_info = get_ami_info(obj['imageId'])
149 |         if ami_info is not None:
150 |             ami_launch.ami_name = ami_info['name']
151 |             ami_launch.ami_description = ami_info['description']
152 | 
153 |         ami_launch.put()
154 | 
155 |         self.response.out.write("thanx\n")
156 | 
157 | 
158 | 
159 | 
160 | application = webapp2.WSGIApplication([
161 |     ('/phone-home', AWSHandler),
162 |     ('/', FrontPage),
163 |     ('/table-data', TableData),
164 | ], debug=True)
165 | 
166 | def is_aws_ip(ip):
167 |     aws_ips="""72.44.32.0/19 (72.44.32.0 - 72.44.63.255)
168 | 67.202.0.0/18 (67.202.0.0 - 67.202.63.255)
169 | 75.101.128.0/17 (75.101.128.0 - 75.101.255.255)
170 | 174.129.0.0/16 (174.129.0.0 - 174.129.255.255)
171 | 204.236.192.0/18 (204.236.192.0 - 204.236.255.255)
172 | 184.73.0.0/16 (184.73.0.0 - 184.73.255.255)
173 | 184.72.128.0/17 (184.72.128.0 - 184.72.255.255)
174 | 184.72.64.0/18 (184.72.64.0 - 184.72.127.255)
175 | 50.16.0.0/15 (50.16.0.0 - 50.17.255.255)
176 | 50.19.0.0/16 (50.19.0.0 - 50.19.255.255)
177 | 107.20.0.0/14 (107.20.0.0 - 107.23.255.255)
178 | 23.20.0.0/14 (23.20.0.0 - 23.23.255.255)
179 | 54.242.0.0/15 (54.242.0.0 - 54.243.255.255)
180 | 54.234.0.0/15 (54.234.0.0 - 54.235.255.255)
181 | 54.236.0.0/15 (54.236.0.0 - 54.237.255.255)
182 | 54.224.0.0/15 (54.224.0.0 - 54.225.255.255)
183 | 54.226.0.0/15 (54.226.0.0 - 54.227.255.255)
184 | 54.208.0.0/15 (54.208.0.0 - 54.209.255.255)
185 | 54.210.0.0/15 (54.210.0.0 - 54.211.255.255)
186 | 54.221.0.0/16 (54.221.0.0 - 54.221.255.255)
187 | 54.204.0.0/15 (54.204.0.0 - 54.205.255.255)
188 | 54.196.0.0/15 (54.196.0.0 - 54.197.255.255)
189 | 54.198.0.0/16 (54.198.0.0 - 54.198.255.255)
190 | 54.80.0.0/13 (54.80.0.0 - 54.87.255.255)
191 | 54.88.0.0/14 (54.88.0.0 - 54.91.255.255)
192 | 54.92.0.0/16 (54.92.0.0 - 54.92.255.255)
193 | 54.92.128.0/17 (54.92.128.0 - 54.92.255.255)
194 | 54.160.0.0/13 (54.160.0.0 - 54.167.255.255)
195 | 54.172.0.0/15 (54.172.0.0 - 54.173.255.255)
196 | 50.112.0.0/16 (50.112.0.0 - 50.112.255.255)
197 | 54.245.0.0/16 (54.245.0.0 - 54.245.255.255)
198 | 54.244.0.0/16 (54.244.0.0 - 54.244.255.255)
199 | 54.214.0.0/16 (54.214.0.0 - 54.214.255.255)
200 | 54.212.0.0/15 (54.212.0.0 - 54.213.255.255)
201 | 54.218.0.0/16 (54.218.0.0 - 54.218.255.255)
202 | 54.200.0.0/15 (54.200.0.0 - 54.201.255.255)
203 | 54.202.0.0/15 (54.202.0.0 - 54.203.255.255)
204 | 54.184.0.0/13 (54.184.0.0 - 54.191.255.255)
205 | 54.68.0.0/14 (54.68.0.0 - 54.71.255.255)
206 | 204.236.128.0/18 (204.236.128.0 - 204.236.191.255)
207 | 184.72.0.0/18 (184.72.0.0 - 184.72.63.255)
208 | 50.18.0.0/16 (50.18.0.0 - 50.18.255.255)
209 | 184.169.128.0/17 (184.169.128.0 - 184.169.255.255)
210 | 54.241.0.0/16 (54.241.0.0 - 54.241.255.255)
211 | 54.215.0.0/16 (54.215.0.0 - 54.215.255.255)
212 | 54.219.0.0/16 (54.219.0.0 - 54.219.255.255)
213 | 54.193.0.0/16 (54.193.0.0 - 54.193.255.255)
214 | 54.176.0.0/15 (54.176.0.0 - 54.177.255.255)
215 | 54.183.0.0/16 (54.183.0.0 - 54.183.255.255)
216 | 54.67.0.0/16 (54.67.0.0 - 54.67.255.255) NEW
217 | 79.125.0.0/17 (79.125.0.0 - 79.125.127.255)
218 | 46.51.128.0/18 (46.51.128.0 - 46.51.191.255)
219 | 46.51.192.0/20 (46.51.192.0 - 46.51.207.255)
220 | 46.137.0.0/17 (46.137.0.0 - 46.137.127.255)
221 | 46.137.128.0/18 (46.137.128.0 - 46.137.191.255)
222 | 176.34.128.0/17 (176.34.128.0 - 176.34.255.255)
223 | 176.34.64.0/18 (176.34.64.0 - 176.34.127.255)
224 | 54.247.0.0/16 (54.247.0.0 - 54.247.255.255)
225 | 54.246.0.0/16 (54.246.0.0 - 54.246.255.255)
226 | 54.228.0.0/16 (54.228.0.0 - 54.228.255.255)
227 | 54.216.0.0/15 (54.216.0.0 - 54.217.255.255)
228 | 54.229.0.0/16 (54.229.0.0 - 54.229.255.255)
229 | 54.220.0.0/16 (54.220.0.0 - 54.220.255.255)
230 | 54.194.0.0/15 (54.194.0.0 - 54.195.255.255)
231 | 54.72.0.0/14 (54.72.0.0 - 54.75.255.255)
232 | 54.76.0.0/15 (54.76.0.0 - 54.77.255.255)
233 | 54.78.0.0/16 (54.78.0.0 - 54.78.255.255)
234 | 54.74.0.0/15 (54.74.0.0 - 54.75.255.255)
235 | 185.48.120.0/22 (185.48.120.0 - 185.48.123.255)
236 | 54.170.0.0/15 (54.170.0.0 - 54.171.255.255)
237 | 175.41.128.0/18 (175.41.128.0 - 175.41.191.255)
238 | 122.248.192.0/18 (122.248.192.0 - 122.248.255.255)
239 | 46.137.192.0/18 (46.137.192.0 - 46.137.255.255)
240 | 46.51.216.0/21 (46.51.216.0 - 46.51.223.255)
241 | 54.251.0.0/16 (54.251.0.0 - 54.251.255.255)
242 | 54.254.0.0/16 (54.254.0.0 - 54.254.255.255)
243 | 54.255.0.0/16 (54.255.0.0 - 54.255.255.255)
244 | 54.179.0.0/16 (54.179.0.0 - 54.179.255.255)
245 | 54.169.0.0/16 (54.169.0.0 - 54.169.255.255)
246 | 54.252.0.0/16 (54.252.0.0 - 54.252.255.255)
247 | 54.253.0.0/16 (54.253.0.0 - 54.253.255.255)
248 | 54.206.0.0/16 (54.206.0.0 - 54.206.255.255)
249 | 54.79.0.0/16 (54.79.0.0 - 54.79.255.255)
250 | 54.66.0.0/16 (54.66.0.0 - 54.66.255.255)
251 | 175.41.192.0/18 (175.41.192.0 - 175.41.255.255)
252 | 46.51.224.0/19 (46.51.224.0 - 46.51.255.255)
253 | 176.32.64.0/19 (176.32.64.0 - 176.32.95.255)
254 | 103.4.8.0/21 (103.4.8.0 - 103.4.15.255)
255 | 176.34.0.0/18 (176.34.0.0 - 176.34.63.255)
256 | 54.248.0.0/15 (54.248.0.0 - 54.249.255.255)
257 | 54.250.0.0/16 (54.250.0.0 - 54.250.255.255)
258 | 54.238.0.0/16 (54.238.0.0 - 54.238.255.255)
259 | 54.199.0.0/16 (54.199.0.0 - 54.199.255.255)
260 | 54.178.0.0/16 (54.178.0.0 - 54.178.255.255)
261 | 54.95.0.0/16 (54.95.0.0-54.95.255.255)
262 | 54.92.0.0/17 (54.92.0.0 - 54.92.127.255)
263 | 54.168.0.0/16 (54.168.0.0 - 54.168.255.255)
264 | 54.64.0.0/15 (54.64.0.0 - 54.65.255.255)
265 | 177.71.128.0/17 (177.71.128.0 - 177.71.255.255)
266 | 54.232.0.0/16 (54.232.0.0 - 54.232.255.255)
267 | 54.233.0.0/18 (54.233.0.0 - 54.233.63.255)
268 | 54.207.0.0/16 (54.207.0.0 - 54.207.255.255)
269 | 54.94.0.0/16 (54.94.0.0 - 54.94.255.255)
270 | 54.223.0.0/16 (54.223.0.0 - 54.223.255.255)
271 | 96.127.0.0/18 (96.127.0.0 - 96.127.63.255)
272 | """
273 |     ip_segs = ip.split(".")
274 | 


--------------------------------------------------------------------------------