├── LAB1 ├── allpairsping.py ├── emulab │ └── howto_get_an_account.png ├── helloworld.py ├── homework │ ├── README │ ├── dist_demo.py │ ├── geoip_uvic.repy │ └── math_uvic.repy ├── infloop.py ├── udpforward.py ├── udpping.py └── udppingserver.py ├── LAB2 ├── README └── lab2.ns ├── LAB3 └── README.md ├── LAB4 └── README.md ├── LAB5 └── README.md ├── LAB6 └── README.md ├── LAB8 ├── README.md ├── mpi_mmul.c ├── mpi_test.c └── mpilab.pbs ├── LAB9 ├── GEE-README.md └── README.md ├── Lab-10 └── Readme.md ├── License ├── PortalCrashUpdate.md ├── ProjectIdea.md └── README.md /LAB1/allpairsping.py: -------------------------------------------------------------------------------- 1 | # send a probe message to each neighbor 2 | def probe_neighbors(port): 3 | 4 | for neighborip in mycontext["neighborlist"]: 5 | mycontext['sendtime'][neighborip] = getruntime() 6 | sendmess(neighborip, port, 'ping',getmyip(),port) 7 | 8 | sendmess(neighborip, port,'share'+encode_row(getmyip(), mycontext["neighborlist"], mycontext['latency'].copy())) 9 | # sleep in between messages to prevent us from getting a huge number of 10 | # responses all at once... 11 | sleep(.5) 12 | 13 | # Call me again in 10 seconds 14 | while True: 15 | try: 16 | settimer(10,probe_neighbors,(port,)) 17 | return 18 | except Exception, e: 19 | if "Resource 'events'" in str(e): 20 | # there are too many events scheduled, I should wait and try again 21 | sleep(.5) 22 | continue 23 | raise 24 | 25 | 26 | 27 | # Handle an incoming message 28 | def got_message(srcip,srcport,mess,ch): 29 | if mess == 'ping': 30 | sendmess(srcip,srcport,'pong') 31 | elif mess == 'pong': 32 | # elapsed time is now - time when I sent the ping 33 | mycontext['latency'][srcip] = getruntime() - mycontext['sendtime'][srcip] 34 | 35 | elif mess.startswith('share'): 36 | mycontext['row'][srcip] = mess[len('share'):] 37 | 38 | 39 | 40 | def encode_row(rowip, neighborlist, latencylist): 41 | 42 | retstring = "
"+ " | ".join(mycontext['neighborlist'])+" | |
'+nodeip+' | No Data Reported |
6 | apt-get update 7 | apt-get install -y build-essential 8 | apt-get install -y python2.7-dev 9 | apt-get install -y python-pip 10 | apt-get install -y wget 11 | apt-get install -y unzip 12 | apt-get install -y uwsgi uwsgi-plugin-python 13 |14 | 15 | ## Prepare Web server directory and install dependencies: 16 | 17 |
18 | cd /root 19 | mkdir www 20 | cd www 21 | wget http://www.scrunch.ca/static/scrunch-code.zip 22 | unzip scrunch-code.zip 23 | chown www-data:www-data -R /root/www 24 | pip install -r requirements/prod.txt 25 | pip install -r requirements/dev.txt 26 |27 | 28 | ## Install nginx: 29 | 30 |
31 | apt-get install -y nginx-full 32 | rm /etc/nginx/sites-enabled/default 33 |34 | 35 | ## Make nginx config file: 36 | 37 |
38 | vi /etc/nginx/sites-enabled/scrunch 39 |40 | 41 |
42 | server { 43 | listen 80; 44 | server_name localhost 45 | server_tokens off; 46 | access_log /var/log/nginx/scrunch_access.log; 47 | error_log /var/log/nginx/scrunch_error.log; 48 | 49 | location / { 50 | root /root/www; 51 | include uwsgi_params; 52 | uwsgi_pass unix:/tmp/uwsgi.sock; 53 | } 54 | 55 | location /static { 56 | alias /root/www; 57 | } 58 | } 59 |60 | 61 | 62 | ## Restart nginx: 63 | 64 |
65 | service nginx restart 66 |67 | 68 | ## Make 'Hello World' program: 69 | 70 |
71 | cd /root/www 72 | vi hello.py 73 |74 | 75 |
76 | from flask import Flask 77 | app = Flask(__name__) 78 | 79 | @app.route('/') 80 | def hello_world(): 81 | return 'Hello World!' 82 | 83 | if __name__ == '__main__': 84 | app.run(host='0.0.0.0') 85 |86 | 87 | ## Make uWSGI configuration yaml script: 88 | 89 |
90 | vi app.yaml 91 | uwsgi: 92 | socket: /tmp/uwsgi.sock 93 | master: 1 94 | workers: 1 95 | chmod-socket: 666 96 | auto-procname: 1 97 | plugins: python 98 | python-path: . 99 | uid: www-data 100 | gid: www-data 101 | pidfile: /tmp/uwsgi.pid 102 | daemonize: /var/log/uwsgi.log 103 | module: hello:app 104 |105 | 106 | ## Start up 'Hello world' program: 107 | 108 |
109 | uwsgi --yaml app.yaml 110 |111 | 112 | ## Setup SSH tunnel forwarding traffic to the GEE nginx server: 113 | 114 |
115 | ssh -L 8080:localhost:80 -i id_rsa -F ssh-config [your slice url] -N 116 |117 | 118 | ## Access your Scrunch server from your local browser: 119 | 120 |
121 | http://localhost:8080 122 |123 | -------------------------------------------------------------------------------- /LAB9/README.md: -------------------------------------------------------------------------------- 1 | # Installing AbeBook's Scrunch on Amazon EC2 2 | 3 | ## Get an Amazon Web Services Account 4 | 1. Get an account here: 5 | 6 | http://aws.amazon.com 7 | 8 | (you will need a Amazon Account/Credit Card & Cell phone to verify your account) 9 | 10 | 2. Create key pair (EC2 Dashboard) 11 | 12 | If you access EC2 from Linux change permission on key 13 |
14 | chmod 400 mykey.pem 15 |16 | 17 | If you access EC2 from Windows using putty you will have to convert your key from *.pem to *.ppk using puttygen, see here: 18 | 19 | http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/get-set-up-for-amazon-ec2.html#prepare-for-putty 20 | 21 | 3. Makes sure your Region is set to 'US-Oregon' (Top right EC2 menu bar) 22 | 23 | 4. Create a Security Group (EC2 Dashboard) 24 | 25 | Allow Inbound: HTTP/HTTPS/SSH 26 | 27 | 5. Launch AMI Instance (EC2 Dashboard) 28 | 29 | Check tick 'Free tier only' check box. 30 | 31 | Choose Amazon Linux AMI 2014.09.2 (HVM) 32 | 33 | Filter for 'Micro Instances' in next dialog (t2.micro) 34 | 35 | Edit Security Group 36 | 37 | 6. Connect to your Micro AWS Instance 38 | 39 | Use ssh with the key you generated/downloaded earlier. 40 | 41 |
42 | ssh -i mykey.pem ec2-user@[machine IP] 43 |44 | 45 | 7. TERMINATE YOUR INSTANCE 46 | 47 | DO IT!!!!! PLEASE!!!!!! 48 | 49 | ## Install basic and required packages 50 | 51 | For the packages below are enough to run a Flask 'Hello World' program we use to get you started (you will need some more packes to run AbeBook's Scrunch server). 52 | 53 |
54 | sudo su 55 | yum upgrade -y 56 |57 | 58 | To begin with, you must install some bare bone packages 59 | Flask, Jinja2 and Werkzeug will be automatically installed below. 60 | 61 |
62 | yum install -y gcc-c++ 63 | yum install -y python-devel 64 | yum -y install python-pip 65 | 66 | pip install uwsgi 67 | pip install flask 68 | pip install flask-debugtoolbar 69 |70 | 71 | ## Install and setup web-server 72 | 73 | You can host your Flask app directly, but nginx, is the better choice if you want your project to scale (you can choose other web-servers like Apache as well, but the instructions here are for nginx . 74 | 75 |
yum install -y nginx76 | 77 | After installation, change configuration file for proxy setup. 78 | 79 |
vi /etc/nginx/nginx.conf80 | 81 | Find the location / section, and change it to as follow: 82 | 83 |
84 | location / { 85 | include uwsgi_params; 86 | uwsgi_pass 127.0.0.1:10080; 87 | } 88 |89 | 90 | Then, start your Web server 91 | 92 |
93 | service nginx start 94 |95 | 96 | ## Create your first 'Hello, World' Flask program 97 | 98 | In your project folder create your first python app. 99 | 100 |
101 | cd /usr/share/nginx/html/ 102 | mkdir hello 103 | cd hello 104 |105 | 106 |
vi hello.py107 | 108 |
109 | from flask import Flask 110 | app = Flask(__name__) 111 | 112 | @app.route('/') 113 | def hello_world(): 114 | return 'Hello World!' 115 | 116 | if __name__ == '__main__': 117 | app.run(host='0.0.0.0', port=10080) 118 |119 | 120 | ## Setup uWSGI 121 | 122 | To configure uWSGI server create a config file as follows: 123 | 124 |
vi app.yaml125 | 126 |
127 | uwsgi: 128 | socket: 127.0.0.1:10080 129 | master: 1 130 | workers: 1 131 | chmod-socket: 666 132 | auto-procname: 1 133 | python-path: . 134 | pidfile: /tmp/uwsgi.pid 135 | daemonize: /var/log/uwsgi.log 136 | module: hello:app 137 |138 | 139 | ## Start uWSGI 140 | 141 | Using the config file, you can easily start uWSGI server: 142 | 143 |
uwsgi --yaml app.yaml144 | 145 | In case, you want to run it as other user's, you can use --uid option additionally. 146 | And, because our config specify that uWSGI processes are executed as daemon, if you want to stop them all, you can run: 147 | 148 |
kill -INT `cat /tmp/uwsgi.pid`149 | 150 | ## Installing AbeBook's Scrunch 151 | Now that you have a running Flask/nginx environment, adapt the code from AbeBook's Scrunch Server to work on your machine such that it will use a database to store the shortened URLs (you may use sqlite as a basic key value store or get fancier and make your key value store distributed using additional AWS instances ). 152 | 153 | Get code for scrunch here: 154 | 155 | http://www.scrunch.ca 156 | 157 |
158 | wget http://www.scrunch.ca/static/scrunch-code.zip 159 |160 | 161 | ## Shut down your running AWS instances! 162 | - In the EC2 Dashboard click 'Running Instances' and right click on your instance in the list and select 'Instance State' > 'Terminate'. 163 | 164 | -------------------------------------------------------------------------------- /Lab-10/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Scaling Challenges and Project Repos set up! 3 | 4 | 5 | ### In Part 1 of this lab, you will design/implement/test as many strategies as you can to continue to scale your Scruncher! 6 | 7 | 1. You want to provide a list of the top 10 trending links, how do you do it? 8 | 2. A market research firm wants to pay you big bucks for a daily report with usage statistics, how do you do it? 9 | 3. Users are complaining they're receiving links to known malware and phishing websites, what do you do? 10 | 4. Users are asking for the ability to share links with only people they approve, how can you give it to them? 11 | 12 | In addition to this, think of another scaling challenge and pose for the rest to the class (and Cliff and Erik) to solve! 13 | Post this as a reply to the "My Scaling Challenge!" issue before Monday March 30th! 14 | 15 | ### In Part 2, make sure you Project 2 repo is set up. 16 | 17 | Feel free to share it under the "My Project 2" issue before the end of lab. For many of you, you are continuing to build on Project 1, so CLEARLY identify the vast improvements you have introduced relative to that early release of your system! :) 18 | -------------------------------------------------------------------------------- /License: -------------------------------------------------------------------------------- 1 | CC0 1.0 Universal 2 | 3 | Statement of Purpose 4 | 5 | The laws of most jurisdictions throughout the world automatically confer 6 | exclusive Copyright and Related Rights (defined below) upon the creator and 7 | subsequent owner(s) (each and all, an "owner") of an original work of 8 | authorship and/or a database (each, a "Work"). 9 | 10 | Certain owners wish to permanently relinquish those rights to a Work for the 11 | purpose of contributing to a commons of creative, cultural and scientific 12 | works ("Commons") that the public can reliably and without fear of later 13 | claims of infringement build upon, modify, incorporate in other works, reuse 14 | and redistribute as freely as possible in any form whatsoever and for any 15 | purposes, including without limitation commercial purposes. These owners may 16 | contribute to the Commons to promote the ideal of a free culture and the 17 | further production of creative, cultural and scientific works, or to gain 18 | reputation or greater distribution for their Work in part through the use and 19 | efforts of others. 20 | 21 | For these and/or other purposes and motivations, and without any expectation 22 | of additional consideration or compensation, the person associating CC0 with a 23 | Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 24 | and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 25 | and publicly distribute the Work under its terms, with knowledge of his or her 26 | Copyright and Related Rights in the Work and the meaning and intended legal 27 | effect of CC0 on those rights. 28 | 29 | 1. Copyright and Related Rights. A Work made available under CC0 may be 30 | protected by copyright and related or neighboring rights ("Copyright and 31 | Related Rights"). Copyright and Related Rights include, but are not limited 32 | to, the following: 33 | 34 | i. the right to reproduce, adapt, distribute, perform, display, communicate, 35 | and translate a Work; 36 | 37 | ii. moral rights retained by the original author(s) and/or performer(s); 38 | 39 | iii. publicity and privacy rights pertaining to a person's image or likeness 40 | depicted in a Work; 41 | 42 | iv. rights protecting against unfair competition in regards to a Work, 43 | subject to the limitations in paragraph 4(a), below; 44 | 45 | v. rights protecting the extraction, dissemination, use and reuse of data in 46 | a Work; 47 | 48 | vi. database rights (such as those arising under Directive 96/9/EC of the 49 | European Parliament and of the Council of 11 March 1996 on the legal 50 | protection of databases, and under any national implementation thereof, 51 | including any amended or successor version of such directive); and 52 | 53 | vii. other similar, equivalent or corresponding rights throughout the world 54 | based on applicable law or treaty, and any national implementations thereof. 55 | 56 | 2. Waiver. To the greatest extent permitted by, but not in contravention of, 57 | applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 58 | unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 59 | and Related Rights and associated claims and causes of action, whether now 60 | known or unknown (including existing as well as future claims and causes of 61 | action), in the Work (i) in all territories worldwide, (ii) for the maximum 62 | duration provided by applicable law or treaty (including future time 63 | extensions), (iii) in any current or future medium and for any number of 64 | copies, and (iv) for any purpose whatsoever, including without limitation 65 | commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 66 | the Waiver for the benefit of each member of the public at large and to the 67 | detriment of Affirmer's heirs and successors, fully intending that such Waiver 68 | shall not be subject to revocation, rescission, cancellation, termination, or 69 | any other legal or equitable action to disrupt the quiet enjoyment of the Work 70 | by the public as contemplated by Affirmer's express Statement of Purpose. 71 | 72 | 3. Public License Fallback. Should any part of the Waiver for any reason be 73 | judged legally invalid or ineffective under applicable law, then the Waiver 74 | shall be preserved to the maximum extent permitted taking into account 75 | Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 76 | is so judged Affirmer hereby grants to each affected person a royalty-free, 77 | non transferable, non sublicensable, non exclusive, irrevocable and 78 | unconditional license to exercise Affirmer's Copyright and Related Rights in 79 | the Work (i) in all territories worldwide, (ii) for the maximum duration 80 | provided by applicable law or treaty (including future time extensions), (iii) 81 | in any current or future medium and for any number of copies, and (iv) for any 82 | purpose whatsoever, including without limitation commercial, advertising or 83 | promotional purposes (the "License"). The License shall be deemed effective as 84 | of the date CC0 was applied by Affirmer to the Work. Should any part of the 85 | License for any reason be judged legally invalid or ineffective under 86 | applicable law, such partial invalidity or ineffectiveness shall not 87 | invalidate the remainder of the License, and in such case Affirmer hereby 88 | affirms that he or she will not (i) exercise any of his or her remaining 89 | Copyright and Related Rights in the Work or (ii) assert any associated claims 90 | and causes of action with respect to the Work, in either case contrary to 91 | Affirmer's express Statement of Purpose. 92 | 93 | 4. Limitations and Disclaimers. 94 | 95 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 96 | surrendered, licensed or otherwise affected by this document. 97 | 98 | b. Affirmer offers the Work as-is and makes no representations or warranties 99 | of any kind concerning the Work, express, implied, statutory or otherwise, 100 | including without limitation warranties of title, merchantability, fitness 101 | for a particular purpose, non infringement, or the absence of latent or 102 | other defects, accuracy, or the present or absence of errors, whether or not 103 | discoverable, all to the greatest extent permissible under applicable law. 104 | 105 | c. Affirmer disclaims responsibility for clearing rights of other persons 106 | that may apply to the Work or any use thereof, including without limitation 107 | any person's Copyright and Related Rights in the Work. Further, Affirmer 108 | disclaims responsibility for obtaining any necessary consents, permissions 109 | or other rights required for any use of the Work. 110 | 111 | d. Affirmer understands and acknowledges that Creative Commons is not a 112 | party to this document and has no duty or obligation with respect to this 113 | CC0 or use of the Work. 114 | 115 | For more information, please see 116 |