├── proxy
├── requirements.txt
├── .idea
│ ├── inspectionProfiles
│ │ └── profiles_settings.xml
│ ├── misc.xml
│ ├── workspace.xml
│ ├── modules.xml
│ └── proxy.iml
├── Dockerfile
└── monitor.py
├── test-client
├── requirements.txt
├── .idea
│ ├── inspectionProfiles
│ │ └── profiles_settings.xml
│ ├── misc.xml
│ ├── workspace.xml
│ ├── modules.xml
│ └── test-client.iml
├── Dockerfile
└── tests
│ ├── testutils.py
│ ├── smoke_test.py
│ ├── basic_test.py
│ └── conftest.py
├── fastapi-server
├── requirements.txt
├── .idea
│ ├── inspectionProfiles
│ │ └── profiles_settings.xml
│ ├── misc.xml
│ ├── workspace.xml
│ ├── modules.xml
│ └── fastapi-server.iml
├── Dockerfile
└── app
│ └── main.py
├── nginx-server
├── html
│ ├── n.gif
│ ├── idc.gif
│ ├── tbi.gif
│ ├── about1.gif
│ ├── action.gif
│ ├── arrowr.gif
│ ├── comdex.gif
│ ├── enter.gif
│ ├── msft.gif
│ ├── one_sm.gif
│ ├── prod.gif
│ ├── search.gif
│ ├── shop.gif
│ ├── spacer.gif
│ ├── sports.gif
│ ├── vrule.gif
│ ├── woofer.gif
│ ├── 1ptrans.gif
│ ├── Pacman1.gif
│ ├── arrowbl.gif
│ ├── arrowgr.gif
│ ├── clinton.gif
│ ├── comdex_6.gif
│ ├── commish.gif
│ ├── home_on.gif
│ ├── nav_home.gif
│ ├── netnow3.gif
│ ├── rolodex.gif
│ ├── spacer1.gif
│ ├── spacer2.gif
│ ├── support.gif
│ ├── tagline.gif
│ ├── appfoundry.gif
│ ├── h_microsoft.gif
│ ├── home_igloo.gif
│ ├── ie_animated.gif
│ ├── inbox_img.gif
│ ├── msinternet.gif
│ ├── solutions.gif
│ ├── worldwide.gif
│ ├── navigation_bar.gif
│ ├── pointcast_small.gif
│ ├── m.d.html
│ └── microscape.html
├── .idea
│ ├── inspectionProfiles
│ │ └── profiles_settings.xml
│ ├── misc.xml
│ ├── workspace.xml
│ ├── modules.xml
│ └── nginx-server.iml
└── default.conf
├── .idea
├── vcs.xml
├── inspectionProfiles
│ └── profiles_settings.xml
├── .gitignore
├── webResources.xml
├── NetworksLab1Grading.iml
└── modules.xml
├── LICENSE.md
├── docker-compose.yml
├── .gitignore
└── README.md
/proxy/requirements.txt:
--------------------------------------------------------------------------------
1 | pyzmq
2 | psutil
--------------------------------------------------------------------------------
/test-client/requirements.txt:
--------------------------------------------------------------------------------
1 | pytest
2 | httpx
3 | pyzmq
--------------------------------------------------------------------------------
/fastapi-server/requirements.txt:
--------------------------------------------------------------------------------
1 | fastapi
2 | uvicorn[standard]
3 | aiofiles
--------------------------------------------------------------------------------
/nginx-server/html/n.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/n.gif
--------------------------------------------------------------------------------
/nginx-server/html/idc.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/idc.gif
--------------------------------------------------------------------------------
/nginx-server/html/tbi.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/tbi.gif
--------------------------------------------------------------------------------
/nginx-server/html/about1.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/about1.gif
--------------------------------------------------------------------------------
/nginx-server/html/action.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/action.gif
--------------------------------------------------------------------------------
/nginx-server/html/arrowr.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/arrowr.gif
--------------------------------------------------------------------------------
/nginx-server/html/comdex.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/comdex.gif
--------------------------------------------------------------------------------
/nginx-server/html/enter.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/enter.gif
--------------------------------------------------------------------------------
/nginx-server/html/msft.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/msft.gif
--------------------------------------------------------------------------------
/nginx-server/html/one_sm.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/one_sm.gif
--------------------------------------------------------------------------------
/nginx-server/html/prod.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/prod.gif
--------------------------------------------------------------------------------
/nginx-server/html/search.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/search.gif
--------------------------------------------------------------------------------
/nginx-server/html/shop.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/shop.gif
--------------------------------------------------------------------------------
/nginx-server/html/spacer.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/spacer.gif
--------------------------------------------------------------------------------
/nginx-server/html/sports.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/sports.gif
--------------------------------------------------------------------------------
/nginx-server/html/vrule.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/vrule.gif
--------------------------------------------------------------------------------
/nginx-server/html/woofer.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/woofer.gif
--------------------------------------------------------------------------------
/nginx-server/html/1ptrans.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/1ptrans.gif
--------------------------------------------------------------------------------
/nginx-server/html/Pacman1.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/Pacman1.gif
--------------------------------------------------------------------------------
/nginx-server/html/arrowbl.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/arrowbl.gif
--------------------------------------------------------------------------------
/nginx-server/html/arrowgr.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/arrowgr.gif
--------------------------------------------------------------------------------
/nginx-server/html/clinton.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/clinton.gif
--------------------------------------------------------------------------------
/nginx-server/html/comdex_6.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/comdex_6.gif
--------------------------------------------------------------------------------
/nginx-server/html/commish.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/commish.gif
--------------------------------------------------------------------------------
/nginx-server/html/home_on.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/home_on.gif
--------------------------------------------------------------------------------
/nginx-server/html/nav_home.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/nav_home.gif
--------------------------------------------------------------------------------
/nginx-server/html/netnow3.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/netnow3.gif
--------------------------------------------------------------------------------
/nginx-server/html/rolodex.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/rolodex.gif
--------------------------------------------------------------------------------
/nginx-server/html/spacer1.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/spacer1.gif
--------------------------------------------------------------------------------
/nginx-server/html/spacer2.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/spacer2.gif
--------------------------------------------------------------------------------
/nginx-server/html/support.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/support.gif
--------------------------------------------------------------------------------
/nginx-server/html/tagline.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/tagline.gif
--------------------------------------------------------------------------------
/nginx-server/html/appfoundry.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/appfoundry.gif
--------------------------------------------------------------------------------
/nginx-server/html/h_microsoft.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/h_microsoft.gif
--------------------------------------------------------------------------------
/nginx-server/html/home_igloo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/home_igloo.gif
--------------------------------------------------------------------------------
/nginx-server/html/ie_animated.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/ie_animated.gif
--------------------------------------------------------------------------------
/nginx-server/html/inbox_img.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/inbox_img.gif
--------------------------------------------------------------------------------
/nginx-server/html/msinternet.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/msinternet.gif
--------------------------------------------------------------------------------
/nginx-server/html/solutions.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/solutions.gif
--------------------------------------------------------------------------------
/nginx-server/html/worldwide.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/worldwide.gif
--------------------------------------------------------------------------------
/nginx-server/html/navigation_bar.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/navigation_bar.gif
--------------------------------------------------------------------------------
/nginx-server/html/pointcast_small.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OpenSUTD/50.012_2023_lab1_tests/main/nginx-server/html/pointcast_small.gif
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/proxy/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/proxy/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/nginx-server/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nginx-server/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/test-client/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/test-client/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/fastapi-server/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/fastapi-server/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /shelf/
3 | /workspace.xml
4 | # Editor-based HTTP Client requests
5 | /httpRequests/
6 | # Datasource local storage ignored files
7 | /dataSources/
8 | /dataSources.local.xml
9 | /aws.xml
10 | /misc.xml
11 |
--------------------------------------------------------------------------------
/proxy/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.11
2 | RUN apt update -y && apt install -y moreutils lsof psmisc net-tools
3 | COPY requirements.txt ./requirements.txt
4 | RUN pip install -r requirements.txt
5 | COPY monitor.py /monitor.py
6 | ENTRYPOINT ["python", "-u", "monitor.py"]
--------------------------------------------------------------------------------
/proxy/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/test-client/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.11
2 | WORKDIR /app
3 | COPY ./requirements.txt requirements.txt
4 | RUN pip install -r requirements.txt
5 | COPY tests /app/tests
6 | WORKDIR /app/tests
7 | ENV PYTHONUNBUFFERED 1
8 | CMD "pytest" "${PYTEST_TESTS:=.}" | tee /var/log/pytest/result.log
9 |
10 |
--------------------------------------------------------------------------------
/proxy/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/test-client/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/fastapi-server/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/fastapi-server/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.11
2 |
3 | WORKDIR /code
4 |
5 | COPY ./requirements.txt /code/requirements.txt
6 |
7 | RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
8 |
9 | COPY ./app /code/app
10 |
11 | CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"]
--------------------------------------------------------------------------------
/nginx-server/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nginx-server/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/test-client/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/fastapi-server/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/nginx-server/.idea/nginx-server.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/proxy/.idea/proxy.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/webResources.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/test-client/.idea/test-client.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/fastapi-server/.idea/fastapi-server.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/.idea/NetworksLab1Grading.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/nginx-server/default.conf:
--------------------------------------------------------------------------------
1 |
2 | log_format log_req_resp '{"time": $msec, "remote": "$remote_addr:$remote_port", "request_line": "$request", "status": $status, "response_hash":"$resp_body_hash"}';
3 |
4 | server {
5 | access_log /var/log/nginx/access.log log_req_resp;
6 | lua_need_request_body on;
7 | set $resp_body "";
8 | set $resp_body_hash "";
9 | body_filter_by_lua '
10 | local resp_body = string.sub(ngx.arg[1], 1, 1000)
11 | ngx.ctx.buffered = (ngx.ctx.buffered or "") .. resp_body
12 | if ngx.arg[2] then
13 | ngx.var.resp_body = ngx.ctx.buffered
14 | end
15 | local resty_md5 = require "resty.md5"
16 | local md5 = resty_md5:new()
17 | md5:update(resp_body)
18 | local digest = md5:final()
19 | local str = require "resty.string"
20 | ngx.var.resp_body_hash = str.to_hex(digest)
21 | ';
22 | location / {
23 | index microscape.html;
24 | }
25 | }
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 OpenSUTD
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
--------------------------------------------------------------------------------
/test-client/tests/testutils.py:
--------------------------------------------------------------------------------
1 | import json
2 | from math import floor
3 | from typing import List, Dict, Any
4 |
5 |
6 | def get_nginx_log_entries_after_time(time: float | int) -> List[Dict[str, Any]]:
7 | start_time = floor(time)
8 | with open("/var/log/nginx/access.log", "r") as f:
9 | while True:
10 | line = f.readline().strip()
11 | if not line:
12 | assert False, "Did not seem to find a future log entry in nginx after firing the request"
13 | log_object = json.loads(line)
14 | if log_object["time"] >= start_time:
15 | log_entries = [log_object] + [json.loads(s) for s in f.readlines()]
16 | return log_entries
17 |
18 |
19 | def get_fastapi_log_entries_after_time(time: float | int) -> List[Dict[str, Any]]:
20 | start_time = floor(time)
21 | with open("/var/log/fastapi/access.log", "r") as f:
22 | while True:
23 | line = f.readline().strip()
24 | if not line:
25 | assert False, "Did not seem to find a future log entry in nginx after firing the request"
26 | log_object = json.loads(line)
27 | if log_object["time"] >= start_time:
28 | log_entries = [log_object] + [json.loads(s) for s in f.readlines()]
29 | return log_entries
30 |
--------------------------------------------------------------------------------
/fastapi-server/app/main.py:
--------------------------------------------------------------------------------
1 | import json
2 | import time
3 |
4 | import aiofiles
5 | from fastapi import FastAPI, Body, Request, Response
6 | from starlette.middleware.base import _StreamingResponse
7 |
8 | app = FastAPI()
9 |
10 |
11 | @app.middleware("http")
12 | async def add_to_log(request: Request, call_next):
13 | response: _StreamingResponse = await call_next(request)
14 |
15 | async with aiofiles.open("/var/log/fastapi/access.log", "a") as f:
16 | log_entry = {
17 | "time": time.time(),
18 | "request_line": "{} {} {}".format(request.method.upper(), request.url, "HTTP/1.1"),
19 | "remote": request.client.host + ":{}".format(
20 | request.client.port) if request.client.port is not None else "",
21 | "status": response.status_code,
22 | # "response_hash": hashlib.md5(body).digest()
23 | }
24 | await f.write(json.dumps(log_entry) + "\n")
25 | return response
26 |
27 |
28 | @app.get("/")
29 | async def root():
30 | return {"message": "Hello World"}
31 |
32 |
33 | @app.get("/empty_body")
34 | def empty_body():
35 | return Response(status_code=200)
36 |
37 |
38 | @app.get("/test_query_parameters")
39 | def test_query_parameters(deez: str, nuts: str):
40 | return "idk"
41 |
42 |
43 | @app.post("/test_post")
44 | def test_post(hello: str = Body()):
45 | return "You just posted something"
46 |
47 |
48 | @app.get("/really_big_header")
49 | def really_big_header(response: Response):
50 | for i in range(1024):
51 | response.headers[f"X-REALLY-BIG-HEADER-{i}"] = "ha" * 16
52 | return "You just got big 4head haha"
53 |
54 |
55 | @app.get("/你好")
56 | def test_chinese():
57 | return "Today is very 风和日丽"
58 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: "3.9"
2 | services:
3 | test-client:
4 | build: test-client
5 | image: 50012-lab1-test-client
6 | volumes:
7 | - type: volume
8 | source: nginx-logs
9 | target: /var/log/nginx
10 | - type: volume
11 | source: fastapi-logs
12 | target: /var/log/fastapi/
13 | - type: bind
14 | source: ./nginx-server/html
15 | target: /var/html
16 | read_only: true
17 | - type: bind
18 | source: ./test-client/result
19 | target: /var/log/pytest
20 | - /etc/timezone:/etc/timezone:ro
21 | - /etc/localtime:/etc/localtime:ro
22 | environment:
23 | - PROXY_HOST=proxy
24 | - PROXY_PORT=8080
25 | # Run basic tests only
26 | - PYTEST_TESTS=basic_test.py
27 | # Run smoke tests only
28 | # - PYTEST_TESTS=smoke_test.py
29 | # Default test options
30 | # - PYTEST_ADDOPTS=-rA --tb=short
31 | # Shorter version for basic tests, only sample a subset of static files
32 | - PYTEST_ADDOPTS=-rA --tb=short --proxytest-nginx-static-files-n-samples=3
33 | depends_on:
34 | - proxy
35 | - fastapi-server
36 | - nginx-server
37 | restart: "no"
38 | stop_grace_period: "1s"
39 | proxy:
40 | build: proxy
41 | image: 50012-lab1-test-proxy
42 | volumes:
43 | - type: bind
44 | source: ./proxy/app
45 | target: /app
46 | - type: bind
47 | source: ./proxy/logs
48 | target: /var/log/proxy/
49 | - /etc/timezone:/etc/timezone:ro
50 | - /etc/localtime:/etc/localtime:ro
51 | depends_on:
52 | - nginx-server
53 | ports:
54 | - "8080:8080"
55 | - "3000:3000"
56 | stop_grace_period: "3s"
57 | proxy-wireshark:
58 | image: linuxserver/wireshark
59 | network_mode: "service:proxy"
60 | cap_add:
61 | - NET_ADMIN
62 | environment:
63 | - PUID=1000
64 | - PGID=1000
65 | - TZ=Asia/Singapore
66 | depends_on:
67 | - proxy
68 | nginx-server:
69 | image: openresty/openresty
70 | volumes:
71 | - type: bind
72 | source: nginx-server/html
73 | target: /usr/local/openresty/nginx/html
74 | read_only: true
75 | - type: bind
76 | source: ./nginx-server/default.conf
77 | target: /etc/nginx/conf.d/default.conf
78 | read_only: true
79 | - type: volume
80 | source: nginx-logs
81 | target: /var/log/nginx/
82 | - /etc/timezone:/etc/timezone:ro
83 | - /etc/localtime:/etc/localtime:ro
84 | ports:
85 | - "8000:80"
86 | fastapi-server:
87 | build: fastapi-server
88 | image: 50012-lab1-fastapi-server
89 | command: uvicorn app.main:app --host 0.0.0.0 --port 80 --reload
90 | environment:
91 | DEBUG: 1
92 | volumes:
93 | - ./fastapi-server/app:/code/app
94 | - type: volume
95 | source: fastapi-logs
96 | target: /var/log/fastapi/
97 | - /etc/timezone:/etc/timezone:ro
98 | - /etc/localtime:/etc/localtime:ro
99 | volumes:
100 | nginx-logs:
101 | name: nginx-logs
102 | fastapi-logs:
103 | name: fastapi-logs
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Specific to this project
2 | **/*.log
3 | proxy/app/cache
4 | proxy/app/*
5 |
6 | # Byte-compiled / optimized / DLL files
7 | __pycache__/
8 | *.py[cod]
9 | *$py.class
10 |
11 | # C extensions
12 | *.so
13 |
14 | # Distribution / packaging
15 | .Python
16 | build/
17 | develop-eggs/
18 | dist/
19 | downloads/
20 | eggs/
21 | .eggs/
22 | lib/
23 | lib64/
24 | parts/
25 | sdist/
26 | var/
27 | wheels/
28 | share/python-wheels/
29 | *.egg-info/
30 | .installed.cfg
31 | *.egg
32 | MANIFEST
33 |
34 | # PyInstaller
35 | # Usually these files are written by a python script from a template
36 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
37 | *.manifest
38 | *.spec
39 |
40 | # Installer logs
41 | pip-log.txt
42 | pip-delete-this-directory.txt
43 |
44 | # Unit test / coverage reports
45 | htmlcov/
46 | .tox/
47 | .nox/
48 | .coverage
49 | .coverage.*
50 | .cache
51 | nosetests.xml
52 | coverage.xml
53 | *.cover
54 | *.py,cover
55 | .hypothesis/
56 | .pytest_cache/
57 | cover/
58 |
59 | # Translations
60 | *.mo
61 | *.pot
62 |
63 | # Django stuff:
64 | *.log
65 | local_settings.py
66 | db.sqlite3
67 | db.sqlite3-journal
68 |
69 | # Flask stuff:
70 | instance/
71 | .webassets-cache
72 |
73 | # Scrapy stuff:
74 | .scrapy
75 |
76 | # Sphinx documentation
77 | docs/_build/
78 |
79 | # PyBuilder
80 | .pybuilder/
81 | target/
82 |
83 | # Jupyter Notebook
84 | .ipynb_checkpoints
85 |
86 | # IPython
87 | profile_default/
88 | ipython_config.py
89 |
90 | # pyenv
91 | # For a library or package, you might want to ignore these files since the code is
92 | # intended to run in multiple environments; otherwise, check them in:
93 | # .python-version
94 |
95 | # pipenv
96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
99 | # install all needed dependencies.
100 | #Pipfile.lock
101 |
102 | # poetry
103 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
104 | # This is especially recommended for binary packages to ensure reproducibility, and is more
105 | # commonly ignored for libraries.
106 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
107 | #poetry.lock
108 |
109 | # pdm
110 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
111 | #pdm.lock
112 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
113 | # in version control.
114 | # https://pdm.fming.dev/#use-with-ide
115 | .pdm.toml
116 |
117 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
118 | __pypackages__/
119 |
120 | # Celery stuff
121 | celerybeat-schedule
122 | celerybeat.pid
123 |
124 | # SageMath parsed files
125 | *.sage.py
126 |
127 | # Environments
128 | .env
129 | .venv
130 | env/
131 | venv/
132 | ENV/
133 | env.bak/
134 | venv.bak/
135 |
136 | # Spyder project settings
137 | .spyderproject
138 | .spyproject
139 |
140 | # Rope project settings
141 | .ropeproject
142 |
143 | # mkdocs documentation
144 | /site
145 |
146 | # mypy
147 | .mypy_cache/
148 | .dmypy.json
149 | dmypy.json
150 |
151 | # Pyre type checker
152 | .pyre/
153 |
154 | # pytype static type analyzer
155 | .pytype/
156 |
157 | # Cython debug symbols
158 | cython_debug/
159 | /.idea/deployment.xml
160 |
--------------------------------------------------------------------------------
/test-client/tests/smoke_test.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | import asyncio
4 | import hashlib
5 | import httpx
6 | from math import floor
7 | from typing import Callable, List, Coroutine
8 |
9 | from testutils import get_nginx_log_entries_after_time, get_fastapi_log_entries_after_time
10 |
11 |
12 | def test_empty_body(make_httpx_client: Callable[..., httpx.Client]):
13 | """
14 | I hope you didn't assume that the body always has something
15 | """
16 | client = make_httpx_client()
17 | response = client.request(
18 | method="GET",
19 | url="http://fastapi-server/empty_body"
20 | )
21 | assert response.status_code == 200
22 | assert len(response.read()) == 0
23 |
24 |
25 | def test_really_big_header(make_httpx_client: Callable[..., httpx.Client]):
26 | """
27 | Did you assume that you could read the entire HTTP header in a single socket.recv(4096) call? Oops!
28 | """
29 | client = make_httpx_client()
30 | response = client.request(
31 | method="GET",
32 | url="http://fastapi-server/really_big_header"
33 | )
34 | for i in range(1024):
35 | assert f"X-REALLY-BIG-HEADER-{i}" in response.headers
36 | assert response.headers.get(f"X-REALLY-BIG-HEADER-{i}") == "ha" * 16
37 |
38 | assert response.read().decode() == """\"You just got big 4head haha\""""
39 |
40 |
41 | def test_chinese(make_httpx_client: Callable[..., httpx.Client]):
42 | """
43 | Mr worldwide :)
44 | """
45 | client = make_httpx_client()
46 | response = client.request(
47 | method="GET",
48 | url="http://fastapi-server/你好"
49 | )
50 | assert response.read().decode() == """\"Today is very 风和日丽\""""
51 |
52 |
53 | def test_cache_stampede_does_not_produce_corrupted_output(make_async_httpx_client: Callable[..., httpx.AsyncClient]):
54 | """
55 | What happens when a horde of elephants (I mean clients) rush to request the same resource from the proxy at once? Only one way to find out...
56 | """
57 | # try to find the original on disk
58 | with open(f"/var/html/home_igloo.gif", "rb") as f:
59 | md5_of_file_on_disk = hashlib.md5(f.read()).digest()
60 | clients: List[httpx.AsyncClient] = [make_async_httpx_client() for i in range(128)]
61 |
62 | async def make_get_request_and_return_hash(client: httpx.AsyncClient) -> bytes:
63 | response = await client.get("http://nginx-server/home_igloo.gif", timeout=60)
64 | return hashlib.md5(response.read()).digest()
65 |
66 | client_get_requests: List[Coroutine] = [make_get_request_and_return_hash(c) for c in clients]
67 | loop = asyncio.get_event_loop()
68 | gathered_task = asyncio.gather(*client_get_requests)
69 | loop.run_until_complete(gathered_task)
70 | result = gathered_task.result()
71 | for md5_of_response in result:
72 | assert md5_of_response == md5_of_file_on_disk
73 |
74 |
75 | def test_404_should_not_be_cached(make_httpx_client: Callable[..., httpx.Client]):
76 | """
77 | Tests that 404 responses should not be cached. We should see two requests in the NGINX log.
78 | :param make_httpx_client: pytest fixture, closure that produces a http client with the configured proxy
79 | """
80 |
81 | client = make_httpx_client()
82 | start_time = floor(time.time())
83 | client.request(method="GET", url=f"http://nginx-server/404")
84 | time.sleep(1)
85 | client.request(method="GET", url=f"http://nginx-server/404")
86 | relevant_log_entries = [l for l in get_nginx_log_entries_after_time(start_time) if
87 | l["request_line"] == "GET http://nginx-server/404 HTTP/1.1"]
88 | assert len(relevant_log_entries) == 2, "There must be exactly two requests made after the start_time"
89 |
90 |
91 | def test_post_should_not_be_cached(make_httpx_client: Callable[..., httpx.Client]):
92 | """
93 | Tests that POST requests should not be cached. We should see two requests in the FastAPI log.
94 | :param make_httpx_client: pytest fixture, closure that produces a http client with the configured proxy
95 | """
96 | client = make_httpx_client()
97 | start_time = floor(time.time())
98 | client.request(method="POST", url=f"http://fastapi-server/test_post", data={"hello": "world"})
99 | time.sleep(1)
100 | client.request(method="POST", url=f"http://fastapi-server/test_post", data={"hello": "world"})
101 | relevant_log_entries = [l for l in get_fastapi_log_entries_after_time(start_time) if
102 | l["request_line"] == "POST http://fastapi-server/test_post HTTP/1.1"]
103 | assert len(relevant_log_entries) == 2, "There must be exactly two requests made after the start_time"
104 |
--------------------------------------------------------------------------------
/test-client/tests/basic_test.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | import hashlib
4 | import httpx
5 | import pytest
6 | from math import floor
7 | from typing import Callable
8 |
9 | from testutils import get_nginx_log_entries_after_time
10 |
11 |
12 | def test_request_neverssl_receives_response(make_httpx_client):
13 | client = make_httpx_client()
14 | response = client.request(
15 | method="GET",
16 | url="http://neverssl.com"
17 | )
18 | body = response.read().decode("utf8")
19 | assert "NeverSSL" in body
20 |
21 |
22 | def test_request_nginx_receives_response(make_httpx_client: Callable[..., httpx.Client], check_proxy_alive):
23 | client = make_httpx_client()
24 | response = client.request(
25 | method="GET",
26 | url=f"http://nginx-server/"
27 | )
28 | body = response.read().decode("utf8")
29 | assert "netscape" in body
30 |
31 |
32 | def test_request_nginx_body_unchanged(make_httpx_client: Callable[..., httpx.Client], nginx_static_file: str):
33 | """
34 | Tests that all files return the exact same value from disk
35 | :param make_httpx_client: pytest fixture, closure that produces a http client with the configured proxy
36 | :param nginx_every_file: pytest parametrized fixture, returns one filename at a time
37 | """
38 | client = make_httpx_client()
39 | response = client.request(
40 | method="GET",
41 | url=f"http://nginx-server/{nginx_static_file}"
42 | )
43 | md5_of_proxy_response = hashlib.md5(response.read()).digest()
44 | # try to find the original on disk
45 | with open(f"/var/html/{nginx_static_file}", "rb") as f:
46 | md5_of_file_on_disk = hashlib.md5(f.read()).digest()
47 | assert md5_of_proxy_response == md5_of_file_on_disk
48 |
49 |
50 | @pytest.mark.parametrize("request_a,request_b,request_c",
51 | [
52 | ("", "", ""),
53 | ("", "", "action.gif"),
54 | ("", "", "404"),
55 | ("", "action.gif", "action.gif"),
56 | ("", "404", "404"),
57 | ("404", "404", "404")
58 | ],
59 | ids=[
60 | "all_request_index",
61 | "first_two_index_last_one_200",
62 | "first_two_index_last_one_404",
63 | "first_index_last_two_200",
64 | "first_index_last_two_404",
65 | "all_404"
66 | ])
67 | def test_closing_one_client_does_not_affect_other_clients(make_httpx_client: Callable[..., httpx.Client],
68 | request_a: str, request_b: str, request_c: str):
69 | """
70 | Tests that after two clients connect to the proxy and make requests, if one closes the other should not be affected
71 | :param make_httpx_client: pytest fixture, closure that produces a http client with the configured proxy
72 | """
73 | client_1 = make_httpx_client()
74 | client_1.request(method="GET", url=f"http://nginx-server/{request_a}")
75 | client_2 = make_httpx_client()
76 | client_2.request(method="GET", url=f"http://nginx-server/{request_b}")
77 | client_1.close()
78 | client_2.request(method="GET", url=f"http://nginx-server/{request_c}")
79 |
80 |
81 | def test_multiple_calls_to_same_resource_should_be_cached_single_client(make_httpx_client: Callable[..., httpx.Client]):
82 | """
83 | Requesting the same resource twice should serve from the cache. We will assert from the NGINX logs that the upstream server (NGINX) only received one request from your proxy.
84 | :param make_httpx_client: pytest fixture, closure that produces a http client with the configured proxy
85 | """
86 | client = make_httpx_client()
87 | start_time = floor(time.time())
88 | client.request(method="GET", url=f"http://nginx-server/")
89 | time.sleep(1)
90 | client.request(method="GET", url=f"http://nginx-server/")
91 | relevant_log_entries = [l for l in get_nginx_log_entries_after_time(start_time) if
92 | l["request_line"] == "GET http://nginx-server/ HTTP/1.1"]
93 | assert len(relevant_log_entries) == 1, "There must be exactly one request made after the start_time"
94 |
95 |
96 | def test_cached_resources_should_be_namespaced_by_domain(make_httpx_client: Callable[..., httpx.Client]):
97 | """
98 | If I make a request to site-a.com/index.html and then a request to site-b.com/index.html, the contents should not be the same
99 | """
100 | client = make_httpx_client()
101 | response_a = client.get(url=f"http://nginx-server/")
102 | response_b = client.get(url=f"http://fastapi-server/")
103 | response_a_hash = hashlib.md5(response_a.read()).digest()
104 | response_b_hash = hashlib.md5(response_b.read()).digest()
105 | assert response_b_hash != response_a_hash
106 |
--------------------------------------------------------------------------------
/test-client/tests/conftest.py:
--------------------------------------------------------------------------------
1 | import time
2 | from os.path import isfile, join
3 |
4 | import asyncio
5 | import httpx
6 | import os
7 | import pytest
8 | import socket
9 | import zmq
10 | from random import sample
11 | from typing import Callable, List, Generator, Optional
12 |
13 |
14 | @pytest.fixture(scope="session")
15 | def proxy_host():
16 | return os.environ["PROXY_HOST"]
17 |
18 |
19 | @pytest.fixture(scope="session")
20 | def proxy_port():
21 | return int(os.environ["PROXY_PORT"])
22 |
23 |
24 | @pytest.fixture(scope="session")
25 | def proxy_address(proxy_host, proxy_port):
26 | return f"{proxy_host}:{proxy_port}"
27 |
28 |
29 | @pytest.fixture()
30 | def check_proxy_alive(proxy_host, proxy_port):
31 | def check_proxy_alive_closure():
32 | attempts = 1
33 | while attempts <= 10:
34 | try:
35 | with socket.create_connection((proxy_host, proxy_port), timeout=10):
36 | return True
37 | except OSError:
38 | attempts += 1
39 | time.sleep(2)
40 | return False
41 |
42 | return check_proxy_alive_closure
43 |
44 |
45 | @pytest.fixture(scope="session")
46 | def zmq_socket(proxy_host):
47 | context = zmq.Context()
48 | _socket = context.socket(zmq.REQ)
49 | _socket.connect(f"tcp://{proxy_host}:5555")
50 | yield _socket
51 | _socket.close()
52 | context.term()
53 |
54 |
55 | def try_and_get_response_from_zmq_server(zmq_socket: zmq.Socket, timeout: float) -> str:
56 | response = None
57 | time_elapsed = 0
58 | while response is None and time_elapsed < timeout:
59 | try:
60 | response = zmq_socket.recv(zmq.NOBLOCK).decode("utf-8")
61 | except zmq.ZMQError:
62 | time.sleep(0.2)
63 | time_elapsed += 0.1
64 | continue
65 | return response
66 |
67 |
68 | premature_exit: bool = False
69 |
70 |
71 | @pytest.fixture()
72 | def restart_proxy(proxy_host, zmq_socket: zmq.Socket, request):
73 | def restart_proxy_closure():
74 | zmq_socket.send(f"begin_test {request.node.name}".encode("utf-8"))
75 | response = try_and_get_response_from_zmq_server(zmq_socket, 15)
76 | if response != "restarted":
77 | global premature_exit
78 | premature_exit = True
79 | pytest.exit("The proxy runner was not able to restart your proxy."
80 | + " Check the logs of the proxy runner container and restart it if necessary.")
81 |
82 | return restart_proxy_closure
83 |
84 |
85 | @pytest.fixture(scope="function", autouse=True)
86 | def setup_proxy_per_test(proxy_host, proxy_port, check_proxy_alive, restart_proxy):
87 | restart_proxy()
88 | assert check_proxy_alive(), "The proxy does not seem to be running before the test starts"
89 | yield
90 | assert check_proxy_alive(), "The proxy seems to have died after running the tests"
91 |
92 |
93 | @pytest.fixture(scope="session", autouse=True)
94 | def cleanup(proxy_host, zmq_socket: zmq.Socket, request):
95 | def send_end_tests_message():
96 | if premature_exit:
97 | return
98 | zmq_socket.send(f"end_tests".encode("utf-8"))
99 | response = try_and_get_response_from_zmq_server(zmq_socket, 15)
100 | if response != "ended":
101 | pytest.exit("The proxy runner was not able to restart your proxy at the end of the test suite."
102 | + " Check the logs of the proxy runner container and restart it if necessary.")
103 |
104 | request.addfinalizer(send_end_tests_message)
105 |
106 |
107 | @pytest.fixture()
108 | def make_httpx_client(proxy_address) -> Generator[Callable[..., httpx.Client], None, None]:
109 | clients: List[httpx.Client] = []
110 |
111 | def httpx_client_closure() -> httpx.Client:
112 | proxies = {
113 | "all://": f"http://{proxy_address}",
114 | }
115 | client = httpx.Client(proxies=proxies, timeout=5)
116 | clients.append(client)
117 | return client
118 |
119 | yield httpx_client_closure
120 | for client in clients:
121 | client.close()
122 |
123 |
124 | @pytest.fixture()
125 | def make_async_httpx_client(proxy_address) -> Generator[Callable[..., httpx.AsyncClient], None, None]:
126 | clients: List[httpx.AsyncClient] = []
127 |
128 | def httpx_client_closure() -> httpx.AsyncClient:
129 | proxies = {
130 | "all://": f"http://{proxy_address}",
131 | }
132 | client = httpx.AsyncClient(proxies=proxies, timeout=5)
133 | clients.append(client)
134 | return client
135 |
136 | yield httpx_client_closure
137 | loop = asyncio.new_event_loop()
138 | client_closing_coroutines = [loop.create_task(c.aclose()) for c in clients]
139 | loop.run_until_complete(asyncio.gather(*client_closing_coroutines))
140 | loop.close()
141 |
142 |
143 | def nginx_list_static_files(n_samples: Optional[int]) -> List[str]:
144 | nginx_file_path = "/var/html"
145 | list_of_files = [f for f in os.listdir(nginx_file_path) if isfile(join(nginx_file_path, f))]
146 | if n_samples:
147 | return sample(list_of_files, n_samples)
148 | else:
149 | return list_of_files
150 |
151 |
152 | def pytest_addoption(parser):
153 | parser.addoption(
154 | "--proxytest-nginx-static-files-n-samples",
155 | default=None,
156 | type=int,
157 | required=False,
158 | help="if set, samples n files for testing with the proxy"
159 | )
160 |
161 |
162 | def pytest_generate_tests(metafunc: pytest.Metafunc):
163 | if "nginx_static_file" in metafunc.fixturenames:
164 | n_samples = metafunc.config.getoption("--proxytest-nginx-static-files-n-samples")
165 | metafunc.parametrize("nginx_static_file", nginx_list_static_files(n_samples))
166 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 50.012 Networks Lab 1 Test Suite
2 |
3 | This project implements an automated test suite for the HTTP proxy lab. It spins up a few test web servers in a docker network and a test monitor that starts new instances of your proxies on demand. Finally there is a run-once docker container containing a pytest suite that that communicates with the monitor to test your proxy. All these docker containers are orchestrated as a single docker-compose project.
4 |
5 | ## Usage
6 |
7 | ### Installing your proxy code
8 |
9 | Place your proxy Python files inside `proxy/app` (create the folder if it doesn't exist). You can have as many files as you like but your application must conform to the following requirements:
10 |
11 | - There must be a `proxy.py` file as this will be the entrypoint called by the monitor.
12 | - Your application must start the proxy on port `8080` when no arguments are passed via the CLI.
13 | - Your application must clear the contents of the cache folder (if you save it on disk) before serving clients on the HTTP proxy.
14 | - Your application must handle `SIGINT` gracefully, releasing all resources and exiting within 5 seconds.
15 |
16 | ### Starting the environment
17 |
18 | Create the folders for the bind mounts, you only need to do this once.
19 |
20 | ```
21 | mkdir -p proxy/logs && mkdir -p test-client/result
22 | ```
23 |
24 | Then deploy the compose file:
25 |
26 | ```
27 | docker compose up
28 | ```
29 |
30 | Deploying the compose project automatically starts the full basic test suite (no smoke tests). Wait for the `test-client` service to exit. In another terminal window you can watch the stdout of the test-client using:
31 |
32 | ```
33 | docker compose logs test-client -f
34 | ```
35 |
36 | ### Viewing results
37 |
38 | #### Overall test results
39 |
40 | The overral test results indicating test successes and failures will be saved in the file `test-client/result/result.log`. Only the latest invocation's results will be saved.
41 |
42 | #### stdout/stderr for each test
43 |
44 | A fresh instance of your proxy is started for each test that is run. You can go to `proxy/logs/{test-name}.log` to see what was printed to stdout and stderr by your proxy during each test. Only the latest invocation results will be saved.
45 |
46 | #### Latest cache results
47 |
48 | The cache directory will be left in the state of the last test run as it is cleaned at the start of each test. If debugging errors it is suggested you run one test at a time using the `-k` argument in `PYTEST_ADDOPTS`.
49 |
50 | > If you open a log file in the editor and re-run the test suite, you need to reload the file from disk to get the latest version.
51 |
52 | #### Proxy Monitor Log
53 |
54 | The monitor is the process that runs your proxy repeatedly on demand. You can inspect the logs of the `proxy` container (`docker compose logs proxy`) to check for any abnormalities, such as detecting the TIME_WAIT bug (see below).
55 |
56 | ### Running the tests again
57 |
58 | You can edit the files in the `proxy/app` folder directly if you are working to pass the tests. When you are ready to re-run the tests, restart the `test-client` container:
59 |
60 | ```
61 | docker-compose start test-client
62 | ```
63 |
64 | #### Running a different suite
65 |
66 | By default, the test client only runs the `basic_test` suite. To run the extended smoke tests, edit the docker-compose.yml file and swap the commented lines under `services.test-client-environment`:
67 |
68 | ```yaml
69 |
70 | # Run basic tests only
71 | - PYTEST_TESTS=basic_test.py
72 | # Run smoke tests only
73 | # - PYTEST_TESTS=smoke_test.py
74 | ```
75 |
76 | Changes to the compose file require you to update the deployment, just run `docker-compose up -d` again
77 |
78 | #### Different test arguments, speeding up the test
79 |
80 | The full test suite that will be using for grading tests every single static file in the `nginx/html` directory. When doing iterative testing this can get quite cumbersome. In the `docker-compose.yml` file, you can pass the add the pytest opt to control how many samples you want to test:
81 |
82 | ```yaml
83 | - PYTEST_ADDOPTS=-ra --tb=short --proxytest-nginx-static-files-n-samples=3
84 | ```
85 |
86 | As above, changes to the compose file require you to update the deployment, just run `docker-compose up -d` again
87 |
88 | #### Running a single test
89 |
90 | Use the `-k` argument in `PYTEST_ADDOPTS`. See pytest documentation on [run tests by keyword expressions](https://docs.pytest.org/en/7.1.x/how-to/usage.html#specifying-which-tests-to-run).
91 |
92 | ### Troubleshooting & Debugging
93 |
94 | #### Test Client Hanging / TCP TIME_WAIT
95 |
96 | If the client does not proceed for more than a minute, check the logs of the proxy container to see if the monitor has crashed. Your code might be suffering from the [TCP `TIME_WAIT` bug](http://hea-www.harvard.edu/~fine/Tech/addrinuse.html) if the monitor reports something like this in the logs:
97 |
98 | ```
99 | WARNING: still found port used by process None in TIME_WAIT state
100 | ```
101 |
102 | If the monitor has indeed crashed or stopped responding without any other errors, it might be a bug in the monitor. Raise issue on Github & contact your TA.
103 |
104 | In any case, if you need to force abort the test, you can stop the test-client container, then stop the proxy container, then restart the proxy container, and finally restart the test-client container when you are ready.
105 |
106 | #### Wireshark
107 |
108 | A wireshark instance served over RDP is available in your web browser at `http://localhost:3000`. It runs on the proxy instance. So for example, you can capture all downstream HTTP-in-TCP datagrams using the packet filter `tcp port 8080`.
109 |
110 | If the application stops responding, restart the `proxy-wireshark` container.
111 |
112 | ## Test Technicals
113 |
114 | ### Static Website tests
115 |
116 | The `nginx-server` container serves static files from the `nginx-server/html` directory. You can add your own files to test here. The parametrized test `basic_test::test_request_nginx_body_unchanged` samples a list of static files in the html directory and tries to fetch the resources.
117 |
118 | ### Smoke Tests
119 |
120 | The `smoke_test` suite is an extended suite for extreme edge cases. You will not be graded based on your results on this, but you can try it for fun.
121 |
122 | ### Your own tests
123 |
124 | Feel free to add your own test files under the `tests` directory.
125 |
126 | ## Development
127 |
128 | First developed by Chester Koh for the Spring 2023 Term.
129 |
130 | ### Things to update for future batches
131 |
132 | - More basic tests, such as caching query parameters
133 | - Bonus tests for extra features like ETAG and HTTPS
134 | - Test downloads of large files (scale of megabytes)
135 | - Test for TIME_WAIT bug
136 | - Rename the `proxy` service in docker compose to `proxy-monitor` for avoidance of doubt.
--------------------------------------------------------------------------------
/proxy/monitor.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | import psutil
4 | import signal
5 | import subprocess
6 | import zmq
7 | from typing import Optional
8 |
9 | context = zmq.Context()
10 | socket = context.socket(zmq.REP)
11 | socket.bind("tcp://*:5555")
12 |
13 | print("Bound ZMQ socket, launching idle proxy")
14 |
15 |
16 | def wait_python_proxy_ready(pid: int):
17 | python_process = psutil.Process(pid)
18 | time_elapsed = 0
19 | while True:
20 | python_connections = python_process.connections(kind="tcp4")
21 | for connection in python_connections:
22 | if connection.laddr.port == 8080:
23 | return
24 | time.sleep(0.1)
25 | if time_elapsed >= 10:
26 | raise TimeoutError("Python proxy did not managed to bind to port 8080 after waiting 10 seconds. "
27 | "Check the individual log file.")
28 |
29 |
30 | def launch_proxy_process(experiment_name: str) -> psutil.Process:
31 | logfile = f"/var/log/proxy/{experiment_name}.log"
32 | p = subprocess.Popen(f"exec python -u /app/proxy.py 2>&1 | ts > {logfile}", shell=True,
33 | stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd="/app")
34 | return psutil.Process(p.pid)
35 |
36 |
37 | def launch_proxy_and_wait(experiment_name: str) -> psutil.Process:
38 | shell_process = launch_proxy_process(experiment_name)
39 | print(f"Launched sh subprocess as {shell_process.pid}")
40 | attempts = 0
41 | while not shell_process.children():
42 | print("Wait for sh subprocess to spawn children...")
43 | time.sleep(0.2)
44 | attempts += 1
45 | if attempts > 10:
46 | raise TimeoutError("sh subprocess did not spawn children after a reasonable time. "
47 | "Check if proxy.py exists in the app folder.")
48 | try:
49 | python_process = next(filter(lambda cp: "python" in cp.name(), shell_process.children()))
50 | except StopIteration:
51 | raise Exception(f"Could not find a python child process - it might have died immediately."
52 | f"Check the log file {experiment_name}.log")
53 | wait_python_proxy_ready(python_process.pid)
54 | print("Proxy ready")
55 | return shell_process
56 |
57 |
58 | def shutdown_proxy_and_wait(shell_process: psutil.Process):
59 | python_process = next(filter(lambda cp: "python" in cp.name(), shell_process.children()))
60 | print(f"Sending SIGINT to process {python_process.pid}")
61 | python_process.send_signal(signal.SIGINT)
62 | time_elapsed = 0
63 | while python_process.is_running() and time_elapsed < 5:
64 | time.sleep(0.1)
65 | time_elapsed += 0.1
66 | while python_process.is_running():
67 | print(f"Process {python_process.pid} is still running, sending SIGTERM")
68 | python_process.kill()
69 | time.sleep(1)
70 | # check if port 8080 is free
71 | port_available = None
72 | time_elapsed = 0
73 | while port_available is None or port_available is False:
74 | port_available = True
75 | for connection in psutil.net_connections(kind="inet"):
76 | if connection.laddr.port == 8080:
77 | print(f"WARNING: still found port used by process {connection.pid} in {connection.status} state. "
78 | "Retrying in 1 second...")
79 | if connection.pid is None:
80 | print("WARNING: netstat reports the port is being used by an exited process - "
81 | "did you close your sockets on program exit??")
82 | port_available = False
83 | time.sleep(1)
84 | time_elapsed += 1
85 | break
86 | if time_elapsed > 5:
87 | raise TimeoutError("Port was still not free after 5 seconds :(")
88 |
89 |
90 | current_shell_process: Optional[psutil.Process] = None
91 |
92 | try:
93 | current_shell_process: Optional[psutil.Process] = launch_proxy_and_wait(experiment_name="pre-experiment-idle")
94 | except Exception as ex:
95 | print("FATAL: Could not start the proxy before experiment starts:")
96 | print(str(ex))
97 | print("The proxy runner will still be available, "
98 | "but it is recommended to investigate and replace the proxy.py file.")
99 |
100 | sigterm_caught = False
101 |
102 |
103 | def on_sigterm(signum, frame):
104 | global sigterm_caught
105 | print("Caught sigterm")
106 | sigterm_caught = True
107 |
108 |
109 | signal.signal(signal.SIGTERM, on_sigterm)
110 | should_say_waiting = True
111 | while not sigterm_caught:
112 | # Wait for next request from client
113 | if should_say_waiting:
114 | print("Waiting for command on ZMQ socket...")
115 | should_say_waiting = False
116 | try:
117 | message_b = socket.recv(zmq.NOBLOCK)
118 | except zmq.ZMQError:
119 | time.sleep(1)
120 | continue
121 | should_say_waiting = True
122 | msg: str = message_b.decode("utf-8")
123 | code, data = msg.split(" ", maxsplit=1) if " " in msg else (msg, None)
124 | print("Received", code)
125 | if code == "begin_test":
126 | print("Beginning test: ", data)
127 | if current_shell_process is not None:
128 | try:
129 | shutdown_proxy_and_wait(current_shell_process)
130 | except TimeoutError as ex:
131 | print("Could not kill the existing proxy instance at the start of test:")
132 | print(str(ex))
133 | socket.send("could_not_end".encode("utf-8"))
134 | current_shell_process = None
135 | continue
136 | try:
137 | current_shell_process = launch_proxy_and_wait(experiment_name=data)
138 | except Exception as ex:
139 | print("Could not start a new instance of the proxy at proxy at the start of the test")
140 | print(str(ex))
141 | socket.send("ended_but_could_not_restart".encode("utf-8"))
142 | continue
143 | socket.send("restarted".encode("utf-8"))
144 | elif code == "end_tests":
145 | if current_shell_process is not None:
146 | try:
147 | shutdown_proxy_and_wait(current_shell_process)
148 | except TimeoutError as ex:
149 | print("Could not kill the proxy instance at the end of test suite:")
150 | print(str(ex))
151 | socket.send("could_not_end".encode("utf-8"))
152 | current_shell_process = None
153 | continue
154 | try:
155 | current_shell_process = launch_proxy_and_wait(experiment_name="post-experiment-idle")
156 | except Exception as ex:
157 | print("Could not start a new instance of the proxy at the end of experiments:")
158 | print(str(ex))
159 | socket.send("ended_but_could_not_restart".encode("utf-8"))
160 | continue
161 | socket.send("ended".encode("utf-8"))
162 | else:
163 | print("Received unknown message")
164 | socket.send(b"???")
165 |
166 | print("Closing ZMQ server socket")
167 | socket.close()
168 | print("Closing ZMQ server context")
169 | context.term()
170 | print("Goodbye")
171 |
--------------------------------------------------------------------------------
/nginx-server/html/m.d.html:
--------------------------------------------------------------------------------
1 | x��<�s�F����!��dל�gl��e�x&�[[�Fj�cI�O-���~�k�Hv<�2Sq� ����>Z�xw5��?Otr;���}o8�J�^�tدׯ�Wa�a�U��JG��{�һW�r1̇z�� a32���b�k�C_��A]vQ���w���'U���/+�AV��*�?������zڬ5����e���y���nk��FMx��G}��Ѭ�}�
;���C|/`ఙ�GSLg�>'#��kҵ��
Jf��0/��E=�!|�����]�`���RxE��:
'��<;;�5ڵ��qz�<�6���{r�^'5�J�Y�呟� �6�g���?W�Ag/�,���ٓ���o��ɔY�Jf��#�g��|�?����o�ȕ�:��&�>ؤ�wϭ�0��W�;�e�a�:�b��'�ɥ����U�ٹF�;���
���2�S��sR[8�qWLwU�K�lhg�i��f`�d���������č>ҧ��x�]@|�E뀫a��?��_��2�N�DZ���)�G��?�����P��]��W��\�&���~*�w��Sxr�~*d8�u�ۏ���u8��w?ԯ�w���`tY���x��u4�bҹxW����a�;}�ͻ�y�ڹ�O�q��
u�ׇ�٤��n.�nҽ�
�>
�淗�f��c���S����Ò�� ]��G��ft t.w7d6�_V֠�����K���I��w:@KF�|��N�
��7���f�$��}|ڹ�{����T����a��=�x��[�_H8�UR���6��xz�OaS�J�f�P�q�f�qCf�u�-i`�5*���ޙu#N�/�x��[P2���r>�V C��¾�g���!"�˅B�:��U�'/,s��[�cK�y�z�Z������Ŏ�����.5����R�>�1�+��Wً�1f�Ə��FvD�~��0}��S��Wp�-��[��,b�kPM�����y&u��z�Z�Y� �D�f���/�-�Fx�I�po�9U;�������i���'��H�${��4�J����4��`vם���_��M���!^��Js�������C 2JN]���'�x��]뵎���|���ò*o�
D�p+��rf��"��n�o���ud�Vh�tnFB��T�r�H����W<����/����oSF�b_$�g������m�nf�2��6��xc%z�ߏ��Q9�B�,] ���
5H�X��n������<�n8���^�&z+�+Ņ#��*��,�s>��D�z��%u9�`�{5Ӳ~�����I�X���pv���}�������j��>�5pi����:�<��x����QS ������q���1)�`2i�Zh?l�npJ�Zւ�S:zXFkl�z��%�w���>[�2���4}NJ��� �j�n���tr0�l#��+�؆Y�M���l��2
�e�2��'f>߰�7�#D9��G% v}
���ۊ�{���@v�Ͽ9��rg�G��a t/#��:�]r��2���[���~s�D�*�0yZ��}7�ts� ��5v0��0i���`k������{�������[���j�V>�]����Y��������#=��Α F=Ol������K�W8}