├── mlab_figures ├── figures │ ├── .gitkeep │ └── .gitignore ├── .gitignore ├── README.md ├── scraped_data │ ├── starlink_countries_2023-10-10.csv │ └── starlink_countries_2024-02-05.csv ├── orbit_utils.py ├── iso3.json ├── geospatial_plot_utils.py └── plot_utils.py ├── artifacts_available.jpg ├── controlled ├── 16a │ ├── schema.sql │ ├── create_sqlite_db.sh │ ├── parse_irtt_json.py │ ├── generate_fig_16a.sh │ └── plot.py ├── 16c │ ├── generate_fig_16c.sh │ └── plot.py ├── 16b │ ├── generate_fig_16b.sh │ ├── parse_iperf.py │ └── plot.py └── README.md ├── .gitmodules ├── zoom ├── process_pcap.sh ├── process_pcaps.sh ├── plot.sh ├── README.md ├── zoom_parser.py ├── plot_xtime_figure.py └── boxplots.py ├── requirements.txt ├── gaming ├── plot.sh ├── preprocessing.sh ├── README.md ├── compute_delay.py ├── inspect_xtime.py └── compute_overall_metrics.py ├── plot_all.sh ├── ripe_atlas_figures └── README.md ├── mno_related ├── README.md └── mno_list_withasn.csv └── README.md /mlab_figures/figures/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mlab_figures/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.html 3 | -------------------------------------------------------------------------------- /mlab_figures/figures/.gitignore: -------------------------------------------------------------------------------- 1 | *.pdf 2 | *.svg 3 | -------------------------------------------------------------------------------- /artifacts_available.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nitindermohan/multifaceted-starlink-performance/HEAD/artifacts_available.jpg -------------------------------------------------------------------------------- /controlled/16a/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE irtt( 2 | epoch INTEGER PRIMARY KEY, 3 | send INTEGER, 4 | receive INTEGER, 5 | rtt INTEGER 6 | ); -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "fig-11-gaming/decaf"] 2 | path = gaming/decaf 3 | url = https://github.com/hendrikcech/decaf.git 4 | branch = webconf-2024 5 | [submodule "fig-10-zoom/zoom-analysis"] 6 | path = zoom/zoom-analysis 7 | url = git@github.com:hendrikcech/zoom-analysis.git 8 | -------------------------------------------------------------------------------- /zoom/process_pcap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -eu 4 | 5 | if [ -z "${1:-}" ]; then 6 | echo "Usage: $0 " 7 | exit 1 8 | fi 9 | 10 | PCAP="$1" 11 | DIR="$(dirname "$PCAP")" 12 | 13 | zoom_flows --in "$PCAP" --zpkt-out "$DIR/zpkt.zpkt" 14 | zoom_rtp --in "$DIR/zpkt.zpkt" -s "$DIR/streams.csv" -p "$DIR/packets.csv" -f "$DIR/frames.csv" -t "$DIR/stats.csv" 15 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==2.2.0 2 | numpy==1.26.4 3 | tqdm==4.66.2 4 | matplotlib==3.8.3 5 | pyasn==1.6.2 6 | geopandas==0.14.3 7 | Cartopy==0.22.0 8 | scipy==1.12.0 9 | haversine==2.8.1 10 | mapclassify==2.6.1 11 | jupyter==1.0.0 12 | python-geohash==0.8.5 13 | descartes==1.1.0 14 | h3==3.7.6 15 | Unidecode==1.3.6 16 | sgp4==2.21 17 | pymap3d==3.0.0 18 | pycountry==23.12.11 19 | lxml==5.1.0 20 | pyarrow==15.0.0 21 | -------------------------------------------------------------------------------- /controlled/16c/generate_fig_16c.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$DATA_PATH" ] || [ -z "$RESULT_PATH" ]; then echo "DATA_PATH and RESULT_PATH need to be set" 4 | exit 1 5 | fi 6 | mkdir -p "$RESULT_PATH/controlled" 7 | 8 | cd "$(dirname -- "$0")" 9 | 10 | # Create the plot 11 | python ./plot.py "$DATA_PATH" 12 | mv fov.pdf "$RESULT_PATH/controlled/fig16c_fov.pdf" 13 | mv fov.svg "$RESULT_PATH/controlled/fig16c_fov.svg" 14 | -------------------------------------------------------------------------------- /gaming/plot.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -z "$DATA_PATH" ] || [ -z "$RESULT_PATH" ]; then echo "DATA_PATH and RESULT_PATH need to be set" 4 | exit 1 5 | fi 6 | RES="$RESULT_PATH/gaming" 7 | mkdir -p "$RES" 8 | 9 | set -xeu 10 | 11 | cd "$(dirname -- "$0")" 12 | 13 | DATA="$DATA_PATH/gaming" 14 | 15 | python inspect_xtime.py \ 16 | --dirs "$DATA/crew_mo_30_cellular2" "$DATA/crew_fr_30_starlink3" \ 17 | --gdelays "$DATA"/gaming_delays_ts_{cellular,starlink}.csv \ 18 | --views 700,730 700,730 \ 19 | --save "$RES/fig-11-gaming.pdf" 20 | -------------------------------------------------------------------------------- /controlled/16b/generate_fig_16b.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$DATA_PATH" ] || [ -z "$RESULT_PATH" ]; then echo "DATA_PATH and RESULT_PATH need to be set" 4 | exit 1 5 | fi 6 | mkdir -p "$RESULT_PATH/controlled" 7 | 8 | cd "$(dirname -- "$0")" 9 | 10 | # Convert the relevent data from the dish from raw iperf logs to an SQLite DB 11 | rm uplink.db downlink.db || true # cleanup 12 | python ./parse_iperf.py "$DATA_PATH/controlled/iperf/dish-2" 13 | 14 | # Create the plot 15 | python ./plot.py 16 | mv throughput.pdf "$RESULT_PATH/controlled/fig16b_throughput.pdf" 17 | mv throughput.svg "$RESULT_PATH/controlled/fig16b_throughput.svg" 18 | -------------------------------------------------------------------------------- /zoom/process_pcaps.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd "$(dirname -- "$0")" 4 | 5 | DATA="../fig-10-zoom-dataset" 6 | 7 | haz() { 8 | if hash $1 > /dev/null 2>&1; then 9 | return 0 10 | else 11 | return 1 12 | fi 13 | } 14 | 15 | if haz parallel; then 16 | echo "Processing the files in parallel" 17 | parallel --eta bash process_pcap.sh "{}" ::: "$DATA"/*/{0,1}*/{local,remote}/*pcap 18 | else 19 | echo "parallel is not available, processing the files sequentially" 20 | for PCAP in "$DATA"/*/{0,1}*/{local,remote}/*pcap; do 21 | bash process_pcap.sh "$PCAP" 22 | done 23 | fi 24 | -------------------------------------------------------------------------------- /controlled/16a/create_sqlite_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [[ $# -ne 1 ]] 3 | then 4 | echo "Usage: ./save_data.sh " 5 | exit 1 6 | fi 7 | 8 | if [ -f irtt.db ]; then 9 | echo 'Database already exists, data will be added...' 10 | else 11 | echo 'Database does not exist, and will be created' 12 | sqlite3 irtt.db '.read schema.sql' 13 | fi 14 | 15 | unzstd -c $1 | jq -c --stream 'select(.[1] and ((.[0][2] == "timestamps" and .[0][3] == "client" and .[0][4] == "send" and .[0][5] == "wall") or (.[0][2] == "delay"))) | del(.[0][0:-1])' | python ./parse_irtt_json.py | sqlite3 -csv ./irtt.db ".separator ','" ".import '|cat -' irtt" 16 | -------------------------------------------------------------------------------- /zoom/plot.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | if [ -z "$DATA_PATH" ] || [ -z "$RESULT_PATH" ]; then echo "DATA_PATH and RESULT_PATH need to be set" 4 | exit 1 5 | fi 6 | RES="$RESULT_PATH/zoom" 7 | mkdir -p "$RES" 8 | 9 | set -xeu 10 | 11 | cd "$(dirname -- "$0")" 12 | 13 | DATA="$DATA_PATH/zoom" 14 | 15 | echo "Plotting Fig 10 Zoom" 16 | python plot_xtime_figure.py \ 17 | "$DATA"/20231011_ter_gpu/01 "$DATA"/20231011_stl_gpu/01 \ 18 | --view-ter 90,145 --view-stl 120,175 \ 19 | --save "$RES/fig-10-zoom.pdf" 20 | 21 | echo "Generating Zoom statistics" 22 | python boxplots.py \ 23 | --stl "$DATA"/20231011_stl_gpu/{01,02,03,04,05,06} \ 24 | --ter "$DATA"/20231011_ter_gpu/{01,02,03,04,05,06} \ 25 | --save "$RES/zoom-boxplot.pdf" \ 26 | --csv "$RES/zoom_metrics.csv" 27 | -------------------------------------------------------------------------------- /gaming/preprocessing.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | if [ -z "$DATA_PATH" ] || [ -z "$RESULT_PATH" ]; then echo "DATA_PATH and RESULT_PATH need to be set" 4 | exit 1 5 | fi 6 | RES="$RESULT_PATH/gaming" 7 | mkdir -p "$RES" 8 | 9 | set -xeu 10 | 11 | cd "$(dirname -- "$0")" 12 | 13 | DATA="$DATA_PATH/gaming" 14 | 15 | for c in ethernet starlink cellular; do 16 | python compute_overall_metrics.py "$DATA"/crew*${c}* --method overall --save "$RES/gaming_metrics_${c}.json" 17 | python compute_overall_metrics.py "$DATA"/crew*${c}* --method per_minute --save "$RES/gaming_metrics_resampled_${c}.json" 18 | done 19 | 20 | for c in ethernet starlink cellular; do 21 | python compute_delay.py "$DATA"/crew*${c}* --save "$RES/gaming_delays_${c}.csv" 22 | python compute_delay.py "$DATA"/crew*${c}* --savets "$RES/gaming_delays_ts_${c}.csv" 23 | done 24 | -------------------------------------------------------------------------------- /plot_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | # Remember that envvar DATA_PATH needs to point to the cloned dataset (see the previous step) 6 | export DATA_PATH="TODO" 7 | export RESULT_PATH="$(pwd)/results" 8 | 9 | echo "Expecting dataset in\tDATA_PATH=$DATA_PATH" 10 | echo "Writing results to\tRESULT_PATH=$RESULT_PATH" 11 | 12 | # RIPE Atlas plots 13 | jupyter nbconvert --to=html --output-dir="$RESULT_PATH" --execute ripe_atlas_figures/ripe_atlas_repr.ipynb 14 | 15 | # MLab plots 16 | jupyter nbconvert --to=html --output-dir="$RESULT_PATH" --execute mlab_figures/mlab_concise.ipynb 17 | 18 | # Zoom plots 19 | bash zoom/plot.sh 20 | 21 | # Cloud gaming plots 22 | bash gaming/plot.sh 23 | 24 | # Controlled experiments (Figure 16) 25 | bash controlled/16a/generate_fig_16a.sh # Generate Figure 16a 26 | bash controlled/16b/generate_fig_16b.sh # Generate Figure 16b 27 | bash controlled/16c/generate_fig_16c.sh # Generate Figure 16c 28 | -------------------------------------------------------------------------------- /mlab_figures/README.md: -------------------------------------------------------------------------------- 1 | # M-Lab Speedtest Starlink Figures 2 | 3 | With the script found in this folder, the following figures can be generated: 4 | 5 | * Figure 6 6 | * Figure 7 7 | * Figure 8 8 | * Figure 9 9 | * Figure 19 10 | * Figure 20 11 | * Figure 21 12 | 13 | ## Requirements 14 | 15 | * Python 3.10 (tested with Python 3.10.12) 16 | * venv 17 | 18 | ## Quick-Start 19 | 20 | Before running the script, make sure you fetch the data and set the `DATA_PATH` variable accordingly. Also set the `RESULT_PATH` variable to control the location of the results. 21 | 22 | ``` 23 | cd /path/to/this/folder 24 | python -m venv .venv 25 | source .venv/bin/activate 26 | pip install -r requirements.txt 27 | export DATA_PATH=/path/to/the/fetched/data 28 | export RESULT_PATH=/path/to/the/result/data 29 | jupyter nbconvert --to=html --execute mlab_concise.ipynb 30 | ``` 31 | 32 | Now, the figures are generated in the `$RESULT_PATH` and you can also see the rendered notebook in `mlab_concise.html`. 33 | -------------------------------------------------------------------------------- /mlab_figures/scraped_data/starlink_countries_2023-10-10.csv: -------------------------------------------------------------------------------- 1 | Australia 2 | Austria 3 | Bahamas 4 | Belgium 5 | Brazil 6 | Bulgaria 7 | Canada 8 | Chile 9 | Colombia 10 | Croatia 11 | Cyprus 12 | Denmark 13 | Dominican Republic 14 | Easter Island 15 | Ecuador 16 | El Salvador 17 | Estonia 18 | Finland 19 | France 20 | Germany 21 | Greece 22 | Guadeloupe 23 | Guatemala 24 | Haiti 25 | Hungary 26 | Iceland 27 | Iran 28 | Ireland 29 | Italy 30 | Jamaica 31 | Japan 32 | Kenya 33 | Latvia 34 | Lithuania 35 | Luxembourg 36 | Malawi 37 | Malaysia 38 | Malta 39 | Martinique 40 | Mexico 41 | Moldova 42 | Mozambique 43 | Netherlands 44 | New Zealand 45 | Nigeria 46 | North Macedonia 47 | Norway 48 | Panama 49 | Peru 50 | Philippines 51 | Pitcairn Islands 52 | Poland 53 | Portugal 54 | Puerto Rico 55 | Romania 56 | Rwanda 57 | Saint Barthélemy 58 | Saint Martin 59 | Slovakia 60 | Slovenia 61 | Spain 62 | Sweden 63 | Switzerland 64 | Tonga 65 | Trinidad and Tobago 66 | Ukraine 67 | United Kingdom 68 | United States 69 | Zambia 70 | Czechia 71 | U.S. Virgin Islands -------------------------------------------------------------------------------- /controlled/16a/parse_irtt_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import fileinput 4 | import re 5 | import json 6 | 7 | wall_pattern = re.compile(r"\[\[\"wall\"\]\,(\d+)\]\n") 8 | receive_pattern = re.compile(r"\[\[\"receive\"\]\,(\d+)\]\n") 9 | rtt_pattern = re.compile(r"\[\[\"rtt\"\]\,(\d+)\]\n") 10 | send_pattern = re.compile(r"\[\[\"send\"\]\,(\d+)\]\n") 11 | 12 | current_dict = None 13 | for line in fileinput.input(): 14 | if match := re.fullmatch(wall_pattern, line): 15 | current_dict = {'epoch': int(match[1])} 16 | elif match := re.fullmatch(receive_pattern, line): 17 | current_dict['receive'] = int(match[1]) 18 | elif match := re.fullmatch(rtt_pattern, line): 19 | current_dict['rtt'] = int(match[1]) 20 | elif match := re.fullmatch(send_pattern, line): 21 | current_dict['send'] = int(match[1]) 22 | try: 23 | print(f'{current_dict["epoch"]},{current_dict["send"]},{current_dict["receive"]},{current_dict["rtt"]}') 24 | except (BrokenPipeError, IOError): 25 | pass 26 | current_dict = None 27 | -------------------------------------------------------------------------------- /controlled/16a/generate_fig_16a.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$DATA_PATH" ] || [ -z "$RESULT_PATH" ]; then echo "DATA_PATH and RESULT_PATH need to be set" 4 | exit 1 5 | fi 6 | mkdir -p "$RESULT_PATH/controlled" 7 | 8 | cd "$(dirname -- "$0")" 9 | 10 | # Check for required installed programmes 11 | if ! which unzstd; then 12 | echo 'unzstd command not found on $PATH. Aborting' 13 | exit 1 14 | fi 15 | if ! which jq; then 16 | echo 'jq command not found on $PATH. Aborting' 17 | exit 1 18 | fi 19 | if ! which sqlite3; then 20 | echo 'sqlite3 command not found on $PATH. Aborting' 21 | exit 1 22 | fi 23 | 24 | # Convert the relevent data from dish 1 from raw IRTT logs to an SQLite DB 25 | ./create_sqlite_db.sh "$DATA_PATH/controlled/irtt/dish-1/230417T120001_irtt.json.zst" 26 | mv irtt.db dish-1.db 27 | 28 | # Convert the relevent data from dish 2 from raw IRTT logs to an SQLite DB 29 | ./create_sqlite_db.sh "$DATA_PATH/controlled/irtt/dish-2/irtt.zst" 30 | mv irtt.db dish-2.db 31 | 32 | # Create the plot 33 | python ./plot.py 34 | mv rtt.pdf "$RESULT_PATH/controlled/fig16a_rtt.pdf" 35 | mv rtt.svg "$RESULT_PATH/controlled/fig16a_rtt.svg" 36 | -------------------------------------------------------------------------------- /mlab_figures/scraped_data/starlink_countries_2024-02-05.csv: -------------------------------------------------------------------------------- 1 | Australia 2 | Austria 3 | Bahamas 4 | Belgium 5 | Benin 6 | Brazil 7 | Bulgaria 8 | Canada 9 | Chile 10 | Colombia 11 | Costa Rica 12 | Croatia 13 | Cyprus 14 | Denmark 15 | Dominican Republic 16 | Easter Island 17 | Ecuador 18 | El Salvador 19 | Estonia 20 | Eswatini 21 | Finland 22 | France 23 | Georgia 24 | Germany 25 | Greece 26 | Guadeloupe 27 | Guam 28 | Guatemala 29 | Haiti 30 | Honduras 31 | Hungary 32 | Iceland 33 | Iran 34 | Ireland 35 | Italy 36 | Jamaica 37 | Japan 38 | Kenya 39 | Latvia 40 | Lithuania 41 | Luxembourg 42 | Malawi 43 | Malaysia 44 | Maldives 45 | Malta 46 | Martinique 47 | Mexico 48 | Moldova 49 | Mozambique 50 | Netherlands 51 | New Zealand 52 | Nigeria 53 | North Macedonia 54 | Northern Mariana Islands 55 | Norway 56 | Panama 57 | Paraguay 58 | Peru 59 | Philippines 60 | Pitcairn Islands 61 | Poland 62 | Portugal 63 | Puerto Rico 64 | Romania 65 | Rwanda 66 | Saint Barthélemy 67 | Saint Martin 68 | Slovakia 69 | Slovenia 70 | Spain 71 | Sweden 72 | Switzerland 73 | Tonga 74 | Trinidad and Tobago 75 | Ukraine 76 | United Kingdom 77 | United States 78 | Zambia 79 | Czechia 80 | U.S. Virgin Islands -------------------------------------------------------------------------------- /ripe_atlas_figures/README.md: -------------------------------------------------------------------------------- 1 | # RIPE Atlas Starlink Figures 2 | 3 | With the script found in this folder, the following figures can be generated: 4 | 5 | * Figure 3 6 | * Figure 12 7 | * Figure 13 8 | * Figure 14 9 | * Figure 15 10 | * Figure 22 11 | 12 | ## Requirements 13 | 14 | * Python 3.9 (tested with Python 3.9.6) 15 | * venv 16 | 17 | ## Quick-Start 18 | 19 | Fetch the data and set the `DATA_PATH` environment variable to the folder where the data is downloaded. Set the `RESULT_PATH` variable to the folder where the results should appear. The Ripe Atlas data must be in a subfolder (./atlas/ripe_Atlas_repr) of this data folder. 20 | 21 | Please Make sure the data is in uncompressed version. Else execute the below command at the 'data' folder level: 22 | 23 | ``` 24 | cd ./atlas/ripe_atlas_repr 25 | unzip '*.zip' 26 | ``` 27 | 28 | Then execute the below commands in the terminal: 29 | 30 | ``` 31 | cd /path/to/this/folder 32 | python -m venv .venv 33 | source .venv/bin/activate 34 | pip install -r requirements.txt 35 | export DATA_PATH=/path/to/data/folder 36 | export RESULT_PATH=/path/to/result/folder 37 | jupyter nbconvert --to=html --execute ripe_atlas_repr.ipynb 38 | ``` 39 | 40 | Now, the figures are generated in the `$RESULT_PATH` folder and you can also see the rendered notebook in `ripe_atlas_repr.html`. 41 | -------------------------------------------------------------------------------- /controlled/16b/parse_iperf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import os 5 | import json 6 | import pandas as pd 7 | import sqlite3 8 | 9 | uplink = [] 10 | downlink = [] 11 | 12 | LOG_DIR = sys.argv[1] 13 | 14 | log_files = [f'{LOG_DIR}/{file}' for file in os.listdir(LOG_DIR) if os.path.isfile(f'{LOG_DIR}/{file}') and file.endswith('txt') and ('uplink' in file or 'downlink' in file)] 15 | 16 | for log in log_files: 17 | with open(log) as lf: 18 | log_json = json.load(lf) 19 | if 'uplink' in log: 20 | try: 21 | log_json = log_json['server_output_json'] 22 | except KeyError: 23 | continue 24 | start_time = log_json['start']['timestamp']['timesecs'] 25 | for interval in log_json['intervals']: 26 | interval = interval['sum'] 27 | interval['start'] += start_time 28 | interval['end'] += start_time 29 | if 'downlink' in log: 30 | downlink.append(interval) 31 | else: 32 | uplink.append(interval) 33 | 34 | uplink_df = pd.DataFrame(uplink) 35 | uplink_conn = sqlite3.connect('uplink.db') 36 | uplink_df.to_sql('iperf', uplink_conn) 37 | 38 | downlink_df = pd.DataFrame(downlink) 39 | downlink_conn = sqlite3.connect('downlink.db') 40 | downlink_df.to_sql('iperf', downlink_conn) 41 | -------------------------------------------------------------------------------- /zoom/README.md: -------------------------------------------------------------------------------- 1 | Fig. 10: Zoom Video Conferencing 2 | ==== 3 | 4 | # Setup 5 | ## Get the data 6 | Download the data set. 7 | 8 | ## Optional: Recompute the Zoom Statistics 9 | We used the Zoom analysis tools published by Michel et al. [0] at [Princeton-Cabernet/zoom-analysis](https://github.com/Princeton-Cabernet/zoom-analysis) to compute Zoom statistics from captured packet traces at the call parties. Our data set already includes the computed metrics but you can optionally recreate them. 10 | 11 | To recompute the traces, load the zoom analysis source by running `git submodule update --init --recursive`. 12 | Please follow their compiliation instructions in [zoom-analysis/README.md](zoom-analysis/README.md) to build the `zoom_flows` and `zoom_rtp` binaries. 13 | Make sure to include them in your PATH: `export PATH="$(pwd)/zoom-analysis/build:$PATH"`. 14 | 15 | Process the files using our utility script: `bash process_pcaps.sh` 16 | 17 | [0] Oliver Michel, Satadal Sengupta, Hyojoon Kim, Ravi Netravali, and Jennifer Rexford. 2022. Enabling passive measurement of zoom performance in production networks. In Proceedings of the 22nd ACM Internet Measurement Conference (IMC '22). Association for Computing Machinery, New York, NY, USA, 244–260. https://doi.org/10.1145/3517745.3561414 18 | 19 | ## Recreate the Plots 20 | Set the environment variables `DATA_PATH` and `RESULT_PATH`. Run `bash plot.sh` to create the figure. 21 | -------------------------------------------------------------------------------- /controlled/README.md: -------------------------------------------------------------------------------- 1 | 2 | Fig. 16: Controlled Measurements 3 | ===== 4 | The scripts in the subdirectories generate the four parts of Figure 16: 5 | 1. Figure 16a: analysis of Starlink round-trip latencies. 6 | 2. Figure 16b: analysis of Starlink uplink and downlink throughput. 7 | 3. Figure 16c (upper): analysis of Starlink round-trip latencies when connected to a single serving satellite. 8 | 4. Figure 16c (lower): analysis of the time between the connectivity window start / end and the previous reconfiguration interval (for more details see the paper). 9 | 10 | # Setup 11 | 12 | ## Dataset 13 | 14 | Please fetch the dataset and point env var `DATA_PATH` to it. Note that only a small subset of the entire dataset is required to produce the figures, specifically the following eight files. 15 | ``` sh 16 | data 17 | └── controlled 18 | ├── irtt 19 | │ ├── dish-1 20 | │ │ └── 230417T120001_irtt.json.zst 21 | │ └── dish-2 22 | │ └── irtt.zst 23 | ├── iperf 24 | │ └── dish-2 25 | │ ├── downlink_20230515T130700.txt 26 | │ └── uplink_20230515T130000.txt 27 | ├── irtt_fov 28 | │ └── 2023-10-08_18-41-11.json 29 | └── grpc_fov 30 | ├── 2023-05-16_to_2023-05-19.db 31 | ├── 2023-10-08_to_2023-10-11.db 32 | └── 2023-10-12_to_2023-10-13.db 33 | ``` 34 | 35 | ## Running the plotting scripts 36 | 37 | To re-create Figure 16 from the paper, simply run the appropriate scripts. 38 | ``` sh 39 | cd 16a && ./generate_fig_16a.sh # Generate Figure 16a 40 | cd ../16b && ./generate_fig_16b.sh # Generate Figure 16b 41 | cd ../16c && ./generate_fig_16c.sh # Generate Figure 16c 42 | ``` 43 | -------------------------------------------------------------------------------- /mlab_figures/orbit_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | import matplotlib.pyplot as plt 4 | from functools import partial 5 | from descartes import PolygonPatch 6 | 7 | # We are often interested in the radius of the coverage area based on altitude and elevation. 8 | # There are two ways to define radius: the radius of the slice depicted in the image above (A), or the geodesic distance (arc length) (B) 9 | 10 | def slant_distance(e, h, R = 6371): 11 | """ 12 | Computes the slant distance based on elevation and altitude. 13 | 14 | e: Elevation in degrees 15 | h: Altitude in km 16 | 17 | Returns 18 | 19 | d: slant distance in km 20 | """ 21 | B = -2 * R * np.cos(np.radians(90 + e)) 22 | C = -2 * R * h - h**2 23 | # Solving a geometric equation where only one is real 24 | d = (-B + np.sqrt(B**2 - 4*C)) / 2.0 25 | return d 26 | 27 | def params(e, h, R = 6371): 28 | """ 29 | With the equations above, we can solve all four parameters 30 | 31 | e: Elevation in degrees 32 | h: Altitude in km 33 | 34 | Returns 35 | 36 | all four parameters discussed above 37 | """ 38 | r = R + h 39 | 40 | d = slant_distance(e, h, R=R) 41 | beta = np.degrees( np.arcsin( d * np.cos(np.radians(e))/r ) ) 42 | alpha = 90 - e - beta 43 | 44 | return e, alpha, beta, d 45 | 46 | def coverage_radius_A(e, h, R = 6371): 47 | """ 48 | With the equations above, we can solve all four parameters 49 | 50 | e: Elevation in degrees 51 | h: Altitude in km 52 | 53 | Returns 54 | 55 | Radius A in km 56 | """ 57 | _, _, beta, _ = params(e, h, R=R) 58 | radius = np.sin(np.radians(beta)) * R 59 | return radius 60 | 61 | def coverage_radius_B(e, h, R = 6371): 62 | """ 63 | With the equations above, we can solve all four parameters 64 | 65 | e: Elevation in degrees 66 | h: Altitude in km 67 | 68 | Returns 69 | 70 | Radius B in km 71 | """ 72 | _, _, beta, _ = params(e, h, R=R) 73 | diameter = 2 * np.pi * R 74 | radius = (beta / 360) * diameter 75 | return radius 76 | -------------------------------------------------------------------------------- /zoom/zoom_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pandas as pd 4 | import os.path 5 | 6 | LOCAL_IP_PREFIXES = ["192.168", "131.159"] 7 | 8 | def read_packets_csv(directory): 9 | df = pd.read_csv(os.path.join(directory, "packets.csv")) 10 | df.rename(columns={'#ts_s':'ts_s'}, inplace=True) 11 | df['ts'] = pd.to_datetime((df['ts_s']*1e6 + df['ts_us']) / 1e6, unit="s") 12 | df['ts_rel'] = df.ts - df.ts.min() 13 | return df 14 | 15 | def merge_packet_dfs(local, remote, join="inner"): # 16 | # cols = ['media_type','pkts_in_frame','ssrc','pt','rtp_seq','rtp_ts','pl_len','rtp_ext1','drop'] 17 | cols = ["ssrc", "rtp_seq", "rtp_ts", "pt"] 18 | m = local.merge(remote, on=cols, how=join, suffixes=('_local', '_remote')) 19 | m['owd'] = ((m['ts_s_local']*1e6 + m['ts_us_local']) - (m['ts_s_remote']*1e6 + m['ts_us_remote'])) / 1e3 20 | min_ts = min(m['ts_local']) 21 | m['ts'] = m['ts_local'] - min_ts 22 | return m 23 | 24 | def find_lost_packets(local, remote): 25 | df = merge_packet_dfs(local, remote, join="left") 26 | # pt 98 are video packets. pt=110 are FEC packets that are only sent, processed by the server, and never received. 27 | df = df[(df.ts_s_remote.isna()) & (df.pt == 98)]#(df.pt_local == 98)] 28 | return df 29 | 30 | def read_streams_csv(directory): 31 | df = pd.read_csv(os.path.join(directory, "streams.csv")) 32 | df["is_local"] = df.ip_src.apply(ip_is_local) 33 | return df 34 | 35 | def read_frames_csv(directory): 36 | df = pd.read_csv(os.path.join(directory, "frames.csv")) 37 | df.rename(columns={' min_ts_us':'min_ts_us'}, inplace=True) 38 | df["min_ts"] = pd.to_datetime((df.min_ts_s * 1e6 + df.min_ts_us) / 1e6, unit="s") 39 | df["max_ts"] = pd.to_datetime((df.max_ts_s * 1e6 + df.max_ts_us) / 1e6, unit="s") 40 | df["is_local"] = df.ip_src.apply(ip_is_local) 41 | return df 42 | 43 | def read_stats_csv(directory): 44 | df = pd.read_csv(os.path.join(directory, "stats.csv")) 45 | df["ts"] = pd.to_datetime(df.ts_s, unit="s") 46 | df["is_local"] = df.ip_src.apply(ip_is_local) 47 | return df 48 | 49 | def ip_is_local(ip): 50 | local_prefixes = LOCAL_IP_PREFIXES 51 | for prefix in local_prefixes: 52 | if ip.startswith(prefix): 53 | return True 54 | return False 55 | 56 | -------------------------------------------------------------------------------- /gaming/README.md: -------------------------------------------------------------------------------- 1 | Fig. 11: Cloud Gaming 2 | ===== 3 | 4 | Our analysis builds on the paper "Dissecting Cloud Gaming Performance with DECAF" [0] who published their code at [decafCG/decaf](https://github.com/decafCG/decaf). 5 | We applied small changes to their analysis script and published our changes at [hendrikcech/decaf](https://github.com/hendrikcech/decaf). 6 | 7 | [0] Hassan Iqbal, Ayesha Khalid, and Muhammad Shahzad. 2021. Dissecting Cloud Gaming Performance with DECAF. Proc. ACM Meas. Anal. Comput. Syst. 5, 3, Article 31 (December 2021), 27 pages. https://doi.org/10.1145/3491043 8 | 9 | # Setup 10 | ## Dataset 11 | Please fetch the gaming dataset and point env var `DATA_PATH` to it. This dataset contains both the raw data captured during the experiment and the artifacts created by the DECAF analysis scripts. 12 | 13 | The dataset contains multiple measurement runs, each 30 minutes long, over different links: Ethernet, Starlink, and cellular 5G. Each folder has the following structure. 14 | ``` sh 15 | crew_fr_30_ethernet1 16 | --- Captured at measurement time 17 | ├── bot_log.csv # timestamped game bot actions performed during the measurement 18 | ├── dump_for_ip.pcapng 19 | ├── dump.pcapng # full packet trace from measurement run 20 | ├── ips.json # detected client and server IPs 21 | ├── rtcStatsCollector.txt # written by Chrome during measurement 22 | ├── video_1684482973.512462.mkv 23 | ---- Created by data_processing.py in post-processing 24 | ├── current_rtt.json 25 | ├── frame_timestamps.json 26 | ├── packet_loss_stats.json 27 | ├── parsed_rtcStatsCollector.json 28 | ├── parsed_videoReceiveStream.json 29 | ├── predicted_files.json 30 | ├── video_cropped.mkv 31 | ├── videoReceiveStream.txt 32 | └─── vrs_summary_stats.json 33 | ``` 34 | 35 | ### 0. Optional: DECAF Artifact Processing 36 | Clone our adapted decaf version: 37 | 38 | ``` sh 39 | git submodule update --init --recursive 40 | ``` 41 | 42 | Please follow the setup instructions in `decaf/data_processing/README.md` and run `decaf/data_processing/data_processing.py`. Afterwards, compute our custom statistics with `bash preprocessing.sh` with `DATA_PATH` and `RESULT_PATH` being set. 43 | 44 | ### 1. Run our Analysis and Plotting Scripts 45 | Set the environment variables `DATA_PATH` and `RESULT_PATH`. To recreate Fig. 11 from our paper, run `bash plot.sh`. 46 | -------------------------------------------------------------------------------- /controlled/16b/plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import json 5 | import matplotlib.pyplot as plt 6 | import sqlite3 7 | import pandas 8 | 9 | OPTIMISATION_START=0 10 | 11 | DOWNLINK_START_TIME = 1684148817 # 15 May 2023 11:06:57 12 | DOWNLINK_END_TIME = DOWNLINK_START_TIME+195 # 15 May 2023 11:10:12 13 | 14 | UPLINK_START_TIME = 1684148397 # 15 May 2023 10:59:57 15 | UPLINK_END_TIME = UPLINK_START_TIME+195 # 15 May 2023 11:03:12 16 | 17 | # It is necessary to shift the dataset because the raw timestamps are incorrect. 18 | # This is due to two errors: 19 | # 1. NTP not being active on the client, resulting in incorrect downlink timestamps 20 | # 2. Iperf rounding down the initial timestamp to the nearest second 21 | # For more information (and the discussion with reviewers), see the publicly-available reviewer comments 22 | DOWNLINK_OFFSET = 1.25 23 | UPLINK_OFFSET = 0.25 24 | 25 | 26 | conn = sqlite3.connect('downlink.db') 27 | downlink_df = pandas.read_sql(f'SELECT * FROM iperf WHERE start >= {DOWNLINK_START_TIME} and end <= {DOWNLINK_END_TIME}', conn) 28 | conn.close() 29 | 30 | conn = sqlite3.connect('uplink.db') 31 | uplink_df = pandas.read_sql(f'SELECT * FROM iperf WHERE start >= {UPLINK_START_TIME} and start <= {UPLINK_END_TIME}', conn) 32 | conn.close() 33 | 34 | fig, axs = plt.subplots(2, sharex=True, gridspec_kw={'height_ratios': [1,1]}, figsize=(2.8, 2.4)) 35 | 36 | axs[0].plot(downlink_df['start']-DOWNLINK_START_TIME + DOWNLINK_OFFSET, downlink_df['bits_per_second'] / (1024*1024), label='Downlink') 37 | axs[1].plot(uplink_df['start']-UPLINK_START_TIME + UPLINK_OFFSET, uplink_df['bits_per_second'] / (1024*1024), label='Uplink') 38 | 39 | optimisation_interval = 0 40 | y_max=340 41 | while optimisation_interval < 300: 42 | for ax in axs: 43 | ax.plot([optimisation_interval, optimisation_interval], [0, y_max], color='#2d000ef4', linestyle='--', label='Reconfiguration Interval' if optimisation_interval == 0 else None) 44 | optimisation_interval += 15 45 | 46 | axs[0].set_xlim(0,195) 47 | axs[0].set_ylim(0, y_max) 48 | 49 | axs[1].set_xlim(0,195) 50 | axs[1].set_ylim(0, 80) 51 | 52 | axs[1].set_xlabel("Experiment time [s]") 53 | axs[1].set_xticks(range(0,195,30)) 54 | fig.text(0,0.5, "Throughput [Mbps]", ha="center", va="center", rotation=90) 55 | 56 | plt.savefig("throughput.pdf", bbox_inches="tight", pad_inches=0) 57 | plt.savefig("throughput.svg", bbox_inches="tight", pad_inches=0) 58 | -------------------------------------------------------------------------------- /mno_related/README.md: -------------------------------------------------------------------------------- 1 | # MNO List 2 | 3 | The `mno_list_withasn.csv` has been curated to contain the top mobile network operators of all countries in which Starlink is available. This table has been last updated on October 13th 2023. 4 | 5 | ## Accessing ASRANK data 6 | 7 | To access the asrank data, we use the [AS Rank v2.1 GraphQL API](https://api.asrank.caida.org/dev/docs). First, we manually curate the list of mobile network operators and their respective autonomous system numbers, and then use the API. The code to generate the table is as follows: 8 | 9 | ```python 10 | import json 11 | import requests 12 | 13 | # 1. We manually curate the dataframe containing "CountryName", "MobileOperator", and "ASN" in the variable `mno_df` 14 | 15 | # 2. Fetching Caida data for each ASN in the `mno_df` dataframe 16 | URL = "https://api.asrank.caida.org/v2/graphql" 17 | decoder = json.JSONDecoder() 18 | encoder = json.JSONEncoder() 19 | 20 | def query_asn(asn): 21 | query = AsnQuery(asn) 22 | request = requests.post(URL,json={'query':query}) 23 | if request.status_code == 200: 24 | return request.json() 25 | else: 26 | print ("Query failed to run returned code of %d " % (request.status_code)) 27 | return None 28 | 29 | def AsnQuery(asn): 30 | return """{ 31 | asn(asn:"%i") { 32 | asn 33 | asnName 34 | rank 35 | country { 36 | iso 37 | name 38 | } 39 | asnDegree { 40 | provider 41 | peer 42 | customer 43 | total 44 | transit 45 | sibling 46 | } 47 | announcing { 48 | numberPrefixes 49 | numberAddresses 50 | } 51 | } 52 | }""" % (int(asn[2:])) 53 | 54 | results = [] 55 | for _, row in mno_df.iterrows(): 56 | results.append(query_asn(row["ASN"])["data"]["asn"]) 57 | caida_df = pd.DataFrame(results) 58 | caida_df = caida_df.set_index("asn") 59 | 60 | # 3. Add an asrank column into the `mno_df` dataframe 61 | 62 | def getrank(asn): 63 | return caida_df.loc[[asn[2:]]].iloc[0]["rank"] 64 | mno_df["asrank"] = mno_df["ASN"].apply(getrank) 65 | 66 | # 4. Store `mno_df` as a .csv file 67 | 68 | mno_df.sort_values(["CountryName", "asrank"])[["CountryName", "MonileOperator", "ASN", "asrank"]].set_index("CountryName").to_csv("mno_list_withasn.csv") 69 | ``` 70 | -------------------------------------------------------------------------------- /mlab_figures/iso3.json: -------------------------------------------------------------------------------- 1 | {"BD": "BGD", "BE": "BEL", "BF": "BFA", "BG": "BGR", "BA": "BIH", "BB": "BRB", "WF": "WLF", "BL": "BLM", "BM": "BMU", "BN": "BRN", "BO": "BOL", "BH": "BHR", "BI": "BDI", "BJ": "BEN", "BT": "BTN", "JM": "JAM", "BV": "BVT", "BW": "BWA", "WS": "WSM", "BQ": "BES", "BR": "BRA", "BS": "BHS", "JE": "JEY", "BY": "BLR", "BZ": "BLZ", "RU": "RUS", "RW": "RWA", "RS": "SRB", "TL": "TLS", "RE": "REU", "TM": "TKM", "TJ": "TJK", "RO": "ROU", "TK": "TKL", "GW": "GNB", "GU": "GUM", "GT": "GTM", "GS": "SGS", "GR": "GRC", "GQ": "GNQ", "GP": "GLP", "JP": "JPN", "GY": "GUY", "GG": "GGY", "GF": "GUF", "GE": "GEO", "GD": "GRD", "GB": "GBR", "GA": "GAB", "SV": "SLV", "GN": "GIN", "GM": "GMB", "GL": "GRL", "GI": "GIB", "GH": "GHA", "OM": "OMN", "TN": "TUN", "JO": "JOR", "HR": "HRV", "HT": "HTI", "HU": "HUN", "HK": "HKG", "HN": "HND", "HM": "HMD", "VE": "VEN", "PR": "PRI", "PS": "PSE", "PW": "PLW", "PT": "PRT", "SJ": "SJM", "PY": "PRY", "IQ": "IRQ", "PA": "PAN", "PF": "PYF", "PG": "PNG", "PE": "PER", "PK": "PAK", "PH": "PHL", "PN": "PCN", "PL": "POL", "PM": "SPM", "ZM": "ZMB", "EH": "ESH", "EE": "EST", "EG": "EGY", "ZA": "ZAF", "EC": "ECU", "IT": "ITA", "VN": "VNM", "SB": "SLB", "ET": "ETH", "SO": "SOM", "ZW": "ZWE", "SA": "SAU", "ES": "ESP", "ER": "ERI", "ME": "MNE", "MD": "MDA", "MG": "MDG", "MF": "MAF", "MA": "MAR", "MC": "MCO", "UZ": "UZB", "MM": "MMR", "ML": "MLI", "MO": "MAC", "MN": "MNG", "MH": "MHL", "MK": "MKD", "MU": "MUS", "MT": "MLT", "MW": "MWI", "MV": "MDV", "MQ": "MTQ", "MP": "MNP", "MS": "MSR", "MR": "MRT", "IM": "IMN", "UG": "UGA", "TZ": "TZA", "MY": "MYS", "MX": "MEX", "IL": "ISR", "FR": "FRA", "IO": "IOT", "SH": "SHN", "FI": "FIN", "FJ": "FJI", "FK": "FLK", "FM": "FSM", "FO": "FRO", "NI": "NIC", "NL": "NLD", "NO": "NOR", "NA": "NAM", "VU": "VUT", "NC": "NCL", "NE": "NER", "NF": "NFK", "NG": "NGA", "NZ": "NZL", "NP": "NPL", "NR": "NRU", "NU": "NIU", "CK": "COK", "XK": "XKX", "CI": "CIV", "CH": "CHE", "CO": "COL", "CN": "CHN", "CM": "CMR", "CL": "CHL", "CC": "CCK", "CA": "CAN", "CG": "COG", "CF": "CAF", "CD": "COD", "CZ": "CZE", "CY": "CYP", "CX": "CXR", "CR": "CRI", "CW": "CUW", "CV": "CPV", "CU": "CUB", "SZ": "SWZ", "SY": "SYR", "SX": "SXM", "KG": "KGZ", "KE": "KEN", "SS": "SSD", "SR": "SUR", "KI": "KIR", "KH": "KHM", "KN": "KNA", "KM": "COM", "ST": "STP", "SK": "SVK", "KR": "KOR", "SI": "SVN", "KP": "PRK", "KW": "KWT", "SN": "SEN", "SM": "SMR", "SL": "SLE", "SC": "SYC", "KZ": "KAZ", "KY": "CYM", "SG": "SGP", "SE": "SWE", "SD": "SDN", "DO": "DOM", "DM": "DMA", "DJ": "DJI", "DK": "DNK", "VG": "VGB", "DE": "DEU", "YE": "YEM", "DZ": "DZA", "US": "USA", "UY": "URY", "YT": "MYT", "UM": "UMI", "LB": "LBN", "LC": "LCA", "LA": "LAO", "TV": "TUV", "TW": "TWN", "TT": "TTO", "TR": "TUR", "LK": "LKA", "LI": "LIE", "LV": "LVA", "TO": "TON", "LT": "LTU", "LU": "LUX", "LR": "LBR", "LS": "LSO", "TH": "THA", "TF": "ATF", "TG": "TGO", "TD": "TCD", "TC": "TCA", "LY": "LBY", "VA": "VAT", "VC": "VCT", "AE": "ARE", "AD": "AND", "AG": "ATG", "AF": "AFG", "AI": "AIA", "VI": "VIR", "IS": "ISL", "IR": "IRN", "AM": "ARM", "AL": "ALB", "AO": "AGO", "AQ": "ATA", "AS": "ASM", "AR": "ARG", "AU": "AUS", "AT": "AUT", "AW": "ABW", "IN": "IND", "AX": "ALA", "AZ": "AZE", "IE": "IRL", "ID": "IDN", "UA": "UKR", "QA": "QAT", "MZ": "MOZ"} -------------------------------------------------------------------------------- /mno_related/mno_list_withasn.csv: -------------------------------------------------------------------------------- 1 | CountryName,MobileOperator,ASN,asrank 2 | Australia,Telstra,AS1221,55 3 | Australia,Optus,AS4804,4540 4 | Australia,Vodafone,AS133612,7298 5 | Austria,A1 Telekom Austria,AS8447,160 6 | Austria,Magenta Telekom,AS25255,522 7 | Austria,T-Mobile Austria,AS8412,555 8 | Belgium,Proximus,AS5432,980 9 | Belgium,Telenet,AS6848,1295 10 | Belgium,Orange,AS47377,2859 11 | Brazil,TIM,AS26615,79 12 | Brazil,Claro,AS28573,7228 13 | Brazil,Vivo,AS18881,7243 14 | Canada,Bell,AS577,95 15 | Canada,Telus,AS852,200 16 | Canada,Rogers,AS812,240 17 | Chile,Movistar,AS7418,5474 18 | Chile,Claro,AS27995,7337 19 | Chile,Entel,AS27651,11937 20 | Colombia,Tigo Colombia/EPM Telecomunicaciones,AS13489,1382 21 | Colombia,Claro,AS10620,11863 22 | Colombia,Colombia Móvil,AS27831,11867 23 | Czechia,Vodafone Czech Republic,AS16019,268 24 | Czechia,O2 Czech Republic,AS5610,830 25 | Czechia,T-Mobile,AS5588,3174 26 | Dominican Republic,Claro,AS6400,2463 27 | Dominican Republic,Altice Dominicana,AS28118,3123 28 | Dominican Republic,Tricom,AS27887,4683 29 | France,Orange,AS3215,213 30 | France,SFR,AS15557,438 31 | France,Bouygues Telecom,AS5410,1426 32 | Germany,Deutsche Telekom,AS3320,18 33 | Germany,Vodafone,AS3209,237 34 | Germany,O2,AS6805,1923 35 | Greece,Vodafone Greece,AS3329,739 36 | Greece,Forthnet GR,AS1241,932 37 | Greece,Cosmote,AS29247,12082 38 | Guadeloupe,Dauphin Telecom,AS33392,6364 39 | Ireland,Eir,AS5466,1755 40 | Ireland,Vodafone,AS15502,3472 41 | Ireland,Three,AS13280,7415 42 | Italy,TIM,AS3269,239 43 | Italy,Wind Tre,AS1267,445 44 | Italy,Vodafone,AS30722,1042 45 | Japan,KDDI,AS2516,123 46 | Japan,SoftBank,AS17676,147 47 | Japan,NTT Docomo,AS9605,11855 48 | Kenya,Safaricom,AS33771,841 49 | Kenya,Airtel Kenya,AS36926,981 50 | Kenya,Telkom Kenya,AS12455,3956 51 | Martinique,Digicel,AS48252,3510 52 | Mexico,AT&T,AS28469,5591 53 | Mexico,Telcel,AS28403,15664 54 | Mozambique,Movitel,AS37342,12384 55 | Mozambique,mCel,AS36945,21793 56 | Netherlands,KPN,AS1136,768 57 | Netherlands,T-Mobile,AS50266,1292 58 | Netherlands,Tele2,AS13127,1434 59 | New Zealand,2degrees,AS9790,263 60 | New Zealand,Vodafone,AS9500,1936 61 | New Zealand,Spark,AS4771,3466 62 | Nigeria,MTN,AS29465,933 63 | Nigeria,Airtel,AS36873,1190 64 | Nigeria,Glo,AS328309,23833 65 | Norway,Telenor,AS2119,256 66 | Norway,NextGenTel AS,AS15659,4600 67 | Norway,TELIA NORGE AS,AS12929,7373 68 | Peru,Movistar,AS6147,1834 69 | Peru,Claro,AS12252,2039 70 | Peru,Entel,AS21575,2047 71 | Philippines,Smart Communications,AS10139,7416 72 | Philippines,Globe Telecom,AS132199,12034 73 | Poland,Orange Polska,AS5617,164 74 | Poland,T-Mobile Poland,AS12912,180 75 | Portugal,NOS,AS2860,801 76 | Portugal,MEO,AS15525,1356 77 | Portugal,Vodafone,AS12353,1601 78 | Puerto Rico,Claro,AS10396,1131 79 | Puerto Rico,Liberty,AS14638,1308 80 | Puerto Rico,T-Mobile,AS21928,5470 81 | Saint Barthélemy,Digicel,AS3215,213 82 | Spain,Vodafone,AS12430,372 83 | Spain,Orange,AS12479,381 84 | Spain,Movistar or Telefonica de Espana,AS3352,403 85 | Sweden,Tele2 Sweden,AS1257,195 86 | Sweden,Telia Company,AS3301,390 87 | Sweden,Telenor Sweden,AS8642,65739 88 | United Kingdom,O2,AS5089,518 89 | United Kingdom,Vodafone,AS5378,5476 90 | United Kingdom,EE,AS12576,11838 91 | United States,T-Mobile,AS21928,5470 92 | United States,AT&T Mobility LLC,AS20057,7244 93 | United States,Verizon,AS22394,12570 94 | -------------------------------------------------------------------------------- /gaming/compute_delay.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import json 5 | import pandas as pd 6 | import numpy as np 7 | import os 8 | import sys 9 | 10 | def compute_blocks(ls): 11 | blocks = [] 12 | block_start = None 13 | prev = None 14 | ls = sorted([int(v) for v in ls]) 15 | for cur in sorted(ls): 16 | if block_start is None: 17 | block_start = cur 18 | if prev is not None: 19 | if prev + 1 != cur: 20 | blocks.append((block_start, prev)) 21 | block_start = cur 22 | prev = cur 23 | return blocks 24 | 25 | def assemble_df(base_path): 26 | action_frame_ts = os.path.join(base_path, "frame_timestamps.json") 27 | bot_log = os.path.join(base_path, "bot_log.csv") 28 | 29 | with open(action_frame_ts) as f: 30 | frame_tss = json.load(f) 31 | 32 | action_frame_ts = [] 33 | action_frames = compute_blocks(frame_tss.keys()) 34 | for (start, _) in action_frames: 35 | try: 36 | frame_ts = frame_tss[str(start)][1] 37 | if frame_ts == "skipped": 38 | continue 39 | ts = pd.Timestamp(frame_ts, unit="s") 40 | except: 41 | print(f"Failed to convert {start}") 42 | breakpoint() 43 | action_frame_ts.append((ts, "action", start)) 44 | 45 | cmds = [] 46 | with open(bot_log) as f: 47 | for line in f.readlines(): 48 | ts, cmd = line.strip().split(",") 49 | cmds.append((pd.Timestamp(int(ts)), cmd, None)) 50 | 51 | columns = ["ts", "event", "frame"] 52 | df1 = pd.DataFrame(action_frame_ts, columns=columns) 53 | df2 = pd.DataFrame(cmds, columns=columns) 54 | df = pd.concat([df1, df2]).set_index("ts").sort_index() 55 | df["delay_ms"] = np.nan 56 | return df 57 | 58 | def compute_delay(df): 59 | ts_triggers = df[(df.event == "s") | (df.event == "r")].index 60 | for cmd_ts in ts_triggers: 61 | next_event = df[(df.index > cmd_ts) & (df.event == "action")] 62 | if len(next_event) == 0: 63 | continue 64 | next_event = next_event.iloc[0] 65 | action_ts = next_event.name 66 | action_delay = (action_ts - cmd_ts).total_seconds() * 1000 67 | if action_delay > 500: 68 | print(f"Discarded large delay of {action_delay:.0f} ms at {action_ts} / frame {next_event.frame}", file=sys.stderr) 69 | continue 70 | if action_delay < 30: 71 | print(f"Discarded small delay of {action_delay:.0f} ms at {action_ts} / frame {next_event.frame}", file=sys.stderr) 72 | # print(df[(df.index >= (cmd_ts - pd.Timedelta(seconds=1)))]) 73 | continue 74 | df.loc[cmd_ts, "delay_ms"] = action_delay 75 | return df.dropna(subset="delay_ms")["delay_ms"]# .to_list() 76 | 77 | def main(): 78 | parser = argparse.ArgumentParser() 79 | parser.add_argument("directories", nargs="+", help="Result directories") 80 | parser.add_argument("--save", help="Save datapoints to csv file") 81 | parser.add_argument("--savets", help="Save datapoints and timestamps to csv file") 82 | args = parser.parse_args() 83 | 84 | tss = [] 85 | delays = [] 86 | for d in args.directories: 87 | print(d, file=sys.stderr) 88 | df = assemble_df(d) 89 | delay = compute_delay(df) 90 | tss.extend(delay.index.to_list()) 91 | delays.extend(delay.to_list()) 92 | 93 | result = dict( 94 | min=min(delays), 95 | q1=np.percentile(delays, 25), 96 | med=np.percentile(delays, 50), 97 | q3=np.percentile(delays, 75), 98 | max=max(delays), 99 | count=len(delays)) 100 | print(result) 101 | 102 | if args.save: 103 | with open(args.save, "w") as f: 104 | for d in delays: 105 | f.write(f"{d:.2f}") 106 | f.write("\n") 107 | if args.savets: 108 | with open(args.savets, "w") as f: 109 | f.write("ts,delay_ms\n") 110 | for ts, d in zip(tss, delays): 111 | f.write(f"{ts.timestamp()},{d:.2f}\n") 112 | 113 | # desc = df.dropna(subset="delay_ms")["delay_ms"].describe() 114 | # print(desc) 115 | # print(df.dtypes) 116 | 117 | 118 | 119 | if __name__ == "__main__": 120 | main() 121 | -------------------------------------------------------------------------------- /controlled/16a/plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import matplotlib.pyplot as plt 4 | import sys 5 | import sqlite3 6 | import pandas 7 | 8 | # unix epoch for the starting time for the duration 9 | START_TIME=1681733787 # 17 April 2023 12:16:27 10 | 11 | # unix epoch for the ending time for the duration 12 | END_TIME = START_TIME + 195 13 | 14 | # do we show a raw plot or a reduced plot 15 | REDUCED=False 16 | 17 | # the first interval between optimisation intervals in the plot 18 | OPTIMISATION_START=0 19 | 20 | 21 | 22 | def read_in_values(irtt_db_file, start_epoch_seconds, start_end_seconds): 23 | """ 24 | Retreive the IRTT round-trip time data for the given duration from the database. 25 | 26 | :param irtt_db_file: the SQLite3 database file to read from 27 | :param start_epoch_seconds: the start time (unix epoch) of the time period 28 | :param start_end_seconds: the end time time (unix epoch) of the time period 29 | :return: a pandas dataframe containing the round-trip-time data 30 | """ 31 | conn = sqlite3.connect(irtt_db_file) 32 | start_time = start_epoch_seconds * 1e9 33 | end_time = start_end_seconds * 1e9 34 | df = pandas.read_sql(f'SELECT epoch,rtt FROM irtt WHERE epoch >= {start_time} AND epoch <= {end_time}', conn) 35 | return df 36 | 37 | 38 | 39 | def reduce_df(df): 40 | """ 41 | Reduce the data by averaging it over one second intervals 42 | 43 | :param df: the pandas dataframe containing the data to reduce 44 | :return: a pandas dataframe containing the reduced data 45 | """ 46 | buckets = range(int((START_TIME-1)*1e9), int((END_TIME+1)*1e9), int(1e9)) 47 | df['epoch category'] = pandas.cut(df['epoch'], buckets) 48 | df['reduced_rtt'] = df.groupby('epoch category')['rtt'].transform('mean') 49 | df = df.drop_duplicates(subset=['epoch category', 'reduced_rtt']) 50 | df = df[['epoch', 'reduced_rtt']] 51 | df = df.reset_index(drop=True) 52 | df = df.rename(columns={'reduced_rtt': 'rtt'}) 53 | return df 54 | 55 | 56 | 57 | def plot_data(dish_a, dish_b): 58 | """ 59 | Draw the plot showing the IRTT values for two dishes 60 | 61 | :param dish_a: a pandas dataframe containing IRTT data to plot for dish A 62 | :param dish_b: a pandas dataframe containing IRTT data to plot for dish B 63 | :return: nothing 64 | """ 65 | 66 | # Plot the raw IRTT measurements 67 | fig, axs = plt.subplots(2, sharex=True, gridspec_kw={'height_ratios': [1,1]}, figsize=(2.8, 2.4)) 68 | axs[0].scatter(dish_a['epoch'], dish_a['rtt'] / 1e6, s=0.5, rasterized=True) 69 | axs[1].scatter(dish_b['epoch'], dish_b['rtt'] / 1e6, s=0.5, rasterized=True) 70 | 71 | # Plot the reduced IRTT measurements (averaged over one second) 72 | dish_a = reduce_df(dish_a) 73 | dish_b = reduce_df(dish_b) 74 | axs[0].plot(dish_a['epoch'], dish_a['rtt'] / 1e6, color='red') 75 | axs[1].plot(dish_b['epoch'], dish_b['rtt'] / 1e6, color='red') 76 | 77 | main_axs = axs[1] 78 | 79 | # limit the x axis to only the IRTT data 80 | y_max = 100 81 | for ax in axs: 82 | ax.set(xlim=(dish_a['epoch'][0], dish_a['epoch'][dish_a['epoch'].size-1]), ylim=(0, y_max)) 83 | 84 | # fix the x axis so the ticks and labels are in seconds 85 | num_seconds = round((dish_a['epoch'][dish_a['epoch'].size-1] - dish_a['epoch'][0]) / 1e9) 86 | secs_step = 30 87 | plt.xticks(list(range(dish_a['epoch'][0], dish_a['epoch'][0]+int(num_seconds*1e9), int(secs_step*1e9))), list(range(0, num_seconds, secs_step))) 88 | 89 | # add the labels 90 | main_axs.set_xlabel("Experiment time [s]") 91 | fig.text(0,0.5, "Dish 1 and 2 RTT [ms]", ha="center", va="center", rotation=90) 92 | 93 | # add the optimisation intervals 94 | optimisation_interval = OPTIMISATION_START 95 | while optimisation_interval < num_seconds: 96 | adjusted_x = dish_a['epoch'][0] + (optimisation_interval*1e9) 97 | for ax in axs: 98 | ax.plot([adjusted_x, adjusted_x], [0, y_max], color='#2d000ef4', linestyle='--') 99 | optimisation_interval += 15 100 | 101 | # show the plot 102 | plt.savefig("rtt.pdf", bbox_inches="tight", pad_inches=0, dpi=600) 103 | plt.savefig("rtt.svg", bbox_inches="tight", pad_inches=0, dpi=600) 104 | 105 | 106 | if __name__ == '__main__': 107 | # get the IRTT data from the database 108 | dish_a = read_in_values('dish-1.db', START_TIME, END_TIME) 109 | dish_b = read_in_values('dish-2.db', START_TIME, END_TIME) 110 | plot_data(dish_a, dish_b) 111 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![](https://img.shields.io/badge/WWW'24-Paper-blue)](https://doi.org/10.1145/3589334.3645328) 2 | 3 | 4 | 5 | # A Multifaceted Look at Starlink Performance 6 | 7 | This is the artifacts repository of the Web Conference (WWW) 2024 paper: A Multifaceted Look at Starlink Performance 8 | 9 | ## 📖 Abstract 10 | In recent years, Low-Earth Orbit (LEO) mega-constellations have emerged as a promising network technology and have ushered in a new era for democratizing Internet access. The Starlink network from SpaceX stands out as the only consumer-facing LEO network with over 2M+ customers and more than 4000 operational satellites. In this paper, we conduct the first-of-its-kind extensive multi-faceted analysis of Starlink network performance leveraging several measurement sources. First, based on 19.2M crowdsourced M-Lab speed test measurements from 34 countries since 2021, we analyze Starlink global performance relative to terrestrial cellular networks. Second, we examine Starlink's ability to support real-time web-based latency and bandwidth-critical applications by analyzing the performance of (i) Zoom video conferencing, and (ii) Luna cloud gaming, comparing it to 5G and terrestrial fiber. Third, we orchestrate targeted measurements from Starlink-enabled RIPE Atlas probes to shed light on the last-mile Starlink access and other factors affecting its performance globally. Finally, we conduct controlled experiments from Starlink dishes in two countries and analyze the impact of globally synchronized "15-second reconfiguration intervals" of the links that cause substantial latency and throughput variations. Our unique analysis provides revealing insights on global Starlink functionality and paints the most comprehensive picture of the LEO network's operation to date. 11 | 12 | ## 📝 Reference 13 | ``` 14 | @inproceedings{multifacetedStarlink-www24, 15 | title={{A Multifaceted Look at Starlink Performance}}, 16 | author={Mohan, Nitinder and Ferguson, Andrew and Cech, Hendrik and Renatin, Prakita Rayyan and Bose, Rohan and Marina, Mahesh and Ott, J{\"o}rg}, 17 | year = {2024}, 18 | publisher = {Association for Computing Machinery}, 19 | address = {New York, NY, USA}, 20 | booktitle = {Proceedings of the Web Conference 2024}, 21 | series = {WWW '24} 22 | } 23 | ``` 24 | 25 | ## 💾 Dataset 26 | 27 | The data necessary for the plots needs to be downloaded before starting and is available at [mediaTUM](https://mediatum.ub.tum.de/1734703) with instructions on how to set it up. 28 | 29 | 30 | ## 📊 Reproducibility Instructions 31 | All plots were created with Python3.10. We recommend following our instructions to create a virtual Python environment with the package versions that we used. 32 | 33 | ``` 34 | git clone https://github.com/nitindermohan/multifaceted-starlink-performance.git 35 | cd multifaceted-starlink-performance 36 | python3.10 -m venv .venv 37 | source .venv/bin/activate 38 | pip install -r requirements.txt 39 | ``` 40 | 41 | To plot Figure 16, you need to have `unzstd`, `jq`, and `sqlite3` available in your `$PATH`. 42 | 43 | The dataset contains both raw data and processed artifacts. To download the data required for plotting, you may use the following command. Please look up the password for rsync access at [mediaTUM](https://mediatum.ub.tum.de/1734703). 44 | 45 | ``` sh 46 | export DATA_PATH="$(pwd)/multifaceted-dataset" 47 | export RSYNC_PASSWORD="TODO: get from mediaTUM" 48 | # You may exclude the large cloud gaming packet and video captures that are not required for recreating our plots 49 | rsync -Pr rsync://m1734703@dataserv.ub.tum.de/m1734703 "$DATA_PATH" --exclude={'*mkv','dump.pcapng','dump_for_ip.pcapng'} 50 | 51 | # Unzip the RIPE Atlas measurement artifacts 52 | unzip 'atlas/ripe_atlas_repr/*zip' -d atlas/ripe_atlas_repr 53 | 54 | $ tree -L 2 "$DATA_PATH" 55 | /multifaceted-dataset/ 56 | . 57 | ├── atlas 58 | ├── controlled 59 | ├── gaming 60 | ├── mlab 61 | ├── zoom 62 | ├── checksums.sha512 63 | └── README.md 64 | ``` 65 | 66 | All plots can be created with `./plot_all.sh`. Make sure to properly configure `DATA_PATH` and `RESULT_PATH`. 67 | ``` sh 68 | # Remember that envvar DATA_PATH needs to point to the cloned dataset (see the previous step) 69 | export DATA_PATH="$(pwd)/multifaceted-dataset" 70 | export RESULT_PATH="$(pwd)/results" 71 | 72 | # RIPE Atlas plots: Figures 3, 12, 13, 14, 15, 22, 23 73 | jupyter nbconvert --to=html --output-dir="$RESULT_PATH" --execute ripe_atlas_figures/ripe_atlas_repr.ipynb 74 | 75 | # MLab plots: Figures 1, 5, 6, 7, 8, 9, 17, 18, 19, 20, 21 76 | jupyter nbconvert --to=html --output-dir="$RESULT_PATH" --execute mlab_figures/mlab_concise.ipynb 77 | 78 | # Zoom plots: Figure 10 79 | bash zoom/plot.sh 80 | 81 | # Cloud gaming plots: Figure 11 82 | bash gaming/plot.sh 83 | 84 | # Controlled experiments: Figure 16 85 | bash controlled/16a/generate_fig_16a.sh # Generate Figure 16a 86 | bash controlled/16b/generate_fig_16b.sh # Generate Figure 16b 87 | bash controlled/16c/generate_fig_16c.sh # Generate Figure 16c 88 | ``` 89 | -------------------------------------------------------------------------------- /controlled/16c/plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import matplotlib.pyplot as plt 4 | import sys 5 | import os 6 | import pandas 7 | import json 8 | import datetime 9 | import sqlite3 10 | import numpy 11 | 12 | # do we show a raw plot or a reduced plot 13 | REDUCED=False 14 | 15 | # the first interval between optimisation intervals in the plot 16 | OPTIMISATION_START=13 17 | 18 | fig, axs = plt.subplots(2, sharex=False, figsize=(2.8, 2.4)) #down from 2.2 19 | 20 | 21 | ###################### 22 | # Load the IRTT Data # 23 | ###################### 24 | 25 | times = [] 26 | rtts = [] 27 | with open(f'{sys.argv[1]}/controlled/irtt_fov/2023-10-08_18-41-11.json') as f: 28 | jj = json.load(f) 29 | for rt in jj['round_trips']: 30 | if rt['delay'] == {}: continue 31 | times.append(rt['timestamps']['client']['send']['wall'] / 1e9) 32 | rtts.append(rt['delay']['rtt']) 33 | irtt = pandas.DataFrame({'epoch': times, 'rtt': rtts}) 34 | 35 | ###################### 36 | # Plot the IRTT data # 37 | ###################### 38 | 39 | axs[0].scatter(irtt['epoch'] - min(irtt['epoch']), irtt['rtt'] / 1e6, s=0.5, rasterized=True) 40 | 41 | x_size = max(irtt['epoch']) - min(irtt['epoch']) 42 | y_max = 100 #max(irtt['rtt'] / 1e6) 43 | axs[0].set(xlim=(0, x_size), ylim=(10, y_max)) 44 | 45 | axs[0].set_xlabel("Experiment Duration [s]") 46 | axs[0].set_ylabel("RTT [ms]") 47 | axs[0].set_yticks([10,55,100]) 48 | 49 | first_time_ds = datetime.datetime.fromtimestamp(min(irtt['epoch'])) 50 | first_interval_ds = datetime.datetime(first_time_ds.year, first_time_ds.month, first_time_ds.day, first_time_ds.hour, first_time_ds.minute-1, 57) 51 | first_interval = first_interval_ds.timestamp() 52 | while first_interval <= max(irtt['epoch']) + 15: 53 | axs[0].plot([first_interval - min(irtt['epoch']), first_interval - min(irtt['epoch']) ], [0, y_max], color='#2d000ef4', linestyle='--', label='Reconfiguration Interval' if first_interval == min(irtt['epoch']) else None) 54 | first_interval += 15 55 | 56 | ###################### 57 | # Plot the GRPC data # 58 | ###################### 59 | 60 | def get_intervals(time_list): 61 | intervals = [] 62 | time_list = sorted(time_list) 63 | current_interval = (time_list[0], time_list[0]) 64 | for ts in time_list[1:]: 65 | if ts == current_interval[1] + 1: 66 | current_interval = (current_interval[0], ts) 67 | elif ts > current_interval[1] + 1: 68 | intervals.append(current_interval) 69 | current_interval = (ts,ts) 70 | elif ts < current_interval[1] + 1: 71 | raise Exception("Somehow gone backwards in time in a sorted list") 72 | # don't forget to append the last interval! 73 | intervals.append(current_interval) 74 | return intervals 75 | 76 | def get_connected_intervals(grpc_df): 77 | connected_ts = sorted(grpc_df[grpc_df['state'] == 'CONNECTED']['time'].tolist()) 78 | return get_intervals(connected_ts) 79 | 80 | def get_prev_interval(ts): 81 | dts = datetime.datetime.fromtimestamp(ts) 82 | if dts.second >= 12 and dts.second < 27: 83 | prev_i = datetime.datetime(year=dts.year, month=dts.month, day=dts.day, hour=dts.hour, minute=dts.minute, second=12) 84 | elif dts.second >= 27 and dts.second < 42: 85 | prev_i = datetime.datetime(year=dts.year, month=dts.month, day=dts.day, hour=dts.hour, minute=dts.minute, second=27) 86 | elif dts.second >= 42 and dts.second < 57: 87 | prev_i = datetime.datetime(year=dts.year, month=dts.month, day=dts.day, hour=dts.hour, minute=dts.minute, second=42) 88 | elif dts.second >= 57: 89 | prev_i = datetime.datetime(year=dts.year, month=dts.month, day=dts.day, hour=dts.hour, minute=dts.minute, second=57) 90 | else: 91 | prev_i = datetime.datetime(year=dts.year, month=dts.month, day=dts.day, hour=dts.hour, minute=dts.minute, second=57) 92 | prev_i -= datetime.timedelta(minutes=1) 93 | return int(prev_i.timestamp()) 94 | 95 | grpc_connection_intervals = [] 96 | 97 | grpc_path = f'{sys.argv[1]}/controlled/grpc_fov' 98 | grpc_files = [f'{grpc_path}/{file}' for file in os.listdir(grpc_path) if os.path.isfile(f'{grpc_path}/{file}') and file.endswith('.db')] 99 | for grpc_file in grpc_files: 100 | conn = sqlite3.connect(grpc_file) 101 | grpc_df = pandas.read_sql(f'SELECT * FROM status', conn) 102 | conn.close() 103 | grpc_connection_intervals += get_connected_intervals(grpc_df) 104 | 105 | 106 | start_seconds_to_prev_interval = [abs(get_prev_interval(iv[0])-iv[0]) for iv in grpc_connection_intervals] 107 | end_seconds_to_prev_interval = [abs(get_prev_interval(iv[1])-iv[1]) for iv in grpc_connection_intervals] 108 | 109 | num_intervals = len(grpc_connection_intervals) 110 | barchart_start = [len([val for val in start_seconds_to_prev_interval if val==i]) / num_intervals for i in range(15)] 111 | barchart_end = [len([val for val in end_seconds_to_prev_interval if val==i]) / num_intervals for i in range(15)] 112 | 113 | rects = axs[1].bar(numpy.arange(15) - 0.2, barchart_start, width=0.4, label='Connectivity Start') 114 | rects = axs[1].bar(numpy.arange(15) + 0.2, barchart_end, width=0.4, label='Connectivity End') 115 | 116 | axs[1].set_xlabel(f'Seconds to prev. R.I.') 117 | axs[1].set_ylabel('Probability') 118 | axs[1].set_xticks(range(0,15,2)) 119 | axs[1].set_yticks([0, 0.2, 0.4]) 120 | 121 | # show the plot 122 | plt.legend(fontsize=8, frameon=False) 123 | plt.savefig("fov.pdf", bbox_inches="tight", pad_inches=0, dpi=600) 124 | plt.savefig("fov.svg", bbox_inches="tight", pad_inches=0, dpi=600) 125 | -------------------------------------------------------------------------------- /gaming/inspect_xtime.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import json 5 | import pandas as pd 6 | import os 7 | import sys 8 | import matplotlib.pyplot as plt 9 | import matplotlib.dates as mdates 10 | from matplotlib import rcParams 11 | from matplotlib.backends.backend_pdf import PdfPages 12 | import shutil 13 | 14 | def plot_optimization_interval(ax, start, end): 15 | res = [] 16 | opt_secs = [12,27,42,57] 17 | tmp = start.replace(second=opt_secs[0]) 18 | for opt_sec in opt_secs: 19 | if opt_sec >= start.second: 20 | tmp = start.replace(second=opt_sec) 21 | break 22 | tmp = tmp.replace(microsecond=0, nanosecond=0) 23 | res.append(tmp) 24 | while True: 25 | tmp += pd.Timedelta(seconds=15) 26 | if tmp > end: 27 | break 28 | else: 29 | res.append(tmp) 30 | for opt in res: 31 | part = ax.axvline(opt, label="SL Interval", 32 | # color="green", alpha=0.5, 33 | color="#360B18", alpha=1, 34 | linestyle="--", zorder=-1) 35 | return part 36 | 37 | def datetime_as_timedelta_formatter(start_ts): 38 | start_mts = mdates.date2num(start_ts) 39 | def formatter(x, pos): 40 | diff = mdates.num2timedelta(x - start_mts).total_seconds() 41 | return f"{diff:g}" 42 | return formatter 43 | 44 | def plot_external_legend(parts, figsize, **kwargs): # ncol = 45 | fig_legend = plt.figure(figsize=figsize) 46 | fig_legend.legend(parts, [p.get_label() for p in parts], **kwargs) 47 | fig_legend.tight_layout() 48 | return fig_legend 49 | 50 | 51 | def main(): 52 | parser = argparse.ArgumentParser() 53 | parser.add_argument("--dirs", nargs="+", help="Result directories") 54 | parser.add_argument("--gdelays", nargs="+", help="delay_ts files") 55 | parser.add_argument("--views", nargs="+", help="Parts to show, in second offsets: start,end") 56 | parser.add_argument("--save", help="Save datapoints to csv file") 57 | args = parser.parse_args() 58 | 59 | if len(args.dirs) != len(args.gdelays) != len(args.views): 60 | print("The same number of arguments need to be passed") 61 | sys.exit(1) 62 | args.views = [(int(view.split(",")[0]), int(view.split(",")[1])) for view in args.views] 63 | 64 | plt.set_cmap("tab10") 65 | rcParams["font.family"] = "CMU Sans Serif" 66 | rcParams["font.size"] = 10.0 67 | plt.rc('text', usetex=True if shutil.which('latex') else False) 68 | 69 | 70 | fig, axes = plt.subplots(figsize=(4, 1.5), ncols=len(args.dirs), sharey=True) 71 | 72 | 73 | for ax_i, (ax, dir, delays_ts, view) in enumerate(zip(axes, args.dirs, args.gdelays, args.views)): 74 | with open(os.path.join(dir, "current_rtt.json"), "r") as f: 75 | rtt = json.load(f) 76 | rtt["ts"] = [pd.Timestamp(v, unit="s") for v in rtt["ts"]] 77 | with open(os.path.join(dir, "parsed_videoReceiveStream.json"), "r") as f: 78 | video = json.load(f) 79 | video = video[list(video.keys())[0]] 80 | video["ts"] = [pd.Timestamp(v, unit="s") for v in video["time_ms"]] 81 | 82 | parts = [] 83 | 84 | colors = list(plt.get_cmap("tab10").colors) 85 | 86 | start_ts, end_ts = rtt["ts"][0], rtt["ts"][-1] 87 | 88 | parts.append(ax.plot(rtt["ts"], rtt["rtts"], label="RTT", color=colors.pop(0))[0]) 89 | 90 | ax_fps = ax.twinx() 91 | 92 | color_fps = colors.pop(0) 93 | ax_fps.yaxis.label.set_color(color_fps) 94 | ax_fps.spines['right'].set_color(color_fps) 95 | ax_fps.tick_params(axis="y", color=color_fps) 96 | 97 | parts.append(ax_fps.plot(video["ts"], video["ren_fps"], label="FPS", color=color_fps)[0]) 98 | colors.pop(0) # skip green; use by axvline 99 | parts.append(ax.plot(video["ts"], video["jb_delay"], label="Jitter Buffer", color=colors.pop(0))[0]) 100 | 101 | gdelay = pd.read_csv(delays_ts) 102 | gdelay["ts"] = pd.to_datetime(gdelay["ts"], unit="s") 103 | gdelay = gdelay[(gdelay.ts >= start_ts) & (gdelay.ts <= end_ts)] 104 | 105 | parts.append(ax.scatter(gdelay["ts"], gdelay["delay_ms"], label="Game Delay", color=colors.pop(0))) 106 | 107 | print(f"dir={dir}") 108 | if dir.find("starlink") > 0: 109 | plot_optimization_interval(ax, start_ts, end_ts) 110 | 111 | if view[0] != 0: 112 | ax.set_xlim(left=start_ts + pd.Timedelta(view[0], unit="seconds")) 113 | if view[1] != 0: 114 | ax.set_xlim(right=start_ts + pd.Timedelta(view[1], unit="seconds")) 115 | 116 | ax_fps.set_ylim(40, 65) 117 | ax.set_ylim(0, 220) 118 | ax.set_yticks([0, 50, 100, 150, 200]) 119 | if ax_i == 0: 120 | ax.set_ylabel("ms") 121 | if ax_i == len(axes)-1: 122 | ax_fps.set_ylabel("FPS") # last column 123 | else: 124 | ax_fps.get_yaxis().set_ticklabels([]) 125 | ax.set_xlabel("Time (s)") 126 | 127 | xfmt = datetime_as_timedelta_formatter(start_ts.replace(microsecond=0, nanosecond=0)) 128 | ax.xaxis.set_major_formatter(xfmt) 129 | 130 | xtick_interval = 10 131 | ax.xaxis.set_major_locator(mdates.SecondLocator(interval=xtick_interval)) 132 | 133 | # Draw ax above ax_fps in terms of zorder 134 | ax.set_zorder(1) 135 | ax.set_frame_on(False) 136 | 137 | fig.tight_layout() 138 | 139 | fig.subplots_adjust(wspace=0.1) 140 | 141 | fig_legend = plot_external_legend(parts, (4, 1), ncols=len(parts), 142 | labelspacing=0.3, handlelength=1.2, handletextpad=0.4, columnspacing=1.2) 143 | 144 | if args.save: 145 | filepath, ext = os.path.splitext(args.save) 146 | if ext == ".pdf": 147 | with PdfPages(args.save) as pdf: 148 | pdf.savefig(fig, bbox_inches="tight", pad_inches=0) 149 | pdf.savefig(fig_legend, bbox_inches="tight") 150 | else: 151 | fig.savefig(filepath + "_plot" + ext, bbox_inches="tight", pad_inches=0) 152 | fig.savefig(filepath + "_legend" + ext, bbox_inches="tight") 153 | else: 154 | plt.show() 155 | 156 | if __name__ == "__main__": 157 | main() 158 | -------------------------------------------------------------------------------- /mlab_figures/geospatial_plot_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pymap3d.ecef import eci2geodetic 3 | from datetime import datetime 4 | from functools import partial 5 | from scipy.spatial.transform import Rotation as R 6 | import geopandas 7 | import cartopy.crs as ccrs 8 | from matplotlib import colors 9 | import matplotlib.pyplot as plt 10 | 11 | from orbit_utils import params 12 | 13 | def get_orbitecis(incl, raan, altitude, n, begin_time=0, Earth_R=6371, portion=1.0, retrograde=False): 14 | """Generates a circular orbit based on given parameters 15 | 16 | Params: 17 | incl: Inclination in Degrees 18 | raan: Right Ascension to Ascending Node in Degrees 19 | altitude: Altitude in km 20 | n: amount of points needed for a full revolution 21 | begin_time: time in which to begin 22 | portion: ratio of the portion of points to be created 23 | 24 | Returns: 25 | A tuple 26 | ( 27 | np array of r_eci points in km, 28 | np array of v_eci in km/s, 29 | delta seconds since beginning) 30 | """ 31 | 32 | # Calculating r_eci 33 | if retrograde: 34 | sign = 1 35 | else: 36 | sign = -1 37 | points = [] 38 | for i in range(int(n*portion)): 39 | points.append([np.sin(sign * 2*np.pi * (i/n)), np.cos(sign * 2*np.pi * (i/n)), 0]) 40 | points = np.array(points) * (altitude + Earth_R) # Plus height in km 41 | 42 | m_raan = R.from_rotvec(np.array([0, 0, 1]) * raan, degrees=True).as_matrix() # Rotation along the north-pole axis 43 | m_incl = R.from_rotvec(np.array([1, 0, 0]) * incl, degrees=True).as_matrix() # Rotation along the axis towards the ascending node 44 | 45 | raan_vec = np.matmul(m_raan, np.array([1, 0, 0])) 46 | incl_vec = np.matmul(m_incl, np.array([0, 1, 0])) 47 | incl_vec = np.matmul(m_raan, incl_vec) 48 | normal = np.cross(raan_vec, incl_vec) 49 | basechange_m = np.array([raan_vec, incl_vec, normal]) 50 | 51 | r_ecis = np.matmul(basechange_m.T, points.T).T 52 | 53 | # Calculating Time periods 54 | mu = 398600.4418 # unit: (km)^3 / s^2 55 | period_s = ((2*np.pi) / np.sqrt(mu)) * np.sqrt(altitude + Earth_R)**3 # Period of one revolution 56 | delta_array = np.arange(0, period_s, period_s/n) 57 | time_array = [datetime.fromtimestamp(dt + begin_time) for dt in delta_array] 58 | 59 | # Calculating v_eci in km/s 60 | velocity = (2 * np.pi * (altitude + Earth_R)) / period_s 61 | v_ecis = [] 62 | for r_eci in r_ecis: 63 | v_eci_ = np.cross(-r_eci, normal) 64 | v_eci_ = v_eci_ / np.linalg.norm(v_eci_) 65 | v_eci_ *= velocity 66 | v_ecis.append(v_eci_) 67 | v_ecis = np.array(v_ecis) 68 | 69 | return r_ecis, v_ecis, time_array 70 | 71 | def get_bounding_points(r_eci, v_eci, time, min_elevation=15): 72 | """ 73 | Given a eci point, we calculate the bounding points of its coverage based 74 | on min_elevation in degrees. 75 | 76 | r_eci: point in ECI coordinates 77 | v_eci: vector tangential forward, also can be seen as velocity 78 | min_elevation: minimum elevation, for coverage 79 | 80 | returns upperbounding point, lowerbounding point as lat long coordinates 81 | """ 82 | altitude = np.linalg.norm(r_eci) - 6371 83 | _, _, beta, _ = params(min_elevation, altitude, R = 6371) 84 | rot_m = R.from_rotvec(v_eci * np.radians(beta) / np.linalg.norm(v_eci)).as_matrix() 85 | 86 | bounding_reci1 = np.matmul(rot_m, r_eci) 87 | bounding_reci2 = np.matmul(np.linalg.inv(rot_m), r_eci) 88 | 89 | lat1, long1, _ = eci2geodetic(*(bounding_reci1 * 1000), time) 90 | lat2, long2, _ = eci2geodetic(*(bounding_reci2 * 1000), time) 91 | return ((lat1, long1), (lat2, long2)) 92 | 93 | def get_coords(incl, raan, altitude, n, begin_time=0, portion=1.0): 94 | """Generates ground track of a circular orbit based on given parameters 95 | 96 | Params: 97 | incl: Inclination in Degrees 98 | raan: Right Ascension to Ascending Node in Degrees 99 | altitude: Altitude in km 100 | n: amount of points to be generated 101 | begin_time: time in which to begin 102 | 103 | Returns: 104 | array of lat long tuples 105 | """ 106 | ps, vs, ts = get_orbitecis(incl, raan, altitude, n, begin_time, portion=portion) 107 | 108 | coords = [] 109 | bound_coords1 = [] 110 | bound_coords2 = [] 111 | for point, fwd, time in zip(ps, vs, ts): 112 | lat, long, altitude = eci2geodetic(*(point * 1000), time) 113 | coords.append((lat[0], long[0])) 114 | bc1, bc2 = get_bounding_points(point, fwd, time) 115 | bound_coords1.append(bc1) 116 | bound_coords2.append(bc2) 117 | return np.array(coords), np.array(bound_coords1), np.array(bound_coords2) 118 | 119 | # partitioning a coordinate list based on continuities 120 | def discontin_partition(coord_list): 121 | latitude, longitude = coord_list[:, 0], coord_list[:, 1] 122 | coords_list_ = [(latitude, longitude)] 123 | last_long = 0 124 | for i, long in enumerate(longitude): 125 | if (np.abs(long - last_long) > 200): 126 | coords_list_ = [(coord_list[:i, 0], coord_list[:i, 1])] 127 | coords_list_.append((coord_list[i:, 0], coord_list[i:, 1])) 128 | break 129 | else: 130 | last_long = long 131 | return coords_list_ 132 | 133 | # Plotting GS points 134 | def plot_period_groundtrack(coord_list, ax, color, label=""): 135 | coords_list = discontin_partition(coord_list) 136 | for lats, longs in coords_list: 137 | ax.plot(longs, lats, 138 | color=color, 139 | zorder = 10, 140 | linewidth = 0.7, label=label) 141 | label = "" 142 | 143 | def construct_polygon(bound_coords1, bound_coords2): 144 | """Constructing polygon based on bounding coords. 145 | 146 | We are working with the following assumptions about the coordinates: 147 | - they are contiguously ordered 148 | - they do one revolution around the longitude coordinates 149 | 150 | This is a rather quick and dirty solution but it somewhat works, so year. The resulting polygon might have some weird 151 | zizags 152 | """ 153 | def get_longitude(c): 154 | return c[1] 155 | bound_coords1 = sorted(bound_coords1, key=get_longitude) 156 | bound_coords2 = sorted(bound_coords2, key=get_longitude, reverse=True) 157 | polygon = np.array([*bound_coords1, *bound_coords2]) 158 | return polygon 159 | 160 | def plot_bounds(bound_coords1, bound_coords2, ax, color): 161 | bound_polygon = construct_polygon(bound_coords1, bound_coords2) 162 | ax.fill(bound_polygon[:, 1], bound_polygon[:, 0], facecolor=color) 163 | -------------------------------------------------------------------------------- /zoom/plot_xtime_figure.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pandas as pd 4 | import matplotlib.pyplot as plt 5 | from matplotlib import rcParams 6 | import matplotlib.patches as mpatches 7 | import matplotlib.ticker as mticker 8 | from matplotlib.backends.backend_pdf import PdfPages 9 | import os.path 10 | import argparse 11 | import sys 12 | import shutil 13 | 14 | import zoom_parser as zp 15 | 16 | def plot_external_legend(parts, figsize, **kwargs): # ncol = 17 | fig_legend = plt.figure(figsize=figsize) 18 | fig_legend.legend(parts, [p.get_label() for p in parts], **kwargs) 19 | fig_legend.tight_layout() 20 | return fig_legend 21 | 22 | def compute_interval_timestamps(ax, start, end): 23 | res = [] 24 | opt_secs = [12,27,42,57] 25 | tmp = start.replace(second=opt_secs[0]) 26 | for opt_sec in opt_secs: 27 | if opt_sec >= start.second: 28 | tmp = start.replace(second=opt_sec) 29 | break 30 | tmp = tmp.replace(microsecond=0, nanosecond=0) 31 | res.append(tmp) 32 | while True: 33 | tmp += pd.Timedelta(seconds=15) 34 | if tmp > end: 35 | break 36 | else: 37 | res.append(tmp) 38 | return res 39 | 40 | def plot_latency(ax, streams, df): # df = packets 41 | media_streams = streams[(streams.stream_type == "m") & (streams.media_type == "v")] 42 | 43 | for stream in media_streams.itertuples(): 44 | df_ssrc = df[(df.ssrc == stream.rtp_ssrc)] 45 | 46 | direction = "UL" if stream.is_local else "DL" 47 | media_type = "Audio" if stream.media_type == "a" else "Video" if stream.media_type == "v" else stream.media_type 48 | # ax.plot(df_ssrc['ts_local'], abs(df_ssrc['owd']), label=f"{direction} {media_type}") 49 | ax.scatter(df_ssrc['ts_local'], abs(df_ssrc['owd']), label=f"{direction} {media_type}", 50 | s=0.3, rasterized=True) 51 | 52 | ctab = list(plt.get_cmap("tab10").colors) 53 | colors = dict(owd=ctab[0], video=ctab[1], fec=ctab[3], 54 | interval="#360B18") 55 | # interval=ctab[2]) 56 | 57 | def plot(ax, dir_path, view, plot_intervals=False): 58 | dir_local = os.path.join(dir_path, "local/") 59 | dir_remote = os.path.join(dir_path, "remote/") 60 | 61 | try: 62 | packets_local = zp.read_packets_csv(dir_local) 63 | packets_remote = zp.read_packets_csv(dir_remote) 64 | packets = zp.merge_packet_dfs(packets_local, packets_remote) 65 | 66 | # lost = zp.find_lost_packets(packets_local, packets_remote) 67 | 68 | # frames_local = zp.read_frames_csv(dir_local) 69 | # frames_remote = zp.read_frames_csv(dir_remote) 70 | 71 | streams = zp.read_streams_csv(dir_local) 72 | 73 | # stats_local = read_stats_csv(dir_local) 74 | # stats_remote = read_stats_csv(dir_remote) 75 | # stats_fig(streams, stats_local, stats_remote) 76 | except Exception as e: 77 | print(f"Failed to parse Zoom packet data from {dir_path}: {e}") 78 | sys.exit(1) 79 | 80 | 81 | media_only = streams.stream_type == "m" 82 | video_only = streams.media_type == "v" 83 | uplink_only = (streams.is_local) 84 | media_streams = streams[media_only & video_only & uplink_only] 85 | 86 | ax2 = ax.twinx() 87 | 88 | def comp_rate(df): 89 | return df.set_index("ts_rel")["pl_len"].resample("1s").sum() * 8 / 1e6 90 | 91 | for stream in media_streams.itertuples(): 92 | owd = abs(packets[(packets.ssrc == stream.rtp_ssrc)].set_index("ts").owd) 93 | packets_local_ssrc = packets_local[packets_local.ssrc == stream.rtp_ssrc] 94 | 95 | direction = "UL" if stream.is_local else "DL" 96 | media_type = "Audio" if stream.media_type == "a" else "Video" if stream.media_type == "v" else stream.media_type 97 | # color = colors[0] if direction == "UL" else colors[3] 98 | 99 | # ax.scatter(df.ts.dt.total_seconds(), abs(df.owd), 100 | ax2.scatter(owd.index.total_seconds(), owd, label=f"{direction} {media_type}", 101 | color=colors["owd"], s=0.3, rasterized=True, 102 | zorder=-1) 103 | 104 | video_bitrate = comp_rate(packets_local_ssrc[packets_local_ssrc.pt == 98]) 105 | fec_bitrate = comp_rate(packets_local_ssrc[packets_local_ssrc.pt == 110]) 106 | ax.plot(video_bitrate.index.total_seconds(), video_bitrate, 107 | label=f"{direction} {media_type} Video", color=colors["video"]) 108 | ax.plot(fec_bitrate.index.total_seconds(), fec_bitrate, 109 | label=f"{direction} {media_type} FEC", color=colors["fec"]) 110 | 111 | if plot_intervals: 112 | opt_tss = compute_interval_timestamps(ax, min(packets.ts_local), max(packets.ts_local)) 113 | min_ts = min(packets.ts_local) 114 | for opt in opt_tss: 115 | xval = (opt - min_ts).total_seconds() 116 | part = ax2.axvline(xval, label="SL Interval", color=colors["interval"], 117 | alpha=1, linestyle="--", zorder=-2) 118 | 119 | ax.set_ylabel("Mbps") 120 | # ax.yaxis.label.set_color(colors[1]) 121 | # ax.spines['right'].set_color(colors[1]) 122 | # ax.tick_params(axis="y", color=colors[1]) 123 | ax.set_ylim(0, 1.5) 124 | 125 | ax2.set_ylabel("Latency (ms)") 126 | ax2.yaxis.label.set_color(colors["owd"]) 127 | ax2.spines['left'].set_color(colors["owd"]) 128 | ax2.tick_params(axis="y", color=colors["owd"]) 129 | ax2.set_ylim(0, 150) 130 | 131 | # Layer ax over ax2 132 | ax.set_zorder(1) 133 | ax.set_frame_on(False) 134 | 135 | ax.set_xlabel("Time (s)") 136 | if view is not None: 137 | ax.set_xlim(left=view[0], right=view[1]) 138 | 139 | def parse_int_tuple(arg): 140 | if arg is None: 141 | return None 142 | parts = arg.split(",") 143 | return (int(parts[0]), int(parts[1])) 144 | 145 | def main(): 146 | parser = argparse.ArgumentParser() 147 | # parser.add_argument("directories", nargs="+", help="Directories with zpkt and parsed csv files") 148 | parser.add_argument("ter", help="Terrestrial: directories with local/ and remote/ directories that contain zpkt and parsed csv files") 149 | parser.add_argument("stl", help="Starlink: directories with local/ and remote/ directories that contain zpkt and parsed csv files") 150 | parser.add_argument("--view-ter", help="Part of stl to show, seconds offsets: start,end") 151 | parser.add_argument("--view-stl", help="Part of stl to show, seconds offsets: start,end") 152 | parser.add_argument("-b", action="store_true", help="Break after parsing csvs") 153 | parser.add_argument("--save", "-w", help="Write plot") 154 | args = parser.parse_args() 155 | 156 | view_stl = parse_int_tuple(args.view_stl) 157 | view_ter = parse_int_tuple(args.view_ter) 158 | 159 | plt.set_cmap("tab10") 160 | rcParams["font.family"] = "CMU Sans Serif" 161 | rcParams["font.size"] = 10.0 162 | plt.rc('text', usetex=True if shutil.which('latex') else False) 163 | 164 | if args.b: 165 | breakpoint() 166 | 167 | fig, axes = plt.subplots(figsize=(4,1.1), ncols=2, dpi=300, sharey=True) 168 | 169 | plot(axes[0], args.ter, view_ter) 170 | plot(axes[1], args.stl, view_stl, plot_intervals=True) 171 | 172 | twin_ax_0 = axes[0].get_shared_x_axes().get_siblings(axes[0])[0] 173 | twin_ax_1 = axes[1].get_shared_x_axes().get_siblings(axes[0])[0] 174 | twin_ax_0.set_ylabel("") 175 | twin_ax_0.set_yticklabels([]) 176 | axes[1].set_ylabel("") 177 | 178 | axes[0].xaxis.set_major_locator(mticker.MultipleLocator(base=15)) 179 | axes[1].xaxis.set_major_locator(mticker.MultipleLocator(base=15)) 180 | 181 | fig.subplots_adjust(wspace=0.1) 182 | 183 | # twin_ax_1.get_shared_y_axes().join(twin_ax_0, twin_ax_1) 184 | 185 | # fig.subplots_adjust(wspace=0.1) 186 | 187 | parts = [ 188 | mpatches.Patch(color=colors["owd"], label="One-Way Delay"), 189 | mpatches.Patch(color=colors["video"], label="Video Throughput"), 190 | mpatches.Patch(color=colors["fec"], label="FEC Throughput") 191 | # mpatches.Patch(color=colors["interval"], label='SL') 192 | ] 193 | fig_legend = plot_external_legend(parts, (4, 1), ncols=len(parts)) 194 | 195 | if args.save: 196 | filepath, ext = os.path.splitext(args.save) 197 | if ext == ".pdf": 198 | with PdfPages(args.save) as pdf: 199 | pdf.savefig(fig, bbox_inches="tight", pad_inches=0) 200 | pdf.savefig(fig_legend, bbox_inches="tight") 201 | else: 202 | # fig.savefig(args.save, bbox_inches="tight", pad_inches=0) 203 | fig.savefig(filepath + "_plot" + ext, bbox_inches="tight", pad_inches=0) 204 | fig_legend.savefig(filepath + "_legend" + ext, bbox_inches="tight") 205 | else: 206 | plt.show() 207 | 208 | if __name__ == "__main__": 209 | main() 210 | -------------------------------------------------------------------------------- /gaming/compute_overall_metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import json 5 | import pandas as pd 6 | import numpy as np 7 | import os 8 | import sys 9 | import matplotlib.pyplot as plt 10 | 11 | import compute_delay 12 | 13 | def compute_overall(args): 14 | metrics = dict( 15 | current_delay = [], 16 | jb_delay = [], 17 | decode_delay = [], 18 | render_delay_ms = [], 19 | gaming_delay = [], 20 | rtt = [], 21 | ren_fps = [], 22 | freeze_count = 0, 23 | total_freezes_duration_ms = 0, 24 | frame_drop = 0, 25 | height = {1080:0, 720:0}, 26 | rtx_bps = [], 27 | rx_bps = [], 28 | target_delay_ms = [], 29 | interframe_delay_max_ms = [], 30 | interframe_delay_ms = [], 31 | duration_s = 0, 32 | frames_rendered = 0 33 | ) 34 | 35 | for d in args.directories: 36 | with open(os.path.join(d, "parsed_videoReceiveStream.json"), "r") as f: 37 | stats = json.load(f) 38 | stats = stats[list(stats.keys())[0]] 39 | # dict_keys(['count', 'current_delay', 'dec_fps', 'decode_delay', 'first_frame_received_to_decoded_ms', 'frame_drop', 'frames_decoded', 'frames_rendered', 'freeze_count', 'height', 'interframe_delay_max_ms', 'jb_cuml_delay', 'jb_delay', 'jb_emit_count', 'max_decode_ms', 'min_playout_delay_ms', 'net_fps', 'pause_count', 'ren_fps', 'render_delay_ms', 'rtx_bps', 'rx_bps', 'sum_squared_frame_durations', 'sync_offset_ms', 'target_delay_ms', 'time_ms', 'total_bps', 'total_decode_time_ms', 'total_frames_duration_ms', 'total_freezes_duration_ms', 'total_inter_frame_delay', 'total_pauses_duration_ms', 'total_squared_inter_frame_delay', 'width']) 40 | 41 | for metric in metrics.keys(): 42 | if metric in ["gaming_delay", "rtt"]: 43 | pass 44 | # computed later 45 | elif metric in ["freeze_count", "total_freezes_duration_ms", "frame_drop"]: 46 | # These values only increase 47 | metrics[metric] += np.max(stats[metric]) 48 | elif metric == "height": 49 | values, counts = np.unique(stats[metric], return_counts=True) 50 | for val, cnt in zip(values, counts): 51 | metrics[metric][int(val)] += cnt 52 | elif metric == "duration_s": 53 | metrics[metric] += np.max(stats["time_ms"]) - np.min(stats["time_ms"]) 54 | elif metric == "frames_rendered": 55 | metrics[metric] += np.max(stats[metric]) - np.min(stats[metric]) 56 | elif metric == "interframe_delay_ms": 57 | stats_key = "total_inter_frame_delay" 58 | metrics[metric] = [(a-b)*1000 for a,b in zip(stats[stats_key][1:], stats[stats_key][:-1])] 59 | else: 60 | metrics[metric].extend(stats[metric]) 61 | 62 | gd_df = compute_delay.assemble_df(d) 63 | gdelay = compute_delay.compute_delay(gd_df) 64 | metrics["gaming_delay"].extend(gdelay.to_list()) 65 | 66 | with open(os.path.join(d, "current_rtt.json"), "r") as f: 67 | rtts = json.load(f) 68 | metrics["rtt"].extend(rtts["rtts"]) 69 | 70 | result = dict() 71 | for metric, data in metrics.items(): 72 | if metric in ["freeze_count"]: 73 | result[metric] = data 74 | elif metric == "total_freezes_duration_ms": 75 | result[metric] = data 76 | result[metric + "_rel"] = float(data / (metrics["duration_s"] * 1000)) 77 | elif metric == "frame_drop": 78 | result[metric] = data 79 | result[metric + "_rel"] = float(data / (data + metrics["frames_rendered"])) 80 | elif metric == "height": 81 | total = data[1080] + data[720] 82 | result["height_1080"] = int(data[1080]) 83 | result["height_1080_rel"] = data[1080] / total 84 | result["height_720"] = int(data[720]) 85 | result["height_720_rel"] = data[720] / total 86 | elif metric == "duration_s": 87 | result[metric] = data 88 | elif metric == "frames_rendered": 89 | result[metric] = int(data) 90 | else: 91 | result[metric] = dict( 92 | mean=np.mean(data), 93 | std=np.std(data), 94 | median=np.percentile(data, 50), 95 | min=np.min(data), 96 | max=np.max(data), 97 | cnt=len(data), 98 | coeff_var=np.std(data) / np.mean(data) 99 | ) 100 | 101 | for k,v in result.items(): 102 | print(f"{k}:{type(v)}") 103 | 104 | print(json.dumps(result, indent=4)) 105 | if args.save: 106 | with open(args.save, "w") as f: 107 | json.dump(result, f, indent=4) 108 | 109 | 110 | def compute_metrics(df): 111 | return dict(mean=df.mean(), 112 | std=df.std(), 113 | median=df.median(), 114 | min=df.min(), 115 | max=df.max(), 116 | cnt=len(df), 117 | coeff_var=df.std() / df.mean()) 118 | 119 | def compute_per_minute(args): 120 | metrics = dict( 121 | ren_fps = "mean", 122 | jb_delay = "mean", 123 | current_delay = "mean", 124 | decode_delay = "mean", 125 | render_delay_ms = "mean", 126 | freeze_count = "diff", 127 | total_freezes_duration_ms = "diff", 128 | frame_drop = "diff", 129 | # height = {1080:0, 720:0}, 130 | rtx_bps = "mean", 131 | rx_bps = "mean", 132 | target_delay_ms = "mean", 133 | interframe_delay_max_ms = "mean", 134 | # duration_s = 0, 135 | frames_rendered = "" # do nothing 136 | ) 137 | 138 | dfs = [] 139 | rtts = [] 140 | gdelays = [] 141 | for di, d in enumerate(args.directories): 142 | print(d) 143 | with open(os.path.join(d, "parsed_videoReceiveStream.json"), "r") as f: 144 | stats = json.load(f) 145 | stats = stats[list(stats.keys())[0]] 146 | # dict_keys(['count', 'current_delay', 'dec_fps', 'decode_delay', 'first_frame_received_to_decoded_ms', 'frame_drop', 'frames_decoded', 'frames_rendered', 'freeze_count', 'height', 'interframe_delay_max_ms', 'jb_cuml_delay', 'jb_delay', 'jb_emit_count', 'max_decode_ms', 'min_playout_delay_ms', 'net_fps', 'pause_count', 'ren_fps', 'render_delay_ms', 'rtx_bps', 'rx_bps', 'sum_squared_frame_durations', 'sync_offset_ms', 'target_delay_ms', 'time_ms', 'total_bps', 'total_decode_time_ms', 'total_frames_duration_ms', 'total_freezes_duration_ms', 'total_inter_frame_delay', 'total_pauses_duration_ms', 'total_squared_inter_frame_delay', 'width']) 147 | 148 | selected_stats = {k:v for k,v in stats.items() if k in metrics.keys()} 149 | df = pd.DataFrame.from_dict(selected_stats) 150 | df.set_index(pd.to_datetime(stats["time_ms"], unit="s"), inplace=True) 151 | df["file"] = di 152 | 153 | total_ifd = stats["total_inter_frame_delay"] 154 | df["interframe_delay_ms"] = [np.nan] + [(a-b)*1000 for a,b in zip(total_ifd[1:], total_ifd[:-1])] 155 | metrics["interframe_delay_ms"] = "mean" 156 | 157 | dfs.append(df) 158 | 159 | # for metric in metrics.keys(): 160 | # metrics[metric].extend(stats[metric]) 161 | 162 | gd_df = compute_delay.assemble_df(d) 163 | gdelay = compute_delay.compute_delay(gd_df) 164 | gdelays.append(gdelay) 165 | 166 | with open(os.path.join(d, "current_rtt.json"), "r") as f: 167 | rtt_stats = json.load(f) 168 | rtt_df = pd.DataFrame(rtt_stats["rtts"], index=pd.to_datetime(rtt_stats["ts"], unit="s")) 169 | rtts.append(rtt_df) 170 | 171 | df = pd.concat(dfs).sort_index() 172 | dfr = df.resample("60s") 173 | test_gap_mask = ~dfr.file.max().isna() 174 | dfr_mean = dfr.mean()[test_gap_mask] 175 | dfr_max = dfr.max()[test_gap_mask] 176 | 177 | df_diff = dfr_max - dfr_max.shift(1).fillna(0) 178 | # reset counters back to zero on file changes 179 | mask = dfr_max["file"] != dfr_max["file"].shift(1) 180 | df_diff[mask] = 0 181 | 182 | # frame_drop relative: (df_diff["frame_drop"] / (df_diff["frames_rendered"] + df_diff["frame_drop"])).describe() 183 | 184 | result = dict() 185 | for metric, how in metrics.items(): 186 | if how == "mean": 187 | result[metric] = compute_metrics(dfr_mean[metric]) 188 | if how == "diff": 189 | result[metric] = compute_metrics(df_diff[metric]) 190 | 191 | dfrtt = pd.concat(rtts).sort_index() 192 | dfrtt_mean = dfrtt.resample("60s").mean() 193 | result["rtt"] = compute_metrics(dfrtt_mean[0]) 194 | 195 | dfg = pd.concat(gdelays).sort_index() 196 | dfg_mean = dfg.resample("60s").mean() 197 | result["gaming_delay"] = compute_metrics(dfg_mean) 198 | 199 | freeze_occurences = df_diff[df_diff["total_freezes_duration_ms"] > 0]["total_freezes_duration_ms"] 200 | result["freeze_occurences"] = compute_metrics(freeze_occurences) 201 | 202 | drop_occurences = df_diff[df_diff["frame_drop"] > 0]["frame_drop"] 203 | result["frame_drop_occurences"] = compute_metrics(drop_occurences) 204 | 205 | # for k,v in result.items(): 206 | # print(f"{k}:{type(v)}") 207 | 208 | print(json.dumps(result, indent=4)) 209 | if args.save: 210 | with open(args.save, "w") as f: 211 | json.dump(result, f, indent=4) 212 | 213 | def main(): 214 | parser = argparse.ArgumentParser() 215 | parser.add_argument("directories", nargs="+", help="Result directories") 216 | parser.add_argument("--method", choices=["per_minute", "overall"], default="per_minute", help="How to group data points") 217 | parser.add_argument("--save", help="Save datapoints to csv file") 218 | args = parser.parse_args() 219 | 220 | if args.method == "per_minute": 221 | compute_per_minute(args) 222 | else: 223 | compute_overall(args) 224 | 225 | 226 | if __name__ == "__main__": 227 | main() 228 | -------------------------------------------------------------------------------- /mlab_figures/plot_utils.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import pandas as pd 3 | import matplotlib.pyplot as plt 4 | import os 5 | import numpy as np 6 | from descartes import PolygonPatch 7 | from unidecode import unidecode 8 | from matplotlib.patches import Patch 9 | cols = plt.get_cmap("tab10").colors 10 | 11 | def plot_cdf(df, cities, column, xlabel, 12 | ax=None, 13 | fig=None, 14 | show=True, 15 | figsize=(7, 4), 16 | savefig="", 17 | title="CDF", 18 | city_colors={}, 19 | lstyles={}, 20 | condition=True, 21 | skiplegend=False, 22 | figures_path="figures", xlim=None, legendsettings={}, legendloc="lower right", xticks=None, xticklabels=None, ylabel="CDF", stripylabels=False, labelmaprule={}): 23 | if ax==None: 24 | fig, ax = plt.subplots(figsize=figsize) 25 | for city in cities: 26 | # & (df["a_MinRTT"] < 600) 27 | xs = df[(df["client_Geo_City"] == city) & (condition)][column].values 28 | xs = sorted(xs) 29 | if len(xs) == 0: 30 | print(city) 31 | raise RuntimeError() 32 | ys = np.arange(1, len(xs) + 1) / len(xs) 33 | indices = [] 34 | current = xs[0] 35 | for i, x in enumerate(xs): # only take max y value at each x value to smoothen out the graph 36 | if x != current: 37 | current = x 38 | indices.append(i - 1) 39 | indices.append(len(ys) - 1) 40 | xs = sorted(set(xs)) 41 | ys = [ys[i] for i in indices] 42 | label = labelmaprule.get(city, city) 43 | ax.plot(xs, ys, label=label, color=city_colors[city], linestyle=lstyles[city]) 44 | 45 | if xlim is not None: 46 | ax.set_xlim(*xlim) 47 | if xticks is not None: 48 | ax.set_xticks(xticks) 49 | if xticklabels is not None: 50 | ax.set_xticks(xticklabels) 51 | ax.set_xlabel(xlabel) 52 | ax.set_ylabel(ylabel) 53 | if stripylabels: 54 | ax.set_yticks(np.arange(0, 1.25, 0.25), labels=[""]*len(np.arange(0, 1.25, 0.25))) 55 | else: 56 | ax.set_yticks(np.arange(0, 1.25, 0.25)) 57 | ax.xaxis.get_major_formatter()._usetex = False 58 | ax.yaxis.get_major_formatter()._usetex = False 59 | 60 | handles, labels = plt.gca().get_legend_handles_labels() 61 | sorted_hl = sorted(zip(handles, labels), key=(lambda t: t[1]), reverse=False) 62 | if not skiplegend: 63 | ax.legend([t[0] for t in sorted_hl], 64 | [t[1] for t in sorted_hl], loc=legendloc, fontsize="small", ncol=2, edgecolor="k", handlelength=1, labelspacing=0.06, 65 | columnspacing=0.5, handletextpad=0.3, fancybox=False) 66 | ax.grid(True, axis='y', linestyle='-', alpha=0.7, linewidth=0.5) 67 | 68 | ax.set_ylim((-0.05, 1.05)) 69 | 70 | #plt.title(title) 71 | if fig != None: 72 | fig.tight_layout() 73 | if(savefig != ""): 74 | if ".pdf" in savefig: 75 | svg_savefig = savefig.replace(".pdf", ".svg") 76 | plt.savefig(os.path.join(figures_path, svg_savefig), bbox_inches="tight", pad_inches=0) 77 | plt.savefig(os.path.join(figures_path, savefig), bbox_inches="tight", pad_inches=0) 78 | if show: 79 | plt.show() 80 | 81 | def plot_probe_map(city_overview_df, city_df, interesting_cities, 82 | lon_bounds = (-180, 180), 83 | lat_bounds = (-90, 90), 84 | figsize=(7, 6), 85 | annotate = True, 86 | annotate_minimal = False, 87 | savefig = "", 88 | figures_path="figures"): 89 | city_colors = {city: cols[i % len(cols)] for i, city in enumerate(interesting_cities)} 90 | lstyles = {city: lstyle_array[i % len(lstyle_array)] for i, city in enumerate(interesting_cities)} 91 | 92 | client_country_coords_df = city_overview_df.groupby(["ClientCountry", "ClientCity"])[["lat", "lon"]].mean() 93 | client_country_coords_df = client_country_coords_df[( 94 | client_country_coords_df["lat"] > lat_bounds[0]) & (client_country_coords_df["lat"] < lat_bounds[1] 95 | )] 96 | client_country_coords_df = client_country_coords_df[( 97 | client_country_coords_df["lon"] > lon_bounds[0]) & (client_country_coords_df["lon"] < lon_bounds[1] 98 | )] 99 | 100 | world = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres')) 101 | world = world[(world.name!="Antarctica")] 102 | 103 | def plotCountryPatch( axes, country_name, fcolor ): 104 | # plot a country on the provided axes 105 | nami = world[world.name == country_name] 106 | namigm = nami.__geo_interface__['features'] # geopandas's geo_interface 107 | namig0 = {'type': namigm[0]['geometry']['type'], \ 108 | 'coordinates': namigm[0]['geometry']['coordinates']} 109 | axes.add_patch(PolygonPatch( namig0, fc=fcolor, ec="black", alpha=0.2, zorder=2, linewidth=0.0 )) 110 | 111 | 112 | fig, ax = plt.subplots(figsize=figsize, subplot_kw={'projection': ccrs.PlateCarree()}) 113 | 114 | ax.set_xlim(lon_bounds) 115 | ax.set_ylim(lat_bounds) 116 | 117 | cmap = colors.ListedColormap(cols[:-2]) 118 | world.plot(facecolor = "grey", edgecolor="black", ax=ax, linewidth=0.2, alpha=0.2, zorder=0) 119 | 120 | for c in countries: 121 | c = "United States of America" if c == "United States" else c 122 | c = "Czechia" if c == "Czech Republic" else c 123 | 124 | if(c in world["name"].values): 125 | plotCountryPatch(ax, c, "#364aa088") 126 | 127 | client_country_df = city_overview_df.groupby(["ClientCountry", "ClientCity"])[["lat", "lon"]].mean() 128 | latitude, longitude = client_country_coords_df["lat"], client_country_coords_df["lon"] 129 | ax.scatter(longitude, latitude, 130 | sizes = [0.7], 131 | color="#a8290a88", 132 | zorder = 10) 133 | 134 | # Filtered with interesting_cities 135 | client_country_df = city_overview_df[city_overview_df["ClientCity"].isin(interesting_cities)].groupby(["ClientCountry", "ClientCity"])[["lat", "lon"]].mean() 136 | latitude, longitude = client_country_df["lat"], client_country_df["lon"] 137 | ax.scatter(longitude, latitude, 138 | sizes = [10.1], 139 | color="green", 140 | zorder = 9, marker="x") 141 | 142 | for x, y, label in zip(client_country_coords_df["lon"], client_country_coords_df["lat"], client_country_coords_df.index): 143 | if(annotate and ((not annotate_minimal) or (label[1] in interesting_cities))): 144 | ax.annotate(label[1], xy=(x, y), xytext=(-7, 5), textcoords="offset points") 145 | 146 | plt.tight_layout() 147 | if(savefig != ""): 148 | if ".pdf" in savefig: 149 | svg_savefig = savefig.replace(".pdf", ".svg") 150 | plt.savefig(os.path.join(figures_path, svg_savefig), bbox_inches="tight", pad_inches=0) 151 | plt.savefig(os.path.join(figures_path, savefig), bbox_inches="tight", pad_inches=0) 152 | plt.show() 153 | 154 | 155 | def plot_boxplot_progression(datetime_array, data_array, 156 | savefig = "", 157 | figsize=(7, 6), 158 | ylabel = "", 159 | figures_path="figures", 160 | nlegendcols=3, 161 | ylim=None, 162 | xlim=None, 163 | decimation=1, 164 | orientation="upper right"): 165 | """ 166 | Plot boxplot progression. As input you give a 167 | """ 168 | fig, ax = plt.subplots(figsize=figsize) 169 | 170 | colnames = data_array[0].keys() 171 | colcolors = [cols[i] for i, _ in enumerate(data_array[0])] 172 | def adjust_box(plot): 173 | for i, (cn, cc) in enumerate(zip(colnames, colcolors)): 174 | plt.setp(plot['boxes'][i], facecolor=cc, linewidth=1) 175 | plt.setp(plot['medians'][i], color='yellow') 176 | 177 | width = 0.5 178 | xspan = len(data_array[0].keys()) + 1 179 | 180 | for i, (dt, data_dict) in enumerate(zip(datetime_array, data_array)): 181 | positions = np.arange( 182 | i * xspan, 183 | i * xspan + len(data_dict.keys()) 184 | ) 185 | bp = ax.boxplot([data_dict[cn] for cn in colnames], positions=positions, 186 | widths=width, showfliers=False, patch_artist=True) 187 | adjust_box(bp) 188 | 189 | [ax.axvspan(i * xspan - 1, i * xspan + xspan - 1, facecolor="k", alpha=0.2) 190 | for i in range(len(datetime_array)) 191 | if i % 2 == 1] 192 | 193 | ax.set_xticks(np.arange( 194 | int(xspan / 2), 195 | 1 + xspan * len(datetime_array), 196 | xspan 197 | )) 198 | ax.set_xticklabels([dt.strftime("%Y/%m") for dt in datetime_array]) 199 | ax.xaxis.set_tick_params(rotation=15) 200 | ax.set_ylabel(ylabel) 201 | 202 | if xlim != None: 203 | if xlim[1] < 0: 204 | xlim_ = [xlim[0] * xspan - 1, (len(datetime_array) - 1) * xspan + xspan - 1] 205 | else: 206 | xlim_ = [xlim[0] * xspan - 1, (xlim[1] + xlim[0]-1) * xspan + xspan - 1] 207 | ax.set_xlim(xlim_) 208 | else: 209 | ax.set_xlim([-1, (len(datetime_array) - 1) * xspan + xspan - 1]) 210 | if ylim != None: 211 | ax.set_ylim(ylim) 212 | #ax.set_yticks(np.arange(0, 700, 100)) 213 | 214 | ax.xaxis.get_major_formatter()._usetex = False 215 | ax.yaxis.get_major_formatter()._usetex = False 216 | 217 | #handles = [Patch(facecolor=continent_colors[target1]), Patch(facecolor=continent_colors[target2]), 218 | # Patch(facecolor=continent_colors[cont])] 219 | handles = [Patch(facecolor=cols[i]) for i, _ in enumerate(data_array[0])] 220 | labels = colnames 221 | 222 | #[label.set_visible(False) for label in ax.xaxis.get_ticklabels()] 223 | #for label in ax.xaxis.get_ticklabels()[::decimation]: 224 | # label.set_visible(True) 225 | 226 | ax.legend(handles, labels, handlelength=1, labelspacing=0.06, columnspacing=0.5, handletextpad=0.3, 227 | loc=orientation, fancybox=False, edgecolor="k", fontsize="small", ncol=nlegendcols) 228 | plt.grid(True, axis='y', linestyle='--') 229 | 230 | fig.tight_layout() 231 | if(savefig != ""): 232 | if ".pdf" in savefig: 233 | svg_savefig = savefig.replace(".pdf", ".svg") 234 | plt.savefig(os.path.join(figures_path, svg_savefig), bbox_inches="tight", pad_inches=0) 235 | plt.savefig(os.path.join(figures_path, savefig), bbox_inches="tight", pad_inches=0) 236 | plt.show() 237 | -------------------------------------------------------------------------------- /zoom/boxplots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pandas as pd 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import matplotlib.patches as mpatches 7 | import os.path 8 | import argparse 9 | from matplotlib.cbook import boxplot_stats 10 | from matplotlib import rcParams 11 | from matplotlib.backends.backend_pdf import PdfPages 12 | import json 13 | import multiprocessing 14 | import sys 15 | import shutil 16 | 17 | import zoom_parser as zp 18 | 19 | def plot_external_legend(parts, figsize, **kwargs): # ncol = 20 | fig_legend = plt.figure(figsize=figsize) 21 | fig_legend.legend(parts, [p.get_label() for p in parts], **kwargs) 22 | fig_legend.tight_layout() 23 | return fig_legend 24 | 25 | def compute_bxp_metrics(paths): 26 | streams = [] 27 | packets = [] 28 | packets_local = [] 29 | packets_remote = [] 30 | frames_local = [] 31 | frames_remote = [] 32 | 33 | for base_path in paths: 34 | local = os.path.join(base_path, "local") 35 | remote = os.path.join(base_path, "remote") 36 | 37 | streams.append(zp.read_streams_csv(local)) 38 | 39 | df_packets_local = zp.read_packets_csv(local) 40 | df_packets_remote = zp.read_packets_csv(remote) 41 | df = zp.merge_packet_dfs(df_packets_local, df_packets_remote) 42 | packets.append(df) 43 | packets_local.append(df_packets_local) 44 | packets_remote.append(df_packets_remote) 45 | 46 | frames_local.append(zp.read_frames_csv(local)) 47 | frames_remote.append(zp.read_frames_csv(remote)) 48 | 49 | 50 | packets = pd.concat(packets) 51 | packets_local = pd.concat(packets_local) 52 | packets_remote = pd.concat(packets_remote) 53 | streams = pd.concat(streams) 54 | frames_local = pd.concat(frames_local) 55 | frames_remote = pd.concat(frames_remote) 56 | 57 | # packets 58 | owd_ul = [] 59 | owd_dl = [] 60 | 61 | tput_total_ul = [] 62 | tput_total_dl = [] 63 | tput_video_ul = [] 64 | tput_video_dl = [] 65 | tput_fec_ul = [] 66 | tput_fec_dl = [] 67 | 68 | gput_total_ul = [] 69 | gput_total_dl = [] 70 | gput_video_ul = [] 71 | gput_video_dl = [] 72 | gput_fec_ul = [] 73 | gput_fec_dl = [] 74 | 75 | # frames 76 | fps_ul = [] # packets sent from local and frames displayed on remote 77 | fps_dl = [] 78 | 79 | jitter_ul = [] # packets sent from local and frames displayed on remote, unit=ms 80 | jitter_dl = [] 81 | 82 | def comp_rate(packets_ssrc): # key = pl_len, pl_len_local, pl_len_remote 83 | return packets_ssrc.set_index("ts")["pl_len"].resample("1s").sum() * 8 / 1e6 84 | 85 | for stream in streams.itertuples(): 86 | packets_owd = abs(packets[packets.ssrc == stream.rtp_ssrc].owd) 87 | packets_local_ssrc = packets_local[packets_local.ssrc == stream.rtp_ssrc] 88 | packets_remote_ssrc = packets_remote[packets_remote.ssrc == stream.rtp_ssrc] 89 | frames_local_ssrc = frames_local[frames_local.ssrc == stream.rtp_ssrc] 90 | frames_remote_ssrc = frames_remote[frames_remote.ssrc == stream.rtp_ssrc] 91 | if not(stream.media_type == "v" and stream.stream_type == "m"): 92 | continue 93 | 94 | rate_local_total = comp_rate(packets_local_ssrc) 95 | rate_local_video = comp_rate(packets_local_ssrc[packets_local_ssrc.pt == 98]) 96 | rate_local_fec = comp_rate(packets_local_ssrc[packets_local_ssrc.pt == 110]) 97 | rate_remote_total = comp_rate(packets_remote_ssrc) 98 | rate_remote_video = comp_rate(packets_remote_ssrc[packets_remote_ssrc.pt == 98]) 99 | rate_remote_fec = comp_rate(packets_remote_ssrc[packets_remote_ssrc.pt == 110]) 100 | 101 | if stream.is_local: 102 | owd_ul.extend(packets_owd) 103 | fps_ul.extend(frames_remote_ssrc.fps) 104 | jitter_ul.extend(frames_remote_ssrc.jitter_ms) 105 | tput_total_ul.extend(rate_local_total) 106 | tput_video_ul.extend(rate_local_video) 107 | tput_fec_ul.extend(rate_local_fec) 108 | gput_total_ul.extend(rate_remote_total) 109 | gput_video_ul.extend(rate_remote_video) 110 | gput_fec_ul.extend(rate_remote_fec) 111 | else: 112 | owd_dl.extend(packets_owd) 113 | fps_dl.extend(frames_local_ssrc.fps) 114 | jitter_dl.extend(frames_local_ssrc.jitter_ms) 115 | tput_total_dl.extend(rate_remote_total) 116 | tput_video_dl.extend(rate_remote_video) 117 | tput_fec_dl.extend(rate_remote_fec) 118 | gput_total_dl.extend(rate_local_total) 119 | gput_video_dl.extend(rate_local_video) 120 | gput_fec_dl.extend(rate_local_fec) 121 | 122 | stats = [("owd_ms_ul", owd_ul), 123 | ("owd_ms_dl", owd_dl), 124 | 125 | ("tput_total_mbps_ul", tput_total_ul), 126 | ("tput_total_mbps_dl", tput_total_dl), 127 | ("tput_video_mbps_ul", tput_video_ul), 128 | ("tput_video_mbps_dl", tput_video_dl), 129 | ("tput_fec_mbps_ul", tput_fec_ul), 130 | ("tput_fec_mbps_dl", tput_fec_dl), 131 | 132 | ("gput_total_mbps_ul", gput_total_ul), 133 | ("gput_total_mbps_dl", gput_total_dl), 134 | ("gput_video_mbps_ul", gput_video_ul), 135 | ("gput_video_mbps_dl", gput_video_dl), 136 | ("gput_fec_mbps_ul", gput_fec_ul), 137 | ("gput_fec_mbps_dl", gput_fec_dl), 138 | 139 | ("fps_ul", fps_ul), 140 | ("fps_dl", fps_dl), 141 | ("jitter_ms_ul", jitter_ul), 142 | ("jitter_ms_dl", jitter_dl)] 143 | labels, metrics = zip(*stats) 144 | bxp_stats = boxplot_stats(metrics, labels=labels) 145 | 146 | for idx in range(len(bxp_stats)): 147 | label = bxp_stats[idx]["label"] 148 | data = [arr for (key, arr) in stats if key == label][0] 149 | bxp_stats[idx]["stddev"] = np.std(data) 150 | bxp_stats[idx]["min"] = np.min(data) if len(data) > 0 else np.nan 151 | bxp_stats[idx]["max"] = np.max(data) if len(data) > 0 else np.nan 152 | 153 | # Remove fliers 154 | for idx in range(len(bxp_stats)): 155 | bxp_stats[idx]["fliers"] = [] 156 | 157 | return bxp_stats 158 | 159 | # def replace_label(ty, label): 160 | # if "_ul" in label: 161 | # return f"{ty.upper()}\nUL" 162 | # if "_dl" in label: 163 | # return f"{ty.upper()}\nDL" 164 | # return label 165 | 166 | class NpEncoder(json.JSONEncoder): 167 | def default(self, obj): 168 | if isinstance(obj, np.integer): 169 | return int(obj) 170 | if isinstance(obj, np.floating): 171 | return float(obj) 172 | if isinstance(obj, np.ndarray): 173 | return obj.tolist() 174 | return super(NpEncoder, self).default(obj) 175 | 176 | def replace_label(stats): 177 | if "label" not in stats: 178 | return stats 179 | if "_ul" in stats["label"]: 180 | stats["label"] = "UL" 181 | elif "_dl" in stats["label"]: 182 | stats["label"] = "DL" 183 | return stats 184 | 185 | def main(): 186 | parser = argparse.ArgumentParser() 187 | # parser.add_argument("directories", nargs="+", help="Directories with zpkt and parsed csv files") 188 | parser.add_argument("--stl", nargs="+", help="Directories with 'local' and remote folders that contain local zpkt and parsed csv files") 189 | parser.add_argument("--ter", nargs="*", help="Directories with 'local' and remote folders that contain local zpkt and parsed csv files") 190 | parser.add_argument("--save", help="Write plot to this file") 191 | parser.add_argument("--csv", help="Write metrics to this path prefix") 192 | parser.add_argument("-b", action="store_true", help="Break after parsing csvs") 193 | args = parser.parse_args() 194 | 195 | if args.stl is None: 196 | parser.print_help() 197 | sys.exit(1) 198 | 199 | print(f"Parsing stl...") 200 | bxp_stl = compute_bxp_metrics(args.stl) 201 | print(f"Parsing ter...") 202 | bxp_ter = compute_bxp_metrics(args.ter) if args.ter else [] 203 | 204 | if args.csv: 205 | filepath, ext = os.path.splitext(args.csv) 206 | with open(filepath + "_stl" + ext, "w") as f: 207 | json.dump(bxp_stl, f, indent=2, cls=NpEncoder) 208 | with open(filepath + "_ter" + ext, "w") as f: 209 | json.dump(bxp_ter, f, indent=2, cls=NpEncoder) 210 | 211 | print(f"Plotting...") 212 | 213 | plt.set_cmap("tab10") 214 | rcParams["font.family"] = "CMU Sans Serif" 215 | rcParams["font.size"] = 9.0 216 | plt.rc('text', usetex=True if shutil.which('latex') else False) 217 | 218 | fig, axes = plt.subplots(figsize=(4, 1.5), ncols=3) 219 | 220 | # facecolor=cols[0], lw=1), medianprops=dict(color="yellow") 221 | colors = plt.get_cmap("tab10").colors 222 | 223 | stl_boxprops = dict(lw=1, facecolor=colors[0]) 224 | ter_boxprops = dict(lw=1, facecolor=colors[1]) 225 | medianprops = dict(color="yellow") 226 | stl_props = dict(boxprops=stl_boxprops, medianprops=medianprops, positions=[0.5,1.5], widths=0.4, patch_artist=True) 227 | ter_props = dict(boxprops=ter_boxprops, medianprops=medianprops, positions=[0.9,1.9], widths=0.4, patch_artist=True) 228 | 229 | stl_stats = [replace_label(s) for s in bxp_stl if s["label"].startswith("owd")] 230 | ter_stats = [replace_label(s) for s in bxp_ter if s["label"].startswith("owd")] 231 | axes[0].bxp(stl_stats, **stl_props) 232 | axes[0].bxp(ter_stats, **ter_props) 233 | axes[0].set_xticks([0.7, 1.7], ["UL", "DL"]) 234 | axes[0].set_ylabel("OWD (ms)") 235 | 236 | stl_stats = [replace_label(s) for s in bxp_stl if s["label"].startswith("tput_total_mbps")] 237 | ter_stats = [replace_label(s) for s in bxp_ter if s["label"].startswith("tput_total_mbps")] 238 | axes[1].bxp(stl_stats, **stl_props) 239 | axes[1].bxp(ter_stats, **ter_props) 240 | axes[1].set_xticks([0.7, 1.7], ["UL", "DL"]) 241 | axes[1].set_ylabel("Throughput (Mbps)") 242 | 243 | stl_stats = [replace_label(s) for s in bxp_stl if s["label"].startswith("fps")] 244 | ter_stats = [replace_label(s) for s in bxp_ter if s["label"].startswith("fps")] 245 | axes[2].bxp(stl_stats, **stl_props) 246 | axes[2].bxp(ter_stats, **ter_props) 247 | axes[2].set_xticks([0.7, 1.7], ["UL", "DL"]) 248 | axes[2].set_ylabel("FPS") 249 | 250 | # stl_stats = [replace_label(s) for s in bxp_stl if s["label"].startswith("jitter")] 251 | # ter_stats = [replace_label(s) for s in bxp_ter if s["label"].startswith("jitter")] 252 | # axes[3].bxp(stl_stats, **stl_props) 253 | # axes[3].bxp(ter_stats, **ter_props) 254 | # axes[3].set_xticks([0.7, 1.7], ["UL", "DL"]) 255 | # axes[3].set_ylabel("Frame Jitter (ms)") 256 | 257 | # axes[0].legend(ncol=2, handles=[ 258 | # mpatches.Patch(color="green", label='Starlink'), 259 | # mpatches.Patch(color="red", label='Terrestrial')]) 260 | 261 | 262 | parts = [ 263 | mpatches.Patch(color=colors[0], label='Starlink'), 264 | mpatches.Patch(color=colors[1], label='Terrestrial') 265 | ] 266 | fig_legend = plot_external_legend(parts, (4, 1), ncols=3) 267 | # labelspacing=0.3, handlelength=1.2, handletextpad=0.4, columnspacing=1.2) 268 | 269 | fig.tight_layout() 270 | fig.subplots_adjust(wspace=0.6) 271 | 272 | # if args.save: 273 | # plt.savefig(args.w, bbox_inches="tight", pad_inches=0) 274 | # else: 275 | # print("Continue to show plot...") 276 | # breakpoint() 277 | # plt.show() 278 | 279 | 280 | if args.save: 281 | filepath, ext = os.path.splitext(args.save) 282 | if ext == ".pdf": 283 | with PdfPages(args.save) as pdf: 284 | pdf.savefig(fig, bbox_inches="tight", pad_inches=0) 285 | pdf.savefig(fig_legend, bbox_inches="tight") 286 | # bbox_inches=Bbox([[0.1,0], [4,4]])) 287 | # 288 | else: 289 | fig.savefig(filepath + "_plot" + ext, bbox_inches="tight", pad_inches=0) 290 | fig_legend.savefig(filepath + "_legend" + ext, bbox_inches="tight") 291 | else: 292 | print("Continue to show plot...") 293 | breakpoint() 294 | plt.show() 295 | 296 | 297 | if __name__ == "__main__": 298 | main() 299 | 300 | # 301 | # media_type = (a)udio, (v)ideo, (s)creen 302 | # stream_type = (f)ec or (m)edia 303 | # 304 | # 305 | # packets.csv 306 | # flow_type = 307 | --------------------------------------------------------------------------------