├── unit_tests └── __init__.py ├── notebooks └── README.md ├── validator_tests ├── utils │ ├── __init__.py │ ├── constants.py │ ├── threshold_utils.py │ ├── weighted_spearman.py │ ├── create_main.py │ ├── plot_score_vs_epoch.py │ ├── plot_val_vs_acc.py │ └── derive.py ├── slurm_configs │ ├── a100.yaml │ └── mnist.yaml ├── flags │ ├── dlogits_accuracy.py │ ├── entropy.py │ ├── diversity.py │ ├── mcc.py │ ├── accuracy.py │ ├── snd.py │ ├── dev.py │ ├── __init__.py │ ├── svd.py │ ├── nearest_source.py │ ├── mmd.py │ ├── knn.py │ └── cluster.py ├── scripts │ ├── 0th_epoch.sh │ ├── mnist.sh │ ├── zip_dfs.sh │ ├── domainnet126.sh │ ├── office31.sh │ ├── officehome.sh │ ├── script_wrapper.sh │ ├── process_df.sh │ ├── scatter_plots.sh │ ├── eval_validators.sh │ ├── delete_DEV_folders.py │ ├── save_resilience_to_noise_dfs.sh │ └── run.py ├── zip_dfs.py ├── count_pkls.py ├── configs │ ├── mcc_config.py │ ├── entropy_config.py │ ├── diversity_config.py │ ├── __init__.py │ ├── snd_config.py │ ├── accuracy_config.py │ ├── d_logits_accuracy_config.py │ ├── dev_config.py │ ├── nearest_source_config.py │ ├── svd_config.py │ ├── base_config.py │ ├── mmd_config.py │ ├── knn_config.py │ └── cluster_config.py ├── delete_pkls.py ├── count_num_checkpoints.py ├── assemble_dataset.py ├── collect_dfs.py ├── plot_resilience_to_noise.py ├── where_nan.py ├── create_plots.py ├── 0th_epoch_gather.py ├── README.md ├── save_resilience_to_noise_dfs.py ├── plotly_test.py └── synthetic_correlation_example.py ├── powerful_benchmarker ├── utils │ ├── __init__.py │ ├── constants.py │ ├── logger.py │ ├── ignite_save_features.py │ ├── get_validator.py │ └── utils.py ├── yaml_configs │ ├── group_configs │ │ ├── fl3_adam_lr1.yaml │ │ ├── fl6_adam_lr1.yaml │ │ ├── office31.yaml │ │ ├── officehome.yaml │ │ ├── domainnet126.yaml │ │ ├── office31_amazon.yaml │ │ ├── default.yaml │ │ ├── 0_epochs.yaml │ │ ├── mnist.yaml │ │ └── trial_run.yaml │ ├── slurm_configs │ │ ├── a100.yaml │ │ └── mnist.yaml │ └── exp_configs │ │ ├── mnist │ │ ├── mnist_fl3_adam_lr1_0th_epoch.yaml │ │ ├── mnist_fl6_adam_lr1_0th_epoch.yaml │ │ ├── mnist_fl3_adam_lr1.yaml │ │ └── mnist_fl6_adam_lr1.yaml │ │ ├── office31 │ │ ├── office31_amazon_fl3_adam_lr1_0th_epoch.yaml │ │ ├── office31_amazon_fl6_adam_lr1_0th_epoch.yaml │ │ ├── office31_fl3_adam_lr1_0th_epoch.yaml │ │ ├── office31_fl6_adam_lr1_0th_epoch.yaml │ │ ├── office31_amazon_fl3_adam_lr1.yaml │ │ ├── office31_amazon_fl6_adam_lr1.yaml │ │ ├── office31_fl3_adam_lr1.yaml │ │ └── office31_fl6_adam_lr1.yaml │ │ ├── test │ │ └── test_jobs.yaml │ │ ├── domainnet126 │ │ ├── trial_run.yaml │ │ └── domainnet126_fl6_adam_lr1_0th_epoch.yaml │ │ └── officehome │ │ ├── officehome_fl3_adam_lr1_0th_epoch.yaml │ │ └── officehome_fl6_adam_lr1_0th_epoch.yaml ├── exp_scripts │ ├── domainnet126.sh │ ├── officehome_jobs.sh │ ├── stress_test.sh │ ├── office31_jobs.sh │ └── 0th_epoch.sh ├── scripts │ ├── process_checker.sh │ └── script_wrapper.sh ├── launch_multiple.py ├── delete_exp_groups.py ├── delete_experiment.py ├── configs │ ├── __init__.py │ ├── adda_config.py │ ├── vada_config.py │ ├── symnets_config.py │ ├── cdan_config.py │ ├── rtn_config.py │ ├── aligner_config.py │ ├── mcd_config.py │ ├── gvb_config.py │ └── gan_config.py └── yaml_creator_helper.py ├── git_assume_unchanged.sh ├── imgs └── Logo.png ├── constants.yaml ├── run_linter.sh ├── scripts ├── script_wrapper.sh ├── gdrive_upload.sh └── upload_logs.sh ├── format_code.sh ├── .flake8 ├── .gitignore ├── requirements.txt ├── kill_all.py ├── latex ├── replace_header_acronyms.py ├── correlation_bar_plot_single_adapter.py ├── scripts │ └── create_all.sh ├── replace_color_map_tags.py ├── correlation_diffs.py ├── correlation_bar_plot_adapter_validator_pairs.py ├── best_accuracy_per_adapter.py ├── best_validator_per_adapter_task.py ├── create_tables.py ├── correlation_bar_plot.py ├── README.md ├── correlation_single_adapter.py ├── color_map_tags.py ├── table_creator.py └── pred_acc_using_best_adapter_validator_pairs.py ├── upload_logs.py ├── delete_slurm_logs.py ├── simple_slurm.py └── print_progress.py /unit_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /validator_tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /powerful_benchmarker/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /git_assume_unchanged.sh: -------------------------------------------------------------------------------- 1 | git update-index --assume-unchanged constants.yaml -------------------------------------------------------------------------------- /imgs/Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinMusgrave/powerful-benchmarker/HEAD/imgs/Logo.png -------------------------------------------------------------------------------- /validator_tests/slurm_configs/a100.yaml: -------------------------------------------------------------------------------- 1 | wckey: "" 2 | cpus-per-task: 2 3 | partition: a100 4 | gpus-per-node: 1 -------------------------------------------------------------------------------- /validator_tests/slurm_configs/mnist.yaml: -------------------------------------------------------------------------------- 1 | wckey: "" 2 | cpus-per-task: 2 3 | gres: "gpu:1" 4 | time: "48:00:00" 5 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/fl3_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | feature_layer: 3 2 | optimizer: Adam 3 | lr_multiplier: 1 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/fl6_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | feature_layer: 6 2 | optimizer: Adam 3 | lr_multiplier: 1 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/slurm_configs/a100.yaml: -------------------------------------------------------------------------------- 1 | wckey: "" 2 | cpus-per-task: 5 3 | partition: a100 4 | gpus-per-node: 2 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/slurm_configs/mnist.yaml: -------------------------------------------------------------------------------- 1 | wckey: "" 2 | cpus-per-task: 2 3 | gres: "gpu:2" 4 | time: "48:00:00" 5 | -------------------------------------------------------------------------------- /validator_tests/flags/dlogits_accuracy.py: -------------------------------------------------------------------------------- 1 | def DLogitsAccuracy(): 2 | return [{"validator": "DLogitsAccuracy", "split": "train"}] 3 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/office31.yaml: -------------------------------------------------------------------------------- 1 | dataset: office31 2 | max_epochs: 2000 3 | val_interval: 100 4 | num_workers: 5 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/officehome.yaml: -------------------------------------------------------------------------------- 1 | dataset: officehome 2 | max_epochs: 200 3 | val_interval: 10 4 | num_workers: 5 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/domainnet126.yaml: -------------------------------------------------------------------------------- 1 | dataset: domainnet126 2 | max_epochs: 40 3 | val_interval: 2 4 | num_workers: 5 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/office31_amazon.yaml: -------------------------------------------------------------------------------- 1 | dataset: office31 2 | max_epochs: 200 3 | val_interval: 10 4 | num_workers: 5 -------------------------------------------------------------------------------- /constants.yaml: -------------------------------------------------------------------------------- 1 | exp_folder: null 2 | 3 | dataset_folder: null 4 | 5 | conda_env: null 6 | 7 | slurm_folder: null 8 | 9 | gdrive_folder: null 10 | -------------------------------------------------------------------------------- /validator_tests/scripts/0th_epoch.sh: -------------------------------------------------------------------------------- 1 | python validator_tests/scripts/run.py --exp_names epoch_0 --other_args "--exp_group_excludes oracle" --slurm_config mnist -------------------------------------------------------------------------------- /validator_tests/scripts/mnist.sh: -------------------------------------------------------------------------------- 1 | python validator_tests/scripts/run.py --other_args "--exp_group_prefix mnist --exp_group_excludes oracle" --slurm_config mnist -------------------------------------------------------------------------------- /validator_tests/scripts/zip_dfs.sh: -------------------------------------------------------------------------------- 1 | cd $1 2 | echo "finding all_dfs.pkl files" 3 | rm all_dfs.zip 4 | find -maxdepth 2 -name all_dfs.pkl | zip -qr all_dfs -@ -------------------------------------------------------------------------------- /powerful_benchmarker/exp_scripts/domainnet126.sh: -------------------------------------------------------------------------------- 1 | python powerful_benchmarker/launch_multiple.py --exp_config domainnet126/domainnet126_fl6_adam_lr1 --slurm_config a100 -------------------------------------------------------------------------------- /run_linter.sh: -------------------------------------------------------------------------------- 1 | flake8 latex powerful_benchmarker unit_tests validator_tests --count --show-source --statistics 2 | nbqa flake8 notebooks --count --show-source --statistics -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/default.yaml: -------------------------------------------------------------------------------- 1 | num_trials: 100 2 | n_startup_trials: 100 3 | batch_size: 64 4 | 5 | save_features: true 6 | use_full_inference: true -------------------------------------------------------------------------------- /validator_tests/scripts/domainnet126.sh: -------------------------------------------------------------------------------- 1 | python validator_tests/scripts/run.py --exp_per_slurm_job_mul $1 --other_args "--exp_group_prefix $2 --partition=learnai" --slurm_config a100 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/0_epochs.yaml: -------------------------------------------------------------------------------- 1 | num_trials: 1 2 | n_startup_trials: 1 3 | max_epochs: 1 4 | epoch_length: 1 5 | val_interval: 10 6 | check_initial_score: true -------------------------------------------------------------------------------- /scripts/script_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | conda_env=$1 4 | command=$2 5 | 6 | echo "Command to be wrapped: $command" 7 | 8 | conda deactivate && conda activate ${conda_env} 9 | 10 | ${command} -------------------------------------------------------------------------------- /validator_tests/scripts/office31.sh: -------------------------------------------------------------------------------- 1 | python validator_tests/scripts/run.py --exp_names im --validators Accuracy DEVBinary --other_args "--exp_group_prefix office31 --exp_group_excludes oracle $1" --slurm_config a100 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/mnist.yaml: -------------------------------------------------------------------------------- 1 | dataset: mnist 2 | src_domains: [mnist] 3 | target_domains: [mnistm] 4 | max_epochs: 100 5 | val_interval: 5 6 | num_workers: 2 7 | 8 | download_datasets: true -------------------------------------------------------------------------------- /validator_tests/scripts/officehome.sh: -------------------------------------------------------------------------------- 1 | python validator_tests/scripts/run.py --exp_names im --validators Accuracy DEVBinary --other_args "--exp_group_prefix officehome --exp_group_excludes oracle $1" --slurm_config a100 -------------------------------------------------------------------------------- /format_code.sh: -------------------------------------------------------------------------------- 1 | black latex powerful_benchmarker unit_tests validator_tests 2 | isort latex powerful_benchmarker unit_tests validator_tests --profile black 3 | nbqa black notebooks 4 | nbqa isort notebooks --profile black -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/group_configs/trial_run.yaml: -------------------------------------------------------------------------------- 1 | num_trials: 1 2 | n_startup_trials: 1 3 | max_epochs: 1 4 | batch_size: 4 5 | 6 | optimizer: DoNothing 7 | use_stat_getter: true 8 | check_initial_score: true -------------------------------------------------------------------------------- /validator_tests/flags/entropy.py: -------------------------------------------------------------------------------- 1 | def Entropy(): 2 | flags = [] 3 | for split in ["src_train", "src_val", "target_train"]: 4 | flags.append({"validator": "Entropy", "split": split}) 5 | return flags 6 | -------------------------------------------------------------------------------- /validator_tests/flags/diversity.py: -------------------------------------------------------------------------------- 1 | def Diversity(): 2 | flags = [] 3 | for split in ["src_train", "src_val", "target_train"]: 4 | flags.append({"validator": "Diversity", "split": split}) 5 | return flags 6 | -------------------------------------------------------------------------------- /validator_tests/flags/mcc.py: -------------------------------------------------------------------------------- 1 | def MCC(): 2 | flags = [] 3 | for split in ["src_train", "target_train", "src_val"]: 4 | for T in [0.1, 1, 10]: 5 | flags.append({"validator": "MCC", "split": split, "T": str(T)}) 6 | return flags 7 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | 3 | extend-ignore = 4 | E266 # too many leading '#' for block comment 5 | E203 # whitespace before ':' 6 | E402 # module level import not at top of file 7 | E501 # line too long 8 | 9 | per-file-ignores = 10 | __init__.py:F401 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.py[cod] 3 | .nfs* 4 | build/ 5 | dist/ 6 | *.egg-info/ 7 | site/ 8 | progress.txt 9 | notebooks/*.pkl 10 | .ipynb_checkpoints 11 | powerful_benchmarker/local_scripts/ 12 | plots/ 13 | tables/ 14 | tables_latex/ 15 | tables_subsets/ -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/mnist/mnist_fl3_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 mnist 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | commands: 5 | - - python powerful_benchmarker/launch_one.py 6 | - *A 7 | - *common -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/mnist/mnist_fl6_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 mnist 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | commands: 5 | - - python powerful_benchmarker/launch_one.py 6 | - *A 7 | - *common -------------------------------------------------------------------------------- /powerful_benchmarker/exp_scripts/officehome_jobs.sh: -------------------------------------------------------------------------------- 1 | python powerful_benchmarker/launch_multiple.py --exp_config officehome/officehome_fl3_adam_lr1 --slurm_config a100 2 | python powerful_benchmarker/launch_multiple.py --exp_config officehome/officehome_fl6_adam_lr1 --slurm_config a100 -------------------------------------------------------------------------------- /validator_tests/flags/accuracy.py: -------------------------------------------------------------------------------- 1 | def Accuracy(): 2 | flags = [] 3 | for average in ["micro", "macro"]: 4 | for split in ["src_train", "src_val", "target_train", "target_val"]: 5 | flags.append({"validator": "Accuracy", "average": average, "split": split}) 6 | return flags 7 | -------------------------------------------------------------------------------- /scripts/gdrive_upload.sh: -------------------------------------------------------------------------------- 1 | echo "Deleting existing files from $2" 2 | gdrive list -q "'$2' in parents" --no-header --max 0 | cut -d" " -f1 - | xargs -L 1 gdrive delete 3 | echo "Starting upload" 4 | for w in $1 5 | do 6 | if [ -f $w ]; then 7 | echo "Uploading $w" 8 | gdrive upload --parent "$2" $w 9 | fi 10 | done 11 | -------------------------------------------------------------------------------- /validator_tests/scripts/script_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | conda_env=$1 4 | devices=$2 5 | command=$3 6 | 7 | echo "Command to be wrapped: $command" 8 | 9 | export CUDA_VISIBLE_DEVICES=${devices} 10 | 11 | echo "CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES" 12 | 13 | conda deactivate && conda activate ${conda_env} 14 | 15 | ${command} -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | black 2 | flake8 3 | h5py==3.6.0 4 | isort 5 | nbqa 6 | optuna 7 | pytorch-adapt[ignite,record-keeper,timm,detection]==0.0.81 8 | PyYAML 9 | seaborn 10 | submitit 11 | tensorboard 12 | tqdm 13 | git+https://github.com/matthijsz/weightedcorr.git 14 | 15 | --find-links https://download.pytorch.org/whl/torch_stable.html 16 | torch==1.10.1+cu111 17 | torchvision==0.11.2+cu111 -------------------------------------------------------------------------------- /validator_tests/scripts/process_df.sh: -------------------------------------------------------------------------------- 1 | # python validator_tests/process_df.py --exp_group_prefix mnist 2 | # python validator_tests/process_df.py --exp_group_prefix office31 3 | # python validator_tests/process_df.py --exp_group_prefix officehome 4 | 5 | python simple_slurm.py --command "python validator_tests/process_df.py" --slurm_config_folder validator_tests --slurm_config mnist --job_name=process_df --cpus-per-task=2 --exp_group_excludes oracle -------------------------------------------------------------------------------- /validator_tests/flags/snd.py: -------------------------------------------------------------------------------- 1 | def SND(): 2 | flags = [] 3 | for layer in ["features", "logits", "preds"]: 4 | for T in [0.01, 0.05, 0.1, 0.5]: 5 | flags.append( 6 | { 7 | "validator": "SND", 8 | "split": "target_train", 9 | "layer": layer, 10 | "T": str(T), 11 | } 12 | ) 13 | return flags 14 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_amazon_fl3_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 office31_amazon 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | commands: 5 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 6 | - *A 7 | - *common 8 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 9 | - *A 10 | - *common 11 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_amazon_fl6_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 office31_amazon 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | commands: 5 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 6 | - *A 7 | - *common 8 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 9 | - *A 10 | - *common 11 | 12 | -------------------------------------------------------------------------------- /validator_tests/flags/dev.py: -------------------------------------------------------------------------------- 1 | def DEV(): 2 | flags = [] 3 | for layer in ["features", "logits", "preds"]: 4 | for normalization in ["None", "max", "standardize"]: 5 | flags.append( 6 | {"validator": "DEV", "layer": layer, "normalization": normalization} 7 | ) 8 | return flags 9 | 10 | 11 | def DEVBinary(): 12 | flags = DEV() 13 | for f in flags: 14 | f["validator"] = "DEVBinary" 15 | return flags 16 | -------------------------------------------------------------------------------- /powerful_benchmarker/exp_scripts/stress_test.sh: -------------------------------------------------------------------------------- 1 | python powerful_benchmarker/launch_multiple.py --exp_config test/stress_test --slurm_config a100 --partition=hipri --gpus-per-node=2 --is_stress_test 2 | python powerful_benchmarker/launch_multiple.py --exp_config test/stress_test --slurm_config a100 --partition=lowpri --gpus-per-node=2 --is_stress_test 3 | python powerful_benchmarker/launch_multiple.py --exp_config test/stress_test --slurm_config a100 --partition=lowpri --gpus-per-node=2 --is_stress_test -------------------------------------------------------------------------------- /powerful_benchmarker/exp_scripts/office31_jobs.sh: -------------------------------------------------------------------------------- 1 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_fl3_adam_lr1 --slurm_config a100 2 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_amazon_fl3_adam_lr1 --slurm_config a100 3 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_fl6_adam_lr1 --slurm_config a100 4 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_amazon_fl6_adam_lr1 --slurm_config a100 -------------------------------------------------------------------------------- /powerful_benchmarker/scripts/process_checker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | input_folder=$1 4 | timeout=$2 5 | latest_dir=$(ls -td ${input_folder}/*/ | head -1) 6 | latest_update_time=$(find ${latest_dir} -printf "%T@\n" | sort | tail -1 | cut -f1 -d".") 7 | current_seconds=$(date +%s) 8 | time_since_latest_update=$((current_seconds - latest_update_time)) 9 | # minutes_since_latest_update=$((time_since_latest_update / 60)) 10 | if (( time_since_latest_update > ${timeout} )); 11 | then 12 | echo 0; 13 | else 14 | echo 1; 15 | fi; -------------------------------------------------------------------------------- /validator_tests/zip_dfs.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import subprocess 3 | import sys 4 | 5 | sys.path.insert(0, ".") 6 | from powerful_benchmarker.utils.constants import add_default_args 7 | 8 | 9 | def main(cfg): 10 | command = "bash -i ./validator_tests/scripts/zip_dfs.sh {0}".format(cfg.exp_folder) 11 | subprocess.run(command.split(" ")) 12 | 13 | 14 | if __name__ == "__main__": 15 | parser = argparse.ArgumentParser(allow_abbrev=False) 16 | add_default_args(parser, ["exp_folder"]) 17 | args = parser.parse_args() 18 | main(args) 19 | -------------------------------------------------------------------------------- /validator_tests/count_pkls.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | 5 | sys.path.insert(0, ".") 6 | from powerful_benchmarker.utils.constants import add_default_args 7 | from validator_tests.utils import utils 8 | 9 | if __name__ == "__main__": 10 | parser = argparse.ArgumentParser(allow_abbrev=False) 11 | add_default_args(parser, ["exp_folder"]) 12 | parser.add_argument("--exp_group", type=str, required=True) 13 | parser.add_argument("--validator", type=str, default="") 14 | args = parser.parse_args() 15 | utils.count_pkls(os.path.join(args.exp_folder, args.exp_group), args.validator) 16 | -------------------------------------------------------------------------------- /validator_tests/configs/mcc_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.layers import MCCLoss 2 | 3 | from .base_config import BaseConfig, get_split_and_layer 4 | 5 | 6 | class MCC(BaseConfig): 7 | def __init__(self, config): 8 | super().__init__(config) 9 | self.validator_args["T"] = float(self.validator_args["T"]) 10 | self.validator = MCCLoss(T=self.validator_args["T"]) 11 | 12 | def score(self, x, exp_config, device): 13 | logits = get_split_and_layer(x, self.split, "logits", device) 14 | return -self.validator(logits).item() 15 | 16 | def expected_keys(self): 17 | return {"split", "T"} 18 | -------------------------------------------------------------------------------- /validator_tests/flags/__init__.py: -------------------------------------------------------------------------------- 1 | from .accuracy import Accuracy 2 | from .cluster import ( 3 | ClassAMI, 4 | ClassAMICentroidInit, 5 | ClassSS, 6 | ClassSSCentroidInit, 7 | DomainCluster, 8 | ) 9 | from .dev import DEV, DEVBinary 10 | from .diversity import Diversity 11 | from .dlogits_accuracy import DLogitsAccuracy 12 | from .entropy import Entropy 13 | from .knn import KNN, TargetKNN, TargetKNNLogits 14 | from .mcc import MCC 15 | from .mmd import MMD, MMDFixedB, MMDPerClass, MMDPerClassFixedB 16 | from .nearest_source import NearestSource, NearestSourceL2 17 | from .snd import SND 18 | from .svd import BNM, BSP, FBNM 19 | -------------------------------------------------------------------------------- /validator_tests/configs/entropy_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.validators import EntropyValidator 2 | 3 | from .base_config import BaseConfig, get_from_hdf5 4 | 5 | 6 | class Entropy(BaseConfig): 7 | def __init__(self, config): 8 | super().__init__(config) 9 | self.validator = EntropyValidator( 10 | key_map={self.split: "target_train"}, 11 | ) 12 | 13 | def score(self, x, exp_config, device): 14 | logits = get_from_hdf5(x, device, f"inference/{self.split}/logits") 15 | return self.validator(**{self.split: {"logits": logits}}) 16 | 17 | def expected_keys(self): 18 | return {"split"} 19 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/test/test_jobs.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 officehome 2 | A: &A --config_names dann mcc --gpus-per-node=4 3 | B: &B --config_names atdoc bsp --gpus-per-node=6 4 | C: &C --config_names cdan im --gpus-per-node=2 5 | 6 | 7 | commands: 8 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains clipart 9 | - *A 10 | - *common 11 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains clipart 12 | - *B 13 | - *common 14 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains clipart 15 | - *C 16 | - *common -------------------------------------------------------------------------------- /validator_tests/configs/diversity_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.validators import DiversityValidator 2 | 3 | from .base_config import BaseConfig, get_from_hdf5 4 | 5 | 6 | class Diversity(BaseConfig): 7 | def __init__(self, config): 8 | super().__init__(config) 9 | self.validator = DiversityValidator( 10 | key_map={self.split: "target_train"}, 11 | ) 12 | 13 | def score(self, x, exp_config, device): 14 | logits = get_from_hdf5(x, device, f"inference/{self.split}/logits") 15 | return self.validator(**{self.split: {"logits": logits}}) 16 | 17 | def expected_keys(self): 18 | return {"split"} 19 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_fl3_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 office31 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | 5 | commands: 6 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 7 | - *A 8 | - *common 9 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 10 | - *A 11 | - *common 12 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 13 | - *A 14 | - *common 15 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 16 | - *A 17 | - *common 18 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_fl6_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 office31 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | 5 | commands: 6 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 7 | - *A 8 | - *common 9 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 10 | - *A 11 | - *common 12 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 13 | - *A 14 | - *common 15 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 16 | - *A 17 | - *common 18 | 19 | -------------------------------------------------------------------------------- /validator_tests/flags/svd.py: -------------------------------------------------------------------------------- 1 | def BNM(): 2 | flags = [] 3 | for split in ["src_train", "target_train", "src_val"]: 4 | flags.append({"validator": "BNM", "layer": "logits", "split": split}) 5 | return flags 6 | 7 | 8 | def BSP(): 9 | flags = [] 10 | for layer in ["features", "logits", "preds"]: 11 | for split in ["src_train", "target_train"]: 12 | for k in [1]: 13 | flags.append( 14 | {"validator": "BSP", "layer": layer, "split": split, "k": str(k)} 15 | ) 16 | return flags 17 | 18 | 19 | def FBNM(): 20 | flags = BNM() 21 | for f in flags: 22 | f["validator"] = "FBNM" 23 | return flags 24 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/mnist/mnist_fl3_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 mnist 2 | A: &A --config_names dann mcc 3 | B: &B --config_names atdoc bsp 4 | C: &C --config_names cdan im 5 | D: &D --config_names bnm gvb 6 | E: &E --config_names mcd mmd 7 | 8 | 9 | commands: 10 | - - python powerful_benchmarker/launch_one.py 11 | - *A 12 | - *common 13 | - - python powerful_benchmarker/launch_one.py 14 | - *B 15 | - *common 16 | - - python powerful_benchmarker/launch_one.py 17 | - *C 18 | - *common 19 | - - python powerful_benchmarker/launch_one.py 20 | - *D 21 | - *common 22 | - - python powerful_benchmarker/launch_one.py 23 | - *E 24 | - *common -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/mnist/mnist_fl6_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 mnist 2 | A: &A --config_names dann mcc 3 | B: &B --config_names atdoc bsp 4 | C: &C --config_names cdan im 5 | D: &D --config_names bnm gvb 6 | E: &E --config_names mcd mmd 7 | 8 | 9 | commands: 10 | - - python powerful_benchmarker/launch_one.py 11 | - *A 12 | - *common 13 | - - python powerful_benchmarker/launch_one.py 14 | - *B 15 | - *common 16 | - - python powerful_benchmarker/launch_one.py 17 | - *C 18 | - *common 19 | - - python powerful_benchmarker/launch_one.py 20 | - *D 21 | - *common 22 | - - python powerful_benchmarker/launch_one.py 23 | - *E 24 | - *common -------------------------------------------------------------------------------- /validator_tests/configs/__init__.py: -------------------------------------------------------------------------------- 1 | from .accuracy_config import Accuracy 2 | from .cluster_config import ( 3 | ClassAMI, 4 | ClassAMICentroidInit, 5 | ClassSS, 6 | ClassSSCentroidInit, 7 | DomainCluster, 8 | ) 9 | from .d_logits_accuracy_config import DLogitsAccuracy 10 | from .dev_config import DEV, DEVBinary 11 | from .diversity_config import Diversity 12 | from .entropy_config import Entropy 13 | from .knn_config import KNN, TargetKNN, TargetKNNLogits 14 | from .mcc_config import MCC 15 | from .mmd_config import MMD, MMDFixedB, MMDPerClass, MMDPerClassFixedB 16 | from .nearest_source_config import NearestSource, NearestSourceL2 17 | from .snd_config import SND 18 | from .svd_config import BNM, BSP, FBNM 19 | -------------------------------------------------------------------------------- /powerful_benchmarker/utils/constants.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | TRIALS_FILENAME = "trials.csv" 4 | BEST_TRIAL_FILENAME = "best_trial.json" 5 | JOBIDS_FILENAME = "all_jobids.json" 6 | 7 | 8 | def get_user_constants(constants_path): 9 | with open(constants_path, "r") as f: 10 | return yaml.safe_load(f) 11 | 12 | 13 | def add_default_args(parser, args_to_add, constants_path="constants.yaml"): 14 | constants = get_user_constants(constants_path) 15 | for x in args_to_add: 16 | k, v = x, x 17 | if not isinstance(x, str): 18 | # then it's a sequence of strings 19 | k, v = x 20 | parser.add_argument( 21 | f"--{v}", 22 | type=str, 23 | default=constants[k], 24 | ) 25 | -------------------------------------------------------------------------------- /kill_all.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from powerful_benchmarker.utils.constants import JOBIDS_FILENAME, add_default_args 4 | from powerful_benchmarker.utils.utils import kill_all_jobs 5 | from validator_tests.utils.constants import JOBIDS_FILENAME as V_JOBSID_FILENAME 6 | 7 | if __name__ == "__main__": 8 | parser = argparse.ArgumentParser(allow_abbrev=False) 9 | add_default_args(parser, ["exp_folder"]) 10 | parser.add_argument("--validator_tests", action="store_true") 11 | parser.add_argument("--old_format", action="store_true") 12 | args = parser.parse_args() 13 | filename = V_JOBSID_FILENAME if args.validator_tests else JOBIDS_FILENAME 14 | if args.old_format: 15 | filename = filename.replace(".json", ".txt") 16 | kill_all_jobs(args.exp_folder, filename) 17 | -------------------------------------------------------------------------------- /latex/replace_header_acronyms.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import os 4 | 5 | 6 | def main(args): 7 | matches = glob.glob(os.path.join(args.input_folder, "**", "*.tex")) 8 | for m in matches: 9 | print(m) 10 | with open(m, "r") as f: 11 | newText = f.read().replace( 12 | "DCP & DCR & DCS & DPC & DPR & DPS & DRC & DRP & DRS & DSC & DSP & DSR", 13 | "CP & CR & CS & PC & PR & PS & RC & RP & RS & SC & SP & SR", 14 | ) 15 | with open(m, "w") as f: 16 | f.write(newText) 17 | 18 | 19 | if __name__ == "__main__": 20 | parser = argparse.ArgumentParser(allow_abbrev=False) 21 | parser.add_argument("--input_folder", type=str, default="tables_latex") 22 | args = parser.parse_args() 23 | main(args) 24 | -------------------------------------------------------------------------------- /validator_tests/flags/nearest_source.py: -------------------------------------------------------------------------------- 1 | def NearestSource(): 2 | flags = [] 3 | for layer in ["logits"]: 4 | for threshold in [-2]: 5 | for weighted in [0, 1]: 6 | flags.append( 7 | { 8 | "validator": "NearestSource", 9 | "layer": layer, 10 | "threshold": threshold, 11 | "weighted": weighted, 12 | } 13 | ) 14 | return flags 15 | 16 | 17 | def NearestSourceL2(): 18 | flags = [] 19 | for layer in ["features", "logits", "preds"]: 20 | flags.append( 21 | { 22 | "validator": "NearestSourceL2", 23 | "layer": layer, 24 | } 25 | ) 26 | return flags 27 | -------------------------------------------------------------------------------- /validator_tests/scripts/scatter_plots.sh: -------------------------------------------------------------------------------- 1 | python validator_tests/create_plots.py --exp_group_prefix mnist --scatter --scatter_no_color --run_single --run_combined 2 | python validator_tests/create_plots.py --exp_group_prefix mnist --scatter --scatter_no_color --run_single --run_combined --scatter_per_adapter 3 | python validator_tests/create_plots.py --exp_group_prefix office31 --scatter --scatter_no_color --run_single --run_combined 4 | python validator_tests/create_plots.py --exp_group_prefix office31 --scatter --scatter_no_color --run_single --run_combined --scatter_per_adapter 5 | python validator_tests/create_plots.py --exp_group_prefix officehome --scatter --scatter_no_color --run_single --run_combined 6 | python validator_tests/create_plots.py --exp_group_prefix officehome --scatter --scatter_no_color --run_single --run_combined --scatter_per_adapter -------------------------------------------------------------------------------- /validator_tests/configs/snd_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.validators import SNDValidator 2 | 3 | from .base_config import BaseConfig, get_split_and_layer 4 | 5 | 6 | class SND(BaseConfig): 7 | def __init__(self, config): 8 | super().__init__(config) 9 | self.validator_args["T"] = float(self.validator_args["T"]) 10 | self.layer = self.validator_args["layer"] 11 | self.validator = SNDValidator( 12 | key_map={self.split: "target_train"}, 13 | layer=self.layer, 14 | T=self.validator_args["T"], 15 | ) 16 | 17 | def score(self, x, exp_config, device): 18 | features = get_split_and_layer(x, self.split, self.layer, device) 19 | return self.validator(**{self.split: {self.layer: features}}) 20 | 21 | def expected_keys(self): 22 | return {"T", "layer", "split"} 23 | -------------------------------------------------------------------------------- /latex/correlation_bar_plot_single_adapter.py: -------------------------------------------------------------------------------- 1 | from latex.correlation import base_filename, get_preprocess_df 2 | from latex.correlation_bar_plot import reshape_and_plot 3 | from latex.correlation_single_adapter import get_postprocess_df 4 | from latex.table_creator import table_creator 5 | 6 | 7 | def correlation_bar_plot_single_adapter(args, name, src_threshold): 8 | basename = base_filename(name, True, src_threshold) 9 | 10 | dfs, output_folder = table_creator( 11 | args, 12 | args.input_folder, 13 | args.output_folder, 14 | basename, 15 | preprocess_df=get_preprocess_df(per_adapter=True), 16 | postprocess_df=get_postprocess_df(remove_index_names=False), 17 | do_save_to_latex=False, 18 | ) 19 | 20 | for adapter, df in dfs.items(): 21 | reshape_and_plot(df, output_folder, f"{basename}_{adapter}", name) 22 | -------------------------------------------------------------------------------- /latex/scripts/create_all.sh: -------------------------------------------------------------------------------- 1 | python latex/create_tables.py --exp_group_excludes domainnet126 --exp_group_excludes_select_best mnist --nlargest 5 2 | python latex/create_tables.py --exp_group_prefix domainnet126 --exp_group_excludes_select_best mnist --nlargest 5 3 | python latex/create_tables.py --exp_group_prefix mnist --exp_group_excludes_select_best mnist --nlargest 5 4 | python latex/create_tables.py --exp_group_prefix office --exp_group_excludes_select_best mnist --nlargest 5 5 | python latex/create_tables.py --exp_group_prefix office31 --exp_group_excludes_select_best mnist --nlargest 5 6 | python latex/create_tables.py --exp_group_prefix officehome --exp_group_excludes_select_best mnist --nlargest 5 7 | python latex/create_tables.py --exp_group_excludes mnist --exp_group_excludes_select_best mnist --nlargest 5 8 | python latex/replace_color_map_tags.py 9 | python latex/replace_header_acronyms.py -------------------------------------------------------------------------------- /validator_tests/utils/constants.py: -------------------------------------------------------------------------------- 1 | JOBIDS_FILENAME = "all_validator_jobids.json" 2 | VALIDATOR_TESTS_FOLDER = "validator_tests" 3 | ALL_DFS_FILENAME = "all_dfs.pkl" 4 | PROCESSED_DF_FILENAME = "all_dfs_processed.pkl" 5 | TARGET_ACCURACY = "target_train_micro" 6 | TARGET_VAL_ACCURACY = "target_val_micro" 7 | NUM_ADAPTERS = 10 8 | EXPECTED_NUMBER_OF_CHECKPOINTS = NUM_ADAPTERS * 100 * 20 9 | 10 | 11 | def exp_group_args(): 12 | return [ 13 | "exp_groups", 14 | "exp_group_prefix", 15 | "exp_group_suffix", 16 | "exp_group_includes", 17 | "exp_group_excludes", 18 | ] 19 | 20 | 21 | def add_exp_group_args(parser, suffix=""): 22 | parser.add_argument(f"--exp_groups{suffix}", nargs="+", type=str, default=[]) 23 | x = exp_group_args() 24 | x.remove("exp_groups") 25 | for k in x: 26 | parser.add_argument(f"--{k}{suffix}", type=str) 27 | -------------------------------------------------------------------------------- /powerful_benchmarker/launch_multiple.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import subprocess 3 | import sys 4 | 5 | import yaml 6 | 7 | sys.path.insert(0, ".") 8 | from powerful_benchmarker.utils.utils import get_yaml_config_path 9 | 10 | 11 | def main(cfg, other_args): 12 | exp_config_file = get_yaml_config_path("exp_configs", cfg.exp_config) 13 | 14 | with open(exp_config_file, "r") as f: 15 | commands = yaml.safe_load(f)["commands"] 16 | 17 | for i, cs in enumerate(commands): 18 | c = " ".join(cs) 19 | c = f"{c} {other_args}" 20 | print(f"launching {c}") 21 | subprocess.run(c.split(" ")) 22 | 23 | 24 | if __name__ == "__main__": 25 | parser = argparse.ArgumentParser(allow_abbrev=False) 26 | parser.add_argument("--exp_config", type=str, required=True) 27 | args, unknown_args = parser.parse_known_args() 28 | other_args = " ".join(unknown_args) 29 | main(args, other_args) 30 | -------------------------------------------------------------------------------- /validator_tests/flags/mmd.py: -------------------------------------------------------------------------------- 1 | def MMD(): 2 | flags = [] 3 | for layer in ["features", "logits", "preds"]: 4 | for exponent in [0]: 5 | for normalize in [0, 1]: 6 | flags.append( 7 | { 8 | "validator": "MMD", 9 | "split": "train", 10 | "layer": layer, 11 | "exponent": str(exponent), 12 | "normalize": str(normalize), 13 | } 14 | ) 15 | return flags 16 | 17 | 18 | def MMDPerClass(): 19 | flags = MMD() 20 | for f in flags: 21 | f["validator"] = "MMDPerClass" 22 | return flags 23 | 24 | 25 | def MMDFixedB(): 26 | flags = MMD() 27 | for f in flags: 28 | f["validator"] = "MMDFixedB" 29 | return flags 30 | 31 | 32 | def MMDPerClassFixedB(): 33 | flags = MMD() 34 | for f in flags: 35 | f["validator"] = "MMDPerClassFixedB" 36 | return flags 37 | -------------------------------------------------------------------------------- /upload_logs.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | sys.path.insert(0, ".") 7 | from powerful_benchmarker.utils.constants import JOBIDS_FILENAME, add_default_args 8 | from validator_tests.utils.constants import JOBIDS_FILENAME as V_JOBSID_FILENAME 9 | 10 | 11 | def main(cfg): 12 | curr_dir = os.getcwd() 13 | 14 | command = "bash -i ./scripts/upload_logs.sh {0} {1} {2} {3} {4}".format( 15 | cfg.exp_folder, 16 | cfg.gdrive_folder, 17 | cfg.sleep_time, 18 | curr_dir, 19 | cfg.conda_env, 20 | ) 21 | command = command.split(" ") 22 | command.append(f"{JOBIDS_FILENAME} {V_JOBSID_FILENAME}") 23 | subprocess.run(command) 24 | 25 | 26 | if __name__ == "__main__": 27 | parser = argparse.ArgumentParser(allow_abbrev=False) 28 | add_default_args(parser, ["exp_folder", "conda_env", "gdrive_folder"]) 29 | parser.add_argument("--sleep_time", type=str, default="120m") 30 | args = parser.parse_args() 31 | main(args) 32 | -------------------------------------------------------------------------------- /latex/replace_color_map_tags.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import os 4 | import sys 5 | 6 | sys.path.insert(0, ".") 7 | from latex.utils import num_to_word 8 | 9 | 10 | def main(args): 11 | matches = glob.glob( 12 | os.path.join( 13 | args.input_folder, "**", "best_accuracy_per_adapter_ranked_by_score*" 14 | ) 15 | ) 16 | for m in matches: 17 | nlargest = os.path.splitext(m)[0].split("_")[-1] 18 | with open(m, "r") as f: 19 | num = num_to_word(int(nlargest)) 20 | newText = f.read().replace( 21 | f"bestaccuracyperadapterrankedbyscore{num}", 22 | f"bestaccuracyperadapter{num}", 23 | ) 24 | 25 | with open(m, "w") as f: 26 | f.write(newText) 27 | 28 | 29 | if __name__ == "__main__": 30 | parser = argparse.ArgumentParser(allow_abbrev=False) 31 | parser.add_argument("--input_folder", type=str, default="tables_latex") 32 | args = parser.parse_args() 33 | main(args) 34 | -------------------------------------------------------------------------------- /validator_tests/delete_pkls.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | 5 | sys.path.insert(0, ".") 6 | 7 | from powerful_benchmarker.utils.constants import add_default_args 8 | from validator_tests.utils import utils 9 | from validator_tests.utils.constants import add_exp_group_args 10 | 11 | 12 | def delete_fn(pkls): 13 | for pkl in pkls: 14 | print("deleting", pkl) 15 | os.remove(pkl) 16 | 17 | 18 | def main(cfg): 19 | fn = delete_fn if cfg.delete else None 20 | exp_groups = utils.get_exp_groups(cfg) 21 | 22 | for exp_group in exp_groups: 23 | utils.count_pkls(os.path.join(cfg.exp_folder, exp_group), cfg.validator, fn) 24 | 25 | 26 | if __name__ == "__main__": 27 | parser = argparse.ArgumentParser(allow_abbrev=False) 28 | add_default_args(parser, ["exp_folder"]) 29 | add_exp_group_args(parser) 30 | parser.add_argument("--validator", type=str, default="") 31 | parser.add_argument("--delete", action="store_true") 32 | args = parser.parse_args() 33 | main(args) 34 | -------------------------------------------------------------------------------- /validator_tests/utils/threshold_utils.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.models import pretrained_src_accuracy 2 | 3 | from . import df_utils 4 | 5 | 6 | def filter_by_acc(df, min_acc, domain_type, filter_action): 7 | if domain_type == "src": 8 | split = "val" 9 | elif domain_type == "target": 10 | split = "train" 11 | 12 | keep_mask = df[f"{domain_type}_{split}_micro"] >= min_acc 13 | if filter_action == "remove": 14 | return df[keep_mask] 15 | elif filter_action == "set_to_nan": 16 | df = df.copy() 17 | df.loc[~keep_mask, "score"] = float("nan") 18 | return df 19 | else: 20 | raise ValueError("incorrect filter_action") 21 | 22 | 23 | def filter_by_src_threshold(df, src_threshold, filter_action): 24 | dataset = df_utils.get_sorted_unique(df, "dataset", assert_one=True)[0] 25 | src_domains = df_utils.get_sorted_unique(df, "src_domains", assert_one=True)[0] 26 | min_acc = pretrained_src_accuracy(dataset, src_domains, "val", "micro") 27 | return filter_by_acc(df, min_acc * src_threshold, "src", filter_action) 28 | -------------------------------------------------------------------------------- /scripts/upload_logs.sh: -------------------------------------------------------------------------------- 1 | conda deactivate && conda activate "$5" 2 | while : 3 | do 4 | cd "$4" 5 | echo "Getting progress" 6 | python print_progress.py --exp_folder "$1" --save_to_file progress.txt --with_validator_progress 7 | progress_fail=$? 8 | if [ "$progress_fail" = "1" ]; 9 | then 10 | echo "fail!" 11 | else 12 | echo "Moving progress.txt" 13 | mv progress.txt "$1" 14 | cd "$1" 15 | echo "Removing existing zip" 16 | rm all_logs.zip csvs.zip 17 | echo "Zipping files" 18 | find -maxdepth 4 -name *.err -o -name *.out | zip -qr all_logs -@ 19 | find -maxdepth 4 -name trials.csv | zip -qr csvs -@ 20 | zip -qu csvs.zip progress.txt 21 | for w in $6 22 | do 23 | if [ -f $w ]; then 24 | zip -qu csvs.zip $w 25 | fi 26 | done 27 | $4/scripts/gdrive_upload.sh "all_logs.zip csvs.zip all_dfs.zip" $2 28 | fi 29 | echo "Sleeping for $3" 30 | sleep "$3" 31 | done -------------------------------------------------------------------------------- /validator_tests/count_num_checkpoints.py: -------------------------------------------------------------------------------- 1 | # number of checkpoints can be calculated manually 2 | # but some checkpoints may have been corrupted somehow etc. 3 | # So here we calculate the actual number of checkpoints used when evaluating the validators 4 | import argparse 5 | import glob 6 | import sys 7 | 8 | import pandas as pd 9 | 10 | sys.path.insert(0, ".") 11 | 12 | from powerful_benchmarker.utils.constants import add_default_args 13 | from validator_tests.utils.constants import PROCESSED_DF_FILENAME 14 | 15 | 16 | def main(exp_folder): 17 | processed = glob.glob(f"{exp_folder}/**/{PROCESSED_DF_FILENAME}") 18 | num_checkpoints = 0 19 | for p in processed: 20 | df = pd.read_pickle(p) 21 | df = df[["epoch", "trial_num", "adapter"]].drop_duplicates() 22 | num_checkpoints += len(df) 23 | print(p, len(df)) 24 | print(f"num_checkpoints = {num_checkpoints}") 25 | 26 | 27 | if __name__ == "__main__": 28 | parser = argparse.ArgumentParser(allow_abbrev=False) 29 | add_default_args(parser, ["exp_folder"]) 30 | args = parser.parse_args() 31 | main(args.exp_folder) 32 | -------------------------------------------------------------------------------- /powerful_benchmarker/delete_exp_groups.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import shutil 4 | import sys 5 | from pprint import pprint 6 | 7 | sys.path.insert(0, ".") 8 | 9 | from powerful_benchmarker.utils.constants import add_default_args 10 | from validator_tests.utils import utils 11 | from validator_tests.utils.constants import add_exp_group_args 12 | 13 | 14 | def main(cfg): 15 | exp_groups = utils.get_exp_groups(cfg) 16 | folders_to_delete = [] 17 | for e in exp_groups: 18 | assert len(e) > 0 19 | full_path = os.path.join(cfg.exp_folder, e) 20 | if cfg.delete: 21 | print("deleting", full_path) 22 | shutil.rmtree(full_path) 23 | else: 24 | folders_to_delete.append(full_path) 25 | 26 | if not cfg.delete: 27 | print("Preview of folders to delete") 28 | pprint(folders_to_delete) 29 | 30 | 31 | if __name__ == "__main__": 32 | parser = argparse.ArgumentParser(allow_abbrev=False) 33 | add_default_args(parser, ["exp_folder"]) 34 | add_exp_group_args(parser) 35 | parser.add_argument("--delete", action="store_true") 36 | args = parser.parse_args() 37 | main(args) 38 | -------------------------------------------------------------------------------- /latex/correlation_diffs.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from latex.correlation import base_filename, get_postprocess_df, get_preprocess_df 4 | from latex.table_creator import table_creator 5 | 6 | 7 | def correlation_diffs(args, per_adapter, names, src_threshold): 8 | dfs = [] 9 | for name in names: 10 | basename = base_filename(name, per_adapter, src_threshold) 11 | 12 | df, output_folder = table_creator( 13 | args, 14 | args.input_folder, 15 | args.output_folder, 16 | basename, 17 | preprocess_df=get_preprocess_df(per_adapter), 18 | postprocess_df=get_postprocess_df(per_adapter), 19 | do_save_to_latex=False, 20 | ) 21 | 22 | df = ( 23 | df.drop(columns=["Mean", "Std"]) 24 | .reset_index() 25 | .melt(id_vars=["level_0", "level_1"], var_name="task", value_name=name) 26 | ) 27 | 28 | dfs.append(df) 29 | 30 | df = dfs[0].merge(dfs[1], on=["level_0", "level_1", "task"]) 31 | df["diff"] = df["weighted_spearman"] - df["spearman"] 32 | df = df.sort_values("diff") 33 | df.to_csv(os.path.join(output_folder, "correlation_diffs.csv")) 34 | -------------------------------------------------------------------------------- /powerful_benchmarker/scripts/script_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exp_name=$1 4 | timeout=$2 5 | exp_folder=$3 6 | full_path="$exp_folder/$exp_name/" 7 | conda_env=$4 8 | devices=$5 9 | best_trial_filename=$6 10 | command=$7 11 | 12 | echo "Command to be wrapped: $command" 13 | 14 | export CUDA_VISIBLE_DEVICES=${devices} 15 | 16 | echo "CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES" 17 | 18 | conda deactivate && conda activate ${conda_env} 19 | 20 | best_trial_full_path="$full_path$best_trial_filename" 21 | 22 | echo "Will run until this file is made: $best_trial_full_path" 23 | 24 | while [ ! -f $best_trial_full_path ] 25 | do 26 | echo "STARTING $script_name" 27 | echo "Checking folder: $full_path" 28 | echo "Will kill script if folder has not updated in the past $timeout seconds" 29 | ${command} & 30 | curr_pid=$! 31 | is_running=1 32 | echo "curr_pid is $curr_pid" 33 | trap "kill -9 $curr_pid & exit" INT 34 | sleep 5m 35 | 36 | while ((is_running == 1)) 37 | do 38 | sleep 1m 39 | if [ -d "$full_path" ]; then 40 | is_running=$(bash ./powerful_benchmarker/scripts/process_checker.sh ${full_path} ${timeout}) 41 | fi 42 | done 43 | pkill -9 -P ${curr_pid} 44 | sleep 10s 45 | done -------------------------------------------------------------------------------- /delete_slurm_logs.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import os 4 | import sys 5 | 6 | sys.path.insert(0, ".") 7 | 8 | from powerful_benchmarker.utils.constants import add_default_args 9 | from validator_tests.utils import utils 10 | from validator_tests.utils.constants import add_exp_group_args 11 | 12 | 13 | def main(cfg): 14 | if not cfg.slurm_folder: 15 | raise ValueError 16 | exp_groups = utils.get_exp_groups(cfg) 17 | num_files = 0 18 | for e in exp_groups: 19 | slurm_folder = os.path.join(cfg.exp_folder, e, cfg.slurm_folder) 20 | if not os.path.isdir(slurm_folder): 21 | continue 22 | files = glob.glob(os.path.join(slurm_folder, "*")) 23 | num_files += len(files) 24 | if cfg.delete: 25 | for f in files: 26 | print("deleting", f) 27 | os.remove(f) 28 | print("num files =", num_files) 29 | 30 | 31 | if __name__ == "__main__": 32 | parser = argparse.ArgumentParser(allow_abbrev=False) 33 | add_default_args(parser, ["exp_folder", "slurm_folder"]) 34 | add_exp_group_args(parser) 35 | parser.add_argument("--delete", action="store_true") 36 | args = parser.parse_args() 37 | main(args) 38 | -------------------------------------------------------------------------------- /validator_tests/scripts/eval_validators.sh: -------------------------------------------------------------------------------- 1 | # exp_prefixes=("mnist_mnist_mnistm" "office31_amazon_dslr" "office31_amazon_webcam" "office31_dslr_amazon" \ 2 | # "office31_dslr_webcam" "office31_webcam_amazon" "office31_webcam_dslr" \ 3 | # "officehome_art_clipart" "officehome_art_product" "officehome_art_real" \ 4 | # "officehome_clipart_art" "officehome_clipart_product" "officehome_clipart_real" \ 5 | # "officehome_product_art" "officehome_product_clipart" "officehome_product_real" \ 6 | # "officehome_real_art" "officehome_real_clipart" "officehome_real_product") 7 | 8 | 9 | # for i in "${exp_prefixes[@]}" 10 | # do 11 | # python simple_slurm.py --command "python validator_tests/eval_validators.py --run_combined" --all_in_one --exp_group_prefix $i --exp_group_excludes oracle \ 12 | # --slurm_config_folder validator_tests --slurm_config mnist --job_name=eval_validators --cpus-per-task=4 13 | # done 14 | 15 | python validator_tests/eval_validators.py --exp_group_prefix mnist --run_combined 16 | python validator_tests/eval_validators.py --exp_group_prefix office31 --run_combined 17 | python validator_tests/eval_validators.py --exp_group_prefix officehome --run_combined 18 | python validator_tests/eval_validators.py --exp_group_prefix domainnet126 --run_single -------------------------------------------------------------------------------- /validator_tests/configs/accuracy_config.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | from pytorch_adapt.datasets import utils as dataset_utils 3 | from pytorch_adapt.validators import AccuracyValidator 4 | 5 | from .base_config import BaseConfig, get_from_hdf5 6 | 7 | 8 | class Accuracy(BaseConfig): 9 | def __init__(self, config): 10 | super().__init__(config) 11 | self.validator = AccuracyValidator 12 | 13 | def actual_init(self, exp_config): 14 | self.validator = self.validator( 15 | key_map={self.split: "src_val"}, 16 | torchmetric_kwargs={ 17 | "average": self.validator_args["average"], 18 | "num_classes": dataset_utils.num_classes(exp_config["dataset"]), 19 | }, 20 | ) 21 | 22 | def score(self, x, exp_config, device): 23 | if self.validator is AccuracyValidator: 24 | self.actual_init(exp_config) 25 | logits = get_from_hdf5(x, device, f"inference/{self.split}/logits") 26 | labels = get_from_hdf5(x, device, f"inference/{self.split}/labels") 27 | preds = F.softmax(logits, dim=1) 28 | return self.validator(**{self.split: {"preds": preds, "labels": labels}}) 29 | 30 | def expected_keys(self): 31 | return {"average", "split"} 32 | -------------------------------------------------------------------------------- /powerful_benchmarker/delete_experiment.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import os 4 | import shutil 5 | import sys 6 | 7 | sys.path.insert(0, ".") 8 | 9 | from powerful_benchmarker.utils.constants import add_default_args 10 | from validator_tests.utils import utils 11 | from validator_tests.utils.constants import add_exp_group_args 12 | 13 | 14 | def delete_experiment(cfg, exp_group): 15 | experiment_paths = sorted(glob.glob(f"{cfg.exp_folder}/{exp_group}/*")) 16 | num_folders = 0 17 | for p in experiment_paths: 18 | if not os.path.isdir(p): 19 | continue 20 | if os.path.basename(p) == cfg.adapter: 21 | num_folders += 1 22 | if cfg.delete: 23 | print("deleting", p) 24 | shutil.rmtree(p) 25 | print("num_folders", num_folders) 26 | 27 | 28 | def main(cfg): 29 | exp_groups = utils.get_exp_groups(cfg) 30 | for e in exp_groups: 31 | delete_experiment(cfg, e) 32 | 33 | 34 | if __name__ == "__main__": 35 | parser = argparse.ArgumentParser(allow_abbrev=False) 36 | add_default_args(parser, ["exp_folder"]) 37 | add_exp_group_args(parser) 38 | parser.add_argument("--adapter", type=str, default="") 39 | parser.add_argument("--delete", action="store_true") 40 | args = parser.parse_args() 41 | main(args) 42 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/__init__.py: -------------------------------------------------------------------------------- 1 | from .adda_config import ADDAConfig 2 | from .aligner_config import AlignerConfig, CORALConfig, JMMDConfig, MMDConfig 3 | from .base_config import BaseConfig 4 | from .cdan_config import CDANConfig, CDANEConfig, CDANEUConfig 5 | from .dann_config import ( 6 | AFNDANNConfig, 7 | ATDOCDANNConfig, 8 | BNMDANNConfig, 9 | BSPDANNConfig, 10 | CDANNEUConfig, 11 | DANNConfig, 12 | DANNEConfig, 13 | DANNEUConfig, 14 | DANNFL8Config, 15 | DANNIMConfig, 16 | DANNMinEntConfig, 17 | MCCDANNConfig, 18 | ) 19 | from .gan_config import ( 20 | DomainConfusionConfig, 21 | GANConfig, 22 | GANEConfig, 23 | GANEUConfig, 24 | GANFL8Config, 25 | GANIMConfig, 26 | GANMinEntConfig, 27 | ) 28 | from .gvb_config import GVBConfig, GVBEConfig, GVBEUConfig 29 | from .mcd_config import MCDConfig, STARConfig, SWDConfig 30 | from .pretrainer_config import ( 31 | AFNConfig, 32 | ATDOCConfig, 33 | BNMConfig, 34 | BSPConfig, 35 | FinetunerConfig, 36 | FinetunerMultiLabelConfig, 37 | IMConfig, 38 | ITLConfig, 39 | MCCConfig, 40 | MinEntConfig, 41 | PretrainerConfig, 42 | PretrainerMultiLabelConfig, 43 | ) 44 | from .rtn_config import RTNConfig 45 | from .symnets_config import SymNetsConfig 46 | from .vada_config import VADAConfig 47 | -------------------------------------------------------------------------------- /powerful_benchmarker/utils/logger.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.frameworks.ignite import IgniteValHookWrapper 2 | from pytorch_adapt.frameworks.ignite.loggers import ( 3 | BasicLossLogger, 4 | IgniteRecordKeeperLogger, 5 | ) 6 | from pytorch_adapt.utils import common_functions as c_f 7 | 8 | 9 | class Logger: 10 | def __init__(self, folder, record_keeper_freq=50): 11 | self.logger1 = BasicLossLogger() 12 | self.logger2 = IgniteRecordKeeperLogger(folder=folder) 13 | self.record_keeper_freq = record_keeper_freq 14 | 15 | def add_training(self, adapter): 16 | fn1 = self.logger1.add_training(adapter) 17 | fn2 = self.logger2.add_training(adapter) 18 | 19 | def fn(engine): 20 | fn1(engine) 21 | if engine.state.iteration % self.record_keeper_freq == 0: 22 | fn2(engine) 23 | 24 | return fn 25 | 26 | def add_validation(self, *args, **kwargs): 27 | self.logger2.add_validation(*args, **kwargs) 28 | 29 | def write(self, *args, **kwargs): 30 | self.logger2.write(*args, **kwargs) 31 | 32 | def get_losses(self): 33 | return self.logger1.get_losses() 34 | 35 | 36 | class IgniteValHookWrapperWithPrint(IgniteValHookWrapper): 37 | def __call__(self, *args, **kwargs): 38 | super().__call__(*args, **kwargs) 39 | c_f.LOGGER.info(self.validator) 40 | -------------------------------------------------------------------------------- /validator_tests/configs/d_logits_accuracy_config.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchmetrics.functional import accuracy 3 | 4 | from .base_config import ( 5 | BaseConfig, 6 | get_from_hdf5, 7 | get_full_split_name, 8 | get_src_domain, 9 | get_target_domain, 10 | ) 11 | 12 | 13 | class DLogitsAccuracy(BaseConfig): 14 | def score(self, x, exp_config, device): 15 | src_split = get_full_split_name("src", self.split) 16 | target_split = get_full_split_name("target", self.split) 17 | 18 | src_logits = get_from_hdf5(x, device, f"inference/{src_split}/d_logits") 19 | target_logits = get_from_hdf5(x, device, f"inference/{target_split}/d_logits") 20 | 21 | src_preds = torch.sigmoid(src_logits) 22 | target_preds = torch.sigmoid(target_logits) 23 | src_labels = get_src_domain(len(src_preds), device) 24 | target_labels = get_target_domain(len(target_preds), device) 25 | 26 | preds = torch.cat([src_preds, target_preds], dim=0) 27 | labels = torch.cat([src_labels, target_labels], dim=0).to( 28 | dtype=torch.long, device=preds.device 29 | ) 30 | 31 | return accuracy( 32 | preds, 33 | labels, 34 | average="macro", 35 | num_classes=2, 36 | multiclass=True, 37 | ).item() 38 | 39 | def expected_keys(self): 40 | return {"split"} 41 | -------------------------------------------------------------------------------- /validator_tests/flags/knn.py: -------------------------------------------------------------------------------- 1 | def KNN(): 2 | flags = [] 3 | for layer in ["features", "logits", "preds"]: 4 | for k in [1000]: 5 | for p in [2]: 6 | for normalize in [0, 1]: 7 | flags.append( 8 | { 9 | "validator": "KNN", 10 | "split": "train", 11 | "layer": layer, 12 | "k": str(k), 13 | "p": str(p), 14 | "normalize": str(normalize), 15 | } 16 | ) 17 | return flags 18 | 19 | 20 | def TargetKNN(): 21 | flags = [] 22 | for k in [1000]: 23 | for p in [2]: 24 | for normalize in [0, 1]: 25 | for T_in_ref in [0, 1]: 26 | flags.append( 27 | { 28 | "validator": "TargetKNN", 29 | "split": "train", 30 | "k": str(k), 31 | "p": str(p), 32 | "normalize": str(normalize), 33 | "T_in_ref": str(T_in_ref), 34 | } 35 | ) 36 | return flags 37 | 38 | 39 | def TargetKNNLogits(): 40 | flags = TargetKNN() 41 | for f in flags: 42 | f["validator"] = "TargetKNNLogits" 43 | return flags 44 | -------------------------------------------------------------------------------- /validator_tests/scripts/delete_DEV_folders.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import os 4 | import shutil 5 | import sys 6 | 7 | sys.path.insert(0, ".") 8 | from powerful_benchmarker.utils.constants import add_default_args 9 | from validator_tests.utils.constants import VALIDATOR_TESTS_FOLDER, add_exp_group_args 10 | from validator_tests.utils.utils import get_exp_groups 11 | 12 | 13 | def main(args): 14 | exp_groups = get_exp_groups(args) 15 | num_folders = 0 16 | for e in exp_groups: 17 | for exp_name in args.exp_names: 18 | folder = os.path.join(args.exp_folder, e, exp_name) 19 | dev_folders = glob.glob( 20 | os.path.join(folder, "*", VALIDATOR_TESTS_FOLDER, "DEV*") 21 | ) 22 | for dev_folder in dev_folders: 23 | if os.path.isdir(dev_folder) and not dev_folder.endswith(".pkl"): 24 | num_folders += 1 25 | if args.delete: 26 | print("deleting", dev_folder) 27 | shutil.rmtree(dev_folder) 28 | print("num folders =", num_folders) 29 | 30 | 31 | if __name__ == "__main__": 32 | parser = argparse.ArgumentParser(allow_abbrev=False) 33 | add_default_args(parser, ["exp_folder"]) 34 | add_exp_group_args(parser) 35 | parser.add_argument("--exp_names", nargs="+", type=str, required=True) 36 | parser.add_argument("--delete", action="store_true") 37 | args = parser.parse_args() 38 | main(args) 39 | -------------------------------------------------------------------------------- /validator_tests/utils/weighted_spearman.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from scipy.stats import rankdata, spearmanr 4 | from WeightedCorr import WeightedCorr 5 | 6 | 7 | def set_nan_inf_to_min(x): 8 | x = x.copy() 9 | is_finite = np.isfinite(x) 10 | x[~is_finite] = np.min(x[is_finite]) 11 | return x 12 | 13 | 14 | def assert_all_finite(validation_scores, target_accuracies): 15 | validation_scores = set_nan_inf_to_min(validation_scores) 16 | assert np.isfinite(target_accuracies).all() 17 | assert np.isfinite(validation_scores).all() 18 | return validation_scores, target_accuracies 19 | 20 | 21 | def weighted_spearman(validation_scores, target_accuracies, pow): 22 | validation_scores, target_accuracies = assert_all_finite( 23 | validation_scores, target_accuracies 24 | ) 25 | v_ranks = rankdata(validation_scores, method="dense").astype(float) 26 | v_ranks /= np.max(v_ranks) 27 | t_ranks = rankdata(target_accuracies, method="dense").astype(float) 28 | t_ranks /= np.max(t_ranks) 29 | weights = np.maximum(v_ranks, t_ranks) ** pow 30 | 31 | return WeightedCorr( 32 | x=pd.Series(validation_scores), 33 | y=pd.Series(target_accuracies), 34 | w=pd.Series(weights), 35 | )(method="spearman") 36 | 37 | 38 | def spearman(validation_scores, target_accuracies): 39 | validation_scores, target_accuracies = assert_all_finite( 40 | validation_scores, target_accuracies 41 | ) 42 | return spearmanr(validation_scores, target_accuracies).correlation 43 | -------------------------------------------------------------------------------- /validator_tests/utils/create_main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import pandas as pd 5 | 6 | sys.path.insert(0, ".") 7 | from validator_tests.utils import utils 8 | from validator_tests.utils.df_utils import ( 9 | get_exp_groups_with_matching_tasks, 10 | get_processed_df, 11 | ) 12 | 13 | 14 | def main(args, fn1, fn2): 15 | exp_groups = utils.get_exp_groups(args) 16 | output_folder = getattr(args, "output_folder", None) 17 | if args.run_single: 18 | for e in exp_groups: 19 | print("exp_group", e) 20 | # do per feature layer 21 | exp_folder = os.path.join(args.exp_folder, e) 22 | df = get_processed_df(exp_folder) 23 | if df is not None: 24 | fn1(output_folder, df) 25 | 26 | if args.run_combined: 27 | # now do with feature layers in one dataframe 28 | # which are saved in args.exp_folder 29 | print("finding combined dfs") 30 | combined_exp_groups = get_exp_groups_with_matching_tasks( 31 | args.exp_folder, exp_groups 32 | ) 33 | for e_group in combined_exp_groups: 34 | print("exp_groups", e_group) 35 | df = [] 36 | for e in e_group: 37 | df.append(get_processed_df(os.path.join(args.exp_folder, e))) 38 | df = pd.concat(df, axis=0, ignore_index=True) 39 | fn2(output_folder, df) 40 | 41 | 42 | def add_main_args(parser): 43 | parser.add_argument("--run_single", action="store_true") 44 | parser.add_argument("--run_combined", action="store_true") 45 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_amazon_fl3_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 office31_amazon 2 | A: &A --config_names dann mcc 3 | B: &B --config_names atdoc bsp 4 | C: &C --config_names cdan im 5 | D: &D --config_names bnm gvb 6 | E: &E --config_names mcd mmd 7 | 8 | 9 | commands: 10 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 11 | - *A 12 | - *common 13 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 14 | - *A 15 | - *common 16 | 17 | 18 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 19 | - *B 20 | - *common 21 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 22 | - *B 23 | - *common 24 | 25 | 26 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 27 | - *C 28 | - *common 29 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 30 | - *C 31 | - *common 32 | 33 | 34 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 35 | - *D 36 | - *common 37 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 38 | - *D 39 | - *common 40 | 41 | 42 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 43 | - *E 44 | - *common 45 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 46 | - *E 47 | - *common 48 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_amazon_fl6_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 office31_amazon 2 | A: &A --config_names dann mcc 3 | B: &B --config_names atdoc bsp 4 | C: &C --config_names cdan im 5 | D: &D --config_names bnm gvb 6 | E: &E --config_names mcd mmd 7 | 8 | 9 | commands: 10 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 11 | - *A 12 | - *common 13 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 14 | - *A 15 | - *common 16 | 17 | 18 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 19 | - *B 20 | - *common 21 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 22 | - *B 23 | - *common 24 | 25 | 26 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 27 | - *C 28 | - *common 29 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 30 | - *C 31 | - *common 32 | 33 | 34 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 35 | - *D 36 | - *common 37 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 38 | - *D 39 | - *common 40 | 41 | 42 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains amazon 43 | - *E 44 | - *common 45 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains amazon 46 | - *E 47 | - *common 48 | -------------------------------------------------------------------------------- /validator_tests/assemble_dataset.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import sys 4 | 5 | sys.path.insert(0, ".") 6 | 7 | import argparse 8 | 9 | from powerful_benchmarker.utils.constants import add_default_args 10 | from validator_tests.utils import utils 11 | from validator_tests.utils.constants import add_exp_group_args 12 | 13 | 14 | def assemble_fn(epoch, x, exp_config, exp_folder): 15 | # TODO: save data to a new hdf5 file 16 | pass 17 | 18 | 19 | # Full dataset is too large. 20 | # Keep every 20th trial. 21 | def condition(iteration, *_): 22 | if iteration % 20 == 0: 23 | return True 24 | return False 25 | 26 | 27 | def assemble(args, exp_group): 28 | exp_folder = os.path.join(args.exp_folder, exp_group) 29 | exp_names = [ 30 | os.path.basename(x) 31 | for x in glob.glob(os.path.join(exp_folder, "*")) 32 | if os.path.isdir(x) 33 | ] 34 | if args.slurm_folder in exp_names: 35 | exp_names.remove(args.slurm_folder) 36 | for e in exp_names: 37 | exp_folders = utils.get_exp_folders(exp_folder, e) 38 | utils.apply_to_data(exp_folders, condition, fn=assemble_fn) 39 | 40 | 41 | def main(args): 42 | exp_groups = utils.get_exp_groups(args) 43 | for e in exp_groups: 44 | assemble(args, e) 45 | 46 | 47 | if __name__ == "__main__": 48 | parser = argparse.ArgumentParser(allow_abbrev=False) 49 | add_default_args(parser, ["exp_folder", "slurm_folder"]) 50 | add_exp_group_args(parser) 51 | parser.add_argument("--output_name", type=str, required=True) 52 | args = parser.parse_args() 53 | main(args) 54 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/domainnet126/trial_run.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs fl6_adam_lr1 domainnet126 trial_run 2 | A: &A --config_names trial_run 3 | 4 | 5 | commands: 6 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains painting 7 | - *A 8 | - *common 9 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains real 10 | - *A 11 | - *common 12 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains sketch 13 | - *A 14 | - *common 15 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains clipart 16 | - *A 17 | - *common 18 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains real 19 | - *A 20 | - *common 21 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains sketch 22 | - *A 23 | - *common 24 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains clipart 25 | - *A 26 | - *common 27 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains painting 28 | - *A 29 | - *common 30 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains sketch 31 | - *A 32 | - *common 33 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains clipart 34 | - *A 35 | - *common 36 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains painting 37 | - *A 38 | - *common 39 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains real 40 | - *A 41 | - *common -------------------------------------------------------------------------------- /latex/correlation_bar_plot_adapter_validator_pairs.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from latex.correlation import base_filename, get_preprocess_df 4 | from latex.correlation_bar_plot import reshape_and_plot 5 | from latex.correlation_single_adapter import get_postprocess_df 6 | from latex.table_creator import table_creator 7 | 8 | 9 | def postprocess_df_wrapper(df): 10 | dfs = get_postprocess_df(remove_index_names=False)(df) 11 | for k, df in dfs.items(): 12 | dfs[k] = pd.concat([df], keys=[k], names=["adapter"]) 13 | return pd.concat(list(dfs.values()), axis=0) 14 | 15 | 16 | def new_col_fn(df): 17 | new_col = df.apply( 18 | lambda x: f'{x["adapter"]} / {x["validator"]}: {x["validator_args"]}', axis=1 19 | ) 20 | return df.assign(validator_as_str=new_col).drop( 21 | columns=["adapter", "validator", "validator_args"] 22 | ) 23 | 24 | 25 | def correlation_bar_plot_adapter_validator_pairs(args, name, src_threshold): 26 | basename = base_filename(name, True, src_threshold) 27 | 28 | df, output_folder = table_creator( 29 | args, 30 | args.input_folder, 31 | args.output_folder, 32 | basename, 33 | preprocess_df=get_preprocess_df(per_adapter=True), 34 | postprocess_df=postprocess_df_wrapper, 35 | do_save_to_latex=False, 36 | ) 37 | 38 | basename = basename.replace("_per_adapter", "") 39 | basename = f"{basename}_adapter_validator_pairs" 40 | reshape_and_plot( 41 | df, 42 | output_folder, 43 | basename, 44 | name, 45 | new_col_fn, 46 | figsize=(12, 72), 47 | y_tick_labelsize=6, 48 | ) 49 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/officehome/officehome_fl3_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 officehome 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | 5 | commands: 6 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains clipart 7 | - *A 8 | - *common 9 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains product 10 | - *A 11 | - *common 12 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains real 13 | - *A 14 | - *common 15 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains art 16 | - *A 17 | - *common 18 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains product 19 | - *A 20 | - *common 21 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains real 22 | - *A 23 | - *common 24 | - - python powerful_benchmarker/launch_one.py --src_domains product --target_domains art 25 | - *A 26 | - *common 27 | - - python powerful_benchmarker/launch_one.py --src_domains product --target_domains clipart 28 | - *A 29 | - *common 30 | - - python powerful_benchmarker/launch_one.py --src_domains product --target_domains real 31 | - *A 32 | - *common 33 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains art 34 | - *A 35 | - *common 36 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains product 37 | - *A 38 | - *common 39 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains clipart 40 | - *A 41 | - *common 42 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/officehome/officehome_fl6_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 officehome 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | 5 | commands: 6 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains clipart 7 | - *A 8 | - *common 9 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains product 10 | - *A 11 | - *common 12 | - - python powerful_benchmarker/launch_one.py --src_domains art --target_domains real 13 | - *A 14 | - *common 15 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains art 16 | - *A 17 | - *common 18 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains product 19 | - *A 20 | - *common 21 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains real 22 | - *A 23 | - *common 24 | - - python powerful_benchmarker/launch_one.py --src_domains product --target_domains art 25 | - *A 26 | - *common 27 | - - python powerful_benchmarker/launch_one.py --src_domains product --target_domains clipart 28 | - *A 29 | - *common 30 | - - python powerful_benchmarker/launch_one.py --src_domains product --target_domains real 31 | - *A 32 | - *common 33 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains art 34 | - *A 35 | - *common 36 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains product 37 | - *A 38 | - *common 39 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains clipart 40 | - *A 41 | - *common 42 | -------------------------------------------------------------------------------- /powerful_benchmarker/exp_scripts/0th_epoch.sh: -------------------------------------------------------------------------------- 1 | python powerful_benchmarker/launch_multiple.py --exp_config mnist/mnist_fl3_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 2 | python powerful_benchmarker/launch_multiple.py --exp_config mnist/mnist_fl6_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 3 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_amazon_fl3_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 4 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_amazon_fl6_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 5 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_fl3_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 6 | python powerful_benchmarker/launch_multiple.py --exp_config office31/office31_fl6_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 7 | python powerful_benchmarker/launch_multiple.py --exp_config officehome/officehome_fl3_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 8 | python powerful_benchmarker/launch_multiple.py --exp_config officehome/officehome_fl6_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 9 | python powerful_benchmarker/launch_multiple.py --exp_config domainnet126/domainnet126_fl6_adam_lr1_0th_epoch --slurm_config mnist --nodelist=belongie-compute-01 --script_wrapper_timeout=300 -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/domainnet126/domainnet126_fl6_adam_lr1_0th_epoch.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 domainnet126 0_epochs 2 | A: &A --config_names epoch_0 3 | 4 | 5 | commands: 6 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains painting 7 | - *A 8 | - *common 9 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains real 10 | - *A 11 | - *common 12 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains sketch 13 | - *A 14 | - *common 15 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains clipart 16 | - *A 17 | - *common 18 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains real 19 | - *A 20 | - *common 21 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains sketch 22 | - *A 23 | - *common 24 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains clipart 25 | - *A 26 | - *common 27 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains painting 28 | - *A 29 | - *common 30 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains sketch 31 | - *A 32 | - *common 33 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains clipart 34 | - *A 35 | - *common 36 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains painting 37 | - *A 38 | - *common 39 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains real 40 | - *A 41 | - *common -------------------------------------------------------------------------------- /validator_tests/scripts/save_resilience_to_noise_dfs.sh: -------------------------------------------------------------------------------- 1 | exp_prefixes=("mnist_mnist_mnistm" "office31_amazon_dslr" "office31_amazon_webcam" "office31_dslr_amazon" \ 2 | "office31_dslr_webcam" "office31_webcam_amazon" "office31_webcam_dslr" \ 3 | "officehome_art_clipart" "officehome_art_product" "officehome_art_real" \ 4 | "officehome_clipart_art" "officehome_clipart_product" "officehome_clipart_real" \ 5 | "officehome_product_art" "officehome_product_clipart" "officehome_product_real" \ 6 | "officehome_real_art" "officehome_real_clipart" "officehome_real_product") 7 | 8 | 9 | for i in "${exp_prefixes[@]}" 10 | do 11 | python simple_slurm.py --command "python validator_tests/save_resilience_to_noise_dfs.py --run_combined" --all_in_one --exp_group_prefix $i --slurm_config_folder validator_tests \ 12 | --slurm_config mnist --job_name=resilience_to_noise --cpus-per-task=8 --exp_group_excludes oracle \ 13 | --gres=gpu:0 14 | done 15 | 16 | 17 | exp_prefixes=("domainnet126_clipart_painting" "domainnet126_clipart_real" "domainnet126_clipart_sketch" \ 18 | "domainnet126_painting_clipart" "domainnet126_painting_real" "domainnet126_painting_sketch" \ 19 | "domainnet126_real_clipart" "domainnet126_real_painting" "domainnet126_real_sketch" \ 20 | "domainnet126_sketch_clipart" "domainnet126_sketch_painting" "domainnet126_sketch_real") 21 | 22 | 23 | for i in "${exp_prefixes[@]}" 24 | do 25 | python simple_slurm.py --command "python validator_tests/save_resilience_to_noise_dfs.py --run_single" --all_in_one --exp_group_prefix $i --slurm_config_folder validator_tests \ 26 | --slurm_config mnist --job_name=resilience_to_noise --cpus-per-task=8 --exp_group_excludes oracle \ 27 | --gres=gpu:0 28 | done 29 | 30 | -------------------------------------------------------------------------------- /validator_tests/flags/cluster.py: -------------------------------------------------------------------------------- 1 | def DomainCluster(): 2 | flags = [] 3 | for layer in ["features", "logits", "preds"]: 4 | for normalize in [0, 1]: 5 | for p in [2]: 6 | flags.append( 7 | { 8 | "validator": "DomainCluster", 9 | "split": "train", 10 | "layer": layer, 11 | "normalize": str(normalize), 12 | "p": str(p), 13 | } 14 | ) 15 | return flags 16 | 17 | 18 | def ClassAMI(): 19 | flags = [] 20 | for layer in ["features", "logits"]: 21 | for normalize in [0, 1]: 22 | for p in [2]: 23 | for with_src in [0, 1]: 24 | flags.append( 25 | { 26 | "validator": "ClassAMI", 27 | "split": "train", 28 | "layer": layer, 29 | "normalize": str(normalize), 30 | "p": str(p), 31 | "with_src": str(with_src), 32 | } 33 | ) 34 | return flags 35 | 36 | 37 | def ClassAMICentroidInit(): 38 | flags = ClassAMI() 39 | for f in flags: 40 | f["validator"] = "ClassAMICentroidInit" 41 | return flags 42 | 43 | 44 | def ClassSS(): 45 | flags = ClassAMI() 46 | for f in flags: 47 | f["validator"] = "ClassSS" 48 | return flags 49 | 50 | 51 | def ClassSSCentroidInit(): 52 | flags = ClassAMI() 53 | for f in flags: 54 | f["validator"] = "ClassSSCentroidInit" 55 | return flags 56 | -------------------------------------------------------------------------------- /powerful_benchmarker/utils/ignite_save_features.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import h5py 4 | import numpy as np 5 | from pytorch_adapt.utils import common_functions as c_f 6 | 7 | 8 | class SaveFeatures: 9 | def __init__(self, folder, logger): 10 | self.folder = os.path.join(folder, "features") 11 | c_f.makedir_if_not_there(self.folder) 12 | self.logger = logger 13 | self.required_data = [ 14 | "src_train", 15 | "src_val", 16 | "target_train_with_labels", 17 | "target_val_with_labels", 18 | ] 19 | 20 | def __call__(self, epoch, **collected_data): 21 | inference_dict = {} 22 | for k, v in collected_data.items(): 23 | curr_k = k.replace("_with_labels", "") 24 | inference_dict[curr_k] = { 25 | name: v[name].cpu().numpy() 26 | for name in v.keys() 27 | if name not in discard_keys() 28 | } 29 | 30 | losses_dict = self.logger.get_losses() 31 | 32 | with h5py.File(os.path.join(self.folder, "features.hdf5"), "a") as hf: 33 | write_nested_dict(hf, inference_dict, epoch, "inference") 34 | write_nested_dict(hf, losses_dict, epoch, "losses") 35 | 36 | 37 | def write_nested_dict(hf, d, epoch, series_name): 38 | for k1, v1 in d.items(): 39 | grp = hf.create_group(f"{epoch}/{series_name}/{k1}") 40 | for k2, v2 in v1.items(): 41 | kwargs = ( 42 | {"compression": "gzip"} if isinstance(v2, (np.ndarray, list)) else {} 43 | ) 44 | grp.create_dataset(k2, data=v2, **kwargs) 45 | 46 | 47 | def discard_keys(): 48 | return ["imgs", "domain", "preds"] 49 | -------------------------------------------------------------------------------- /validator_tests/configs/dev_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.validators import DeepEmbeddedValidator 2 | from pytorch_adapt.validators.deep_embedded_validator import dev_binary_fn 3 | 4 | from .base_config import BaseConfig, get_split_and_layer 5 | 6 | 7 | class DEV(BaseConfig): 8 | def __init__(self, config): 9 | super().__init__(config) 10 | self.layer = self.validator_args["layer"] 11 | if self.validator_args["normalization"] == "None": 12 | self.validator_args["normalization"] = None 13 | self.validator = DeepEmbeddedValidator( 14 | temp_folder=None, 15 | batch_size=256, 16 | layer=self.layer, 17 | normalization=self.validator_args["normalization"], 18 | ) 19 | 20 | def score(self, x, exp_config, device): 21 | src_train = get_split_and_layer(x, "src_train", self.layer, device) 22 | src_val = get_split_and_layer(x, "src_val", self.layer, device) 23 | src_val_labels = get_split_and_layer(x, "src_val", "labels", device) 24 | src_val_logits = get_split_and_layer(x, "src_val", "logits", device) 25 | target_train = get_split_and_layer(x, "target_train", self.layer, device) 26 | return self.validator( 27 | src_train={self.layer: src_train}, 28 | src_val={ 29 | self.layer: src_val, 30 | "labels": src_val_labels, 31 | "logits": src_val_logits, 32 | }, 33 | target_train={self.layer: target_train}, 34 | ) 35 | 36 | def expected_keys(self): 37 | return {"layer", "normalization"} 38 | 39 | 40 | class DEVBinary(DEV): 41 | def __init__(self, config): 42 | super().__init__(config) 43 | self.validator.error_fn = dev_binary_fn 44 | -------------------------------------------------------------------------------- /validator_tests/collect_dfs.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import os 4 | import sys 5 | 6 | import pandas as pd 7 | 8 | sys.path.insert(0, ".") 9 | from powerful_benchmarker.utils.constants import add_default_args 10 | from validator_tests.utils import utils 11 | from validator_tests.utils.constants import ( 12 | ALL_DFS_FILENAME, 13 | VALIDATOR_TESTS_FOLDER, 14 | add_exp_group_args, 15 | ) 16 | 17 | 18 | def collect_dfs(args, exp_group): 19 | df = [] 20 | exp_folder = os.path.join(args.exp_folder, exp_group) 21 | exp_names = [os.path.basename(x) for x in glob.glob(os.path.join(exp_folder, "*"))] 22 | if args.slurm_folder in exp_names: 23 | exp_names.remove(args.slurm_folder) 24 | for e in exp_names: 25 | exp_folders = utils.get_exp_folders(exp_folder, e, use_glob=True) 26 | for ef in exp_folders: 27 | print(ef, flush=True) 28 | for v in args.validators: 29 | search_term = f"{v}*.pkl" 30 | df_files = glob.glob( 31 | os.path.join(ef, VALIDATOR_TESTS_FOLDER, search_term) 32 | ) 33 | for dff in df_files: 34 | df.append(pd.read_pickle(dff)) 35 | 36 | df = pd.concat(df, axis=0, ignore_index=True) 37 | filename = os.path.join(exp_folder, ALL_DFS_FILENAME) 38 | df.to_pickle(filename) 39 | 40 | 41 | def main(args): 42 | exp_groups = utils.get_exp_groups(args) 43 | for e in exp_groups: 44 | collect_dfs(args, e) 45 | 46 | 47 | if __name__ == "__main__": 48 | parser = argparse.ArgumentParser(allow_abbrev=False) 49 | add_default_args(parser, ["exp_folder", "slurm_folder"]) 50 | add_exp_group_args(parser) 51 | parser.add_argument("--validators", nargs="+", type=str, default=[""]) 52 | args = parser.parse_args() 53 | main(args) 54 | -------------------------------------------------------------------------------- /validator_tests/plot_resilience_to_noise.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | 4 | import pandas as pd 5 | import seaborn as sns 6 | 7 | 8 | def save_plot(output_folder, df): 9 | Ns = [100, 10, 1] 10 | x_axis_title = "Standard deviation of noise added to target-domain accuracy" 11 | y_axis_title = "Correlation with 0-noise ranking" 12 | keys = [f"Top {N} Accuracy" for N in Ns] + [ 13 | "Noise Standard Deviation", 14 | "Correlation with original data", 15 | ] 16 | new_keys = [f"AATN, N={N}" for N in Ns] + [x_axis_title, y_axis_title] 17 | df = df.rename(columns={k: v for k, v in zip(keys, new_keys)}) 18 | 19 | df = pd.melt( 20 | df, 21 | id_vars=[x_axis_title], 22 | value_vars=[ 23 | "Weighted Spearman Correlation", 24 | *new_keys[:-2], 25 | ], 26 | var_name="Metric", 27 | value_name=y_axis_title, 28 | ) 29 | 30 | sns.set(font_scale=1, style="whitegrid", rc={"figure.figsize": (10, 7.5)}) 31 | plot = sns.lineplot( 32 | data=df, 33 | x=x_axis_title, 34 | y=y_axis_title, 35 | hue="Metric", 36 | ) 37 | sns.move_legend(plot, "lower left") 38 | fig = plot.get_figure() 39 | fig.savefig( 40 | os.path.join(output_folder, "resilience.png"), 41 | bbox_inches="tight", 42 | ) 43 | fig.clf() 44 | 45 | 46 | def main(): 47 | folder = "plots/resilience_to_noise" 48 | folders = glob.glob(os.path.join(folder, "*")) 49 | 50 | all_dfs = [] 51 | for f in folders: 52 | print("plotting", f) 53 | filename = os.path.join(f, "df.pkl") 54 | if os.path.isfile(filename): 55 | df = pd.read_pickle(filename) 56 | all_dfs.append(df) 57 | save_plot(f, df) 58 | 59 | df = pd.concat(all_dfs, axis=0).reset_index() 60 | save_plot(folder, df) 61 | 62 | 63 | main() 64 | -------------------------------------------------------------------------------- /latex/best_accuracy_per_adapter.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from latex import utils as latex_utils 4 | from latex.table_creator import table_creator 5 | from validator_tests.utils.constants import TARGET_ACCURACY 6 | 7 | 8 | def preprocess_df(df): 9 | latex_utils.convert_adapter_name(df) 10 | return df 11 | 12 | 13 | def reshape_into_best_accuracy_table(df): 14 | df = df.pivot(index="adapter", columns="task").droplevel(level=0, axis=1) 15 | df = latex_utils.add_source_only(df, TARGET_ACCURACY) 16 | df = latex_utils.shortened_task_names(df) 17 | df = (df * 100).round(1) 18 | return df 19 | 20 | 21 | def postprocess_df(df): 22 | df = pd.concat(df, axis=0) 23 | df = df.drop(columns=[f"{TARGET_ACCURACY}_std"]) 24 | return reshape_into_best_accuracy_table(df) 25 | 26 | 27 | def min_value_fn(x, *_): 28 | return x.loc["Source only"] 29 | 30 | 31 | def best_accuracy_per_adapter(args, do_save_to_latex=True): 32 | nlargest = args.nlargest 33 | basename = f"best_accuracy_per_adapter_{nlargest}" 34 | color_map_tag_kwargs = { 35 | "tag_prefix": latex_utils.get_tag_prefix(basename), 36 | "min_value_fn": min_value_fn, 37 | } 38 | caption = ( 39 | f"The average of the top {nlargest} target domain accuracies per adapter/task pair. " 40 | "Green cells have an average accuracy greater than than the source-only model. " 41 | "A stronger green color indicates higher accuracy. The highest value per column is bolded." 42 | ) 43 | return table_creator( 44 | args, 45 | args.input_folder, 46 | args.output_folder, 47 | basename, 48 | preprocess_df, 49 | postprocess_df, 50 | color_map_tag_kwargs, 51 | add_resizebox=True, 52 | caption=caption, 53 | final_str_hook=latex_utils.adapter_final_str_hook, 54 | do_save_to_latex=do_save_to_latex, 55 | ) 56 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/adda_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.adapters import ADDA 2 | from pytorch_adapt.containers import Models, Optimizers 3 | from pytorch_adapt.inference import adda_full_fn 4 | from pytorch_adapt.weighters import MeanWeighter 5 | 6 | from ..utils import main_utils 7 | from .base_config import BaseConfig 8 | 9 | 10 | class ADDAConfig(BaseConfig): 11 | def get_adapter_kwargs( 12 | self, 13 | models, 14 | optimizers, 15 | before_training_starts, 16 | lr_multiplier, 17 | use_full_inference, 18 | **kwargs 19 | ): 20 | models = Models(models) 21 | optimizers = Optimizers( 22 | optimizers, 23 | multipliers={"D": lr_multiplier}, 24 | ) 25 | d_scale = self.optuna_trial.suggest_float("d_scale", 0, 1) 26 | g_scale = self.optuna_trial.suggest_float("g_scale", 0, 1) 27 | d_weighter = MeanWeighter(scale=d_scale) 28 | g_weighter = MeanWeighter(scale=g_scale) 29 | d_accuracy_threshold = self.optuna_trial.suggest_float( 30 | "d_accuracy_threshold", 0, 1 31 | ) 32 | hook_kwargs = { 33 | "threshold": d_accuracy_threshold, 34 | "d_weighter": d_weighter, 35 | "g_weighter": g_weighter, 36 | } 37 | inference_fn = adda_full_fn if use_full_inference else None 38 | 39 | return { 40 | "models": models, 41 | "optimizers": optimizers, 42 | "misc": None, 43 | "before_training_starts": before_training_starts, 44 | "inference_fn": inference_fn, 45 | "hook_kwargs": hook_kwargs, 46 | } 47 | 48 | def get_new_adapter(self, *args, **kwargs): 49 | return ADDA(**self.get_adapter_kwargs(*args, **kwargs)) 50 | 51 | def save(self, folder): 52 | super().save(folder) 53 | main_utils.save_this_file(__file__, folder) 54 | -------------------------------------------------------------------------------- /latex/best_validator_per_adapter_task.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import pandas as pd 5 | 6 | from latex.correlation import base_filename 7 | from latex.correlation import get_preprocess_df as get_preprocess_df_correlation 8 | from latex.correlation_single_adapter import ( 9 | get_postprocess_df as get_postprocess_df_correlation, 10 | ) 11 | from latex.table_creator import table_creator 12 | from validator_tests.utils import utils 13 | 14 | 15 | def get_best_validators(args, name, src_threshold): 16 | basename = base_filename(name, True, src_threshold) 17 | exp_groups = utils.get_exp_groups(args, args.input_folder) 18 | 19 | dfs, output_folder = table_creator( 20 | args, 21 | args.input_folder, 22 | args.output_folder, 23 | basename, 24 | preprocess_df=get_preprocess_df_correlation(per_adapter=True), 25 | postprocess_df=get_postprocess_df_correlation(remove_index_names=False), 26 | do_save_to_latex=False, 27 | exp_groups=exp_groups, 28 | ) 29 | 30 | best_validators = {} 31 | for adapter, df in dfs.items(): 32 | best_validators[adapter] = {} 33 | for task in df.columns: 34 | if task in ["Mean", "Std"]: 35 | continue 36 | best_validators[adapter][task] = df.loc[df[task].idxmax()].name 37 | 38 | return best_validators, output_folder 39 | 40 | 41 | def best_validator_per_adapter_task(args, name, src_threshold): 42 | best_validators, output_folder = get_best_validators(args, name, src_threshold) 43 | 44 | df = pd.DataFrame.from_dict(best_validators).transpose() 45 | 46 | for c in df.columns: 47 | df[c] = df[c].agg(" ".join) 48 | 49 | split_columns = np.array_split(df.columns, 5) 50 | for i, c in enumerate(split_columns): 51 | df[c].style.to_latex( 52 | os.path.join(output_folder, f"best_validator_per_adapter_task_{i}.tex"), 53 | hrules=True, 54 | position_float="centering", 55 | ) 56 | -------------------------------------------------------------------------------- /validator_tests/configs/nearest_source_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.validators import NearestSourceL2Validator, NearestSourceValidator 2 | 3 | from .base_config import BaseConfig, get_split_and_layer 4 | 5 | 6 | def get_score(x, layer, device, validator): 7 | src_features = get_split_and_layer(x, "src_val", layer, device) 8 | src_preds = get_split_and_layer(x, "src_val", "preds", device) 9 | src_labels = get_split_and_layer(x, "src_val", "labels", device) 10 | target_features = get_split_and_layer(x, "target_train", layer, device) 11 | return validator( 12 | src_val={ 13 | layer: src_features, 14 | "preds": src_preds, 15 | "labels": src_labels, 16 | }, 17 | target_train={layer: target_features}, 18 | ) 19 | 20 | 21 | class NearestSource(BaseConfig): 22 | def __init__(self, config): 23 | super().__init__(config) 24 | self.validator_args["threshold"] = float(self.validator_args["threshold"]) 25 | self.validator_args["weighted"] = bool(int(self.validator_args["weighted"])) 26 | self.layer = self.validator_args["layer"] 27 | self.validator = NearestSourceValidator( 28 | layer=self.layer, 29 | threshold=self.validator_args["threshold"], 30 | weighted=self.validator_args["weighted"], 31 | ) 32 | 33 | def score(self, x, exp_config, device): 34 | return get_score(x, self.layer, device, self.validator) 35 | 36 | def expected_keys(self): 37 | return {"threshold", "weighted", "layer"} 38 | 39 | 40 | class NearestSourceL2(BaseConfig): 41 | def __init__(self, config): 42 | super().__init__(config) 43 | self.layer = self.validator_args["layer"] 44 | self.validator = NearestSourceL2Validator( 45 | layer=self.layer, 46 | ) 47 | 48 | def score(self, x, exp_config, device): 49 | return get_score(x, self.layer, device, self.validator) 50 | 51 | def expected_keys(self): 52 | return {"layer"} 53 | -------------------------------------------------------------------------------- /validator_tests/utils/plot_score_vs_epoch.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import seaborn as sns 4 | from pytorch_adapt.utils import common_functions as c_f 5 | 6 | from .plot_utils import filter_and_plot 7 | 8 | 9 | def line_plot( 10 | plots_folder, 11 | df, 12 | x, 13 | y, 14 | filename, 15 | font_scale=0.8, 16 | figsize=(4.8, 4.8), 17 | ): 18 | sns.set(font_scale=font_scale, style="whitegrid", rc={"figure.figsize": figsize}) 19 | 20 | best_trial_num = df.loc[df["score"].idxmax()]["trial_num"] 21 | best_trial = df[df["trial_num"] == best_trial_num] 22 | worst_trial_num = df.loc[df["score"].idxmin()]["trial_num"] 23 | worst_trial = df[df["trial_num"] == worst_trial_num] 24 | 25 | sns.lineplot(data=df, x=x, y=y, ci="sd") 26 | sns.lineplot(data=best_trial, x=x, y=y, color="green") 27 | plot = sns.lineplot(data=worst_trial, x=x, y=y, color="red") 28 | fig = plot.get_figure() 29 | c_f.makedir_if_not_there(plots_folder) 30 | fig.savefig( 31 | os.path.join(plots_folder, f"{filename}.png"), 32 | bbox_inches="tight", 33 | ) 34 | fig.clf() 35 | 36 | 37 | def get_score_vs_epoch_fn(**kwargs): 38 | def fn(curr_plots_folder, curr_df, filename): 39 | curr_df = curr_df.astype({"epoch": int}) 40 | input_kwargs = { 41 | "plots_folder": curr_plots_folder, 42 | "df": curr_df, 43 | "x": "epoch", 44 | "y": "score", 45 | "filename": filename, 46 | } 47 | input_kwargs.update(kwargs) 48 | line_plot(**input_kwargs) 49 | 50 | return fn 51 | 52 | 53 | def plot_score_vs_epoch( 54 | df, 55 | plots_folder, 56 | per_adapter, 57 | per_feature_layer, 58 | validator_set=None, 59 | src_threshold=None, 60 | adapter=None, 61 | **kwargs, 62 | ): 63 | plots_folder = os.path.join(plots_folder, "score_vs_epoch") 64 | 65 | filter_and_plot( 66 | df, 67 | get_score_vs_epoch_fn(**kwargs), 68 | plots_folder, 69 | per_adapter, 70 | per_feature_layer, 71 | validator_set, 72 | src_threshold, 73 | adapter, 74 | ) 75 | -------------------------------------------------------------------------------- /simple_slurm.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import subprocess 4 | 5 | import submitit 6 | 7 | from powerful_benchmarker.utils.constants import add_default_args 8 | from powerful_benchmarker.utils.utils import create_slurm_args 9 | from validator_tests.utils import utils 10 | from validator_tests.utils.constants import add_exp_group_args, exp_group_args 11 | 12 | 13 | def exp_launcher(conda_env, command): 14 | full_command = f"bash -i ./scripts/script_wrapper.sh {conda_env}".split(" ") 15 | full_command += [command] 16 | subprocess.run(full_command) 17 | 18 | 19 | def run(args, slurm_args, exp_group): 20 | executor = submitit.AutoExecutor( 21 | folder=os.path.join(args.exp_folder, args.slurm_folder) 22 | ) 23 | executor.update_parameters( 24 | timeout_min=0, 25 | tasks_per_node=1, 26 | slurm_additional_parameters=slurm_args, 27 | ) 28 | 29 | command = args.command 30 | if exp_group: 31 | command = f"{command} --exp_groups {exp_group}" 32 | 33 | job = executor.submit(exp_launcher, args.conda_env, command) 34 | print("started", job.job_id) 35 | 36 | 37 | def main(args, slurm_args): 38 | if not any(getattr(args, k) for k in exp_group_args()): 39 | run(args, slurm_args, None) 40 | return 41 | exp_groups = utils.get_exp_groups(args) 42 | if args.all_in_one: 43 | run(args, slurm_args, " ".join(exp_groups)) 44 | else: 45 | for e in exp_groups: 46 | run(args, slurm_args, e) 47 | 48 | 49 | if __name__ == "__main__": 50 | parser = argparse.ArgumentParser(allow_abbrev=False) 51 | add_default_args(parser, ["exp_folder", "conda_env", "slurm_folder"]) 52 | add_exp_group_args(parser) 53 | parser.add_argument("--command", type=str, required=True) 54 | parser.add_argument("--slurm_config_folder", type=str, required=True) 55 | parser.add_argument("--slurm_config", type=str, required=True) 56 | parser.add_argument("--all_in_one", action="store_true") 57 | args, unknown_args = parser.parse_known_args() 58 | slurm_args = create_slurm_args(args, unknown_args, args.slurm_config_folder) 59 | main(args, slurm_args) 60 | -------------------------------------------------------------------------------- /latex/create_tables.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | 4 | sys.path.insert(0, ".") 5 | from latex.best_accuracy_per_adapter import best_accuracy_per_adapter 6 | from latex.best_validator_per_adapter_task import best_validator_per_adapter_task 7 | from latex.correlation import correlation 8 | from latex.correlation_bar_plot import correlation_bar_plot 9 | from latex.correlation_bar_plot_adapter_validator_pairs import ( 10 | correlation_bar_plot_adapter_validator_pairs, 11 | ) 12 | from latex.correlation_bar_plot_single_adapter import ( 13 | correlation_bar_plot_single_adapter, 14 | ) 15 | from latex.correlation_diffs import correlation_diffs 16 | from latex.correlation_single_adapter import correlation_single_adapter 17 | from latex.pred_acc_using_best_adapter_validator_pairs import ( 18 | pred_acc_using_best_adapter_validator_pairs, 19 | ) 20 | from latex.validator_parameter_explanations import validator_parameter_explanations 21 | from validator_tests.utils.constants import add_exp_group_args 22 | 23 | 24 | def main(args): 25 | src_threshold = 0.0 26 | wsp = "weighted_spearman" 27 | best_accuracy_per_adapter(args) 28 | validator_parameter_explanations(args, wsp, src_threshold) 29 | pred_acc_using_best_adapter_validator_pairs(args, wsp, src_threshold) 30 | best_validator_per_adapter_task(args, wsp, src_threshold) 31 | correlation_diffs(args, False, [wsp, "spearman"], src_threshold) 32 | correlation_single_adapter(args, wsp, src_threshold) 33 | correlation_bar_plot_single_adapter(args, wsp, src_threshold) 34 | correlation_bar_plot_adapter_validator_pairs(args, wsp, src_threshold) 35 | 36 | for per_adapter in [False, True]: 37 | correlation(args, per_adapter, wsp, src_threshold) 38 | correlation_bar_plot(args, per_adapter, wsp, src_threshold) 39 | 40 | 41 | if __name__ == "__main__": 42 | parser = argparse.ArgumentParser(allow_abbrev=False) 43 | add_exp_group_args(parser) 44 | add_exp_group_args(parser, "_select_best") 45 | parser.add_argument("--input_folder", type=str, default="tables") 46 | parser.add_argument("--output_folder", type=str, default="tables_latex") 47 | parser.add_argument("--nlargest", type=int, default=5) 48 | args = parser.parse_args() 49 | main(args) 50 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_creator_helper.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import string 3 | 4 | 5 | def main(cfg): 6 | base_string = """ 7 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains painting 8 | - *A 9 | - *common 10 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains real 11 | - *A 12 | - *common 13 | - - python powerful_benchmarker/launch_one.py --src_domains clipart --target_domains sketch 14 | - *A 15 | - *common 16 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains clipart 17 | - *A 18 | - *common 19 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains real 20 | - *A 21 | - *common 22 | - - python powerful_benchmarker/launch_one.py --src_domains painting --target_domains sketch 23 | - *A 24 | - *common 25 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains clipart 26 | - *A 27 | - *common 28 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains painting 29 | - *A 30 | - *common 31 | - - python powerful_benchmarker/launch_one.py --src_domains real --target_domains sketch 32 | - *A 33 | - *common 34 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains clipart 35 | - *A 36 | - *common 37 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains painting 38 | - *A 39 | - *common 40 | - - python powerful_benchmarker/launch_one.py --src_domains sketch --target_domains real 41 | - *A 42 | - *common 43 | """ 44 | 45 | out_string = f"{base_string}\n" 46 | for i in range(1, 5): 47 | curr_char = string.ascii_uppercase[i] 48 | new_block = base_string.replace("*A", f"*{curr_char}") 49 | out_string += f"{new_block}\n" 50 | if cfg.save_to_file: 51 | with open(cfg.save_to_file, "w") as f: 52 | f.write(out_string) 53 | else: 54 | print(out_string) 55 | 56 | 57 | if __name__ == "__main__": 58 | parser = argparse.ArgumentParser(allow_abbrev=False) 59 | parser.add_argument( 60 | "--save_to_file", 61 | type=str, 62 | default=None, 63 | ) 64 | args = parser.parse_args() 65 | main(args) 66 | -------------------------------------------------------------------------------- /validator_tests/configs/svd_config.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorch_adapt.layers import BatchSpectralLoss 3 | from pytorch_adapt.validators import BNMValidator 4 | 5 | from .base_config import BaseConfig, get_split_and_layer 6 | 7 | 8 | class BSP(BaseConfig): 9 | def __init__(self, config): 10 | super().__init__(config) 11 | self.validator_args["k"] = int(self.validator_args["k"]) 12 | self.layer = self.validator_args["layer"] 13 | self.validator = BatchSpectralLoss(k=self.validator_args["k"]) 14 | 15 | def score(self, x, exp_config, device): 16 | features = get_split_and_layer(x, self.split, self.layer, device) 17 | try: 18 | return -self.validator(features).item() 19 | except RuntimeError as e: 20 | if "svd_cuda" in str(e): 21 | return float("nan") 22 | raise 23 | 24 | def expected_keys(self): 25 | return {"k", "split", "layer"} 26 | 27 | 28 | class BNM(BaseConfig): 29 | def __init__(self, config): 30 | super().__init__(config) 31 | self.layer = self.validator_args["layer"] 32 | self.validator = BNMValidator( 33 | key_map={self.split: "target_train"}, 34 | ) 35 | 36 | def score(self, x, exp_config, device): 37 | features = get_split_and_layer(x, self.split, self.layer, device) 38 | return self.validator(**{self.split: {self.layer: features}}) 39 | 40 | def expected_keys(self): 41 | return {"split", "layer"} 42 | 43 | 44 | # from https://github.com/cuishuhao/BNM 45 | # TODO: move to pytorch-adapt 46 | def FBNM_loss(X): 47 | X = torch.nn.functional.softmax(X, dim=1) 48 | list_svd, _ = torch.sort( 49 | torch.sqrt(torch.sum(torch.pow(X, 2), dim=0)), descending=True 50 | ) 51 | nums = min(X.shape[0], X.shape[1]) 52 | return -torch.sum(list_svd[:nums]) 53 | 54 | 55 | class FBNM(BaseConfig): 56 | def __init__(self, config): 57 | super().__init__(config) 58 | self.layer = self.validator_args["layer"] 59 | 60 | def score(self, x, exp_config, device): 61 | features = get_split_and_layer(x, self.split, self.layer, device) 62 | return -FBNM_loss(features).item() 63 | 64 | def expected_keys(self): 65 | return {"split", "layer"} 66 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/vada_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.adapters import VADA 2 | from pytorch_adapt.containers import Models, Optimizers 3 | from pytorch_adapt.inference import default_with_d 4 | from pytorch_adapt.layers import VATLoss 5 | from pytorch_adapt.weighters import MeanWeighter 6 | 7 | from ..utils import main_utils 8 | from .base_config import BaseConfig 9 | 10 | 11 | class VADAConfig(BaseConfig): 12 | def get_adapter_kwargs( 13 | self, 14 | models, 15 | optimizers, 16 | before_training_starts, 17 | lr_multiplier, 18 | use_full_inference, 19 | **kwargs 20 | ): 21 | models = Models(models) 22 | optimizers = Optimizers(optimizers, multipliers={"D": lr_multiplier}) 23 | d_weight = self.optuna_trial.suggest_float("d_weight", 0, 1) 24 | g_weight = self.optuna_trial.suggest_float("g_weight", 0, 1) 25 | src_weight = self.optuna_trial.suggest_float("src_weight", 0, 1) 26 | target_weight = self.optuna_trial.suggest_float("target_weight", 0, 1) 27 | d_loss_weighter = MeanWeighter(scale=d_weight) 28 | g_loss_weighter = MeanWeighter( 29 | weights={ 30 | "src_vat_loss": src_weight, 31 | "target_vat_loss": target_weight, 32 | "entropy_loss": target_weight, 33 | "g_src_domain_loss": g_weight, 34 | "g_target_domain_loss": g_weight, 35 | } 36 | ) 37 | 38 | vat_loss_fn = VATLoss(epsilon=2) 39 | 40 | hook_kwargs = { 41 | "d_weighter": d_loss_weighter, 42 | "g_weighter": g_loss_weighter, 43 | "vat_loss_fn": vat_loss_fn, 44 | } 45 | 46 | inference_fn = default_with_d if use_full_inference else None 47 | 48 | return { 49 | "models": models, 50 | "optimizers": optimizers, 51 | "misc": None, 52 | "before_training_starts": before_training_starts, 53 | "inference_fn": inference_fn, 54 | "hook_kwargs": hook_kwargs, 55 | } 56 | 57 | def get_new_adapter(self, *args, **kwargs): 58 | return VADA(**self.get_adapter_kwargs(*args, **kwargs)) 59 | 60 | def save(self, folder): 61 | super().save(folder) 62 | main_utils.save_this_file(__file__, folder) 63 | -------------------------------------------------------------------------------- /validator_tests/where_nan.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import sys 5 | 6 | import h5py 7 | import pandas as pd 8 | import torch 9 | 10 | sys.path.insert(0, ".") 11 | from powerful_benchmarker.utils.constants import add_default_args 12 | from validator_tests import configs 13 | from validator_tests.utils.constants import get_all_dfs 14 | from validator_tests.utils.df_utils import print_validators_with_nan 15 | 16 | DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") 17 | 18 | 19 | def main(args): 20 | pd.options.display.max_colwidth = 100 21 | exp_folder = os.path.join(args.exp_folder, args.exp_group) 22 | df = get_all_dfs(exp_folder) 23 | df = print_validators_with_nan(df, return_df=True) 24 | validator_name = "MMDPerClassFixedB" 25 | df = df[df["validator"] == validator_name] 26 | df = df.iloc[0] 27 | 28 | validator = getattr(configs, validator_name)(json.loads(df["validator_args"])) 29 | if validator_name == "DEV": 30 | validator.validator.temp_folder = "DEV_temp_folder" 31 | 32 | features_file = os.path.join( 33 | exp_folder, df["exp_name"], str(df["trial_num"]), "features", "features.hdf5" 34 | ) 35 | with h5py.File(features_file, "r") as data: 36 | epoch_data = data[f"{df['epoch']}"] 37 | score = validator.score(epoch_data, None, DEVICE) 38 | 39 | print(score) 40 | 41 | 42 | if __name__ == "__main__": 43 | parser = argparse.ArgumentParser(allow_abbrev=False) 44 | add_default_args(parser, ["exp_folder"]) 45 | parser.add_argument("--exp_group", type=str, required=True) 46 | args = parser.parse_args() 47 | main(args) 48 | 49 | 50 | # MMD and MMDPerClass NaNs are caused by collapsed embeddings. 51 | # The distance between the embeddings is 0, so the median scaling (1/median(x)) results in inf 52 | 53 | # DEV NaN is also caused by collapsed embeddings. 54 | # The softmaxed output of the discriminator end up being entirely for one class 55 | # so the weights calculation becomes 1/0 56 | 57 | # MMDPerClassFixedB NaNs are caused by either the source or target domain 58 | # having only 1 embedding, i.e. the shape of src_features or target_features is 59 | # (1, feature_size). Since self-distances are excluded, the denominator 60 | # used for normalization is 0. See denom calculation in 61 | # pytorch_adapt.layers.utils.get_mmd_quadratic_batched 62 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/symnets_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.adapters import SymNets 2 | from pytorch_adapt.containers import Models, Optimizers 3 | from pytorch_adapt.inference import symnets_full_fn 4 | from pytorch_adapt.weighters import MeanWeighter 5 | 6 | from ..utils import main_utils 7 | from .mcd_config import MCDConfig 8 | 9 | 10 | class SymNetsConfig(MCDConfig): 11 | def get_adapter_kwargs( 12 | self, 13 | models, 14 | optimizers, 15 | before_training_starts, 16 | lr_multiplier, 17 | use_full_inference, 18 | **kwargs 19 | ): 20 | models = Models(models) 21 | optimizers = Optimizers( 22 | optimizers, 23 | multipliers={"C": lr_multiplier}, 24 | ) 25 | 26 | domain_weight = self.optuna_trial.suggest_float("domain_weight", 0, 1) 27 | category_weight = self.optuna_trial.suggest_float("category_weight", 0, 1) 28 | confusion_weight = self.optuna_trial.suggest_float("confusion_weight", 0, 1) 29 | entropy_weight = self.optuna_trial.suggest_float("entropy_weight", 0, 1) 30 | 31 | c_weighter = MeanWeighter( 32 | weights={ 33 | "c_symnets_src_domain_loss_0": domain_weight, 34 | "c_symnets_target_domain_loss_1": domain_weight, 35 | } 36 | ) 37 | g_weighter = MeanWeighter( 38 | weights={ 39 | "symnets_category_loss": category_weight, 40 | "g_symnets_target_domain_loss_0": confusion_weight, 41 | "g_symnets_target_domain_loss_1": confusion_weight, 42 | "symnets_entropy_loss": entropy_weight, 43 | } 44 | ) 45 | 46 | hook_kwargs = {"c_weighter": c_weighter, "g_weighter": g_weighter} 47 | 48 | inference_fn = symnets_full_fn if use_full_inference else None 49 | 50 | return { 51 | "models": models, 52 | "optimizers": optimizers, 53 | "misc": None, 54 | "before_training_starts": before_training_starts, 55 | "inference_fn": inference_fn, 56 | "hook_kwargs": hook_kwargs, 57 | } 58 | 59 | def get_new_adapter(self, *args, **kwargs): 60 | return SymNets(**self.get_adapter_kwargs(*args, **kwargs)) 61 | 62 | def save(self, folder): 63 | super().save(folder) 64 | main_utils.save_this_file(__file__, folder) 65 | -------------------------------------------------------------------------------- /powerful_benchmarker/utils/get_validator.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.frameworks.ignite import CheckpointFnCreator 2 | from pytorch_adapt.validators import AccuracyValidator, APValidator, ScoreHistory 3 | 4 | 5 | def get_validator_cls_and_kwargs(num_classes, average, multilabel): 6 | kwargs = {"num_classes": num_classes, "average": average} 7 | validator_cls = APValidator if multilabel else AccuracyValidator 8 | return validator_cls, kwargs 9 | 10 | 11 | def src_accuracy(num_classes, average, split, multilabel): 12 | validator_cls, torchmetric_kwargs = get_validator_cls_and_kwargs( 13 | num_classes, average, multilabel 14 | ) 15 | return validator_cls( 16 | torchmetric_kwargs=torchmetric_kwargs, key_map={f"src_{split}": "src_val"} 17 | ) 18 | 19 | 20 | def target_accuracy(num_classes, average, split, multilabel): 21 | validator_cls, torchmetric_kwargs = get_validator_cls_and_kwargs( 22 | num_classes, average, multilabel 23 | ) 24 | return validator_cls( 25 | torchmetric_kwargs=torchmetric_kwargs, 26 | key_map={f"target_{split}_with_labels": "src_val"}, 27 | ) 28 | 29 | 30 | def get_validator( 31 | num_classes, 32 | validator_name, 33 | checkpoint_path, 34 | ): 35 | if validator_name is None: 36 | return None, None 37 | 38 | if validator_name == "oracle": 39 | validator = target_accuracy( 40 | num_classes, average="macro", split="train", multilabel=False 41 | ) 42 | elif validator_name == "oracle_micro": 43 | validator = target_accuracy( 44 | num_classes, average="micro", split="train", multilabel=False 45 | ) 46 | elif validator_name == "src_accuracy": 47 | validator = src_accuracy( 48 | num_classes, average="macro", split="val", multilabel=False 49 | ) 50 | elif validator_name == "oracle_multilabel": 51 | validator = target_accuracy( 52 | num_classes, average="macro", split="train", multilabel=True 53 | ) 54 | elif validator_name == "src_accuracy_multilabel": 55 | validator = src_accuracy( 56 | num_classes, average="macro", split="val", multilabel=True 57 | ) 58 | else: 59 | raise ValueError 60 | 61 | validator = ScoreHistory(validator, ignore_epoch=0) 62 | checkpoint_fn = CheckpointFnCreator(dirname=checkpoint_path, require_empty=False) 63 | return validator, checkpoint_fn 64 | -------------------------------------------------------------------------------- /validator_tests/scripts/run.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import subprocess 3 | 4 | 5 | def main(args): 6 | validators = [ 7 | {"flags": "Accuracy", "exp_per_slurm_job": "4", "trials_per_exp": "100"}, 8 | {"flags": "Entropy", "exp_per_slurm_job": "4", "trials_per_exp": "100"}, 9 | {"flags": "DEVBinary", "exp_per_slurm_job": "4", "trials_per_exp": "100"}, 10 | {"flags": "SND", "exp_per_slurm_job": "3", "trials_per_exp": "100"}, 11 | { 12 | "flags": "ClassAMICentroidInit", 13 | "exp_per_slurm_job": "4", 14 | "trials_per_exp": "100", 15 | }, 16 | { 17 | "flags": "ClassSSCentroidInit", 18 | "exp_per_slurm_job": "4", 19 | "trials_per_exp": "100", 20 | }, 21 | {"flags": "BNM", "exp_per_slurm_job": "4", "trials_per_exp": "100"}, 22 | ] 23 | 24 | if args.validators: 25 | validators = [v for v in validators if v["flags"] in args.validators] 26 | 27 | exp_names = [ 28 | "atdoc", 29 | "bnm", 30 | "bsp", 31 | "cdan", 32 | "dann", 33 | "gvb", 34 | "im", 35 | "mcc", 36 | "mcd", 37 | "mmd", 38 | "epoch_0", 39 | ] 40 | if args.exp_names is not None: 41 | exp_names = args.exp_names 42 | exp_names = " ".join(exp_names) 43 | 44 | for x in validators: 45 | exp_per_slurm_job = int( 46 | int(x["exp_per_slurm_job"]) * args.exp_per_slurm_job_mul 47 | ) 48 | trials_per_exp = int(int(x["trials_per_exp"]) * args.trials_per_exp_mul) 49 | command = f"python validator_tests/run_validators.py {args.other_args} --slurm_config {args.slurm_config} --run" 50 | command += f" --exp_names {exp_names} --flags {x['flags']} --exp_per_slurm_job {exp_per_slurm_job} --trials_per_exp {trials_per_exp}" 51 | subprocess.run(command.split(" ")) 52 | 53 | 54 | if __name__ == "__main__": 55 | parser = argparse.ArgumentParser(allow_abbrev=False) 56 | parser.add_argument("--exp_names", nargs="+", type=str, default=None) 57 | parser.add_argument("--validators", nargs="+", type=str, default=[]) 58 | parser.add_argument("--other_args", type=str, default="") 59 | parser.add_argument("--slurm_config", type=str, required=True) 60 | parser.add_argument("--exp_per_slurm_job_mul", type=float, default=1) 61 | parser.add_argument("--trials_per_exp_mul", type=float, default=1) 62 | args = parser.parse_args() 63 | main(args) 64 | -------------------------------------------------------------------------------- /latex/correlation_bar_plot.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import seaborn as sns 4 | 5 | from latex.correlation import base_filename, get_postprocess_df, get_preprocess_df 6 | from latex.table_creator import table_creator 7 | 8 | 9 | def combine_levels(df): 10 | new_col = df.apply(lambda x: f'{x["validator"]}: {x["validator_args"]}', axis=1) 11 | return df.assign(validator_as_str=new_col).drop( 12 | columns=["validator", "validator_args"] 13 | ) 14 | 15 | 16 | def reshape_and_plot( 17 | df, 18 | output_folder, 19 | basename, 20 | name, 21 | new_col_fn=None, 22 | figsize=(12, 12), 23 | y_tick_labelsize=None, 24 | ): 25 | new_col_fn = combine_levels if new_col_fn is None else new_col_fn 26 | df = df.reset_index() 27 | df = new_col_fn(df) 28 | df["validator_as_str"] = df["validator_as_str"].str.replace( 29 | "\\tau", "τ", regex=False 30 | ) 31 | dfm = df.drop(columns=["Mean", "Std"]).melt(id_vars=["validator_as_str"]) 32 | assert "Mean" not in dfm["variable"].values 33 | assert "Std" not in dfm["variable"].values 34 | order = df.sort_values("Mean", ascending=False)["validator_as_str"].values 35 | xlabel = ( 36 | "Weighted Spearman Correlation" 37 | if name == "weighted_spearman" 38 | else "Spearman Correlation" 39 | ) 40 | 41 | sns.set_theme(style="whitegrid") 42 | plt = sns.barplot( 43 | x="value", 44 | y="validator_as_str", 45 | data=dfm, 46 | ci="sd", 47 | order=order, 48 | color="lightskyblue", 49 | errwidth=1.5, 50 | capsize=0.1, 51 | ) 52 | plt.set(xlabel=xlabel, ylabel="Validator", xlim=(-100, 100)) 53 | if y_tick_labelsize: 54 | plt.set_yticklabels(plt.get_yticklabels(), size=y_tick_labelsize) 55 | fig = plt.get_figure() 56 | fig.set_size_inches(*figsize) 57 | fig.savefig( 58 | os.path.join(output_folder, f"{basename}_barplot.png"), bbox_inches="tight" 59 | ) 60 | fig.clf() 61 | 62 | 63 | def correlation_bar_plot(args, per_adapter, name, src_threshold): 64 | basename = base_filename(name, per_adapter, src_threshold) 65 | 66 | df, output_folder = table_creator( 67 | args, 68 | args.input_folder, 69 | args.output_folder, 70 | basename, 71 | preprocess_df=get_preprocess_df(per_adapter), 72 | postprocess_df=get_postprocess_df(per_adapter, remove_index_names=False), 73 | do_save_to_latex=False, 74 | ) 75 | 76 | reshape_and_plot(df, output_folder, basename, name) 77 | -------------------------------------------------------------------------------- /latex/README.md: -------------------------------------------------------------------------------- 1 | ## powerful-benchmarker/latex 2 | 3 | The following commands reads the dataframes created by `../validator_tests/eval_validators.py`, and creates latex tables that summarize the results. 4 | 5 | The `--exp_group_excludes_select_best mnist` flag tells the code to ignore `mnist` when determining the best validator per algorithm. 6 | 7 | The `--nlargest 5` flag refers to the top-5 checkpoints for computing accuracy. 8 | 9 | ``` 10 | python latex/create_tables.py --exp_group_excludes domainnet126 --exp_group_excludes_select_best mnist --nlargest 5 11 | python latex/create_tables.py --exp_group_prefix domainnet126 --exp_group_excludes_select_best mnist --nlargest 5 12 | python latex/create_tables.py --exp_group_prefix mnist --exp_group_excludes_select_best mnist --nlargest 5 13 | python latex/create_tables.py --exp_group_prefix office --exp_group_excludes_select_best mnist --nlargest 5 14 | python latex/create_tables.py --exp_group_excludes mnist --exp_group_excludes_select_best mnist --nlargest 5 15 | ``` 16 | 17 | Make the `best_accuracy_per_adapter_ranked_by_score` tables use the same color tags as the `best_accuracy_per_adapter` tables. 18 | ``` 19 | python latex/replace_color_map_tags.py 20 | ``` 21 | 22 | Replace the header for domainnet: 23 | ``` 24 | python latex/replace_header_acronyms.py 25 | ``` 26 | 27 | 28 | ### File summary 29 | 30 | 31 | |Filename|Description| 32 | |-|-| 33 | |`best_accuracy_per_adapter.py`| Best accuracy per adapter per task when using an oracle | 34 | |`best_validator_per_adapter_task.py`| Best validator per algorithm per task | 35 | |`color_map_tags.py`| Code for colorizing latex table cells | 36 | |`correlation_bar_plot_adapter_validator_pairs.py`| Bar plot showing the correlation of every algorithm/validator pair | 37 | |`correlation_bar_plot_single_adapter.py`| Bar plot showing correlation of validators for a single algorithm | 38 | |`correlation_bar_plot.py`| Bar plot showing correlation of validators when applied to all algorithms simultaneously | 39 | |`correlation_diffs.py`| CSV of validator/task pairs with the largest diff between spearman and weighted spearman | 40 | |`correlation_single_adapter.py`| Table showing correlation of validators for a single algorithm | 41 | |`correlation.py`| Table showing correlation of validators when applied to all algorithms simultaneously | 42 | |`create_tables.py`| The main file which calls all other functions | 43 | |`pred_acc_using_best_adapter_validator_pairs.py`| Best accuracy per adapter per task when using the best validator for that algorithm | 44 | |`validator_parameter_explanations.py`| Table explaining validator parameters | -------------------------------------------------------------------------------- /validator_tests/configs/base_config.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | 6 | 7 | def get_from_hdf5(x, device, key): 8 | return torch.from_numpy(x[key][()]).to(device) 9 | 10 | 11 | def get_split_and_layer(x, split, layer, device): 12 | hdf5_layer = "logits" if layer == "preds" else layer 13 | features = get_from_hdf5(x, device, f"inference/{split}/{hdf5_layer}") 14 | if layer == "preds": 15 | features = F.softmax(features, dim=1) 16 | return features 17 | 18 | 19 | def get_full_split_name(domain, split): 20 | return f"{domain}_{split}" 21 | 22 | 23 | def get_src_domain(length, device): 24 | return torch.zeros(length).to(device=device, dtype=torch.long) 25 | 26 | 27 | def get_target_domain(length, device): 28 | return torch.ones(length).to(device=device, dtype=torch.long) 29 | 30 | 31 | def use_src_and_target(x, device, validator, src_split_name, target_split_name, layer): 32 | src = get_split_and_layer(x, src_split_name, layer, device) 33 | target = get_split_and_layer(x, target_split_name, layer, device) 34 | return pass_src_and_target_to_validator( 35 | validator, src_split_name, target_split_name, layer, src, target, device 36 | ) 37 | 38 | 39 | def pass_src_and_target_to_validator( 40 | validator, src_split_name, target_split_name, layer, src, target, device 41 | ): 42 | return validator( 43 | **{ 44 | src_split_name: { 45 | layer: src, 46 | "domain": get_src_domain(len(src), device), 47 | }, 48 | target_split_name: { 49 | layer: target, 50 | "domain": get_target_domain(len(target), device), 51 | }, 52 | } 53 | ) 54 | 55 | 56 | def use_labels_and_logits( 57 | x, device, validator, src_split_name, target_split_name, layer 58 | ): 59 | src = { 60 | k: get_split_and_layer(x, src_split_name, k, device) for k in [layer, "labels"] 61 | } 62 | target = { 63 | k: get_split_and_layer(x, target_split_name, k, device) 64 | for k in [layer, "logits"] 65 | } 66 | kwargs = {src_split_name: src, target_split_name: target} 67 | return validator(**kwargs) 68 | 69 | 70 | class BaseConfig: 71 | def __init__(self, config): 72 | self.validator_args = copy.deepcopy(config) 73 | if self.validator_args.keys() != self.expected_keys(): 74 | raise ValueError( 75 | f"expected {self.expected_keys()} but got {self.validator_args.keys()}" 76 | ) 77 | self.split = self.validator_args.get("split", None) 78 | -------------------------------------------------------------------------------- /latex/correlation_single_adapter.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from latex import utils as latex_utils 4 | from latex.correlation import ( 5 | base_filename, 6 | get_add_resizebox, 7 | get_caption, 8 | get_highlight_max_subset, 9 | get_highlight_min, 10 | get_highlight_min_subset, 11 | get_label_prefix, 12 | get_preprocess_df, 13 | interval_fn, 14 | max_value_fn, 15 | min_value_fn, 16 | operation_fn, 17 | ) 18 | from latex.table_creator import table_creator 19 | 20 | 21 | def get_postprocess_df(remove_index_names=True): 22 | def fn(df): 23 | df = pd.concat(df, axis=0).reset_index(drop=True) 24 | df = latex_utils.rename_validator_args(df) 25 | df = df.pivot(index=["validator", "validator_args"], columns="task") 26 | df_dict = {} 27 | for i in df.columns.levels[0]: 28 | df_dict[i] = df[i] 29 | for k, df in df_dict.items(): 30 | df = latex_utils.shortened_task_names(df) 31 | df = latex_utils.add_mean_std_column(df) 32 | df = (df * 100).round(1) 33 | df.columns.names = (None,) 34 | if remove_index_names: 35 | df.index.names = (None, None) 36 | df_dict[k] = df 37 | return df_dict 38 | 39 | return fn 40 | 41 | 42 | def caption_hook(caption, k): 43 | return caption.replace("pair", f"pair for \\textbf{{{k}}}") 44 | 45 | 46 | def correlation_single_adapter(args, name, src_threshold): 47 | basename = base_filename(name, True, src_threshold) 48 | color_map_tag_kwargs = { 49 | "tag_prefix": latex_utils.get_tag_prefix(basename), 50 | "min_value_fn": min_value_fn, 51 | "max_value_fn": max_value_fn, 52 | "num_steps": 11, 53 | "interval_fn": interval_fn, 54 | "operation_fn": operation_fn, 55 | } 56 | 57 | caption = get_caption(per_adapter=False, short_caption=True) 58 | 59 | highlight_max_subset = get_highlight_max_subset(per_adapter=False) 60 | highlight_min_subset = get_highlight_min_subset() 61 | 62 | table_creator( 63 | args, 64 | args.input_folder, 65 | args.output_folder, 66 | basename, 67 | get_preprocess_df(per_adapter=True), 68 | get_postprocess_df(), 69 | color_map_tag_kwargs, 70 | add_resizebox=get_add_resizebox(args), 71 | clines="skip-last;data", 72 | caption=caption, 73 | highlight_min=get_highlight_min(args), 74 | highlight_max_subset=highlight_max_subset, 75 | highlight_min_subset=highlight_min_subset, 76 | final_str_hook=latex_utils.validator_final_str_hook, 77 | caption_hook=caption_hook, 78 | position="H", 79 | label_prefix=get_label_prefix(args), 80 | ) 81 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_fl3_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl3_adam_lr1 office31 2 | A: &A --config_names dann mcc 3 | B: &B --config_names atdoc bsp 4 | C: &C --config_names cdan im 5 | D: &D --config_names bnm gvb 6 | E: &E --config_names mcd mmd 7 | 8 | 9 | commands: 10 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 11 | - *A 12 | - *common 13 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 14 | - *A 15 | - *common 16 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 17 | - *A 18 | - *common 19 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 20 | - *A 21 | - *common 22 | 23 | 24 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 25 | - *B 26 | - *common 27 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 28 | - *B 29 | - *common 30 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 31 | - *B 32 | - *common 33 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 34 | - *B 35 | - *common 36 | 37 | 38 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 39 | - *C 40 | - *common 41 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 42 | - *C 43 | - *common 44 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 45 | - *C 46 | - *common 47 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 48 | - *C 49 | - *common 50 | 51 | 52 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 53 | - *D 54 | - *common 55 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 56 | - *D 57 | - *common 58 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 59 | - *D 60 | - *common 61 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 62 | - *D 63 | - *common 64 | 65 | 66 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 67 | - *E 68 | - *common 69 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 70 | - *E 71 | - *common 72 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 73 | - *E 74 | - *common 75 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 76 | - *E 77 | - *common 78 | 79 | -------------------------------------------------------------------------------- /powerful_benchmarker/yaml_configs/exp_configs/office31/office31_fl6_adam_lr1.yaml: -------------------------------------------------------------------------------- 1 | common: &common --group_configs default fl6_adam_lr1 office31 2 | A: &A --config_names dann mcc 3 | B: &B --config_names atdoc bsp 4 | C: &C --config_names cdan im 5 | D: &D --config_names bnm gvb 6 | E: &E --config_names mcd mmd 7 | 8 | 9 | commands: 10 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 11 | - *A 12 | - *common 13 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 14 | - *A 15 | - *common 16 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 17 | - *A 18 | - *common 19 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 20 | - *A 21 | - *common 22 | 23 | 24 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 25 | - *B 26 | - *common 27 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 28 | - *B 29 | - *common 30 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 31 | - *B 32 | - *common 33 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 34 | - *B 35 | - *common 36 | 37 | 38 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 39 | - *C 40 | - *common 41 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 42 | - *C 43 | - *common 44 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 45 | - *C 46 | - *common 47 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 48 | - *C 49 | - *common 50 | 51 | 52 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 53 | - *D 54 | - *common 55 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 56 | - *D 57 | - *common 58 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 59 | - *D 60 | - *common 61 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 62 | - *D 63 | - *common 64 | 65 | 66 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains dslr 67 | - *E 68 | - *common 69 | - - python powerful_benchmarker/launch_one.py --src_domains amazon --target_domains webcam 70 | - *E 71 | - *common 72 | - - python powerful_benchmarker/launch_one.py --src_domains dslr --target_domains webcam 73 | - *E 74 | - *common 75 | - - python powerful_benchmarker/launch_one.py --src_domains webcam --target_domains dslr 76 | - *E 77 | - *common 78 | 79 | -------------------------------------------------------------------------------- /latex/color_map_tags.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def format_tag(prefix, column_name): 5 | return f"\\{prefix}{column_name.replace(' ', '')}" 6 | 7 | 8 | def format_tag_and_float(prefix, column_name): 9 | def return_fn(x): 10 | return f"{format_tag(prefix, column_name)}{{{x:.1f}}}" 11 | 12 | return return_fn 13 | 14 | 15 | def default_operation_fn(*_): 16 | return ">" 17 | 18 | 19 | # operation fn 20 | def absolute_value_greater_than(lower_bound, *_): 21 | if lower_bound >= 0: 22 | return ">" 23 | return "<" 24 | 25 | 26 | def default_interval_fn(min_value, max_value, num_steps, *_): 27 | intervals = np.linspace(min_value, max_value, num_steps) 28 | return [intervals[::-1]] 29 | 30 | 31 | def absolute_value_interval_fn(min_value, max_value, num_steps, *_): 32 | intervals = np.linspace(np.abs(min_value), max_value, num_steps) 33 | intervals = intervals[::-1] 34 | return [intervals, intervals * -1] 35 | 36 | 37 | def reverse_interval_fn(*args): 38 | return [default_interval_fn(*args)[0][::-1]] 39 | 40 | 41 | def create_color_map_tags( 42 | df, 43 | tag_prefix, 44 | min_value_fn=None, 45 | max_value_fn=None, 46 | operation_fn=None, 47 | interval_fn=None, 48 | num_steps=11, 49 | ): 50 | output_strs = [] 51 | if operation_fn is None: 52 | operation_fn = default_operation_fn 53 | if interval_fn is None: 54 | interval_fn = default_interval_fn 55 | for column_name in df.columns.values: 56 | curr_df = df[column_name] 57 | min_value = 0 if min_value_fn is None else min_value_fn(curr_df, column_name) 58 | max_value = ( 59 | curr_df.max() 60 | if max_value_fn is None 61 | else max_value_fn(curr_df, column_name) 62 | ) 63 | intervals_list = interval_fn(min_value, max_value, num_steps, column_name) 64 | 65 | curr_str = f"\\def{format_tag(tag_prefix, column_name)}" + "#1{" 66 | ifdim_count = 0 67 | for intervals in intervals_list: 68 | for i, lower_bound in enumerate(intervals): 69 | if i == 0: 70 | continue 71 | greenness = (10 - i + 1) * 10 72 | operation = operation_fn(lower_bound, column_name) 73 | curr_str += ( 74 | f"\\ifdim#1pt{operation}{lower_bound:.1f}" 75 | + f"pt\\cellcolor{{lime!{greenness}}}\\else" 76 | ) 77 | ifdim_count += 1 78 | curr_str += "\\cellcolor{lime!0}" 79 | curr_str += "\\fi" * ifdim_count 80 | curr_str += "#1}" 81 | output_strs.append(curr_str) 82 | 83 | return "\n\n".join(output_strs) 84 | 85 | 86 | def get_tags_dict(tag_prefix, task_columns): 87 | return { 88 | column_name: format_tag_and_float(tag_prefix, column_name) 89 | for column_name in task_columns 90 | } 91 | -------------------------------------------------------------------------------- /validator_tests/configs/mmd_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.layers.utils import get_kernel_scales 2 | from pytorch_adapt.validators import MMDValidator, PerClassValidator 3 | from pytorch_metric_learning.distances import LpDistance 4 | 5 | from .base_config import ( 6 | BaseConfig, 7 | get_full_split_name, 8 | use_labels_and_logits, 9 | use_src_and_target, 10 | ) 11 | 12 | 13 | class MMD(BaseConfig): 14 | def __init__(self, config): 15 | super().__init__(config) 16 | self.validator_args["exponent"] = int(self.validator_args["exponent"]) 17 | self.validator_args["normalize"] = bool(int(self.validator_args["normalize"])) 18 | self.layer = self.validator_args["layer"] 19 | self.src_split_name = get_full_split_name("src", self.split) 20 | self.target_split_name = get_full_split_name("target", self.split) 21 | 22 | self.validator = MMDValidator( 23 | key_map={ 24 | self.src_split_name: "src_train", 25 | self.target_split_name: "target_train", 26 | }, 27 | layer=self.validator_args["layer"], 28 | batch_size=512, 29 | mmd_kwargs=self.get_mmd_kwargs(), 30 | ) 31 | 32 | def score(self, x, exp_config, device): 33 | return use_src_and_target( 34 | x, 35 | device, 36 | self.validator, 37 | self.src_split_name, 38 | self.target_split_name, 39 | self.layer, 40 | ) 41 | 42 | def expected_keys(self): 43 | return {"exponent", "normalize", "layer", "split"} 44 | 45 | def get_mmd_kwargs(self): 46 | exponent = self.validator_args["exponent"] 47 | num_kernels = (exponent * 2) + 1 48 | kernel_scales = get_kernel_scales( 49 | low=-exponent, high=exponent, num_kernels=num_kernels 50 | ) 51 | dist_func = LpDistance( 52 | normalize_embeddings=self.validator_args["normalize"], p=2, power=2 53 | ) 54 | return { 55 | "kernel_scales": kernel_scales, 56 | "dist_func": dist_func, 57 | "mmd_type": "quadratic", 58 | } 59 | 60 | 61 | class MMDPerClass(MMD): 62 | def __init__(self, config): 63 | super().__init__(config) 64 | self.validator = PerClassValidator(self.validator) 65 | 66 | def score(self, x, exp_config, device): 67 | return use_labels_and_logits( 68 | x, 69 | device, 70 | self.validator, 71 | self.src_split_name, 72 | self.target_split_name, 73 | self.layer, 74 | ) 75 | 76 | 77 | class MMDFixedB(MMD): 78 | def get_mmd_kwargs(self): 79 | kwargs = super().get_mmd_kwargs() 80 | kwargs["bandwidth"] = 1 81 | return kwargs 82 | 83 | 84 | class MMDPerClassFixedB(MMDPerClass): 85 | def get_mmd_kwargs(self): 86 | kwargs = super().get_mmd_kwargs() 87 | kwargs["bandwidth"] = 1 88 | return kwargs 89 | -------------------------------------------------------------------------------- /validator_tests/create_plots.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | 4 | sys.path.insert(0, ".") 5 | from powerful_benchmarker.utils.constants import add_default_args 6 | from validator_tests.utils import create_main 7 | from validator_tests.utils.constants import add_exp_group_args 8 | from validator_tests.utils.plot_score_vs_epoch import plot_score_vs_epoch 9 | from validator_tests.utils.plot_val_vs_acc import plot_val_vs_acc 10 | 11 | 12 | def scatter(plots_folder, df, per_feature_layer): 13 | kwargs = {} 14 | if args.no_color: 15 | kwargs["c"] = None 16 | if args.dot_size: 17 | kwargs["s"] = args.dot_size 18 | if args.font_scale: 19 | kwargs["font_scale"] = args.font_scale 20 | if args.figsize: 21 | kwargs["figsize"] = args.figsize 22 | plot_val_vs_acc( 23 | df, 24 | plots_folder, 25 | per_adapter=args.per_adapter, 26 | per_feature_layer=per_feature_layer, 27 | validator_set=args.validator_set, 28 | src_threshold=args.src_threshold, 29 | adapter=args.adapter, 30 | **kwargs 31 | ) 32 | 33 | 34 | def score_vs_epoch(plots_folder, df, per_feature_layer): 35 | plot_score_vs_epoch( 36 | df, 37 | plots_folder, 38 | per_adapter=args.per_adapter, 39 | per_feature_layer=per_feature_layer, 40 | validator_set=args.validator_set, 41 | src_threshold=args.src_threshold, 42 | adapter=args.adapter, 43 | ) 44 | 45 | 46 | def get_fns(fn_list): 47 | if fn_list == []: 48 | fn_list = ["scatter", "over_time"] 49 | 50 | def fn1(*args): 51 | if "scatter" in fn_list: 52 | scatter(*args, per_feature_layer=True) 53 | if "score_vs_epoch" in fn_list: 54 | score_vs_epoch(*args, per_feature_layer=True) 55 | 56 | def fn2(*args): 57 | if "scatter" in fn_list: 58 | scatter(*args, per_feature_layer=False) 59 | if "score_vs_epoch" in fn_list: 60 | score_vs_epoch(*args, per_feature_layer=False) 61 | 62 | return fn1, fn2 63 | 64 | 65 | if __name__ == "__main__": 66 | parser = argparse.ArgumentParser(allow_abbrev=False) 67 | add_default_args(parser, ["exp_folder"]) 68 | add_exp_group_args(parser) 69 | create_main.add_main_args(parser) 70 | parser.add_argument("--output_folder", type=str, default="plots") 71 | parser.add_argument("--validator_set", nargs="+", type=str, default=None) 72 | parser.add_argument("--src_threshold", type=float) 73 | parser.add_argument("--no_color", action="store_true") 74 | parser.add_argument("--dot_size", type=float, default=None) 75 | parser.add_argument("--font_scale", type=float, default=None) 76 | parser.add_argument("--figsize", nargs="+", type=float, default=None) 77 | parser.add_argument("--per_adapter", action="store_true") 78 | parser.add_argument("--adapter", type=str) 79 | parser.add_argument("--fn_list", nargs="+", type=str, default=[]) 80 | args = parser.parse_args() 81 | create_main.main(args, *get_fns(args.fn_list)) 82 | -------------------------------------------------------------------------------- /validator_tests/0th_epoch_gather.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import os 4 | import sys 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | sys.path.insert(0, ".") 10 | from pytorch_adapt.models.pretrained_scores import ( 11 | pretrained_src_accuracy, 12 | pretrained_target_accuracy, 13 | ) 14 | 15 | from powerful_benchmarker.utils.constants import add_default_args 16 | from validator_tests.utils import df_utils, utils 17 | from validator_tests.utils.constants import ( 18 | ALL_DFS_FILENAME, 19 | VALIDATOR_TESTS_FOLDER, 20 | add_exp_group_args, 21 | ) 22 | 23 | 24 | def collect_dfs(args, exp_group): 25 | df = [] 26 | exp_folder = os.path.join(args.exp_folder, exp_group) 27 | exp_folders = utils.get_exp_folders(exp_folder, "epoch_0", use_glob=True) 28 | for ef in exp_folders: 29 | print(ef, flush=True) 30 | for v in args.validators: 31 | search_term = f"{v}*.pkl" 32 | df_files = glob.glob(os.path.join(ef, VALIDATOR_TESTS_FOLDER, search_term)) 33 | for dff in df_files: 34 | curr_df = pd.read_pickle(dff) 35 | curr_df = curr_df[curr_df["epoch"] == "0"] 36 | df.append(curr_df) 37 | 38 | if len(df) > 0: 39 | df = pd.concat(df, axis=0, ignore_index=True) 40 | 41 | # check that the computed accuracies match the ones provided by pytorch adapt 42 | for split in df_utils.SPLIT_NAMES: 43 | for average in ["micro", "macro"]: 44 | acc = df_utils.get_acc_df(df, split, average) 45 | dataset = acc["dataset"].item() 46 | src_domains = acc["src_domains"].item() 47 | target_domains = acc["target_domains"].item() 48 | if split.startswith("src"): 49 | correct = pretrained_src_accuracy( 50 | dataset, src_domains, split.replace("src_", ""), average 51 | ) 52 | else: 53 | correct = pretrained_target_accuracy( 54 | dataset, 55 | src_domains, 56 | target_domains, 57 | split.replace("target_", ""), 58 | average, 59 | ) 60 | 61 | assert ( 62 | np.round( 63 | acc[df_utils.acc_score_column_name(split, average)].item(), 4 64 | ) 65 | == correct 66 | ) 67 | 68 | all_dfs = pd.read_pickle(os.path.join(exp_folder, ALL_DFS_FILENAME)) 69 | df = pd.concat([all_dfs, df], axis=0) 70 | df.to_pickle(os.path.join(exp_folder, ALL_DFS_FILENAME)) 71 | 72 | 73 | def main(args): 74 | exp_groups = utils.get_exp_groups(args) 75 | for e in exp_groups: 76 | collect_dfs(args, e) 77 | 78 | 79 | if __name__ == "__main__": 80 | parser = argparse.ArgumentParser(allow_abbrev=False) 81 | add_default_args(parser, ["exp_folder", "slurm_folder"]) 82 | add_exp_group_args(parser) 83 | parser.add_argument("--validators", nargs="+", type=str, default=[""]) 84 | args = parser.parse_args() 85 | main(args) 86 | -------------------------------------------------------------------------------- /validator_tests/utils/plot_val_vs_acc.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import matplotlib.pyplot as plt 4 | import seaborn as sns 5 | from pytorch_adapt.utils import common_functions as c_f 6 | 7 | from .constants import TARGET_ACCURACY 8 | from .plot_utils import filter_and_plot 9 | 10 | 11 | def _scatter_plot( 12 | df, 13 | x, 14 | y, 15 | c=None, 16 | colobar_label=None, 17 | log_x=False, 18 | s=0.1, 19 | font_scale=0.8, 20 | figsize=(4.8, 4.8), 21 | show_x_label=True, 22 | show_y_label=True, 23 | colorbar=True, 24 | x_label=None, 25 | y_label=None, 26 | alpha=None, 27 | cmap="rainbow", 28 | invert_cmap_axis=False, 29 | ): 30 | sns.set(font_scale=font_scale, style="whitegrid", rc={"figure.figsize": figsize}) 31 | if colorbar: 32 | points = plt.scatter( 33 | df[x], 34 | df[y], 35 | c=df[c] if c is not None else None, 36 | s=s, 37 | cmap=cmap, 38 | alpha=alpha, 39 | ) 40 | if c: 41 | cbar = plt.colorbar(points) 42 | if invert_cmap_axis: 43 | cbar.ax.invert_yaxis() 44 | if colobar_label: 45 | cbar.set_label(colobar_label) 46 | if show_x_label: 47 | plt.xlabel(x if x_label is None else x_label) 48 | if show_y_label: 49 | plt.ylabel(y if y_label is None else y_label) 50 | if log_x: 51 | plt.xscale("symlog") 52 | fig = plt 53 | else: 54 | plot = sns.scatterplot(data=df, x=x, y=y, hue=c, s=s, alpha=alpha) 55 | fig = plot.get_figure() 56 | return fig 57 | 58 | 59 | def scatter_plot(plots_folder, df, x, y, filename, **kwargs): 60 | fig = _scatter_plot(df, x, y, **kwargs) 61 | c_f.makedir_if_not_there(plots_folder) 62 | fig.savefig( 63 | os.path.join(plots_folder, f"{filename}.png"), 64 | bbox_inches="tight", 65 | ) 66 | fig.clf() 67 | 68 | 69 | def get_score_vs_target_accuracy_fn(**kwargs): 70 | def fn(curr_plots_folder, curr_df, filename): 71 | input_kwargs = { 72 | "plots_folder": curr_plots_folder, 73 | "df": curr_df, 74 | "x": "score", 75 | "y": TARGET_ACCURACY, 76 | "filename": filename, 77 | "c": "src_val_micro", 78 | "x_label": "Validation Score", 79 | "y_label": "Target Accuracy", 80 | } 81 | input_kwargs.update(kwargs) 82 | scatter_plot(**input_kwargs) 83 | 84 | return fn 85 | 86 | 87 | def plot_val_vs_acc( 88 | df, 89 | plots_folder, 90 | per_adapter, 91 | per_feature_layer, 92 | validator_set=None, 93 | src_threshold=None, 94 | adapter=None, 95 | **kwargs, 96 | ): 97 | plots_folder = os.path.join(plots_folder, "val_vs_acc") 98 | 99 | filter_and_plot( 100 | df, 101 | get_score_vs_target_accuracy_fn(**kwargs), 102 | plots_folder, 103 | per_adapter, 104 | per_feature_layer, 105 | validator_set, 106 | src_threshold, 107 | adapter, 108 | ) 109 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/cdan_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.adapters import CDAN, CDANE 2 | from pytorch_adapt.containers import Misc, Models, Optimizers 3 | from pytorch_adapt.datasets import utils as dataset_utils 4 | from pytorch_adapt.inference import cdan_full_fn 5 | from pytorch_adapt.layers import RandomizedDotProduct 6 | from pytorch_adapt.weighters import MeanWeighter 7 | 8 | from ..utils import main_utils 9 | from .base_config import BaseConfig 10 | 11 | 12 | class CDANConfig(BaseConfig): 13 | def get_models(self, dataset, *args, **kwargs): 14 | num_classes = dataset_utils.num_classes(dataset) 15 | models, framework = super().get_models(dataset, *args, **kwargs) 16 | models["feature_combiner"] = RandomizedDotProduct( 17 | [self.feature_size, num_classes], self.feature_size 18 | ) 19 | return models, framework 20 | 21 | def get_adapter_kwargs( 22 | self, 23 | models, 24 | optimizers, 25 | before_training_starts, 26 | lr_multiplier, 27 | use_full_inference, 28 | **kwargs 29 | ): 30 | feature_combiner = models.pop("feature_combiner") 31 | models = Models(models) 32 | optimizers = Optimizers(optimizers, multipliers={"D": lr_multiplier}) 33 | misc = Misc({"feature_combiner": feature_combiner}) 34 | 35 | d_weight = self.optuna_trial.suggest_float("d_weight", 0, 1) 36 | g_weight = self.optuna_trial.suggest_float("g_weight", 0, 1) 37 | label_weight = self.optuna_trial.suggest_float("label_weight", 0, 1) 38 | 39 | d_weighter = MeanWeighter(scale=d_weight) 40 | g_weighter = MeanWeighter( 41 | weights={ 42 | "g_src_domain_loss": g_weight, 43 | "g_target_domain_loss": g_weight, 44 | "c_loss": label_weight, 45 | }, 46 | ) 47 | hook_kwargs = { 48 | "d_weighter": d_weighter, 49 | "g_weighter": g_weighter, 50 | } 51 | inference_fn = cdan_full_fn if use_full_inference else None 52 | return { 53 | "models": models, 54 | "optimizers": optimizers, 55 | "misc": misc, 56 | "before_training_starts": before_training_starts, 57 | "inference_fn": inference_fn, 58 | "hook_kwargs": hook_kwargs, 59 | } 60 | 61 | def get_new_adapter(self, *args, **kwargs): 62 | return CDAN(**self.get_adapter_kwargs(*args, **kwargs)) 63 | 64 | def save(self, folder): 65 | super().save(folder) 66 | main_utils.save_this_file(__file__, folder) 67 | 68 | 69 | class CDANEConfig(CDANConfig): 70 | def get_adapter_kwargs(self, *args, **kwargs): 71 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 72 | all_kwargs["hook_kwargs"]["detach_entropy_reducer"] = True 73 | return all_kwargs 74 | 75 | def get_new_adapter(self, *args, **kwargs): 76 | return CDANE(**self.get_adapter_kwargs(*args, **kwargs)) 77 | 78 | 79 | class CDANEUConfig(CDANEConfig): 80 | def get_adapter_kwargs(self, *args, **kwargs): 81 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 82 | all_kwargs["hook_kwargs"]["detach_entropy_reducer"] = False 83 | return all_kwargs 84 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/rtn_config.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorch_adapt.adapters import RTN 3 | from pytorch_adapt.containers import Misc, Models, Optimizers 4 | from pytorch_adapt.datasets import utils as dataset_utils 5 | from pytorch_adapt.inference import rtn_full_fn 6 | from pytorch_adapt.layers import MMDLoss, PlusResidual, RandomizedDotProduct 7 | from pytorch_adapt.layers.utils import get_kernel_scales 8 | from pytorch_adapt.weighters import MeanWeighter 9 | 10 | from ..utils import main_utils 11 | from .base_config import BaseConfig 12 | 13 | 14 | class RTNConfig(BaseConfig): 15 | def get_models(self, dataset, *args, **kwargs): 16 | models, framework = super().get_models(dataset, *args, **kwargs) 17 | num_classes = dataset_utils.num_classes(dataset) 18 | models["residual_model"] = PlusResidual( 19 | torch.nn.Sequential( 20 | torch.nn.Linear(num_classes, num_classes), 21 | torch.nn.ReLU(), 22 | torch.nn.Linear(num_classes, num_classes), 23 | ) 24 | ) 25 | del models["D"] 26 | models["feature_combiner"] = RandomizedDotProduct( 27 | [self.feature_size, num_classes], self.feature_size 28 | ) 29 | return models, framework 30 | 31 | def get_adapter_kwargs( 32 | self, 33 | models, 34 | optimizers, 35 | before_training_starts, 36 | lr_multiplier, 37 | use_full_inference, 38 | **kwargs 39 | ): 40 | feature_combiner = models.pop("feature_combiner") 41 | models = Models(models) 42 | optimizers = Optimizers( 43 | optimizers, multipliers={"residual_model": lr_multiplier} 44 | ) 45 | misc = Misc({"feature_combiner": feature_combiner}) 46 | 47 | label_weight = self.optuna_trial.suggest_float("label_weight", 0, 1) 48 | 49 | confusion_weight = self.optuna_trial.suggest_float("confusion_weight", 0, 1) 50 | entropy_weight = self.optuna_trial.suggest_float("entropy_weight", 0, 1) 51 | weighter = MeanWeighter( 52 | weights={ 53 | "c_loss": label_weight, 54 | "features_confusion_loss": confusion_weight, 55 | "entropy_loss": entropy_weight, 56 | } 57 | ) 58 | 59 | exponent = self.optuna_trial.suggest_int("exponent", 1, 8) 60 | num_kernels = (exponent * 2) + 1 61 | kernel_scales = get_kernel_scales( 62 | low=-exponent, high=exponent, num_kernels=num_kernels 63 | ) 64 | hook_kwargs = { 65 | "weighter": weighter, 66 | "aligner_loss_fn": MMDLoss(kernel_scales=kernel_scales), 67 | } 68 | 69 | inference_fn = rtn_full_fn if use_full_inference else None 70 | 71 | return { 72 | "models": models, 73 | "optimizers": optimizers, 74 | "misc": misc, 75 | "before_training_starts": before_training_starts, 76 | "inference_fn": inference_fn, 77 | "hook_kwargs": hook_kwargs, 78 | } 79 | 80 | def get_new_adapter(self, *args, **kwargs): 81 | return RTN(**self.get_adapter_kwargs(*args, **kwargs)) 82 | 83 | def save(self, folder): 84 | super().save(folder) 85 | main_utils.save_this_file(__file__, folder) 86 | -------------------------------------------------------------------------------- /validator_tests/README.md: -------------------------------------------------------------------------------- 1 | ## powerful-benchmarker/validator_tests 2 | 3 | 4 | ### Order of operations 5 | 6 | 1. main.py (or run_validators.py) to compute scores 7 | 2. collect_dfs.py to gather all dataframe pkls into one dataframe pkl 8 | 3. process_df.py 9 | 4. eval_validators.py 10 | 11 | 12 | ### Common command line flags 13 | The following flags allow for filtering of experiment groups: 14 | | Command-line argument | Description | 15 | | - | - | 16 | |`--exp_groups` | A space delimited list of experiment group names. 17 | |`--exp_group_prefix` | Matches all experiment groups that start with this. 18 | |`--exp_group_suffix` | Matches all experiment groups that end with this. 19 | |`--exp_group_includes` | Matches all experiment groups that have this in their name. 20 | |`--exp_group_excludes` | Matches all experiment groups that do not have this in their name. 21 | 22 | These flags are available in the following scripts: 23 | 24 | - collect_dfs.py 25 | - create_plots.py 26 | - eval_validators.py 27 | - delete_pkls.py 28 | - run_validators.py 29 | 30 | --- 31 | ### main.py 32 | 33 | This runs a single validator configuration on a single experiment's trials. It will save a pkl file containing the validation scores, within each trial's folder. For example, the following command will compute micro-averaged accuracy on the source validation set for all trials within `/mnist_mnist_mnistm_fl6_Adam_lr1/dann`: 34 | 35 | ``` 36 | python validator_tests/main.py --exp_group mnist_mnist_mnistm_fl6_Adam_lr1 --exp_name dann \ 37 | --validator Accuracy --average=micro --split=src_val 38 | ``` 39 | 40 | --- 41 | ### run_validators.py 42 | 43 | This uses slurm to run a single validator with all of its configurations on multiple experiments. For example, the following will compute source and target accuracies on all 100 trials of the atdoc, dann, and mcc experiments. It will run 4 config/experiment combinations per slurm job: 44 | 45 | ``` 46 | python validator_tests/run_validators.py --slurm_config a100 --run --exp_names atdoc dann mcc --flags Accuracy \ 47 | --exp_per_slurm_job 4 --trials_per_exp 100 48 | ``` 49 | 50 | See [scripts/run.py](https://github.com/KevinMusgrave/powerful-benchmarker/blob/domain-adaptation/validator_tests/scripts/run.py), [scripts/mnist.sh](https://github.com/KevinMusgrave/powerful-benchmarker/blob/domain-adaptation/validator_tests/scripts/mnist.sh), [scripts/office31.sh](https://github.com/KevinMusgrave/powerful-benchmarker/blob/domain-adaptation/validator_tests/scripts/office31.sh), and [scripts/officehome.sh](https://github.com/KevinMusgrave/powerful-benchmarker/blob/domain-adaptation/validator_tests/scripts/officehome.sh) for examples. 51 | 52 | 53 | --- 54 | ### collect_dfs.py 55 | 56 | After computing validation scores and saving them to pkl files, you can gather them into larger files using `collect_dfs.py`. For example, this will gather all pkls under each experiment group that starts with "mnist", and save them as `all_dfs.pkl` within that same experiment group: 57 | 58 | ``` 59 | python collect_dfs.py --exp_group_prefix mnist 60 | ``` 61 | 62 | --- 63 | ### process_df.py 64 | This makes some modifications to `all_dfs.pkl`, like removing irrelevant column names. The new file will be `all_dfs_processed.pkl`, saved in the same folder as `all_dfs.pkl`: 65 | 66 | ``` 67 | python process_df.py --exp_group_prefix mnist 68 | ``` 69 | 70 | --- 71 | ### eval_validators.py 72 | The next step is to compute the weighted Spearman correlation and top-N accuracies. 73 | 74 | --- 75 | ### create_plots.py 76 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/aligner_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.adapters import Aligner 2 | from pytorch_adapt.containers import Models, Optimizers 3 | from pytorch_adapt.hooks import JointAlignerHook 4 | from pytorch_adapt.layers import CORALLoss, MMDLoss 5 | from pytorch_adapt.layers.utils import get_kernel_scales 6 | from pytorch_adapt.weighters import MeanWeighter 7 | 8 | from ..utils import main_utils 9 | from .base_config import BaseConfig 10 | 11 | 12 | class AlignerConfig(BaseConfig): 13 | def get_models(self, *args, **kwargs): 14 | models, framework = super().get_models(*args, **kwargs) 15 | del models["D"] 16 | return models, framework 17 | 18 | def get_adapter_kwargs( 19 | self, 20 | models, 21 | optimizers, 22 | before_training_starts, 23 | lr_multiplier, 24 | use_full_inference, 25 | **kwargs 26 | ): 27 | models = Models(models) 28 | optimizers = Optimizers(optimizers, multipliers={"C": lr_multiplier}) 29 | 30 | confusion_weight = self.optuna_trial.suggest_float("confusion_weight", 0, 1) 31 | label_weight = self.optuna_trial.suggest_float("label_weight", 0, 1) 32 | weighter = MeanWeighter( 33 | weights={ 34 | "features_confusion_loss": confusion_weight, 35 | "logits_confusion_loss": confusion_weight, 36 | "c_loss": label_weight, 37 | } 38 | ) 39 | hook_kwargs = {"weighter": weighter} 40 | return { 41 | "models": models, 42 | "optimizers": optimizers, 43 | "misc": None, 44 | "before_training_starts": before_training_starts, 45 | "hook_kwargs": hook_kwargs, 46 | } 47 | 48 | def get_new_adapter(self, *args, **kwargs): 49 | return Aligner(**self.get_adapter_kwargs(*args, **kwargs)) 50 | 51 | def save(self, folder): 52 | super().save(folder) 53 | main_utils.save_this_file(__file__, folder) 54 | 55 | 56 | class CORALConfig(AlignerConfig): 57 | def get_adapter_kwargs(self, *args, **kwargs): 58 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 59 | all_kwargs["hook_kwargs"].update({"loss_fn": CORALLoss(), "softmax": False}) 60 | return all_kwargs 61 | 62 | 63 | class MMDConfig(AlignerConfig): 64 | def get_adapter_kwargs(self, *args, **kwargs): 65 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 66 | exponent = self.optuna_trial.suggest_int("exponent", 1, 8) 67 | num_kernels = (exponent * 2) + 1 68 | kernel_scales = get_kernel_scales( 69 | low=-exponent, high=exponent, num_kernels=num_kernels 70 | ) 71 | all_kwargs["hook_kwargs"].update( 72 | { 73 | "loss_fn": MMDLoss(kernel_scales=kernel_scales), 74 | "softmax": True, 75 | } 76 | ) 77 | return all_kwargs 78 | 79 | 80 | class JMMDConfig(MMDConfig): 81 | def get_adapter_kwargs(self, *args, **kwargs): 82 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 83 | weighter = MeanWeighter( 84 | weights={ 85 | "joint_confusion_loss": self.optuna_trial.params["confusion_weight"], 86 | "c_loss": self.optuna_trial.params["label_weight"], 87 | } 88 | ) 89 | aligner_hook = JointAlignerHook( 90 | loss_fn=all_kwargs["hook_kwargs"].pop("loss_fn") 91 | ) 92 | all_kwargs["hook_kwargs"].update( 93 | {"weighter": weighter, "aligner_hook": aligner_hook} 94 | ) 95 | return all_kwargs 96 | -------------------------------------------------------------------------------- /validator_tests/configs/knn_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.validators import KNNValidator, TargetKNNValidator 2 | from pytorch_metric_learning.distances import LpDistance 3 | from pytorch_metric_learning.utils.inference import CustomKNN 4 | 5 | from .base_config import ( 6 | BaseConfig, 7 | get_full_split_name, 8 | use_labels_and_logits, 9 | use_src_and_target, 10 | ) 11 | 12 | 13 | class KNN(BaseConfig): 14 | def __init__(self, config): 15 | super().__init__(config) 16 | self.validator_args["p"] = float(self.validator_args["p"]) 17 | self.validator_args["normalize"] = bool(int(self.validator_args["normalize"])) 18 | self.set_k() 19 | self.set_layer() 20 | self.src_split_name = get_full_split_name("src", self.split) 21 | self.target_split_name = get_full_split_name("target", self.split) 22 | 23 | knn_func = CustomKNN( 24 | LpDistance( 25 | normalize_embeddings=self.validator_args["normalize"], 26 | p=self.validator_args["p"], 27 | ), 28 | batch_size=512, 29 | ) 30 | 31 | self.validator = self.create_validator(knn_func) 32 | 33 | def score(self, x, exp_config, device): 34 | return use_src_and_target( 35 | x, 36 | device, 37 | self.validator, 38 | self.src_split_name, 39 | self.target_split_name, 40 | self.layer, 41 | ) 42 | 43 | def create_validator(self, knn_func): 44 | batch_size = None if self.validator_args["k"] <= 1000 else 256 45 | return KNNValidator( 46 | key_map={ 47 | self.src_split_name: "src_train", 48 | self.target_split_name: "target_train", 49 | }, 50 | layer=self.layer, 51 | knn_func=knn_func, 52 | k=self.validator_args["k"], 53 | metric="mean_average_precision", 54 | batch_size=batch_size, 55 | ) 56 | 57 | def set_k(self): 58 | self.validator_args["k"] = int(self.validator_args["k"]) 59 | 60 | def set_layer(self): 61 | self.layer = self.validator_args["layer"] 62 | 63 | def expected_keys(self): 64 | return {"k", "p", "normalize", "layer", "split"} 65 | 66 | 67 | class TargetKNN(KNN): 68 | def score(self, x, exp_config, device): 69 | return use_labels_and_logits( 70 | x, 71 | device, 72 | self.validator, 73 | self.src_split_name, 74 | self.target_split_name, 75 | self.layer, 76 | ) 77 | 78 | def create_validator(self, knn_func): 79 | self.validator_args["T_in_ref"] = bool(int(self.validator_args["T_in_ref"])) 80 | batch_size = None if self.validator_args["k"] <= 1000 else 256 81 | return TargetKNNValidator( 82 | key_map={ 83 | self.src_split_name: "src_train", 84 | self.target_split_name: "target_train", 85 | }, 86 | layer=self.layer, 87 | knn_func=knn_func, 88 | k=self.validator_args["k"], 89 | metric="mean_average_precision", 90 | batch_size=batch_size, 91 | add_target_to_ref=self.validator_args["T_in_ref"], 92 | ) 93 | 94 | def set_layer(self): 95 | self.layer = "features" 96 | 97 | def expected_keys(self): 98 | return {"k", "p", "normalize", "T_in_ref", "split"} 99 | 100 | 101 | class TargetKNNLogits(TargetKNN): 102 | def set_layer(self): 103 | self.layer = "logits" 104 | -------------------------------------------------------------------------------- /powerful_benchmarker/utils/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pathlib 4 | import subprocess 5 | 6 | import yaml 7 | 8 | 9 | def convert_unknown_args(unknown_args): 10 | args = {} 11 | for s in unknown_args: 12 | if s == "": 13 | continue 14 | k, v = s.split("=") 15 | args[k.lstrip("--")] = v 16 | return args 17 | 18 | 19 | def create_slurm_args(args, other_args, folder): 20 | slurm_config_file = os.path.join( 21 | folder, "slurm_configs", f"{args.slurm_config}.yaml" 22 | ) 23 | 24 | with open(slurm_config_file, "r") as f: 25 | slurm_args = yaml.safe_load(f) 26 | 27 | slurm_args.update(convert_unknown_args(other_args)) 28 | 29 | return slurm_args 30 | 31 | 32 | def rotate(x, n): 33 | return x[n:] + x[:n] 34 | 35 | 36 | def get_yaml_config_folder(): 37 | return os.path.join("powerful_benchmarker", "yaml_configs") 38 | 39 | 40 | def get_yaml_config_path(category, name): 41 | return os.path.join(get_yaml_config_folder(), category, f"{name}.yaml") 42 | 43 | 44 | def append_jobid_to_file(jobid, jobname, filename): 45 | print(f"running job_id = {jobid}") 46 | if os.path.isfile(filename): 47 | with open(filename, "r") as f: 48 | jobids = json.load(f) 49 | else: 50 | jobids = {} 51 | jobids[jobid] = jobname 52 | with open(filename, "w") as f: 53 | json.dump(jobids, f, indent=2) 54 | 55 | 56 | def kill_all_jobs(exp_folder, jobids_file): 57 | all_jobids_filename = os.path.join(exp_folder, jobids_file) 58 | if not os.path.isfile(all_jobids_filename): 59 | print(f"{all_jobids_filename} file not found, skipping") 60 | return 61 | 62 | filetype = pathlib.Path(all_jobids_filename).suffix 63 | 64 | if filetype == ".json": 65 | with open(all_jobids_filename, "r") as f: 66 | jobids = json.load(f) 67 | jobids = " ".join(list(jobids.keys())) 68 | 69 | else: 70 | with open(all_jobids_filename, "r") as f: 71 | jobids = " ".join([line.rstrip("\n") for line in f]) 72 | 73 | command = f"scancel {jobids}" 74 | print("killing slurm jobs") 75 | subprocess.run(command.split(" ")) 76 | print(f"deleting {jobids_file}") 77 | os.remove(all_jobids_filename) 78 | 79 | 80 | def jobs_that_are_still_running(exp_folder, jobids_file): 81 | x = subprocess.run("squeue --nohead --format %F".split(" "), capture_output=True) 82 | jobid_list = x.stdout.decode("utf-8").split("\n") 83 | 84 | all_jobids_filename = os.path.join(exp_folder, jobids_file) 85 | 86 | if os.path.isfile(all_jobids_filename): 87 | with open(all_jobids_filename, "r") as f: 88 | y = list(json.load(f).keys()) 89 | 90 | return set(y).intersection(jobid_list) 91 | return {} 92 | 93 | 94 | def create_exp_group_name( 95 | dataset, 96 | src_domains, 97 | target_domains, 98 | feature_layers, 99 | optimizers, 100 | lr_multipliers, 101 | validator=None, 102 | ): 103 | src_domains = "_".join(src_domains) 104 | target_domains = "_".join(target_domains) 105 | exp_group_name = f"{dataset}_{src_domains}_{target_domains}" 106 | if validator: 107 | exp_group_name += f"_{validator}" 108 | feature_layers = "".join(f"fl{str(x)}" for x in sorted(feature_layers)) 109 | optimizers = "".join(x for x in sorted(optimizers)) 110 | lr_multipliers = "".join( 111 | f"lr{f'{x:.1f}' if x < 1 else int(x)}" for x in sorted(lr_multipliers) 112 | ) 113 | exp_group_name += f"_{feature_layers}_{optimizers}_{lr_multipliers}" 114 | return exp_group_name 115 | -------------------------------------------------------------------------------- /validator_tests/save_resilience_to_noise_dfs.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | from collections import defaultdict 5 | 6 | import numpy as np 7 | import pandas as pd 8 | from pytorch_adapt.utils import common_functions as c_f 9 | from scipy.stats import spearmanr 10 | 11 | sys.path.insert(0, ".") 12 | 13 | from powerful_benchmarker.utils.constants import add_default_args 14 | from validator_tests.eval_validators import _get_correlation, group_by_task_validator 15 | from validator_tests.plot_ranks_vs_acc import get_global_ranks 16 | from validator_tests.utils import create_main 17 | from validator_tests.utils.constants import TARGET_ACCURACY, add_exp_group_args 18 | from validator_tests.utils.df_utils import get_name_from_df 19 | 20 | 21 | def add_noise(original_df, scale): 22 | df = original_df.copy() 23 | df.groupby(["adapter", "trial_num", "epoch"]).size().reset_index().rename( 24 | columns={0: "count"} 25 | ) 26 | df = df.rename(columns={"count": "noise"}) 27 | df["noise"] = np.random.normal(scale=scale, size=(len(df))) 28 | df = original_df.merge(df) 29 | df[TARGET_ACCURACY] = df[TARGET_ACCURACY] + df["noise"] 30 | df[TARGET_ACCURACY] = df[TARGET_ACCURACY].clip(lower=0, upper=1) 31 | return df 32 | 33 | 34 | def get_correlation(df, per_adapter): 35 | return _get_correlation( 36 | df.copy(), per_adapter=per_adapter, src_threshold=0.0, name="weighted_spearman" 37 | ) 38 | 39 | 40 | def get_acc(df, per_adapter, N): 41 | df = get_global_ranks(df, rank_by="score", per_adapter=per_adapter) 42 | df = df[df["rank"] <= N] 43 | df[f"top_{N}_acc"] = df.groupby(group_by_task_validator(per_adapter))[ 44 | TARGET_ACCURACY 45 | ].transform("mean") 46 | keep = ["validator", "validator_args", f"top_{N}_acc"] 47 | if per_adapter: 48 | keep += ["adapter"] 49 | return df[keep].drop_duplicates() 50 | 51 | 52 | def save_df(output_folder, df): 53 | accs = {} 54 | corr = get_correlation(df, per_adapter=False) 55 | Ns = [1, 5, 10, 50, 100, 500, 1000, 5000] 56 | for N in Ns: 57 | accs[N] = get_acc(df, per_adapter=False, N=N) 58 | 59 | s = defaultdict(list) 60 | for scale in np.linspace(0, 0.2, 21): 61 | s["Noise Standard Deviation"].append(scale) 62 | df_with_noise = add_noise(df, scale) 63 | corr_with_noise = get_correlation(df_with_noise, per_adapter=False) 64 | s["Weighted Spearman Correlation"].append( 65 | spearmanr( 66 | corr_with_noise["weighted_spearman"].values, 67 | corr["weighted_spearman"].values, 68 | ).correlation 69 | ) 70 | for N in Ns: 71 | acc_with_noise = get_acc(df_with_noise, per_adapter=False, N=N) 72 | s[f"Top {N} Accuracy"].append( 73 | spearmanr( 74 | acc_with_noise[f"top_{N}_acc"].values, 75 | accs[N][f"top_{N}_acc"].values, 76 | ).correlation 77 | ) 78 | 79 | print(s) 80 | 81 | sdf = pd.DataFrame.from_dict(s) 82 | 83 | output_folder = os.path.join( 84 | output_folder, get_name_from_df(df, assert_one_task=True) 85 | ) 86 | c_f.makedir_if_not_there(output_folder) 87 | sdf.to_pickle(os.path.join(output_folder, "df.pkl")) 88 | 89 | 90 | if __name__ == "__main__": 91 | parser = argparse.ArgumentParser(allow_abbrev=False) 92 | add_default_args(parser, ["exp_folder"]) 93 | add_exp_group_args(parser) 94 | parser.add_argument( 95 | "--output_folder", type=str, default="plots/resilience_to_noise" 96 | ) 97 | create_main.add_main_args(parser) 98 | args = parser.parse_args() 99 | create_main.main(args, save_df, save_df) 100 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/mcd_config.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from pytorch_adapt.adapters import MCD 4 | from pytorch_adapt.containers import Models, Optimizers 5 | from pytorch_adapt.inference import mcd_full_fn 6 | from pytorch_adapt.layers import ( 7 | MCDLoss, 8 | MultipleModels, 9 | SlicedWasserstein, 10 | StochasticLinear, 11 | ) 12 | from pytorch_adapt.utils.common_functions import reinit 13 | from pytorch_adapt.weighters import MeanWeighter 14 | 15 | from ..utils import main_utils 16 | from .base_config import BaseConfig 17 | 18 | 19 | class MCDConfig(BaseConfig): 20 | def get_models(self, dataset, *args, **kwargs): 21 | models, framework = super().get_models(dataset, *args, **kwargs) 22 | c1 = reinit(copy.deepcopy(models["C"])) 23 | models["C"] = MultipleModels(models["C"], c1) 24 | del models["D"] 25 | return models, framework 26 | 27 | def get_adapter_kwargs( 28 | self, 29 | models, 30 | optimizers, 31 | before_training_starts, 32 | lr_multiplier, 33 | use_full_inference, 34 | **kwargs 35 | ): 36 | models = Models(models) 37 | optimizers = Optimizers( 38 | optimizers, 39 | multipliers={"C": lr_multiplier}, 40 | ) 41 | 42 | num_repeat = self.optuna_trial.suggest_int("num_repeat", 1, 10) 43 | 44 | label_weight = self.optuna_trial.suggest_float("label_weight", 0, 1) 45 | discrepancy_weight = self.optuna_trial.suggest_float("discrepancy_weight", 0, 1) 46 | 47 | x_weighter = MeanWeighter(scale=label_weight) 48 | y_weighter = MeanWeighter( 49 | weights={ 50 | "c_loss0": label_weight, 51 | "c_loss1": label_weight, 52 | "discrepancy_loss": discrepancy_weight, 53 | } 54 | ) 55 | z_weighter = MeanWeighter(scale=discrepancy_weight) 56 | hook_kwargs = { 57 | "repeat": num_repeat, 58 | "x_weighter": x_weighter, 59 | "y_weighter": y_weighter, 60 | "z_weighter": z_weighter, 61 | } 62 | inference_fn = mcd_full_fn if use_full_inference else None 63 | 64 | return { 65 | "models": models, 66 | "optimizers": optimizers, 67 | "misc": None, 68 | "before_training_starts": before_training_starts, 69 | "hook_kwargs": hook_kwargs, 70 | "inference_fn": inference_fn, 71 | } 72 | 73 | def get_new_adapter(self, *args, **kwargs): 74 | return MCD(**self.get_adapter_kwargs(*args, **kwargs)) 75 | 76 | def save(self, folder): 77 | super().save(folder) 78 | main_utils.save_this_file(__file__, folder) 79 | 80 | 81 | class STARConfig(MCDConfig): 82 | def get_models(self, dataset, *args, **kwargs): 83 | models, framework = super().get_models(dataset, *args, **kwargs) 84 | last_linear = models["C"].models[0].net[-1] 85 | in_features = last_linear.in_features 86 | out_features = last_linear.out_features 87 | new_linear = StochasticLinear(in_features, out_features) 88 | new_linear.weight_mean.data = last_linear.weight.data.t() 89 | new_linear.bias_mean.data = last_linear.bias.data.t() 90 | models["C"].models[0].net[-1] = new_linear 91 | models["C"].models[1] = models["C"].models[0] 92 | return models, framework 93 | 94 | 95 | class SWDConfig(MCDConfig): 96 | def get_adapter_kwargs(self, *args, **kwargs): 97 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 98 | m = 128 99 | all_kwargs["hook_kwargs"]["discrepancy_loss_fn"] = MCDLoss( 100 | dist_fn=SlicedWasserstein(m=m) 101 | ) 102 | return all_kwargs 103 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/gvb_config.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorch_adapt.adapters import GVB, GVBE 3 | from pytorch_adapt.containers import Models, Optimizers 4 | from pytorch_adapt.inference import gvb_full_fn 5 | from pytorch_adapt.layers import ModelWithBridge 6 | from pytorch_adapt.models import Discriminator 7 | from pytorch_adapt.weighters import MeanWeighter 8 | 9 | from ..utils import main_utils 10 | from .base_config import BaseConfig 11 | 12 | 13 | class Bridge(torch.nn.Module): 14 | def __init__(self, in_f, out_f): 15 | super().__init__() 16 | self.fc = torch.nn.Linear(in_f, out_f) 17 | 18 | def forward(self, x): 19 | return self.fc(x).squeeze(1) 20 | 21 | 22 | class GVBConfig(BaseConfig): 23 | def get_models(self, *args, **kwargs): 24 | models, framework = super().get_models(*args, **kwargs) 25 | num_classes = kwargs["num_classes"] 26 | h = models["D"].h 27 | models["D"] = Discriminator(in_size=num_classes, h=h) 28 | models["D"] = ModelWithBridge(models["D"], Bridge(num_classes, 1)) 29 | models["C"] = ModelWithBridge( 30 | models["C"], Bridge(self.feature_size, num_classes) 31 | ) 32 | return models, framework 33 | 34 | def get_adapter_kwargs( 35 | self, 36 | models, 37 | optimizers, 38 | before_training_starts, 39 | lr_multiplier, 40 | use_full_inference, 41 | **kwargs 42 | ): 43 | models = Models(models) 44 | optimizers = Optimizers( 45 | optimizers, 46 | multipliers={ 47 | "C": lr_multiplier, 48 | "D": lr_multiplier, 49 | }, 50 | ) 51 | domain_weight = self.optuna_trial.suggest_float("domain_weight", 0, 1) 52 | bridge_G_weight = self.optuna_trial.suggest_float("bridge_G_weight", 0, 1) 53 | bridge_D_weight = self.optuna_trial.suggest_float("bridge_D_weight", 0, 1) 54 | weighter = MeanWeighter( 55 | weights={ 56 | "src_domain_loss": domain_weight, 57 | "target_domain_loss": domain_weight, 58 | "g_src_bridge_loss": bridge_G_weight, 59 | "d_src_bridge_loss": bridge_D_weight, 60 | "g_target_bridge_loss": bridge_G_weight, 61 | "d_target_bridge_loss": bridge_D_weight, 62 | } 63 | ) 64 | 65 | grl_weight = self.optuna_trial.suggest_float("grl_weight", 0.1, 10, log=True) 66 | hook_kwargs = {"weighter": weighter, "gradient_reversal_weight": grl_weight} 67 | inference_fn = gvb_full_fn if use_full_inference else None 68 | 69 | return { 70 | "models": models, 71 | "optimizers": optimizers, 72 | "misc": None, 73 | "before_training_starts": before_training_starts, 74 | "inference_fn": inference_fn, 75 | "hook_kwargs": hook_kwargs, 76 | } 77 | 78 | def get_new_adapter(self, *args, **kwargs): 79 | return GVB(**self.get_adapter_kwargs(*args, **kwargs)) 80 | 81 | def save(self, folder): 82 | super().save(folder) 83 | main_utils.save_this_file(__file__, folder) 84 | 85 | 86 | class GVBEConfig(GVBConfig): 87 | def get_adapter_kwargs(self, *args, **kwargs): 88 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 89 | all_kwargs["hook_kwargs"]["detach_entropy_reducer"] = True 90 | return all_kwargs 91 | 92 | def get_new_adapter(self, *args, **kwargs): 93 | return GVBE(**self.get_adapter_kwargs(*args, **kwargs)) 94 | 95 | 96 | class GVBEUConfig(GVBEConfig): 97 | def get_adapter_kwargs(self, *args, **kwargs): 98 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 99 | all_kwargs["hook_kwargs"]["detach_entropy_reducer"] = False 100 | return all_kwargs 101 | -------------------------------------------------------------------------------- /validator_tests/plotly_test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import sys 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import plotly.graph_objects as go 8 | 9 | sys.path.insert(0, ".") 10 | 11 | from powerful_benchmarker.utils.constants import add_default_args 12 | from validator_tests.eval_validators import group_by_task_validator 13 | from validator_tests.utils import create_main 14 | from validator_tests.utils.constants import TARGET_ACCURACY, add_exp_group_args 15 | from validator_tests.utils.df_utils import validator_str 16 | 17 | 18 | def get_best_accuracy_per_adapter(df): 19 | rank_by = "score" 20 | groupby = group_by_task_validator(per_adapter=True) 21 | groupby_with_trial_params = groupby + ["trial_params"] 22 | 23 | # best score per trial param 24 | ranked = df.groupby(groupby_with_trial_params)[rank_by].rank( 25 | method="min", ascending=False 26 | ) 27 | to_save = df[ranked <= 1] 28 | 29 | # remove duplicate scores for a trial by taking the earliest epoch 30 | return to_save.sort_values(by=["epoch"]).drop_duplicates( 31 | subset=groupby_with_trial_params 32 | ) 33 | 34 | 35 | # https://stackoverflow.com/a/64146570 36 | def add_dummy_validator_column(df): 37 | dfg = pd.DataFrame({"validator": df["validator"].unique()}) 38 | dfg["validator_index"] = dfg.index 39 | return pd.merge(df, dfg, on="validator", how="left") 40 | 41 | 42 | def create_plot(original_df, validator_name, validator_args): 43 | true_max = original_df[TARGET_ACCURACY].max() 44 | true_min = original_df[TARGET_ACCURACY].min() 45 | df = original_df.copy() 46 | df = df[ 47 | (df["adapter"] == "DANNConfig") 48 | & (df["validator"] == validator_name) 49 | & (df["validator_args"] == validator_args) 50 | ] 51 | if len(df) == 0: 52 | return 53 | df = get_best_accuracy_per_adapter(df) 54 | 55 | applied_df = df.apply( 56 | lambda row: json.loads(row.trial_params), axis="columns", result_type="expand" 57 | ) 58 | df = pd.concat([df, applied_df], axis="columns") 59 | 60 | trial_param_keys = json.loads(df["trial_params"].iloc[0]).keys() 61 | trial_param_dimensions = [ 62 | dict( 63 | label="log_lr", 64 | values=np.log10(df[x].values), 65 | tickvals=np.linspace( 66 | np.log10(df[x].min()), np.log10(df[x].max()), 10, endpoint=True 67 | ), 68 | ) 69 | if x == "lr" 70 | else dict(label=x, values=df[x]) 71 | for x in trial_param_keys 72 | ] 73 | 74 | fig = go.Figure( 75 | data=go.Parcoords( 76 | line=dict( 77 | color=df[TARGET_ACCURACY], 78 | colorscale="viridis", 79 | showscale=True, 80 | cmin=true_min, 81 | cmax=true_max, 82 | ), 83 | dimensions=list( 84 | [ 85 | *trial_param_dimensions, 86 | dict(label="Feature Layer", values=df["feature_layer"]), 87 | ] 88 | ), 89 | ) 90 | ) 91 | fig.write_html(f"plotly_test_{validator_str(validator_name, validator_args)}.html") 92 | 93 | 94 | def create_subsets(output_folder, original_df): 95 | for validator_name in original_df["validator"].unique(): 96 | curr_df = original_df[original_df["validator"] == validator_name] 97 | for validator_args in curr_df["validator_args"].unique(): 98 | print(validator_name, validator_args) 99 | create_plot(original_df, validator_name, validator_args) 100 | 101 | 102 | if __name__ == "__main__": 103 | parser = argparse.ArgumentParser(allow_abbrev=False) 104 | add_default_args(parser, ["exp_folder"]) 105 | add_exp_group_args(parser) 106 | create_main.add_main_args(parser) 107 | args = parser.parse_args() 108 | create_main.main(args, create_subsets, create_subsets) 109 | -------------------------------------------------------------------------------- /validator_tests/configs/cluster_config.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | from pytorch_adapt.validators import ClassClusterValidator, KNNValidator 3 | from sklearn.cluster import KMeans 4 | from sklearn.metrics import adjusted_mutual_info_score, silhouette_score 5 | 6 | from .base_config import BaseConfig, get_full_split_name, use_labels_and_logits 7 | from .knn_config import KNN 8 | 9 | 10 | def kmeans_func(normalize, p): 11 | def fn(x, n_clusters): 12 | if normalize: 13 | x = F.normalize(x, dim=1, p=p) 14 | return KMeans(n_clusters=n_clusters).fit_predict(x.cpu().numpy()) 15 | 16 | return fn 17 | 18 | 19 | class DomainCluster(KNN): 20 | def create_validator(self, knn_func): 21 | return KNNValidator( 22 | key_map={ 23 | self.src_split_name: "src_train", 24 | self.target_split_name: "target_train", 25 | }, 26 | layer=self.layer, 27 | knn_func=knn_func, 28 | kmeans_func=kmeans_func( 29 | self.validator_args["normalize"], self.validator_args["p"] 30 | ), 31 | metric="AMI", 32 | ) 33 | 34 | def set_k(self): 35 | pass 36 | 37 | def expected_keys(self): 38 | return {"p", "normalize", "layer", "split"} 39 | 40 | 41 | def feat_normalizer_fn(normalize, p): 42 | def fn(x): 43 | if normalize: 44 | return F.normalize(x, dim=1, p=p) 45 | return x 46 | 47 | return fn 48 | 49 | 50 | class ClassAMI(BaseConfig): 51 | def __init__(self, config): 52 | super().__init__(config) 53 | self.layer = self.validator_args["layer"] 54 | self.validator_args["p"] = float(self.validator_args["p"]) 55 | self.validator_args["with_src"] = bool(int(self.validator_args["with_src"])) 56 | self.validator_args["normalize"] = bool(int(self.validator_args["normalize"])) 57 | self.src_split_name = get_full_split_name("src", self.split) 58 | self.target_split_name = get_full_split_name("target", self.split) 59 | self.create_validator() 60 | 61 | def create_validator(self): 62 | score_fn, score_fn_type = self.get_score_fn() 63 | self.validator = ClassClusterValidator( 64 | key_map={ 65 | self.src_split_name: "src_train", 66 | self.target_split_name: "target_train", 67 | }, 68 | layer=self.layer, 69 | score_fn=score_fn, 70 | score_fn_type=score_fn_type, 71 | with_src=self.validator_args["with_src"], 72 | pca_size=None, 73 | centroid_init=self.get_centroid_init(), 74 | feat_normalizer=feat_normalizer_fn( 75 | self.validator_args["normalize"], self.validator_args["p"] 76 | ), 77 | ) 78 | 79 | def get_score_fn(self): 80 | return adjusted_mutual_info_score, "labels" 81 | 82 | def get_centroid_init(self): 83 | return None 84 | 85 | def score(self, x, exp_config, device): 86 | return use_labels_and_logits( 87 | x, 88 | device, 89 | self.validator, 90 | self.src_split_name, 91 | self.target_split_name, 92 | self.layer, 93 | ) 94 | 95 | def expected_keys(self): 96 | return {"p", "with_src", "normalize", "layer", "split"} 97 | 98 | 99 | class ClassAMICentroidInit(ClassAMI): 100 | def get_centroid_init(self): 101 | return "label_centers" 102 | 103 | 104 | class ClassSS(ClassAMI): 105 | def get_score_fn(self): 106 | return silhouette_score, "features" 107 | 108 | 109 | class ClassSSCentroidInit(ClassSS): 110 | def get_centroid_init(self): 111 | return "label_centers" 112 | 113 | def score(self, *args, **kwargs): 114 | try: 115 | return super().score(*args, **kwargs) 116 | except ValueError as e: 117 | if ( 118 | "Number of labels is 1. Valid values are 2 to n_samples - 1 (inclusive)" 119 | in str(e) 120 | ): 121 | return float("nan") 122 | raise 123 | -------------------------------------------------------------------------------- /validator_tests/utils/derive.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from .df_utils import ( 4 | drop_validator_cols, 5 | exp_specific_columns, 6 | remove_arg_from_validator_args, 7 | ) 8 | 9 | 10 | def add_derived_scores(df): 11 | for x in [ 12 | add_IM, 13 | # add_NegSND, 14 | add_BNMSummed, 15 | add_BNMSummedSrcVal, 16 | add_BSPSummed, 17 | add_EntropySummed, 18 | add_EntropySummedSrcVal, 19 | add_DiversitySummed, 20 | add_IMSummed, 21 | add_IMSummedSrcVal, 22 | ]: 23 | df = x(df) 24 | return df 25 | 26 | 27 | def add_IM(df): 28 | e = df[df["validator"] == "Entropy"] 29 | d = df[df["validator"] == "Diversity"] 30 | 31 | if len(e) == 0 or len(d) == 0: 32 | return df 33 | 34 | e = drop_validator_cols(e, drop_validator_args=False).rename( 35 | columns={"score": "entropy_score"} 36 | ) 37 | d = drop_validator_cols(d, drop_validator_args=False).rename( 38 | columns={"score": "diversity_score"} 39 | ) 40 | im = e.merge( 41 | d, on=exp_specific_columns(e, exclude=["entropy_score", "diversity_score"]) 42 | ) 43 | 44 | im = im.assign( 45 | score=im["entropy_score"] + im["diversity_score"], 46 | validator="IM", 47 | ) 48 | im = im.drop(columns=["entropy_score", "diversity_score"]) 49 | 50 | return pd.concat([df, im], axis=0, ignore_index=True) 51 | 52 | 53 | def add_NegSND(df): 54 | x = df[df["validator"] == "SND"] 55 | if len(x) == 0: 56 | return df 57 | x = drop_validator_cols(x, drop_validator_args=False).rename( 58 | columns={"score": "SND_score"} 59 | ) 60 | x = x.assign(score=-x["SND_score"], validator="NegSND") 61 | x = x.drop(columns=["SND_score"]) 62 | return pd.concat([df, x], axis=0, ignore_index=True) 63 | 64 | 65 | def _add_src_and_target(df, validator_name, src_split="train", new_name=None): 66 | x = df[df["validator"] == validator_name] 67 | src = x[x["validator_args"].str.contains(f'"split": "src_{src_split}"')] 68 | target = x[x["validator_args"].str.contains('"split": "target_train"')] 69 | 70 | if len(src) == 0 or len(target) == 0: 71 | return df 72 | 73 | src_score_name = f"src_{validator_name}_score" 74 | target_score_name = f"target_{validator_name}_score" 75 | 76 | src = drop_validator_cols(src, drop_validator_args=False).rename( 77 | columns={"score": src_score_name} 78 | ) 79 | target = drop_validator_cols(target, drop_validator_args=False).rename( 80 | columns={"score": target_score_name} 81 | ) 82 | 83 | src = remove_arg_from_validator_args(src, ["split"]) 84 | target = remove_arg_from_validator_args(target, ["split"]) 85 | 86 | summed = src.merge( 87 | target, 88 | on=exp_specific_columns(src, exclude=[src_score_name, target_score_name]), 89 | ) 90 | if new_name is None: 91 | new_name = f"{validator_name}Summed" 92 | 93 | summed = summed.assign( 94 | score=summed[src_score_name] + summed[target_score_name], 95 | validator=new_name, 96 | ) 97 | summed = summed.drop(columns=[src_score_name, target_score_name]) 98 | return pd.concat([df, summed], axis=0, ignore_index=True) 99 | 100 | 101 | def add_BNMSummed(df): 102 | return _add_src_and_target(df, "BNM") 103 | 104 | 105 | def add_BNMSummedSrcVal(df): 106 | return _add_src_and_target(df, "BNM", src_split="val", new_name="BNMSummedSrcVal") 107 | 108 | 109 | def add_BSPSummed(df): 110 | return _add_src_and_target(df, "BSP") 111 | 112 | 113 | def add_EntropySummed(df): 114 | return _add_src_and_target(df, "Entropy") 115 | 116 | 117 | def add_EntropySummedSrcVal(df): 118 | return _add_src_and_target( 119 | df, "Entropy", src_split="val", new_name="EntropySummedSrcVal" 120 | ) 121 | 122 | 123 | def add_DiversitySummed(df): 124 | return _add_src_and_target(df, "Diversity") 125 | 126 | 127 | def add_IMSummed(df): 128 | return _add_src_and_target(df, "IM") 129 | 130 | 131 | def add_IMSummedSrcVal(df): 132 | return _add_src_and_target(df, "IM", src_split="val", new_name="IMSummedSrcVal") 133 | -------------------------------------------------------------------------------- /latex/table_creator.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pandas as pd 4 | from pytorch_adapt.utils import common_functions as c_f 5 | 6 | from latex.color_map_tags import create_color_map_tags, get_tags_dict 7 | from latex.utils import resizebox 8 | from validator_tests.utils import utils 9 | from validator_tests.utils.df_utils import get_name_from_exp_groups 10 | 11 | 12 | def maybe_set_to_null_fn(x): 13 | if x is None: 14 | return lambda _: None 15 | return x 16 | 17 | 18 | def save_to_latex( 19 | df, 20 | folder, 21 | filename, 22 | color_map_tag_kwargs, 23 | add_resizebox, 24 | highlight_max=True, 25 | highlight_min=False, 26 | highlight_max_subset=None, 27 | highlight_min_subset=None, 28 | final_str_hook=None, 29 | **kwargs, 30 | ): 31 | if len(df.columns) == 3 and "Mean" in df.columns and "Std" in df.columns: 32 | df = df.drop(columns=["Mean", "Std"]) 33 | 34 | highlight_max_subset = maybe_set_to_null_fn(highlight_max_subset) 35 | highlight_min_subset = maybe_set_to_null_fn(highlight_min_subset) 36 | 37 | c_f.makedir_if_not_there(folder) 38 | tags_dict, color_map_tags = None, "" 39 | if color_map_tag_kwargs: 40 | color_map_tags = create_color_map_tags(df, **color_map_tag_kwargs) 41 | tags_dict = get_tags_dict(color_map_tag_kwargs["tag_prefix"], df.columns.values) 42 | 43 | df_style = df.style 44 | if highlight_max: 45 | df_style = df_style.highlight_max( 46 | subset=highlight_max_subset(df), props="textbf:--rwrap" 47 | ) 48 | if highlight_min: 49 | df_style = df_style.highlight_min( 50 | subset=highlight_min_subset(df), props="textbf:--rwrap" 51 | ) 52 | latex_str = df_style.format( 53 | tags_dict, 54 | escape="latex", 55 | na_rep="-", 56 | ).to_latex(hrules=True, position_float="centering", **kwargs) 57 | full_path = os.path.join(folder, f"{filename}.tex") 58 | 59 | if color_map_tags: 60 | newlines = "\n" * 10 61 | latex_str = f"{color_map_tags}{newlines}{latex_str}" 62 | if add_resizebox: 63 | latex_str = resizebox(latex_str) 64 | if final_str_hook: 65 | latex_str = final_str_hook(latex_str) 66 | with open(full_path, "w") as text_file: 67 | text_file.write(latex_str) 68 | 69 | 70 | def table_creator( 71 | args, 72 | input_folder, 73 | output_folder, 74 | basename, 75 | preprocess_df, 76 | postprocess_df, 77 | color_map_tag_kwargs=None, 78 | add_resizebox=False, 79 | do_save_to_latex=True, 80 | caption_hook=None, 81 | label_prefix=None, 82 | exp_groups=None, 83 | **kwargs, 84 | ): 85 | exp_groups = c_f.default(exp_groups, utils.get_exp_groups, [args, input_folder]) 86 | df = [] 87 | for e in exp_groups: 88 | filename = os.path.join(input_folder, e, f"{basename}.pkl") 89 | curr_df = pd.read_pickle(filename) 90 | curr_df = preprocess_df(curr_df) 91 | df.append(curr_df) 92 | 93 | df = postprocess_df(df) 94 | output_folder = os.path.join(output_folder, get_name_from_exp_groups(exp_groups)) 95 | # df.to_csv(os.path.join(output_folder, f"{basename}.csv")) 96 | if do_save_to_latex: 97 | if isinstance(df, dict): 98 | original_caption = kwargs.pop("caption", None) 99 | for k, x in df.items(): 100 | curr_basename = f"{basename}_{k}" 101 | if caption_hook: 102 | caption = caption_hook(original_caption, k) 103 | save_to_latex( 104 | x, 105 | output_folder, 106 | curr_basename, 107 | color_map_tag_kwargs, 108 | add_resizebox, 109 | label=f"{label_prefix}{curr_basename}", 110 | caption=caption, 111 | **kwargs, 112 | ) 113 | else: 114 | save_to_latex( 115 | df, 116 | output_folder, 117 | basename, 118 | color_map_tag_kwargs, 119 | add_resizebox, 120 | label=f"{label_prefix}{basename}", 121 | **kwargs, 122 | ) 123 | return df, output_folder 124 | -------------------------------------------------------------------------------- /validator_tests/synthetic_correlation_example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import seaborn as sns 8 | from pytorch_adapt.utils import common_functions as c_f 9 | from scipy.stats import spearmanr 10 | 11 | sys.path.insert(0, ".") 12 | 13 | from validator_tests.utils.plot_val_vs_acc import scatter_plot 14 | from validator_tests.utils.weighted_spearman import weighted_spearman 15 | 16 | 17 | def save_plot(x, y, plots_folder, filename): 18 | df = pd.DataFrame({"Validation Score": x, "Target Accuracy": y}) 19 | sns.set(font_scale=2, style="whitegrid", rc={"figure.figsize": (10, 10)}) 20 | plot = sns.scatterplot(data=df, x="Validation Score", y="Target Accuracy", s=4) 21 | fig = plot.get_figure() 22 | c_f.makedir_if_not_there(plots_folder) 23 | fig.savefig( 24 | os.path.join(plots_folder, f"{filename}.png"), 25 | bbox_inches="tight", 26 | ) 27 | fig.clf() 28 | 29 | 30 | def normalize(x): 31 | return (x - np.min(x)) / (np.max(x) - np.min(x)) 32 | 33 | 34 | def step_example(): 35 | num_samples = 10000 36 | x = np.arange(num_samples) / num_samples 37 | y = np.arange(num_samples) / num_samples 38 | y_bad = np.concatenate([y[:9000], np.arange(1000) / num_samples], axis=0) 39 | y_good = y 40 | return x, normalize(y_bad), normalize(y_good) 41 | 42 | 43 | def outliers_example1(): 44 | num_samples = 10000 45 | x = np.arange(num_samples) / num_samples 46 | y = np.arange(num_samples) / num_samples 47 | y_bad = np.copy(y) 48 | y_bad[:1000] = np.random.randn(1000) + 10 49 | y_good = y 50 | return x, normalize(y_bad), normalize(y_good) 51 | 52 | 53 | def outliers_example2(): 54 | num_samples = 10000 55 | x = np.arange(num_samples) / num_samples 56 | y = np.arange(num_samples) / num_samples 57 | y_bad = np.copy(y) 58 | y_bad[-1000:] = np.random.randn(1000) - 10 59 | y_good = y 60 | return x, normalize(y_bad), normalize(y_good) 61 | 62 | 63 | def noise_example(): 64 | num_samples = 40000 65 | x = np.arange(num_samples) / num_samples 66 | y = np.arange(num_samples) * 2 / num_samples 67 | noise = np.random.randn(num_samples) 68 | y_bad = y - noise * x * 0.5 69 | y_good = y - noise * (x[::-1]) * 0.5 70 | return x, normalize(y_bad), normalize(y_good) 71 | 72 | 73 | def main(args): 74 | for name, (x, y_bad, y_good) in [ 75 | ("step", step_example()), 76 | ("noise", noise_example()), 77 | ("outliers1", outliers_example1()), 78 | ("outliers2", outliers_example2()), 79 | ]: 80 | 81 | print("\n\n", name) 82 | spearman_bad = spearmanr(x, y_bad).correlation 83 | weighted_spearman_bad = weighted_spearman(x, y_bad, 2) 84 | spearman_good = spearmanr(x, y_good).correlation 85 | weighted_spearman_good = weighted_spearman(x, y_good, 2) 86 | 87 | print("spearman_bad", spearman_bad) 88 | print("weighted_spearman_bad", weighted_spearman_bad) 89 | print("spearman_good", spearman_good) 90 | print("weighted_spearman_good", weighted_spearman_good) 91 | 92 | plots_folder = os.path.join( 93 | args.output_folder, "synthetic_correlation_examples" 94 | ) 95 | x_label = "Validation Score" 96 | y_label = "Target Accuracy" 97 | kwargs = { 98 | "x_label": x_label, 99 | "y_label": y_label, 100 | "font_scale": 2, 101 | "figsize": (10, 10), 102 | "s": 0.2, 103 | } 104 | 105 | scatter_plot( 106 | plots_folder, 107 | pd.DataFrame({x_label: x, y_label: y_bad}), 108 | x_label, 109 | y_label, 110 | f"{name}_bad", 111 | **kwargs, 112 | ) 113 | 114 | scatter_plot( 115 | plots_folder, 116 | pd.DataFrame({x_label: x, y_label: y_good}), 117 | x_label, 118 | y_label, 119 | f"{name}_good", 120 | **kwargs, 121 | ) 122 | 123 | 124 | if __name__ == "__main__": 125 | parser = argparse.ArgumentParser(allow_abbrev=False) 126 | parser.add_argument("--output_folder", type=str, default="plots") 127 | args = parser.parse_args() 128 | main(args) 129 | -------------------------------------------------------------------------------- /powerful_benchmarker/configs/gan_config.py: -------------------------------------------------------------------------------- 1 | from pytorch_adapt.adapters import GAN, GANE, DomainConfusion 2 | from pytorch_adapt.containers import Models, Optimizers 3 | from pytorch_adapt.hooks import CLossHook, TargetDiversityHook, TargetEntropyHook 4 | from pytorch_adapt.inference import default_with_d, default_with_d_logits_layer 5 | from pytorch_adapt.layers import NLLLoss 6 | from pytorch_adapt.models import Discriminator 7 | from pytorch_adapt.weighters import MeanWeighter 8 | 9 | from ..utils import main_utils 10 | from .base_config import BaseConfig 11 | 12 | 13 | class GANConfig(BaseConfig): 14 | def get_adapter_kwargs( 15 | self, 16 | models, 17 | optimizers, 18 | before_training_starts, 19 | lr_multiplier, 20 | use_full_inference, 21 | **kwargs 22 | ): 23 | models = Models(models) 24 | optimizers = Optimizers(optimizers, multipliers={"D": lr_multiplier}) 25 | label_weight = self.optuna_trial.suggest_float("label_weight", 0, 1) 26 | d_weight = self.optuna_trial.suggest_float("d_weight", 0, 1) 27 | g_weight = self.optuna_trial.suggest_float("g_weight", 0, 1) 28 | 29 | d_loss_weighter = MeanWeighter(scale=d_weight) 30 | g_loss_weighter = MeanWeighter( 31 | weights={ 32 | "c_loss": label_weight, 33 | "g_src_domain_loss": g_weight, 34 | "g_target_domain_loss": g_weight, 35 | }, 36 | ) 37 | 38 | hook_kwargs = {"d_weighter": d_loss_weighter, "g_weighter": g_loss_weighter} 39 | inference_fn = default_with_d if use_full_inference else None 40 | 41 | return { 42 | "models": models, 43 | "optimizers": optimizers, 44 | "misc": None, 45 | "before_training_starts": before_training_starts, 46 | "inference_fn": inference_fn, 47 | "hook_kwargs": hook_kwargs, 48 | } 49 | 50 | def get_new_adapter(self, *args, **kwargs): 51 | return GAN(**self.get_adapter_kwargs(*args, **kwargs)) 52 | 53 | def save(self, folder): 54 | super().save(folder) 55 | main_utils.save_this_file(__file__, folder) 56 | 57 | 58 | class DomainConfusionConfig(GANConfig): 59 | def get_models(self, dataset, *args, **kwargs): 60 | models, framework = super().get_models(dataset, *args, **kwargs) 61 | h = models["D"].h 62 | models["D"] = Discriminator(in_size=self.feature_size, h=h, out_size=2) 63 | return models, framework 64 | 65 | def get_new_adapter(self, *args, **kwargs): 66 | return DomainConfusion(**self.get_adapter_kwargs(*args, **kwargs)) 67 | 68 | 69 | class GANEConfig(GANConfig): 70 | def get_adapter_kwargs(self, *args, **kwargs): 71 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 72 | all_kwargs["hook_kwargs"]["detach_entropy_reducer"] = True 73 | return all_kwargs 74 | 75 | def get_new_adapter(self, *args, **kwargs): 76 | return GANE(**self.get_adapter_kwargs(*args, **kwargs)) 77 | 78 | 79 | class GANEUConfig(GANEConfig): 80 | def get_adapter_kwargs(self, *args, **kwargs): 81 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 82 | all_kwargs["hook_kwargs"]["detach_entropy_reducer"] = False 83 | return all_kwargs 84 | 85 | 86 | class GANMinEntConfig(GANConfig): 87 | def get_adapter_kwargs(self, *args, **kwargs): 88 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 89 | all_kwargs["hook_kwargs"]["post_g"] = [TargetEntropyHook()] 90 | entropy_weight = self.optuna_trial.suggest_float("entropy_weight", 0, 1) 91 | all_kwargs["hook_kwargs"]["g_weighter"].weights["entropy_loss"] = entropy_weight 92 | return all_kwargs 93 | 94 | 95 | class GANIMConfig(GANMinEntConfig): 96 | def get_adapter_kwargs(self, *args, **kwargs): 97 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 98 | all_kwargs["hook_kwargs"]["post_g"] += [TargetDiversityHook()] 99 | weights = all_kwargs["hook_kwargs"]["g_weighter"].weights 100 | weights["diversity_loss"] = weights["entropy_loss"] 101 | return all_kwargs 102 | 103 | 104 | class GANFL8Config(GANConfig): 105 | def get_adapter_kwargs(self, *args, **kwargs): 106 | all_kwargs = super().get_adapter_kwargs(*args, **kwargs) 107 | all_kwargs["hook_kwargs"]["c_hook"] = CLossHook( 108 | loss_fn=NLLLoss(reduction="none") 109 | ) 110 | if kwargs["use_full_inference"]: 111 | all_kwargs["inference_fn"] = default_with_d_logits_layer 112 | return all_kwargs 113 | -------------------------------------------------------------------------------- /latex/pred_acc_using_best_adapter_validator_pairs.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pandas as pd 4 | 5 | from latex import utils as latex_utils 6 | from latex.best_accuracy_per_adapter import ( 7 | best_accuracy_per_adapter, 8 | min_value_fn, 9 | reshape_into_best_accuracy_table, 10 | ) 11 | from latex.correlation import base_filename, filter_and_process_validator_args 12 | from latex.correlation import get_preprocess_df as get_preprocess_df_correlation 13 | from latex.correlation_single_adapter import ( 14 | get_postprocess_df as get_postprocess_df_correlation, 15 | ) 16 | from latex.table_creator import table_creator 17 | from validator_tests.utils import utils 18 | from validator_tests.utils.constants import TARGET_ACCURACY 19 | 20 | 21 | def get_postprocess_df(best_validators): 22 | def fn(df): 23 | df = pd.concat(df, axis=0) 24 | latex_utils.convert_adapter_name(df) 25 | df = latex_utils.rename_validator_args(df) 26 | filter = False 27 | for k, v in best_validators.items(): 28 | filter |= ( 29 | (df["adapter"] == k) 30 | & (df["validator"] == v[0]) 31 | & (df["validator_args"] == v[1]) 32 | ) 33 | df = df[filter] 34 | df = df.drop(columns=[f"{TARGET_ACCURACY}_std", "validator", "validator_args"]) 35 | return reshape_into_best_accuracy_table(df) 36 | 37 | return fn 38 | 39 | 40 | def get_best_validators(args, name, src_threshold): 41 | basename = base_filename(name, True, src_threshold) 42 | exp_groups = utils.get_exp_groups(args, args.input_folder, "_select_best") 43 | 44 | dfs, _ = table_creator( 45 | args, 46 | args.input_folder, 47 | args.output_folder, 48 | basename, 49 | preprocess_df=get_preprocess_df_correlation(per_adapter=True), 50 | postprocess_df=get_postprocess_df_correlation(remove_index_names=False), 51 | do_save_to_latex=False, 52 | exp_groups=exp_groups, 53 | ) 54 | 55 | best_validators = {} 56 | for adapter, df in dfs.items(): 57 | df = df.loc[df["Mean"].idxmax()] 58 | best_validators[adapter] = df.name + (rf"${str(df.Mean)} \pm {str(df.Std)}$",) 59 | 60 | tasks = [x for x in list(dfs.values())[0].columns if x not in ["Mean", "Std"]] 61 | 62 | return best_validators, tasks 63 | 64 | 65 | def get_meanstd(df, name): 66 | mean = df.mean(axis=1).round(1).astype(str) 67 | std = df.std(axis=1).round(1).astype(str) 68 | return ( 69 | ("$" + mean + r" \pm " + std + "$") 70 | .to_frame(name) 71 | .reset_index() 72 | .rename(columns={"index": "Algorithm"}) 73 | ) 74 | 75 | 76 | def pred_acc_using_best_adapter_validator_pairs(args, name, src_threshold): 77 | best_validators, tasks = get_best_validators(args, name, src_threshold) 78 | 79 | nlargest = args.nlargest 80 | basename = f"best_accuracy_per_adapter_ranked_by_score_{nlargest}" 81 | color_map_tag_kwargs = { 82 | "tag_prefix": latex_utils.get_tag_prefix(basename), 83 | "min_value_fn": min_value_fn, 84 | } 85 | best_accs, output_folder = table_creator( 86 | args, 87 | args.input_folder, 88 | args.output_folder, 89 | basename, 90 | preprocess_df=filter_and_process_validator_args, 91 | postprocess_df=get_postprocess_df(best_validators), 92 | color_map_tag_kwargs=color_map_tag_kwargs, 93 | add_resizebox=True, 94 | final_str_hook=latex_utils.adapter_final_str_hook, 95 | ) 96 | 97 | best_accs_oracle, _ = best_accuracy_per_adapter(args, do_save_to_latex=False) 98 | 99 | tasks = [x for x in tasks if x in best_accs.columns] 100 | best_accs = best_accs[tasks] 101 | best_accs_oracle = best_accs_oracle[tasks] 102 | diff_from_oracle = best_accs - best_accs_oracle 103 | 104 | best_accs_meanstd = get_meanstd(best_accs, "Average Accuracy") 105 | diff_from_oracle_meanstd = get_meanstd(diff_from_oracle, "Degradation") 106 | 107 | to_save = ( 108 | pd.DataFrame(best_validators) 109 | .transpose() 110 | .reset_index() 111 | .rename( 112 | columns={ 113 | "index": "Algorithm", 114 | 0: "Validator", 115 | 1: "Validator Parameters", 116 | 2: "Weighted Spearman Correlation", 117 | } 118 | ) 119 | ) 120 | 121 | to_save = ( 122 | to_save.merge(best_accs_meanstd) 123 | .drop(columns=["Weighted Spearman Correlation"]) 124 | .sort_values(by="Average Accuracy", ascending=False) 125 | ) 126 | 127 | to_save = to_save.merge(diff_from_oracle_meanstd) 128 | 129 | to_save.style.hide(axis="index").to_latex( 130 | os.path.join(output_folder, "best_validator_per_algorithm.tex"), 131 | hrules=True, 132 | position_float="centering", 133 | ) 134 | -------------------------------------------------------------------------------- /print_progress.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import json 4 | import os 5 | import sys 6 | from collections import defaultdict 7 | from pathlib import Path 8 | 9 | import pandas as pd 10 | 11 | sys.path.insert(0, ".") 12 | from powerful_benchmarker.utils.constants import ( 13 | BEST_TRIAL_FILENAME, 14 | JOBIDS_FILENAME, 15 | TRIALS_FILENAME, 16 | add_default_args, 17 | ) 18 | from powerful_benchmarker.utils.utils import jobs_that_are_still_running 19 | from validator_tests.utils.constants import JOBIDS_FILENAME as V_JOBSID_FILENAME 20 | from validator_tests.utils.constants import VALIDATOR_TESTS_FOLDER 21 | 22 | 23 | def num_jobs_running(exp_folder): 24 | x = jobs_that_are_still_running(exp_folder, JOBIDS_FILENAME) 25 | y = jobs_that_are_still_running(exp_folder, V_JOBSID_FILENAME) 26 | return f"{len(x)} algorithm jobs and {len(y)} validator jobs still running\n" 27 | 28 | 29 | def update_validator_progress_dicts(exp_name, contents, val, val_details): 30 | curr_val = validator_test_progress(contents) 31 | curr_details = {} 32 | for k, v in curr_val.items(): 33 | val[k] += v 34 | x = str(v) 35 | if v < 100: 36 | x += " Validator Not Done" 37 | curr_details[k] = x 38 | val_details[exp_name] = curr_details 39 | 40 | 41 | def validator_test_progress(contents): 42 | output = defaultdict(int) 43 | for x in contents: 44 | contents = glob.glob(os.path.join(x, VALIDATOR_TESTS_FOLDER, "*.pkl")) 45 | for pkl_file in contents: 46 | output[Path(pkl_file).stem] += 1 47 | return output 48 | 49 | 50 | def is_done(e): 51 | best_trial_file = os.path.join(e, BEST_TRIAL_FILENAME) 52 | return os.path.isfile(best_trial_file) 53 | 54 | 55 | def read_trials_csv(e): 56 | filepath = os.path.join(e, TRIALS_FILENAME) 57 | if os.path.isfile(filepath): 58 | trials = pd.read_csv(filepath) 59 | num_success = len(trials[trials["state"] == "COMPLETE"]) 60 | return f"{num_success} / {len(trials)}" 61 | return "0 / 0" 62 | 63 | 64 | def count_exp_folders(contents): 65 | num_folders = 0 66 | for x in contents: 67 | basename = os.path.basename(x) 68 | if os.path.isdir(x) and ( 69 | basename.isdigit() or basename.startswith("reproduction") 70 | ): 71 | num_folders += 1 72 | return num_folders 73 | 74 | 75 | def progress(cfg, exps): 76 | folder_progress = {} 77 | validator_progress = defaultdict(int) 78 | validator_progress_details = {} 79 | for e in exps: 80 | e = os.path.normpath(e) 81 | exp_name = os.path.basename(e) 82 | if not os.path.isdir(e) or exp_name == cfg.slurm_folder: 83 | continue 84 | contents = glob.glob(f"{e}/*") 85 | num_folders = count_exp_folders(contents) 86 | num_success_str = read_trials_csv(e) 87 | output_str = f"{num_folders} folders: {num_success_str}" 88 | if not is_done(e): 89 | output_str += " In Progress" 90 | folder_progress[exp_name] = output_str 91 | if cfg.with_validator_progress: 92 | update_validator_progress_dicts( 93 | exp_name, contents, validator_progress, validator_progress_details 94 | ) 95 | return folder_progress, validator_progress, validator_progress_details 96 | 97 | 98 | def main(cfg): 99 | experiment_paths = sorted(glob.glob(f"{cfg.exp_folder}/*")) 100 | all_folders = {} 101 | for p in experiment_paths: 102 | if os.path.isdir(p): 103 | exps = sorted(glob.glob(f"{p}/*")) 104 | folder_progress, validator_progress, validator_progress_details = progress( 105 | cfg, exps 106 | ) 107 | curr_dict = { 108 | "folder_progress": folder_progress, 109 | } 110 | if cfg.with_validator_progress: 111 | curr_dict.update( 112 | { 113 | "validator_progress": validator_progress, 114 | "validator_progress_details": validator_progress_details, 115 | } 116 | ) 117 | all_folders[os.path.basename(p)] = curr_dict 118 | 119 | out_string = num_jobs_running(cfg.exp_folder) 120 | out_string += json.dumps(all_folders, indent=4, sort_keys=True) 121 | if cfg.save_to_file: 122 | with open(cfg.save_to_file, "w") as f: 123 | f.write(out_string) 124 | else: 125 | print(out_string) 126 | 127 | 128 | if __name__ == "__main__": 129 | parser = argparse.ArgumentParser(allow_abbrev=False) 130 | add_default_args(parser, ["exp_folder", "slurm_folder"]) 131 | parser.add_argument( 132 | "--save_to_file", 133 | type=str, 134 | default=None, 135 | ) 136 | parser.add_argument("--with_validator_progress", action="store_true") 137 | args = parser.parse_args() 138 | main(args) 139 | --------------------------------------------------------------------------------