├── FLARE21 ├── Evaluation │ ├── DSC_NSD_eval.py │ ├── Efficiency.py │ ├── SurfaceDice.py │ ├── Time_GPUMem_eval.py │ ├── load_json.py │ └── logger.py ├── README.md └── ShortPapers │ ├── BIT-Winner.pdf │ ├── CCCCS.pdf │ ├── DaManGo.pdf │ ├── DeepInsthink.pdf │ ├── EK.pdf │ ├── IMT-Atlantique.pdf │ ├── LetsGo.pdf │ ├── MIMT-MIS.pdf │ ├── Ocean.pdf │ ├── RRR_MCR.pdf │ ├── SJTU_MathImaging.pdf │ ├── SuperX.pdf │ ├── TXD.pdf │ ├── The-Flash.pdf │ ├── UIT-VNU.pdf │ ├── aq_enib_flare_seg.pdf │ ├── fosun_aitrox.pdf │ ├── hitcs.pdf │ ├── icg.pdf │ ├── jiujiuhaiziba.pdf │ ├── pmcc.pdf │ ├── ttime.pdf │ └── xf4j.pdf ├── FLARE22 └── Evaluation │ ├── Efficiency.py │ ├── FLARE22_DSC_NSD_Eval.py │ ├── Segmentation Efficiency Evaluation.md │ ├── SurfaceDice.py │ ├── load_json.py │ ├── logger.py │ └── resource_eval.py ├── FLARE23 ├── Efficiency.py ├── FLARE23_DSC_NSD_Eval.py ├── README.md ├── SurfaceDice.py ├── load_json.py ├── logger.py └── resource_eval.py ├── FLARE24 ├── Efficiency.py ├── README.md ├── SurfaceDice.py ├── T1_FLARE24_DSC_NSD_Eval.py ├── T2andT3_FLARE24_DSC_NSD_Eval.py ├── __pycache__ │ └── SurfaceDice.cpython-310.pyc ├── load_json.py ├── logger.py └── resource_eval.py ├── LICENSE └── README.md /FLARE21/Evaluation/DSC_NSD_eval.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import nibabel as nb 3 | import os 4 | from collections import OrderedDict 5 | import pandas as pd 6 | from SurfaceDice import compute_surface_distances, compute_surface_dice_at_tolerance, compute_dice_coefficient 7 | join = os.path.join 8 | 9 | 10 | seg_path = r'../seg' 11 | gt_path = r'../gt' 12 | save_path = r'../result' 13 | save_name = 'eval.csv' 14 | filenames = os.listdir(seg_path) 15 | filenames = [x for x in filenames if x.endswith('.nii.gz')] 16 | filenames.sort() 17 | 18 | seg_metrics = OrderedDict() 19 | seg_metrics['Name'] = list() 20 | for i in range(1, 5): 21 | seg_metrics['DSC_{}'.format(i)] = list() 22 | seg_metrics['NSD-1mm_{}'.format(i)] = list() 23 | 24 | 25 | for name in filenames: 26 | seg_metrics['Name'].append(name) 27 | # load grond truth and segmentation 28 | gt_nii = nb.load(join(gt_path, name)) 29 | case_spacing = gt_nii.header.get_zooms() 30 | gt_data = np.uint8(gt_nii.get_fdata()) 31 | seg_data = nb.load(join(seg_path, name)).get_fdata() 32 | 33 | for i in range(1, 5): 34 | if np.sum(gt_data==i)==0 and np.sum(seg_data==i)==0: 35 | DSC_i = 1 36 | NSD_i = 1 37 | elif np.sum(gt_data==i)==0 and np.sum(seg_data==i)>0: 38 | DSC_i = 0 39 | NSD_i = 0 40 | else: 41 | surface_distances = compute_surface_distances(gt_data==i, seg_data==i, case_spacing) 42 | DSC_i = compute_dice_coefficient(gt_data==i, seg_data==i) 43 | NSD_i = compute_surface_dice_at_tolerance(surface_distances, 1) 44 | seg_metrics['DSC_{}'.format(i)].append(DSC_i) 45 | seg_metrics['NSD-1mm_{}'.format(i)].append(NSD_i) 46 | 47 | dataframe = pd.DataFrame(seg_metrics) 48 | dataframe.to_csv(join(save_path, save_name), index=False) 49 | -------------------------------------------------------------------------------- /FLARE21/Evaluation/Efficiency.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import json 4 | import os 5 | import time 6 | from multiprocessing import Manager, Process 7 | 8 | from pynvml.smi import nvidia_smi 9 | 10 | from logger import add_file_handler_to_logger, logger 11 | 12 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 13 | 14 | manager = Manager() 15 | gpu_list = manager.list() 16 | 17 | 18 | def daemon_process(time_interval, json_path, gpu_index=1): 19 | while True: 20 | nvsmi = nvidia_smi.getInstance() 21 | dictm = nvsmi.DeviceQuery("memory.free, memory.total") 22 | gpu_memory = ( 23 | dictm["gpu"][gpu_index]["fb_memory_usage"]["total"] - dictm["gpu"][gpu_index]["fb_memory_usage"]["free"] 24 | ) 25 | gpu_list.append(gpu_memory) 26 | # with open(json_path, 'w')as f: 27 | # #js['gpu_memory'] = gpu_memory_max 28 | # js['gpu_memory'].append(gpu_memory) 29 | # json.dump(js, f, indent=4) 30 | time.sleep(time_interval) 31 | 32 | 33 | def save_result(start_time, sleep_time, json_path, gpu_list_): 34 | if os.path.exists(json_path): 35 | with open(json_path) as f: 36 | js = json.load(f) 37 | else: 38 | js = {"gpu_memory": []} 39 | # raise ValueError(f"{json_path} don't exist!") 40 | 41 | with open(json_path, "w") as f: 42 | # js['gpu_memory'] = gpu_memory_max 43 | js["gpu_memory"] = gpu_list_ 44 | json.dump(js, f, indent=4) 45 | 46 | infer_time = time.time() - start_time 47 | with open(json_path, "r") as f: 48 | js = json.load(f) 49 | with open(json_path, "w") as f: 50 | js["time"] = infer_time 51 | json.dump(js, f, indent=4) 52 | time.sleep(2) 53 | logger.info("save result end") 54 | 55 | 56 | if __name__ == "__main__": 57 | parser = argparse.ArgumentParser() 58 | parser.add_argument("-time_interval", default=0.1, help="time_interval") 59 | parser.add_argument("-sleep_time", default=5, help="sleep time") 60 | parser.add_argument("-shell_path", default="predict.sh", help="time_interval") 61 | # XXX: in case of someone use lower docker, please use specified GPU !!! 62 | parser.add_argument("-gpus", default=1, help="CUDA_VISIBLE_DEVICES") 63 | parser.add_argument("-docker_input_file", default="./inputs/", help="docker input folder") 64 | parser.add_argument("-docker_name", default="nnunet", help="docker output folder") 65 | args = parser.parse_args() 66 | logger.info(f"We are evaluating {args.docker_name}") 67 | json_dir = "./results/{}".format(args.docker_name) 68 | json_path = os.path.join( 69 | json_dir, glob.glob(args.docker_input_file + "/*")[0].split("/")[-1].split(".")[0] + ".json", 70 | ) 71 | 72 | try: 73 | p1 = Process(target=daemon_process, args=(args.time_interval, json_path, args.gpus,)) 74 | p1.daemon = True 75 | p1.start() 76 | t0 = time.time() 77 | # XXX: in case of someone use lower docker, please use specified GPU !!! 78 | # cmd = 'docker container run --runtime="nvidia" -e NVIDIA_VISIBLE_DEVICES={} --name {} --rm -v $PWD/inputs/:/workspace/input/ -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/sh -c "sh {}"'.format( 79 | # args.gpus, args.docker_name, args.docker_name, args.shell_path 80 | # ) 81 | cmd = 'docker container run --gpus="device={}" --name {} --rm -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/bash -c "sh predict.sh" '.format( 82 | args.gpus, args.docker_name, args.docker_name 83 | ) 84 | logger.info(f"cmd is : {cmd}") 85 | logger.info("start predict...") 86 | os.system(cmd) 87 | gpu_list = list(gpu_list) 88 | gpu_list_copy = gpu_list.copy() 89 | save_result(t0, args.sleep_time, json_path, gpu_list_copy) 90 | time.sleep(args.sleep_time) 91 | except Exception as error: 92 | logger.exception(error) 93 | -------------------------------------------------------------------------------- /FLARE21/Evaluation/SurfaceDice.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.ndimage 3 | 4 | # neighbour_code_to_normals is a lookup table. 5 | # For every binary neighbour code 6 | # (2x2x2 neighbourhood = 8 neighbours = 8 bits = 256 codes) 7 | # it contains the surface normals of the triangles (called "surfel" for 8 | # "surface element" in the following). The length of the normal 9 | # vector encodes the surfel area. 10 | # 11 | # created by compute_surface_area_lookup_table.ipynb using the 12 | # marching_cube algorithm, see e.g. https://en.wikipedia.org/wiki/Marching_cubes 13 | # credit to: http://medicaldecathlon.com/files/Surface_distance_based_measures.ipynb 14 | neighbour_code_to_normals = [ 15 | [[0,0,0]], 16 | [[0.125,0.125,0.125]], 17 | [[-0.125,-0.125,0.125]], 18 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 19 | [[0.125,-0.125,0.125]], 20 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 21 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 22 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 23 | [[-0.125,0.125,0.125]], 24 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 25 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 26 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 27 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 28 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 29 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 30 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 31 | [[0.125,-0.125,-0.125]], 32 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 33 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 34 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 35 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 36 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 37 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 38 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 39 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 40 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 41 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 42 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 43 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 44 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 45 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 46 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 47 | [[0.125,-0.125,0.125]], 48 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 49 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 50 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 51 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 52 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 53 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 54 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 55 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 56 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 57 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 58 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 59 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 60 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 61 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 62 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 63 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 64 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 65 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 66 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 67 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 68 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 69 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 70 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 71 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 72 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 73 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 74 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 75 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 76 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 77 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 78 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 79 | [[-0.125,-0.125,0.125]], 80 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 81 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 82 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 83 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 84 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 85 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 86 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 87 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 88 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 89 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 90 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 91 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 92 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 93 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 94 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 95 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 96 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 97 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 98 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 99 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 100 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 101 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 102 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 103 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 104 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 105 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 106 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 107 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 108 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 109 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 110 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 111 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 112 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 113 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 114 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 115 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 116 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 117 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 118 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 119 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 120 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 121 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 122 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 123 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 124 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 125 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 126 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 127 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 128 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 129 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 130 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 131 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 132 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 133 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 134 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 135 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 136 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 137 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 138 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 139 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 140 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 141 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 142 | [[0.125,0.125,0.125]], 143 | [[0.125,0.125,0.125]], 144 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 145 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 146 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 147 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 148 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 149 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 150 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 151 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 152 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 153 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 154 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 155 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 156 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 157 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 158 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 159 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 160 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 161 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 162 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 163 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 164 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 165 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 166 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 167 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 168 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.0,0.25,0.25],[0.0,0.25,0.25]], 169 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 170 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 171 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 172 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 173 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 174 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 175 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 176 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 177 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 178 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 179 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 180 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.25,0.0,0.25],[0.25,0.0,0.25]], 181 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 182 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 183 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 184 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 185 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 186 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 187 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 188 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 189 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 190 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 191 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 192 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 193 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 194 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 195 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 196 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 197 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 198 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 199 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 200 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 201 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 202 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 203 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 204 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 205 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 206 | [[-0.125,-0.125,0.125]], 207 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 208 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 209 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 210 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 211 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 212 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 213 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 214 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 215 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 216 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 217 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 218 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 219 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 220 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 221 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 222 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 223 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 224 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 225 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 226 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 227 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 228 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 229 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 230 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 231 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 232 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 233 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 234 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 235 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 236 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 237 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 238 | [[0.125,-0.125,0.125]], 239 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 240 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 241 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 242 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 243 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 244 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 245 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 246 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 247 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 248 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 249 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 250 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 251 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 252 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 253 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 254 | [[0.125,-0.125,-0.125]], 255 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 256 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 257 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 258 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 259 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 260 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 261 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 262 | [[-0.125,0.125,0.125]], 263 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 264 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 265 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 266 | [[0.125,-0.125,0.125]], 267 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 268 | [[-0.125,-0.125,0.125]], 269 | [[0.125,0.125,0.125]], 270 | [[0,0,0]]] 271 | 272 | 273 | def compute_surface_distances(mask_gt, mask_pred, spacing_mm): 274 | """Compute closest distances from all surface points to the other surface. 275 | 276 | Finds all surface elements "surfels" in the ground truth mask `mask_gt` and 277 | the predicted mask `mask_pred`, computes their area in mm^2 and the distance 278 | to the closest point on the other surface. It returns two sorted lists of 279 | distances together with the corresponding surfel areas. If one of the masks 280 | is empty, the corresponding lists are empty and all distances in the other 281 | list are `inf` 282 | 283 | Args: 284 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 285 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 286 | spacing_mm: 3-element list-like structure. Voxel spacing in x0, x1 and x2 287 | direction 288 | 289 | Returns: 290 | A dict with 291 | "distances_gt_to_pred": 1-dim numpy array of type float. The distances in mm 292 | from all ground truth surface elements to the predicted surface, 293 | sorted from smallest to largest 294 | "distances_pred_to_gt": 1-dim numpy array of type float. The distances in mm 295 | from all predicted surface elements to the ground truth surface, 296 | sorted from smallest to largest 297 | "surfel_areas_gt": 1-dim numpy array of type float. The area in mm^2 of 298 | the ground truth surface elements in the same order as 299 | distances_gt_to_pred 300 | "surfel_areas_pred": 1-dim numpy array of type float. The area in mm^2 of 301 | the predicted surface elements in the same order as 302 | distances_pred_to_gt 303 | 304 | """ 305 | 306 | # compute the area for all 256 possible surface elements 307 | # (given a 2x2x2 neighbourhood) according to the spacing_mm 308 | neighbour_code_to_surface_area = np.zeros([256]) 309 | for code in range(256): 310 | normals = np.array(neighbour_code_to_normals[code]) 311 | sum_area = 0 312 | for normal_idx in range(normals.shape[0]): 313 | # normal vector 314 | n = np.zeros([3]) 315 | n[0] = normals[normal_idx,0] * spacing_mm[1] * spacing_mm[2] 316 | n[1] = normals[normal_idx,1] * spacing_mm[0] * spacing_mm[2] 317 | n[2] = normals[normal_idx,2] * spacing_mm[0] * spacing_mm[1] 318 | area = np.linalg.norm(n) 319 | sum_area += area 320 | neighbour_code_to_surface_area[code] = sum_area 321 | 322 | # compute the bounding box of the masks to trim 323 | # the volume to the smallest possible processing subvolume 324 | mask_all = mask_gt | mask_pred 325 | bbox_min = np.zeros(3, np.int64) 326 | bbox_max = np.zeros(3, np.int64) 327 | 328 | # max projection to the x0-axis 329 | proj_0 = np.max(np.max(mask_all, axis=2), axis=1) 330 | idx_nonzero_0 = np.nonzero(proj_0)[0] 331 | if len(idx_nonzero_0) == 0: 332 | return {"distances_gt_to_pred": np.array([]), 333 | "distances_pred_to_gt": np.array([]), 334 | "surfel_areas_gt": np.array([]), 335 | "surfel_areas_pred": np.array([])} 336 | 337 | bbox_min[0] = np.min(idx_nonzero_0) 338 | bbox_max[0] = np.max(idx_nonzero_0) 339 | 340 | # max projection to the x1-axis 341 | proj_1 = np.max(np.max(mask_all, axis=2), axis=0) 342 | idx_nonzero_1 = np.nonzero(proj_1)[0] 343 | bbox_min[1] = np.min(idx_nonzero_1) 344 | bbox_max[1] = np.max(idx_nonzero_1) 345 | 346 | # max projection to the x2-axis 347 | proj_2 = np.max(np.max(mask_all, axis=1), axis=0) 348 | idx_nonzero_2 = np.nonzero(proj_2)[0] 349 | bbox_min[2] = np.min(idx_nonzero_2) 350 | bbox_max[2] = np.max(idx_nonzero_2) 351 | 352 | # print("bounding box min = {}".format(bbox_min)) 353 | # print("bounding box max = {}".format(bbox_max)) 354 | 355 | # crop the processing subvolume. 356 | # we need to zeropad the cropped region with 1 voxel at the lower, 357 | # the right and the back side. This is required to obtain the "full" 358 | # convolution result with the 2x2x2 kernel 359 | cropmask_gt = np.zeros((bbox_max - bbox_min)+2, np.uint8) 360 | cropmask_pred = np.zeros((bbox_max - bbox_min)+2, np.uint8) 361 | 362 | cropmask_gt[0:-1, 0:-1, 0:-1] = mask_gt[bbox_min[0]:bbox_max[0]+1, 363 | bbox_min[1]:bbox_max[1]+1, 364 | bbox_min[2]:bbox_max[2]+1] 365 | 366 | cropmask_pred[0:-1, 0:-1, 0:-1] = mask_pred[bbox_min[0]:bbox_max[0]+1, 367 | bbox_min[1]:bbox_max[1]+1, 368 | bbox_min[2]:bbox_max[2]+1] 369 | 370 | # compute the neighbour code (local binary pattern) for each voxel 371 | # the resultsing arrays are spacially shifted by minus half a voxel in each axis. 372 | # i.e. the points are located at the corners of the original voxels 373 | kernel = np.array([[[128,64], 374 | [32,16]], 375 | [[8,4], 376 | [2,1]]]) 377 | neighbour_code_map_gt = scipy.ndimage.filters.correlate(cropmask_gt.astype(np.uint8), kernel, mode="constant", cval=0) 378 | neighbour_code_map_pred = scipy.ndimage.filters.correlate(cropmask_pred.astype(np.uint8), kernel, mode="constant", cval=0) 379 | 380 | # create masks with the surface voxels 381 | borders_gt = ((neighbour_code_map_gt != 0) & (neighbour_code_map_gt != 255)) 382 | borders_pred = ((neighbour_code_map_pred != 0) & (neighbour_code_map_pred != 255)) 383 | 384 | # compute the distance transform (closest distance of each voxel to the surface voxels) 385 | if borders_gt.any(): 386 | distmap_gt = scipy.ndimage.morphology.distance_transform_edt(~borders_gt, sampling=spacing_mm) 387 | else: 388 | distmap_gt = np.Inf * np.ones(borders_gt.shape) 389 | 390 | if borders_pred.any(): 391 | distmap_pred = scipy.ndimage.morphology.distance_transform_edt(~borders_pred, sampling=spacing_mm) 392 | else: 393 | distmap_pred = np.Inf * np.ones(borders_pred.shape) 394 | 395 | # compute the area of each surface element 396 | surface_area_map_gt = neighbour_code_to_surface_area[neighbour_code_map_gt] 397 | surface_area_map_pred = neighbour_code_to_surface_area[neighbour_code_map_pred] 398 | 399 | # create a list of all surface elements with distance and area 400 | distances_gt_to_pred = distmap_pred[borders_gt] 401 | distances_pred_to_gt = distmap_gt[borders_pred] 402 | surfel_areas_gt = surface_area_map_gt[borders_gt] 403 | surfel_areas_pred = surface_area_map_pred[borders_pred] 404 | 405 | # sort them by distance 406 | if distances_gt_to_pred.shape != (0,): 407 | sorted_surfels_gt = np.array(sorted(zip(distances_gt_to_pred, surfel_areas_gt))) 408 | distances_gt_to_pred = sorted_surfels_gt[:,0] 409 | surfel_areas_gt = sorted_surfels_gt[:,1] 410 | 411 | if distances_pred_to_gt.shape != (0,): 412 | sorted_surfels_pred = np.array(sorted(zip(distances_pred_to_gt, surfel_areas_pred))) 413 | distances_pred_to_gt = sorted_surfels_pred[:,0] 414 | surfel_areas_pred = sorted_surfels_pred[:,1] 415 | 416 | 417 | return {"distances_gt_to_pred": distances_gt_to_pred, 418 | "distances_pred_to_gt": distances_pred_to_gt, 419 | "surfel_areas_gt": surfel_areas_gt, 420 | "surfel_areas_pred": surfel_areas_pred} 421 | 422 | 423 | def compute_average_surface_distance(surface_distances): 424 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 425 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 426 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 427 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 428 | average_distance_gt_to_pred = np.sum( distances_gt_to_pred * surfel_areas_gt) / np.sum(surfel_areas_gt) 429 | average_distance_pred_to_gt = np.sum( distances_pred_to_gt * surfel_areas_pred) / np.sum(surfel_areas_pred) 430 | return (average_distance_gt_to_pred, average_distance_pred_to_gt) 431 | 432 | def compute_robust_hausdorff(surface_distances, percent): 433 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 434 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 435 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 436 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 437 | if len(distances_gt_to_pred) > 0: 438 | surfel_areas_cum_gt = np.cumsum(surfel_areas_gt) / np.sum(surfel_areas_gt) 439 | idx = np.searchsorted(surfel_areas_cum_gt, percent/100.0) 440 | perc_distance_gt_to_pred = distances_gt_to_pred[min(idx, len(distances_gt_to_pred)-1)] 441 | else: 442 | perc_distance_gt_to_pred = np.Inf 443 | 444 | if len(distances_pred_to_gt) > 0: 445 | surfel_areas_cum_pred = np.cumsum(surfel_areas_pred) / np.sum(surfel_areas_pred) 446 | idx = np.searchsorted(surfel_areas_cum_pred, percent/100.0) 447 | perc_distance_pred_to_gt = distances_pred_to_gt[min(idx, len(distances_pred_to_gt)-1)] 448 | else: 449 | perc_distance_pred_to_gt = np.Inf 450 | 451 | return max( perc_distance_gt_to_pred, perc_distance_pred_to_gt) 452 | 453 | def compute_surface_overlap_at_tolerance(surface_distances, tolerance_mm): 454 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 455 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 456 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 457 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 458 | rel_overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) / np.sum(surfel_areas_gt) 459 | rel_overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) / np.sum(surfel_areas_pred) 460 | return (rel_overlap_gt, rel_overlap_pred) 461 | 462 | def compute_surface_dice_at_tolerance(surface_distances, tolerance_mm): 463 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 464 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 465 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 466 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 467 | overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) 468 | overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) 469 | surface_dice = (overlap_gt + overlap_pred) / ( 470 | np.sum(surfel_areas_gt) + np.sum(surfel_areas_pred)) 471 | return surface_dice 472 | 473 | 474 | def compute_dice_coefficient(mask_gt, mask_pred): 475 | """Compute soerensen-dice coefficient. 476 | 477 | compute the soerensen-dice coefficient between the ground truth mask `mask_gt` 478 | and the predicted mask `mask_pred`. 479 | 480 | Args: 481 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 482 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 483 | 484 | Returns: 485 | the dice coeffcient as float. If both masks are empty, the result is NaN 486 | """ 487 | volume_sum = mask_gt.sum() + mask_pred.sum() 488 | if volume_sum == 0: 489 | return np.NaN 490 | volume_intersect = (mask_gt & mask_pred).sum() 491 | return 2*volume_intersect / volume_sum 492 | 493 | 494 | #%% Some Simple Tests 495 | # single pixels, 2mm away 496 | mask_gt = np.zeros((128,128,128), np.uint8) 497 | mask_pred = np.zeros((128,128,128), np.uint8) 498 | mask_gt[50,60,70] = 1 499 | mask_pred[50,60,72] = 1 500 | surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3,2,1)) 501 | print("surface dice at 1mm: {}".format(compute_surface_dice_at_tolerance(surface_distances, 1))) 502 | print("volumetric dice: {}".format(compute_dice_coefficient(mask_gt, mask_pred))) 503 | 504 | 505 | #%% two cubes. cube 1 is 100x100x100 mm^3 and cube 2 is 102x100x100 mm^3 506 | mask_gt = np.zeros((100,100,100), np.uint8) 507 | mask_pred = np.zeros((100,100,100), np.uint8) 508 | spacing_mm=(2,1,1) 509 | mask_gt[0:50, :, :] = 1 510 | mask_pred[0:51, :, :] = 1 511 | surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm) 512 | print("surface dice at 1mm: {}".format(compute_surface_dice_at_tolerance(surface_distances, 1))) 513 | print("volumetric dice: {}".format(compute_dice_coefficient(mask_gt, mask_pred))) 514 | print("") 515 | print("expected average_distance_gt_to_pred = 1./6 * 2mm = {}mm".format(1./6 * 2)) 516 | print("expected volumetric dice: {}".format(2.*100*100*100 / (100*100*100 + 102*100*100) )) 517 | 518 | 519 | 520 | #%% test empty mask in prediction 521 | mask_gt = np.zeros((128,128,128), np.uint8) 522 | mask_pred = np.zeros((128,128,128), np.uint8) 523 | mask_gt[50,60,70] = 1 524 | #mask_pred[50,60,72] = 1 525 | surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3,2,1)) 526 | print("average surface distance: {} mm".format(compute_average_surface_distance(surface_distances))) 527 | print("hausdorff (100%): {} mm".format(compute_robust_hausdorff(surface_distances, 100))) 528 | print("hausdorff (95%): {} mm".format(compute_robust_hausdorff(surface_distances, 95))) 529 | print("surface overlap at 1mm: {}".format(compute_surface_overlap_at_tolerance(surface_distances, 1))) 530 | print("surface dice at 1mm: {}".format(compute_surface_dice_at_tolerance(surface_distances, 1))) 531 | print("volumetric dice: {}".format(compute_dice_coefficient(mask_gt, mask_pred))) 532 | 533 | 534 | #%% test empty mask in ground truth 535 | mask_gt = np.zeros((128,128,128), np.uint8) 536 | mask_pred = np.zeros((128,128,128), np.uint8) 537 | #mask_gt[50,60,70] = 1 538 | mask_pred[50,60,72] = 1 539 | surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3,2,1)) 540 | print("average surface distance: {} mm".format(compute_average_surface_distance(surface_distances))) 541 | print("hausdorff (100%): {} mm".format(compute_robust_hausdorff(surface_distances, 100))) 542 | print("hausdorff (95%): {} mm".format(compute_robust_hausdorff(surface_distances, 95))) 543 | print("surface overlap at 1mm: {}".format(compute_surface_overlap_at_tolerance(surface_distances, 1))) 544 | print("surface dice at 1mm: {}".format(compute_surface_dice_at_tolerance(surface_distances, 1))) 545 | print("volumetric dice: {}".format(compute_dice_coefficient(mask_gt, mask_pred))) 546 | 547 | 548 | #%% test both masks empty 549 | mask_gt = np.zeros((128,128,128), np.uint8) 550 | mask_pred = np.zeros((128,128,128), np.uint8) 551 | #mask_gt[50,60,70] = 1 552 | #mask_pred[50,60,72] = 1 553 | surface_distances = compute_surface_distances(mask_gt, mask_pred, spacing_mm=(3,2,1)) 554 | print("average surface distance: {} mm".format(compute_average_surface_distance(surface_distances))) 555 | print("hausdorff (100%): {} mm".format(compute_robust_hausdorff(surface_distances, 100))) 556 | print("hausdorff (95%): {} mm".format(compute_robust_hausdorff(surface_distances, 95))) 557 | print("surface overlap at 1mm: {}".format(compute_surface_overlap_at_tolerance(surface_distances, 1))) 558 | print("surface dice at 1mm: {}".format(compute_surface_dice_at_tolerance(surface_distances, 1))) 559 | print("volumetric dice: {}".format(compute_dice_coefficient(mask_gt, mask_pred))) 560 | -------------------------------------------------------------------------------- /FLARE21/Evaluation/Time_GPUMem_eval.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | import time 5 | import torch 6 | from pathlib import Path 7 | join = os.path.join 8 | from logger import add_file_handler_to_logger, logger 9 | 10 | 11 | def check_dir(file_path): 12 | file_path = Path(file_path) 13 | files = [f for f in file_path.iterdir() if ".nii.gz" in str(f)] 14 | if len(files) != 0: 15 | return False 16 | return True 17 | 18 | 19 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 20 | 21 | docker_path = './team_docker' 22 | test_img_path = './ValidationImg/' 23 | save_path = './results/' 24 | 25 | os.makedirs(save_path, exist_ok=True) 26 | os.makedirs('./inputs/', exist_ok=True) 27 | os.makedirs('./outputs/', exist_ok=True) 28 | 29 | dockers = sorted(os.listdir(docker_path)) 30 | test_cases = sorted(os.listdir(test_img_path)) 31 | 32 | for docker in dockers: 33 | try: 34 | name = docker.split('.')[0].lower() 35 | print('teamname docker: ', docker) 36 | os.system('docker image load < {}'.format(join(docker_path, docker))) 37 | team_outpath = join(save_path, name) 38 | if os.path.exists(team_outpath): 39 | shutil.rmtree(team_outpath) 40 | os.mkdir(team_outpath) 41 | for case in test_cases: 42 | if not check_dir('./inputs'): 43 | logger.error("please check inputs folder") 44 | raise 45 | shutil.copy(join(test_img_path, case), './inputs') 46 | start_time = time.time() 47 | os.system('python Efficiency.py -docker_name {}'.format(name)) 48 | logger.info(f"{case} finished!") 49 | os.remove(join('./inputs', case)) 50 | # shutil.rmtree('./inputs') 51 | logger.info(f"{case} cost time: {time.time() - start_time}") 52 | 53 | os.system("python load_json.py -docker_name {} -save_path {}".format(name, save_path)) 54 | shutil.move("./outputs", team_outpath) 55 | os.mkdir("./outputs") 56 | torch.cuda.empty_cache() 57 | os.system("docker rmi {}:latest".format(name)) 58 | except Exception as e: 59 | logger.exception(e) 60 | -------------------------------------------------------------------------------- /FLARE21/Evaluation/load_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | import csv 3 | import argparse 4 | import glob 5 | import matplotlib 6 | import os 7 | join = os.path.join 8 | 9 | matplotlib.use("Agg") 10 | import matplotlib.pyplot as plt 11 | 12 | from logger import add_file_handler_to_logger, logger 13 | 14 | add_file_handler_to_logger(name="main", dir_path=f"logs/", level="DEBUG") 15 | 16 | 17 | if __name__ == "__main__": 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument("-docker_name", default="fully_suplearn_subtask1", help="team docker name") 20 | parser.add_argument("-save_path", default="./results", help="save_path") 21 | args = parser.parse_args() 22 | logger.info("we are counting: {args.docker_name}") 23 | json_dir = join(args.save_path, args.docker_name) 24 | csv_path = join(json_dir, args.docker_name + '_Efficiency.csv') 25 | jsonl = sorted(glob.glob(json_dir + "/*.json")) 26 | alldata = [] 27 | for item in jsonl: 28 | csv_l = [] 29 | name = item.split("/")[-1].split(".")[0] 30 | csv_l.append(name) 31 | zitem = item 32 | with open(zitem) as f: 33 | try: 34 | js = json.load(f) 35 | except Exception as error: 36 | logger.error(f"{item} have error") 37 | logger.exception(error) 38 | if "time" not in js: 39 | logger.error(f"{item} don't have time!!!!") 40 | logger.info(f"Manually compute {item}") 41 | time = 0.1 * len(js["gpu_memory"]) 42 | else: 43 | time = js["time"] 44 | csv_l.append(time) 45 | mem = js["gpu_memory"] 46 | x = [item * 0.1 for item in range(len(mem))] 47 | plt.cla() 48 | plt.xlabel("Time (s)", fontsize="large") 49 | plt.ylabel("GPU Memory (MB)", fontsize="large") 50 | plt.plot(x, mem, "b", ms=10, label="a") 51 | plt.savefig(zitem.replace(".json", ".jpg")) 52 | count_set = set(mem) 53 | count_list = [] 54 | for citem in count_set: 55 | cts = mem.count(citem) 56 | if cts > 0.02 * len(mem): 57 | count_list.append(citem) 58 | max_mem = max(count_list) 59 | csv_l.append(max_mem) 60 | csv_l.append(sum(mem) * 0.1) 61 | alldata.append(csv_l) 62 | f = open(csv_path, "w") 63 | writer = csv.writer(f) 64 | writer.writerow(["name", "time", "gpu_memory", "time_multiply_memory"]) 65 | for i in alldata: 66 | writer.writerow(i) 67 | f.close() 68 | -------------------------------------------------------------------------------- /FLARE21/Evaluation/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from loguru import logger as loguru_logger 5 | 6 | loguru_logger.configure( 7 | handlers=[dict(sink=sys.stderr, filter=lambda record: record["extra"]["console"], level="DEBUG",),], 8 | ) 9 | 10 | 11 | logger = loguru_logger.bind(console=True) 12 | 13 | 14 | def add_file_handler_to_logger( 15 | name: str, dir_path="./logs", level="DEBUG", 16 | ): 17 | loguru_logger.add( 18 | sink=os.path.join(dir_path, f"{name}-{level}.log"), 19 | level=level, 20 | filter=lambda record: "console" in record["extra"], 21 | enqueue=True, 22 | ) 23 | -------------------------------------------------------------------------------- /FLARE21/README.md: -------------------------------------------------------------------------------- 1 | # Official repository of MICCAI 2021 Challenge: [FLARE21](https://flare.grand-challenge.org/FLARE21/). 2 | 3 | We provide the evaluation code and a [video demo](https://www.bilibili.com/video/BV1mU4y1n7V6) of the evaluation process. You can use the code to evaluate the `Running time` and `Maximum used GPU memory` of your Docker. 4 | 5 | Please feel free to raise any issues if you have questions about the challenge, e.g., dataset, evaluation measures, ranking scheme and so on. 6 | 7 | ### Enviroment and requirement 8 | 9 | - python 3.8+ 10 | - torch 11 | - loguru 12 | - pynvml 13 | 14 | 15 | ### Compute running time and GPU memory 16 | 17 | Set `docker_path`, `test_img_path`, and `save_path` in `Time_GPUMem_eval.py` and run 18 | 19 | `nohup python Time_GPUMem_eval.py >> infos.log &` 20 | 21 | ### Compute DSC and NSD 22 | 23 | Set `seg_path`, `gt_path`, `save_path`, `save_name` in `DSC_NSD_eval.py` and run 24 | 25 | `python DSC_NSD_eval.py` 26 | 27 | -------------------------------------------------------------------------------- /FLARE21/ShortPapers/BIT-Winner.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/BIT-Winner.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/CCCCS.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/CCCCS.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/DaManGo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/DaManGo.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/DeepInsthink.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/DeepInsthink.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/EK.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/EK.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/IMT-Atlantique.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/IMT-Atlantique.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/LetsGo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/LetsGo.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/MIMT-MIS.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/MIMT-MIS.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/Ocean.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/Ocean.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/RRR_MCR.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/RRR_MCR.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/SJTU_MathImaging.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/SJTU_MathImaging.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/SuperX.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/SuperX.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/TXD.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/TXD.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/The-Flash.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/The-Flash.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/UIT-VNU.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/UIT-VNU.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/aq_enib_flare_seg.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/aq_enib_flare_seg.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/fosun_aitrox.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/fosun_aitrox.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/hitcs.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/hitcs.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/icg.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/icg.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/jiujiuhaiziba.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/jiujiuhaiziba.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/pmcc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/pmcc.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/ttime.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/ttime.pdf -------------------------------------------------------------------------------- /FLARE21/ShortPapers/xf4j.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE21/ShortPapers/xf4j.pdf -------------------------------------------------------------------------------- /FLARE22/Evaluation/Efficiency.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import json 4 | import os 5 | import time 6 | import psutil as p 7 | from multiprocessing import Manager, Process 8 | from pynvml.smi import nvidia_smi 9 | import numpy as np 10 | from logger import add_file_handler_to_logger, logger 11 | 12 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 13 | 14 | manager = Manager() 15 | gpu_list = manager.list() 16 | cpu_list = manager.list() 17 | RAM_list = manager.list() 18 | def cpu_usage(): 19 | t = p.cpu_times() 20 | return [t.user, t.system, t.idle] 21 | before = cpu_usage() 22 | def get_cpu_usage(): 23 | global before 24 | now = cpu_usage() 25 | delta = [now[i] - before[i] for i in range(len(now))] 26 | total = sum(delta) 27 | before = now 28 | return [(100.0*dt)/(total+0.1) for dt in delta] 29 | 30 | 31 | def daemon_process(time_interval, json_path, gpu_index=1): 32 | while True: 33 | nvsmi = nvidia_smi.getInstance() 34 | dictm = nvsmi.DeviceQuery("memory.free, memory.total") 35 | gpu_memory = ( 36 | dictm["gpu"][gpu_index]["fb_memory_usage"]["total"] - dictm["gpu"][gpu_index]["fb_memory_usage"]["free"] 37 | ) 38 | cpu_usage =get_cpu_usage() 39 | RAM = p.virtual_memory().used/1048576 40 | gpu_list.append(gpu_memory) 41 | RAM_list.append(RAM) 42 | cpu_list.append(cpu_usage) 43 | # with open(json_path, 'w')as f: 44 | # #js['gpu_memory'] = gpu_memory_max 45 | # js['gpu_memory'].append(gpu_memory) 46 | # json.dump(js, f, indent=4) 47 | time.sleep(time_interval) 48 | 49 | 50 | def save_result(start_time, sleep_time, json_path, gpu_list_ ,cpu_list_copy,RAM_list_copy): 51 | if os.path.exists(json_path): 52 | with open(json_path) as f: 53 | js = json.load(f) 54 | else: 55 | js = {"gpu_memory": []} 56 | # raise ValueError(f"{json_path} don't exist!") 57 | 58 | with open(json_path, "w") as f: 59 | # js['gpu_memory'] = gpu_memory_max 60 | js["gpu_memory"] = gpu_list_ 61 | js["cpu_list"] = cpu_list_copy 62 | js["RAM_list"] = RAM_list_copy 63 | 64 | json.dump(js, f, indent=4) 65 | 66 | infer_time = time.time() - start_time 67 | with open(json_path, "r") as f: 68 | js = json.load(f) 69 | with open(json_path, "w") as f: 70 | js["time"] = infer_time 71 | json.dump(js, f, indent=4) 72 | time.sleep(2) 73 | logger.info("save result end") 74 | 75 | 76 | if __name__ == "__main__": 77 | parser = argparse.ArgumentParser() 78 | parser.add_argument("-time_interval", default=0.1, help="time_interval") 79 | parser.add_argument("-sleep_time", default=5, help="sleep time") 80 | parser.add_argument("-shell_path", default="predict.sh", help="time_interval") 81 | # XXX: in case of someone use lower docker, please use specified GPU !!! 82 | parser.add_argument("-gpus", default=1, help="CUDA_VISIBLE_DEVICES") 83 | parser.add_argument("-docker_input_file", default="./inputs/", help="docker input folder") 84 | parser.add_argument("-save_file", default="results", help="data save folder") 85 | parser.add_argument("-docker_name", default="nnunet", help="docker output folder") 86 | args = parser.parse_args() 87 | logger.info(f"We are evaluating {args.docker_name}") 88 | json_dir = "./{}/{}".format(args.save_file,args.docker_name) 89 | json_path = os.path.join( 90 | json_dir, glob.glob(args.docker_input_file + "/*")[0].split("/")[-1].split(".")[0] + ".json", 91 | ) 92 | 93 | try: 94 | p1 = Process(target=daemon_process, args=(args.time_interval, json_path, args.gpus,)) 95 | p1.daemon = True 96 | p1.start() 97 | t0 = time.time() 98 | # XXX: in case of someone use lower docker, please use specified GPU !!! 99 | # cmd = 'docker container run --runtime="nvidia" -e NVIDIA_VISIBLE_DEVICES={} --name {} --rm -v $PWD/inputs/:/workspace/input/ -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/sh -c "sh {}"'.format( 100 | # args.gpus, args.docker_name, args.docker_name, args.shell_path 101 | # ) 102 | cmd = 'docker container run --gpus="device=1" --name {} --rm -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/bash -c "sh predict.sh" '.format(args.docker_name, args.docker_name) 103 | logger.info(f"cmd is : {cmd}") 104 | logger.info(f"cmd is : {cmd}") 105 | logger.info("start predict...") 106 | os.system(cmd) 107 | RAM_list= list(RAM_list) 108 | RAM_list_copy = RAM_list.copy() 109 | cpu_list= list(cpu_list) 110 | cpu_list_copy = cpu_list.copy() 111 | gpu_list = list(gpu_list) 112 | gpu_list_copy = gpu_list.copy() 113 | save_result(t0, args.sleep_time, json_path, gpu_list_copy ,cpu_list_copy,RAM_list_copy) 114 | time.sleep(args.sleep_time) 115 | except Exception as error: 116 | logger.exception(error) 117 | -------------------------------------------------------------------------------- /FLARE22/Evaluation/FLARE22_DSC_NSD_Eval.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Apr 15 12:59:48 2022 4 | 5 | @author: 12593 6 | """ 7 | 8 | import numpy as np 9 | import nibabel as nb 10 | import os 11 | join = os.path.join 12 | from collections import OrderedDict 13 | import pandas as pd 14 | from SurfaceDice import compute_surface_distances, compute_surface_dice_at_tolerance, compute_dice_coefficient 15 | 16 | 17 | seg_path = 'path to segmentation' 18 | gt_path = 'path to ground truth' 19 | save_path = 'path to Results' 20 | save_name = 'DSC_NSD_teamname.csv' 21 | filenames = os.listdir(seg_path) 22 | filenames = [x for x in filenames if x.endswith('.nii.gz')] 23 | filenames.sort() 24 | 25 | seg_metrics = OrderedDict() 26 | seg_metrics['Name'] = list() 27 | label_tolerance = OrderedDict({'Liver': 5, 'RK':3, 'Spleen':3, 'Pancreas':5, 28 | 'Aorta': 2, 'IVC':2, 'RAG':2, 'LAG':2, 'Gallbladder': 2, 29 | 'Esophagus':3, 'Stomach': 5, 'Duodenum': 7, 'LK':3}) 30 | for organ in label_tolerance.keys(): 31 | seg_metrics['{}_DSC'.format(organ)] = list() 32 | for organ in label_tolerance.keys(): 33 | seg_metrics['{}_NSD'.format(organ)] = list() 34 | 35 | def find_lower_upper_zbound(organ_mask): 36 | """ 37 | Parameters 38 | ---------- 39 | seg : TYPE 40 | DESCRIPTION. 41 | 42 | Returns 43 | ------- 44 | z_lower: lower bound in z axis: int 45 | z_upper: upper bound in z axis: int 46 | 47 | """ 48 | organ_mask = np.uint8(organ_mask) 49 | assert np.max(organ_mask) ==1, print('mask label error!') 50 | z_index = np.where(organ_mask>0)[2] 51 | z_lower = np.min(z_index) 52 | z_upper = np.max(z_index) 53 | 54 | return z_lower, z_upper 55 | 56 | 57 | 58 | for name in filenames: 59 | seg_metrics['Name'].append(name) 60 | # load grond truth and segmentation 61 | gt_nii = nb.load(join(gt_path, name)) 62 | case_spacing = gt_nii.header.get_zooms() 63 | gt_data = np.uint8(gt_nii.get_fdata()) 64 | seg_data = np.uint8(nb.load(join(seg_path, name)).get_fdata()) 65 | 66 | for i, organ in enumerate(label_tolerance.keys(),1): 67 | if np.sum(gt_data==i)==0 and np.sum(seg_data==i)==0: 68 | DSC_i = 1 69 | NSD_i = 1 70 | elif np.sum(gt_data==i)==0 and np.sum(seg_data==i)>0: 71 | DSC_i = 0 72 | NSD_i = 0 73 | elif np.sum(gt_data==i)>0 and np.sum(seg_data==i)==0: 74 | DSC_i = 0 75 | NSD_i = 0 76 | else: 77 | if i==5 or i==6 or i==10: # for Aorta, IVC, and Esophagus, only evaluate the labelled slices in ground truth 78 | z_lower, z_upper = find_lower_upper_zbound(gt_data==i) 79 | organ_i_gt, organ_i_seg = gt_data[:,:,z_lower:z_upper]==i, seg_data[:,:,z_lower:z_upper]==i 80 | else: 81 | organ_i_gt, organ_i_seg = gt_data==i, seg_data==i 82 | DSC_i = compute_dice_coefficient(organ_i_gt, organ_i_seg) 83 | if DSC_i < 0.2: 84 | NSD_i = 0 85 | else: 86 | surface_distances = compute_surface_distances(organ_i_gt, organ_i_seg, case_spacing) 87 | NSD_i = compute_surface_dice_at_tolerance(surface_distances, label_tolerance[organ]) 88 | seg_metrics['{}_DSC'.format(organ)].append(round(DSC_i, 4)) 89 | seg_metrics['{}_NSD'.format(organ)].append(round(NSD_i, 4)) 90 | print(name, organ, round(DSC_i,4), 'tol:', label_tolerance[organ], round(NSD_i,4)) 91 | 92 | dataframe = pd.DataFrame(seg_metrics) 93 | dataframe.to_csv(join(save_path, save_name), index=False) 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /FLARE22/Evaluation/Segmentation Efficiency Evaluation.md: -------------------------------------------------------------------------------- 1 | # Segmentation Efficiency Evaluation 2 | 3 | 4 | 5 | ### Enviroment and requirements 6 | 7 | ```python 8 | - python 3.8+ 9 | - torch 10 | - loguru 11 | - pynvml 12 | - psutil 13 | ``` 14 | 15 | 16 | 17 | Set `docker_path`, `test_img_path`, and `save_path` in `resource_eval.py` and run 18 | 19 | `nohup python resource_eval.py >> infos.log &` 20 | 21 | 22 | 23 | Q: How the `Area under GPU memory-time curve` and `Area under CPU utilization-time curve` is computed? 24 | 25 | > A: We record the GPU memory and GPU utilization every 0.1s. The `Area under GPU memory-time curve` and `Area under CPU utilization-time curve` are the cumulative values along running time. 26 | 27 | 28 | 29 | Note: For DSC and NSD, please refer to `FLARE22_DSC_NSD_Eval.py`. 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /FLARE22/Evaluation/SurfaceDice.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Apr 15 13:01:08 2022 4 | 5 | @author: 12593 6 | """ 7 | 8 | import numpy as np 9 | import scipy.ndimage 10 | 11 | # neighbour_code_to_normals is a lookup table. 12 | # For every binary neighbour code 13 | # (2x2x2 neighbourhood = 8 neighbours = 8 bits = 256 codes) 14 | # it contains the surface normals of the triangles (called "surfel" for 15 | # "surface element" in the following). The length of the normal 16 | # vector encodes the surfel area. 17 | # 18 | # created by compute_surface_area_lookup_table.ipynb using the 19 | # marching_cube algorithm, see e.g. https://en.wikipedia.org/wiki/Marching_cubes 20 | # credit to: http://medicaldecathlon.com/files/Surface_distance_based_measures.ipynb 21 | neighbour_code_to_normals = [ 22 | [[0,0,0]], 23 | [[0.125,0.125,0.125]], 24 | [[-0.125,-0.125,0.125]], 25 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 26 | [[0.125,-0.125,0.125]], 27 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 28 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 29 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 30 | [[-0.125,0.125,0.125]], 31 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 32 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 33 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 34 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 35 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 36 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 37 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 38 | [[0.125,-0.125,-0.125]], 39 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 40 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 41 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 42 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 43 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 44 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 45 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 46 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 47 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 48 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 49 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 50 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 51 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 52 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 53 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 54 | [[0.125,-0.125,0.125]], 55 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 56 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 57 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 58 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 59 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 60 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 61 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 62 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 63 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 64 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 65 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 66 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 67 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 68 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 69 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 70 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 71 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 72 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 73 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 74 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 75 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 76 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 77 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 78 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 79 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 80 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 81 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 82 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 83 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 84 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 85 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 86 | [[-0.125,-0.125,0.125]], 87 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 88 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 89 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 90 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 91 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 92 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 93 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 94 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 95 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 96 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 97 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 98 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 99 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 100 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 101 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 102 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 103 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 104 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 105 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 106 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 107 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 108 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 109 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 110 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 111 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 112 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 113 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 114 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 115 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 116 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 117 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 118 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 119 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 120 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 121 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 122 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 123 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 124 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 125 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 126 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 127 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 128 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 129 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 130 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 131 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 132 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 133 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 134 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 135 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 136 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 137 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 138 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 139 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 140 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 141 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 142 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 143 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 144 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 145 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 146 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 147 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 148 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 149 | [[0.125,0.125,0.125]], 150 | [[0.125,0.125,0.125]], 151 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 152 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 153 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 154 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 155 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 156 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 157 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 158 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 159 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 160 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 161 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 162 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 163 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 164 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 165 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 166 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 167 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 168 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 169 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 170 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 171 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 172 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 173 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 174 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 175 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.0,0.25,0.25],[0.0,0.25,0.25]], 176 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 177 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 178 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 179 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 180 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 181 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 182 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 183 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 184 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 185 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 186 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 187 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.25,0.0,0.25],[0.25,0.0,0.25]], 188 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 189 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 190 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 191 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 192 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 193 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 194 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 195 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 196 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 197 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 198 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 199 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 200 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 201 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 202 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 203 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 204 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 205 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 206 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 207 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 208 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 209 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 210 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 211 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 212 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 213 | [[-0.125,-0.125,0.125]], 214 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 215 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 216 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 217 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 218 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 219 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 220 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 221 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 222 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 223 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 224 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 225 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 226 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 227 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 228 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 229 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 230 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 231 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 232 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 233 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 234 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 235 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 236 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 237 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 238 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 239 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 240 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 241 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 242 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 243 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 244 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 245 | [[0.125,-0.125,0.125]], 246 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 247 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 248 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 249 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 250 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 251 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 252 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 253 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 254 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 255 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 256 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 257 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 258 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 259 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 260 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 261 | [[0.125,-0.125,-0.125]], 262 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 263 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 264 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 265 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 266 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 267 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 268 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 269 | [[-0.125,0.125,0.125]], 270 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 271 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 272 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 273 | [[0.125,-0.125,0.125]], 274 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 275 | [[-0.125,-0.125,0.125]], 276 | [[0.125,0.125,0.125]], 277 | [[0,0,0]]] 278 | 279 | 280 | def compute_surface_distances(mask_gt, mask_pred, spacing_mm): 281 | """Compute closest distances from all surface points to the other surface. 282 | 283 | Finds all surface elements "surfels" in the ground truth mask `mask_gt` and 284 | the predicted mask `mask_pred`, computes their area in mm^2 and the distance 285 | to the closest point on the other surface. It returns two sorted lists of 286 | distances together with the corresponding surfel areas. If one of the masks 287 | is empty, the corresponding lists are empty and all distances in the other 288 | list are `inf` 289 | 290 | Args: 291 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 292 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 293 | spacing_mm: 3-element list-like structure. Voxel spacing in x0, x1 and x2 294 | direction 295 | 296 | Returns: 297 | A dict with 298 | "distances_gt_to_pred": 1-dim numpy array of type float. The distances in mm 299 | from all ground truth surface elements to the predicted surface, 300 | sorted from smallest to largest 301 | "distances_pred_to_gt": 1-dim numpy array of type float. The distances in mm 302 | from all predicted surface elements to the ground truth surface, 303 | sorted from smallest to largest 304 | "surfel_areas_gt": 1-dim numpy array of type float. The area in mm^2 of 305 | the ground truth surface elements in the same order as 306 | distances_gt_to_pred 307 | "surfel_areas_pred": 1-dim numpy array of type float. The area in mm^2 of 308 | the predicted surface elements in the same order as 309 | distances_pred_to_gt 310 | 311 | """ 312 | 313 | # compute the area for all 256 possible surface elements 314 | # (given a 2x2x2 neighbourhood) according to the spacing_mm 315 | neighbour_code_to_surface_area = np.zeros([256]) 316 | for code in range(256): 317 | normals = np.array(neighbour_code_to_normals[code]) 318 | sum_area = 0 319 | for normal_idx in range(normals.shape[0]): 320 | # normal vector 321 | n = np.zeros([3]) 322 | n[0] = normals[normal_idx,0] * spacing_mm[1] * spacing_mm[2] 323 | n[1] = normals[normal_idx,1] * spacing_mm[0] * spacing_mm[2] 324 | n[2] = normals[normal_idx,2] * spacing_mm[0] * spacing_mm[1] 325 | area = np.linalg.norm(n) 326 | sum_area += area 327 | neighbour_code_to_surface_area[code] = sum_area 328 | 329 | # compute the bounding box of the masks to trim 330 | # the volume to the smallest possible processing subvolume 331 | mask_all = mask_gt | mask_pred 332 | bbox_min = np.zeros(3, np.int64) 333 | bbox_max = np.zeros(3, np.int64) 334 | 335 | # max projection to the x0-axis 336 | proj_0 = np.max(np.max(mask_all, axis=2), axis=1) 337 | idx_nonzero_0 = np.nonzero(proj_0)[0] 338 | if len(idx_nonzero_0) == 0: 339 | return {"distances_gt_to_pred": np.array([]), 340 | "distances_pred_to_gt": np.array([]), 341 | "surfel_areas_gt": np.array([]), 342 | "surfel_areas_pred": np.array([])} 343 | 344 | bbox_min[0] = np.min(idx_nonzero_0) 345 | bbox_max[0] = np.max(idx_nonzero_0) 346 | 347 | # max projection to the x1-axis 348 | proj_1 = np.max(np.max(mask_all, axis=2), axis=0) 349 | idx_nonzero_1 = np.nonzero(proj_1)[0] 350 | bbox_min[1] = np.min(idx_nonzero_1) 351 | bbox_max[1] = np.max(idx_nonzero_1) 352 | 353 | # max projection to the x2-axis 354 | proj_2 = np.max(np.max(mask_all, axis=1), axis=0) 355 | idx_nonzero_2 = np.nonzero(proj_2)[0] 356 | bbox_min[2] = np.min(idx_nonzero_2) 357 | bbox_max[2] = np.max(idx_nonzero_2) 358 | 359 | # print("bounding box min = {}".format(bbox_min)) 360 | # print("bounding box max = {}".format(bbox_max)) 361 | 362 | # crop the processing subvolume. 363 | # we need to zeropad the cropped region with 1 voxel at the lower, 364 | # the right and the back side. This is required to obtain the "full" 365 | # convolution result with the 2x2x2 kernel 366 | cropmask_gt = np.zeros((bbox_max - bbox_min)+2, np.uint8) 367 | cropmask_pred = np.zeros((bbox_max - bbox_min)+2, np.uint8) 368 | 369 | cropmask_gt[0:-1, 0:-1, 0:-1] = mask_gt[bbox_min[0]:bbox_max[0]+1, 370 | bbox_min[1]:bbox_max[1]+1, 371 | bbox_min[2]:bbox_max[2]+1] 372 | 373 | cropmask_pred[0:-1, 0:-1, 0:-1] = mask_pred[bbox_min[0]:bbox_max[0]+1, 374 | bbox_min[1]:bbox_max[1]+1, 375 | bbox_min[2]:bbox_max[2]+1] 376 | 377 | # compute the neighbour code (local binary pattern) for each voxel 378 | # the resultsing arrays are spacially shifted by minus half a voxel in each axis. 379 | # i.e. the points are located at the corners of the original voxels 380 | kernel = np.array([[[128,64], 381 | [32,16]], 382 | [[8,4], 383 | [2,1]]]) 384 | neighbour_code_map_gt = scipy.ndimage.filters.correlate(cropmask_gt.astype(np.uint8), kernel, mode="constant", cval=0) 385 | neighbour_code_map_pred = scipy.ndimage.filters.correlate(cropmask_pred.astype(np.uint8), kernel, mode="constant", cval=0) 386 | 387 | # create masks with the surface voxels 388 | borders_gt = ((neighbour_code_map_gt != 0) & (neighbour_code_map_gt != 255)) 389 | borders_pred = ((neighbour_code_map_pred != 0) & (neighbour_code_map_pred != 255)) 390 | 391 | # compute the distance transform (closest distance of each voxel to the surface voxels) 392 | if borders_gt.any(): 393 | distmap_gt = scipy.ndimage.morphology.distance_transform_edt(~borders_gt, sampling=spacing_mm) 394 | else: 395 | distmap_gt = np.Inf * np.ones(borders_gt.shape) 396 | 397 | if borders_pred.any(): 398 | distmap_pred = scipy.ndimage.morphology.distance_transform_edt(~borders_pred, sampling=spacing_mm) 399 | else: 400 | distmap_pred = np.Inf * np.ones(borders_pred.shape) 401 | 402 | # compute the area of each surface element 403 | surface_area_map_gt = neighbour_code_to_surface_area[neighbour_code_map_gt] 404 | surface_area_map_pred = neighbour_code_to_surface_area[neighbour_code_map_pred] 405 | 406 | # create a list of all surface elements with distance and area 407 | distances_gt_to_pred = distmap_pred[borders_gt] 408 | distances_pred_to_gt = distmap_gt[borders_pred] 409 | surfel_areas_gt = surface_area_map_gt[borders_gt] 410 | surfel_areas_pred = surface_area_map_pred[borders_pred] 411 | 412 | # sort them by distance 413 | if distances_gt_to_pred.shape != (0,): 414 | sorted_surfels_gt = np.array(sorted(zip(distances_gt_to_pred, surfel_areas_gt))) 415 | distances_gt_to_pred = sorted_surfels_gt[:,0] 416 | surfel_areas_gt = sorted_surfels_gt[:,1] 417 | 418 | if distances_pred_to_gt.shape != (0,): 419 | sorted_surfels_pred = np.array(sorted(zip(distances_pred_to_gt, surfel_areas_pred))) 420 | distances_pred_to_gt = sorted_surfels_pred[:,0] 421 | surfel_areas_pred = sorted_surfels_pred[:,1] 422 | 423 | 424 | return {"distances_gt_to_pred": distances_gt_to_pred, 425 | "distances_pred_to_gt": distances_pred_to_gt, 426 | "surfel_areas_gt": surfel_areas_gt, 427 | "surfel_areas_pred": surfel_areas_pred} 428 | 429 | 430 | def compute_average_surface_distance(surface_distances): 431 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 432 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 433 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 434 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 435 | average_distance_gt_to_pred = np.sum( distances_gt_to_pred * surfel_areas_gt) / np.sum(surfel_areas_gt) 436 | average_distance_pred_to_gt = np.sum( distances_pred_to_gt * surfel_areas_pred) / np.sum(surfel_areas_pred) 437 | return (average_distance_gt_to_pred, average_distance_pred_to_gt) 438 | 439 | def compute_robust_hausdorff(surface_distances, percent): 440 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 441 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 442 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 443 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 444 | if len(distances_gt_to_pred) > 0: 445 | surfel_areas_cum_gt = np.cumsum(surfel_areas_gt) / np.sum(surfel_areas_gt) 446 | idx = np.searchsorted(surfel_areas_cum_gt, percent/100.0) 447 | perc_distance_gt_to_pred = distances_gt_to_pred[min(idx, len(distances_gt_to_pred)-1)] 448 | else: 449 | perc_distance_gt_to_pred = np.Inf 450 | 451 | if len(distances_pred_to_gt) > 0: 452 | surfel_areas_cum_pred = np.cumsum(surfel_areas_pred) / np.sum(surfel_areas_pred) 453 | idx = np.searchsorted(surfel_areas_cum_pred, percent/100.0) 454 | perc_distance_pred_to_gt = distances_pred_to_gt[min(idx, len(distances_pred_to_gt)-1)] 455 | else: 456 | perc_distance_pred_to_gt = np.Inf 457 | 458 | return max( perc_distance_gt_to_pred, perc_distance_pred_to_gt) 459 | 460 | def compute_surface_overlap_at_tolerance(surface_distances, tolerance_mm): 461 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 462 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 463 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 464 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 465 | rel_overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) / np.sum(surfel_areas_gt) 466 | rel_overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) / np.sum(surfel_areas_pred) 467 | return (rel_overlap_gt, rel_overlap_pred) 468 | 469 | def compute_surface_dice_at_tolerance(surface_distances, tolerance_mm): 470 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 471 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 472 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 473 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 474 | overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) 475 | overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) 476 | surface_dice = (overlap_gt + overlap_pred) / ( 477 | np.sum(surfel_areas_gt) + np.sum(surfel_areas_pred)) 478 | return surface_dice 479 | 480 | 481 | def compute_dice_coefficient(mask_gt, mask_pred): 482 | """Compute soerensen-dice coefficient. 483 | 484 | compute the soerensen-dice coefficient between the ground truth mask `mask_gt` 485 | and the predicted mask `mask_pred`. 486 | 487 | Args: 488 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 489 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 490 | 491 | Returns: 492 | the dice coeffcient as float. If both masks are empty, the result is NaN 493 | """ 494 | volume_sum = mask_gt.sum() + mask_pred.sum() 495 | if volume_sum == 0: 496 | return np.NaN 497 | volume_intersect = (mask_gt & mask_pred).sum() 498 | return 2*volume_intersect / volume_sum 499 | -------------------------------------------------------------------------------- /FLARE22/Evaluation/load_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | import csv 3 | import argparse 4 | import glob 5 | import matplotlib 6 | import os 7 | join = os.path.join 8 | import numpy as np 9 | 10 | matplotlib.use("Agg") 11 | import matplotlib.pyplot as plt 12 | 13 | from logger import add_file_handler_to_logger, logger 14 | 15 | add_file_handler_to_logger(name="main", dir_path=f"logs/", level="DEBUG") 16 | 17 | 18 | if __name__ == "__main__": 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("-docker_name", default="fully_suplearn_subtask1", help="team docker name") 21 | parser.add_argument("-save_path", default="./results", help="save_path") 22 | time_interval = 0.1 23 | args = parser.parse_args() 24 | logger.info("we are counting: {args.docker_name}") 25 | json_dir = join(args.save_path, args.docker_name) 26 | csv_path = join(json_dir, args.docker_name + '_Efficiency.csv') 27 | jsonl = sorted(glob.glob(json_dir + "/*.json")) 28 | alldata = [] 29 | for item in jsonl: 30 | csv_l = [] 31 | name = item.split(os.sep)[-1].split(".")[0] 32 | csv_l.append(name + '.nii.gz') 33 | zitem = item 34 | with open(zitem) as f: 35 | try: 36 | js = json.load(f) 37 | except Exception as error: 38 | logger.error(f"{item} have error") 39 | logger.exception(error) 40 | if "time" not in js: 41 | logger.error(f"{item} don't have time!!!!") 42 | logger.info(f"Manually compute {item}") 43 | time = time_interval * len(js["gpu_memory"]) 44 | else: 45 | time = js["time"] 46 | csv_l.append(np.round(time,2)) 47 | #CPU 48 | user, system, all_cpu_used = [item[0] for item in js['cpu_list']], [item[1] for item in js['cpu_list']], [ 49 | 100 - item[2] for item in js['cpu_list']] 50 | plt.cla() 51 | x = [item * time_interval for item in range(len(user))] 52 | plt.xlabel("Time (s)", fontsize="large") 53 | plt.ylabel("CPU Utlization (%)", fontsize="large") 54 | # plt.plot(x, user, "b", ms=10, label="User %") 55 | # plt.plot(x, system, "r", ms=10, label="System %") 56 | plt.plot(x, all_cpu_used, "b", ms=10, label="Used %") 57 | plt.legend() 58 | plt.savefig(zitem.replace(".json", "_CPU-Time.png")) 59 | #RAM 60 | RAM = js["RAM_list"] 61 | plt.cla() 62 | x = [item * time_interval for item in range(len(RAM))] 63 | plt.xlabel("Time (s)", fontsize="large") 64 | plt.ylabel("RAM (MB)", fontsize="large") 65 | plt.plot(x, RAM, "b", ms=10, label="RAM") 66 | plt.legend() 67 | plt.savefig(zitem.replace(".json", "_RAM-Time.png")) 68 | mem = js["gpu_memory"] 69 | x = [item * time_interval for item in range(len(mem))] 70 | plt.cla() 71 | plt.xlabel("Time (s)", fontsize="large") 72 | plt.ylabel("GPU Memory (MB)", fontsize="large") 73 | plt.plot(x, mem, "b", ms=10, label="a") 74 | plt.savefig(zitem.replace(".json", "_GPU-Time.png")) 75 | count_set = set(mem) 76 | count_list = [] 77 | for citem in count_set: 78 | cts = mem.count(citem) 79 | if cts > 0.02 * len(mem): 80 | count_list.append(citem) 81 | max_mem = max(count_list) 82 | csv_l.append(np.round(max_mem)) 83 | csv_l.append(np.round(sum(mem) * time_interval)) 84 | csv_l.append(np.round(max(all_cpu_used),2)) 85 | csv_l.append(np.round(sum(all_cpu_used) * time_interval,2)) 86 | csv_l.append(np.round(max(RAM),2)) 87 | csv_l.append(np.round(sum(RAM) * time_interval)) 88 | alldata.append(csv_l) 89 | 90 | f = open(csv_path, "w",newline='') 91 | writer = csv.writer(f) 92 | writer.writerow(["Name", "Time", "GPU_Mem", "AUC_GPU_Time",'CPU_Utilization','AUC_CPU_Time','RAM' 93 | ,'AUC_RAM_Time' ]) 94 | for i in alldata: 95 | writer.writerow(i) 96 | f.close() 97 | -------------------------------------------------------------------------------- /FLARE22/Evaluation/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from loguru import logger as loguru_logger 5 | 6 | loguru_logger.configure( 7 | handlers=[dict(sink=sys.stderr, filter=lambda record: record["extra"]["console"], level="DEBUG",),], 8 | ) 9 | 10 | 11 | logger = loguru_logger.bind(console=True) 12 | 13 | 14 | def add_file_handler_to_logger( 15 | name: str, dir_path="./logs", level="DEBUG", 16 | ): 17 | loguru_logger.add( 18 | sink=os.path.join(dir_path, f"{name}-{level}.log"), 19 | level=level, 20 | filter=lambda record: "console" in record["extra"], 21 | enqueue=True, 22 | ) 23 | -------------------------------------------------------------------------------- /FLARE22/Evaluation/resource_eval.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | import time 5 | import torch 6 | from pathlib import Path 7 | join = os.path.join 8 | from logger import add_file_handler_to_logger, logger 9 | 10 | 11 | def check_dir(file_path): 12 | file_path = Path(file_path) 13 | files = [f for f in file_path.iterdir() if ".nii.gz" in str(f)] 14 | if len(files) != 0: 15 | return False 16 | return True 17 | 18 | 19 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 20 | 21 | docker_path = './team_docker' 22 | test_img_path = './test_demo/' 23 | save_path = './results/' 24 | 25 | os.makedirs(save_path, exist_ok=True) 26 | os.makedirs('./inputs/', exist_ok=True) 27 | os.makedirs('./outputs/', exist_ok=True) 28 | 29 | dockers = sorted(os.listdir(docker_path)) 30 | test_cases = sorted(os.listdir(test_img_path)) 31 | 32 | for docker in dockers: 33 | try: 34 | name = docker.split('.')[0].lower() 35 | print('teamname docker: ', docker) 36 | os.system('docker image load < {}'.format(join(docker_path, docker))) 37 | team_outpath = join(save_path, name) 38 | if os.path.exists(team_outpath): 39 | shutil.rmtree(team_outpath) 40 | os.mkdir(team_outpath) 41 | for case in test_cases: 42 | if not check_dir('./inputs'): 43 | logger.error("please check inputs folder") 44 | raise 45 | shutil.copy(join(test_img_path, case), './inputs') 46 | start_time = time.time() 47 | os.system('python Efficiency.py -docker_name {} -save_file {}'.format(name, save_path)) 48 | logger.info(f"{case} finished!") 49 | os.remove(join('./inputs', case)) 50 | # shutil.rmtree('./inputs') 51 | logger.info(f"{case} cost time: {time.time() - start_time}") 52 | 53 | os.system("python load_json.py -docker_name {} -save_path {}".format(name, save_path)) 54 | shutil.move("./outputs", team_outpath) 55 | os.mkdir("./outputs") 56 | torch.cuda.empty_cache() 57 | # os.system("docker rmi {}:latest".format(name)) 58 | except Exception as e: 59 | logger.exception(e) 60 | -------------------------------------------------------------------------------- /FLARE23/Efficiency.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import json 4 | import os 5 | import time 6 | import psutil as p 7 | from multiprocessing import Manager, Process 8 | from pynvml.smi import nvidia_smi 9 | import numpy as np 10 | from logger import add_file_handler_to_logger, logger 11 | 12 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 13 | 14 | manager = Manager() 15 | gpu_list = manager.list() 16 | cpu_list = manager.list() 17 | RAM_list = manager.list() 18 | def cpu_usage(): 19 | t = p.cpu_times() 20 | return [t.user, t.system, t.idle] 21 | before = cpu_usage() 22 | def get_cpu_usage(): 23 | global before 24 | now = cpu_usage() 25 | delta = [now[i] - before[i] for i in range(len(now))] 26 | total = sum(delta) 27 | before = now 28 | return [(100.0*dt)/(total+0.1) for dt in delta] 29 | 30 | 31 | def daemon_process(time_interval, json_path, gpu_index=1): 32 | while True: 33 | nvsmi = nvidia_smi.getInstance() 34 | dictm = nvsmi.DeviceQuery("memory.free, memory.total") 35 | gpu_memory = ( 36 | dictm["gpu"][gpu_index]["fb_memory_usage"]["total"] - dictm["gpu"][gpu_index]["fb_memory_usage"]["free"] 37 | ) 38 | cpu_usage =get_cpu_usage() 39 | RAM = p.virtual_memory().used/1048576 40 | gpu_list.append(gpu_memory) 41 | RAM_list.append(RAM) 42 | cpu_list.append(cpu_usage) 43 | # with open(json_path, 'w')as f: 44 | # #js['gpu_memory'] = gpu_memory_max 45 | # js['gpu_memory'].append(gpu_memory) 46 | # json.dump(js, f, indent=4) 47 | time.sleep(time_interval) 48 | 49 | 50 | def save_result(start_time, sleep_time, json_path, gpu_list_ , cpu_list_copy, RAM_list_copy): 51 | if os.path.exists(json_path): 52 | with open(json_path) as f: 53 | js = json.load(f) 54 | else: 55 | js = {"gpu_memory": []} 56 | # raise ValueError(f"{json_path} don't exist!") 57 | 58 | with open(json_path, "w") as f: 59 | # js['gpu_memory'] = gpu_memory_max 60 | js["gpu_memory"] = gpu_list_ 61 | js["cpu_list"] = cpu_list_copy 62 | js["RAM_list"] = RAM_list_copy 63 | 64 | json.dump(js, f, indent=4) 65 | 66 | infer_time = time.time() - start_time 67 | with open(json_path, "r") as f: 68 | js = json.load(f) 69 | with open(json_path, "w") as f: 70 | js["time"] = infer_time 71 | json.dump(js, f, indent=4) 72 | time.sleep(2) 73 | logger.info("save result end") 74 | 75 | 76 | if __name__ == "__main__": 77 | parser = argparse.ArgumentParser() 78 | parser.add_argument("-time_interval", default=0.1, help="time_interval") 79 | parser.add_argument("-sleep_time", default=5, help="sleep time") 80 | parser.add_argument("-shell_path", default="predict.sh", help="time_interval") 81 | # XXX: in case of someone use lower docker, please use specified GPU !!! 82 | parser.add_argument("-gpus", default=1, help="CUDA_VISIBLE_DEVICES") 83 | parser.add_argument("-docker_input_file", default="./inputs/", help="docker input folder") 84 | parser.add_argument("-save_file", default="results", help="data save folder") 85 | parser.add_argument("-docker_name", default="nnunet", help="docker output folder") 86 | args = parser.parse_args() 87 | logger.info(f"We are evaluating {args.docker_name}") 88 | json_dir = "./{}/{}".format(args.save_file,args.docker_name) 89 | json_path = os.path.join( 90 | json_dir, glob.glob(args.docker_input_file + "/*.nii.gz")[0].split("/")[-1].split(".nii.gz")[0] + ".json", 91 | ) 92 | 93 | try: 94 | p1 = Process(target=daemon_process, args=(args.time_interval, json_path, args.gpus,)) 95 | p1.daemon = True 96 | p1.start() 97 | t0 = time.time() 98 | # XXX: in case of someone use lower docker, please use specified GPU !!! 99 | # cmd = 'docker container run --runtime="nvidia" -e NVIDIA_VISIBLE_DEVICES={} --name {} --rm -v $PWD/inputs/:/workspace/input/ -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/sh -c "sh {}"'.format( 100 | # args.gpus, args.docker_name, args.docker_name, args.shell_path 101 | # ) 102 | cmd = 'docker container run --shm-size=28gb -m 28G --gpus="device=1" --name {} --rm -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/bash -c "sh predict.sh" '.format(args.docker_name, args.docker_name) 103 | logger.info(f"cmd is : {cmd}") 104 | logger.info("start predict...") 105 | os.system(cmd) 106 | RAM_list= list(RAM_list) 107 | RAM_list_copy = RAM_list.copy() 108 | cpu_list= list(cpu_list) 109 | cpu_list_copy = cpu_list.copy() 110 | gpu_list = list(gpu_list) 111 | gpu_list_copy = gpu_list.copy() 112 | print(f"{json_path=}") 113 | save_result(t0, args.sleep_time, json_path, gpu_list_copy ,cpu_list_copy,RAM_list_copy) 114 | time.sleep(args.sleep_time) 115 | except Exception as error: 116 | logger.exception(error) 117 | -------------------------------------------------------------------------------- /FLARE23/FLARE23_DSC_NSD_Eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import os 4 | import nibabel as nb 5 | import numpy as np 6 | import glob 7 | import gc 8 | from collections import OrderedDict 9 | from SurfaceDice import compute_surface_distances, compute_surface_dice_at_tolerance, compute_dice_coefficient 10 | 11 | 12 | def find_lower_upper_zbound(organ_mask): 13 | """ 14 | Parameters 15 | ---------- 16 | seg : TYPE 17 | DESCRIPTION. 18 | Returns 19 | ------- 20 | z_lower: lower bound in z axis: int 21 | z_upper: upper bound in z axis: int 22 | """ 23 | organ_mask = np.uint8(organ_mask) 24 | assert np.max(organ_mask) ==1, print('mask label error!') 25 | z_index = np.where(organ_mask>0)[2] 26 | z_lower = np.min(z_index) 27 | z_upper = np.max(z_index) 28 | 29 | return z_lower, z_upper 30 | 31 | 32 | # Check input directories. 33 | submit_dir = os.path.join(sys.argv[1], 'res') 34 | truth_dir = os.path.join(sys.argv[1], 'ref') 35 | if not os.path.isdir(submit_dir): 36 | print("submit_dir {} doesn't exist".format(submit_dir)) 37 | sys.exit() 38 | if not os.path.isdir(truth_dir): 39 | print("truth_dir {} doesn't exist".format(submit_dir)) 40 | sys.exit() 41 | 42 | # Create output directory. 43 | output_dir = sys.argv[2] 44 | if not os.path.exists(output_dir): 45 | os.makedirs(output_dir) 46 | 47 | # -------------------------- flare metrics 48 | seg_metrics = OrderedDict() 49 | seg_metrics['Name'] = list() 50 | label_tolerance = OrderedDict({'Liver': 5, 'RK':3, 'Spleen':3, 'Pancreas':5, 51 | 'Aorta': 2, 'IVC':2, 'RAG':2, 'LAG':2, 'Gallbladder': 2, 52 | 'Esophagus':3, 'Stomach': 5, 'Duodenum': 7, 'LK':3, 'Lesion':2}) 53 | for organ in label_tolerance.keys(): 54 | seg_metrics['{}_DSC'.format(organ)] = list() 55 | for organ in label_tolerance.keys(): 56 | seg_metrics['{}_NSD'.format(organ)] = list() 57 | # -------------------------- flare metrics 58 | 59 | # Iterate over all volumes in the reference list. 60 | reference_volume_list = sorted(glob.glob(truth_dir+'/*.nii.gz')) 61 | for reference_volume_fn in reference_volume_list: 62 | print("Starting with volume {}".format(reference_volume_fn)) 63 | submission_volume_path = os.path.join(submit_dir, os.path.basename(reference_volume_fn)) 64 | if not os.path.exists(submission_volume_path): 65 | raise ValueError("Submission volume not found - terminating!\n" 66 | "Missing volume: {}".format(submission_volume_path)) 67 | print("Found corresponding submission file {} for reference file {}" 68 | "".format(reference_volume_fn, submission_volume_path)) 69 | 70 | # Load reference and submission volumes with Nibabel. 71 | reference_volume = nb.load(reference_volume_fn) 72 | submission_volume = nb.load(submission_volume_path) 73 | 74 | # Get the current voxel spacing. 75 | voxel_spacing = reference_volume.header.get_zooms()[:3] 76 | 77 | # Get Numpy data and compress to int8. 78 | reference_volume = (reference_volume.get_fdata()).astype(np.int8) 79 | submission_volume = (submission_volume.get_fdata()).astype(np.int8) 80 | 81 | # Ensure that the shapes of the masks match. 82 | if submission_volume.shape!=reference_volume.shape: 83 | raise AttributeError("Shapes do not match! Prediction mask {}, " 84 | "ground truth mask {}" 85 | "".format(submission_volume.shape, 86 | reference_volume.shape)) 87 | print("Done loading files ({:.2f} seconds)".format(t())) 88 | 89 | # ----------------------- flare metrics 90 | seg_metrics['Name'].append(os.path.basename(reference_volume_fn)) 91 | for i, organ in enumerate(label_tolerance.keys(),1): 92 | if np.sum(reference_volume==i)==0 and np.sum(submission_volume==i)==0: 93 | DSC_i = 1 94 | NSD_i = 1 95 | elif np.sum(reference_volume==i)==0 and np.sum(submission_volume==i)>0: 96 | DSC_i = 0 97 | NSD_i = 0 98 | elif np.sum(reference_volume==i)>0 and np.sum(submission_volume==i)==0: 99 | DSC_i = 0 100 | NSD_i = 0 101 | else: 102 | if i==5 or i==6 or i==10: # for Aorta, IVC, and Esophagus, only evaluate the labelled slices in ground truth 103 | z_lower, z_upper = find_lower_upper_zbound(reference_volume==i) 104 | organ_i_gt, organ_i_seg = reference_volume[:,:,z_lower:z_upper]==i, submission_volume[:,:,z_lower:z_upper]==i 105 | else: 106 | organ_i_gt, organ_i_seg = reference_volume==i, submission_volume==i 107 | DSC_i = compute_dice_coefficient(organ_i_gt, organ_i_seg) 108 | if DSC_i < 0.2: 109 | NSD_i = 0 110 | else: 111 | surface_distances = compute_surface_distances(organ_i_gt, organ_i_seg, voxel_spacing) 112 | NSD_i = compute_surface_dice_at_tolerance(surface_distances, label_tolerance[organ]) 113 | seg_metrics['{}_DSC'.format(organ)].append(round(DSC_i, 4)) 114 | seg_metrics['{}_NSD'.format(organ)].append(round(NSD_i, 4)) 115 | # print(name, organ, round(DSC_i,4), 'tol:', label_tolerance[organ], round(NSD_i,4)) 116 | # ----------------------- flare metrics 117 | 118 | 119 | print("Done processing volume (total time: {:.2f} seconds)" 120 | "".format(t.total_elapsed())) 121 | gc.collect() 122 | 123 | overall_metrics = {} 124 | for key, value in seg_metrics.items(): 125 | if 'Name' not in key: 126 | overall_metrics[key] = round(np.mean(value), 4) 127 | 128 | organ_dsc = [] 129 | organ_nsd = [] 130 | for key, value in overall_metrics.items(): 131 | if 'Lesion' not in key: 132 | if 'DSC' in key: 133 | organ_dsc.append(value) 134 | if 'NSD' in key: 135 | organ_nsd.append(value) 136 | overall_metrics['Organ_DSC'] = round(np.mean(organ_dsc), 4) 137 | overall_metrics['Organ_NSD'] = round(np.mean(organ_nsd), 4) 138 | 139 | print("Computed metrics:") 140 | for key, value in overall_metrics.items(): 141 | print("{}: {:.4f}".format(key, float(value))) 142 | 143 | # Write metrics to file. 144 | output_filename = os.path.join(output_dir, 'scores.txt') 145 | output_file = open(output_filename, 'w') 146 | for key, value in overall_metrics.items(): 147 | output_file.write("{}: {:.4f}\n".format(key, float(value))) 148 | output_file.close() 149 | -------------------------------------------------------------------------------- /FLARE23/README.md: -------------------------------------------------------------------------------- 1 | # Segmentation Efficiency Evaluation 2 | 3 | 4 | 5 | ### Enviroment and requirements 6 | 7 | ```python 8 | - python 3.8+ 9 | - torch 10 | - loguru 11 | - pynvml 12 | - psutil 13 | ``` 14 | 15 | 16 | 17 | Set `docker_path`, `test_img_path`, and `save_path` in `resource_eval.py` and run 18 | 19 | `nohup python resource_eval.py >> infos.log &` 20 | 21 | 22 | 23 | Q: How the `Area under GPU memory-time curve` is computed? 24 | 25 | > A: We record the GPU memory and GPU utilization every 0.1s. The `Area under GPU memory-time curve` and `Area under CPU utilization-time curve` are the cumulative values along running time. 26 | 27 | 28 | 29 | Note: 30 | - The above command will also generate the evaluation results of CPU and RAM usage. They are used for debug purpose (not for ranking purpose). 31 | - For DSC and NSD, please refer to `FLARE23_DSC_NSD_Eval.py`. 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /FLARE23/SurfaceDice.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Apr 15 13:01:08 2022 4 | 5 | @author: 12593 6 | """ 7 | 8 | import numpy as np 9 | import scipy.ndimage 10 | 11 | # neighbour_code_to_normals is a lookup table. 12 | # For every binary neighbour code 13 | # (2x2x2 neighbourhood = 8 neighbours = 8 bits = 256 codes) 14 | # it contains the surface normals of the triangles (called "surfel" for 15 | # "surface element" in the following). The length of the normal 16 | # vector encodes the surfel area. 17 | # 18 | # created by compute_surface_area_lookup_table.ipynb using the 19 | # marching_cube algorithm, see e.g. https://en.wikipedia.org/wiki/Marching_cubes 20 | # credit to: http://medicaldecathlon.com/files/Surface_distance_based_measures.ipynb 21 | neighbour_code_to_normals = [ 22 | [[0,0,0]], 23 | [[0.125,0.125,0.125]], 24 | [[-0.125,-0.125,0.125]], 25 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 26 | [[0.125,-0.125,0.125]], 27 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 28 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 29 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 30 | [[-0.125,0.125,0.125]], 31 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 32 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 33 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 34 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 35 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 36 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 37 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 38 | [[0.125,-0.125,-0.125]], 39 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 40 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 41 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 42 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 43 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 44 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 45 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 46 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 47 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 48 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 49 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 50 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 51 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 52 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 53 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 54 | [[0.125,-0.125,0.125]], 55 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 56 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 57 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 58 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 59 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 60 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 61 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 62 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 63 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 64 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 65 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 66 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 67 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 68 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 69 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 70 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 71 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 72 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 73 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 74 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 75 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 76 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 77 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 78 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 79 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 80 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 81 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 82 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 83 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 84 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 85 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 86 | [[-0.125,-0.125,0.125]], 87 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 88 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 89 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 90 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 91 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 92 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 93 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 94 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 95 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 96 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 97 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 98 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 99 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 100 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 101 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 102 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 103 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 104 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 105 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 106 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 107 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 108 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 109 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 110 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 111 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 112 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 113 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 114 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 115 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 116 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 117 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 118 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 119 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 120 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 121 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 122 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 123 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 124 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 125 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 126 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 127 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 128 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 129 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 130 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 131 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 132 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 133 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 134 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 135 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 136 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 137 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 138 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 139 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 140 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 141 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 142 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 143 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 144 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 145 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 146 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 147 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 148 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 149 | [[0.125,0.125,0.125]], 150 | [[0.125,0.125,0.125]], 151 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 152 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 153 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 154 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 155 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 156 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 157 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 158 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 159 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 160 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 161 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 162 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 163 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 164 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 165 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 166 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 167 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 168 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 169 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 170 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 171 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 172 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 173 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 174 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 175 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.0,0.25,0.25],[0.0,0.25,0.25]], 176 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 177 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 178 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 179 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 180 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 181 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 182 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 183 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 184 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 185 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 186 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 187 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.25,0.0,0.25],[0.25,0.0,0.25]], 188 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 189 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 190 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 191 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 192 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 193 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 194 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 195 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 196 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 197 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 198 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 199 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 200 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 201 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 202 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 203 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 204 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 205 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 206 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 207 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 208 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 209 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 210 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 211 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 212 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 213 | [[-0.125,-0.125,0.125]], 214 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 215 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 216 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 217 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 218 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 219 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 220 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 221 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 222 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 223 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 224 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 225 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 226 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 227 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 228 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 229 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 230 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 231 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 232 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 233 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 234 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 235 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 236 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 237 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 238 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 239 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 240 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 241 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 242 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 243 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 244 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 245 | [[0.125,-0.125,0.125]], 246 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 247 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 248 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 249 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 250 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 251 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 252 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 253 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 254 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 255 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 256 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 257 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 258 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 259 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 260 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 261 | [[0.125,-0.125,-0.125]], 262 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 263 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 264 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 265 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 266 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 267 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 268 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 269 | [[-0.125,0.125,0.125]], 270 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 271 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 272 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 273 | [[0.125,-0.125,0.125]], 274 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 275 | [[-0.125,-0.125,0.125]], 276 | [[0.125,0.125,0.125]], 277 | [[0,0,0]]] 278 | 279 | 280 | def compute_surface_distances(mask_gt, mask_pred, spacing_mm): 281 | """Compute closest distances from all surface points to the other surface. 282 | 283 | Finds all surface elements "surfels" in the ground truth mask `mask_gt` and 284 | the predicted mask `mask_pred`, computes their area in mm^2 and the distance 285 | to the closest point on the other surface. It returns two sorted lists of 286 | distances together with the corresponding surfel areas. If one of the masks 287 | is empty, the corresponding lists are empty and all distances in the other 288 | list are `inf` 289 | 290 | Args: 291 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 292 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 293 | spacing_mm: 3-element list-like structure. Voxel spacing in x0, x1 and x2 294 | direction 295 | 296 | Returns: 297 | A dict with 298 | "distances_gt_to_pred": 1-dim numpy array of type float. The distances in mm 299 | from all ground truth surface elements to the predicted surface, 300 | sorted from smallest to largest 301 | "distances_pred_to_gt": 1-dim numpy array of type float. The distances in mm 302 | from all predicted surface elements to the ground truth surface, 303 | sorted from smallest to largest 304 | "surfel_areas_gt": 1-dim numpy array of type float. The area in mm^2 of 305 | the ground truth surface elements in the same order as 306 | distances_gt_to_pred 307 | "surfel_areas_pred": 1-dim numpy array of type float. The area in mm^2 of 308 | the predicted surface elements in the same order as 309 | distances_pred_to_gt 310 | 311 | """ 312 | 313 | # compute the area for all 256 possible surface elements 314 | # (given a 2x2x2 neighbourhood) according to the spacing_mm 315 | neighbour_code_to_surface_area = np.zeros([256]) 316 | for code in range(256): 317 | normals = np.array(neighbour_code_to_normals[code]) 318 | sum_area = 0 319 | for normal_idx in range(normals.shape[0]): 320 | # normal vector 321 | n = np.zeros([3]) 322 | n[0] = normals[normal_idx,0] * spacing_mm[1] * spacing_mm[2] 323 | n[1] = normals[normal_idx,1] * spacing_mm[0] * spacing_mm[2] 324 | n[2] = normals[normal_idx,2] * spacing_mm[0] * spacing_mm[1] 325 | area = np.linalg.norm(n) 326 | sum_area += area 327 | neighbour_code_to_surface_area[code] = sum_area 328 | 329 | # compute the bounding box of the masks to trim 330 | # the volume to the smallest possible processing subvolume 331 | mask_all = mask_gt | mask_pred 332 | bbox_min = np.zeros(3, np.int64) 333 | bbox_max = np.zeros(3, np.int64) 334 | 335 | # max projection to the x0-axis 336 | proj_0 = np.max(np.max(mask_all, axis=2), axis=1) 337 | idx_nonzero_0 = np.nonzero(proj_0)[0] 338 | if len(idx_nonzero_0) == 0: 339 | return {"distances_gt_to_pred": np.array([]), 340 | "distances_pred_to_gt": np.array([]), 341 | "surfel_areas_gt": np.array([]), 342 | "surfel_areas_pred": np.array([])} 343 | 344 | bbox_min[0] = np.min(idx_nonzero_0) 345 | bbox_max[0] = np.max(idx_nonzero_0) 346 | 347 | # max projection to the x1-axis 348 | proj_1 = np.max(np.max(mask_all, axis=2), axis=0) 349 | idx_nonzero_1 = np.nonzero(proj_1)[0] 350 | bbox_min[1] = np.min(idx_nonzero_1) 351 | bbox_max[1] = np.max(idx_nonzero_1) 352 | 353 | # max projection to the x2-axis 354 | proj_2 = np.max(np.max(mask_all, axis=1), axis=0) 355 | idx_nonzero_2 = np.nonzero(proj_2)[0] 356 | bbox_min[2] = np.min(idx_nonzero_2) 357 | bbox_max[2] = np.max(idx_nonzero_2) 358 | 359 | # print("bounding box min = {}".format(bbox_min)) 360 | # print("bounding box max = {}".format(bbox_max)) 361 | 362 | # crop the processing subvolume. 363 | # we need to zeropad the cropped region with 1 voxel at the lower, 364 | # the right and the back side. This is required to obtain the "full" 365 | # convolution result with the 2x2x2 kernel 366 | cropmask_gt = np.zeros((bbox_max - bbox_min)+2, np.uint8) 367 | cropmask_pred = np.zeros((bbox_max - bbox_min)+2, np.uint8) 368 | 369 | cropmask_gt[0:-1, 0:-1, 0:-1] = mask_gt[bbox_min[0]:bbox_max[0]+1, 370 | bbox_min[1]:bbox_max[1]+1, 371 | bbox_min[2]:bbox_max[2]+1] 372 | 373 | cropmask_pred[0:-1, 0:-1, 0:-1] = mask_pred[bbox_min[0]:bbox_max[0]+1, 374 | bbox_min[1]:bbox_max[1]+1, 375 | bbox_min[2]:bbox_max[2]+1] 376 | 377 | # compute the neighbour code (local binary pattern) for each voxel 378 | # the resultsing arrays are spacially shifted by minus half a voxel in each axis. 379 | # i.e. the points are located at the corners of the original voxels 380 | kernel = np.array([[[128,64], 381 | [32,16]], 382 | [[8,4], 383 | [2,1]]]) 384 | neighbour_code_map_gt = scipy.ndimage.filters.correlate(cropmask_gt.astype(np.uint8), kernel, mode="constant", cval=0) 385 | neighbour_code_map_pred = scipy.ndimage.filters.correlate(cropmask_pred.astype(np.uint8), kernel, mode="constant", cval=0) 386 | 387 | # create masks with the surface voxels 388 | borders_gt = ((neighbour_code_map_gt != 0) & (neighbour_code_map_gt != 255)) 389 | borders_pred = ((neighbour_code_map_pred != 0) & (neighbour_code_map_pred != 255)) 390 | 391 | # compute the distance transform (closest distance of each voxel to the surface voxels) 392 | if borders_gt.any(): 393 | distmap_gt = scipy.ndimage.morphology.distance_transform_edt(~borders_gt, sampling=spacing_mm) 394 | else: 395 | distmap_gt = np.Inf * np.ones(borders_gt.shape) 396 | 397 | if borders_pred.any(): 398 | distmap_pred = scipy.ndimage.morphology.distance_transform_edt(~borders_pred, sampling=spacing_mm) 399 | else: 400 | distmap_pred = np.Inf * np.ones(borders_pred.shape) 401 | 402 | # compute the area of each surface element 403 | surface_area_map_gt = neighbour_code_to_surface_area[neighbour_code_map_gt] 404 | surface_area_map_pred = neighbour_code_to_surface_area[neighbour_code_map_pred] 405 | 406 | # create a list of all surface elements with distance and area 407 | distances_gt_to_pred = distmap_pred[borders_gt] 408 | distances_pred_to_gt = distmap_gt[borders_pred] 409 | surfel_areas_gt = surface_area_map_gt[borders_gt] 410 | surfel_areas_pred = surface_area_map_pred[borders_pred] 411 | 412 | # sort them by distance 413 | if distances_gt_to_pred.shape != (0,): 414 | sorted_surfels_gt = np.array(sorted(zip(distances_gt_to_pred, surfel_areas_gt))) 415 | distances_gt_to_pred = sorted_surfels_gt[:,0] 416 | surfel_areas_gt = sorted_surfels_gt[:,1] 417 | 418 | if distances_pred_to_gt.shape != (0,): 419 | sorted_surfels_pred = np.array(sorted(zip(distances_pred_to_gt, surfel_areas_pred))) 420 | distances_pred_to_gt = sorted_surfels_pred[:,0] 421 | surfel_areas_pred = sorted_surfels_pred[:,1] 422 | 423 | 424 | return {"distances_gt_to_pred": distances_gt_to_pred, 425 | "distances_pred_to_gt": distances_pred_to_gt, 426 | "surfel_areas_gt": surfel_areas_gt, 427 | "surfel_areas_pred": surfel_areas_pred} 428 | 429 | 430 | def compute_average_surface_distance(surface_distances): 431 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 432 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 433 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 434 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 435 | average_distance_gt_to_pred = np.sum( distances_gt_to_pred * surfel_areas_gt) / np.sum(surfel_areas_gt) 436 | average_distance_pred_to_gt = np.sum( distances_pred_to_gt * surfel_areas_pred) / np.sum(surfel_areas_pred) 437 | return (average_distance_gt_to_pred, average_distance_pred_to_gt) 438 | 439 | def compute_robust_hausdorff(surface_distances, percent): 440 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 441 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 442 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 443 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 444 | if len(distances_gt_to_pred) > 0: 445 | surfel_areas_cum_gt = np.cumsum(surfel_areas_gt) / np.sum(surfel_areas_gt) 446 | idx = np.searchsorted(surfel_areas_cum_gt, percent/100.0) 447 | perc_distance_gt_to_pred = distances_gt_to_pred[min(idx, len(distances_gt_to_pred)-1)] 448 | else: 449 | perc_distance_gt_to_pred = np.Inf 450 | 451 | if len(distances_pred_to_gt) > 0: 452 | surfel_areas_cum_pred = np.cumsum(surfel_areas_pred) / np.sum(surfel_areas_pred) 453 | idx = np.searchsorted(surfel_areas_cum_pred, percent/100.0) 454 | perc_distance_pred_to_gt = distances_pred_to_gt[min(idx, len(distances_pred_to_gt)-1)] 455 | else: 456 | perc_distance_pred_to_gt = np.Inf 457 | 458 | return max( perc_distance_gt_to_pred, perc_distance_pred_to_gt) 459 | 460 | def compute_surface_overlap_at_tolerance(surface_distances, tolerance_mm): 461 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 462 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 463 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 464 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 465 | rel_overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) / np.sum(surfel_areas_gt) 466 | rel_overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) / np.sum(surfel_areas_pred) 467 | return (rel_overlap_gt, rel_overlap_pred) 468 | 469 | def compute_surface_dice_at_tolerance(surface_distances, tolerance_mm): 470 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 471 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 472 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 473 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 474 | overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) 475 | overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) 476 | surface_dice = (overlap_gt + overlap_pred) / ( 477 | np.sum(surfel_areas_gt) + np.sum(surfel_areas_pred)) 478 | return surface_dice 479 | 480 | 481 | def compute_dice_coefficient(mask_gt, mask_pred): 482 | """Compute soerensen-dice coefficient. 483 | 484 | compute the soerensen-dice coefficient between the ground truth mask `mask_gt` 485 | and the predicted mask `mask_pred`. 486 | 487 | Args: 488 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 489 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 490 | 491 | Returns: 492 | the dice coeffcient as float. If both masks are empty, the result is NaN 493 | """ 494 | volume_sum = mask_gt.sum() + mask_pred.sum() 495 | if volume_sum == 0: 496 | return np.NaN 497 | volume_intersect = (mask_gt & mask_pred).sum() 498 | return 2*volume_intersect / volume_sum 499 | 500 | -------------------------------------------------------------------------------- /FLARE23/load_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | import csv 3 | import argparse 4 | import glob 5 | import matplotlib 6 | import os 7 | join = os.path.join 8 | import numpy as np 9 | 10 | matplotlib.use("Agg") 11 | import matplotlib.pyplot as plt 12 | 13 | from logger import add_file_handler_to_logger, logger 14 | 15 | add_file_handler_to_logger(name="main", dir_path=f"logs/", level="DEBUG") 16 | 17 | 18 | if __name__ == "__main__": 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("-docker_name", default="fully_suplearn_subtask1", help="team docker name") 21 | parser.add_argument("-save_path", default="./results", help="save_path") 22 | time_interval = 0.1 23 | args = parser.parse_args() 24 | logger.info("we are counting: {args.docker_name}") 25 | json_dir = join(args.save_path, args.docker_name) 26 | csv_path = join(json_dir, args.docker_name + '_Efficiency.csv') 27 | jsonl = sorted(glob.glob(json_dir + "/*.json")) 28 | alldata = [] 29 | for item in jsonl: 30 | csv_l = [] 31 | name = item.split(os.sep)[-1].split(".")[0] 32 | csv_l.append(name + '.nii.gz') 33 | zitem = item 34 | with open(zitem) as f: 35 | try: 36 | js = json.load(f) 37 | except Exception as error: 38 | logger.error(f"{item} have error") 39 | logger.exception(error) 40 | if "time" not in js: 41 | logger.error(f"{item} don't have time!!!!") 42 | logger.info(f"Manually compute {item}") 43 | time = time_interval * len(js["gpu_memory"]) 44 | else: 45 | time = js["time"] 46 | csv_l.append(np.round(time,2)) 47 | #CPU 48 | user, system, all_cpu_used = [item[0] for item in js['cpu_list']], [item[1] for item in js['cpu_list']], [ 49 | 100 - item[2] for item in js['cpu_list']] 50 | plt.cla() 51 | x = [item * time_interval for item in range(len(user))] 52 | plt.xlabel("Time (s)", fontsize="large") 53 | plt.ylabel("CPU Utilization (%)", fontsize="large") 54 | # plt.plot(x, user, "b", ms=10, label="User %") 55 | # plt.plot(x, system, "r", ms=10, label="System %") 56 | plt.plot(x, all_cpu_used, "b", ms=10, label="Used %") 57 | plt.legend() 58 | plt.savefig(zitem.replace(".json", "_CPU-Time.png")) 59 | #RAM 60 | RAM = js["RAM_list"] 61 | plt.cla() 62 | x = [item * time_interval for item in range(len(RAM))] 63 | plt.xlabel("Time (s)", fontsize="large") 64 | plt.ylabel("RAM (MB)", fontsize="large") 65 | plt.plot(x, RAM, "b", ms=10, label="RAM") 66 | plt.legend() 67 | plt.savefig(zitem.replace(".json", "_RAM-Time.png")) 68 | mem = js["gpu_memory"] 69 | x = [item * time_interval for item in range(len(mem))] 70 | plt.cla() 71 | plt.xlabel("Time (s)", fontsize="large") 72 | plt.ylabel("GPU Memory (MB)", fontsize="large") 73 | plt.plot(x, mem, "b", ms=10, label="a") 74 | plt.savefig(zitem.replace(".json", "_GPU-Time.png"), dpi=300) 75 | count_set = set(mem) 76 | # select the stable memory 77 | # count_list = [] 78 | # for citem in count_set: 79 | # cts = mem.count(citem) 80 | # if cts > 0.02 * len(mem): 81 | # count_list.append(citem) 82 | # directly count the max GPU memory 83 | max_mem = max(count_set) 84 | csv_l.append(np.round(max_mem)) 85 | csv_l.append(np.round(sum(mem) * time_interval)) 86 | csv_l.append(np.round(max(all_cpu_used),2)) 87 | csv_l.append(np.round(sum(all_cpu_used) * time_interval,2)) 88 | csv_l.append(np.round(max(RAM),2)) 89 | csv_l.append(np.round(sum(RAM) * time_interval)) 90 | alldata.append(csv_l) 91 | 92 | f = open(csv_path, "w",newline='') 93 | writer = csv.writer(f) 94 | writer.writerow(["Name", "Time", "MaxGPU_Mem", "AUC_GPU_Time",'MaxCPU_Utilization','AUC_CPU_Time','MaxRAM' 95 | ,'AUC_RAM_Time' ]) 96 | for i in alldata: 97 | writer.writerow(i) 98 | f.close() 99 | -------------------------------------------------------------------------------- /FLARE23/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from loguru import logger as loguru_logger 5 | 6 | loguru_logger.configure( 7 | handlers=[dict(sink=sys.stderr, filter=lambda record: record["extra"]["console"], level="DEBUG",),], 8 | ) 9 | 10 | logger = loguru_logger.bind(console=True) 11 | 12 | 13 | def add_file_handler_to_logger( 14 | name: str, dir_path="./logs", level="DEBUG", 15 | ): 16 | loguru_logger.add( 17 | sink=os.path.join(dir_path, f"{name}-{level}.log"), 18 | level=level, 19 | filter=lambda record: "console" in record["extra"], 20 | enqueue=True, 21 | ) 22 | -------------------------------------------------------------------------------- /FLARE23/resource_eval.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | import time 5 | import torch 6 | from pathlib import Path 7 | join = os.path.join 8 | from logger import add_file_handler_to_logger, logger 9 | 10 | 11 | def check_dir(file_path): 12 | file_path = Path(file_path) 13 | files = [f for f in file_path.iterdir() if ".nii.gz" in str(f)] 14 | if len(files) != 0: 15 | return False 16 | return True 17 | 18 | 19 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 20 | 21 | docker_path = './team_docker/' # put docker in this folder 22 | test_img_path = './data/validation' # all validation cases 23 | save_path = './FLARE23_ValResults/' # evaluation results will be saved in this folder 24 | 25 | temp_in = './inputs/' 26 | temp_out = './outputs' 27 | os.makedirs(save_path, exist_ok=True) 28 | os.makedirs(temp_in, exist_ok=True) 29 | os.makedirs(temp_out, exist_ok=True) 30 | os.system("chmod -R 777 outputs/") 31 | 32 | dockers = sorted(os.listdir(docker_path)) 33 | test_cases = sorted(os.listdir(test_img_path)) 34 | 35 | for docker in dockers: 36 | try: 37 | name = docker.split('.')[0].lower() 38 | print('loading docker:', docker) 39 | os.system('docker load -i {}'.format(join(docker_path, docker))) 40 | team_outpath = join(save_path, name) 41 | if os.path.exists(team_outpath): 42 | shutil.rmtree(team_outpath) 43 | os.mkdir(team_outpath) 44 | for case in test_cases: 45 | if not check_dir(temp_in): 46 | logger.error("please check inputs folder", temp_in) 47 | raise 48 | shutil.copy(join(test_img_path, case), temp_in) 49 | start_time = time.time() 50 | try: 51 | shutil.rmtree('./inputs/nnUNet_raw/') 52 | shutil.rmtree('./inputs/nnUNet_preprocessed/') 53 | except: 54 | print('no temp files') 55 | os.system('python Efficiency.py -docker_name {} -save_file {}'.format(name, save_path)) 56 | logger.info(f"{case} finished!") 57 | os.remove(join('./inputs', case)) 58 | try: 59 | shutil.rmtree('./inputs/nnUNet_cropped_data/') 60 | shutil.rmtree('./inputs/nnUNet_raw_data/') 61 | shutil.rmtree('./inputs/nnUNet_preprocessed/') 62 | except: 63 | print('no temp files') 64 | # shutil.rmtree('./inputs') 65 | logger.info(f"{case} cost time: {time.time() - start_time}") 66 | # move segmentation file 67 | seg_name = case.split('_0000.nii.gz')[0]+'.nii.gz' 68 | if os.path.exists(join(temp_out, seg_name)): 69 | os.rename(join(temp_out, seg_name), join(team_outpath, seg_name)) 70 | 71 | os.system("python load_json.py -docker_name {} -save_path {}".format(name, save_path)) 72 | shutil.move(temp_out, team_outpath) 73 | os.mkdir(temp_out) 74 | torch.cuda.empty_cache() 75 | shutil.rmtree(temp_in) 76 | os.mkdir(temp_in) 77 | os.system("docker rmi {}:latest".format(name)) 78 | except Exception as e: 79 | logger.exception(e) 80 | -------------------------------------------------------------------------------- /FLARE24/Efficiency.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import json 4 | import os 5 | import time 6 | import psutil as p 7 | from multiprocessing import Manager, Process 8 | from pynvml.smi import nvidia_smi 9 | import numpy as np 10 | from logger import add_file_handler_to_logger, logger 11 | 12 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 13 | 14 | manager = Manager() 15 | gpu_list = manager.list() 16 | cpu_list = manager.list() 17 | RAM_list = manager.list() 18 | def cpu_usage(): 19 | t = p.cpu_times() 20 | return [t.user, t.system, t.idle] 21 | before = cpu_usage() 22 | def get_cpu_usage(): 23 | global before 24 | now = cpu_usage() 25 | delta = [now[i] - before[i] for i in range(len(now))] 26 | total = sum(delta) 27 | before = now 28 | return [(100.0*dt)/(total+0.1) for dt in delta] 29 | 30 | 31 | def daemon_process(time_interval, json_path, gpu_index=1): 32 | while True: 33 | nvsmi = nvidia_smi.getInstance() 34 | dictm = nvsmi.DeviceQuery("memory.free, memory.total") 35 | gpu_memory = ( 36 | dictm["gpu"][gpu_index]["fb_memory_usage"]["total"] - dictm["gpu"][gpu_index]["fb_memory_usage"]["free"] 37 | ) 38 | cpu_usage =get_cpu_usage() 39 | RAM = p.virtual_memory().used/1048576 40 | gpu_list.append(gpu_memory) 41 | RAM_list.append(RAM) 42 | cpu_list.append(cpu_usage) 43 | # with open(json_path, 'w')as f: 44 | # #js['gpu_memory'] = gpu_memory_max 45 | # js['gpu_memory'].append(gpu_memory) 46 | # json.dump(js, f, indent=4) 47 | time.sleep(time_interval) 48 | 49 | 50 | def save_result(start_time, sleep_time, json_path, gpu_list_ , cpu_list_copy, RAM_list_copy): 51 | if os.path.exists(json_path): 52 | with open(json_path) as f: 53 | js = json.load(f) 54 | else: 55 | js = {"gpu_memory": []} 56 | # raise ValueError(f"{json_path} don't exist!") 57 | 58 | with open(json_path, "w") as f: 59 | # js['gpu_memory'] = gpu_memory_max 60 | js["gpu_memory"] = gpu_list_ 61 | js["cpu_list"] = cpu_list_copy 62 | js["RAM_list"] = RAM_list_copy 63 | 64 | json.dump(js, f, indent=4) 65 | 66 | infer_time = time.time() - start_time 67 | with open(json_path, "r") as f: 68 | js = json.load(f) 69 | with open(json_path, "w") as f: 70 | js["time"] = infer_time 71 | json.dump(js, f, indent=4) 72 | time.sleep(2) 73 | logger.info("save result end") 74 | 75 | 76 | if __name__ == "__main__": 77 | parser = argparse.ArgumentParser() 78 | parser.add_argument("-time_interval", default=0.1, help="time_interval") 79 | parser.add_argument("-sleep_time", default=5, help="sleep time") 80 | parser.add_argument("-shell_path", default="predict.sh", help="time_interval") 81 | # XXX: in case of someone use lower docker, please use specified GPU !!! 82 | parser.add_argument("-gpus", default=1, help="CUDA_VISIBLE_DEVICES") 83 | parser.add_argument("-docker_input_file", default="./inputs/", help="docker input folder") 84 | parser.add_argument("-save_file", default="results", help="data save folder") 85 | parser.add_argument("-docker_name", default="nnunet", help="docker output folder") 86 | args = parser.parse_args() 87 | logger.info(f"We are evaluating {args.docker_name}") 88 | json_dir = "./{}/{}".format(args.save_file,args.docker_name) 89 | json_path = os.path.join( 90 | json_dir, glob.glob(args.docker_input_file + "/*.nii.gz")[0].split("/")[-1].split(".nii.gz")[0] + ".json", 91 | ) 92 | 93 | try: 94 | p1 = Process(target=daemon_process, args=(args.time_interval, json_path, args.gpus,)) 95 | p1.daemon = True 96 | p1.start() 97 | t0 = time.time() 98 | # XXX: in case of someone use lower docker, please use specified GPU !!! 99 | # cmd = 'docker container run --runtime="nvidia" -e NVIDIA_VISIBLE_DEVICES={} --name {} --rm -v $PWD/inputs/:/workspace/input/ -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/sh -c "sh {}"'.format( 100 | # args.gpus, args.docker_name, args.docker_name, args.shell_path 101 | # ) 102 | cmd = 'docker container run --shm-size=28gb -m 28G --gpus="device=1" --name {} --rm -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/bash -c "sh predict.sh" '.format(args.docker_name, args.docker_name) 103 | logger.info(f"cmd is : {cmd}") 104 | logger.info("start predict...") 105 | os.system(cmd) 106 | RAM_list= list(RAM_list) 107 | RAM_list_copy = RAM_list.copy() 108 | cpu_list= list(cpu_list) 109 | cpu_list_copy = cpu_list.copy() 110 | gpu_list = list(gpu_list) 111 | gpu_list_copy = gpu_list.copy() 112 | print(f"{json_path=}") 113 | save_result(t0, args.sleep_time, json_path, gpu_list_copy ,cpu_list_copy,RAM_list_copy) 114 | time.sleep(args.sleep_time) 115 | except Exception as error: 116 | logger.exception(error) 117 | -------------------------------------------------------------------------------- /FLARE24/README.md: -------------------------------------------------------------------------------- 1 | # Segmentation Efficiency Evaluation 2 | 3 | ### Enviroment and requirements 4 | 5 | ```python 6 | - python 3.8+ 7 | - torch 8 | - loguru 9 | - pynvml 10 | - psutil 11 | ``` 12 | 13 | 14 | 15 | Set `docker_path`, `test_img_path`, and `save_path` in `resource_eval.py` and run 16 | 17 | `nohup python resource_eval.py >> infos.log &` 18 | 19 | 20 | 21 | Q: How the `Area under GPU memory-time curve` is computed? 22 | 23 | > A: We record the GPU memory and GPU utilization every 0.1s. The `Area under GPU memory-time curve` and `Area under CPU utilization-time curve` are the cumulative values along running time. 24 | 25 | 26 | Note: 27 | - The above command will also generate the evaluation results of CPU and RAM usage. They are used for debug purpose (not for ranking purpose). 28 | - For DSC and NSD, please refer to `T1_FLARE24_DSC_NSD_Eval.py` and `T2andT3_FLARE24_DSC_NSD_Eval.py`. 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /FLARE24/SurfaceDice.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Apr 15 13:01:08 2022 4 | 5 | @author: 12593 6 | """ 7 | 8 | import numpy as np 9 | import scipy.ndimage 10 | 11 | # neighbour_code_to_normals is a lookup table. 12 | # For every binary neighbour code 13 | # (2x2x2 neighbourhood = 8 neighbours = 8 bits = 256 codes) 14 | # it contains the surface normals of the triangles (called "surfel" for 15 | # "surface element" in the following). The length of the normal 16 | # vector encodes the surfel area. 17 | # 18 | # created by compute_surface_area_lookup_table.ipynb using the 19 | # marching_cube algorithm, see e.g. https://en.wikipedia.org/wiki/Marching_cubes 20 | # credit to: http://medicaldecathlon.com/files/Surface_distance_based_measures.ipynb 21 | neighbour_code_to_normals = [ 22 | [[0,0,0]], 23 | [[0.125,0.125,0.125]], 24 | [[-0.125,-0.125,0.125]], 25 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 26 | [[0.125,-0.125,0.125]], 27 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 28 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 29 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 30 | [[-0.125,0.125,0.125]], 31 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 32 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 33 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 34 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 35 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 36 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 37 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 38 | [[0.125,-0.125,-0.125]], 39 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 40 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 41 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 42 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 43 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 44 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 45 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 46 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 47 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 48 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 49 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 50 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 51 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 52 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 53 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 54 | [[0.125,-0.125,0.125]], 55 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 56 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 57 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 58 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 59 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 60 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 61 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 62 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 63 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 64 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 65 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 66 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 67 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 68 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 69 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 70 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 71 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 72 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 73 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 74 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 75 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 76 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 77 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 78 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 79 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 80 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 81 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 82 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 83 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 84 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 85 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 86 | [[-0.125,-0.125,0.125]], 87 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 88 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 89 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 90 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 91 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 92 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 93 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 94 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 95 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 96 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 97 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 98 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 99 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 100 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 101 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 102 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 103 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 104 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 105 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 106 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 107 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 108 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 109 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 110 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 111 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 112 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 113 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 114 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 115 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 116 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 117 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 118 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 119 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 120 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 121 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 122 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 123 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 124 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 125 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 126 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 127 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 128 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 129 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 130 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 131 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 132 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 133 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 134 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 135 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 136 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 137 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 138 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 139 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 140 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 141 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 142 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 143 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 144 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 145 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 146 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 147 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 148 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 149 | [[0.125,0.125,0.125]], 150 | [[0.125,0.125,0.125]], 151 | [[0.125,0.125,0.125],[0.125,0.125,0.125]], 152 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 153 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0],[0.125,0.125,0.125]], 154 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 155 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.125,0.125,0.125]], 156 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 157 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125],[0.125,0.125,0.125]], 158 | [[0.0,0.25,0.25],[0.0,0.25,0.25]], 159 | [[0.125,0.125,0.125],[0.0,0.25,0.25],[0.0,0.25,0.25]], 160 | [[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.0,0.0,0.5]], 161 | [[-0.375,-0.375,0.375],[0.25,-0.25,0.0],[0.0,0.25,0.25],[-0.125,-0.125,0.125]], 162 | [[0.0,-0.5,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 163 | [[0.375,-0.375,0.375],[0.0,0.25,0.25],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 164 | [[-0.25,0.25,0.25],[-0.125,0.125,0.125],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 165 | [[0.5,0.0,-0.0],[0.25,-0.25,-0.25],[0.125,-0.125,-0.125]], 166 | [[0.125,0.125,0.125],[0.125,-0.125,-0.125]], 167 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.125,0.125,0.125]], 168 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 169 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 170 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 171 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 172 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125],[0.125,0.125,0.125]], 173 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 174 | [[0.0,0.25,0.25],[0.0,0.25,0.25],[0.125,-0.125,-0.125]], 175 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[0.0,0.25,0.25],[0.0,0.25,0.25]], 176 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 177 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[0.125,-0.125,0.125]], 178 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 179 | [[-0.125,-0.125,0.125],[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 180 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 181 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125]], 182 | [[0.25,0.0,0.25],[0.25,0.0,0.25]], 183 | [[0.125,0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 184 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 185 | [[0.25,0.0,0.25],[-0.375,-0.375,0.375],[-0.25,0.25,0.0],[-0.125,-0.125,0.125]], 186 | [[0.125,-0.125,0.125],[0.25,0.0,0.25],[0.25,0.0,0.25]], 187 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25],[0.25,0.0,0.25],[0.25,0.0,0.25]], 188 | [[-0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 189 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25],[-0.125,0.125,0.125]], 190 | [[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 191 | [[0.125,0.125,0.125],[0.125,0.125,0.125],[0.25,0.25,0.25],[0.0,0.0,0.5]], 192 | [[-0.0,0.0,0.5],[0.0,0.0,0.5]], 193 | [[0.0,0.0,-0.5],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 194 | [[-0.25,-0.0,-0.25],[-0.375,0.375,0.375],[-0.25,-0.25,0.0],[-0.125,0.125,0.125]], 195 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 196 | [[-0.0,0.0,0.5],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 197 | [[-0.25,0.0,0.25],[0.25,0.0,-0.25]], 198 | [[0.5,0.0,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 199 | [[-0.25,0.0,-0.25],[0.375,-0.375,-0.375],[0.0,0.25,-0.25],[-0.125,0.125,0.125]], 200 | [[-0.25,0.25,-0.25],[-0.25,0.25,-0.25],[-0.125,0.125,-0.125],[-0.125,0.125,-0.125]], 201 | [[-0.0,0.5,0.0],[-0.25,0.25,-0.25],[0.125,-0.125,0.125]], 202 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 203 | [[-0.125,-0.125,0.125],[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 204 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 205 | [[-0.125,-0.125,0.125],[-0.125,0.125,0.125]], 206 | [[0.375,-0.375,0.375],[0.0,-0.25,-0.25],[-0.125,0.125,-0.125],[0.25,0.25,0.0]], 207 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25],[-0.125,-0.125,0.125]], 208 | [[0.0,0.0,0.5],[0.25,-0.25,0.25],[0.125,-0.125,0.125]], 209 | [[0.0,-0.25,0.25],[0.0,-0.25,0.25]], 210 | [[-0.125,-0.125,0.125],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 211 | [[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 212 | [[0.125,0.125,0.125],[-0.125,-0.125,0.125]], 213 | [[-0.125,-0.125,0.125]], 214 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 215 | [[0.125,0.125,0.125],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0]], 216 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.125,-0.125,0.125]], 217 | [[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 218 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125]], 219 | [[-0.375,0.375,-0.375],[-0.25,-0.25,0.0],[-0.125,0.125,-0.125],[-0.25,0.0,0.25]], 220 | [[0.0,0.5,0.0],[0.25,0.25,-0.25],[-0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 221 | [[-0.125,0.125,0.125],[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 222 | [[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 223 | [[0.125,0.125,0.125],[0.0,-0.5,0.0],[-0.25,-0.25,-0.25],[-0.125,-0.125,-0.125]], 224 | [[-0.375,-0.375,-0.375],[-0.25,0.0,0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 225 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0],[0.125,-0.125,0.125]], 226 | [[0.0,0.5,0.0],[0.0,-0.5,0.0]], 227 | [[0.0,0.5,0.0],[0.125,-0.125,0.125],[-0.25,0.25,-0.25]], 228 | [[0.0,0.5,0.0],[-0.25,0.25,0.25],[0.125,-0.125,-0.125]], 229 | [[0.25,-0.25,0.0],[-0.25,0.25,0.0]], 230 | [[-0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 231 | [[0.0,0.25,-0.25],[0.375,-0.375,-0.375],[-0.125,0.125,0.125],[0.25,0.25,0.0]], 232 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125],[0.125,-0.125,0.125]], 233 | [[0.125,-0.125,0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 234 | [[0.25,0.25,-0.25],[0.25,0.25,-0.25],[0.125,0.125,-0.125],[-0.125,-0.125,0.125]], 235 | [[-0.0,0.0,0.5],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 236 | [[0.125,0.125,0.125],[0.125,-0.125,0.125],[-0.125,0.125,0.125]], 237 | [[-0.125,0.125,0.125],[0.125,-0.125,0.125]], 238 | [[-0.375,-0.375,0.375],[-0.0,0.25,0.25],[0.125,0.125,-0.125],[-0.25,-0.0,-0.25]], 239 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25],[0.125,-0.125,0.125]], 240 | [[0.125,-0.125,0.125],[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 241 | [[0.125,-0.125,0.125],[0.125,-0.125,0.125]], 242 | [[0.0,-0.5,0.0],[0.125,0.125,-0.125],[0.25,0.25,-0.25]], 243 | [[0.0,-0.25,0.25],[0.0,0.25,-0.25]], 244 | [[0.125,0.125,0.125],[0.125,-0.125,0.125]], 245 | [[0.125,-0.125,0.125]], 246 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25]], 247 | [[-0.5,0.0,0.0],[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.125,0.125,0.125]], 248 | [[0.375,0.375,0.375],[0.0,0.25,-0.25],[-0.125,-0.125,-0.125],[-0.25,0.25,0.0]], 249 | [[0.125,-0.125,-0.125],[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 250 | [[0.125,0.125,0.125],[0.375,0.375,0.375],[0.0,-0.25,0.25],[-0.25,0.0,0.25]], 251 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25],[0.125,-0.125,-0.125]], 252 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25],[-0.125,0.125,0.125]], 253 | [[-0.125,0.125,0.125],[0.125,-0.125,-0.125]], 254 | [[-0.125,-0.125,-0.125],[-0.25,-0.25,-0.25],[0.25,0.25,0.25],[0.125,0.125,0.125]], 255 | [[-0.125,-0.125,0.125],[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 256 | [[0.0,0.0,-0.5],[0.25,0.25,0.25],[-0.125,-0.125,-0.125]], 257 | [[0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 258 | [[0.0,-0.5,0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 259 | [[-0.125,-0.125,0.125],[0.125,-0.125,-0.125]], 260 | [[0.0,-0.25,-0.25],[0.0,0.25,0.25]], 261 | [[0.125,-0.125,-0.125]], 262 | [[0.5,0.0,0.0],[0.5,0.0,0.0]], 263 | [[-0.5,0.0,0.0],[-0.25,0.25,0.25],[-0.125,0.125,0.125]], 264 | [[0.5,0.0,0.0],[0.25,-0.25,0.25],[-0.125,0.125,-0.125]], 265 | [[0.25,-0.25,0.0],[0.25,-0.25,0.0]], 266 | [[0.5,0.0,0.0],[-0.25,-0.25,0.25],[-0.125,-0.125,0.125]], 267 | [[-0.25,0.0,0.25],[-0.25,0.0,0.25]], 268 | [[0.125,0.125,0.125],[-0.125,0.125,0.125]], 269 | [[-0.125,0.125,0.125]], 270 | [[0.5,0.0,-0.0],[0.25,0.25,0.25],[0.125,0.125,0.125]], 271 | [[0.125,-0.125,0.125],[-0.125,-0.125,0.125]], 272 | [[-0.25,-0.0,-0.25],[0.25,0.0,0.25]], 273 | [[0.125,-0.125,0.125]], 274 | [[-0.25,-0.25,0.0],[0.25,0.25,-0.0]], 275 | [[-0.125,-0.125,0.125]], 276 | [[0.125,0.125,0.125]], 277 | [[0,0,0]]] 278 | 279 | 280 | def compute_surface_distances(mask_gt, mask_pred, spacing_mm): 281 | """Compute closest distances from all surface points to the other surface. 282 | 283 | Finds all surface elements "surfels" in the ground truth mask `mask_gt` and 284 | the predicted mask `mask_pred`, computes their area in mm^2 and the distance 285 | to the closest point on the other surface. It returns two sorted lists of 286 | distances together with the corresponding surfel areas. If one of the masks 287 | is empty, the corresponding lists are empty and all distances in the other 288 | list are `inf` 289 | 290 | Args: 291 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 292 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 293 | spacing_mm: 3-element list-like structure. Voxel spacing in x0, x1 and x2 294 | direction 295 | 296 | Returns: 297 | A dict with 298 | "distances_gt_to_pred": 1-dim numpy array of type float. The distances in mm 299 | from all ground truth surface elements to the predicted surface, 300 | sorted from smallest to largest 301 | "distances_pred_to_gt": 1-dim numpy array of type float. The distances in mm 302 | from all predicted surface elements to the ground truth surface, 303 | sorted from smallest to largest 304 | "surfel_areas_gt": 1-dim numpy array of type float. The area in mm^2 of 305 | the ground truth surface elements in the same order as 306 | distances_gt_to_pred 307 | "surfel_areas_pred": 1-dim numpy array of type float. The area in mm^2 of 308 | the predicted surface elements in the same order as 309 | distances_pred_to_gt 310 | 311 | """ 312 | 313 | # compute the area for all 256 possible surface elements 314 | # (given a 2x2x2 neighbourhood) according to the spacing_mm 315 | neighbour_code_to_surface_area = np.zeros([256]) 316 | for code in range(256): 317 | normals = np.array(neighbour_code_to_normals[code]) 318 | sum_area = 0 319 | for normal_idx in range(normals.shape[0]): 320 | # normal vector 321 | n = np.zeros([3]) 322 | n[0] = normals[normal_idx,0] * spacing_mm[1] * spacing_mm[2] 323 | n[1] = normals[normal_idx,1] * spacing_mm[0] * spacing_mm[2] 324 | n[2] = normals[normal_idx,2] * spacing_mm[0] * spacing_mm[1] 325 | area = np.linalg.norm(n) 326 | sum_area += area 327 | neighbour_code_to_surface_area[code] = sum_area 328 | 329 | # compute the bounding box of the masks to trim 330 | # the volume to the smallest possible processing subvolume 331 | mask_all = mask_gt | mask_pred 332 | bbox_min = np.zeros(3, np.int64) 333 | bbox_max = np.zeros(3, np.int64) 334 | 335 | # max projection to the x0-axis 336 | proj_0 = np.max(np.max(mask_all, axis=2), axis=1) 337 | idx_nonzero_0 = np.nonzero(proj_0)[0] 338 | if len(idx_nonzero_0) == 0: 339 | return {"distances_gt_to_pred": np.array([]), 340 | "distances_pred_to_gt": np.array([]), 341 | "surfel_areas_gt": np.array([]), 342 | "surfel_areas_pred": np.array([])} 343 | 344 | bbox_min[0] = np.min(idx_nonzero_0) 345 | bbox_max[0] = np.max(idx_nonzero_0) 346 | 347 | # max projection to the x1-axis 348 | proj_1 = np.max(np.max(mask_all, axis=2), axis=0) 349 | idx_nonzero_1 = np.nonzero(proj_1)[0] 350 | bbox_min[1] = np.min(idx_nonzero_1) 351 | bbox_max[1] = np.max(idx_nonzero_1) 352 | 353 | # max projection to the x2-axis 354 | proj_2 = np.max(np.max(mask_all, axis=1), axis=0) 355 | idx_nonzero_2 = np.nonzero(proj_2)[0] 356 | bbox_min[2] = np.min(idx_nonzero_2) 357 | bbox_max[2] = np.max(idx_nonzero_2) 358 | 359 | # print("bounding box min = {}".format(bbox_min)) 360 | # print("bounding box max = {}".format(bbox_max)) 361 | 362 | # crop the processing subvolume. 363 | # we need to zeropad the cropped region with 1 voxel at the lower, 364 | # the right and the back side. This is required to obtain the "full" 365 | # convolution result with the 2x2x2 kernel 366 | cropmask_gt = np.zeros((bbox_max - bbox_min)+2, np.uint8) 367 | cropmask_pred = np.zeros((bbox_max - bbox_min)+2, np.uint8) 368 | 369 | cropmask_gt[0:-1, 0:-1, 0:-1] = mask_gt[bbox_min[0]:bbox_max[0]+1, 370 | bbox_min[1]:bbox_max[1]+1, 371 | bbox_min[2]:bbox_max[2]+1] 372 | 373 | cropmask_pred[0:-1, 0:-1, 0:-1] = mask_pred[bbox_min[0]:bbox_max[0]+1, 374 | bbox_min[1]:bbox_max[1]+1, 375 | bbox_min[2]:bbox_max[2]+1] 376 | 377 | # compute the neighbour code (local binary pattern) for each voxel 378 | # the resultsing arrays are spacially shifted by minus half a voxel in each axis. 379 | # i.e. the points are located at the corners of the original voxels 380 | kernel = np.array([[[128,64], 381 | [32,16]], 382 | [[8,4], 383 | [2,1]]]) 384 | neighbour_code_map_gt = scipy.ndimage.filters.correlate(cropmask_gt.astype(np.uint8), kernel, mode="constant", cval=0) 385 | neighbour_code_map_pred = scipy.ndimage.filters.correlate(cropmask_pred.astype(np.uint8), kernel, mode="constant", cval=0) 386 | 387 | # create masks with the surface voxels 388 | borders_gt = ((neighbour_code_map_gt != 0) & (neighbour_code_map_gt != 255)) 389 | borders_pred = ((neighbour_code_map_pred != 0) & (neighbour_code_map_pred != 255)) 390 | 391 | # compute the distance transform (closest distance of each voxel to the surface voxels) 392 | if borders_gt.any(): 393 | distmap_gt = scipy.ndimage.morphology.distance_transform_edt(~borders_gt, sampling=spacing_mm) 394 | else: 395 | distmap_gt = np.Inf * np.ones(borders_gt.shape) 396 | 397 | if borders_pred.any(): 398 | distmap_pred = scipy.ndimage.morphology.distance_transform_edt(~borders_pred, sampling=spacing_mm) 399 | else: 400 | distmap_pred = np.Inf * np.ones(borders_pred.shape) 401 | 402 | # compute the area of each surface element 403 | surface_area_map_gt = neighbour_code_to_surface_area[neighbour_code_map_gt] 404 | surface_area_map_pred = neighbour_code_to_surface_area[neighbour_code_map_pred] 405 | 406 | # create a list of all surface elements with distance and area 407 | distances_gt_to_pred = distmap_pred[borders_gt] 408 | distances_pred_to_gt = distmap_gt[borders_pred] 409 | surfel_areas_gt = surface_area_map_gt[borders_gt] 410 | surfel_areas_pred = surface_area_map_pred[borders_pred] 411 | 412 | # sort them by distance 413 | if distances_gt_to_pred.shape != (0,): 414 | sorted_surfels_gt = np.array(sorted(zip(distances_gt_to_pred, surfel_areas_gt))) 415 | distances_gt_to_pred = sorted_surfels_gt[:,0] 416 | surfel_areas_gt = sorted_surfels_gt[:,1] 417 | 418 | if distances_pred_to_gt.shape != (0,): 419 | sorted_surfels_pred = np.array(sorted(zip(distances_pred_to_gt, surfel_areas_pred))) 420 | distances_pred_to_gt = sorted_surfels_pred[:,0] 421 | surfel_areas_pred = sorted_surfels_pred[:,1] 422 | 423 | 424 | return {"distances_gt_to_pred": distances_gt_to_pred, 425 | "distances_pred_to_gt": distances_pred_to_gt, 426 | "surfel_areas_gt": surfel_areas_gt, 427 | "surfel_areas_pred": surfel_areas_pred} 428 | 429 | 430 | def compute_average_surface_distance(surface_distances): 431 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 432 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 433 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 434 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 435 | average_distance_gt_to_pred = np.sum( distances_gt_to_pred * surfel_areas_gt) / np.sum(surfel_areas_gt) 436 | average_distance_pred_to_gt = np.sum( distances_pred_to_gt * surfel_areas_pred) / np.sum(surfel_areas_pred) 437 | return (average_distance_gt_to_pred, average_distance_pred_to_gt) 438 | 439 | def compute_robust_hausdorff(surface_distances, percent): 440 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 441 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 442 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 443 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 444 | if len(distances_gt_to_pred) > 0: 445 | surfel_areas_cum_gt = np.cumsum(surfel_areas_gt) / np.sum(surfel_areas_gt) 446 | idx = np.searchsorted(surfel_areas_cum_gt, percent/100.0) 447 | perc_distance_gt_to_pred = distances_gt_to_pred[min(idx, len(distances_gt_to_pred)-1)] 448 | else: 449 | perc_distance_gt_to_pred = np.Inf 450 | 451 | if len(distances_pred_to_gt) > 0: 452 | surfel_areas_cum_pred = np.cumsum(surfel_areas_pred) / np.sum(surfel_areas_pred) 453 | idx = np.searchsorted(surfel_areas_cum_pred, percent/100.0) 454 | perc_distance_pred_to_gt = distances_pred_to_gt[min(idx, len(distances_pred_to_gt)-1)] 455 | else: 456 | perc_distance_pred_to_gt = np.Inf 457 | 458 | return max( perc_distance_gt_to_pred, perc_distance_pred_to_gt) 459 | 460 | def compute_surface_overlap_at_tolerance(surface_distances, tolerance_mm): 461 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 462 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 463 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 464 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 465 | rel_overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) / np.sum(surfel_areas_gt) 466 | rel_overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) / np.sum(surfel_areas_pred) 467 | return (rel_overlap_gt, rel_overlap_pred) 468 | 469 | def compute_surface_dice_at_tolerance(surface_distances, tolerance_mm): 470 | distances_gt_to_pred = surface_distances["distances_gt_to_pred"] 471 | distances_pred_to_gt = surface_distances["distances_pred_to_gt"] 472 | surfel_areas_gt = surface_distances["surfel_areas_gt"] 473 | surfel_areas_pred = surface_distances["surfel_areas_pred"] 474 | overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) 475 | overlap_pred = np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) 476 | surface_dice = (overlap_gt + overlap_pred) / ( 477 | np.sum(surfel_areas_gt) + np.sum(surfel_areas_pred)) 478 | return surface_dice 479 | 480 | 481 | def compute_dice_coefficient(mask_gt, mask_pred): 482 | """Compute soerensen-dice coefficient. 483 | 484 | compute the soerensen-dice coefficient between the ground truth mask `mask_gt` 485 | and the predicted mask `mask_pred`. 486 | 487 | Args: 488 | mask_gt: 3-dim Numpy array of type bool. The ground truth mask. 489 | mask_pred: 3-dim Numpy array of type bool. The predicted mask. 490 | 491 | Returns: 492 | the dice coeffcient as float. If both masks are empty, the result is NaN 493 | """ 494 | volume_sum = mask_gt.sum() + mask_pred.sum() 495 | if volume_sum == 0: 496 | return np.NaN 497 | volume_intersect = (mask_gt & mask_pred).sum() 498 | return 2*volume_intersect / volume_sum 499 | 500 | -------------------------------------------------------------------------------- /FLARE24/T1_FLARE24_DSC_NSD_Eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on March 2024 5 | 6 | @author: junma 7 | """ 8 | 9 | import numpy as np 10 | import nibabel as nb 11 | import os 12 | join = os.path.join 13 | from collections import OrderedDict 14 | import pandas as pd 15 | from SurfaceDice import compute_surface_distances, compute_surface_dice_at_tolerance, compute_dice_coefficient 16 | 17 | 18 | seg_path = 'path to segmentation' 19 | gt_path = 'path to ground truth' 20 | save_path = 'path to Results' 21 | save_name = 'DSC_NSD_teamname.csv' 22 | filenames = os.listdir(seg_path) 23 | filenames = [x for x in filenames if x.endswith('.nii.gz')] 24 | filenames.sort() 25 | 26 | tumor_tolerance = 2 27 | seg_metrics = OrderedDict() 28 | seg_metrics['Name'] = list() 29 | seg_metrics['Lesion_DSC'] = list() 30 | seg_metrics['Lesion_NSD'] = list() 31 | 32 | for name in filenames: 33 | seg_metrics['Name'].append(name) 34 | # load grond truth and segmentation 35 | gt_nii = nb.load(join(gt_path, name)) 36 | case_spacing = gt_nii.header.get_zooms() 37 | gt_data = np.uint8(gt_nii.get_fdata()) 38 | seg_data = np.uint8(nb.load(join(seg_path, name)).get_fdata()) 39 | 40 | if np.max(seg_data) > 1: 41 | DSC_i = 0 42 | NSD_i = 0 43 | elif np.sum(gt_data)==0 and np.sum(seg_data)==0: 44 | DSC_i = 1 45 | NSD_i = 1 46 | elif np.sum(gt_data)==0 and np.sum(seg_data)>0: 47 | DSC_i = 0 48 | NSD_i = 0 49 | elif np.sum(gt_data)>0 and np.sum(seg_data)==0: 50 | DSC_i = 0 51 | NSD_i = 0 52 | else: 53 | DSC_i = compute_dice_coefficient(gt_data, seg_data) 54 | if DSC_i < 0.2: # don't compute NSD if DSC is too low 55 | NSD_i = 0 56 | else: 57 | surface_distances = compute_surface_distances(gt_data, seg_data, case_spacing) 58 | NSD_i = compute_surface_dice_at_tolerance(surface_distances, tumor_tolerance) 59 | seg_metrics['Lesion_DSC'].append(round(DSC_i, 4)) 60 | seg_metrics['Lesion_NSD'].append(round(NSD_i, 4)) 61 | print(name, 'DSC:', round(DSC_i, 4), 'NSD (tol: 2)', round(NSD_i, 4)) 62 | 63 | dataframe = pd.DataFrame(seg_metrics) 64 | dataframe.to_csv(join(save_path, save_name), index=False) 65 | 66 | -------------------------------------------------------------------------------- /FLARE24/T2andT3_FLARE24_DSC_NSD_Eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on March 2024 5 | 6 | @author: junma 7 | """ 8 | 9 | import numpy as np 10 | import nibabel as nb 11 | import os 12 | join = os.path.join 13 | from collections import OrderedDict 14 | import pandas as pd 15 | from SurfaceDice import compute_surface_distances, compute_surface_dice_at_tolerance, compute_dice_coefficient 16 | 17 | 18 | seg_path = 'path to segmentation' 19 | gt_path = 'path to ground truth' 20 | save_path = 'path to Results' 21 | save_name = 'DSC_NSD_teamname.csv' 22 | filenames = os.listdir(seg_path) 23 | filenames = [x for x in filenames if x.endswith('.nii.gz')] 24 | filenames.sort() 25 | 26 | seg_metrics = OrderedDict() 27 | seg_metrics['Name'] = list() 28 | label_tolerance = OrderedDict({'Liver': 5, 'RK':3, 'Spleen':3, 'Pancreas':5, 29 | 'Aorta': 2, 'IVC':2, 'RAG':2, 'LAG':2, 'Gallbladder': 2, 30 | 'Esophagus':3, 'Stomach': 5, 'Duodenum': 7, 'LK':3}) 31 | 32 | for organ in label_tolerance.keys(): 33 | seg_metrics['{}_DSC'.format(organ)] = list() 34 | for organ in label_tolerance.keys(): 35 | seg_metrics['{}_NSD'.format(organ)] = list() 36 | 37 | def find_lower_upper_zbound(organ_mask): 38 | """ 39 | Parameters 40 | ---------- 41 | seg : TYPE 42 | DESCRIPTION. 43 | 44 | Returns 45 | ------- 46 | z_lower: lower bound in z axis: int 47 | z_upper: upper bound in z axis: int 48 | 49 | """ 50 | organ_mask = np.uint8(organ_mask) 51 | assert np.max(organ_mask) ==1, print('mask label error!') 52 | z_index = np.where(organ_mask>0)[2] 53 | z_lower = np.min(z_index) 54 | z_upper = np.max(z_index) 55 | 56 | return z_lower, z_upper 57 | 58 | 59 | 60 | for name in filenames: 61 | seg_metrics['Name'].append(name) 62 | # load grond truth and segmentation 63 | gt_nii = nb.load(join(gt_path, name)) 64 | case_spacing = gt_nii.header.get_zooms() 65 | gt_data = np.uint8(gt_nii.get_fdata()) 66 | seg_data = np.uint8(nb.load(join(seg_path, name)).get_fdata()) 67 | 68 | for i, organ in enumerate(label_tolerance.keys(),1): 69 | if np.sum(gt_data==i)==0 and np.sum(seg_data==i)==0: 70 | DSC_i = 1 71 | NSD_i = 1 72 | elif np.sum(gt_data==i)==0 and np.sum(seg_data==i)>0: 73 | DSC_i = 0 74 | NSD_i = 0 75 | elif np.sum(gt_data==i)>0 and np.sum(seg_data==i)==0: 76 | DSC_i = 0 77 | NSD_i = 0 78 | else: 79 | if i==5 or i==6 or i==10: # for Aorta, IVC, and Esophagus, only evaluate the labelled slices in ground truth 80 | z_lower, z_upper = find_lower_upper_zbound(gt_data==i) 81 | organ_i_gt, organ_i_seg = gt_data[:,:,z_lower:z_upper]==i, seg_data[:,:,z_lower:z_upper]==i 82 | else: 83 | organ_i_gt, organ_i_seg = gt_data==i, seg_data==i 84 | DSC_i = compute_dice_coefficient(organ_i_gt, organ_i_seg) 85 | if DSC_i < 0.2: # don't compute NSD if DSC is too low 86 | NSD_i = 0 87 | else: 88 | surface_distances = compute_surface_distances(organ_i_gt, organ_i_seg, case_spacing) 89 | NSD_i = compute_surface_dice_at_tolerance(surface_distances, label_tolerance[organ]) 90 | seg_metrics['{}_DSC'.format(organ)].append(round(DSC_i, 4)) 91 | seg_metrics['{}_NSD'.format(organ)].append(round(NSD_i, 4)) 92 | print(name, organ, round(DSC_i,4), 'tol:', label_tolerance[organ], round(NSD_i,4)) 93 | 94 | dataframe = pd.DataFrame(seg_metrics) 95 | dataframe.to_csv(join(save_path, save_name), index=False) 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /FLARE24/__pycache__/SurfaceDice.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JunMa11/FLARE/aad2cc2813d11135d014bf578d4f62cea84ab865/FLARE24/__pycache__/SurfaceDice.cpython-310.pyc -------------------------------------------------------------------------------- /FLARE24/load_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | import csv 3 | import argparse 4 | import glob 5 | import matplotlib 6 | import os 7 | join = os.path.join 8 | import numpy as np 9 | 10 | matplotlib.use("Agg") 11 | import matplotlib.pyplot as plt 12 | 13 | from logger import add_file_handler_to_logger, logger 14 | 15 | add_file_handler_to_logger(name="main", dir_path=f"logs/", level="DEBUG") 16 | 17 | 18 | if __name__ == "__main__": 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("-docker_name", default="fully_suplearn_subtask1", help="team docker name") 21 | parser.add_argument("-save_path", default="./results", help="save_path") 22 | time_interval = 0.1 23 | args = parser.parse_args() 24 | logger.info("we are counting: {args.docker_name}") 25 | json_dir = join(args.save_path, args.docker_name) 26 | csv_path = join(json_dir, args.docker_name + '_Efficiency.csv') 27 | jsonl = sorted(glob.glob(json_dir + "/*.json")) 28 | alldata = [] 29 | for item in jsonl: 30 | csv_l = [] 31 | name = item.split(os.sep)[-1].split(".")[0] 32 | csv_l.append(name + '.nii.gz') 33 | zitem = item 34 | with open(zitem) as f: 35 | try: 36 | js = json.load(f) 37 | except Exception as error: 38 | logger.error(f"{item} have error") 39 | logger.exception(error) 40 | if "time" not in js: 41 | logger.error(f"{item} don't have time!!!!") 42 | logger.info(f"Manually compute {item}") 43 | time = time_interval * len(js["gpu_memory"]) 44 | else: 45 | time = js["time"] 46 | csv_l.append(np.round(time,2)) 47 | #CPU 48 | user, system, all_cpu_used = [item[0] for item in js['cpu_list']], [item[1] for item in js['cpu_list']], [ 49 | 100 - item[2] for item in js['cpu_list']] 50 | plt.cla() 51 | x = [item * time_interval for item in range(len(user))] 52 | plt.xlabel("Time (s)", fontsize="large") 53 | plt.ylabel("CPU Utilization (%)", fontsize="large") 54 | # plt.plot(x, user, "b", ms=10, label="User %") 55 | # plt.plot(x, system, "r", ms=10, label="System %") 56 | plt.plot(x, all_cpu_used, "b", ms=10, label="Used %") 57 | plt.legend() 58 | plt.savefig(zitem.replace(".json", "_CPU-Time.png")) 59 | #RAM 60 | RAM = js["RAM_list"] 61 | plt.cla() 62 | x = [item * time_interval for item in range(len(RAM))] 63 | plt.xlabel("Time (s)", fontsize="large") 64 | plt.ylabel("RAM (MB)", fontsize="large") 65 | plt.plot(x, RAM, "b", ms=10, label="RAM") 66 | plt.legend() 67 | plt.savefig(zitem.replace(".json", "_RAM-Time.png")) 68 | mem = js["gpu_memory"] 69 | x = [item * time_interval for item in range(len(mem))] 70 | plt.cla() 71 | plt.xlabel("Time (s)", fontsize="large") 72 | plt.ylabel("GPU Memory (MB)", fontsize="large") 73 | plt.plot(x, mem, "b", ms=10, label="a") 74 | plt.savefig(zitem.replace(".json", "_GPU-Time.png"), dpi=300) 75 | count_set = set(mem) 76 | # select the stable memory 77 | # count_list = [] 78 | # for citem in count_set: 79 | # cts = mem.count(citem) 80 | # if cts > 0.02 * len(mem): 81 | # count_list.append(citem) 82 | # directly count the max GPU memory 83 | max_mem = max(count_set) 84 | csv_l.append(np.round(max_mem)) 85 | csv_l.append(np.round(sum(mem) * time_interval)) 86 | csv_l.append(np.round(max(all_cpu_used),2)) 87 | csv_l.append(np.round(sum(all_cpu_used) * time_interval,2)) 88 | csv_l.append(np.round(max(RAM),2)) 89 | csv_l.append(np.round(sum(RAM) * time_interval)) 90 | alldata.append(csv_l) 91 | 92 | f = open(csv_path, "w",newline='') 93 | writer = csv.writer(f) 94 | writer.writerow(["Name", "Time", "MaxGPU_Mem", "AUC_GPU_Time",'MaxCPU_Utilization','AUC_CPU_Time','MaxRAM' 95 | ,'AUC_RAM_Time' ]) 96 | for i in alldata: 97 | writer.writerow(i) 98 | f.close() 99 | -------------------------------------------------------------------------------- /FLARE24/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from loguru import logger as loguru_logger 5 | 6 | loguru_logger.configure( 7 | handlers=[dict(sink=sys.stderr, filter=lambda record: record["extra"]["console"], level="DEBUG",),], 8 | ) 9 | 10 | logger = loguru_logger.bind(console=True) 11 | 12 | 13 | def add_file_handler_to_logger( 14 | name: str, dir_path="./logs", level="DEBUG", 15 | ): 16 | loguru_logger.add( 17 | sink=os.path.join(dir_path, f"{name}-{level}.log"), 18 | level=level, 19 | filter=lambda record: "console" in record["extra"], 20 | enqueue=True, 21 | ) 22 | -------------------------------------------------------------------------------- /FLARE24/resource_eval.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | import time 5 | import torch 6 | from pathlib import Path 7 | join = os.path.join 8 | from logger import add_file_handler_to_logger, logger 9 | 10 | 11 | def check_dir(file_path): 12 | file_path = Path(file_path) 13 | files = [f for f in file_path.iterdir() if ".nii.gz" in str(f)] 14 | if len(files) != 0: 15 | return False 16 | return True 17 | 18 | 19 | add_file_handler_to_logger(name="main", dir_path="logs/", level="DEBUG") 20 | 21 | docker_path = './team_docker/' # put docker in this folder 22 | test_img_path = './data/validation' # all validation cases 23 | save_path = './FLARE23_ValResults/' # evaluation results will be saved in this folder 24 | 25 | temp_in = './inputs/' 26 | temp_out = './outputs' 27 | os.makedirs(save_path, exist_ok=True) 28 | os.makedirs(temp_in, exist_ok=True) 29 | os.makedirs(temp_out, exist_ok=True) 30 | os.system("chmod -R 777 outputs/") 31 | 32 | dockers = sorted(os.listdir(docker_path)) 33 | test_cases = sorted(os.listdir(test_img_path)) 34 | 35 | for docker in dockers: 36 | try: 37 | name = docker.split('.')[0].lower() 38 | print('loading docker:', docker) 39 | os.system('docker load -i {}'.format(join(docker_path, docker))) 40 | team_outpath = join(save_path, name) 41 | if os.path.exists(team_outpath): 42 | shutil.rmtree(team_outpath) 43 | os.mkdir(team_outpath) 44 | for case in test_cases: 45 | if not check_dir(temp_in): 46 | logger.error("please check inputs folder", temp_in) 47 | raise 48 | shutil.copy(join(test_img_path, case), temp_in) 49 | start_time = time.time() 50 | try: 51 | shutil.rmtree('./inputs/nnUNet_raw/') 52 | shutil.rmtree('./inputs/nnUNet_preprocessed/') 53 | except: 54 | print('no temp files') 55 | os.system('python Efficiency.py -docker_name {} -save_file {}'.format(name, save_path)) 56 | logger.info(f"{case} finished!") 57 | os.remove(join('./inputs', case)) 58 | try: 59 | shutil.rmtree('./inputs/nnUNet_cropped_data/') 60 | shutil.rmtree('./inputs/nnUNet_raw_data/') 61 | shutil.rmtree('./inputs/nnUNet_preprocessed/') 62 | except: 63 | print('no temp files') 64 | # shutil.rmtree('./inputs') 65 | logger.info(f"{case} cost time: {time.time() - start_time}") 66 | # move segmentation file 67 | seg_name = case.split('_0000.nii.gz')[0]+'.nii.gz' 68 | if os.path.exists(join(temp_out, seg_name)): 69 | os.rename(join(temp_out, seg_name), join(team_outpath, seg_name)) 70 | 71 | os.system("python load_json.py -docker_name {} -save_path {}".format(name, save_path)) 72 | shutil.move(temp_out, team_outpath) 73 | os.mkdir(temp_out) 74 | torch.cuda.empty_cache() 75 | shutil.rmtree(temp_in) 76 | os.mkdir(temp_in) 77 | os.system("docker rmi {}:latest".format(name)) 78 | except Exception as e: 79 | logger.exception(e) 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Jun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Official Repository of FLARE Challenge 2 | 3 | This repository provides the evaluation code of the FLARE Challenges. More details are available on the corresponding websites. 4 | 5 | ## [MICCAI FLARE 2021](https://flare.grand-challenge.org/FLARE21/): Fast and Low GPU memory Abdominal oRgan sEgmentation in CT Scans 6 | 7 | - 511 cases from 11 medical centers 8 | - Fully supervised setting 9 | - Four segmentation targets: liver, kidney, spleen, and pancreas 10 | - Evaluation metrics: DSC, NSD, Running time, Maximum GPU memory consumption 11 | 12 | 13 | 14 | ## [MICCAI FLARE 2022](https://flare22.grand-challenge.org/): Fast and Low-resource semi-supervised Abdominal oRgan sEgmentation in CT Scans 15 | 16 | - 2300 CT scans from 50+ medical centers 17 | - Semi-supervised setting 18 | - 13 segmentation targets: liver, spleen, pancreas, right kidney, left kidney, stomach, gallbladder, esophagus, aorta, inferior vena cava, right adrenal gland, left adrenal gland, and duodenum 19 | - Evaluation metrics: DSC, NSD, Running time, Area under GPU memory-time curve and Area under CPU utilization-time curve 20 | 21 | 22 | ## [MICCAI FLARE 2023](https://codalab.lisn.upsaclay.fr/competitions/12239): Fast, Low-resource, and Accurate oRgan and Pan-cancer sEgmentation in Abdomen CT 23 | 24 | - 4000 CT scans from 50+ medical centers 25 | - Partial-label setting 26 | - 14 segmentation targets: liver, spleen, pancreas, right kidney, left kidney, stomach, gallbladder, esophagus, aorta, inferior vena cava, right adrenal gland, left adrenal gland, duodenum, and tumor 27 | - Evaluation metrics: DSC, NSD, Running time, and Area under GPU memory-time curve 28 | 29 | 30 | ## MICCAI FLARE 2024: Fast, Low-resource, and Accurate oRgan and Pan-cancer sEgmentation 31 | 32 | 33 | - [Task 1: Pan-cancer segmentation in CT scans](https://www.codabench.org/competitions/2319/) 34 | - [Task 2: Abdominal CT Organ Segmentation on Laptop](https://www.codabench.org/competitions/2320/) 35 | - [Task 3: Unsupervised Domain Adaptation for Abdominal Organ Segmentation in MRI Scans](https://www.codabench.org/competitions/2296/) 36 | --------------------------------------------------------------------------------