├── .gitignore
├── .idea
├── CASED-Tensorflow.iml
├── inspectionProfiles
│ └── Project_Default.xml
├── modules.xml
├── vcs.xml
└── workspace.xml
├── CASED_test.py
├── CASED_train.py
├── LICENSE
├── README.md
├── assests
├── framework.JPG
├── lr.JPG
├── network.JPG
├── nodule.png
├── nodule_label.png
├── patch.png
├── result1.JPG
├── result2.JPG
└── stride.png
├── main_test.py
├── main_train.py
├── ops.py
├── preprocessing
├── README
│ ├── convert_luna_to_npy_README.md
│ └── h5py_patch_README.md
├── all_in_one.py
├── convert_luna_to_npy.py
├── h5py_patch.py
└── preprocess_utils.py
└── utils.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 |
49 | # Translations
50 | *.mo
51 | *.pot
52 |
53 | # Django stuff:
54 | *.log
55 | local_settings.py
56 |
57 | # Flask stuff:
58 | instance/
59 | .webassets-cache
60 |
61 | # Scrapy stuff:
62 | .scrapy
63 |
64 | # Sphinx documentation
65 | docs/_build/
66 |
67 | # PyBuilder
68 | target/
69 |
70 | # Jupyter Notebook
71 | .ipynb_checkpoints
72 |
73 | # pyenv
74 | .python-version
75 |
76 | # celery beat schedule file
77 | celerybeat-schedule
78 |
79 | # SageMath parsed files
80 | *.sage.py
81 |
82 | # dotenv
83 | .env
84 |
85 | # virtualenv
86 | .venv
87 | venv/
88 | ENV/
89 |
90 | # Spyder project settings
91 | .spyderproject
92 | .spyproject
93 |
94 | # Rope project settings
95 | .ropeproject
96 |
97 | # mkdocs documentation
98 | /site
99 |
100 | # mypy
101 | .mypy_cache/
102 |
--------------------------------------------------------------------------------
/.idea/CASED-Tensorflow.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
13 |
14 |
15 |
16 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
126 |
127 |
128 |
129 |
130 | true
131 | DEFINITION_ORDER
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 | CSS
148 |
149 |
150 | Probable bugsCSS
151 |
152 |
153 | RELAX NG
154 |
155 |
156 |
157 |
158 | AngularJS
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
108 |
109 |
110 |
39 |
40 | * `stride` = It means how many spaces to move around the nodule
41 | * It is possible to stride `8 times(4 * 2)` in the ***x, y, z, xy, xz, yz, and xyz*** directions respectively. (*Then, get 56 patch*)
42 | * Therefore, It will be make the patch per single nodule point is 56 + 1 = `57`
43 | * `stride` = 8, `move` = 4
44 |
45 | 
46 |
47 | ### 3. get the coordinates of the nodule
48 | ```python
49 | def world_2_voxel(world_coord, origin, spacing):
50 | stretched_voxel_coord = np.absolute(world_coord - origin)
51 | voxel_coord = stretched_voxel_coord / spacing
52 | return voxel_coord
53 | ```
54 |
55 | ### 4. extract the patch that contains the nodule and any patches that are not included
56 | * the number of non-nodule patch = 3 * the number of nodule patch
57 | * For the label patch, resize the patch using `max-pooling` in `skimage`, since the output size of the network (U-net) is 8 * 8 * 8
58 | * Don't use `scipy.resize`... Because, if you use it, there is a phenomenon that the pixel value is 1 (True) disappears
59 |
60 | ```python
61 | def get_patch(image, coords, offset, patch_list, patch_flag=True):
62 | xyz = image[int(coords[0] - offset): int(coords[0] + offset),
63 | int(coords[1] - offset): int(coords[1] + offset),
64 | int(coords[2] - offset): int(coords[2] + offset)]
65 |
66 | if patch_flag:
67 | output = np.expand_dims(xyz, axis=-1)
68 | else: # label
69 | # resize xyz
70 | xyz = skimage.measure.block_reduce(xyz, (9, 9, 9), np.max)
71 | output = np.expand_dims(xyz, axis=-1)
72 |
73 | output = indices_to_one_hot(output.astype(np.int32), 2)
74 | output = np.reshape(output, (label_size, label_size, label_size, 2))
75 | output = output.astype(np.float32)
76 |
77 | patch_list.append(output)
78 | ```
79 |
80 | ### 5. create the patch with h5py
81 | * See [link](https://www.safaribooksonline.com/library/view/python-and-hdf5/9781491944981/ch04.html) for reasons why I chose `lzf type`
82 | ```python
83 | with h5py.File(save_path + 'subset' + str(i) + '.h5', 'w') as hf:
84 | hf.create_dataset('nodule', data=nodule[:], compression='lzf')
85 | hf.create_dataset('label_nodule', data=nodule_label[:], compression='lzf')
86 |
87 | hf.create_dataset('non_nodule', data=non_nodule[:], compression='lzf')
88 | hf.create_dataset('label_non_nodule', data=non_nodule_label[:], compression='lzf')
89 | ```
90 |
91 | ### 6. data load
92 | * Use multiprocessing.Pool
93 | * `process_num` is good for exponentiation of 2.
94 | * If you want to know how many cpu you have available in Linux... `grep -c processor /proc/cpuinfo`
95 | ```python
96 | from multiprocessing import Pool
97 |
98 | def nodule_hf(idx):
99 | with h5py.File(image_patch, 'r') as hf:
100 | nodule = hf['nodule'][idx:idx + get_data_num]
101 | return nodule
102 |
103 | process_num = 32
104 | get_data_num = 64
105 |
106 | with h5py.File(image_patch, 'r') as fin:
107 | nodule_range = range(0, len(fin['nodule']), get_data_num)
108 |
109 | pool = Pool(processes = process_num)
110 | pool_nodule = pool.map(nodule_hf, nodule_range)
111 | pool.close()
112 |
113 | nodule = []
114 |
115 | for p in pool_nodule :
116 | nodule.extend(p)
117 | ```
118 |
119 | ## Author
120 | Junho Kim / [@Lunit](http://lunit.io/)
121 |
--------------------------------------------------------------------------------
/preprocessing/all_in_one.py:
--------------------------------------------------------------------------------
1 | from preprocessing.preprocess_utils import *
2 | from random import randint
3 | import h5py
4 | import numpy as np
5 | OUTPUT_SPACING = [1.25, 1.25, 1.25]
6 | patch_size = 68
7 | label_size = 8
8 | stride = 8
9 | move = (patch_size // 2) // stride # 4
10 |
11 |
12 | if __name__=="__main__":
13 |
14 | from matplotlib.patches import Circle
15 |
16 | coord = False
17 |
18 | dst_spacing = OUTPUT_SPACING
19 | src_root = '/data/jhkim/LUNA16/original'
20 | #dst_root = '/lunit/data/LUNA16/npydata'
21 |
22 | save_path = '/data2/jhkim/LUNA16/patch/SH/'
23 |
24 | # if not os.path.exists(dst_root):
25 | # os.makedirs(dst_root)
26 |
27 | annotation_csv = os.path.join(src_root,'CSVFILES/annotations.csv')
28 | annotations = read_csv(annotation_csv)
29 |
30 | src_mhd = []
31 | coord_dict = {}
32 | for sub_n in range(10) :
33 | idx = 1
34 | sub_n_str = 'subset' + str(sub_n)
35 | image_paths = glob.glob(os.path.join(src_root,sub_n_str,'*.mhd'))
36 |
37 | nodule = []
38 | non_nodule = []
39 | nodule_label = []
40 | non_nodule_label = []
41 |
42 | for i in image_paths:
43 | filename = os.path.split(i)[-1]
44 | series_uid = os.path.splitext(filename)[0]
45 |
46 | subset_num = i.split('/')[-2]
47 | # dst_subset_path = os.path.join(dst_root,subset_num)
48 | #
49 | # if not os.path.exists(dst_subset_path):
50 | # os.makedirs(dst_subset_path)
51 |
52 | np_img, origin, spacing = load_itk_image(i)
53 |
54 | resampled_img, new_spacing = resample(np_img, spacing, dst_spacing)
55 | resampled_img_shape = resampled_img.shape
56 |
57 | norm_img = normalize_planes(resampled_img)
58 | norm_img = zero_center(norm_img)
59 | norm_img_shape = norm_img.shape
60 | nodule_coords = []
61 | try:
62 | nodule_coords = annotations[series_uid]
63 | label = create_label(resampled_img_shape, nodule_coords, origin, new_spacing, coord=coord)
64 | except:
65 | if coord:
66 | label = []
67 | else:
68 | label = np.zeros(resampled_img_shape, dtype='bool')
69 |
70 | image = np.transpose(norm_img)
71 | label = np.transpose(label)
72 | #origin = np.transpose(origin)
73 | #new_spacing = np.transpose(new_spacing)
74 |
75 | #np.save(os.path.join(dst_subset_path,series_uid+'.npy'), norm_img)
76 | #np.save(os.path.join(dst_subset_path,series_uid+'.label.npy'), label)
77 | #coord_dict[os.path.join(dst_subset_path,series_uid+'.npy')] = label
78 |
79 | # padding
80 | offset = patch_size // 2
81 |
82 | non_pad = image
83 | non_label_pad = label
84 |
85 | non_pad = np.pad(non_pad, offset, 'constant', constant_values=np.min(non_pad))
86 | non_label_pad = np.pad(non_label_pad, offset, 'constant', constant_values=np.min(non_label_pad))
87 |
88 | image = np.pad(image, offset + (stride * move), 'constant', constant_values=np.min(image))
89 | label = np.pad(label, offset + (stride * move), 'constant', constant_values=np.min(label))
90 |
91 | nodule_list = []
92 | nodule_label_list = []
93 |
94 | for nodule_coord in nodule_coords :
95 | worldCoord = nodule_coord['position']
96 | worldCoord = np.asarray([worldCoord[2], worldCoord[1], worldCoord[0]])
97 |
98 | # new_spacing came from resample
99 | voxelCoord = compute_coord(worldCoord, origin, new_spacing)
100 | voxelCoord = np.asarray([int(i) + offset + (stride * move) for i in voxelCoord])
101 | voxelCoord = np.transpose(voxelCoord)
102 |
103 | patch_stride(image, voxelCoord, offset, nodule_list) # x,y,z, xy,xz,yz, xyz ... get stride patch
104 | patch_stride(label, voxelCoord, offset, nodule_label_list, patch_flag=False)
105 |
106 | #print(series_uid)
107 |
108 | nodule_num = len(nodule_list)
109 |
110 | non_nodule_list = []
111 | non_nodule_label_list = []
112 | x_coords = non_pad.shape[0] - offset - 1
113 | y_coords = non_pad.shape[1] - offset - 1
114 | z_coords = non_pad.shape[2] - offset - 1
115 |
116 | while len(non_nodule_list) < 3 * nodule_num:
117 | rand_x = randint(offset, x_coords)
118 | rand_y = randint(offset, y_coords)
119 | rand_z = randint(offset, z_coords)
120 |
121 | coords = np.array([rand_x, rand_y, rand_z])
122 |
123 | get_patch(non_pad, coords, offset, non_nodule_list)
124 | get_patch(non_label_pad, coords, offset, non_nodule_label_list, patch_flag=False)
125 |
126 | nodule.extend(nodule_list)
127 | non_nodule.extend(non_nodule_list)
128 |
129 | nodule_label.extend(nodule_label_list)
130 | non_nodule_label.extend(non_nodule_label_list)
131 |
132 | print(sub_n_str + ' / ' + str(idx) + ' / ' + str(len(image_paths)))
133 | print('nodule : ', np.shape(nodule))
134 | print('nodule_label : ', np.shape(nodule_label))
135 | print('non-nodule : ', np.shape(non_nodule))
136 | print('non-nodule_label : ', np.shape(non_nodule_label))
137 | idx += 1
138 |
139 | np.random.seed(0)
140 | np.random.shuffle(nodule)
141 | np.random.seed(0)
142 | np.random.shuffle(nodule_label)
143 |
144 | np.random.seed(0)
145 | np.random.shuffle(non_nodule)
146 | np.random.seed(0)
147 | np.random.shuffle(non_nodule_label)
148 |
149 | train_nodule_len = int(len(nodule) * 0.7)
150 | train_non_nodule_len = int(len(non_nodule) * 0.7)
151 |
152 | with h5py.File(save_path + 'subset' + str(sub_n) + '.h5', 'w') as hf:
153 | hf.create_dataset('nodule', data=nodule[:], compression='lzf')
154 | hf.create_dataset('label_nodule', data=nodule_label[:], compression='lzf')
155 |
156 | hf.create_dataset('non_nodule', data=non_nodule[:], compression='lzf')
157 | hf.create_dataset('label_non_nodule', data=non_nodule_label[:], compression='lzf')
158 |
159 |
160 |
161 | # with h5py.File(save_path + 'subset' + str(sub_n) + '.h5', 'w') as hf:
162 | # hf.create_dataset('nodule', data=nodule[:train_nodule_len], compression='lzf')
163 | # hf.create_dataset('label_nodule', data=nodule_label[:train_nodule_len], compression='lzf')
164 | #
165 | # hf.create_dataset('non_nodule', data=non_nodule[:train_non_nodule_len], compression='lzf')
166 | # hf.create_dataset('label_non_nodule', data=non_nodule_label[:train_non_nodule_len], compression='lzf')
167 | #
168 | # with h5py.File(save_path + 't_subset' + str(sub_n) + '.h5', 'w') as hf:
169 | # hf.create_dataset('nodule', data=nodule[train_nodule_len:], compression='lzf')
170 | # hf.create_dataset('label_nodule', data=nodule_label[train_nodule_len:], compression='lzf')
171 | #
172 | # hf.create_dataset('non_nodule', data=non_nodule[train_non_nodule_len:], compression='lzf')
173 | # hf.create_dataset('label_non_nodule', data=non_nodule_label[train_non_nodule_len:], compression='lzf')
174 |
175 |
176 |
--------------------------------------------------------------------------------
/preprocessing/convert_luna_to_npy.py:
--------------------------------------------------------------------------------
1 | import SimpleITK as sitk
2 | import numpy as np
3 | import csv
4 | import os, glob
5 | from PIL import Image
6 | import matplotlib.pyplot as plt
7 | import pickle
8 |
9 | import scipy.ndimage
10 | from skimage import measure, morphology
11 | from mpl_toolkits.mplot3d.art3d import Poly3DCollection
12 | from time import time
13 |
14 | OUTPUT_SPACING = [1.25, 1.25, 1.25]
15 | def load_itk_image(filename):
16 | itkimage = sitk.ReadImage(filename)
17 | numpyImage = sitk.GetArrayFromImage(itkimage)
18 |
19 | numpyOrigin = np.array(list(reversed(itkimage.GetOrigin())))
20 | numpySpacing = np.array(list(reversed(itkimage.GetSpacing())))
21 |
22 | return numpyImage, numpyOrigin, numpySpacing
23 |
24 | def read_csv(filename):
25 | lines = []
26 | with open(filename, 'r') as f:
27 | csvreader = csv.reader(f)
28 | for line in csvreader:
29 | lines.append(line)
30 |
31 | lines = lines[1:] # remove csv headers
32 | annotations_dict = {}
33 | for i in lines:
34 | series_uid, x, y, z, diameter = i
35 | value = {'position':[float(x),float(y),float(z)],
36 | 'diameter':float(diameter)}
37 | if series_uid in annotations_dict.keys():
38 | annotations_dict[series_uid].append(value)
39 | else:
40 | annotations_dict[series_uid] = [value]
41 |
42 | return annotations_dict
43 |
44 | def compute_coord(worldCoord, origin, spacing):
45 | stretchedVoxelCoord = np.absolute(worldCoord - origin)
46 | voxelCoord = stretchedVoxelCoord / spacing
47 | return voxelCoord
48 |
49 | def normalize_planes(npzarray):
50 | #maxHU = 600.
51 | #minHU = -1200.
52 | maxHU = 400.
53 | minHU = -1000.
54 | npzarray = (npzarray - minHU) / (maxHU - minHU)
55 | npzarray[npzarray>1] = 1.
56 | npzarray[npzarray<0] = 0.
57 | return npzarray
58 |
59 | def zero_center(image):
60 | PIXEL_MEAN = 0.25
61 | image = image - PIXEL_MEAN
62 |
63 | return image
64 |
65 | def resample(image, org_spacing, new_spacing=OUTPUT_SPACING):
66 |
67 | resize_factor = org_spacing / new_spacing
68 | new_real_shape = image.shape * resize_factor
69 | new_shape = np.round(new_real_shape)
70 | real_resize_factor = new_shape / image.shape
71 | new_spacing = org_spacing / real_resize_factor
72 |
73 | image = scipy.ndimage.interpolation.zoom(image, real_resize_factor, mode='nearest')
74 |
75 | return image, new_spacing
76 |
77 | def create_label(arr_shape, nodules, new_spacing, coord=False):
78 | """
79 | nodules = list of dict {'position', 'diameter'}
80 | """
81 | def _create_mask(arr_shape, position, diameter):
82 |
83 | z_dim, y_dim, x_dim = arr_shape
84 | z_pos, y_pos, x_pos = position
85 |
86 | z,y,x = np.ogrid[-z_pos:z_dim-z_pos, -y_pos:y_dim-y_pos, -x_pos:x_dim-x_pos]
87 | mask = z**2 + y**2 + x**2 <= int(diameter//2)**2
88 |
89 | return mask
90 |
91 | if coord:
92 | label = []
93 | else:
94 | label = np.zeros(arr_shape, dtype='bool')
95 |
96 | for nodule in nodules:
97 | worldCoord = nodule['position']
98 | worldCoord = np.asarray([worldCoord[2],worldCoord[1],worldCoord[0]])
99 |
100 | # new_spacing came from resample
101 | voxelCoord = compute_coord(worldCoord, origin, new_spacing)
102 | voxelCoord = [int(i) for i in voxelCoord]
103 |
104 | diameter = nodule['diameter']
105 | diameter = diameter / new_spacing[1]
106 |
107 | if coord:
108 | label.append(voxelCoord + [diameter])
109 | else:
110 | mask = _create_mask(arr_shape, voxelCoord, diameter)
111 | label = np.logical_or(label, mask)
112 |
113 | return label
114 |
115 |
116 | def plot(image, label, z_idx):
117 |
118 | fig = plt.figure()
119 | ax = fig.add_subplot(1,2,1)
120 | ax.imshow(image[z_idx,:,:],cmap='gray')
121 |
122 | ax = fig.add_subplot(1,2,2)
123 | ax.imshow(label[z_idx,:,:],cmap='gray')
124 |
125 | fig.show()
126 |
127 | if __name__=="__main__":
128 |
129 | from matplotlib.patches import Circle
130 |
131 | coord = True
132 | exclude_flag = True
133 |
134 | dst_spacing = OUTPUT_SPACING
135 | #src_root = '/lunit/data/LUNA16/rawdata'
136 | dst_root = '/data2/jhkim/npydata'
137 |
138 | src_root = '/data/jhkim/LUNA16/original'
139 |
140 |
141 | if not os.path.exists(dst_root):
142 | os.makedirs(dst_root)
143 |
144 | if exclude_flag :
145 | annotation_csv = os.path.join(src_root,'CSVFILES/annotations_excluded.csv')
146 | else :
147 | annotation_csv = os.path.join(src_root,'CSVFILES/annotations.csv')
148 |
149 | annotations = read_csv(annotation_csv)
150 |
151 | src_mhd = []
152 | coord_dict = {}
153 | all_file = len(glob.glob(os.path.join(src_root,'subset[0-9]','*.mhd')))
154 | cnt = 1
155 |
156 | for i in glob.glob(os.path.join(src_root,'subset[0-9]','*.mhd')):
157 | st = time()
158 | filename = os.path.split(i)[-1]
159 | series_uid = os.path.splitext(filename)[0]
160 |
161 | subset_num = i.split('/')[-2]
162 | dst_subset_path = os.path.join(dst_root,subset_num)
163 |
164 | if not os.path.exists(dst_subset_path):
165 | os.makedirs(dst_subset_path)
166 |
167 | np_img, origin, spacing = load_itk_image(i)
168 |
169 | resampled_img, new_spacing = resample(np_img, spacing, dst_spacing)
170 | resampled_img_shape = resampled_img.shape
171 |
172 | norm_img = normalize_planes(resampled_img)
173 | norm_img = zero_center(norm_img)
174 |
175 | try:
176 | nodules = annotations[series_uid]
177 | label = create_label(resampled_img_shape, nodules, new_spacing, coord=coord)
178 | except:
179 | if coord:
180 | label = []
181 | else:
182 | label = np.zeros(resampled_img_shape, dtype='bool')
183 |
184 | # np.save(os.path.join(dst_subset_path,series_uid+'.npy'), norm_img)
185 | # np.save(os.path.join(dst_subset_path,series_uid+'.label.npy'), label)
186 |
187 | coord_dict[series_uid] = label
188 | with open('exclude.pkl', 'wb') as f:
189 | pickle.dump(coord_dict, f, protocol=pickle.HIGHEST_PROTOCOL)
190 | print('{} / {} / {}'.format(cnt, all_file, time() - st))
191 | cnt += 1
192 |
193 |
194 |
195 |
--------------------------------------------------------------------------------
/preprocessing/h5py_patch.py:
--------------------------------------------------------------------------------
1 | import glob
2 | import os
3 | from random import randint
4 | import h5py
5 | import numpy as np
6 | import pandas as pd
7 | from utils import load_itk, world_2_voxel
8 | import scipy.ndimage
9 | from skimage.measure import block_reduce
10 |
11 | OUTPUT_SPACING = [1.25, 1.25, 1.25]
12 | patch_size = 68
13 | label_size = 8
14 | stride = 8
15 |
16 | move = (patch_size // 2) // stride # 4
17 | annotations = pd.read_csv('/data/jhkim/LUNA16/CSVFILES/annotations.csv') # save dict format may be..
18 |
19 |
20 | def indices_to_one_hot(data, nb_classes):
21 | """Convert an iterable of indices to one-hot encoded labels."""
22 | targets = np.array(data).reshape(-1)
23 | return np.eye(nb_classes)[targets]
24 |
25 |
26 | def normalize(image):
27 | maxHU = 400.
28 | minHU = -1000.
29 |
30 | image = (image - minHU) / (maxHU - minHU)
31 | image[image > 1] = 1.
32 | image[image < 0] = 0.
33 | return image
34 |
35 |
36 | def zero_center(image):
37 | PIXEL_MEAN = 0.25
38 | image = image - PIXEL_MEAN
39 |
40 | return image
41 |
42 |
43 | def get_patch(image, coords, offset, nodule_list, patch_flag=True):
44 | xyz = image[int(coords[0] - offset): int(coords[0] + offset), int(coords[1] - offset): int(coords[1] + offset),
45 | int(coords[2] - offset): int(coords[2] + offset)]
46 |
47 | if patch_flag:
48 | output = np.expand_dims(xyz, axis=-1)
49 | else:
50 | # resize xyz
51 | """
52 | xyz = scipy.ndimage.zoom(input=xyz, zoom=1/8, order=1) # nearest
53 | xyz = np.where(xyz > 0, 1.0, 0.0)
54 | """
55 | xyz = block_reduce(xyz, (9, 9, 9), np.max)
56 | output = np.expand_dims(xyz, axis=-1)
57 |
58 | output = indices_to_one_hot(output.astype(np.int32), 2)
59 | output = np.reshape(output, (label_size, label_size, label_size, 2))
60 | output = output.astype(np.float32)
61 |
62 | # print('------------------')
63 | # print(output)
64 |
65 | # print(output)
66 | # print(np.shape(output))
67 |
68 | nodule_list.append(output)
69 |
70 |
71 | def patch_stride(image, coords, offset, nodule_list, patch_flag=True):
72 | # get stride * 7
73 | # In this case, get 8*7 = 56
74 | original_coords = coords
75 | move_stride = stride
76 |
77 | # center
78 | get_patch(image, coords, offset, nodule_list, patch_flag)
79 |
80 | # x
81 | for idx in range(1, move + 1):
82 | move_stride = stride * idx
83 | coords[0] = coords[0] + move_stride
84 | get_patch(image, coords, offset, nodule_list, patch_flag)
85 | coords = original_coords
86 |
87 | coords[0] = coords[0] - move_stride
88 | get_patch(image, coords, offset, nodule_list, patch_flag)
89 | coords = original_coords
90 |
91 | # y
92 | for idx in range(1, move + 1):
93 | move_stride = stride * idx
94 | coords[1] = coords[1] + move_stride
95 | get_patch(image, coords, offset, nodule_list, patch_flag)
96 | coords = original_coords
97 |
98 | coords[1] = coords[1] - move_stride
99 | get_patch(image, coords, offset, nodule_list, patch_flag)
100 | coords = original_coords
101 |
102 | # z
103 | for idx in range(1, move + 1):
104 | move_stride = stride * idx
105 | coords[2] = coords[2] + move_stride
106 | get_patch(image, coords, offset, nodule_list, patch_flag)
107 | coords = original_coords
108 |
109 | coords[2] = coords[2] - move_stride
110 | get_patch(image, coords, offset, nodule_list, patch_flag)
111 | coords = original_coords
112 |
113 | # xy
114 | for idx in range(1, move + 1):
115 | move_stride = stride * idx
116 | coords[0] = coords[0] + move_stride
117 | coords[1] = coords[1] + move_stride
118 | get_patch(image, coords, offset, nodule_list, patch_flag)
119 | coords = original_coords
120 |
121 | coords[0] = coords[0] - move_stride
122 | coords[1] = coords[1] - move_stride
123 | get_patch(image, coords, offset, nodule_list, patch_flag)
124 | coords = original_coords
125 |
126 | # xz
127 | for idx in range(1, move + 1):
128 | move_stride = stride * idx
129 | coords[0] = coords[0] + move_stride
130 | coords[2] = coords[2] + move_stride
131 | get_patch(image, coords, offset, nodule_list, patch_flag)
132 | coords = original_coords
133 |
134 | coords[0] = coords[0] - move_stride
135 | coords[2] = coords[2] - move_stride
136 | get_patch(image, coords, offset, nodule_list, patch_flag)
137 | coords = original_coords
138 |
139 | # yz
140 | for idx in range(1, move + 1):
141 | move_stride = stride * idx
142 | coords[1] = coords[1] + move_stride
143 | coords[2] = coords[2] + move_stride
144 | get_patch(image, coords, offset, nodule_list, patch_flag)
145 | coords = original_coords
146 |
147 | coords[1] = coords[1] - move_stride
148 | coords[2] = coords[2] - move_stride
149 | get_patch(image, coords, offset, nodule_list, patch_flag)
150 | coords = original_coords
151 |
152 | # xyz
153 | for idx in range(1, move + 1):
154 | move_stride = stride * idx
155 | coords += move_stride
156 | get_patch(image, coords, offset, nodule_list, patch_flag)
157 | coords = original_coords
158 |
159 | coords -= move_stride
160 | get_patch(image, coords, offset, nodule_list, patch_flag)
161 | coords = original_coords
162 |
163 |
164 | def process_image(image_path, annotations, nodule, non_nodule, nodule_label, non_nodule_label):
165 | image, origin, spacing = load_itk(image_path) # 512 512 119
166 | image_name = os.path.split(image_path)[1].replace('.mhd', '')
167 |
168 | subset_name = image_path.split('/')[-2]
169 | SH_path = '/data2/jhkim/npydata/' + subset_name + '/' + image_name + '.npy'
170 | label_name = '/data2/jhkim/npydata/' + subset_name + '/' + image_name + '.label.npy'
171 |
172 | # calculate resize factor
173 | resize_factor = spacing / OUTPUT_SPACING
174 | new_real_shape = image.shape * resize_factor
175 | new_shape = np.round(new_real_shape)
176 | real_resize = new_shape / image.shape
177 | new_spacing = spacing / real_resize
178 |
179 | image = np.transpose(np.load(SH_path))
180 | label = np.transpose(np.load(label_name))
181 |
182 | # image = normalize(image)
183 | # image = zero_center(image)
184 |
185 | # padding
186 | offset = patch_size // 2
187 |
188 | non_pad = image
189 | non_label_pad = label
190 |
191 | non_pad = np.pad(non_pad, offset, 'constant', constant_values=np.min(non_pad))
192 | non_label_pad = np.pad(non_label_pad, offset, 'constant', constant_values=np.min(non_label_pad))
193 |
194 | image = np.pad(image, offset + (stride * move), 'constant', constant_values=np.min(image))
195 | label = np.pad(label, offset + (stride * move), 'constant', constant_values=np.min(label))
196 |
197 |
198 | indices = annotations[annotations['seriesuid'] == image_name].index
199 |
200 | nodule_list = []
201 | nodule_label_list = []
202 | for i in indices:
203 | row = annotations.iloc[i]
204 | world_coords = np.array([row.coordX, row.coordY, row.coordZ])
205 |
206 | coords = np.floor(world_2_voxel(world_coords, origin, new_spacing)) + offset + (stride * move) # center
207 | patch_stride(image, coords, offset, nodule_list) # x,y,z, xy,xz,yz, xyz ... get stride patch
208 | patch_stride(label, coords, offset, nodule_label_list, patch_flag=False)
209 |
210 | nodule_num = len(nodule_list)
211 |
212 | non_nodule_list = []
213 | non_nodule_label_list = []
214 | x_coords = non_pad.shape[0] - offset - 1
215 | y_coords = non_pad.shape[1] - offset - 1
216 | z_coords = non_pad.shape[2] - offset - 1
217 |
218 | while len(non_nodule_list) < 3 * nodule_num:
219 | rand_x = randint(offset, x_coords)
220 | rand_y = randint(offset, y_coords)
221 | rand_z = randint(offset, z_coords)
222 |
223 | coords = np.array([rand_x, rand_y, rand_z])
224 |
225 | get_patch(non_pad, coords, offset, non_nodule_list)
226 | get_patch(non_label_pad, coords, offset, non_nodule_label_list, patch_flag=False)
227 |
228 | nodule.extend(nodule_list)
229 | non_nodule.extend(non_nodule_list)
230 | nodule_label.extend(nodule_label_list)
231 | non_nodule_label.extend(non_nodule_label_list)
232 |
233 | print('nodule : ', np.shape(nodule))
234 | print('nodule_label : ', np.shape(nodule_label))
235 | print('non-nodule : ', np.shape(non_nodule))
236 | print('non-nodule_label : ', np.shape(non_nodule_label))
237 |
238 |
239 | for i in range(10):
240 | image_paths = glob.glob("/data/jhkim/LUNA16/original/subset" + str(i) + '/*.mhd')
241 | nodule = []
242 | non_nodule = []
243 | nodule_label = []
244 | non_nodule_label = []
245 |
246 | idx = 1
247 | flag = 1
248 | save_path = '/data2/jhkim/LUNA16/patch/SH/'
249 |
250 | for image_path in image_paths:
251 | print('subset' + str(i) + ' / ' + str(idx) + ' / ' + str(len(image_paths)))
252 | process_image(image_path, annotations, nodule, non_nodule, nodule_label, non_nodule_label)
253 | idx += 1
254 |
255 | np.random.seed(0)
256 | np.random.shuffle(nodule)
257 | np.random.seed(0)
258 | np.random.shuffle(nodule_label)
259 |
260 | np.random.seed(0)
261 | np.random.shuffle(non_nodule)
262 | np.random.seed(0)
263 | np.random.shuffle(non_nodule_label)
264 |
265 | train_nodule_len = int(len(nodule) * 0.7)
266 | train_non_nodule_len = int(len(non_nodule) * 0.7)
267 |
268 | # print(np.shape(nodule))
269 | # print(np.shape(non_nodule))
270 |
271 | # print(np.shape(nodule[:train_nodule_len]))
272 |
273 | with h5py.File(save_path + 'subset' + str(i) + '.h5', 'w') as hf:
274 | hf.create_dataset('nodule', data=nodule[:], compression='lzf')
275 | hf.create_dataset('label_nodule', data=nodule_label[:], compression='lzf')
276 |
277 | hf.create_dataset('non_nodule', data=non_nodule[:], compression='lzf')
278 | hf.create_dataset('label_non_nodule', data=non_nodule_label[:], compression='lzf')
279 |
280 | # with h5py.File(save_path + 't_subset' + str(i) + '.h5', 'w') as hf:
281 | # hf.create_dataset('nodule', data=nodule[train_nodule_len:], compression='lzf')
282 | # hf.create_dataset('label_nodule', data=nodule_label[train_nodule_len:], compression='lzf')
283 | #
284 | # hf.create_dataset('non_nodule', data=non_nodule[train_non_nodule_len:], compression='lzf')
285 | # hf.create_dataset('label_non_nodule', data=non_nodule_label[train_non_nodule_len:], compression='lzf')
286 |
--------------------------------------------------------------------------------
/preprocessing/preprocess_utils.py:
--------------------------------------------------------------------------------
1 | import SimpleITK as sitk
2 | import numpy as np
3 | import csv
4 | import os, glob
5 | from PIL import Image
6 | import matplotlib.pyplot as plt
7 |
8 | import scipy.ndimage
9 | from skimage import measure, morphology
10 | from mpl_toolkits.mplot3d.art3d import Poly3DCollection
11 | from skimage.measure import block_reduce
12 |
13 | OUTPUT_SPACING = [1.25, 1.25, 1.25]
14 | patch_size = 68
15 | label_size = 8
16 | stride = 8
17 |
18 | move = (patch_size // 2) // stride # 4
19 |
20 | def load_itk_image(filename):
21 | itkimage = sitk.ReadImage(filename)
22 | numpyImage = sitk.GetArrayFromImage(itkimage)
23 |
24 | numpyOrigin = np.array(list(reversed(itkimage.GetOrigin())))
25 | numpySpacing = np.array(list(reversed(itkimage.GetSpacing())))
26 |
27 | return numpyImage, numpyOrigin, numpySpacing
28 |
29 | def read_csv(filename):
30 | lines = []
31 | with open(filename, 'r') as f:
32 | csvreader = csv.reader(f)
33 | for line in csvreader:
34 | lines.append(line)
35 |
36 | lines = lines[1:] # remove csv headers
37 | annotations_dict = {}
38 | for i in lines:
39 | series_uid, x, y, z, diameter = i
40 | value = {'position':[float(x),float(y),float(z)],
41 | 'diameter':float(diameter)}
42 | if series_uid in annotations_dict.keys():
43 | annotations_dict[series_uid].append(value)
44 | else:
45 | annotations_dict[series_uid] = [value]
46 |
47 | return annotations_dict
48 |
49 | def compute_coord(worldCoord, origin, spacing):
50 | stretchedVoxelCoord = np.absolute(worldCoord - origin)
51 | voxelCoord = stretchedVoxelCoord / spacing
52 | return voxelCoord
53 |
54 | def normalize_planes(npzarray):
55 | maxHU = 400. # 600
56 | minHU = -1000. # -1200
57 |
58 | npzarray = (npzarray - minHU) / (maxHU - minHU)
59 | npzarray[npzarray>1] = 1.
60 | npzarray[npzarray<0] = 0.
61 | return npzarray
62 |
63 | def zero_center(image):
64 | PIXEL_MEAN = 0.25
65 | image = image - PIXEL_MEAN
66 |
67 | return image
68 |
69 | def indices_to_one_hot(data, nb_classes):
70 | """Convert an iterable of indices to one-hot encoded labels."""
71 | targets = np.array(data).reshape(-1)
72 | return np.eye(nb_classes)[targets]
73 |
74 | def get_patch(image, coords, offset, nodule_list, patch_flag=True):
75 | xyz = image[int(coords[0] - offset): int(coords[0] + offset), int(coords[1] - offset): int(coords[1] + offset),
76 | int(coords[2] - offset): int(coords[2] + offset)]
77 |
78 | if patch_flag:
79 | output = np.expand_dims(xyz, axis=-1)
80 | print(coords, np.shape(output))
81 | else:
82 | # resize xyz
83 | """
84 | xyz = scipy.ndimage.zoom(input=xyz, zoom=1/8, order=1) # nearest
85 | xyz = np.where(xyz > 0, 1.0, 0.0)
86 | """
87 | xyz = block_reduce(xyz, (9, 9, 9), np.max)
88 | output = np.expand_dims(xyz, axis=-1)
89 |
90 | output = indices_to_one_hot(output.astype(np.int32), 2)
91 | output = np.reshape(output, (label_size, label_size, label_size, 2))
92 | output = output.astype(np.float32)
93 |
94 | # print('------------------')
95 | # print(output)
96 |
97 | # print(output)
98 | # print(np.shape(output))
99 |
100 | nodule_list.append(output)
101 |
102 | def patch_stride(image, coords, offset, nodule_list, patch_flag=True):
103 | # get stride * 7
104 | # In this case, get 8*7 = 56
105 | original_coords = coords
106 | move_stride = stride
107 |
108 | # center
109 | get_patch(image, coords, offset, nodule_list, patch_flag)
110 |
111 | # x
112 | for idx in range(1, move + 1):
113 | move_stride = stride * idx
114 | coords[0] = coords[0] + move_stride
115 | get_patch(image, coords, offset, nodule_list, patch_flag)
116 | coords = original_coords
117 |
118 | coords[0] = coords[0] - move_stride
119 | get_patch(image, coords, offset, nodule_list, patch_flag)
120 | coords = original_coords
121 |
122 | # y
123 | for idx in range(1, move + 1):
124 | move_stride = stride * idx
125 | coords[1] = coords[1] + move_stride
126 | get_patch(image, coords, offset, nodule_list, patch_flag)
127 | coords = original_coords
128 |
129 | coords[1] = coords[1] - move_stride
130 | get_patch(image, coords, offset, nodule_list, patch_flag)
131 | coords = original_coords
132 |
133 | # z
134 | for idx in range(1, move + 1):
135 | move_stride = stride * idx
136 | coords[2] = coords[2] + move_stride
137 | get_patch(image, coords, offset, nodule_list, patch_flag)
138 | coords = original_coords
139 |
140 | coords[2] = coords[2] - move_stride
141 | get_patch(image, coords, offset, nodule_list, patch_flag)
142 | coords = original_coords
143 |
144 | # xy
145 | for idx in range(1, move + 1):
146 | move_stride = stride * idx
147 | coords[0] = coords[0] + move_stride
148 | coords[1] = coords[1] + move_stride
149 | get_patch(image, coords, offset, nodule_list, patch_flag)
150 | coords = original_coords
151 |
152 | coords[0] = coords[0] - move_stride
153 | coords[1] = coords[1] - move_stride
154 | get_patch(image, coords, offset, nodule_list, patch_flag)
155 | coords = original_coords
156 |
157 | # xz
158 | for idx in range(1, move + 1):
159 | move_stride = stride * idx
160 | coords[0] = coords[0] + move_stride
161 | coords[2] = coords[2] + move_stride
162 | get_patch(image, coords, offset, nodule_list, patch_flag)
163 | coords = original_coords
164 |
165 | coords[0] = coords[0] - move_stride
166 | coords[2] = coords[2] - move_stride
167 | get_patch(image, coords, offset, nodule_list, patch_flag)
168 | coords = original_coords
169 |
170 | # yz
171 | for idx in range(1, move + 1):
172 | move_stride = stride * idx
173 | coords[1] = coords[1] + move_stride
174 | coords[2] = coords[2] + move_stride
175 | get_patch(image, coords, offset, nodule_list, patch_flag)
176 | coords = original_coords
177 |
178 | coords[1] = coords[1] - move_stride
179 | coords[2] = coords[2] - move_stride
180 | get_patch(image, coords, offset, nodule_list, patch_flag)
181 | coords = original_coords
182 |
183 | # xyz
184 | for idx in range(1, move + 1):
185 | move_stride = stride * idx
186 | coords += move_stride
187 | get_patch(image, coords, offset, nodule_list, patch_flag)
188 | coords = original_coords
189 |
190 | coords -= move_stride
191 | get_patch(image, coords, offset, nodule_list, patch_flag)
192 | coords = original_coords
193 |
194 | def resample(image, org_spacing, new_spacing):
195 |
196 | resize_factor = org_spacing / new_spacing
197 | new_real_shape = image.shape * resize_factor
198 | new_shape = np.round(new_real_shape)
199 | real_resize_factor = new_shape / image.shape
200 | new_spacing = org_spacing / real_resize_factor
201 |
202 | image = scipy.ndimage.interpolation.zoom(image, real_resize_factor, mode='nearest')
203 |
204 | return image, new_spacing
205 |
206 | def create_label(arr_shape, nodules, origin, new_spacing, coord=False):
207 | """
208 | nodules = list of dict {'position', 'diameter'}
209 | """
210 |
211 | def _create_mask(arr_shape, position, diameter):
212 |
213 | z_dim, y_dim, x_dim = arr_shape
214 | z_pos, y_pos, x_pos = position
215 |
216 | z,y,x = np.ogrid[-z_pos:z_dim-z_pos, -y_pos:y_dim-y_pos, -x_pos:x_dim-x_pos]
217 | mask = z**2 + y**2 + x**2 <= int(diameter//2)**2
218 |
219 | return mask
220 |
221 | if coord:
222 | label = []
223 | else:
224 | label = np.zeros(arr_shape, dtype='bool')
225 |
226 | for nodule in nodules:
227 | worldCoord = nodule['position']
228 | worldCoord = np.asarray([worldCoord[2],worldCoord[1],worldCoord[0]])
229 |
230 | # new_spacing came from resample
231 | voxelCoord = compute_coord(worldCoord, origin, new_spacing)
232 | voxelCoord = [int(i) for i in voxelCoord]
233 |
234 | diameter = nodule['diameter']
235 | diameter = diameter / new_spacing[1]
236 |
237 | if coord:
238 | label.append(voxelCoord + [diameter])
239 | else:
240 | mask = _create_mask(arr_shape, voxelCoord, diameter)
241 | label = np.logical_or(label, mask)
242 |
243 | return label
244 |
245 |
246 | def plot(image, label, z_idx):
247 |
248 | fig = plt.figure()
249 | ax = fig.add_subplot(1,2,1)
250 | ax.imshow(image[z_idx,:,:],cmap='gray')
251 |
252 | ax = fig.add_subplot(1,2,2)
253 | ax.imshow(label[z_idx,:,:],cmap='gray')
254 |
255 | fig.show()
256 |
--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import SimpleITK as sitk
3 | import tensorflow as tf
4 | import os, h5py, glob
5 | import tensorflow.contrib.slim as slim
6 | from multiprocessing import Pool
7 | from sklearn.metrics import roc_curve, auc
8 | import matplotlib.pyplot as plt
9 | from matplotlib.ticker import FixedFormatter
10 | import matplotlib
11 |
12 |
13 | seed = 0
14 | file = ""
15 |
16 | process_num = 32
17 | get_data_num = 64
18 |
19 |
20 | def load_itk(filename):
21 | itkimage = sitk.ReadImage(filename)
22 | image = np.transpose(sitk.GetArrayFromImage(itkimage))
23 | origin = np.array(itkimage.GetOrigin())
24 | spacing = np.array(itkimage.GetSpacing())
25 | return image, origin, spacing
26 |
27 |
28 | def world_2_voxel(world_coord, origin, spacing):
29 | stretched_voxel_coord = np.absolute(world_coord - origin)
30 | voxel_coord = stretched_voxel_coord / spacing
31 | return voxel_coord
32 |
33 |
34 | def voxel_2_world(voxel_coord, origin, spacing):
35 | stretched_voxel_coord = voxel_coord * spacing
36 | world_coord = stretched_voxel_coord + origin
37 | return world_coord
38 |
39 |
40 | def show_all_variables():
41 | model_vars = tf.trainable_variables()
42 | slim.model_analyzer.analyze_vars(model_vars, print_info=True)
43 |
44 |
45 | def check_folder(log_dir):
46 | if not os.path.exists(log_dir):
47 | os.makedirs(log_dir)
48 | return log_dir
49 |
50 |
51 | def nodule_hf(idx):
52 | with h5py.File(file, 'r') as hf:
53 | nodule = hf['nodule'][idx:idx + get_data_num]
54 | # print(np.shape(nodule))
55 | return nodule
56 |
57 |
58 | def non_nodule_hf(idx):
59 | with h5py.File(file, 'r') as hf:
60 | non_nodule = hf['non_nodule'][idx:idx + get_data_num]
61 | return non_nodule
62 |
63 | def label_nodule_hf(idx):
64 | with h5py.File(file, 'r') as hf:
65 | nodule = hf['label_nodule'][idx:idx + get_data_num]
66 | # print(np.shape(nodule))
67 | return nodule
68 |
69 |
70 | def label_non_nodule_hf(idx):
71 | with h5py.File(file, 'r') as hf:
72 | non_nodule = hf['label_non_nodule'][idx:idx + get_data_num]
73 | return non_nodule
74 |
75 |
76 | def prepare_data(sub_n):
77 | global file
78 | dir = '/data2/jhkim/LUNA16/patch/SH/subset'
79 | file = ''.join([dir, str(sub_n), '.h5'])
80 |
81 | with h5py.File(file, 'r') as fin:
82 | nodule_range = range(0, len(fin['nodule']), get_data_num)
83 | non_nodule_range = range(0, len(fin['non_nodule']), get_data_num)
84 | label_nodule_range = range(0, len(fin['label_nodule']), get_data_num)
85 | label_non_nodule_range = range(0, len(fin['label_non_nodule']), get_data_num)
86 |
87 |
88 | pool = Pool(processes=process_num)
89 | pool_nodule = pool.map(nodule_hf, nodule_range)
90 | pool_non_nodule = pool.map(non_nodule_hf, non_nodule_range)
91 | pool_label_nodule = pool.map(label_nodule_hf, label_nodule_range)
92 | pool_label_non_nodule = pool.map(label_non_nodule_hf, label_non_nodule_range)
93 |
94 | pool.close()
95 |
96 | # print(np.shape(nodule[0]))
97 | nodule = []
98 | non_nodule = []
99 |
100 | nodule_y = []
101 | non_nodule_y = []
102 |
103 | for p in pool_nodule:
104 | nodule.extend(p)
105 | for p in pool_non_nodule:
106 | non_nodule.extend(p)
107 |
108 | for p in pool_label_nodule:
109 | nodule_y.extend(p)
110 | for p in pool_label_non_nodule:
111 | non_nodule_y.extend(p)
112 |
113 | nodule = np.asarray(nodule)
114 | non_nodule = np.asarray(non_nodule)
115 | nodule_y = np.asarray(nodule_y)
116 | non_nodule_y = np.asarray(non_nodule_y)
117 |
118 | #print(np.shape(nodule_y))
119 | #print(np.shape(non_nodule_y))
120 |
121 | all_y = np.concatenate([nodule_y, non_nodule_y], axis=0)
122 | all_patch = np.concatenate([nodule, non_nodule], axis=0)
123 |
124 | # seed = 777
125 | # np.random.seed(seed)
126 | # np.random.shuffle(nodule)
127 | # np.random.seed(seed)
128 | # np.random.shuffle(nodule_y)
129 | #
130 | # np.random.seed(seed)
131 | # np.random.shuffle(all_patch)
132 | # np.random.seed(seed)
133 | # np.random.shuffle(all_y)
134 |
135 |
136 | return nodule, all_patch, nodule_y, all_y
137 |
138 | def validation_data(sub_n) :
139 | global file
140 | dir = '/data2/jhkim/LUNA16/patch/SH/subset'
141 | file = ''.join([dir, str(sub_n), '.h5'])
142 |
143 | with h5py.File(file, 'r') as fin:
144 | nodule_range = range(0, len(fin['nodule']), get_data_num)
145 | non_nodule_range = range(0, len(fin['non_nodule']), get_data_num)
146 | label_nodule_range = range(0, len(fin['label_nodule']), get_data_num)
147 | label_non_nodule_range = range(0, len(fin['label_non_nodule']), get_data_num)
148 |
149 |
150 | pool = Pool(processes=process_num)
151 | pool_nodule = pool.map(nodule_hf, nodule_range)
152 | pool_non_nodule = pool.map(non_nodule_hf, non_nodule_range)
153 | pool_label_nodule = pool.map(label_nodule_hf, label_nodule_range)
154 | pool_label_non_nodule = pool.map(label_non_nodule_hf, label_non_nodule_range)
155 |
156 | pool.close()
157 |
158 | # print(np.shape(nodule[0]))
159 | nodule = []
160 | non_nodule = []
161 |
162 | nodule_y = []
163 | non_nodule_y = []
164 |
165 | for p in pool_nodule:
166 | nodule.extend(p)
167 | for p in pool_non_nodule:
168 | non_nodule.extend(p)
169 |
170 | for p in pool_label_nodule:
171 | nodule_y.extend(p)
172 | for p in pool_label_non_nodule:
173 | non_nodule_y.extend(p)
174 |
175 | nodule = np.asarray(nodule)
176 | non_nodule = np.asarray(non_nodule)
177 | nodule_y = np.asarray(nodule_y)
178 | non_nodule_y = np.asarray(non_nodule_y)
179 |
180 | all_y = np.concatenate([nodule_y, non_nodule_y], axis=0)
181 | all_patch = np.concatenate([nodule, non_nodule], axis=0)
182 |
183 | # seed = 777
184 | # np.random.seed(seed)
185 | # np.random.shuffle(all_patch)
186 | # np.random.seed(seed)
187 | # np.random.shuffle(all_y)
188 |
189 |
190 | return all_patch, all_y
191 |
192 | def test_data(sub_n):
193 | global file
194 | dir = '/data2/jhkim/LUNA16/patch/SH/t_subset'
195 | file = ''.join([dir, str(sub_n), '.h5'])
196 |
197 | with h5py.File(file, 'r') as fin:
198 | nodule_range = range(0, len(fin['nodule']), get_data_num)
199 | non_nodule_range = range(0, len(fin['non_nodule']), get_data_num)
200 | label_nodule_range = range(0, len(fin['label_nodule']), get_data_num)
201 | label_non_nodule_range = range(0, len(fin['label_non_nodule']), get_data_num)
202 |
203 |
204 | pool = Pool(processes=process_num)
205 | pool_nodule = pool.map(nodule_hf, nodule_range)
206 | pool_non_nodule = pool.map(non_nodule_hf, non_nodule_range)
207 | pool_label_nodule = pool.map(label_nodule_hf, label_nodule_range)
208 | pool_label_non_nodule = pool.map(label_non_nodule_hf, label_non_nodule_range)
209 |
210 | pool.close()
211 |
212 | # print(np.shape(nodule[0]))
213 | nodule = []
214 | non_nodule = []
215 |
216 | nodule_y = []
217 | non_nodule_y = []
218 |
219 | for p in pool_nodule:
220 | nodule.extend(p)
221 | for p in pool_non_nodule:
222 | non_nodule.extend(p)
223 |
224 | for p in pool_label_nodule:
225 | nodule_y.extend(p)
226 | for p in pool_label_non_nodule:
227 | non_nodule_y.extend(p)
228 |
229 | nodule = np.asarray(nodule)
230 | non_nodule = np.asarray(non_nodule)
231 | nodule_y = np.asarray(nodule_y)
232 | non_nodule_y = np.asarray(non_nodule_y)
233 |
234 | all_y = np.concatenate([nodule_y, non_nodule_y], axis=0)
235 | all_patch = np.concatenate([nodule, non_nodule], axis=0)
236 |
237 | # seed = 777
238 | # np.random.seed(seed)
239 | # np.random.shuffle(all_patch)
240 | # np.random.seed(seed)
241 | # np.random.shuffle(all_y)
242 |
243 |
244 | return all_patch, all_y
245 |
246 |
247 | def sensitivity(logits, labels):
248 | predictions = tf.argmax(logits, axis=-1)
249 | actuals = tf.argmax(labels, axis=-1)
250 |
251 |
252 | nodule_actuals = tf.ones_like(actuals)
253 | non_nodule_actuals = tf.zeros_like(actuals)
254 | nodule_predictions = tf.ones_like(predictions)
255 | non_nodule_predictions = tf.zeros_like(predictions)
256 |
257 | tp_op = tf.reduce_sum(
258 | tf.cast(
259 | tf.logical_and(
260 | tf.equal(actuals, nodule_actuals),
261 | tf.equal(predictions, nodule_predictions)
262 | ),
263 | tf.float32
264 | )
265 | )
266 |
267 | tn_op = tf.reduce_sum(
268 | tf.cast(
269 | tf.logical_and(
270 | tf.equal(actuals, non_nodule_actuals),
271 | tf.equal(predictions, non_nodule_predictions)
272 | ),
273 | tf.float32
274 | )
275 | )
276 |
277 | fp_op = tf.reduce_sum(
278 | tf.cast(
279 | tf.logical_and(
280 | tf.equal(actuals, non_nodule_actuals),
281 | tf.equal(predictions, nodule_predictions)
282 | ),
283 | tf.float32
284 | )
285 | )
286 |
287 | fn_op = tf.reduce_sum(
288 | tf.cast(
289 | tf.logical_and(
290 | tf.equal(actuals, nodule_actuals),
291 | tf.equal(predictions, non_nodule_predictions)
292 | ),
293 | tf.float32
294 | )
295 | )
296 |
297 | false_positive_rate = fp_op / (fp_op + tn_op)
298 |
299 | recall = tp_op / (tp_op + fn_op)
300 |
301 | return recall, false_positive_rate
302 |
303 | def Snapshot(t, T, M, alpha_zero) :
304 | """
305 |
306 | t = # of current iteration
307 | T = # of total iteration
308 | M = # of snapshot
309 | alpha_zero = init learning rate
310 |
311 | """
312 |
313 | x = (np.pi * (t % (T // M))) / (T // M)
314 | x = np.cos(x) + 1
315 |
316 | lr = (alpha_zero / 2) * x
317 |
318 | return lr
319 |
320 | def indices_to_one_hot(data, nb_classes):
321 | """Convert an iterable of indices to one-hot encoded labels."""
322 | targets = np.array(data).reshape(-1)
323 | return np.eye(nb_classes)[targets]
324 |
325 | def create_exclude_mask(arr_shape, position, diameter):
326 | x_dim, y_dim, z_dim = arr_shape
327 | x_pos, y_pos, z_pos = position
328 |
329 | x, y, z = np.ogrid[-x_pos:x_dim - x_pos, -y_pos:y_dim - y_pos, -z_pos:z_dim - z_pos]
330 | mask = x ** 2 + y ** 2 + z ** 2 > int(diameter // 2) ** 2
331 |
332 | return mask
333 |
334 | def fp_per_scan(logit, label) :
335 | logit = np.reshape(logit, -1)
336 | label = np.reshape(label, -1)
337 |
338 | logit = logit[logit >= 0]
339 | label = label[label >= 0]
340 |
341 | logit = np.where(logit >= 1.0, logit-1, logit)
342 | label = np.where(label >= 1.0, label-1, label)
343 |
344 | fpr, tpr, th = roc_curve(label, logit, pos_label=1.0)
345 | negative_samples = np.count_nonzero(label == 0.0)
346 | fps = fpr * negative_samples
347 |
348 |
349 | """
350 | mean_sens = np.mean(sens_list)
351 | matplotlib.use('Agg')
352 |
353 | ax = plt.gca()
354 | plt.plot(fps_itp, sens_itp)
355 | # https://matplotlib.org/devdocs/api/_as_gen/matplotlib.pyplot.grid.html
356 | plt.xlim(MIN_FROC, MAX_FROC)
357 | plt.ylim(0, 1.1)
358 | plt.xlabel('Average number of false positives per scan')
359 | plt.ylabel('Sensitivity')
360 | # plt.legend(loc='lower right')
361 | # plt.legend(loc=9)
362 | plt.title('Average sensitivity = %.4f' % (mean_sens))
363 |
364 | plt.xscale('log', basex=2)
365 | ax.xaxis.set_major_formatter(FixedFormatter(fp_list))
366 |
367 | ax.xaxis.set_ticks(fp_list)
368 | ax.yaxis.set_ticks(np.arange(0, 1.1, 0.1))
369 | plt.grid(b=True, linestyle='dotted', which='both')
370 | plt.tight_layout()
371 |
372 | # plt.show()
373 | plt.savefig('result.png', bbox_inches=0, dpi=300)
374 | """
375 |
376 | return np.asarray(fps), np.asarray(tpr)
--------------------------------------------------------------------------------