├── .gitignore ├── Calibrations └── calibration.json ├── ClassifierFunctions.py ├── ClassifierFunctions2.py ├── ClientSide.py ├── ClientSide2.py ├── Data └── profile_test │ └── 04_Cs_zxxx_profile_Trans.csv ├── DiffractionClassifier.py ├── DiffractionClassifier2.0.py ├── DiffractionClassifierCombinatorial.py ├── DiffractionClassifierCombinatorial2.0.py ├── LICENSE.md ├── Notation ├── HSGdict.txt ├── SpaceGroupsDict.py └── SpaceGroupsDict.pyc ├── PeakFinding.py ├── PeakFinding2.py ├── ProfileExtraction.py ├── README.txt ├── Sessions ├── EV_XRD.json ├── EV_sesion.json ├── session_gen_1.json ├── session_gen_2.json └── session_w_chem.json ├── UniversalLoader.py ├── UniversalLoader2.py ├── dm3_lib ├── _dm3_lib.py ├── _dm3_lib.pyc └── demo │ ├── demo.py │ └── utilities.py ├── requirements.txt ├── server.json └── user_profile.json /.gitignore: -------------------------------------------------------------------------------- 1 | __*__ -------------------------------------------------------------------------------- /Calibrations/calibration.json: -------------------------------------------------------------------------------- 1 | {"pixel_size":1, 2 | "camera_distance":1, 3 | "wavelength":1.54056} 4 | -------------------------------------------------------------------------------- /ClassifierFunctions.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | import numpy as np 4 | 5 | from matplotlib import pyplot as plt 6 | from future.builtins.misc import input 7 | 8 | 9 | def validate_calibration(prompt,name): 10 | """ 11 | ensures proper options are chosen for calibration 12 | """ 13 | 14 | while True: 15 | temp_value = input(prompt) 16 | try: 17 | float(temp_value) 18 | 19 | if temp_value > 0: 20 | return float(temp_value) 21 | else: 22 | print("invalid {}".format(name)) 23 | except: 24 | print("invalid {}".format(name)) 25 | 26 | def validate_profile_choice(dims): 27 | """ 28 | ensure proper options are chosen for selecting a single profile 29 | 30 | """ 31 | 32 | if dims[0] > 1: 33 | profile_choice = int(input("Multiple profiles detected.\nplease choose which profile to use.\n")) 34 | while profile_choice not in range(dims[0]): 35 | profile_choice = int(input("Incorrect selection.\nplease choose {}.\n".format(range(dims[0])))) 36 | else: 37 | profile_choice = 0 38 | 39 | 40 | return profile_choice 41 | 42 | 43 | def set_calibration(is_profile): 44 | """ 45 | prompt user to fill in proper calibration parameters 46 | """ 47 | 48 | 49 | print("No calibration could be extracted from the file metadata\n Please enter calibration paramters") 50 | 51 | pixel_size = validate_calibration("Please enter a pixel size (in angstroms) e.g. 14.1\n","pixel_size") 52 | 53 | wavelength = validate_calibration("Please enter beam wavelength (in microns) e.g. .02354\n","wavelength") 54 | 55 | if not is_profile: 56 | camera_dist = validate_calibration("Please enter camera distance from sample (in millimeters) e.g. 223.1\n","camera distance") 57 | else: 58 | camera_dist = 0 59 | 60 | return { "pixel_size": pixel_size, 61 | "wavelength": wavelength, 62 | "camera_distance": camera_dist} 63 | 64 | def choose_profile(image_data): 65 | """ 66 | prompts user to select a profile if multiple are detected 67 | """ 68 | print("The data is a profile.") 69 | 70 | if len(image_data.shape) == 1: 71 | 72 | plt.plot(image_data) 73 | plt.show(block=False) 74 | 75 | return image_data,np.array(range(image_data.shape[0])) 76 | 77 | elif len(image_data.shape) == 2: 78 | if image_data.shape[0] == 1: 79 | plt.plot(image_data[0,:]) 80 | plt.show(block=False) 81 | #plt.show() 82 | 83 | return image_data[0,:],np.array(range(image_data.shape[1])) 84 | 85 | 86 | elif image_data.shape[1] == 1: 87 | plt.plot(image_data[1,:]) 88 | plt.show(block=False) 89 | #plt.show() 90 | return image_data[0,:],np.array(range(image_data.shape[1])) 91 | 92 | 93 | 94 | else: 95 | "" 96 | 97 | return image_data[1,:],image_data[0,:] 98 | 99 | else: 100 | # show the user which profiles are present 101 | for i in range(image_data.shape[0]): 102 | plt.plot(image_data[i,:], label="profile {}".format(i)) 103 | plt.legend() 104 | plt.show(block=False) 105 | 106 | # have the user select the profile 107 | profile_choice = validate_profile_choice(image_data.shape) 108 | plt.close() 109 | 110 | image_data = image_data[profile_choice] 111 | 112 | # show chosen profile 113 | plt.plot(image_data) 114 | plt.show(block=False) 115 | 116 | return image_data 117 | 118 | def choose_display(): 119 | """ 120 | prompt user to choose scale bar display 121 | """ 122 | 123 | choices = ["d","theta","both"] 124 | 125 | temp_choice = "false" 126 | 127 | while temp_choice not in choices: 128 | temp_choice = input("Please choose the scale to display.\nd, theta, both\n") 129 | if temp_choice not in choices: 130 | print("incorrect choice\n") 131 | 132 | return temp_choice 133 | 134 | 135 | def choose_peaks(peak_locs,display_type): 136 | """ 137 | prompt user to select which peaks to classify on 138 | """ 139 | d = peak_locs["d_spacing"] 140 | theta = peak_locs["2theta"] 141 | vec = peak_locs["vec"] 142 | 143 | if display_type == "d" or display_type =="both": 144 | print(d) 145 | 146 | if display_type == "theta" or display_type =="both": 147 | print(theta) 148 | 149 | maximum = min(len(d),len(theta)) 150 | print(maximum,len(d),len(theta)) 151 | raw_choices = input("Choose which peaks you'd like to select separated by spaces.\n").split(" ") 152 | 153 | temp_choices = [] 154 | 155 | for choice in raw_choices: 156 | try: 157 | temp_index = int(choice) 158 | if temp_index > 0 and temp_index <= maximum and temp_index not in temp_choices: 159 | temp_choices.append(temp_index) 160 | else: 161 | print("index {} outside of available peaks".format(temp_index)) 162 | except: 163 | print("couldn't convert {} into an index".format(choice)) 164 | 165 | print(temp_choices) 166 | 167 | temp_locs = { 168 | "d_spacing":[d[i-1] for i in temp_choices], 169 | "2theta":[theta[i-1] for i in temp_choices], 170 | "vec":[vec[i-1] for i in temp_choices] 171 | } 172 | 173 | return temp_locs 174 | 175 | def provide_family(): 176 | """ 177 | prompt user and ensure proper selection of base Crystal family 178 | """ 179 | 180 | family = None 181 | 182 | while family is None: 183 | temp_choice = input("Would you like to suggest a crystal family? yes or no\n") 184 | 185 | if temp_choice =="yes": 186 | family = temp_choice 187 | elif temp_choice =="no": 188 | family = temp_choice 189 | else: 190 | print("Invalid choice. Please choose yes or no\n") 191 | 192 | return family 193 | 194 | 195 | def write_to_csv(path,data_dict): 196 | """ 197 | save new row of results to csv 198 | """ 199 | 200 | 201 | schema = ["file_name","family","genus","genus_confidence", 202 | "species_1","confidence_1","hall_1", 203 | "species_2","confidence_2","hall_2", 204 | "species_3","confidence_3","hall_3", 205 | "species_4","confidence_4","hall_4","peaks"] 206 | 207 | # if no file exists create a one and inform the user 208 | if not os.path.exists(path): 209 | print("creating new output file {}".format(path)) 210 | with open(path, "w") as csv_file: 211 | filewriter = csv.writer(csv_file, delimiter=",") 212 | filewriter.writerow(schema) 213 | 214 | row = [] 215 | 216 | row.append(data_dict["file_name"]) 217 | row.append(data_dict["family"]) 218 | 219 | row.append(data_dict["genus_1"]) 220 | row.append(data_dict["genus_confidence_1"][:5]) 221 | 222 | row.append(data_dict["species_1"]) 223 | row.append(data_dict["confidence_1"][:5]) 224 | row.append(data_dict["hall_1"]) 225 | 226 | row.append(data_dict["species_2"]) 227 | row.append(data_dict["confidence_2"][:5]) 228 | row.append(data_dict["hall_2"]) 229 | 230 | row.append(data_dict["species_3"]) 231 | row.append(data_dict["confidence_3"][:5]) 232 | row.append(data_dict["hall_3"]) 233 | 234 | row.append(data_dict["species_4"]) 235 | row.append(data_dict["confidence_4"][:5]) 236 | row.append(data_dict["hall_4"]) 237 | 238 | row.append(data_dict["peaks"]) 239 | 240 | with open(path, "a") as csv_file: 241 | filewriter = csv.writer(csv_file, delimiter=",") 242 | filewriter.writerow(row) 243 | 244 | 245 | 246 | -------------------------------------------------------------------------------- /ClassifierFunctions2.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | import numpy as np 4 | import time 5 | 6 | from matplotlib import pyplot as plt 7 | from future.builtins.misc import input 8 | 9 | from mendeleev import element 10 | 11 | 12 | def choose_peaks(peaks,peak_h): 13 | """ 14 | prompt user to select which peaks to classify on 15 | """ 16 | d = peaks 17 | maximum = len(d['d_spacing']) 18 | 19 | plt.title('select peaks. Enter to stop.') 20 | 21 | raw_choices = [] 22 | 23 | ax = plt.gca() 24 | if ax.figure.canvas.manager not in plt._pylab_helpers.Gcf.figs.values(): 25 | raise ValueError('Window closed') 26 | while True: 27 | pts = [] 28 | pts = plt.ginput(100, timeout=-1) 29 | 30 | print(pts) 31 | print(len(pts)) 32 | 33 | index = [] 34 | print(pts) 35 | for p in pts: 36 | index.append(np.argmin(np.abs(d['d_spacing']-p[0]))) 37 | # else: 38 | # index = [*range(0,maximum)] 39 | 40 | index.sort() 41 | 42 | 43 | 44 | for i in index: 45 | peak_h[i][0].set_linewidth(5) 46 | 47 | if ax.figure.canvas.manager not in plt._pylab_helpers.Gcf.figs.values(): 48 | raise ValueError('Window closed') 49 | 50 | 51 | plt.title('Enter to keep peaks, or reselect points') 52 | 53 | # time.sleep(1) # Wait a second 54 | 55 | if plt.waitforbuttonpress(): 56 | break 57 | 58 | 59 | 60 | #raw_choices = input("Choose which peaks you'd like to select separated by spaces.\n").split(" ") 61 | 62 | raw_choices = index 63 | 64 | temp_choices = [] 65 | 66 | for choice in raw_choices: 67 | try: 68 | temp_index = int(choice) 69 | if temp_index > 0 and temp_index <= maximum and temp_index not in temp_choices: 70 | temp_choices.append(temp_index) 71 | else: 72 | print("index {} outside of available peaks".format(temp_index)) 73 | except: 74 | print("couldn't convert {} into an index".format(choice)) 75 | 76 | print(temp_choices) 77 | 78 | 79 | temp_locs = { 80 | "d_spacing":[d['d_spacing'][i-1] for i in temp_choices], 81 | #"2theta":[theta[i-1] for i in temp_choices], 82 | "vec":[d['vec'][i-1] for i in temp_choices] 83 | } 84 | 85 | return temp_locs 86 | 87 | def provide_family(): 88 | """ 89 | prompt user and ensure proper selection of base Crystal family 90 | """ 91 | 92 | family = None 93 | 94 | while family is None: 95 | temp_choice = input("Would you like to suggest a crystal family? yes or no\n") 96 | 97 | if temp_choice =="yes": 98 | family = temp_choice 99 | elif temp_choice =="no": 100 | family = temp_choice 101 | else: 102 | print("Invalid choice. Please choose yes or no\n") 103 | 104 | return family 105 | 106 | 107 | def write_to_csv(path, data_dict, prediction_per_level): 108 | """ 109 | save new row of results to csv 110 | """ 111 | 112 | 113 | # schema = ["file_name","family","confidence", "genus 1st pred","confidence", "species_1", "confidence", "species_2", "confidence", "genus 2nd pred","confidence","species_3", "confidence", "species_4", "confidence", "peaks"] 114 | 115 | ppl = prediction_per_level 116 | 117 | # if no file exists create a one and inform the user 118 | if not os.path.exists(path): 119 | schema = ["file_name"] 120 | for k in range(ppl[0]): 121 | schema.append("family_"+str(k+1)) 122 | schema.append("family_confidence_"+str(k+1)) 123 | for l in range(ppl[1]): 124 | gn=k*ppl[1]+l 125 | schema.append("genus_"+str(gn+1)) 126 | schema.append("genus_confidence_"+str(gn+1)) 127 | for m in range(ppl[2]): 128 | schema.append("species_"+str(gn*ppl[2]+m+1)) 129 | schema.append("species_confidence_"+str(gn*ppl[2]+m+1)) 130 | schema.append("hall_"+str(gn*ppl[2]+m+1)) 131 | schema.append("peaks") 132 | 133 | print("creating new output file {}".format(path)) 134 | with open(path, "w") as csv_file: 135 | filewriter = csv.writer(csv_file, delimiter=",") 136 | filewriter.writerow(schema) 137 | 138 | row = [] 139 | 140 | row.append(data_dict["file_name"]) 141 | 142 | for k in range(ppl[0]): 143 | row.append(data_dict["family_"+str(k+1)]) 144 | row.append(data_dict["fam_confidence_"+str(k+1)]) 145 | for l in range(ppl[1]): 146 | gn=k*ppl[1]+l 147 | row.append(data_dict["genus_"+str(gn+1)]) 148 | row.append(data_dict["gen_confidence_"+str(gn+1)]) 149 | for m in range(ppl[2]): 150 | row.append(data_dict["species_"+str(gn*ppl[2]+m+1)]) 151 | row.append(data_dict["spec_confidence_"+str(gn*ppl[2]+m+1)]) 152 | row.append(data_dict["hall_"+str(gn*ppl[2]+m+1)]) 153 | 154 | row.append(data_dict["peaks"]) 155 | 156 | with open(path, "a") as csv_file: 157 | filewriter = csv.writer(csv_file, delimiter=",") 158 | filewriter.writerow(row) 159 | 160 | def check_for_chemistry(session): 161 | 162 | # tries to identify chemistry information from session file 163 | 164 | if "chemistry" not in session or not session["chemistry"]: 165 | return [] 166 | 167 | 168 | if "atomic_percentage" in session: 169 | # print('percentage of each element by count') 170 | chem_vec = session["atomic_percentage"] 171 | 172 | elif "chemical_formula" in session: 173 | # print('expected chemical formula') 174 | chem_vec = str2chem(session["chemical_formula"]) 175 | tot_elem = 0 176 | for cv in chem_vec: 177 | tot_elem+=cv[1] 178 | for k in range(len(chem_vec)): 179 | chem_vec[k][1] /= tot_elem 180 | 181 | elif "atomic_density" in session: 182 | # print('percentage of each element by mass') 183 | print("Warning: atomic density may not improve the accuracy, especially if atomic weights of the elements are significantly different") 184 | chem_vec = session["atomic_density"] 185 | 186 | elif "cemical_contents" in session: 187 | # print('list of elements to expect') 188 | cc = session["cemical_contents"] 189 | chem_vec = [] 190 | for elem in cc: 191 | chem_vec.append([element(elem).atomic_number, 1/len(cc)]) 192 | 193 | else: 194 | print("not enough data to run chemistry prediction. Ignoring") 195 | return [] 196 | 197 | return chem_vec 198 | 199 | def str2chem(string): 200 | 201 | elem_list = [] 202 | new_elem = False 203 | prev_elem = '' 204 | prev_num = '' 205 | for k,c in enumerate(string): 206 | 207 | if c.isdigit(): 208 | prev_num+=c 209 | elif c.islower(): 210 | prev_elem+=c 211 | 212 | if c.isupper() or k==len(string)-1: 213 | if prev_elem: 214 | try: 215 | elem = element(prev_elem).atomic_number 216 | except: 217 | raise ValueError("Something wrong with Chemical formula input") 218 | if prev_num: 219 | num = int(prev_num) 220 | else: 221 | num = 1 222 | 223 | elem_list.append([elem,num]) 224 | prev_elem = c 225 | prev_num = '' 226 | return elem_list 227 | 228 | 229 | 230 | -------------------------------------------------------------------------------- /ClientSide.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | 5 | from Notation import SpaceGroupsDict as spgs 6 | 7 | SpGr = spgs.spacegroups() 8 | 9 | import ProfileExtraction as pfex #custom library to handle the functions behind Extract_Profile 10 | import PeakFinding as pfnd #custom library to handle the functions behind Find_Peaks 11 | import UniversalLoader as uvll #custom library to handle the functions behind UniversalLoader 12 | import requests 13 | 14 | 15 | def Load_Image(path,get_metadata=False): 16 | """ 17 | Loads an image and extracts the relevant metadata for the image based in file type 18 | 19 | Inputs: 20 | 21 | path : string, contains the location of the file on the local machine 22 | 23 | Outputs: 24 | 25 | image_data : np.array, the array of values in the image 26 | calibration : dictionary, the essential metadata to convert from pixels to two_theta space 27 | 28 | """ 29 | 30 | """ 31 | NEEDS A BUNCH OF CONDITIONALS TO DETERMINE 32 | THE NATURE OF THE DATA, BEING IN SOOOOO MANY FORMS 33 | """ 34 | valid_filetypes={".tif":uvll.tif_extract, 35 | ".dm3":uvll.dm3_extract, 36 | ".csv":uvll.csv_extract, 37 | ".txt":uvll.txt_extract} 38 | 39 | file_type = path[-4:] 40 | 41 | # Check that the provided file is a supported file type 42 | if file_type in valid_filetypes.keys(): 43 | 44 | # Call the appropriate extraction function 45 | image_data = valid_filetypes[file_type](path) 46 | 47 | else: 48 | 49 | raise ValueError("Unsupported file type: please use a {}".format(valid_filetypes.keys())) 50 | 51 | 52 | 53 | 54 | return image_data 55 | 56 | 57 | def Extract_Profile(image_data): 58 | """ 59 | Azimuthally integrates a 2D diffraction pattern 60 | 61 | Inputs: 62 | 63 | image_data : np.array, the array of values in the image 64 | 65 | Outputs: 66 | 67 | 68 | profile: dictionary, contains intensity profile and pixel scale of 69 | diffraction pattern 70 | 71 | """ 72 | 73 | #Find the center of the 74 | center = pfex.find_center(image_data) 75 | 76 | #Make Radial Profile based on the image dimensions 77 | pixel_range,brightness = pfex.radial_profile(image_data,center) 78 | 79 | 80 | return {"pixel_range":pixel_range, 81 | "brightness":brightness} 82 | 83 | 84 | 85 | """ 86 | """ 87 | def Find_Peaks(profile,calibration,is_profile=False,display_type="d",scale_bar="pixel"): 88 | """ 89 | Pulls out the peaks from a radial profile 90 | 91 | 92 | Inputs: 93 | 94 | profile : dictionary, contains intensity profile and pixel scale of 95 | diffraction pattern 96 | calibration : dictionary, contains camera parameters to scale data 97 | properly in two theta space 98 | is_profile : boolean, changes processing for profiles vs 2D patterns 99 | scale_bar : string, determines which conversions need to be run 100 | to convert to two theta 101 | display_type: string, determines which plots to show 102 | 103 | 104 | Outputs: 105 | 106 | peak_locs : dictionary, contains two_theta, d_spacings, and input_vector arrays 107 | peaks locations found in the profile 108 | 109 | """ 110 | 111 | filter_size=max(int(profile["pixel_range"].shape[0]/50),3) 112 | 113 | # find the location of the peaks in pixel space 114 | peaks_pixel = pfnd.vote_peaks(profile["brightness"],filter_size=filter_size) 115 | 116 | if is_profile: 117 | 118 | if scale_bar == "pixel": 119 | peaks_theta, peaks_d = pfnd.profile2theta(profile["pixel_range"][peaks_pixel>0], 120 | calibration['pixel_size'],calibration["wavelength"]) 121 | scale_t, scale_d = pfnd.pixel2theta(profile["pixel_range"],calibration['pixel_size'], 122 | calibration["camera_distance"],calibration["wavelength"]) 123 | 124 | print(peaks_theta,peaks_d) 125 | 126 | elif scale_bar == "d": 127 | 128 | peaks_theta = pfnd.d2theta(profile["pixel_range"][peaks_pixel>0],calibration["wavelength"]) 129 | 130 | peaks_d = profile["pixel_range"][peaks_pixel>0] 131 | scale_t = pfnd.d2theta(profile["pixel_range"],calibration["wavelength"]) 132 | scale_d = profile["pixel_range"] 133 | 134 | print(peaks_d) 135 | 136 | elif scale_bar =="theta": 137 | 138 | peaks_theta = peaks_pixel 139 | 140 | else: 141 | print("Invalid scale bar selection. Choose pixel or d") 142 | 143 | else: 144 | 145 | # convert pixel locations into d and two_theta positions 146 | peaks_theta, peaks_d = pfnd.pixel2theta(profile["pixel_range"][peaks_pixel>0],calibration['pixel_size'], 147 | calibration["camera_distance"],calibration["wavelength"]) 148 | scale_t, scale_d = pfnd.pixel2theta(profile["pixel_range"],calibration['pixel_size'], 149 | calibration["camera_distance"],calibration["wavelength"]) 150 | 151 | peak_locs = {"d_spacing":[x for x in peaks_d if x<10 and x >.9], 152 | "2theta":[x for x in peaks_theta if x<90 and x >10], 153 | "vec":[int(round(2*x)) for x in peaks_theta.tolist() if x < 90 and x > 10] 154 | } 155 | 156 | # Display the data on the selected scale bar 157 | if display_type == "d": 158 | pfnd.plot_peaks(profile['brightness'],scale_d,peaks_pixel,"d") 159 | 160 | elif display_type == "theta": 161 | pfnd.plot_peaks(profile['brightness'],scale_d,peaks_pixel,"theta") 162 | 163 | elif display_type == "both": 164 | pfnd.plot_peaks(profile['brightness'],scale_d,peaks_pixel,"d") 165 | pfnd.plot_peaks(profile['brightness'],scale_t,peaks_pixel,"theta") 166 | elif display_type == "none": 167 | pass 168 | else: 169 | print("Error invalid display_type") 170 | 171 | 172 | if len(peak_locs) <= 2: 173 | print("WARNING: only {} peaks were detected, this is lower than the recommended 4+ peaks needed\nfor best results. Please check calibration.") 174 | 175 | 176 | return peak_locs 177 | 178 | 179 | def Send_For_Classification(peak_locations,user_info,URL,fam=None): 180 | """ 181 | Input: 182 | 183 | peak_locs : dictionary, contains two_theta, d_spacings, and input_vector arrays 184 | peaks locations found in the profile 185 | 186 | user_info : dictionary, contains user profile information for tracking 187 | and security purposes 188 | 189 | Outputs: 190 | 191 | payload : dictionary, contains classification statistics and predictions 192 | 193 | Calls: 194 | 195 | URL: POST, sends peak locations to the server for classification 196 | 197 | """ 198 | 199 | int_to_fam = {"0":"triclinic", 200 | "1":"monoclinic", 201 | "2":"orthorhombic", 202 | "3":"tetragonal", 203 | "4":"trigonal", 204 | "5":"hexagonal", 205 | "6":"cubic"} 206 | 207 | 208 | payload = {'peaks':peak_locations['vec'], 209 | 'family':fam, 210 | 'genus':None, 211 | 'genus_confidence':None, 212 | 'First Prediction':None, 213 | 'Second Prediction':None 214 | } 215 | 216 | 217 | # If no family is provided then it tries to predict the family 218 | if fam is None: 219 | family = requests.post(URL+"predict/family", json=payload).text 220 | payload['family'] = int_to_fam[family] 221 | print(payload['family']) 222 | print(requests.post(URL+"predict/family", json=payload).text) 223 | 224 | 225 | # Once the family is known, predicts the genus 226 | print(payload) 227 | print(requests.post(URL+"predict/genera", json=payload).text) 228 | genus = requests.post(URL+"predict/genera", json=payload).json() 229 | 230 | print(genus) 231 | 232 | payload['genus_1'] = genus["genus_1"] 233 | payload['genus_confidence_1'] = genus["genus_confidence_1"] 234 | payload['genus_2'] = genus["genus_2"] 235 | payload['genus_confidence_2'] = genus["genus_confidence_2"] 236 | 237 | # Once the genera are predicted give the top two from each 238 | print(payload) 239 | print(requests.post(URL+"predict/species", json=payload,timeout=30).text) 240 | species = requests.post(URL+"predict/species", json=payload,timeout=30).json() 241 | print(species) 242 | 243 | # Formatting the response to be saved more easily 244 | 245 | # First prediction 246 | payload["species_1"]=str(species["prediction1"][0]) 247 | payload["confidence_1"]=str(float(species["prediction1"][1])*float(genus["genus_confidence_1"])) 248 | payload["hall_1"]=SpGr.sgs_to_group[str(species["prediction1"][0])] 249 | 250 | # Second prediction 251 | payload["species_2"]=str(species["prediction2"][0]) 252 | payload["confidence_2"]=str(float(species["prediction2"][1])*float(genus["genus_confidence_1"])) 253 | payload["hall_2"]=SpGr.sgs_to_group[str(species["prediction2"][0])] 254 | 255 | # Third prediction 256 | payload["species_3"]=str(species["prediction3"][0]) 257 | payload["confidence_3"]=str(float(species["prediction3"][1])*float(genus["genus_confidence_2"])) 258 | payload["hall_3"]=SpGr.sgs_to_group[str(species["prediction3"][0])] 259 | 260 | # Fourth prediction 261 | payload["species_4"]=str(species["prediction4"][0]) 262 | payload["confidence_4"]=str(float(species["prediction4"][1])*float(genus["genus_confidence_2"])) 263 | payload["hall_4"]=SpGr.sgs_to_group[str(species["prediction4"][0])] 264 | 265 | return payload 266 | -------------------------------------------------------------------------------- /ClientSide2.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | from Notation import SpaceGroupsDict as spgs 5 | 6 | SpGr = spgs.spacegroups() 7 | 8 | notation_dictionary = spgs.spacegroups() 9 | 10 | import PeakFinding2 as pfnd #custom library to handle the functions behind Find_Peaks 11 | import UniversalLoader2 as uvll #custom library to handle the functions behind UniversalLoader 12 | import requests 13 | 14 | import numpy as np 15 | 16 | def Load_Profile(path,get_metadata=False): 17 | """ 18 | Loads an image and extracts the relevant metadata for the image based in file type 19 | 20 | Inputs: 21 | 22 | path : string, contains the location of the file on the local machine 23 | 24 | Outputs: 25 | 26 | image_data : np.array, the array of values in the image 27 | calibration : dictionary, the essential metadata to convert from pixels to two_theta space 28 | 29 | """ 30 | 31 | """ 32 | NEEDS A BUNCH OF CONDITIONALS TO DETERMINE 33 | THE NATURE OF THE DATA, BEING IN SOOOOO MANY FORMS 34 | """ 35 | valid_filetypes={".csv":uvll.csv_extract, 36 | ".txt":uvll.txt_extract} 37 | 38 | file_type = path[-4:] 39 | 40 | # Check that the provided file is a supported file type 41 | if file_type in valid_filetypes.keys(): 42 | 43 | # Call the appropriate extraction function 44 | profile,scale = valid_filetypes[file_type](path) 45 | 46 | else: 47 | 48 | raise ValueError("Unsupported file type: please use a {}".format(valid_filetypes.keys())) 49 | 50 | return profile,scale 51 | 52 | def Find_Peaks(profile, scale, **kwargs): 53 | """ 54 | Pulls out the peaks from a radial profile 55 | 56 | 57 | Inputs: 58 | 59 | profile : dictionary, contains intensity profile and pixel scale of 60 | diffraction pattern 61 | calibration : dictionary, contains camera parameters to scale data 62 | properly in two theta space 63 | is_profile : boolean, changes processing for profiles vs 2D patterns 64 | scale_bar : string, determines which conversions need to be run 65 | to convert to two theta 66 | display_type: string, determines which plots to show 67 | 68 | 69 | Outputs: 70 | 71 | peak_locs : dictionary, contains two_theta, d_spacings, and input_vector arrays 72 | peaks locations found in the profile 73 | 74 | """ 75 | 76 | 77 | max_numpeaks = kwargs.get('max_numpeaks', 75) 78 | scale_range = kwargs.get('dspace_range',[0.5, 6]) 79 | 80 | squished_scale = [True if xscale_range[0] else False for x in scale] 81 | 82 | print(squished_scale) 83 | 84 | 85 | filter_size_default=max(int(scale[squished_scale].shape[0]/50),3) 86 | print(filter_size_default) 87 | kwargs['filter_size'] = kwargs.get('filter_size',filter_size_default) 88 | print('filter size') 89 | print(kwargs['filter_size']) 90 | # find the location of the peaks in pixel space 91 | peaks = pfnd.vote_peaks(profile[squished_scale], **kwargs) 92 | 93 | peaks_d = scale[squished_scale][peaks>0] 94 | scale_d = scale 95 | thresh = 0 96 | orig_length = len(peaks_d) 97 | 98 | if len(peaks_d) > max_numpeaks: 99 | print(len(peaks_d)) 100 | print("WARNING: {} peaks were detected," + 101 | " some of the peaks will be trimmed."+ 102 | "\nFor best results. Please check calibration or run manual peak detection.".format(len(peaks_d))) 103 | srt_peaks = np.sort(peaks[peaks>0]) 104 | thresh = srt_peaks[len(peaks_d)-max_numpeaks] 105 | if len(scale[squished_scale][peaks>thresh]) ==0 and thresh>0: 106 | thresh -=1 107 | peaks_d = scale[squished_scale][peaks>thresh] 108 | print(len(peaks_d)) 109 | print(thresh) 110 | print(srt_peaks) 111 | 112 | if len(peaks_d) == orig_length: 113 | print("WARNING: reduction based on votes unsuccessful. try other parameters") 114 | elif len(peaks_d)> max_numpeaks: 115 | print("WARNING: partial reduction to {} peaks.".format(len(peaks_d))) 116 | 117 | 118 | peak_locs = {"d_spacing":scale[squished_scale][peaks>thresh], 119 | "vec":[int(round((x-.5)*164))-1 for x in peaks_d] 120 | } 121 | 122 | 123 | 124 | # Display the data 125 | peaks_h = pfnd.plot_peaks(profile[squished_scale], scale[squished_scale], peaks, thresh, **kwargs) 126 | 127 | if len(peak_locs['vec']) <= 4: 128 | print("WARNING: only {} peaks were detected," + 129 | " this is lower than the recommended 4+ peaks needed"+ 130 | "\nFor best results. Please check calibration.".format(len(peaks_d))) 131 | 132 | 133 | 134 | return peak_locs, peaks_h 135 | 136 | def find_name_in_dict(name,dict): 137 | o_ind = False 138 | for ind, nm in dict.items(): 139 | if nm == name: 140 | o_ind = ind 141 | 142 | return o_ind 143 | 144 | 145 | 146 | def Send_For_Classification(peak_locations, chem_vec, mode, crystal_family, user_info, URL, prediction_per_level, fam=None): 147 | """ 148 | Input: 149 | 150 | peak_locs : dictionary, contains two_theta, d_spacings, and input_vector arrays 151 | peaks locations found in the profile 152 | 153 | user_info : dictionary, contains user profile information for tracking 154 | and security purposes 155 | 156 | Outputs: 157 | 158 | payload : dictionary, contains classification statistics and predictions 159 | 160 | Calls: 161 | 162 | URL: POST, sends peak locations to the server for classification 163 | 164 | """ 165 | 166 | int_to_fam = {0:"triclinic", 167 | 1:"monoclinic", 168 | 2:"orthorhombic", 169 | 3:"tetragonal", 170 | 4:"trigonal", 171 | 5:"hexagonal", 172 | 6:"cubic"} 173 | 174 | 175 | payload = {'peaks':peak_locations['vec'], 176 | 'chemistry':chem_vec, 177 | 'level':"Family", 178 | 'mode': mode, 179 | 'number':0 180 | } 181 | 182 | # print(payload) 183 | 184 | payload['prediction_per_level'] = prediction_per_level 185 | 186 | skip_family = False 187 | # reproduce the gen 1 ability to specify the family to look it. Use this if the family prediction seems suspect. 188 | if crystal_family: 189 | print(" setting the family to search in is old functionality that is no longer needed for most predictions") 190 | number = find_name_in_dict(crystal_family,int_to_fam) 191 | if number: 192 | payload['family'] = crystal_family 193 | payload['family_1'] = crystal_family 194 | payload['fam_confidence_1'] = float("nan") 195 | 196 | 197 | payload['number'] = number+1 198 | skip_family = True 199 | 200 | payload = Classify_Family(payload, user_info, URL, 1, 1) 201 | 202 | for k in range(1,prediction_per_level[0]): 203 | payload['family_'+str(1+k)] = float("nan") 204 | payload['fam_confidence_'+str(1+k)] = float("nan") 205 | for l in range(0,prediction_per_level[1]): 206 | num_l = (k)*prediction_per_level[1]+l+1 207 | payload['genus_'+str(num_l)] = float("nan") 208 | payload['gen_confidence_'+str(num_l)] = float("nan") 209 | for m in range(0,prediction_per_level[2]): 210 | num_m = (num_l-1)*prediction_per_level[2]+m+1 211 | payload['species_'+str(num_m)] = float("nan") 212 | payload['spec_confidence_'+str(num_m)] = float("nan") 213 | 214 | 215 | 216 | else: 217 | print("family name not recognized, ignoring input.") 218 | 219 | if not skip_family: 220 | print(requests.post(URL+"predict", json=payload).text) 221 | family = requests.post(URL+"predict", json=payload).json() 222 | print(family['votes']) 223 | fam_votes = family['votes'] 224 | pred = [] 225 | # pred.append(np.argmax(family['votes'])) 226 | # payload['family_1'] = int_to_fam[pred[0]] 227 | fam_confidence = confidence(fam_votes) 228 | # payload['fam_confidence_1'] = fam_confidence[pred[0]] 229 | # payload['number'] = int(pred[0])+1 230 | # 231 | # print(pred[0]) 232 | # Classify_Family(peak_locations,payload,user_info,URL,1) 233 | # print(fam_confidence) 234 | # print(payload) 235 | 236 | for k in range(prediction_per_level[0]): 237 | pred.append(np.argmax(fam_votes)) 238 | payload['family'] = int_to_fam[pred[k]] 239 | payload['family_'+str(k+1)] = int_to_fam[pred[k]] 240 | payload['fam_confidence_'+str(k+1)] =fam_confidence[pred[k]] 241 | payload['number'] = int(pred[k])+1 242 | w = fam_confidence[pred[k]] 243 | payload = Classify_Family(payload, user_info, URL, w, k+1) 244 | 245 | # for next iteration 246 | fam_votes[pred[k]] = -float("inf") 247 | # print(pred[k]) 248 | # print(fam_votes) 249 | 250 | 251 | return payload 252 | 253 | 254 | def confidence(array): 255 | # softmax like normalization 256 | np_array = np.array(array) 257 | 258 | total = np.sum(np.exp(np_array)) 259 | # total = np.sum(np_array[np_array>0]) 260 | 261 | # print('softmax -') 262 | # print(np_array) 263 | # print(total) 264 | # print(np.exp(np_array)/total) 265 | 266 | #L = -np_array+np.log(total) 267 | #L = -np.log(np.exp(np_array)/total) 268 | L = np.exp(np_array)/total 269 | # L = np_array/total 270 | 271 | return L 272 | 273 | def Classify_Family(payload, user_info, URL, weight, pred_number): 274 | 275 | payload['level'] = "Genera" 276 | 277 | # Once the family is known, predicts the genus 278 | # print(requests.post(URL+"predict", json=payload,timeout=30)) 279 | print("----") 280 | print(payload) 281 | genus = requests.post(URL+"predict", json=payload,timeout=30).json() 282 | 283 | print("---genus---") 284 | # print(genus['votes']) 285 | 286 | # genera_votes = np.sum(genus['votes'],axis=0).tolist() 287 | # genera_votes_1 = int(np.argmax(genus['votes'])) 288 | genera_votes = genus['votes'] 289 | genera_con = confidence(genera_votes) 290 | pred=[] 291 | genera_pred = [] 292 | 293 | for k in range(payload['prediction_per_level'][1]): 294 | pred.append(int(np.argmax(genera_votes))) 295 | # print(pred[k]) 296 | g_pred_num = (pred_number-1)*payload['prediction_per_level'][1]+k+1 297 | genera_pred.append(pred[k]+ notation_dictionary.edges["genus"][payload['family']][0]) 298 | payload['genus_'+str(g_pred_num)] = genera_pred[k] 299 | payload['gen_confidence_'+str(g_pred_num)] = genera_con[pred[k]] * weight 300 | 301 | payload['number'] = genera_pred[k] 302 | # print('genus prediction = ',genera_pred[k]) 303 | # print('genus_number = ',g_pred_num) 304 | w = genera_con[pred[k]] * weight 305 | payload = Classify_Genus(payload,user_info,URL,w, g_pred_num) 306 | 307 | genera_votes[pred[k]] = - float("inf") 308 | # print(pred[k]) 309 | # print(genera_votes) 310 | 311 | return payload 312 | 313 | # pred_2 = int(np.argmax(genera_votes)) 314 | # genera_pred_2 = pred_2+ notation_dictionary.edges["genus"][payload['family']][0] 315 | # 316 | # 317 | # payload['genus_2'] = genera_pred_2 318 | # payload['gen_confidence_2'] = genera_con[pred_2] 319 | 320 | # Configure payload json for next request 321 | 322 | 323 | 324 | 325 | 326 | 327 | def Classify_Genus(payload, user_info, URL, weight, pred_number): 328 | 329 | # species prediction 1 330 | print("---species ---") 331 | payload['level'] = "Species" 332 | 333 | # print(requests.post(URL+"predict", json=payload,timeout=30)) 334 | species = requests.post(URL+"predict", json=payload,timeout=30).json() 335 | 336 | # print(species) 337 | 338 | 339 | # print(species['votes']) 340 | 341 | # Formatting the response to be saved more easily 342 | species_votes = species['votes'] 343 | spec_confidence = confidence(species_votes) 344 | pred = [] 345 | species_pred = [] 346 | 347 | # print(payload) 348 | 349 | for k in range(payload['prediction_per_level'][2]): 350 | pred.append(int(np.argmax(species_votes))) 351 | species_pred.append(pred[k] + notation_dictionary.edges["species"][payload['genus_'+str(pred_number)]][0]) 352 | num = (pred_number-1)*payload['prediction_per_level'][2]+k+1 353 | # print('species number = ',num) 354 | payload["species_"+str(num)] = species_pred[k] 355 | payload["spec_confidence_"+str(num)] = spec_confidence[pred[k]] * weight 356 | payload["hall_"+str(num)] = SpGr.sgs_to_group[str(species_pred[k])] 357 | 358 | species_votes[pred[k]] = -float("inf") 359 | 360 | return payload 361 | 362 | 363 | 364 | 365 | -------------------------------------------------------------------------------- /Data/profile_test/04_Cs_zxxx_profile_Trans.csv: -------------------------------------------------------------------------------- 1 | 0,0 2 | 0.0149599,6415.75 3 | 0.0299197,13407.5 4 | 0.0448796,19224.3 5 | 0.0598395,21258.5 6 | 0.0747994,17923.3 7 | 0.0897592,14420.7 8 | 0.104719,10920.1 9 | 0.119679,8281.14 10 | 0.134639,6457.18 11 | 0.149599,5161.91 12 | 0.164559,4223.83 13 | 0.179518,3559.65 14 | 0.194478,2948.32 15 | 0.209438,2638.05 16 | 0.224398,2379.53 17 | 0.239358,2217 18 | 0.254318,2065.65 19 | 0.269278,1904.7 20 | 0.284238,1771.12 21 | 0.299197,1684.84 22 | 0.314157,1628.38 23 | 0.329117,1517.84 24 | 0.344077,1457.54 25 | 0.359037,1402.26 26 | 0.373997,1369.79 27 | 0.388957,1307.68 28 | 0.403917,1289.25 29 | 0.418876,1253.17 30 | 0.433836,1224.19 31 | 0.448796,1166.49 32 | 0.463756,1110.93 33 | 0.478716,1065.12 34 | 0.493676,1086.46 35 | 0.508636,1069.06 36 | 0.523596,1042.26 37 | 0.538555,1039.5 38 | 0.553515,1008.9 39 | 0.568475,979.094 40 | 0.583435,958.802 41 | 0.598395,937.458 42 | 0.613355,941.794 43 | 0.628315,986.205 44 | 0.643275,995.719 45 | 0.658234,976.609 46 | 0.673194,961.71 47 | 0.688154,926.372 48 | 0.703114,921.007 49 | 0.718074,888.154 50 | 0.733034,904.566 51 | 0.747994,873.653 52 | 0.762954,852.316 53 | 0.777913,806.5 54 | 0.792873,819.33 55 | 0.807833,820.833 56 | 0.822793,798.747 57 | 0.837753,811.018 58 | 0.852713,818.539 59 | 0.867673,837.635 60 | 0.882633,816.463 61 | 0.897592,773.265 62 | 0.912552,797.436 63 | 0.927512,842.405 64 | 0.942472,835.421 65 | 0.957432,816.333 66 | 0.972392,798.182 67 | 0.987352,753.57 68 | 1.00231,770.179 69 | 1.01727,754.571 70 | 1.03223,757.98 71 | 1.04719,760 72 | 1.06215,752.262 73 | 1.07711,746.334 74 | 1.09207,757.037 75 | 1.10703,731.868 76 | 1.12199,759.534 77 | 1.13695,743.309 78 | 1.15191,763.491 79 | 1.16687,762.349 80 | 1.18183,725.095 81 | 1.19679,729.625 82 | 1.21175,731.324 83 | 1.22671,722.104 84 | 1.24167,744.976 85 | 1.25663,771.931 86 | 1.27159,768.792 87 | 1.28655,747.819 88 | 1.30151,719.479 89 | 1.31647,732.465 90 | 1.33143,778.18 91 | 1.34639,810 92 | 1.36135,761.86 93 | 1.37631,776.434 94 | 1.39127,747.895 95 | 1.40623,765.406 96 | 1.42119,743.719 97 | 1.43615,784.274 98 | 1.45111,735.358 99 | 1.46607,742.047 100 | 1.48103,739.818 101 | 1.49599,787.946 102 | 1.51095,795.474 103 | 1.52591,759.062 104 | 1.54087,776.002 105 | 1.55583,805.557 106 | 1.57079,804.72 107 | 1.58575,773.595 108 | 1.60071,832.869 109 | 1.61567,780.131 110 | 1.63063,806.272 111 | 1.64559,782.112 112 | 1.66055,757.052 113 | 1.67551,796.221 114 | 1.69047,813.134 115 | 1.70543,818.033 116 | 1.72039,820.367 117 | 1.73535,851.954 118 | 1.75031,856.13 119 | 1.76527,804.039 120 | 1.78022,846.113 121 | 1.79518,787.019 122 | 1.81014,855.09 123 | 1.8251,825.236 124 | 1.84006,844.652 125 | 1.85502,849.942 126 | 1.86998,892.49 127 | 1.88494,843.058 128 | 1.8999,939.119 129 | 1.91486,961.716 130 | 1.92982,877.676 131 | 1.94478,933.461 132 | 1.95974,884.503 133 | 1.9747,924 134 | 1.98966,931.986 135 | 2.00462,925.421 136 | 2.01958,954.172 137 | 2.03454,941.589 138 | 2.0495,984.496 139 | 2.06446,935.61 140 | 2.07942,940.142 141 | 2.09438,946.983 142 | 2.10934,942.42 143 | 2.1243,999.601 144 | 2.13926,910.196 145 | 2.15422,911.439 146 | 2.16918,940.209 147 | 2.18414,907.805 148 | 2.1991,930.161 149 | 2.21406,928.393 150 | 2.22902,951.474 151 | 2.24398,966.993 152 | 2.25894,992.286 153 | 2.2739,1038.96 154 | 2.28886,1038.36 155 | 2.30382,1090.02 156 | 2.31878,1118.38 157 | 2.33374,1120.5 158 | 2.3487,1110.12 159 | 2.36366,1107.9 160 | 2.37862,1152.38 161 | 2.39358,1136.48 162 | 2.40854,1117.55 163 | 2.4235,1133.83 164 | 2.43846,1153.8 165 | 2.45342,1189.16 166 | 2.46838,1215.96 167 | 2.48334,1214.23 168 | 2.4983,1267.85 169 | 2.51326,1281.06 170 | 2.52822,1261.44 171 | 2.54318,1286.02 172 | 2.55814,1242.28 173 | 2.5731,1257.29 174 | 2.58806,1340.79 175 | 2.60302,1335.09 176 | 2.61798,1298.6 177 | 2.63294,1284.42 178 | 2.6479,1292.32 179 | 2.66286,1313.49 180 | 2.67782,1291.45 181 | 2.69278,1338.34 182 | 2.70774,1316.3 183 | 2.7227,1280.79 184 | 2.73766,1303.53 185 | 2.75262,1306.31 186 | 2.76758,1331.83 187 | 2.78254,1370.46 188 | 2.7975,1424.08 189 | 2.81246,1386.25 190 | 2.82742,1356.47 191 | 2.84238,1387.35 192 | 2.85734,1438.71 193 | 2.8723,1450.97 194 | 2.88726,1437.68 195 | 2.90222,1413.84 196 | 2.91718,1476.89 197 | 2.93214,1408.33 198 | 2.9471,1435.26 199 | 2.96205,1446.3 200 | 2.97701,1454.11 201 | 2.99197,1399.02 202 | 3.00693,1433 203 | 3.02189,1465.95 204 | 3.03685,1424.97 205 | 3.05181,1491.02 206 | 3.06677,1449.87 207 | 3.08173,1532.65 208 | 3.09669,1446.21 209 | 3.11165,1506.04 210 | 3.12661,1428.81 211 | 3.14157,1476.55 212 | 3.15653,1450.22 213 | 3.17149,1403.01 214 | 3.18645,1503.26 215 | 3.20141,1494.82 216 | 3.21637,1543.53 217 | 3.23133,1479.35 218 | 3.24629,1512.63 219 | 3.26125,1516.24 220 | 3.27621,1597.86 221 | 3.29117,1567.75 222 | 3.30613,1535.31 223 | 3.32109,1536.22 224 | 3.33605,1606.7 225 | 3.35101,1615.6 226 | 3.36597,1622.75 227 | 3.38093,1590.41 228 | 3.39589,1657.75 229 | 3.41085,1709.75 230 | 3.42581,1699.29 231 | 3.44077,1668.49 232 | 3.45573,1664.06 233 | 3.47069,1639.79 234 | 3.48565,1681.59 235 | 3.50061,1630.62 236 | 3.51557,1718.97 237 | 3.53053,1694.68 238 | 3.54549,1684.41 239 | 3.56045,1749.81 240 | 3.57541,1690.3 241 | 3.59037,1744.18 242 | 3.60533,1819.9 243 | 3.62029,1825.53 244 | 3.63525,1758.69 245 | 3.65021,1696.53 246 | 3.66517,1859.45 247 | 3.68013,1826.32 248 | 3.69509,1830.28 249 | 3.71005,1856.69 250 | 3.72501,1819.03 251 | 3.73997,1828.35 252 | 3.75493,1770.84 253 | 3.76989,1892.84 254 | 3.78485,1940.16 255 | 3.79981,1988.43 256 | 3.81477,1975.59 257 | 3.82973,2037.17 258 | 3.84469,2001.14 259 | 3.85965,1927.39 260 | 3.87461,2005.51 261 | 3.88957,2087.04 262 | 3.90453,2114.43 263 | 3.91949,2110.68 264 | 3.93445,2150.46 265 | 3.94941,2114.17 266 | 3.96437,2206.62 267 | 3.97933,2147.59 268 | 3.99429,2205.8 269 | 4.00925,2330.84 270 | 4.02421,2291.24 271 | 4.03917,2317.98 272 | 4.05413,2367.14 273 | 4.06909,2397.99 274 | 4.08405,2448.1 275 | 4.09901,2441.21 276 | 4.11397,2388.59 277 | 4.12893,2507.14 278 | 4.14388,2515.31 279 | 4.15884,2544.64 280 | 4.1738,2617.8 281 | 4.18876,2664.74 282 | 4.20372,2684.87 283 | 4.21868,2665.14 284 | 4.23364,2786.67 285 | 4.2486,2804.91 286 | 4.26356,2868.19 287 | 4.27852,3003.51 288 | 4.29348,3071.81 289 | 4.30844,3108.46 290 | 4.3234,3147.71 291 | 4.33836,3267.3 292 | 4.35332,3306.77 293 | 4.36828,3256.84 294 | 4.38324,3536.33 295 | 4.3982,3591.02 296 | 4.41316,3939.74 297 | 4.42812,4030.87 298 | 4.44308,4318.45 299 | 4.45804,4697.51 300 | 4.473,5019.89 301 | 4.48796,5476.65 302 | 4.50292,6238.48 303 | 4.51788,7199.62 304 | 4.53284,8964.41 305 | 4.5478,11060.6 306 | 4.56276,13486.6 307 | 4.57772,17676.9 308 | 4.59268,22463 309 | 4.60764,29716.9 310 | 4.6226,34310.9 311 | 4.63756,38247.7 312 | 4.65252,34358.8 313 | 4.66748,28368.3 314 | 4.68244,22474.6 315 | 4.6974,16226.9 316 | 4.71236,13092.5 317 | 4.72732,10543.6 318 | 4.74228,8281.11 319 | 4.75724,6947.95 320 | 4.7722,5964.08 321 | 4.78716,5235.17 322 | 4.80212,4766.75 323 | 4.81708,4482.54 324 | 4.83204,4226.15 325 | 4.847,4172.62 326 | 4.86196,3878.52 327 | 4.87692,3674.41 328 | 4.89188,3502.97 329 | 4.90684,3474.65 330 | 4.9218,3392.95 331 | 4.93676,3262.93 332 | 4.95172,3203.69 333 | 4.96668,3112.56 334 | 4.98164,2946.13 335 | 4.9966,2966.94 336 | 5.01156,2956 337 | 5.02652,2896.91 338 | 5.04148,2890.91 339 | 5.05644,2737.58 340 | 5.0714,2697.48 341 | 5.08636,2697.07 342 | 5.10132,2656.62 343 | 5.11628,2578.69 344 | 5.13124,2597.34 345 | 5.1462,2549.43 346 | 5.16116,2478.03 347 | 5.17612,2466.16 348 | 5.19108,2405.11 349 | 5.20604,2514.62 350 | 5.221,2495.57 351 | 5.23596,2424.79 352 | 5.25092,2438.55 353 | 5.26588,2386.11 354 | 5.28084,2385.66 355 | 5.2958,2448.88 356 | 5.31076,2433.42 357 | 5.32572,2384.67 358 | 5.34067,2377.98 359 | 5.35563,2307.6 360 | 5.37059,2361.88 361 | 5.38555,2331.62 362 | 5.40051,2333.7 363 | 5.41547,2351.84 364 | 5.43043,2305.25 365 | 5.44539,2271.25 366 | 5.46035,2277.45 367 | 5.47531,2249.48 368 | 5.49027,2190.64 369 | 5.50523,2306.55 370 | 5.52019,2219.95 371 | 5.53515,2235.6 372 | 5.55011,2294.96 373 | 5.56507,2297.73 374 | 5.58003,2217.9 375 | 5.59499,2258.54 376 | 5.60995,2250 377 | 5.62491,2244.93 378 | 5.63987,2246.57 379 | 5.65483,2256.08 380 | 5.66979,2240.38 381 | 5.68475,2114.84 382 | 5.69971,2150.95 383 | 5.71467,2188.05 384 | 5.72963,2193.22 385 | 5.74459,2115.18 386 | 5.75955,2121.63 387 | 5.77451,2156.84 388 | 5.78947,2130.88 389 | 5.80443,2248.1 390 | 5.81939,2195.68 391 | 5.83435,2123.22 392 | 5.84931,2187.53 393 | 5.86427,2175.3 394 | 5.87923,2199.54 395 | 5.89419,2100.78 396 | 5.90915,2160.27 397 | 5.92411,2175 398 | 5.93907,2183.41 399 | 5.95403,2205.51 400 | 5.96899,2191.47 401 | 5.98395,2172.2 402 | 5.99891,2243.06 403 | 6.01387,2294.15 404 | 6.02883,2288.89 405 | 6.04379,2200.48 406 | 6.05875,2240.47 407 | 6.07371,2124.6 408 | 6.08867,2302.84 409 | 6.10363,2172.33 410 | 6.11859,2125.39 411 | 6.13355,2221.58 412 | 6.14851,2185.89 413 | 6.16347,2255.69 414 | 6.17843,2200.7 415 | 6.19339,2307.9 416 | 6.20835,2229.93 417 | 6.22331,2247.86 418 | 6.23827,2243.72 419 | 6.25323,2221.23 420 | 6.26819,2305.08 421 | 6.28315,2302.4 422 | 6.29811,2295.46 423 | 6.31307,2374.44 424 | 6.32803,2223.09 425 | 6.34299,2226.5 426 | 6.35795,2336.41 427 | 6.37291,2281.62 428 | 6.38787,2338.14 429 | 6.40283,2452.65 430 | 6.41779,2337.49 431 | 6.43275,2317.96 432 | 6.44771,2339.18 433 | 6.46267,2250.43 434 | 6.47763,2291.09 435 | 6.49259,2281.02 436 | 6.50755,2402.92 437 | 6.5225,2432.63 438 | 6.53746,2487.99 439 | 6.55242,2423.15 440 | 6.56738,2392.21 441 | 6.58234,2415.78 442 | 6.5973,2554.39 443 | 6.61226,2561.36 444 | 6.62722,2624.39 445 | 6.64218,2675.47 446 | 6.65714,2599.87 447 | 6.6721,2644.41 448 | 6.68706,2706.58 449 | 6.70202,2725.32 450 | 6.71698,2750.96 451 | 6.73194,2962.33 452 | 6.7469,3052.44 453 | 6.76186,3143.27 454 | 6.77682,3112.24 455 | 6.79178,3388.41 456 | 6.80674,3565.39 457 | 6.8217,3730.32 458 | 6.83666,4013.99 459 | 6.85162,4577.29 460 | 6.86658,5253.22 461 | 6.88154,6409.16 462 | 6.8965,7630.96 463 | 6.91146,9895.73 464 | 6.92642,13266.1 465 | 6.94138,14495.6 466 | 6.95634,15787 467 | 6.9713,14659.6 468 | 6.98626,12968.1 469 | 7.00122,10078.1 470 | 7.01618,8259.94 471 | 7.03114,6411.47 472 | 7.0461,5341.42 473 | 7.06106,4705.56 474 | 7.07602,4180.51 475 | 7.09098,3894.44 476 | 7.10594,3653.96 477 | 7.1209,3492.95 478 | 7.13586,3403.34 479 | 7.15082,3105.44 480 | 7.16578,3200.74 481 | 7.18074,3084.73 482 | 7.1957,3038.86 483 | 7.21066,3000.46 484 | 7.22562,2908.44 485 | 7.24058,2890.16 486 | 7.25554,2801.83 487 | 7.2705,2737.56 488 | 7.28546,2781.17 489 | 7.30042,2801.1 490 | 7.31538,2772.32 491 | 7.33034,2833.13 492 | 7.3453,2771.64 493 | 7.36026,2694.64 494 | 7.37522,2687.87 495 | 7.39018,2594.49 496 | 7.40514,2635.21 497 | 7.4201,2566.47 498 | 7.43506,2552.87 499 | 7.45002,2571.5 500 | 7.46498,2579.58 501 | 7.47994,2631.82 502 | 7.4949,2609.24 503 | 7.50986,2677 504 | 7.52482,2597.52 505 | 7.53978,2659.18 506 | 7.55474,2585.88 507 | 7.5697,2606.16 508 | 7.58466,2649.06 509 | 7.59962,2535.57 510 | 7.61458,2533.43 511 | 7.62954,2502.57 512 | 7.6445,2490.51 513 | 7.65946,2473.38 514 | 7.67442,2546.04 515 | 7.68938,2485.29 516 | 7.70433,2399.55 517 | 7.71929,2437.67 518 | 7.73425,2580.77 519 | 7.74921,2549.34 520 | 7.76417,2683.23 521 | 7.77913,2563.88 522 | 7.79409,2538.83 523 | 7.80905,2555.27 524 | 7.82401,2617.03 525 | 7.83897,2519.21 526 | 7.85393,2679.45 527 | 7.86889,2534.18 528 | 7.88385,2600.92 529 | 7.89881,2543.98 530 | 7.91377,2777.63 531 | 7.92873,2806.16 532 | 7.94369,2713.59 533 | 7.95865,2792.52 534 | 7.97361,2872.31 535 | 7.98857,2820.82 536 | 8.00353,2847.44 537 | 8.01849,3042.14 538 | 8.03345,3018.02 539 | 8.04841,3037.31 540 | 8.06337,3195.35 541 | 8.07833,3336.98 542 | 8.09329,3617.97 543 | 8.10825,3751.47 544 | 8.12321,4169.19 545 | 8.13817,4781.61 546 | 8.15313,5658.52 547 | 8.16809,6554 548 | 8.18305,7983.01 549 | 8.19801,9752.18 550 | 8.21297,12893.9 551 | 8.22793,13198.2 552 | 8.24289,13783.8 553 | 8.25785,12828.2 554 | 8.27281,11590.3 555 | 8.28777,9436.74 556 | 8.30273,7084.78 557 | 8.31769,6416.69 558 | 8.33265,5745.43 559 | 8.34761,5347.99 560 | 8.36257,4912.1 561 | 8.37753,4904.68 562 | 8.39249,5054.82 563 | 8.40745,5092.4 564 | 8.42241,5691.33 565 | 8.43737,6145.14 566 | 8.45233,6938.27 567 | 8.46729,7156.61 568 | 8.48225,7247.13 569 | 8.49721,7500.12 570 | 8.51217,6583.18 571 | 8.52713,6034.49 572 | 8.54209,5232.75 573 | 8.55705,4628.97 574 | 8.57201,4365.29 575 | 8.58697,3961.47 576 | 8.60193,3725.87 577 | 8.61689,3424.78 578 | 8.63185,3260.99 579 | 8.64681,3208.55 580 | 8.66177,3036.29 581 | 8.67673,2885.58 582 | 8.69169,2858.85 583 | 8.70665,2812.71 584 | 8.72161,2819.39 585 | 8.73657,2746.08 586 | 8.75153,2664 587 | 8.76649,2632.11 588 | 8.78145,2695 589 | 8.79641,2633.51 590 | 8.81137,2589.86 591 | 8.82633,2532.14 592 | 8.84129,2521.49 593 | 8.85625,2494.95 594 | 8.87121,2569.56 595 | 8.88616,2409.67 596 | 8.90112,2577.36 597 | 8.91608,2489.13 598 | 8.93104,2539.03 599 | 8.946,2513.79 600 | 8.96096,2521.82 601 | 8.97592,2476.62 602 | 8.99088,2384.17 603 | 9.00584,2400.9 604 | 9.0208,2409.42 605 | 9.03576,2569.03 606 | 9.05072,2502.33 607 | 9.06568,2507.16 608 | 9.08064,2587.5 609 | 9.0956,2442.19 610 | 9.11056,2488.08 611 | 9.12552,2535.66 612 | 9.14048,2688.2 613 | 9.15544,2702.86 614 | 9.1704,2745 615 | 9.18536,2711.79 616 | 9.20032,2798.06 617 | 9.21528,3007.18 618 | 9.23024,2968.4 619 | 9.2452,3074.8 620 | 9.26016,3147.36 621 | 9.27512,3210.87 622 | 9.29008,3233.94 623 | 9.30504,3094.94 624 | 9.32,2941.3 625 | 9.33496,2786.55 626 | 9.34992,2680.71 627 | 9.36488,2698.7 628 | 9.37984,2615.77 629 | 9.3948,2487.79 630 | 9.40976,2316.1 631 | 9.42472,2373.97 632 | 9.43968,2302.85 633 | 9.45464,2254.16 634 | 9.4696,2246.03 635 | 9.48456,2330.96 636 | 9.49952,2166.5 637 | 9.51448,2257.35 638 | 9.52944,2273.03 639 | 9.5444,2284.66 640 | 9.55936,2152.92 641 | 9.57432,2069.1 642 | 9.58928,2002.01 643 | 9.60424,1930.06 644 | 9.6192,2018.15 645 | 9.63416,2060.43 646 | 9.64912,2075.96 647 | 9.66408,1979.34 648 | 9.67904,1989.53 649 | 9.694,1982.05 650 | 9.70896,1970.35 651 | 9.72392,1924.1 652 | 9.73888,1931.56 653 | 9.75384,1939.1 654 | 9.7688,1887.68 655 | 9.78376,1835.29 656 | 9.79872,1922.13 657 | 9.81368,1888.55 658 | 9.82864,1890.83 659 | 9.8436,1772.13 660 | 9.85856,1896.04 661 | 9.87352,1899.22 662 | 9.88848,1895.77 663 | 9.90344,1927.24 664 | 9.9184,1816.42 665 | 9.93336,1825.26 666 | 9.94832,1838.33 667 | 9.96328,1854.69 668 | 9.97824,1976.01 669 | 9.9932,1830.78 670 | 10.0082,1902.03 671 | 10.0231,1759.3 672 | 10.0381,1746.8 673 | 10.053,1876.74 674 | 10.068,1873.65 675 | 10.083,1946.94 676 | 10.0979,1843.78 677 | 10.1129,1921.83 678 | 10.1278,1794.1 679 | 10.1428,1900.42 680 | 10.1578,1957.57 681 | 10.1727,1914.55 682 | 10.1877,1967.63 683 | 10.2026,1808.29 684 | 10.2176,1844.53 685 | 10.2326,1819.81 686 | 10.2475,1931.83 687 | 10.2625,1918.31 688 | 10.2774,1842.4 689 | 10.2924,1727.18 690 | 10.3074,1881.19 691 | 10.3223,1799.65 692 | 10.3373,1772.29 693 | 10.3522,1740.36 694 | 10.3672,1853.31 695 | 10.3822,1787.1 696 | 10.3971,1808.08 697 | 10.4121,1821.81 698 | 10.427,1832.7 699 | 10.442,1855.15 700 | 10.457,1777.74 701 | 10.4719,1724.53 702 | 10.4869,1878.87 703 | 10.5018,1714.31 704 | 10.5168,1766.83 705 | 10.5318,1753.9 706 | 10.5467,1733.77 707 | 10.5617,1859.34 708 | 10.5766,1980.76 709 | 10.5916,1790.14 710 | 10.6066,1924.11 711 | 10.6215,1836.64 712 | 10.6365,1954.99 713 | 10.6514,1960.91 714 | 10.6664,1953.71 715 | 10.6813,1904.64 716 | 10.6963,1920.04 717 | 10.7113,1973.83 718 | 10.7262,1935.56 719 | 10.7412,1819.03 720 | 10.7561,2026.38 721 | 10.7711,1902.02 722 | 10.7861,1958.24 723 | 10.801,2002.15 724 | 10.816,1956.03 725 | 10.8309,1920.98 726 | 10.8459,1944.5 727 | 10.8609,1959.92 728 | 10.8758,1881.22 729 | 10.8908,1947.33 730 | 10.9057,1858.3 731 | 10.9207,1934.74 732 | 10.9357,1889.34 733 | 10.9506,1961.68 734 | 10.9656,2032.11 735 | 10.9805,1938.74 736 | 10.9955,2063.31 737 | 11.0105,2071.65 738 | 11.0254,2056.88 739 | 11.0404,1882.19 740 | 11.0553,2067 741 | 11.0703,2077.05 742 | 11.0853,2120.52 743 | 11.1002,2077.6 744 | 11.1152,1887.68 745 | 11.1301,1947.8 746 | 11.1451,1955.6 747 | 11.1601,2009.51 748 | 11.175,2085.92 749 | 11.19,2109.82 750 | 11.2049,2182.29 751 | 11.2199,2106.82 752 | 11.2349,2087.12 753 | 11.2498,2048.88 754 | 11.2648,2272.73 755 | 11.2797,2197.42 756 | 11.2947,2264.65 757 | 11.3097,2354.54 758 | 11.3246,2506.18 759 | 11.3396,2525.1 760 | 11.3545,2499.44 761 | 11.3695,2346.84 762 | 11.3845,2437.27 763 | 11.3994,2467.45 764 | 11.4144,2463.13 765 | 11.4293,2466.34 766 | 11.4443,2461.3 767 | 11.4593,2346.13 768 | 11.4742,2341.93 769 | 11.4892,2229.62 770 | 11.5041,2275.85 771 | 11.5191,2113 772 | 11.5341,2080.26 773 | 11.549,2062.77 774 | 11.564,2070.11 775 | 11.5789,1842.65 776 | 11.5939,1939.56 777 | 11.6089,2115.91 778 | 11.6238,2105.35 779 | 11.6388,1995.97 780 | 11.6537,1962.58 781 | 11.6687,1968.87 782 | 11.6837,2020.91 783 | 11.6986,2098.15 784 | 11.7136,2027.77 785 | 11.7285,2090.21 786 | 11.7435,2042.56 787 | 11.7585,2107.64 788 | 11.7734,2017.83 789 | 11.7884,2132.72 790 | 11.8033,1967.31 791 | 11.8183,2056.96 792 | 11.8333,1999.32 793 | 11.8482,2081.49 794 | 11.8632,1967.61 795 | 11.8781,2013.93 796 | 11.8931,1944.47 797 | 11.9081,1781.56 798 | 11.923,1768.58 799 | 11.938,1849.05 800 | 11.9529,1822.7 801 | 11.9679,1893.32 802 | 11.9829,1876.13 803 | 11.9978,1863.39 804 | 12.0128,1791.09 805 | 12.0277,1805.14 806 | 12.0427,1748.34 807 | 12.0577,1873.99 808 | 12.0726,1823.18 809 | 12.0876,1698.59 810 | 12.1025,1706.94 811 | 12.1175,1775.8 812 | 12.1325,1720.27 813 | 12.1474,1733.39 814 | 12.1624,1625.32 815 | 12.1773,1646.79 816 | 12.1923,1590.11 817 | 12.2073,1774 818 | 12.2222,1596.41 819 | 12.2372,1744.47 820 | 12.2521,1653.81 821 | 12.2671,1709.65 822 | 12.2821,1790.82 823 | 12.297,1762.58 824 | 12.312,1726.56 825 | 12.3269,1724.06 826 | 12.3419,1586.89 827 | 12.3569,1697.69 828 | 12.3718,1558.86 829 | 12.3868,1542.54 830 | 12.4017,1621.91 831 | 12.4167,1631.79 832 | 12.4317,1667.74 833 | 12.4466,1661.56 834 | 12.4616,1586.13 835 | 12.4765,1680.42 836 | 12.4915,1630.7 837 | 12.5065,1689.03 838 | 12.5214,1649.04 839 | 12.5364,1593.04 840 | 12.5513,1707.85 841 | 12.5663,1690.25 842 | 12.5813,1592.04 843 | 12.5962,1728.82 844 | 12.6112,1590.91 845 | 12.6261,1719 846 | 12.6411,1709.72 847 | 12.6561,1618.17 848 | 12.671,1687.33 849 | 12.686,1539.56 850 | 12.7009,1670.56 851 | 12.7159,1677.39 852 | 12.7309,1710.54 853 | 12.7458,1564.26 854 | 12.7608,1584.81 855 | 12.7757,1569.25 856 | 12.7907,1589.62 857 | 12.8057,1641.44 858 | 12.8206,1720.35 859 | 12.8356,1563.37 860 | 12.8505,1702.7 861 | 12.8655,1706.87 862 | 12.8805,1726.69 863 | 12.8954,1746.78 864 | 12.9104,1742.47 865 | 12.9253,1786.63 866 | 12.9403,1898.36 867 | 12.9553,1880.02 868 | 12.9702,1732.6 869 | 12.9852,1713.52 870 | 13.0001,1633.75 871 | 13.0151,1579.52 872 | 13.03,1589 873 | 13.045,1569.32 874 | 13.06,1616.4 875 | 13.0749,1541.74 876 | 13.0899,1510.66 877 | 13.1048,1393.68 878 | 13.1198,1585.81 879 | 13.1348,1491.95 880 | 13.1497,1512.96 881 | 13.1647,1501.29 882 | 13.1796,1491.04 883 | 13.1946,1533.67 884 | 13.2096,1580.3 885 | 13.2245,1518.04 886 | 13.2395,1539.02 887 | 13.2544,1527.07 888 | 13.2694,1558.59 889 | 13.2844,1461.92 890 | 13.2993,1518.49 891 | 13.3143,1519.96 892 | 13.3292,1562.23 893 | 13.3442,1626.55 894 | 13.3592,1477.39 895 | 13.3741,1454.42 896 | 13.3891,1482.06 897 | 13.404,1456.5 898 | 13.419,1452.62 899 | 13.434,1633.19 900 | 13.4489,1458.24 901 | 13.4639,1448.09 902 | 13.4788,1595.03 903 | 13.4938,1551.31 904 | 13.5088,1582.42 905 | 13.5237,1640.93 906 | 13.5387,1592.22 907 | 13.5536,1496.48 908 | 13.5686,1497.18 909 | 13.5836,1643.41 910 | 13.5985,1561.77 911 | 13.6135,1552.37 912 | 13.6284,1530.43 913 | 13.6434,1563.9 914 | 13.6584,1462.98 915 | 13.6733,1561.44 916 | 13.6883,1535.23 917 | 13.7032,1531.61 918 | 13.7182,1518.45 919 | 13.7332,1459.12 920 | 13.7481,1511.58 921 | 13.7631,1513.77 922 | 13.778,1520.53 923 | 13.793,1473.24 924 | 13.808,1495.42 925 | 13.8229,1672.97 926 | 13.8379,1583.78 927 | 13.8528,1612.07 928 | 13.8678,1571.84 929 | 13.8828,1526.96 930 | 13.8977,1491.95 931 | 13.9127,1518.12 932 | 13.9276,1367.09 933 | 13.9426,1510.79 934 | 13.9576,1615.14 935 | 13.9725,1574.68 936 | 13.9875,1479.55 937 | 14.0024,1661.42 938 | 14.0174,1454.81 939 | 14.0324,1377.18 940 | 14.0473,1442.35 941 | 14.0623,1419.86 942 | 14.0772,1444.98 943 | 14.0922,1450.13 944 | 14.1072,1583.53 945 | 14.1221,1456.18 946 | 14.1371,1489.51 947 | 14.152,1556.78 948 | 14.167,1399.89 949 | 14.182,1500.97 950 | 14.1969,1509.91 951 | 14.2119,1540.7 -------------------------------------------------------------------------------- /DiffractionClassifier.py: -------------------------------------------------------------------------------- 1 | import ClientSide #custom package 2 | 3 | import numpy as np 4 | import argparse 5 | import json 6 | import os 7 | import ClassifierFunctions as cf 8 | 9 | from matplotlib import pyplot as plt 10 | from builtins import input 11 | 12 | 13 | 14 | # Initialize essential global variables 15 | USER_INFO = "user_profile.json" 16 | #URL = ""#you'll need me to send you the link 17 | SERVER_INFO = "server_gen1.json" 18 | FAMILIES = ["triclinic","monoclinic","orthorhombic","tetragonal", 19 | "trigonal","hexagonal","cubic"] 20 | 21 | DEFAULT_SESSION = os.path.join ("Sessions","session.json") 22 | 23 | 24 | def build_parser(): 25 | parser = argparse.ArgumentParser() 26 | 27 | # This will be implemented as rollout broadens 28 | parser.add_argument('--apikey', type=str, 29 | dest='key', help='api key to securely access service', 30 | metavar='KEY', required=False) 31 | 32 | parser.add_argument('--session', 33 | dest='session',help='Keep user preferences for multirun sessions', 34 | metavar='SESSION',required=False, default=None) 35 | return parser 36 | 37 | 38 | 39 | def main(): 40 | 41 | parser = build_parser() 42 | options = parser.parse_args() 43 | 44 | print(options.session) 45 | 46 | # opens the user specified session 47 | if options.session: 48 | with open(os.path.join("Sessions",options.session),'r') as f: 49 | session = json.load(f) 50 | 51 | # opens the default session 52 | else: 53 | with open(DEFAULT_SESSION,'r') as f: 54 | 55 | session = json.load(f) 56 | 57 | # set variables from loaded session data 58 | fam = session["crystal_family"] 59 | provide_family = session["known_family"] 60 | display_type = session["display_type"] 61 | auto_calibrate = session["auto_calibrate"] 62 | file_path = session["file_path"] 63 | output_file = session["output_file"] 64 | is_profile = session["is_profile"] 65 | manual_peak_selection = session["manual_peak_selection"] 66 | scale_bar = session["scale_bar"] 67 | 68 | # Load calibration from specified file (json) 69 | try: 70 | print("Loading calibration from {}".format(os.path.join("Calibrations",auto_calibrate))) 71 | with open(os.path.join("Calibrations",auto_calibrate),'r') as f: 72 | calibration = json.load(f) 73 | 74 | except: 75 | print("No calibration could be loaded from {}\nPlease check the file exists and is formatted properly".format(auto_calibrate)) 76 | calibration = cf.set_calibration(is_profile) 77 | print(calibration) 78 | 79 | with open(SERVER_INFO,'r') as f: 80 | server_info = json.load(f) 81 | 82 | if server_info['URL']: 83 | URL = server_info['URL'] 84 | else: 85 | raise ValueError('you need to have the server URL provided to you') 86 | 87 | 88 | 89 | # Load user from provided path, [IN PROGRESS] 90 | with open(USER_INFO) as f: 91 | user_info = json.load(f) 92 | 93 | # Determine if the path is a directory or a file 94 | if os.path.isdir(file_path): 95 | print("loading files from directory") 96 | file_paths = [] 97 | for dirpath,dirnames,fpath in os.walk(file_path): 98 | for path in fpath: 99 | file_paths.append(os.path.join(dirpath,path)) 100 | print("found {} files to load.".format(len(file_paths))) 101 | 102 | else: 103 | file_paths = [file_path] 104 | 105 | print(file_paths) 106 | for f_path in file_paths: 107 | 108 | # Load Data from specified file (DM3, TIFF, CSV etc....) 109 | try: 110 | print("loading data from {}".format(f_path)) 111 | image_data = ClientSide.Load_Image(f_path) 112 | print("I successfully loaded the data") 113 | except: 114 | print("Invalid file path given ({}).\n Please enter filepath to your data".format(f_path)) 115 | options.fpath = input() 116 | 117 | # Change the processing based on data type 118 | if is_profile: 119 | 120 | # Choose which profile if there are multiple 121 | image_data,scale = cf.choose_profile(image_data) 122 | 123 | else: 124 | plt.imshow(np.log(image_data)) 125 | plt.show(block=False) 126 | #plt.show() 127 | 128 | # Change the Processing based on the type of data 129 | if is_profile: 130 | print("identifying from profile") 131 | radial_profile = {"brightness":image_data, 132 | "pixel_range":scale} 133 | 134 | else: 135 | radial_profile = ClientSide.Extract_Profile(image_data) 136 | 137 | 138 | peak_locs = ClientSide.Find_Peaks(radial_profile,calibration,is_profile,display_type,scale_bar) 139 | 140 | # Choose which peaks to classify on 141 | if manual_peak_selection: 142 | peak_locs = cf.choose_peaks(peak_locs,display_type) 143 | 144 | 145 | if provide_family =="yes": 146 | 147 | while fam is None: 148 | temp_fam = input("What family does the Crystal belong to?\n") 149 | if temp_fam in FAMILIES: 150 | fam = temp_fam 151 | else: 152 | print("Invalid choice. choose from {}\n".format(str(FAMILIES)[1:-1])) 153 | elif provide_family =="no": 154 | fam = None 155 | 156 | classificated = ClientSide.Send_For_Classification(peak_locs,user_info,URL,fam) 157 | 158 | classificated["file_name"] = f_path 159 | 160 | # update the user on the results before saving 161 | print(classificated) 162 | 163 | # write results out to the specified file 164 | if not os.path.exists("Results"): 165 | os.makedirs("Results") 166 | cf.write_to_csv(os.path.join("Results", output_file), classificated) 167 | 168 | if __name__ == "__main__": 169 | main() 170 | 171 | 172 | -------------------------------------------------------------------------------- /DiffractionClassifier2.0.py: -------------------------------------------------------------------------------- 1 | import ClientSide2 #custom package 2 | import logging 3 | 4 | import numpy as np 5 | import argparse 6 | import json 7 | import os 8 | import ClassifierFunctions2 as cf 9 | 10 | from matplotlib import pyplot as plt 11 | from builtins import input 12 | 13 | 14 | 15 | # Initialize essential global variables 16 | #URL = "" #you'll need me to send you the link 17 | FAMILIES = ["triclinic","monoclinic","orthorhombic","tetragonal", 18 | "trigonal","hexagonal","cubic"] 19 | 20 | DEFAULT_SESSION = os.path.join ("Sessions","session.json") 21 | DEFAULT_USER = "user_profile.json" 22 | SERVER_INFO = "server_gen2.json" 23 | 24 | # list of three, one per level 25 | prediction_per_level = [2, 3, 3] 26 | 27 | DEFAULT_FILTER_SETTINGS = { "max_numpeaks": 20, 28 | "dspace_range" : [0.1,6], 29 | "peak_threshold": 1, 30 | "filter_size" : 15, 31 | "passes" : 2 32 | } 33 | 34 | 35 | def build_parser(): 36 | parser = argparse.ArgumentParser() 37 | 38 | # This will be implemented as rollout broadens 39 | parser.add_argument('--apikey', type=str, 40 | dest='key', help='api key to securely access service', 41 | metavar='KEY', required=False) 42 | 43 | parser.add_argument('--session', 44 | dest='session',help='Keep user preferences for multirun sessions', 45 | metavar='SESSION',required=False, default=None) 46 | return parser 47 | 48 | 49 | 50 | def main(): 51 | 52 | parser = build_parser() 53 | options = parser.parse_args() 54 | 55 | #print(options.session) 56 | 57 | # opens the user specified session 58 | if options.session: 59 | with open(os.path.join("Sessions",options.session),'r') as f: 60 | session = json.load(f) 61 | 62 | # opens the default session 63 | else: 64 | with open(DEFAULT_SESSION,'r') as f: 65 | session = json.load(f) 66 | 67 | # set variables from loaded session data 68 | # print(session) 69 | file_path = session["file_path"] 70 | output_file = session["output_file"] 71 | manual_peak_selection = session["manual_peak_selection"] 72 | known_family = session["known_family"] 73 | chemistry = session["chemistry"] 74 | diffraction = session["diffraction"] 75 | 76 | mode = "" 77 | 78 | if diffraction: 79 | if chemistry: 80 | mode="DiffChem" 81 | else: 82 | mode="DiffOnly" 83 | else: 84 | if chemistry: 85 | raise ValueError('Running chemistry only predictions is currently not implemented') 86 | else: 87 | raise ValueError('Invalid prediction type. Either diffraction or chemistry must be enabled') 88 | 89 | if known_family and known_family=='yes': 90 | print('known family') 91 | crystal_family = session["crystal_family"] 92 | prediction_per_level[0] = 1 93 | else: 94 | crystal_family = None 95 | 96 | 97 | if session["filter_settings"]: 98 | FILTER_SETTINGS = session["filter_settings"] 99 | else: 100 | FILTER_SETTINGS = {} 101 | 102 | print(FILTER_SETTINGS) 103 | 104 | # Load user from provided path, [IN PROGRESS] 105 | if session["user_info"]: 106 | with open(session["user_info"],'r') as f: 107 | user_info = json.load(f) 108 | else: 109 | with open(DEFAULT_USER,'r') as f: 110 | user_info = json.load(f) 111 | 112 | with open(session["server_info"],'r') as f: 113 | server_info = json.load(f) 114 | 115 | if server_info['URL']: 116 | url = server_info['URL'] 117 | else: 118 | raise ValueError('you need to have the server URL provided to you') 119 | 120 | chem_vec = cf.check_for_chemistry(session) 121 | 122 | 123 | 124 | # Determine if the path is a directory or a file 125 | if os.path.isdir(file_path): 126 | print("loading files from directory") 127 | file_paths = [] 128 | for dirpath,dirnames,fpath in os.walk(file_path): 129 | for path in fpath: 130 | if not path[0] == '.': 131 | file_paths.append(os.path.join(dirpath,path)) 132 | print("found {} files to load.".format(len(file_paths))) 133 | 134 | else: 135 | file_paths = [file_path] 136 | 137 | 138 | for f_path in file_paths: 139 | 140 | # Load Data from specified file (DM3, TIFF, CSV etc....) 141 | 142 | print("loading data from {}".format(f_path)) 143 | image_data,scale = ClientSide2.Load_Profile(f_path) 144 | print("I successfully loaded the data") 145 | 146 | # print(scale) 147 | print("length",len(image_data)) 148 | print("max",np.max(image_data)) 149 | if diffraction: 150 | 151 | peak_locs,peaks_h = ClientSide2.Find_Peaks(image_data,scale, **FILTER_SETTINGS) 152 | # Choose which peaks to classify on 153 | if manual_peak_selection: 154 | peak_locs = cf.choose_peaks(peak_locs,peaks_h) 155 | #raise NotImplementedError 156 | else: 157 | peak_locs = [] 158 | peaks_h = [] 159 | # 160 | # print(peak_locs) 161 | # print(chem_vec) 162 | 163 | 164 | classificated = ClientSide2.Send_For_Classification(peak_locs, chem_vec, mode, crystal_family, user_info, url, prediction_per_level) 165 | 166 | classificated["file_name"] = f_path 167 | 168 | # update the user on the results before saving 169 | print(classificated) 170 | 171 | # write results out to the specified file 172 | if not os.path.exists("Results"): 173 | os.makedirs("Results") 174 | 175 | cf.write_to_csv(os.path.join("Results",output_file), classificated, prediction_per_level) 176 | 177 | if __name__ == "__main__": 178 | main() 179 | 180 | 181 | -------------------------------------------------------------------------------- /DiffractionClassifierCombinatorial.py: -------------------------------------------------------------------------------- 1 | import ClientSide as ClientSide #custom package 2 | 3 | import numpy as np 4 | import argparse 5 | import json 6 | import os 7 | import ClassifierFunctions as cf 8 | import csv 9 | from matplotlib import pyplot as plt 10 | from builtins import input 11 | 12 | from Notation import SpaceGroupsDict as spgs 13 | SpGr = spgs.spacegroups() 14 | 15 | from itertools import combinations,chain 16 | 17 | 18 | 19 | # Initialize essential global variables 20 | USER_INFO = "user_profile.json" 21 | #URL = "" # you'll need me to send you the link 22 | SERVER_INFO = "server_gen1.json" 23 | FAMILIES = ["triclinic","monoclinic","orthorhombic","tetragonal", 24 | "trigonal","hexagonal","cubic"] 25 | 26 | DEFAULT_SESSION = os.path.join ("Sessions","session.json") 27 | 28 | 29 | def build_parser(): 30 | parser = argparse.ArgumentParser() 31 | 32 | # This will be implemented as rollout broadens 33 | parser.add_argument('--apikey', type=str, 34 | dest='key', help='api key to securely access service', 35 | metavar='KEY', required=False) 36 | 37 | parser.add_argument('--session', 38 | dest='session',help='Keep user preferences for multirun sessions', 39 | metavar='SESSION',required=False, default=None) 40 | return parser 41 | 42 | 43 | def powerset(iterable): 44 | "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" 45 | s = list(iterable) 46 | return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) 47 | 48 | def write_to_csv(path,data_dict): 49 | schema = ["file_name","family","genus","genus_confidence", 50 | "species_1","confidence_1","hall_1", 51 | "species_2","confidence_2","hall_2", 52 | "species_3","confidence_3","hall_3", 53 | "species_4","confidence_4","hall_4","peaks"] # if no file exists create a one and warn the user 54 | if not os.path.exists(path): 55 | print("creating new output file {}".format(path)) 56 | with open(path, "w") as csv_file: 57 | filewriter = csv.writer(csv_file, delimiter=",") 58 | filewriter.writerow(schema) 59 | 60 | row = [] 61 | row.append(data_dict["file_name"]) 62 | row.append(data_dict["family"]) 63 | 64 | row.append(data_dict["genus_1"]) 65 | row.append(data_dict["genus_confidence_1"][:5]) 66 | 67 | row.append(data_dict["species_1"]) 68 | row.append(data_dict["confidence_1"][:5]) 69 | row.append(data_dict["hall_1"]) 70 | 71 | row.append(data_dict["species_2"]) 72 | row.append(data_dict["confidence_2"][:5]) 73 | row.append(data_dict["hall_2"]) 74 | 75 | row.append(data_dict["species_3"]) 76 | row.append(data_dict["confidence_3"][:5]) 77 | row.append(data_dict["hall_3"]) 78 | 79 | row.append(data_dict["species_4"]) 80 | row.append(data_dict["confidence_4"][:5]) 81 | row.append(data_dict["hall_4"]) 82 | 83 | row.append(data_dict["peaks"]) 84 | 85 | with open(path, "a") as csv_file: 86 | filewriter = csv.writer(csv_file, delimiter=",") 87 | filewriter.writerow(row) 88 | 89 | def combination_peaks(peak_batch,temp_name,user_info,URL,fam): 90 | outpath = "Ready" 91 | if not os.path.exists(outpath): 92 | os.makedirs(outpath) 93 | 94 | find_valid_peaks = list(powerset(peak_batch["vec"])) 95 | find_valid_peaks = [item for item in find_valid_peaks if len(item) > 3 and len(item) < 6] 96 | print(len(find_valid_peaks),"valid peak combinations") 97 | 98 | valid_peaks_combinations = [{"vec":proto_combo} for proto_combo in find_valid_peaks] 99 | found = False 100 | threshold = 0 101 | guesses = {"species_1":[], 102 | "species_2":[], 103 | "species_3":[], 104 | "species_4":[]} 105 | 106 | common_peaks = [] 107 | failed_combos = valid_peaks_combinations 108 | #peak_locs,user_info,URL,fam 109 | persistance = 0 110 | LIMIT = 3 111 | 112 | while len(failed_combos) > 0 and persistance < LIMIT: 113 | for combo in failed_combos: 114 | try: 115 | classificated = ClientSide.Send_For_Classification(combo, user_info, URL, fam) 116 | print(classificated) 117 | classificated["file_name"] = temp_name 118 | write_to_csv(os.path.join(outpath,temp_name) + ".csv", classificated) 119 | 120 | guesses['species_1'].append(classificated["species_1"]) 121 | guesses['species_2'].append(classificated["species_2"]) 122 | guesses['species_3'].append(classificated["species_3"]) 123 | guesses['species_4'].append(classificated["species_4"]) 124 | 125 | common_peaks.append(classificated["species_1"]) 126 | common_peaks.append(classificated["species_2"]) 127 | common_peaks.append(classificated["species_3"]) 128 | common_peaks.append(classificated["species_4"]) 129 | 130 | # remove the classified combination 131 | failed_combos.remove(combo) 132 | except KeyboardInterrupt: 133 | raise 134 | except: 135 | print("An error occured this combination was not classified.\nIt will be retried {} more times".format(LIMIT-persistance)) 136 | 137 | persistance += 1 138 | 139 | if len(failed_combos)>0: 140 | print("there were {} failed combinations".format(len(failed_combos))) 141 | 142 | return common_peaks, guesses 143 | 144 | def main(): 145 | 146 | parser = build_parser() 147 | options = parser.parse_args() 148 | 149 | print(options.session) 150 | 151 | # opens the user specified session 152 | if options.session: 153 | with open(os.path.join("Sessions",options.session),'r') as f: 154 | session = json.load(f) 155 | 156 | # opens the default session 157 | else: 158 | with open(DEFAULT_SESSION,'r') as f: 159 | 160 | session = json.load(f) 161 | 162 | # set variables from loaded session data 163 | fam = session["crystal_family"] 164 | provide_family = session["known_family"] 165 | display_type = session["display_type"] 166 | auto_calibrate = session["auto_calibrate"] 167 | file_path = session["file_path"] 168 | output_file = session["output_file"] 169 | is_profile = session["is_profile"] 170 | manual_peak_selection = session["manual_peak_selection"] 171 | scale_bar = session["scale_bar"] 172 | 173 | # Load calibration from specified file (json) 174 | try: 175 | print("Loading calibration from {}".format(os.path.join("Calibrations",auto_calibrate))) 176 | with open(os.path.join("Calibrations",auto_calibrate),'r') as f: 177 | calibration = json.load(f) 178 | 179 | except: 180 | print("No calibration could be loaded from {}\nPlease check the file exists and is formatted properly".format(auto_calibrate)) 181 | calibration = cf.set_calibration(is_profile) 182 | print(calibration) 183 | 184 | with open(SERVER_INFO,'r') as f: 185 | server_info = json.load(f) 186 | 187 | if server_info['URL']: 188 | URL = server_info['URL'] 189 | else: 190 | raise ValueError('you need to have the server URL provided to you') 191 | 192 | 193 | # Load user from provided path, [IN PROGRESS] 194 | with open(USER_INFO) as f: 195 | user_info = json.load(f) 196 | 197 | if not os.path.exists(file_path): 198 | print("The path provided could not be found. Please check your session file path") 199 | return 200 | 201 | # Determine if the path is a directory or a file 202 | if os.path.isdir(file_path): 203 | print("loading files from directory") 204 | file_paths = [] 205 | for dirpath,dirnames,fpath in os.walk(file_path): 206 | for path in fpath: 207 | file_paths.append(os.path.join(dirpath,path)) 208 | print("found {} files to load.".format(len(file_paths))) 209 | 210 | else: 211 | file_paths = [file_path] 212 | 213 | print(file_paths) 214 | for f_path in file_paths: 215 | 216 | # Load Data from specified file (DM3, TIFF, CSV etc....) 217 | try: 218 | print("loading data from {}".format(f_path)) 219 | image_data = ClientSide.Load_Image(f_path) 220 | print("I successfully loaded the data") 221 | except: 222 | print("Invalid file path given ({}).\n Please enter filepath to your data".format(f_path)) 223 | options.fpath = input() 224 | 225 | # Change the processing based on data type 226 | if is_profile: 227 | 228 | # Choose which profile if there are multiple 229 | image_data,scale = cf.choose_profile(image_data) 230 | 231 | else: 232 | pass 233 | #plt.imshow(np.log(image_data)) 234 | #plt.show(block=False) 235 | #plt.show() 236 | 237 | # Change the Processing based on the type of data 238 | if is_profile: 239 | print("identifying from profile") 240 | radial_profile = {"brightness":image_data, 241 | "pixel_range":scale} 242 | 243 | else: 244 | radial_profile = ClientSide.Extract_Profile(image_data) 245 | 246 | 247 | peak_locs = ClientSide.Find_Peaks(radial_profile,calibration,is_profile,display_type,scale_bar) 248 | print(peak_locs) 249 | 250 | 251 | # Choose which peaks to classify on 252 | if manual_peak_selection: 253 | peak_locs = cf.choose_peaks(peak_locs,display_type) 254 | 255 | 256 | if provide_family =="yes": 257 | 258 | while fam is None: 259 | temp_fam = input("What family does the Crystal belong to?\n") 260 | if temp_fam in FAMILIES: 261 | fam = temp_fam 262 | else: 263 | print("Invalid choice. choose from {}\n".format(str(FAMILIES)[1:-1])) 264 | elif provide_family == "no": 265 | fam = None 266 | 267 | print(peak_locs) 268 | 269 | with open(os.path.join("Results",f_path.split(os.sep)[-1][:-4]+".json"), "w") as o: 270 | o.write(json.dumps(peak_locs)) 271 | 272 | lower_gen = SpGr.edges["genus"][fam][0] 273 | upper_gen = SpGr.edges["genus"][fam][1] 274 | fam_range = range(SpGr.edges["species"][lower_gen][0],1+SpGr.edges["species"][upper_gen][1]) 275 | 276 | common_peaks,guesses = combination_peaks(peak_locs,f_path.split(os.sep)[-1][:-4],user_info,URL,fam) 277 | 278 | plt.figure(figsize=(len(fam_range)//2,4)) 279 | prev_histograms = [] 280 | plots = [] 281 | 282 | for rank in range(1,5): 283 | histo = np.histogram([int(g) for g in guesses["species_{}".format(rank)]],bins=fam_range) 284 | #histo[0] = histo[0]*(2-(rank/5.0)) 285 | 286 | if rank > 1: 287 | plot = plt.bar(histo[1][:-1],histo[0], 288 | bottom=np.sum(np.vstack(prev_histograms),axis=0),align="center") 289 | else: 290 | plot = plt.bar(histo[1][:-1],histo[0],align="center",color='red') 291 | 292 | plots.append(plot) 293 | plt.yticks(rotation='vertical') 294 | plt.xticks(histo[1][:-1],rotation='vertical') 295 | prev_histograms.append(histo[0]) 296 | 297 | plt.xlabel("Prediction",fontsize=10,rotation='vertical') 298 | plt.ylabel("Counts",fontsize=10) 299 | #plt.legend(plots,("species_1","species_2","species_3","species_4")) 300 | if not os.path.exists("Results"): 301 | os.makedirs("Results") 302 | print("Results/"+f_path.split(os.sep)[-1][:-4]+"gen1.png") 303 | plt.savefig("Results/" + f_path.split(os.sep)[-1][:-4] + "_gen1.png") 304 | plt.show(block=False) 305 | 306 | 307 | if __name__ == "__main__": 308 | main() 309 | 310 | 311 | 312 | -------------------------------------------------------------------------------- /DiffractionClassifierCombinatorial2.0.py: -------------------------------------------------------------------------------- 1 | import ClientSide2 #custom package 2 | 3 | import numpy as np 4 | import argparse 5 | import json 6 | import os 7 | import ClassifierFunctions2 as cf 8 | import random 9 | import logging 10 | 11 | from matplotlib import pyplot as plt 12 | from builtins import input 13 | 14 | from Notation import SpaceGroupsDict as spgs 15 | SpGr = spgs.spacegroups() 16 | 17 | 18 | 19 | from itertools import combinations,chain 20 | 21 | 22 | # Initialize essential global variables 23 | #URL = "" #you'll need me to send you the link 24 | FAMILIES = ["triclinic","monoclinic","orthorhombic","tetragonal", 25 | "trigonal","hexagonal","cubic"] 26 | 27 | DEFAULT_SESSION = os.path.join ("Sessions","session.json") 28 | DEFAULT_USER = "user_profile.json" 29 | SERVER_INFO = "server_gen2.json" 30 | 31 | # list of three, one per level 32 | prediction_per_level = [1, 1, 2] 33 | num_peaks = [1, 5] 34 | 35 | DEFAULT_FILTER_SETTINGS = { "max_numpeaks": 75, 36 | "dspace_range" : [0.5,6], 37 | "peak_threshold": 0.7, 38 | "filter_size" : 15, 39 | "passes" : 2 40 | } 41 | 42 | 43 | def build_parser(): 44 | parser = argparse.ArgumentParser() 45 | 46 | # This will be implemented as rollout broadens 47 | parser.add_argument('--apikey', type=str, 48 | dest='key', help='api key to securely access service', 49 | metavar='KEY', required=False) 50 | 51 | parser.add_argument('--session', 52 | dest='session', help='Keep user preferences for multirun sessions', metavar='SESSION',required=False, default=None) 53 | parser.add_argument('--subset', 54 | dest='subset',help='Run a small number of the possible combinations. Mostly for testing. Input the number of combos to run.', metavar='NO_OF_COMBOS',required=False, default=None) 55 | parser.add_argument('--dataonly', 56 | dest='data_only',help='run the classification without plotting', metavar='True/[False]',required=False, default=False) 57 | parser.add_argument('--figuresonly', 58 | dest='figures_only',help='Plot the figures without running data. Data must be saved previously.', metavar='True/[False]',required=False, default=False) 59 | 60 | return parser 61 | 62 | def powerset(iterable): 63 | "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" 64 | s = list(iterable) 65 | return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) 66 | 67 | 68 | 69 | def combination_peaks(peak_batch, chem_vec, mode, temp_name, crystal_family, user_info, URL, prediction_per_level, subset, num_peaks): 70 | 71 | outpath = "Ready" 72 | if not os.path.exists(outpath): 73 | os.makedirs(outpath) 74 | find_valid_peaks = list(powerset(peak_batch["vec"])) 75 | find_valid_peaks = [item for item in find_valid_peaks if len(item) > num_peaks[0] and len(item) < num_peaks[1]] 76 | print(len(find_valid_peaks),"valid peak combinations") 77 | 78 | valid_peaks_combinations = [{"vec":proto_combo} for proto_combo in find_valid_peaks] 79 | found = False 80 | threshold = 0 81 | tot_spec = 1 82 | for p in prediction_per_level: 83 | tot_spec *= p 84 | guesses = {"num_pred": tot_spec} 85 | for k in range(1,tot_spec+1): 86 | guesses["species_"+str(k)]=[] 87 | guesses["spec_confidence_"+str(k)]=[] 88 | # print(guesses) 89 | common_peaks = [] 90 | failed_combos = valid_peaks_combinations 91 | #peak_locs,user_info,URL,fam 92 | persistance = 0 93 | LIMIT = 3 94 | # print(failed_combos) 95 | 96 | 97 | if subset >0 and subset 0 and persistance < LIMIT: 102 | for combo in failed_combos: 103 | try: 104 | # print('---classifying---') 105 | # print(combo) 106 | classificated = ClientSide2.Send_For_Classification(combo, chem_vec, mode, crystal_family, user_info, URL, prediction_per_level) 107 | print(classificated) 108 | classificated["file_name"] = temp_name 109 | # print('name =') 110 | # print(temp_name) 111 | print(os.path.join(outpath,temp_name)) 112 | cf.write_to_csv(os.path.join(outpath,temp_name) + ".csv", classificated, prediction_per_level) 113 | print(tot_spec) 114 | for k in range(1,tot_spec+1): 115 | print(guesses) 116 | guesses['species_'+str(k)].append( classificated["species_"+str(k)] ) 117 | guesses['spec_confidence_'+str(k)].append( classificated["spec_confidence_"+str(k)] ) 118 | common_peaks.append(classificated["peaks"]) 119 | 120 | 121 | 122 | # remove the classified combination 123 | failed_combos.remove(combo) 124 | 125 | except KeyboardInterrupt: 126 | raise 127 | except: 128 | print("An error occured this combination was not classified.\nIt will be retried {} more times".format(LIMIT-persistance)) 129 | 130 | persistance += 1 131 | 132 | if len(failed_combos)>0: 133 | print("there were {} failed combinations".format(len(failed_combos))) 134 | print('returning') 135 | return common_peaks, guesses 136 | 137 | def make_figures(guesses,crystal_family,froot): 138 | 139 | if crystal_family: 140 | lower_gen = SpGr.edges["genus"][crystal_family][0] 141 | upper_gen = SpGr.edges["genus"][crystal_family][1] 142 | else: 143 | lower_gen = SpGr.edges["genus"][FAMILIES[0]][0] 144 | upper_gen = SpGr.edges["genus"][FAMILIES[-1]][1] 145 | fam_range = range(SpGr.edges["species"][lower_gen][0],1+SpGr.edges["species"][upper_gen][1]) 146 | 147 | # phi = 2*np.pi/360 148 | fig_ang = 300 149 | phi = (2*np.pi*fig_ang/360)/(max(fam_range)-min(fam_range)+1) 150 | thet = fig_ang/(max(fam_range)-min(fam_range)+1) 151 | fam_axes = [1,3,16,75,143,168,195] 152 | 153 | # fig1 = plt.figure(1,figsize=(len(fam_range),16)) 154 | fig1 = plt.figure(2,figsize=(16,8)) 155 | plt.clf() 156 | ax1 = fig1.add_axes([0.03,0.1,.96,.8]) 157 | # ax1.set_yscale('log') 158 | fam_color = ['k','g','b','c','m','y','k'] 159 | for k in range(len(fam_axes)-1): 160 | ax1.axvspan(fam_axes[k]-0.5,fam_axes[k+1]-0.5,facecolor = fam_color[k], alpha=0.5) 161 | # ax1.axvspan(fam_axes[0],fam_axes[1]-1,alpha=0.5) 162 | 163 | ax1.axvspan(fam_axes[-1]-0.5,np.max(fam_range)-0.5,alpha=0.3) 164 | plt.ion 165 | 166 | fig2 = plt.figure(3,figsize=(8,8)) 167 | plt.clf() 168 | plt.ion 169 | ax2 = fig2.add_axes([0.1,0.1,0.8,0.8],polar=True) 170 | ax2.set_thetamin(1) 171 | ax2.set_rmin(0) 172 | ax2.set_thetamax(fig_ang) 173 | ax2.set_rlabel_position(30) 174 | ax2.set_theta_direction(-1) 175 | ax2.set_theta_zero_location("S",offset=-(360-fig_ang)/2) 176 | # ax2.set_rscale('log') 177 | prev_histograms_1 = [] 178 | prev_histograms_2 = [] 179 | plots_1 = [] 180 | plots_2 = [] 181 | # print('guesses = ') 182 | # print(guesses) 183 | num_pred = np.prod(prediction_per_level) 184 | for rank in range(1,num_pred+1): 185 | histo = np.histogram([g for g in guesses["species_{}".format(rank)]], weights = [g for g in guesses["spec_confidence_{}".format(rank)]], bins = np.arange(min(fam_range)-0.5, max(fam_range)+1.5)) 186 | histo_log = np.array([np.log10(float(h))+1 if h>0 else 0 for h in histo[0]]) 187 | # print('log_histo = ') 188 | # print(histo_log.tolist()) 189 | if rank > 1: 190 | plt.figure(2) 191 | plot_1 = plt.bar(fam_range, histo[0], bottom = np.sum(np.vstack(prev_histograms_1), axis=0), align="center", width = 1.1) 192 | plt.figure(3) 193 | sum_hist = np.sum(np.vstack(prev_histograms_1), axis=0) 194 | log_sum = np.array([np.log10(float(h))-1 if h>0 else -1 for h in sum_hist]) 195 | # print('log_sum = ') 196 | # print(log_sum.tolist()) 197 | plot_2 = plt.bar([f*phi for f in fam_range], histo_log, bottom = log_sum, align="center", width = phi) 198 | else: 199 | plt.figure(2) 200 | plot_1 = plt.bar(fam_range, histo[0], align="center", color='red', width = 1.1) 201 | plt.figure(3) 202 | plot_2 = plt.bar([f*phi for f in fam_range], histo_log, bottom = -1, align="center", color='red', width = phi) 203 | 204 | plots_1.append(plot_1) 205 | plots_2.append(plot_2) 206 | plt.figure(2) 207 | plt.yticks(rotation='vertical') 208 | plt.xticks(fam_range,rotation='vertical') 209 | prev_histograms_1.append(histo[0]) 210 | prev_histograms_2.append(histo[0]) 211 | # plt.figure(3) 212 | # ax2.set_xticks(histo[1][:-1]) 213 | 214 | plt.figure(2) 215 | # ym = ax1.get_ymax()*.9 216 | 217 | r_max = 0 218 | for rect in plot_1: 219 | n_max = rect.get_height()+rect.get_y() 220 | if n_max>r_max: 221 | r_max = n_max 222 | 223 | 224 | for k in range(len(FAMILIES)-1): 225 | if k ==0: 226 | ym_t = r_max*0.7 227 | cent = 'left' 228 | else: 229 | ym_t = r_max*0.6 230 | cent = 'center' 231 | ax1.text((fam_axes[k+1]+fam_axes[k])/2,ym_t, FAMILIES[k], horizontalalignment=cent) 232 | 233 | 234 | ax1.text((fam_axes[-1]+np.max(fam_range))/2,ym_t, FAMILIES[-1], horizontalalignment='center') 235 | 236 | ax1.autoscale(enable=True, axis='x', tight=True) 237 | ax1.tick_params(axis='x', which='major', labelsize=6) 238 | plt.xlabel("Prediction",fontsize=10) 239 | plt.ylabel("Counts",fontsize=10) 240 | # plt.legend(plots,("species_1","species_2","species_3","species_4")) 241 | leg_list = [ "species_{}".format(k+1) for k in range(num_pred) ] 242 | plt.legend(plots_1,leg_list) 243 | print("Results/"+froot+"_gen2.png") 244 | plt.savefig("Results/"+froot+"_gen2.png",dpi = 300) 245 | 246 | plt.figure(3) 247 | # plt.xlabel("Prediction",fontsize=10,rotation='vertical') 248 | # plt.ylabel("Counts",fontsize=10) 249 | r_ticks = list(range(int(np.floor(ax2.get_rmin())),int(np.ceil(ax2.get_rmax())+1))) 250 | ax2.set_rgrids(r_ticks, labels = ['10e'+str(r) for r in r_ticks]) 251 | ax2.set_thetagrids([f*thet for f in fam_axes],labels = FAMILIES) 252 | plt.legend(plots_2,leg_list) 253 | # plt.legend(plots,("species_1","species_2","species_3","species_4")) 254 | # print("Results/"+froot+"_gen2_polar.png") 255 | # plt.savefig("Results/"+froot+"_gen2_polar.png",dpi = 300) 256 | # plt.show() 257 | 258 | 259 | def main(): 260 | 261 | parser = build_parser() 262 | options = parser.parse_args() 263 | 264 | if options.subset: 265 | subset = int(options.subset) 266 | else: 267 | subset = -1 268 | 269 | 270 | print(options.session) 271 | 272 | # opens the user specified session 273 | if options.session: 274 | with open(os.path.join("Sessions",options.session),'r') as f: 275 | session = json.load(f) 276 | 277 | # opens the default session 278 | else: 279 | with open(DEFAULT_SESSION,'r') as f: 280 | session = json.load(f) 281 | 282 | # set variables from loaded session data 283 | # print(session) 284 | file_path = session["file_path"] 285 | if "output_file" in session: 286 | output_file = session["output_file"] 287 | else: 288 | output_file = '' 289 | if "output_file_root" in session: 290 | output_file_root = session["output_file_root"] 291 | else: 292 | output_file_root = '' 293 | if not (output_file or output_file_root): 294 | raise ValueError('output_file or output_file_root must be defined in session file.') 295 | manual_peak_selection = session["manual_peak_selection"] 296 | known_family = session["known_family"] 297 | chemistry = session["chemistry"] 298 | diffraction = session["diffraction"] 299 | 300 | print('file inputs') 301 | print(output_file) 302 | print(output_file_root) 303 | 304 | mode = "" 305 | 306 | if diffraction: 307 | if chemistry: 308 | mode="DiffChem" 309 | else: 310 | mode="DiffOnly" 311 | else: 312 | if chemistry: 313 | raise ValueError('Running chemistry only predictions is currently not implemented') 314 | else: 315 | raise ValueError('Invalid prediction type. Either diffraction or chemistry must be enabled') 316 | 317 | if known_family and known_family=='yes': 318 | print('known family') 319 | crystal_family = session["crystal_family"] 320 | prediction_per_level[0] = 1 321 | else: 322 | crystal_family = None 323 | 324 | # Load user from provided path, [IN PROGRESS] 325 | if session["user_info"]: 326 | with open(session["user_info"],'r') as f: 327 | user_info = json.load(f) 328 | else: 329 | with open(DEFAULT_USER,'r') as f: 330 | user_info = json.load(f) 331 | 332 | with open(session["server_info"],'r') as f: 333 | server_info = json.load(f) 334 | 335 | if server_info['URL']: 336 | url = server_info['URL'] 337 | else: 338 | raise ValueError('you need to have the server URL provided to you') 339 | 340 | chem_vec = cf.check_for_chemistry(session) 341 | 342 | print(file_path) 343 | print('---starting loop--') 344 | # Determine if the path is a directory or a file 345 | if os.path.isdir(file_path): 346 | print("loading files from directory") 347 | file_paths = [] 348 | for dirpath,dirnames,fpath in os.walk(file_path): 349 | for path in fpath: 350 | if not path[0] == '.': 351 | file_paths.append(os.path.join(dirpath,path)) 352 | print("found {} files to load.".format(len(file_paths))) 353 | 354 | else: 355 | file_paths = [file_path] 356 | 357 | if not os.path.exists("Results"): 358 | os.makedirs("Results") 359 | 360 | 361 | print(file_paths) 362 | for f_path in file_paths: 363 | 364 | # Load Data from specified file (DM3, TIFF, CSV etc....) 365 | 366 | print("loading data from {}".format(f_path)) 367 | image_data,scale = ClientSide2.Load_Profile(f_path) 368 | print("I successfully loaded the data") 369 | 370 | # print(scale) 371 | 372 | print(options.figures_only) 373 | print(options.data_only) 374 | 375 | # difining filepaths here to facilitate loading data. 376 | froot = os.path.splitext(os.path.basename(f_path))[0] 377 | if output_file_root: 378 | outfile = 'Results/'+output_file_root+froot+'.json' 379 | outfile_2 = 'Results/'+output_file_root+froot+'_peaks.json' 380 | else: 381 | output_file_root='' #for the figure filenames 382 | [outroot, ext] = os.path.splitext(output_file) 383 | if not ext=='.json': 384 | output_file = outroot+'.json' 385 | output_file_2 = outroot+'_peaks.json' 386 | outfile = 'Results/'+output_file 387 | outfile_2 = 'Results/'+output_file_2 388 | 389 | # optional skipping the data creation 390 | if options.figures_only: 391 | print('Only creating figures') 392 | with open(outfile, 'r') as fp: 393 | guesses = json.load(fp) 394 | else: 395 | if diffraction: 396 | peak_locs,peaks_h = ClientSide2.Find_Peaks(image_data, scale, **FILTER_SETTINGS) 397 | # Choose which peaks to classify on 398 | if manual_peak_selection: 399 | peak_locs = cf.choose_peaks(peak_locs,peaks_h) 400 | #raise NotImplementedError 401 | else: 402 | peak_locs = [] 403 | peaks_h = [] 404 | 405 | 406 | # Script hangs when there are too many peaks. 407 | # TODO: implement something better. 408 | if len(peak_locs['d_spacing'])>25: 409 | print("\n\n======================================================") 410 | print("there are "+ str(len(peak_locs['d_spacing']))+" peaks, which is too many.") 411 | print(f_path) 412 | print("======================================================\n\n") 413 | continue 414 | 415 | 416 | 417 | common_peaks,guesses = combination_peaks(peak_locs, chem_vec, mode, froot, crystal_family, user_info, url, prediction_per_level, subset, num_peaks) 418 | # print("--- peak_locs ---") 419 | # print(peak_locs) 420 | guesses["pk_d_spacing"] = peak_locs["d_spacing"].tolist() 421 | guesses["pk_vec"] = peak_locs["vec"] 422 | 423 | print(guesses) 424 | 425 | # save data 426 | with open(outfile, 'w') as fp: 427 | json.dump(guesses, fp) 428 | with open(outfile_2, 'w') as fp: 429 | json.dump(common_peaks, fp) 430 | 431 | 432 | if options.data_only: 433 | print('skipping figures') 434 | else: 435 | make_figures(guesses,crystal_family,output_file_root+froot) 436 | # TODO: Split up this function and enable plotting on precomupted data. 437 | 438 | 439 | # plt.show(block=False) 440 | 441 | 442 | if __name__ == "__main__": 443 | main() 444 | 445 | 446 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 The Scientific Computing and Imaging Institute 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Notation/HSGdict.txt: -------------------------------------------------------------------------------- 1 | P1 triclinic 1 1 2 | P-1 triclinic 2 2 3 | P2 monoclinic 3 3 4 | P2_1 monoclinic 4 3 5 | C2 monoclinic 5 3 6 | Pm monoclinic 6 4 7 | Pc monoclinic 7 4 8 | Cm monoclinic 8 4 9 | Cc monoclinic 9 4 10 | P2/m monoclinic 10 5 11 | P2_1/m monoclinic 11 5 12 | C2/m monoclinic 12 5 13 | P2/c monoclinic 13 5 14 | P2_1/c monoclinic 14 5 15 | C2/c monoclinic 15 5 16 | P222 orthorhombic 16 6 17 | P222_1 orthorhombic 17 6 18 | P2_12_12 orthorhombic 18 6 19 | P2_12_12_1 orthorhombic 19 6 20 | C222_1 orthorhombic 20 6 21 | C222 orthorhombic 21 6 22 | F222 orthorhombic 22 6 23 | I222 orthorhombic 23 6 24 | I2_12_12_1 orthorhombic 24 6 25 | Pmm2 orthorhombic 25 7 26 | Pmc2_1 orthorhombic 26 7 27 | Pcc2 orthorhombic 27 7 28 | Pma2 orthorhombic 28 7 29 | Pca2_1 orthorhombic 29 7 30 | Pnc2 orthorhombic 30 7 31 | Pmn2_1 orthorhombic 31 7 32 | Pba2 orthorhombic 32 7 33 | Pna2_1 orthorhombic 33 7 34 | Pnn2 orthorhombic 34 7 35 | Cmm2 orthorhombic 35 7 36 | Cmc2_1 orthorhombic 36 7 37 | Ccc2 orthorhombic 37 7 38 | Amm2 orthorhombic 38 7 39 | Aem2 orthorhombic 39 7 40 | Ama2 orthorhombic 40 7 41 | Aea2 orthorhombic 41 7 42 | Fmm2 orthorhombic 42 7 43 | Fdd2 orthorhombic 43 7 44 | Imm2 orthorhombic 44 7 45 | Iba2 orthorhombic 45 7 46 | Ima2 orthorhombic 46 7 47 | Pmmm orthorhombic 47 8 48 | Pnnn orthorhombic 48 8 49 | Pccm orthorhombic 49 8 50 | Pban orthorhombic 50 8 51 | Pmma orthorhombic 51 8 52 | Pnna orthorhombic 52 8 53 | Pmna orthorhombic 53 8 54 | Pcca orthorhombic 54 8 55 | Pbam orthorhombic 55 8 56 | Pccn orthorhombic 56 8 57 | Pbcm orthorhombic 57 8 58 | Pnnm orthorhombic 58 8 59 | Pmmn orthorhombic 59 8 60 | Pbcn orthorhombic 60 8 61 | Pbca orthorhombic 61 8 62 | Pnma orthorhombic 62 8 63 | Cmcm orthorhombic 63 8 64 | Cmce orthorhombic 64 8 65 | Cmmm orthorhombic 65 8 66 | Cccm orthorhombic 66 8 67 | Cmme orthorhombic 67 8 68 | Ccce orthorhombic 68 8 69 | Fmmm orthorhombic 69 8 70 | Fddd orthorhombic 70 8 71 | Immm orthorhombic 71 8 72 | Ibam orthorhombic 72 8 73 | Ibca orthorhombic 73 8 74 | Imma orthorhombic 74 8 75 | P4 tetragonal 75 9 76 | P4_1 tetragonal 76 9 77 | P4_2 tetragonal 77 9 78 | P4_3 tetragonal 78 9 79 | I4 tetragonal 79 9 80 | I4_1 tetragonal 80 9 81 | P-4 tetragonal 81 10 82 | I-4 tetragonal 82 10 83 | P4/m tetragonal 83 11 84 | P4_2/m tetragonal 84 11 85 | P4/n tetragonal 85 11 86 | P4_2/n tetragonal 86 11 87 | I4/m tetragonal 87 11 88 | I4_1/a tetragonal 88 11 89 | P422 tetragonal 89 12 90 | P42_12 tetragonal 90 12 91 | P4_122 tetragonal 91 12 92 | P4_12_12 tetragonal 92 12 93 | P4_222 tetragonal 93 12 94 | P4_22_12 tetragonal 94 12 95 | P4_322 tetragonal 95 12 96 | P4_32_12 tetragonal 96 12 97 | I422 tetragonal 97 12 98 | I4_122 tetragonal 98 12 99 | P4mm tetragonal 99 13 100 | P4bm tetragonal 100 13 101 | P4_2cm tetragonal 101 13 102 | P4_2nm tetragonal 102 13 103 | P4cc tetragonal 103 13 104 | P4nc tetragonal 104 13 105 | P4_2mc tetragonal 105 13 106 | P4_2bc tetragonal 106 13 107 | I4mm tetragonal 107 13 108 | I4cm tetragonal 108 13 109 | I4_1md tetragonal 109 13 110 | I4_1cd tetragonal 110 13 111 | P-42m tetragonal 111 14 112 | P-42c tetragonal 112 14 113 | P-42_1m tetragonal 113 14 114 | P-42_1c tetragonal 114 14 115 | P-4m2 tetragonal 115 14 116 | P-4c2 tetragonal 116 14 117 | P-4b2 tetragonal 117 14 118 | P-4n2 tetragonal 118 14 119 | I-4m2 tetragonal 119 14 120 | I-4c2 tetragonal 120 14 121 | I-42m tetragonal 121 14 122 | I-42d tetragonal 122 14 123 | P4/mmm tetragonal 123 15 124 | P4/mcc tetragonal 124 15 125 | P4/nbm tetragonal 125 15 126 | P4/nnc tetragonal 126 15 127 | P4/mbm tetragonal 127 15 128 | P4/mnc tetragonal 128 15 129 | P4/nmm tetragonal 129 15 130 | P4/ncc tetragonal 130 15 131 | P4_2/mmc tetragonal 131 15 132 | P4_2/mcm tetragonal 132 15 133 | P4_2/nbc tetragonal 133 15 134 | P4_2/nnm tetragonal 134 15 135 | P4_2/mbc tetragonal 135 15 136 | P4_2/mnm tetragonal 136 15 137 | P4_2/nmc tetragonal 137 15 138 | P4_2/ncm tetragonal 138 15 139 | I4/mmm tetragonal 139 15 140 | I4/mcm tetragonal 140 15 141 | I4_1/amd tetragonal 141 15 142 | I4_1/acd tetragonal 142 15 143 | P3 trigonal 143 16 144 | P3_1 trigonal 144 16 145 | P3_2 trigonal 145 16 146 | R3 trigonal 146 16 147 | P-3 trigonal 147 17 148 | R-3 trigonal 148 17 149 | P312 trigonal 149 18 150 | P321 trigonal 150 18 151 | P3_112 trigonal 151 18 152 | P3_121 trigonal 152 18 153 | P3_212 trigonal 153 18 154 | P3_221 trigonal 154 18 155 | R32 trigonal 155 18 156 | P3m1 trigonal 156 19 157 | P31m trigonal 157 19 158 | P3c1 trigonal 158 19 159 | P31c trigonal 159 19 160 | R3m trigonal 160 19 161 | R3c trigonal 161 19 162 | P-31m trigonal 162 20 163 | P-31c trigonal 163 20 164 | P-3m1 trigonal 164 20 165 | P-3c1 trigonal 165 20 166 | R-3m trigonal 166 20 167 | R-3c trigonal 167 20 168 | P6 hexagonal 168 21 169 | P6_1 hexagonal 169 21 170 | P6_5 hexagonal 170 21 171 | P6_2 hexagonal 171 21 172 | P6_4 hexagonal 172 21 173 | P6_3 hexagonal 173 21 174 | P-6 hexagonal 174 22 175 | P6/m hexagonal 175 23 176 | P6_3/m hexagonal 176 23 177 | P622 hexagonal 177 24 178 | P6_122 hexagonal 178 24 179 | P6_522 hexagonal 179 24 180 | P6_222 hexagonal 180 24 181 | P6_422 hexagonal 181 24 182 | P6_322 hexagonal 182 24 183 | P6mm hexagonal 183 25 184 | P6cc hexagonal 184 25 185 | P6_3cm hexagonal 185 25 186 | P6_3mc hexagonal 186 25 187 | P-6m2 hexagonal 187 26 188 | P-6c2 hexagonal 188 26 189 | P-62m hexagonal 189 26 190 | P-62c hexagonal 190 26 191 | P6/mmm hexagonal 191 27 192 | P6/mcc hexagonal 192 27 193 | P6_3/mcm hexagonal 193 27 194 | P6_3/mmc hexagonal 194 27 195 | P23 cubic 195 28 196 | F23 cubic 196 28 197 | I23 cubic 197 28 198 | P2_13 cubic 198 28 199 | I2_13 cubic 199 28 200 | Pm-3 cubic 200 29 201 | Pn-3 cubic 201 29 202 | Fm-3 cubic 202 29 203 | Fd-3 cubic 203 29 204 | Im-3 cubic 204 29 205 | Pa-3 cubic 205 29 206 | Ia-3 cubic 206 29 207 | P432 cubic 207 30 208 | P4_232 cubic 208 30 209 | F432 cubic 209 30 210 | F4_132 cubic 210 30 211 | I432 cubic 211 30 212 | P4_332 cubic 212 30 213 | P4_132 cubic 213 30 214 | I4_132 cubic 214 30 215 | P-43m cubic 215 31 216 | F-43m cubic 216 31 217 | I-43m cubic 217 31 218 | P-43n cubic 218 31 219 | F-43c cubic 219 31 220 | I-43d cubic 220 31 221 | Pm-3m cubic 221 32 222 | Pn-3n cubic 222 32 223 | Pm-3n cubic 223 32 224 | Pn-3m cubic 224 32 225 | Fm-3m cubic 225 32 226 | Fm-3c cubic 226 32 227 | Fd-3m cubic 227 32 228 | Fd-3c cubic 228 32 229 | Im-3m cubic 229 32 230 | Ia-3d cubic 230 32 231 | -------------------------------------------------------------------------------- /Notation/SpaceGroupsDict.py: -------------------------------------------------------------------------------- 1 | import os 2 | dirname = os.path.dirname(__file__) 3 | class spacegroups: 4 | 5 | def __init__(self): 6 | 7 | groups = [] 8 | families = [] 9 | sgs = [] 10 | geni = [] 11 | with open(os.path.join(dirname,"HSGdict.txt"), 'r') as proto: 12 | for line in proto.readlines(): 13 | group, family, sg, genus = line.strip().split('\t') 14 | groups.append(group) 15 | families.append(family) 16 | sgs.append(sg) 17 | geni.append(genus) 18 | 19 | self.family_as_int = {} 20 | counter = 0 21 | for item in families: 22 | if item not in self.family_as_int.keys(): 23 | counter += 1 24 | self.family_as_int[item] = counter 25 | 26 | 27 | self.group_to_family = dict(zip(groups,families)) 28 | self.group_to_sgs = dict(zip(groups,sgs)) 29 | self.group_to_geni = dict(zip(groups,geni)) 30 | self.genera_to_family = dict(zip(geni,families)) 31 | self.sgs_to_group = dict(zip(sgs,groups)) 32 | self.sgs_to_genus = dict(zip(sgs,geni)) 33 | self.genus = set(geni) 34 | 35 | edges_genus = {"triclinic":[1,2], 36 | "monoclinic":[3,5], 37 | "orthorhombic":[6,8], 38 | "tetragonal":[9,15], 39 | "trigonal":[16,20], 40 | "hexagonal":[21,27], 41 | "cubic":[28,32]} 42 | 43 | edges_species = {1:[1,1], 44 | 2:[2,2], 45 | 3:[3,5], 46 | 4:[6,9], 47 | 5:[10,15], 48 | 6:[16,24], 49 | 7:[25,46], 50 | 8:[47,74], 51 | 9:[75,80], 52 | 10:[81,82], 53 | 11:[83,88], 54 | 12:[89,98], 55 | 13:[99,110], 56 | 14:[111,122], 57 | 15:[123,142], 58 | 16:[143,146], 59 | 17:[147,148], 60 | 18:[149,155], 61 | 19:[156,161], 62 | 20:[162,167], 63 | 21:[168,173], 64 | 22:[174,174], 65 | 23:[175,176], 66 | 24:[177,182], 67 | 25:[183,186], 68 | 26:[187,190], 69 | 27:[191,194], 70 | 28:[195,199], 71 | 29:[200,206], 72 | 30:[207,214], 73 | 31:[215,220], 74 | 32:[221,230]} 75 | 76 | self.edges = {"genus":edges_genus, 77 | "species":edges_species} 78 | 79 | self.sgs_to_family = {} 80 | for family in self.edges["genus"].keys(): 81 | startstop = self.edges['genus'][family] 82 | 83 | for genera in range(startstop[0],startstop[1]+1): 84 | ss = self.edges["species"][genera] 85 | for sgroup in range(ss[0],ss[1]+1): 86 | self.sgs_to_family[sgroup] = self.family_as_int[family] 87 | 88 | -------------------------------------------------------------------------------- /Notation/SpaceGroupsDict.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SCIInstitute/DiffractionClassification/68be6cf3960f09388253c79bab13cbd9dc07edbb/Notation/SpaceGroupsDict.pyc -------------------------------------------------------------------------------- /PeakFinding.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | import numpy as np 5 | from matplotlib import pyplot as plt 6 | 7 | def vote_peaks(signal, filter_size=1,passes=2,threshold=.8): 8 | """ 9 | Input: 10 | 11 | signal : dictionary, contains two_theta, d_spacings, and input_vector arrays 12 | peaks locations found in the profile 13 | 14 | filter_size : int, starting bin width of each pass 15 | 16 | passes : int, number of voting rounds 17 | 18 | thresh : 0 < float < 1, percentage votes to be considered a peak 19 | 20 | Outputs: 21 | 22 | payload : dictionary, contains classification statistics and predictions 23 | 24 | """ 25 | 26 | # determine size of filter 27 | size = len(signal) 28 | 29 | # Pad profile so it can be indexed properly 30 | signal = np.pad(signal,(filter_size+passes,filter_size+passes),'constant', constant_values=(0)) 31 | 32 | 33 | # create vote holder array 34 | votes = np.zeros_like(signal) 35 | 36 | # iterate over the array accruing votes for the tallest peaks 37 | for j in range(passes): 38 | 39 | # step through the array and vote for peak locations 40 | for i in range(0,size+j): 41 | scalar = 1 42 | votes[np.argmax(signal[i:i+filter_size+j])+i] += scalar 43 | 44 | # Pare down the votes to remove extraneous peaks but preserve candidates 45 | votes[votesdegrees*2 76 | 77 | return theta, d 78 | 79 | def profile2theta(pixel_profile,SIZE=1e9,WAVE=.15406): 80 | """ 81 | Inputs: 82 | pixel_profile : array, vector of bins in inverse D 83 | SIZE: float, pixel_size in microns/pixel 84 | WAVE: float, wave length in nanometers 85 | Outputs: 86 | 87 | t_scale: array, vector of two theta values 88 | rd_scale: array, vector of d space values 89 | 90 | """ 91 | 92 | id_scale = np.linspace(0.00001,len(pixel_profile),len(pixel_profile))*SIZE 93 | 94 | rd_scale = 1/id_scale 95 | 96 | t_scale = d2theta(rd_scale,WAVE) 97 | 98 | return t_scale, rd_scale 99 | 100 | def d2theta(d,wavelength=.15406): 101 | """ 102 | Inputs: 103 | pixel_profile : array, vector of bins in D 104 | WAVE: float, wave length in nanometers 105 | Outputs: 106 | 107 | theta_vec: array, vector of two theta values 108 | """ 109 | 110 | theta_vec = 2 * np.arcsin(wavelength/(d*2)) * 180/np.pi 111 | return theta_vec 112 | 113 | def plot_peaks(signal,scale,votes,display_type): 114 | 115 | """ 116 | Inputs: 117 | signal : array, vector of bins in inverse D 118 | scale: array, pixel_size in microns/pixel 119 | votes: array, wave length in nanometers 120 | display_type: string, d or theta 121 | 122 | Outputs: 123 | 124 | None 125 | 126 | """ 127 | 128 | indicies = np.where(votes>0) 129 | plt.figure(figsize=(6,2)) 130 | plt.plot(scale,signal,linewidth=3) 131 | sig_min = np.amin(signal) 132 | 133 | counter = 1 134 | for index in indicies[0]: 135 | x = np.array([scale[index],scale[index]]) 136 | y = np.array([signal[index],sig_min]) 137 | plt.plot(x,y,linewidth=2,label="peak {}".format(counter)) 138 | counter += 1 139 | 140 | if display_type == "d": 141 | plt.xlim(.5,6) 142 | plt.title("d spacing") 143 | elif display_type == "theta": 144 | plt.xlim(0,110) 145 | plt.title("two theta") 146 | 147 | 148 | plt.legend() 149 | plt.show(block=False) 150 | 151 | -------------------------------------------------------------------------------- /PeakFinding2.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | import numpy as np 5 | from matplotlib import pyplot as plt 6 | 7 | def vote_peaks(signal, **kwargs): 8 | """ 9 | Input: 10 | 11 | signal : list, contains d_spacings in the profile 12 | 13 | **kwargs: 14 | 15 | filter_size : int, starting bin width of each pass 16 | 17 | passes : int, number of voting rounds 18 | 19 | thresh : 0 < float < 1, percentage votes to be considered a peak 20 | 21 | Outputs: 22 | 23 | payload : dictionary, contains classification statistics and predictions 24 | 25 | """ 26 | filter_size = kwargs.get('filter_size',10) 27 | passes = kwargs.get('passes',2) 28 | threshold = kwargs.get('peak_threshold',.6) 29 | 30 | # determine size of filter 31 | size = len(signal) 32 | 33 | # Pad profile so it can be indexed properly 34 | signal = np.pad(signal,(filter_size,filter_size),'constant', constant_values=(0)) 35 | print(size,signal.shape) 36 | 37 | # create vote holder array 38 | votes = np.zeros_like(signal) 39 | scalar = 1 40 | 41 | # step through the array and vote for peak locations 42 | for i in range(0,size): 43 | 44 | #print(np.argmax(signal[i:i+filter_size])+i) 45 | votes[np.argmax(signal[i:i+filter_size])+i] += scalar 46 | 47 | # Pare down the votes to remove extraneous peaks but preserve candidates 48 | votes[votesy_res] = y_res 94 | 95 | y_tick[0] = "%7.3g" % np.max(histo[0][1:]) 96 | y_tick[int(y_res/2)] = "%7.3g" % (np.max(histo[0][1:])* (int(y_res/2)/y_res)) 97 | y_tick[y_res-1] = "%7.3g" % 0 98 | else: 99 | raise ValueError("print_histogram: mode not recognized") 100 | 101 | 102 | spacer = ' ' 103 | l_sp = len(spacer)-1 104 | x_tick = ' ' 105 | x_tick += "%7.3g" % histo[1][0] 106 | h1 = int(len(y_hist)/4) 107 | h2 = int(len(y_hist)/2) 108 | h3 = int(3*len(y_hist)/4) 109 | x_tick+=' '*(h1-l_sp) 110 | x_tick+= "%7.3g" % (histo[1][-1] * h1/len(y_hist)) 111 | x_tick+=' '*(h2-h1-l_sp) 112 | x_tick+= "%7.3g" % (histo[1][-1] * h2/len(y_hist)) 113 | x_tick+=' '*(h3-h2-l_sp) 114 | x_tick+= "%7.3g" % (histo[1][-1] * h3/len(y_hist)) 115 | x_tick+=' '*(len(y_hist)-h3-l_sp) 116 | x_tick+= "%7.3g" % (histo[1][-1]) 117 | 118 | bar_string = spacer+title+'\n' 119 | for l in range(y_res,0,-1): 120 | string = ['%' if y>=l else ' ' for y in y_hist] 121 | string = ''.join(string) 122 | bar_string += y_tick[y_res-l]+'|'+string+'\n' 123 | 124 | 125 | bar_string+= spacer + '-'*len(y_hist) +'\n' 126 | bar_string+= x_tick 127 | 128 | 129 | return bar_string 130 | 131 | 132 | def plot_peaks(signal,scale,votes,thresh = 0, **kwargs): 133 | 134 | """ 135 | Inputs: 136 | signal : array, vector of bins in inverse D 137 | scale: array, pixel_size in microns/pixel 138 | votes: array, wave length in nanometers 139 | display_type: string, d or theta 140 | 141 | Outputs: 142 | 143 | None 144 | 145 | """ 146 | 147 | scale_range = kwargs.get('dspace_range',[0.5, 6]) 148 | 149 | indicies = np.where(votes>thresh) 150 | plt.figure(1,figsize=(6,2)) 151 | plt.cla() 152 | plt.ion() 153 | plt.plot(scale,signal,linewidth=3) 154 | 155 | sig_min = np.amin(signal) 156 | 157 | 158 | peaks_h =[] 159 | counter = 1 160 | for index in indicies[0]: 161 | x = np.array([scale[index],scale[index]]) 162 | y = np.array([signal[index],sig_min]) 163 | line = plt.plot(x,y,linewidth=2,label="peak {}".format(counter)) 164 | counter += 1 165 | peaks_h.append(line) 166 | 167 | plt.xlim(scale_range[0],scale_range[1]) 168 | plt.xlabel("d spacing") 169 | plt.ylabel("intensity") 170 | 171 | 172 | return peaks_h 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /ProfileExtraction.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | 5 | import numpy as np 6 | 7 | 8 | def find_center(image,beam_stop=False): 9 | """ 10 | Find the center of the image 11 | 12 | Inputs: 13 | 14 | image : np.array, contains the diffraction data to analyze 15 | beam_stop : boolean, represents whether the beamstop is present in the image 16 | 17 | Outputs: 18 | 19 | center: tuple(x,y), the index coordinates of the central peak 20 | 21 | """ 22 | 23 | if beam_stop: 24 | raise ValueError("Beam stop analysis not implemented yet.") 25 | else: 26 | 27 | # We exclude the top n=3 pixels to avoid weird interactions on the detector 28 | ordered_points = np.dstack(np.unravel_index(np.argsort(image.ravel()), image.shape))[:,:-2] 29 | 30 | # Extract the nth pixel from the list ordered on intensity. 31 | center = ordered_points[:,-3][0] 32 | 33 | 34 | return center 35 | 36 | 37 | def radial_profile(data, center,bins=1): 38 | """ 39 | Create the azimuthal integration from the data 40 | 41 | Inputs: 42 | 43 | data : np.array, contains the diffraction data to analyze 44 | center : tuple(x,y), the index coordinates of the central peak 45 | 46 | Outputs: 47 | 48 | radius : np.array(1,N), range of pixel distance 49 | brightness : np.array(1,N), array of values for the integration 50 | 51 | """ 52 | 53 | # Create the indicies array masked array 54 | y,x = np.indices((data.shape)) # first determine radii of all pixels 55 | 56 | # Modify the indicies to incorporate pixel distance 57 | r = np.sqrt((x-center[1])**2+(y-center[0])**2) 58 | 59 | # Radius of the image 60 | r_max = np.max(r) 61 | 62 | # Evaluate the brightnesses of each ring in the indicies array 63 | ring_brightness, radius = np.histogram(r, weights=data, bins=int(r_max)//bins) 64 | 65 | return radius[1:], ring_brightness 66 | 67 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | README.txt 2 | 3 | ####Welcome to the DeepdiveCrystallography diffraction classification web service 4 | 5 | ###Quickstart Guide: 6 | 7 | prerequisites: python 3.6 and an internet connection 8 | 9 | To predict the crystal structure simply start the Diffraction Classifier by 10 | running the command: 11 | 12 | python DiffractionClassifier.py 13 | 14 | Then follow the series of prompts. 15 | 16 | Advanced usage: 17 | You can specify the data you'd like to load by adding --filepath FILEPATH_TO_YOUR_DATA 18 | to the function call. 19 | 20 | Similarly you specify the calibration by modifying the calibration.json file and adding --calibration calibration.json to the 21 | function call. 22 | 23 | ### Acknowledgements 24 | 25 | Work supported through the INL Laboratory Directed Research& Development (LDRD) Program under DOE Idaho Operations Office Contract DE-AC07-05ID145142. Thanks to Ian Harvey for many useful discussions and contributions to this work. 26 | 27 | 28 | ### Citations 29 | 30 | - J. A. Aguiar, M. L. Gong, R. R. Unocic, T. Tasdizen, & B. D. Miller. Decoding Crystallography from High-Resolution Electron Imaging and Diffraction Datasets with Deep Learning. Sci. Adv. aaw1949 (2019). 31 | 32 | - J. A. Aguiar, M. L. Gong & T. Tasdizen. Crystallographic prediction from diffraction and chemistry data for higher throughput classification using machine learning. Comput. Mater. Sci. 173, 109409 (2020). 33 | 34 | 35 | -------------------------------------------------------------------------------- /Sessions/EV_XRD.json: -------------------------------------------------------------------------------- 1 | {"file_path":"ExperimentalValidation", 2 | "auto_calibrate":"EV.json", 3 | "manual_peak_selection":true, 4 | "is_profile":false, 5 | "output_file":"ResultsEV_XRD.csv", 6 | "known_family":"yes", 7 | "crystal_family":"cubic", 8 | "user_info":"user_info.json", 9 | "display_type":"both", 10 | "scale_bar":"d"} -------------------------------------------------------------------------------- /Sessions/EV_sesion.json: -------------------------------------------------------------------------------- 1 | {"file_path":"ExperimentalValidation", 2 | "auto_calibrate":"EV.json", 3 | "manual_peak_selection":true, 4 | "is_profile":false, 5 | "output_file":"ResultsEV.csv", 6 | "known_family":"yes", 7 | "crystal_family":"cubic", 8 | "user_info":"user_info.json", 9 | "display_type":"both", 10 | "scale_bar":"d"} -------------------------------------------------------------------------------- /Sessions/session_gen_1.json: -------------------------------------------------------------------------------- 1 | {"file_path":"Data/profile_test/04_Cs_summed.csv", 2 | "auto_calibrate":"calibration.json", 3 | "manual_peak_selection":false, 4 | "is_profile":true, 5 | "output_file":"ResultsPTest_gen_1.csv", 6 | "known_family":"yes", 7 | "crystal_family":"tetragonal", 8 | "user_info":"user_profile.json", 9 | "display_type":"both", 10 | "scale_bar":"d", 11 | "server_info":"server_gen1.json" 12 | } 13 | -------------------------------------------------------------------------------- /Sessions/session_gen_2.json: -------------------------------------------------------------------------------- 1 | {"file_path":"Data/profile_test/04_Cs_summed.csv", 2 | "auto_calibrate":"calibration.json", 3 | "diffraction":true, 4 | "chemistry":false, 5 | "manual_peak_selection":true, 6 | "is_profile":true, 7 | "output_file":"ResultsPTest.csv", 8 | "known_family":"no", 9 | "crystal_family":"tetragonal", 10 | "user_info":"user_profile.json", 11 | "display_type":"both", 12 | "scale_bar":"d", 13 | "server_info":"server_gen2.json", 14 | "filter_settings": {"max_numpeaks": 20, 15 | "dspace_range" : [0.9,4], 16 | "peak_threshold": 0.5, 17 | "filter_size" : 250, 18 | "passes" : 2} 19 | } 20 | -------------------------------------------------------------------------------- /Sessions/session_w_chem.json: -------------------------------------------------------------------------------- 1 | {"file_path":"Data/profile_test/04_Cs_summed.csv", 2 | "auto_calibrate":"calibration.json", 3 | "diffraction":true, 4 | "chemistry":true, 5 | "manual_peak_selection":false, 6 | "is_profile":true, 7 | "output_file":"ResultsPTest.csv", 8 | "known_family":"no", 9 | "crystal_family":"tetragonal", 10 | "user_info":"user_profile.json", 11 | "display_type":"both", 12 | "scale_bar":"d", 13 | "chemical_formula":"Cs2O3", 14 | "chemical_contents":["Cs","O"], 15 | "atomic_percentage":[[58,0.4],[8,0.6]], 16 | "atomic_density":[[58,0.854],[8,0.146]], 17 | "server_info":"server_gen2.json" 18 | } 19 | -------------------------------------------------------------------------------- /UniversalLoader.py: -------------------------------------------------------------------------------- 1 | import dm3_lib as dm3 2 | import numpy as np 3 | 4 | from matplotlib import pyplot as plt 5 | 6 | 7 | def tif_extract(filepath): 8 | """ 9 | TIFs are assumed to be images unless stated in the session 10 | 11 | 12 | Inputs: 13 | 14 | filepath: string, location of data on drive 15 | 16 | Outputs: 17 | 18 | profile_data: array, 19 | 20 | """ 21 | 22 | 23 | # Read in the dat as a Numpy array 24 | image_data = plt.imread(fpath) 25 | 26 | print(image_data.shape) 27 | 28 | # Due to the lack of a universal schema for metatags at the current time 29 | # we leave it to the user to proscribe the relevant calibrations 30 | 31 | return image_data 32 | 33 | 34 | def dm3_extract(filepath): 35 | """ 36 | DM3 data is assumed to be an image unless stated in the session 37 | 38 | 39 | Inputs: 40 | 41 | filepath: string, location of data on drive 42 | 43 | Outputs: 44 | 45 | image_data: array, 46 | 47 | """ 48 | 49 | # Read in the data as a numpy array 50 | dm3_file = dm3.DM3(filepath) 51 | 52 | image_data = dm3_file.imagedata 53 | 54 | print(image_data.shape) 55 | 56 | # Due to the lack of a universal schema for metatags at the current time 57 | # we leave it to the user to proscribe the relevant calibrations 58 | 59 | return image_data 60 | 61 | def csv_extract(filepath): 62 | """ 63 | Data is expected to be a profile in rows or columns with no headers. 64 | Singular profiles are expected to be in pixels unless a scale bar is explicitly supplied 65 | 66 | Inputs: 67 | 68 | filepath: string, location of data on drive 69 | 70 | Outputs: 71 | 72 | profile_data: array, 73 | 74 | """ 75 | 76 | csv_file = np.genfromtxt(open(filepath, "rb"), delimiter=",") 77 | profile_data = csv_file 78 | 79 | # Orients columnar data into rows if necessary 80 | print(profile_data.shape) 81 | if len(profile_data.shape) != 1: 82 | if profile_data.shape[0] > profile_data.shape[1]: 83 | profile_data = profile_data.T 84 | print(profile_data.shape) 85 | 86 | # Due to the lack of a universal schema for metatags at the current time 87 | # we leave it to the user to proscribe the relevant calibrations 88 | 89 | return profile_data 90 | 91 | def txt_extract(filepath): 92 | """ 93 | Data is expected to be a profile in rows or columns with no headers. 94 | Singular profiles are expected to be in pixels unless a scale bar is explicitly supplied 95 | 96 | Inputs: 97 | 98 | filepath: string, location of data on drive 99 | 100 | Outputs: 101 | 102 | profile_data: array, 103 | 104 | """ 105 | 106 | # Read in the data as a numpy array 107 | text_file = np.genfromtxt(open(filepath, "rb"), delimiter="\t") 108 | profile_data = text_file 109 | 110 | # Orients columnar data into rows if necessary 111 | print(profile_data.shape) 112 | if len(profile_data.shape) != 1: 113 | if profile_data.shape[0] > profile_data.shape[1]: 114 | profile_data = profile_data.T 115 | print(profile_data.shape) 116 | 117 | 118 | # Due to the lack of a universal schema for metatags at the current time 119 | # we leave it to the user to proscribe the relevant calibrations 120 | 121 | return profile_data -------------------------------------------------------------------------------- /UniversalLoader2.py: -------------------------------------------------------------------------------- 1 | import dm3_lib as dm3 2 | import numpy as np 3 | 4 | from matplotlib import pyplot as plt 5 | 6 | 7 | def tif_extract(filepath): 8 | """ 9 | TIFs are assumed to be images unless stated in the session 10 | 11 | 12 | Inputs: 13 | 14 | filepath: string, location of data on drive 15 | 16 | Outputs: 17 | 18 | profile_data: array, 19 | 20 | """ 21 | 22 | raise NotImplementedError 23 | 24 | # Due to the lack of a universal schema for metatags at the current time 25 | # we leave it to the user to proscribe the relevant calibrations 26 | 27 | return image_data 28 | 29 | 30 | def dm3_extract(filepath): 31 | """ 32 | DM3 data is assumed to be an image unless stated in the session 33 | 34 | 35 | Inputs: 36 | 37 | filepath: string, location of data on drive 38 | 39 | Outputs: 40 | 41 | image_data: array, 42 | 43 | """ 44 | 45 | # Read in the data as a numpy array 46 | raise NotImplementedError 47 | # Due to the lack of a universal schema for metatags at the current time 48 | # we leave it to the user to proscribe the relevant calibrations 49 | 50 | return image_data 51 | 52 | def csv_extract(filepath): 53 | """ 54 | Data is expected to be a profile in rows or columns with no headers. 55 | Singular profiles are expected to be in pixels unless a scale bar is explicitly supplied 56 | 57 | Inputs: 58 | 59 | filepath: string, location of data on drive 60 | 61 | Outputs: 62 | 63 | profile_data: array, 64 | 65 | """ 66 | 67 | csv_file = np.genfromtxt(open(filepath, "r"), delimiter=",") 68 | profile_data = csv_file 69 | 70 | # Orients columnar data into rows if necessary 71 | print(profile_data.shape) 72 | if len(profile_data.shape) != 1: 73 | if profile_data.shape[0] > profile_data.shape[1]: 74 | profile_data = profile_data.T 75 | print(profile_data.shape) 76 | 77 | # Due to the lack of a universal schema for metatags at the current time 78 | # we leave it to the user to proscribe the relevant calibrations 79 | 80 | return profile_data[1,:],profile_data[0,:] 81 | 82 | def txt_extract(filepath): 83 | """ 84 | Data is expected to be a profile in rows or columns with no headers. 85 | Singular profiles are expected to be in pixels unless a scale bar is explicitly supplied 86 | 87 | Inputs: 88 | 89 | filepath: string, location of data on drive 90 | 91 | Outputs: 92 | 93 | profile_data: array, 94 | 95 | """ 96 | 97 | # Read in the data as a numpy array 98 | text_file = np.genfromtxt(open(filepath, "r"), delimiter="\t") 99 | profile_data = text_file 100 | 101 | # Orients columnar data into rows if necessary 102 | print(profile_data.shape) 103 | if len(profile_data.shape) != 1: 104 | if profile_data.shape[0] > profile_data.shape[1]: 105 | profile_data = profile_data.T 106 | print(profile_data.shape) 107 | 108 | 109 | # Due to the lack of a universal schema for metatags at the current time 110 | # we leave it to the user to proscribe the relevant calibrations 111 | 112 | return profile_data 113 | -------------------------------------------------------------------------------- /dm3_lib/_dm3_lib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """Python module for parsing GATAN DM3 files""" 3 | 4 | ################################################################################ 5 | ## Python script for parsing GATAN DM3 (DigitalMicrograph) files 6 | ## -- 7 | ## based on the DM3_Reader plug-in (v 1.3.4) for ImageJ 8 | ## by Greg Jefferis 9 | ## http://rsb.info.nih.gov/ij/plugins/DM3_Reader.html 10 | ## -- 11 | ## Python adaptation: Pierre-Ivan Raynal 12 | ## http://microscopies.med.univ-tours.fr/ 13 | ################################################################################ 14 | 15 | from __future__ import print_function 16 | 17 | import sys 18 | import os 19 | import struct 20 | 21 | from PIL import Image 22 | 23 | import numpy 24 | 25 | __all__ = ["DM3", "VERSION", "SUPPORTED_DATA_TYPES"] 26 | 27 | VERSION = '1.2' 28 | 29 | debugLevel = 0 # 0=none, 1-3=basic, 4-5=simple, 6-10 verbose 30 | 31 | ## check for Python version 32 | PY3 = (sys.version_info[0] == 3) 33 | 34 | ## - adjust for Python3 35 | if PY3: 36 | # unicode() deprecated in Python 3 37 | unicode_str = str 38 | else: 39 | unicode_str = unicode 40 | 41 | ### utility fuctions ### 42 | 43 | ### binary data reading functions ### 44 | 45 | def readLong(f): 46 | """Read 4 bytes as integer in file f""" 47 | read_bytes = f.read(4) 48 | return struct.unpack('>l', read_bytes)[0] 49 | 50 | def readShort(f): 51 | """Read 2 bytes as integer in file f""" 52 | read_bytes = f.read(2) 53 | return struct.unpack('>h', read_bytes)[0] 54 | 55 | def readByte(f): 56 | """Read 1 byte as integer in file f""" 57 | read_bytes = f.read(1) 58 | return struct.unpack('>b', read_bytes)[0] 59 | 60 | def readBool(f): 61 | """Read 1 byte as boolean in file f""" 62 | read_val = readByte(f) 63 | return (read_val!=0) 64 | 65 | def readChar(f): 66 | """Read 1 byte as char in file f""" 67 | read_bytes = f.read(1) 68 | return struct.unpack('c', read_bytes)[0] 69 | 70 | def readString(f, len_=1): 71 | """Read len_ bytes as a string in file f""" 72 | read_bytes = f.read(len_) 73 | str_fmt = '>'+str(len_)+'s' 74 | return struct.unpack( str_fmt, read_bytes )[0] 75 | 76 | def readLEShort(f): 77 | """Read 2 bytes as *little endian* integer in file f""" 78 | read_bytes = f.read(2) 79 | return struct.unpack(' reading function 122 | readFunc = { 123 | SHORT: readLEShort, 124 | LONG: readLELong, 125 | USHORT: readLEUShort, 126 | ULONG: readLEULong, 127 | FLOAT: readLEFloat, 128 | DOUBLE: readLEDouble, 129 | BOOLEAN: readBool, 130 | CHAR: readChar, 131 | OCTET: readChar, # difference with char??? 132 | } 133 | 134 | ## list of image DataTypes ## 135 | dataTypes = { 136 | 0: 'NULL_DATA', 137 | 1: 'SIGNED_INT16_DATA', 138 | 2: 'REAL4_DATA', 139 | 3: 'COMPLEX8_DATA', 140 | 4: 'OBSELETE_DATA', 141 | 5: 'PACKED_DATA', 142 | 6: 'UNSIGNED_INT8_DATA', 143 | 7: 'SIGNED_INT32_DATA', 144 | 8: 'RGB_DATA', 145 | 9: 'SIGNED_INT8_DATA', 146 | 10: 'UNSIGNED_INT16_DATA', 147 | 11: 'UNSIGNED_INT32_DATA', 148 | 12: 'REAL8_DATA', 149 | 13: 'COMPLEX16_DATA', 150 | 14: 'BINARY_DATA', 151 | 15: 'RGB_UINT8_0_DATA', 152 | 16: 'RGB_UINT8_1_DATA', 153 | 17: 'RGB_UINT16_DATA', 154 | 18: 'RGB_FLOAT32_DATA', 155 | 19: 'RGB_FLOAT64_DATA', 156 | 20: 'RGBA_UINT8_0_DATA', 157 | 21: 'RGBA_UINT8_1_DATA', 158 | 22: 'RGBA_UINT8_2_DATA', 159 | 23: 'RGBA_UINT8_3_DATA', 160 | 24: 'RGBA_UINT16_DATA', 161 | 25: 'RGBA_FLOAT32_DATA', 162 | 26: 'RGBA_FLOAT64_DATA', 163 | 27: 'POINT2_SINT16_0_DATA', 164 | 28: 'POINT2_SINT16_1_DATA', 165 | 29: 'POINT2_SINT32_0_DATA', 166 | 30: 'POINT2_FLOAT32_0_DATA', 167 | 31: 'RECT_SINT16_1_DATA', 168 | 32: 'RECT_SINT32_1_DATA', 169 | 33: 'RECT_FLOAT32_1_DATA', 170 | 34: 'RECT_FLOAT32_0_DATA', 171 | 35: 'SIGNED_INT64_DATA', 172 | 36: 'UNSIGNED_INT64_DATA', 173 | 37: 'LAST_DATA', 174 | } 175 | 176 | ## supported Data Types 177 | dT_supported = [1, 2, 6, 7, 9, 10, 11, 14] 178 | SUPPORTED_DATA_TYPES = {i: dataTypes[i] for i in dT_supported} 179 | 180 | ## other constants ## 181 | IMGLIST = "root.ImageList." 182 | OBJLIST = "root.DocumentObjectList." 183 | MAXDEPTH = 64 184 | 185 | DEFAULTCHARSET = 'utf-8' 186 | ## END constants ## 187 | 188 | 189 | class DM3(object): 190 | """DM3 object. """ 191 | 192 | ## utility functions 193 | def _makeGroupString(self): 194 | tString = str(self._curGroupAtLevelX[0]) 195 | for i in range( 1, self._curGroupLevel+1 ): 196 | tString += '.{}'.format(self._curGroupAtLevelX[i]) 197 | return tString 198 | 199 | def _makeGroupNameString(self): 200 | tString = self._curGroupNameAtLevelX[0] 201 | for i in range( 1, self._curGroupLevel+1 ): 202 | tString += '.' + str( self._curGroupNameAtLevelX[i] ) 203 | return tString 204 | 205 | def _readTagGroup(self): 206 | # go down a level 207 | self._curGroupLevel += 1 208 | # increment group counter 209 | self._curGroupAtLevelX[self._curGroupLevel] += 1 210 | # set number of current tag to -1 211 | # --- readTagEntry() pre-increments => first gets 0 212 | self._curTagAtLevelX[self._curGroupLevel] = -1 213 | if ( debugLevel > 5): 214 | print("rTG: Current Group Level:", self._curGroupLevel) 215 | # is the group sorted? 216 | sorted_ = readByte(self._f) 217 | isSorted = (sorted_ == 1) 218 | # is the group open? 219 | opened = readByte(self._f) 220 | isOpen = (opened == 1) 221 | # number of Tags 222 | nTags = readLong(self._f) 223 | if ( debugLevel > 5): 224 | print("rTG: Iterating over the", nTags, "tag entries in this group") 225 | # read Tags 226 | for i in range( nTags ): 227 | self._readTagEntry() 228 | # go back up one level as reading group is finished 229 | self._curGroupLevel += -1 230 | return 1 231 | 232 | def _readTagEntry(self): 233 | # is data or a new group? 234 | data = readByte(self._f) 235 | isData = (data == 21) 236 | self._curTagAtLevelX[self._curGroupLevel] += 1 237 | # get tag label if exists 238 | lenTagLabel = readShort(self._f) 239 | if ( lenTagLabel != 0 ): 240 | tagLabel = readString(self._f, lenTagLabel).decode('latin-1') 241 | else: 242 | tagLabel = str( self._curTagAtLevelX[self._curGroupLevel] ) 243 | if ( debugLevel > 5): 244 | print("{}|{}:".format(self._curGroupLevel, self._makeGroupString()), 245 | end=' ') 246 | print("Tag label = "+tagLabel) 247 | elif ( debugLevel > 1 ): 248 | print(str(self._curGroupLevel)+": Tag label = "+tagLabel) 249 | if isData: 250 | # give it a name 251 | self._curTagName = self._makeGroupNameString()+"."+tagLabel 252 | # read it 253 | self._readTagType() 254 | else: 255 | # it is a tag group 256 | self._curGroupNameAtLevelX[self._curGroupLevel+1] = tagLabel 257 | self._readTagGroup() # increments curGroupLevel 258 | return 1 259 | 260 | def _readTagType(self): 261 | delim = readString(self._f, 4).decode('latin-1') 262 | if ( delim != '%%%%' ): 263 | raise Exception(hex( self._f.tell() ) 264 | + ": Tag Type delimiter not %%%%") 265 | nInTag = readLong(self._f) 266 | self._readAnyData() 267 | return 1 268 | 269 | def _encodedTypeSize(self, eT): 270 | # returns the size in bytes of the data type 271 | if eT == 0: 272 | width = 0 273 | elif eT in (BOOLEAN, CHAR, OCTET): 274 | width = 1 275 | elif eT in (SHORT, USHORT): 276 | width = 2 277 | elif eT in (LONG, ULONG, FLOAT): 278 | width = 4 279 | elif eT == DOUBLE: 280 | width = 8 281 | else: 282 | # returns -1 for unrecognised types 283 | width = -1 284 | return width 285 | 286 | def _readAnyData(self): 287 | ## higher level function dispatching to handling data types 288 | ## to other functions 289 | # - get Type category (short, long, array...) 290 | encodedType = readLong(self._f) 291 | # - calc size of encodedType 292 | etSize = self._encodedTypeSize(encodedType) 293 | if ( debugLevel > 5): 294 | print("rAnD, " + hex( self._f.tell() ) + ":", end=' ') 295 | print("Tag Type = " + str(encodedType) + ",", end=' ') 296 | print("Tag Size = " + str(etSize)) 297 | if ( etSize > 0 ): 298 | self._storeTag( self._curTagName, 299 | self._readNativeData(encodedType, etSize) ) 300 | elif ( encodedType == STRING ): 301 | stringSize = readLong(self._f) 302 | self._readStringData(stringSize) 303 | elif ( encodedType == STRUCT ): 304 | # does not store tags yet 305 | structTypes = self._readStructTypes() 306 | self._readStructData(structTypes) 307 | elif ( encodedType == ARRAY ): 308 | # does not store tags yet 309 | # indicates size of skipped data blocks 310 | arrayTypes = self._readArrayTypes() 311 | self._readArrayData(arrayTypes) 312 | else: 313 | raise Exception("rAnD, " + hex(self._f.tell()) 314 | + ": Can't understand encoded type") 315 | return 1 316 | 317 | def _readNativeData(self, encodedType, etSize): 318 | # reads ordinary data types 319 | if encodedType in readFunc: 320 | val = readFunc[encodedType](self._f) 321 | else: 322 | raise Exception("rND, " + hex(self._f.tell()) 323 | + ": Unknown data type " + str(encodedType)) 324 | if ( debugLevel > 3 ): 325 | print("rND, " + hex(self._f.tell()) + ": " + str(val)) 326 | elif ( debugLevel > 1 ): 327 | print(val) 328 | return val 329 | 330 | def _readStringData(self, stringSize): 331 | # reads string data 332 | if ( stringSize <= 0 ): 333 | rString = "" 334 | else: 335 | if ( debugLevel > 3 ): 336 | print("rSD @ " + str(self._f.tell()) + "/" + hex(self._f.tell()) +" :", end=' ') 337 | rString = readString(self._f, stringSize) 338 | # /!\ UTF-16 unicode string => convert to Python unicode str 339 | rString = rString.decode('utf-16-le') 340 | if ( debugLevel > 3 ): 341 | print(rString + " <" + repr( rString ) + ">") 342 | if ( debugLevel > 1 ): 343 | print("StringVal:", rString) 344 | self._storeTag( self._curTagName, rString ) 345 | return rString 346 | 347 | def _readArrayTypes(self): 348 | # determines the data types in an array data type 349 | arrayType = readLong(self._f) 350 | itemTypes = [] 351 | if ( arrayType == STRUCT ): 352 | itemTypes = self._readStructTypes() 353 | elif ( arrayType == ARRAY ): 354 | itemTypes = self._readArrayTypes() 355 | else: 356 | itemTypes.append( arrayType ) 357 | return itemTypes 358 | 359 | def _readArrayData(self, arrayTypes): 360 | # reads array data 361 | 362 | arraySize = readLong(self._f) 363 | 364 | if ( debugLevel > 3 ): 365 | print("rArD, " + hex( self._f.tell() ) + ":", end=' ') 366 | print("Reading array of size = " + str(arraySize)) 367 | 368 | itemSize = 0 369 | encodedType = 0 370 | 371 | for i in range( len(arrayTypes) ): 372 | encodedType = int( arrayTypes[i] ) 373 | etSize = self._encodedTypeSize(encodedType) 374 | itemSize += etSize 375 | if ( debugLevel > 5 ): 376 | print("rArD: Tag Type = " + str(encodedType) + ",", end=' ') 377 | print("Tag Size = " + str(etSize)) 378 | ##! readNativeData( encodedType, etSize ) !## 379 | 380 | if ( debugLevel > 5 ): 381 | print("rArD: Array Item Size = " + str(itemSize)) 382 | 383 | bufSize = arraySize * itemSize 384 | 385 | if ( (not self._curTagName.endswith("ImageData.Data")) 386 | and ( len(arrayTypes) == 1 ) 387 | and ( encodedType == USHORT ) 388 | and ( arraySize < 256 ) ): 389 | # treat as string 390 | val = self._readStringData( bufSize ) 391 | else: 392 | # treat as binary data 393 | # - store data size and offset as tags 394 | self._storeTag( self._curTagName + ".Size", bufSize ) 395 | self._storeTag( self._curTagName + ".Offset", self._f.tell() ) 396 | # - skip data w/o reading 397 | self._f.seek( self._f.tell() + bufSize ) 398 | 399 | return 1 400 | 401 | def _readStructTypes(self): 402 | # analyses data types in a struct 403 | 404 | if ( debugLevel > 3 ): 405 | print("Reading Struct Types at Pos = " + hex(self._f.tell())) 406 | 407 | structNameLength = readLong(self._f) 408 | nFields = readLong(self._f) 409 | 410 | if ( debugLevel > 5 ): 411 | print("nFields = ", nFields) 412 | 413 | if ( nFields > 100 ): 414 | raise Exception(hex(self._f.tell())+": Too many fields") 415 | 416 | fieldTypes = [] 417 | nameLength = 0 418 | for i in range( nFields ): 419 | nameLength = readLong(self._f) 420 | if ( debugLevel > 9 ): 421 | print("{}th nameLength = {}".format(i, nameLength)) 422 | fieldType = readLong(self._f) 423 | fieldTypes.append( fieldType ) 424 | 425 | return fieldTypes 426 | 427 | def _readStructData(self, structTypes): 428 | # reads struct data based on type info in structType 429 | for i in range( len(structTypes) ): 430 | encodedType = structTypes[i] 431 | etSize = self._encodedTypeSize(encodedType) 432 | 433 | if ( debugLevel > 5 ): 434 | print("Tag Type = " + str(encodedType) + ",", end=' ') 435 | print("Tag Size = " + str(etSize)) 436 | 437 | # get data 438 | self._readNativeData(encodedType, etSize) 439 | 440 | return 1 441 | 442 | def _storeTag(self, tagName, tagValue): 443 | # store Tags as list and dict 444 | # NB: all tag values (and names) stored as unicode objects; 445 | # => can then be easily converted to any encoding 446 | if ( debugLevel == 1 ): 447 | print(" - storing Tag:") 448 | print(" -- name: ", tagName) 449 | print(" -- value: ", tagValue, type(tagValue)) 450 | # - convert tag value to unicode if not already unicode object 451 | self._storedTags.append( tagName + " = " + unicode_str(tagValue) ) 452 | self._tagDict[tagName] = unicode_str(tagValue) 453 | 454 | ### END utility functions ### 455 | 456 | def __init__(self, filename, debug=0): 457 | """DM3 object: parses DM3 file.""" 458 | 459 | ## initialize variables ## 460 | self._debug = debug 461 | self._outputcharset = DEFAULTCHARSET 462 | self._filename = filename 463 | self._chosenImage = 1 464 | # - track currently read group 465 | self._curGroupLevel = -1 466 | self._curGroupAtLevelX = [ 0 for x in range(MAXDEPTH) ] 467 | self._curGroupNameAtLevelX = [ '' for x in range(MAXDEPTH) ] 468 | # - track current tag 469 | self._curTagAtLevelX = [ '' for x in range(MAXDEPTH) ] 470 | self._curTagName = '' 471 | # - open file for reading 472 | self._f = open( self._filename, 'rb' ) 473 | # - create Tags repositories 474 | self._storedTags = [] 475 | self._tagDict = {} 476 | 477 | isDM3 = True 478 | ## read header (first 3 4-byte int) 479 | # get version 480 | fileVersion = readLong(self._f) 481 | if ( fileVersion != 3 ): 482 | isDM3 = False 483 | # get indicated file size 484 | fileSize = readLong(self._f) 485 | # get byte-ordering 486 | lE = readLong(self._f) 487 | littleEndian = (lE == 1) 488 | if not littleEndian: 489 | isDM3 = False 490 | # check file header, raise Exception if not DM3 491 | if not isDM3: 492 | raise Exception("%s does not appear to be a DM3 file." 493 | % os.path.split(self._filename)[1]) 494 | elif self._debug > 0: 495 | print("%s appears to be a DM3 file" % (self._filename)) 496 | 497 | if ( debugLevel > 5 or self._debug > 1): 498 | print("Header info.:") 499 | print("- file version:", fileVersion) 500 | print("- lE:", lE) 501 | print("- file size:", fileSize, "bytes") 502 | 503 | # set name of root group (contains all data)... 504 | self._curGroupNameAtLevelX[0] = "root" 505 | # ... then read it 506 | self._readTagGroup() 507 | if self._debug > 0: 508 | print("-- %s Tags read --" % len(self._storedTags)) 509 | 510 | # fetch image characteristics 511 | tag_root = 'root.ImageList.1' 512 | self._data_type = int( self.tags["%s.ImageData.DataType" % tag_root] ) 513 | self._im_width = int( self.tags["%s.ImageData.Dimensions.0" % tag_root] ) 514 | self._im_height = int( self.tags["%s.ImageData.Dimensions.1" % tag_root] ) 515 | try: 516 | self._im_depth = int( self.tags['root.ImageList.1.ImageData.Dimensions.2'] ) 517 | except KeyError: 518 | self._im_depth = 1 519 | 520 | if self._debug > 0: 521 | print("Notice: image size: %sx%s px" % (self._im_width, self._im_height)) 522 | if self._im_depth>1: 523 | print("Notice: %s image stack" % (self._im_depth)) 524 | 525 | @property 526 | def data_type(self): 527 | """Returns image DataType.""" 528 | return self._data_type 529 | 530 | @property 531 | def data_type_str(self): 532 | """Returns image DataType string.""" 533 | return dataTypes[self._data_type] 534 | 535 | @property 536 | def width(self): 537 | """Returns image width (px).""" 538 | return self._im_width 539 | 540 | @property 541 | def height(self): 542 | """Returns image height (px).""" 543 | return self._im_height 544 | 545 | @property 546 | def depth(self): 547 | """Returns image depth (i.e. number of images in stack).""" 548 | return self._im_depth 549 | 550 | @property 551 | def size(self): 552 | """Returns image size (width,height[,depth]).""" 553 | if self._im_depth > 1: 554 | return (self._im_width, self._im_height, self._im_depth) 555 | else: 556 | return (self._im_width, self._im_height) 557 | 558 | @property 559 | def outputcharset(self): 560 | """Returns Tag dump/output charset.""" 561 | return self._outputcharset 562 | 563 | @outputcharset.setter 564 | def outputcharset(self, value): 565 | """Set Tag dump/output charset.""" 566 | self._outputcharset = value 567 | 568 | @property 569 | def filename(self): 570 | """Returns full file path.""" 571 | return self._filename 572 | 573 | @property 574 | def tags(self): 575 | """Returns all image Tags.""" 576 | return self._tagDict 577 | 578 | def dumpTags(self, dump_dir='/tmp'): 579 | """Dumps image Tags in a txt file.""" 580 | dump_file = os.path.join(dump_dir, 581 | os.path.split(self._filename)[1] 582 | + ".tagdump.txt") 583 | try: 584 | dumpf = open( dump_file, 'w' ) 585 | except: 586 | print("Warning: cannot generate dump file.") 587 | else: 588 | for tag in self._storedTags: 589 | dumpf.write( "{}\n".format(tag.encode(self._outputcharset))) 590 | dumpf.close 591 | 592 | @property 593 | def info(self): 594 | """Extracts useful experiment info from DM3 file.""" 595 | # define useful information 596 | tag_root = 'root.ImageList.1' 597 | info_keys = { 598 | 'descrip': "%s.Description" % tag_root, 599 | 'acq_date': "%s.ImageTags.DataBar.Acquisition Date" % tag_root, 600 | 'acq_time': "%s.ImageTags.DataBar.Acquisition Time" % tag_root, 601 | 'name': "%s.ImageTags.Microscope Info.Name" % tag_root, 602 | 'micro': "%s.ImageTags.Microscope Info.Microscope" % tag_root, 603 | 'hv': "%s.ImageTags.Microscope Info.Voltage" % tag_root, 604 | 'mag': "%s.ImageTags.Microscope Info.Indicated Magnification" % tag_root, 605 | 'mode': "%s.ImageTags.Microscope Info.Operation Mode" % tag_root, 606 | 'operator': "%s.ImageTags.Microscope Info.Operator" % tag_root, 607 | 'specimen': "%s.ImageTags.Microscope Info.Specimen" % tag_root, 608 | # 'image_notes': "root.DocumentObjectList.10.Text' # = Image Notes 609 | } 610 | # get experiment information 611 | infoDict = {} 612 | for key, tag_name in info_keys.items(): 613 | if tag_name in self.tags: 614 | # tags supplied as Python unicode str; convert to chosen charset 615 | # (typically latin-1 or utf-8) 616 | infoDict[key] = self.tags[tag_name].encode(self._outputcharset) 617 | # return experiment information 618 | return infoDict 619 | 620 | @property 621 | def imagedata(self): 622 | """Extracts image data as numpy.array""" 623 | 624 | # numpy dtype strings associated to the various image dataTypes 625 | dT_str = { 626 | 1: ' 0: 646 | print("Notice: image data in %s starts at %s" % ( 647 | os.path.split(self._filename)[1], hex(data_offset) 648 | )) 649 | 650 | # check if image DataType is implemented, then read 651 | if data_type in dT_str: 652 | np_dt = numpy.dtype( dT_str[data_type] ) 653 | if self._debug > 0: 654 | print("Notice: image data type: %s ('%s'), read as %s" % ( 655 | data_type, dataTypes[data_type], np_dt 656 | )) 657 | self._f.seek( data_offset ) 658 | # - fetch image data 659 | rawdata = self._f.read(data_size) 660 | # - convert raw to numpy array w/ correct dtype 661 | ima = numpy.fromstring(rawdata, dtype=np_dt) 662 | # - reshape to matrix or stack 663 | if im_depth > 1: 664 | ima = ima.reshape(im_depth, im_height, im_width) 665 | else: 666 | ima = ima.reshape(im_height, im_width) 667 | else: 668 | raise Exception( 669 | "Cannot extract image data from %s: unimplemented DataType (%s:%s)." % 670 | (os.path.split(self._filename)[1], data_type, dataTypes[data_type]) 671 | ) 672 | 673 | # if image dataType is BINARY, binarize image 674 | # (i.e., px_value>0 is True) 675 | if data_type == 14: 676 | ima[ima>0] = 1 677 | 678 | return ima 679 | 680 | 681 | @property 682 | def Image(self): 683 | """Returns image data as PIL Image""" 684 | 685 | # define PIL Image mode for the various (supported) image dataTypes, 686 | # among: 687 | # - '1': 1-bit pixels, black and white, stored as 8-bit pixels 688 | # - 'L': 8-bit pixels, gray levels 689 | # - 'I': 32-bit integer pixels 690 | # - 'F': 32-bit floating point pixels 691 | dT_modes = { 692 | 1: 'I', # 16-bit LE signed integer 693 | 2: 'F', # 32-bit LE floating point 694 | 6: 'L', # 8-bit unsigned integer 695 | 7: 'I', # 32-bit LE signed integer 696 | 9: 'I', # 8-bit signed integer 697 | 10: 'I', # 16-bit LE unsigned integer 698 | 11: 'I', # 32-bit LE unsigned integer 699 | 14: 'L', # "binary" 700 | } 701 | 702 | # define loaded array dtype if has to be fixed to match Image mode 703 | dT_newdtypes = { 704 | 1: 'int32', # 16-bit LE integer to 32-bit int 705 | 2: 'float32', # 32-bit LE float to 32-bit float 706 | 9: 'int32', # 8-bit signed integer to 32-bit int 707 | 10: 'int32', # 16-bit LE u. integer to 32-bit int 708 | } 709 | 710 | # get relevant Tags 711 | data_type = self._data_type 712 | im_width = self._im_width 713 | im_height = self._im_height 714 | im_depth = self._im_depth 715 | 716 | # fetch image data array 717 | ima = self.imagedata 718 | # assign Image mode 719 | mode_ = dT_modes[data_type] 720 | 721 | # reshape array if image stack 722 | if im_depth > 1: 723 | ima = ima.reshape(im_height*im_depth, im_width) 724 | 725 | # load image data array into Image object (recast array if necessary) 726 | if data_type in dT_newdtypes: 727 | im = Image.fromarray(ima.astype(dT_newdtypes[data_type]),mode_) 728 | else: 729 | im = Image.fromarray(ima,mode_) 730 | 731 | return im 732 | 733 | 734 | @property 735 | def contrastlimits(self): 736 | """Returns display range (cuts).""" 737 | tag_root = 'root.DocumentObjectList.0' 738 | low = int(float(self.tags["%s.ImageDisplayInfo.LowLimit" % tag_root])) 739 | high = int(float(self.tags["%s.ImageDisplayInfo.HighLimit" % tag_root])) 740 | cuts = (low, high) 741 | return cuts 742 | 743 | @property 744 | def cuts(self): 745 | """Returns display range (cuts).""" 746 | return self.contrastlimits 747 | 748 | @property 749 | def pxsize(self): 750 | """Returns pixel size and unit.""" 751 | tag_root = 'root.ImageList.1' 752 | pixel_size = float( 753 | self.tags["%s.ImageData.Calibrations.Dimension.0.Scale" % tag_root]) 754 | unit = self.tags["%s.ImageData.Calibrations.Dimension.0.Units" % 755 | tag_root] 756 | if unit == u'\xb5m': 757 | unit = 'micron' 758 | else: 759 | unit = unit.encode('ascii') 760 | if self._debug > 0: 761 | print("pixel size = %s %s" % (pixel_size, unit)) 762 | return (pixel_size, unit) 763 | 764 | 765 | @property 766 | def tnImage(self): 767 | """Returns thumbnail as PIL Image.""" 768 | # get thumbnail 769 | tag_root = 'root.ImageList.0' 770 | tn_size = int( self.tags["%s.ImageData.Data.Size" % tag_root] ) 771 | tn_offset = int( self.tags["%s.ImageData.Data.Offset" % tag_root] ) 772 | tn_width = int( self.tags["%s.ImageData.Dimensions.0" % tag_root] ) 773 | tn_height = int( self.tags["%s.ImageData.Dimensions.1" % tag_root] ) 774 | 775 | if self._debug > 0: 776 | print("Notice: tn data in %s starts at %s" % ( 777 | os.path.split(self._filename)[1], hex(tn_offset) 778 | )) 779 | print("Notice: tn size: %sx%s px" % (tn_width, tn_height)) 780 | 781 | if (tn_width*tn_height*4) != tn_size: 782 | raise Exception("Cannot extract thumbnail from %s" 783 | % os.path.split(self._filename)[1]) 784 | else: 785 | self._f.seek( tn_offset ) 786 | rawdata = self._f.read(tn_size) 787 | # - read as 32-bit LE unsigned integer 788 | tn = Image.frombytes( 'F', (tn_width, tn_height), rawdata, 789 | 'raw', 'F;32' ) 790 | # - rescale and convert px data 791 | tn = tn.point(lambda x: x * (1./65536) + 0) 792 | tn = tn.convert('L') 793 | # - return image 794 | return tn 795 | 796 | @property 797 | def thumbnaildata(self): 798 | """Fetch thumbnail image data as numpy.array""" 799 | 800 | # get useful thumbnail Tags 801 | tag_root = 'root.ImageList.0' 802 | tn_size = int( self.tags["%s.ImageData.Data.Size" % tag_root] ) 803 | tn_offset = int( self.tags["%s.ImageData.Data.Offset" % tag_root] ) 804 | tn_width = int( self.tags["%s.ImageData.Dimensions.0" % tag_root] ) 805 | tn_height = int( self.tags["%s.ImageData.Dimensions.1" % tag_root] ) 806 | 807 | if self._debug > 0: 808 | print("Notice: tn data in %s starts at %s" % ( 809 | os.path.split(self._filename)[1], hex(tn_offset) 810 | )) 811 | print("Notice: tn size: %sx%s px" % (tn_width, tn_height)) 812 | 813 | # get thumbnail data 814 | if (tn_width*tn_height*4) == tn_size: 815 | self._f.seek(tn_offset) 816 | rawtndata = self._f.read(tn_size) 817 | print('## rawdata:', len(rawtndata)) 818 | # - read as 32-bit LE unsigned integer 819 | np_dt_tn = numpy.dtype(' 0: 847 | print("Thumbnail saved as '%s'." % tn_path) 848 | except: 849 | print("Warning: could not save thumbnail.") 850 | 851 | 852 | ## MAIN ## 853 | if __name__ == '__main__': 854 | print("dm3_lib %s" % VERSION) 855 | 856 | -------------------------------------------------------------------------------- /dm3_lib/_dm3_lib.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SCIInstitute/DiffractionClassification/68be6cf3960f09388253c79bab13cbd9dc07edbb/dm3_lib/_dm3_lib.pyc -------------------------------------------------------------------------------- /dm3_lib/demo/demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | from __future__ import print_function, division 4 | 5 | import os.path 6 | import argparse 7 | 8 | import numpy as np 9 | import matplotlib.pyplot as plt 10 | 11 | from PIL import Image 12 | 13 | import dm3_lib as dm3 14 | 15 | from utilities import calcHistogram, calcDisplayRange 16 | 17 | # CONSTANTS 18 | 19 | savedir = os.path.expanduser("~/Desktop") 20 | debug = 0 21 | 22 | # define command line arguments 23 | parser = argparse.ArgumentParser() 24 | 25 | parser.add_argument("file", help="path to DM3 file to parse") 26 | parser.add_argument("-v", "--verbose", help="increase output verbosity", 27 | action="store_true") 28 | parser.add_argument("--dump", help="dump DM3 tags in text file", 29 | action="store_true") 30 | parser.add_argument("--convert", help="save image in various formats", 31 | action="store_true") 32 | 33 | # parse command line arguments 34 | args = parser.parse_args() 35 | if args.verbose: 36 | debug = 1 37 | filepath = args.file 38 | 39 | # get filename 40 | filename = os.path.split(filepath)[1] 41 | fileref = os.path.splitext(filename)[0] 42 | 43 | # pyplot interactive mode 44 | plt.ion() 45 | plt.close('all') 46 | 47 | # parse DM3 file 48 | dm3f = dm3.DM3(filepath, debug=debug) 49 | 50 | # get some useful tag data and print 51 | print("file:", dm3f.filename) 52 | print("file info.:") 53 | print(dm3f.info) 54 | print("scale: %.3g %s/px"%dm3f.pxsize) 55 | cuts = dm3f.cuts 56 | print("cuts:",cuts) 57 | 58 | # dump image Tags in txt file 59 | if args.dump: 60 | dm3f.dumpTags(savedir) 61 | 62 | # get image data 63 | aa = dm3f.imagedata 64 | 65 | # display image 66 | # - w/o cuts 67 | if args.verbose: 68 | plt.matshow(aa, cmap=plt.cm.pink) 69 | plt.title("%s (w/o cuts)"%filename) 70 | plt.colorbar(shrink=.8) 71 | # - w/ cuts (if cut values different) 72 | if cuts[0] != cuts[1]: 73 | plt.matshow(aa, cmap=plt.cm.pink, vmin=cuts[0], vmax=cuts[1]) 74 | plt.title("%s"%filename) 75 | plt.colorbar(shrink=.8) 76 | 77 | # - display image histogram 78 | if args.verbose: 79 | hh,bb = calcHistogram(aa) 80 | plt.figure('Image histogram') 81 | plt.plot(bb[:-1],hh,drawstyle='steps') 82 | plt.xlim(bb[0],bb[-1]) 83 | plt.xlabel('Intensity') 84 | plt.ylabel('Number') 85 | 86 | # convert image to various formats 87 | if args.convert: 88 | # save image as TIFF 89 | tif_file = os.path.join(savedir,fileref+'.tif') 90 | im = Image.fromarray(aa) 91 | im.save(tif_file) 92 | # check TIFF dynamic range 93 | tim = Image.open(tif_file) 94 | if tim.mode == 'L': 95 | tif_range = "8-bit" 96 | else: 97 | tif_range = "32-bit" 98 | print("Image saved as %s TIFF."%tif_range) 99 | 100 | # save image as PNG and JPG files 101 | # - normalize image for conversion to 8-bit 102 | aa_norm = aa.copy() 103 | # -- apply cuts (optional) 104 | if cuts[0] != cuts[1]: 105 | aa_norm[ (aa <= min(cuts)) ] = float(min(cuts)) 106 | aa_norm[ (aa >= max(cuts)) ] = float(max(cuts)) 107 | # -- normalize 108 | aa_norm = (aa_norm - np.min(aa_norm)) / (np.max(aa_norm) - np.min(aa_norm)) 109 | # -- scale to 0--255, convert to (8-bit) integer 110 | aa_norm = np.uint8(np.round( aa_norm * 255 )) 111 | 112 | if args.verbose: 113 | # - display normalized image 114 | plt.matshow(aa_norm, cmap=plt.cm.Greys_r) 115 | plt.title("%s [8-bit display]"%filename) 116 | plt.colorbar(shrink=.8) 117 | 118 | # - save as PNG and JPG 119 | im_dsp = Image.fromarray(aa_norm) 120 | im_dsp.save(os.path.join(savedir,fileref+'.png')) 121 | im_dsp.save(os.path.join(savedir,fileref+'.jpg')) 122 | 123 | -------------------------------------------------------------------------------- /dm3_lib/demo/utilities.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import numpy as np 4 | 5 | # histogram, re-compute cuts 6 | 7 | def calcHistogram(imdata, bins_=256): 8 | '''Compute image histogram.''' 9 | im_values = np.ravel(imdata) 10 | hh, bins_ = np.histogram( im_values, bins=bins_ ) 11 | return hh, bins_ 12 | 13 | def calcDisplayRange(imdata, cutoff=.1, bins_=512): 14 | '''Compute display range, i.e., cuts. 15 | (ignore the 'cutoff'% lowest/highest value pixels)''' 16 | # compute image histogram 17 | hh, bins_ = calcHistogram(imdata, bins_) 18 | # check histogram format 19 | if len(bins_)==len(hh): 20 | bb = bins_ 21 | else: 22 | bb = bins_[:-1] # 'bins' == bin_edges 23 | # number of pixels 24 | Npx = np.sum(hh) 25 | # calc. lower limit : 26 | i = 1 27 | while np.sum( hh[:i] ) < Npx*cutoff/100.: 28 | i += 1 29 | cut0 = round( bb[i] ) 30 | # calc. higher limit 31 | j = 1 32 | while np.sum( hh[-j:] ) < Npx*cutoff/100.: 33 | j += 1 34 | cut1 = round( bb[-j] ) 35 | return cut0,cut1 36 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | ######### Requirements without specific versions 3 | 4 | requests 5 | numpy 6 | matplotlib 7 | 8 | -------------------------------------------------------------------------------- /server.json: -------------------------------------------------------------------------------- 1 | { 2 | "URL" : "" 3 | } 4 | -------------------------------------------------------------------------------- /user_profile.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"John Doe", 3 | "organization": "GenericPlaceOfBusiness", 4 | "key": "lorem ipsun" 5 | } 6 | --------------------------------------------------------------------------------