├── models ├── __init__.py └── bilstm.py ├── .gitignore ├── imgs ├── labels_test.jpg ├── ground_train.jpg ├── height_train.jpg ├── normals_train.jpg ├── regions_train.jpg ├── confusion_matrix.png ├── intensity_train.jpg ├── planarity_train.jpg ├── predictions_test.jpg ├── sphericity_train.jpg └── ground_rasterized_train.jpg ├── requirements.txt ├── utils ├── pts.py ├── dataloader.py └── ply.py ├── cfg ├── config_bilstm.yaml └── config_features_extraction.yaml ├── fetch_data.sh ├── preprocessing.py ├── features ├── region_growing.py ├── descriptors.py └── ground_extraction.py ├── README.md ├── test.py ├── compute_features.py └── train.py /models/__init__.py: -------------------------------------------------------------------------------- 1 | from .bilstm import BiLSTM 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | data/ 3 | ckpts/ 4 | **/__pycache__/ 5 | -------------------------------------------------------------------------------- /imgs/labels_test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/labels_test.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | numpy 3 | tqdm 4 | torch>=1.3.1 5 | scipy 6 | scikit_learn 7 | pandas 8 | -------------------------------------------------------------------------------- /imgs/ground_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/ground_train.jpg -------------------------------------------------------------------------------- /imgs/height_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/height_train.jpg -------------------------------------------------------------------------------- /imgs/normals_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/normals_train.jpg -------------------------------------------------------------------------------- /imgs/regions_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/regions_train.jpg -------------------------------------------------------------------------------- /imgs/confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/confusion_matrix.png -------------------------------------------------------------------------------- /imgs/intensity_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/intensity_train.jpg -------------------------------------------------------------------------------- /imgs/planarity_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/planarity_train.jpg -------------------------------------------------------------------------------- /imgs/predictions_test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/predictions_test.jpg -------------------------------------------------------------------------------- /imgs/sphericity_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/sphericity_train.jpg -------------------------------------------------------------------------------- /imgs/ground_rasterized_train.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/theobdt/aerial_pc_classification/HEAD/imgs/ground_rasterized_train.jpg -------------------------------------------------------------------------------- /utils/pts.py: -------------------------------------------------------------------------------- 1 | from numpy import genfromtxt, savetxt 2 | 3 | 4 | def read_pts(filename, delimiter=' '): 5 | data = genfromtxt(filename, delimiter=delimiter) 6 | return data 7 | 8 | 9 | def write_pts(data, filename, delimiter=' '): 10 | savetxt(filename, data, fmt='%.2f') 11 | -------------------------------------------------------------------------------- /cfg/config_bilstm.yaml: -------------------------------------------------------------------------------- 1 | network: 2 | hidden_size: 200 3 | num_layers: 1 4 | relation_type: 2 5 | training: 6 | batch_size: 1000 7 | max_batches: 50 # max number of batches per epoch 8 | epoch_end: 50 9 | num_workers: 2 10 | shuffle: True 11 | test: 12 | batch_size: 1000 13 | max_batches: 50 # max number of batches per epoch 14 | num_workers: 2 15 | shuffle: True 16 | optimizer: 17 | learning_rate: 0.001 18 | data: 19 | features: ['height_above_ground', 'planarity', 'sphericity', 'verticality', 'anisotropy', 'surface_variation', 'intensity'] 20 | n_neighbors: 60 21 | all_labels: False 22 | -------------------------------------------------------------------------------- /fetch_data.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | echo "To download data, please request access at http://www2.isprs.org/commissions/comm3/wg4/detection-and-reconstruction.html (it is totally free).\n" 3 | echo -n "Login: "; read login 4 | echo -n "Password: "; stty -echo; read passwd; stty echo; echo 5 | 6 | URL_TRAIN="ftp://ftp.ipi.uni-hannover.de/ISPRS_BENCHMARK_DATASETS/Vaihingen/3DLabeling/Vaihingen3D_Traininig.pts" 7 | URL_TEST="ftp://ftp.ipi.uni-hannover.de/ISPRS_BENCHMARK_DATASETS/Vaihingen/3DLabeling/Vaihingen3D_EVAL_WITH_REF.pts.gz" 8 | PATH_TRAIN_DL="data/vaihingen3D_train.pts" 9 | PATH_TEST_DL="data/vaihingen3D_test.pts" 10 | 11 | mkdir -p data 12 | wget --user="$login" --password="$passwd" "$URL_TRAIN" -O "$PATH_TRAIN_DL" && \ 13 | echo "Downloaded training data to $PATH_TRAIN_DL" 14 | wget --user="$login" --password="$passwd" "$URL_TEST" -O - | gzip -cd > "$PATH_TEST_DL" && \ 15 | echo "Downloaded test data to $PATH_TEST_DL" 16 | -------------------------------------------------------------------------------- /cfg/config_features_extraction.yaml: -------------------------------------------------------------------------------- 1 | # Local descriptors available: 2 | # ["normals", "verticality", "linearity", "planarity", "sphericity", "curvature"] 3 | descriptors: 4 | descriptors: ["all"] 5 | radius: 2 6 | preferred_orientation: "+z" # normals orientation 7 | epsilon: 0.01 # Epsilon added to denominator for some descriptors 8 | 9 | region_growing: 10 | n_regions: 200 # max number of regions to grow 11 | radius: 1 # radius used to select candidate points from queue points 12 | minimize: False # maximize(False) or minimize(True) the following descriptor 13 | descriptor: "planarity" # descriptor to maximize/minimize 14 | thresholds: 15 | height: 0.1 # max diff of height between queue points and candidate points 16 | angle: 0.1 # max diff of normals angle (rads) for candidate points 17 | descriptor: 0.1 # max/min value of [descriptor] in candidate points 18 | 19 | ground_extraction: 20 | slope_intra_max: 0.1 # Maximum internal relative slope for candidate regions (diff_height / span) 21 | slope_inter_max: 0.2 # Maximum relative slope between current ground and candidate regions (avf diff_height / avg distance) 22 | percentile_closest: 0.01 # Percentile of closest points in candidate region used to compute slope_inter 23 | 24 | ground_rasterization: 25 | method: "delaunay" # "closest_neighbor" or "delaunay" 26 | step_size: 0.5 # step size of the grid used to rasterize 27 | 28 | height_above_ground: 29 | -------------------------------------------------------------------------------- /preprocessing.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import argparse 4 | 5 | from utils.pts import read_pts 6 | from utils.ply import dict2ply 7 | 8 | PATH_PREPROCESSED = "data/preprocessed" 9 | 10 | 11 | def center(coords): 12 | means = np.mean(coords, axis=0) 13 | centered = coords - means 14 | return centered 15 | 16 | 17 | def preprocess_pts(path, centering, scale): 18 | print(f"Reading points from {path}") 19 | data_pts = read_pts(path) 20 | 21 | # center/scale point cloud 22 | coords = data_pts[:, :3] 23 | features = data_pts[:, 3:-1].astype(np.uint8) 24 | labels = data_pts[:, -1].astype(np.uint8) 25 | 26 | if centering: 27 | coords = center(coords) 28 | coords = (coords * scale).astype(np.float32) 29 | data_ply = { 30 | "x": coords[:, 0], 31 | "y": coords[:, 1], 32 | "z": coords[:, 2], 33 | "intensity": features[:, 0], 34 | "return_number": features[:, 1], 35 | "number_of_returns": features[:, 2], 36 | "labels": labels, 37 | } 38 | 39 | # save preprocessed point cloud 40 | os.makedirs(PATH_PREPROCESSED, exist_ok=True) 41 | filename = os.path.split(path)[-1].split(".")[-2] 42 | path_ply = os.path.join(PATH_PREPROCESSED, filename + ".ply") 43 | if dict2ply(data_ply, path_ply): 44 | print(f"PLY point cloud successfully saved to {path_ply}") 45 | 46 | 47 | if __name__ == "__main__": 48 | parser = argparse.ArgumentParser( 49 | description="Center and rescale point cloud" 50 | ) 51 | parser.add_argument( 52 | "--files", "-f", type=str, nargs="+", help="Path to point cloud file" 53 | ) 54 | parser.add_argument( 55 | "--scale", "-s", type=float, default=1, help="Scale factor" 56 | ) 57 | parser.add_argument( 58 | "--centering", 59 | "-c", 60 | action="store_true", 61 | help="Recenter point cloud coordinates", 62 | ) 63 | args = parser.parse_args() 64 | # Path of the file 65 | 66 | # Load point cloud 67 | print(f"Centering : {args.centering}") 68 | print(f"Scale factor : {args.scale}") 69 | for path in args.files: 70 | preprocess_pts(path, args.centering, args.scale) 71 | -------------------------------------------------------------------------------- /utils/dataloader.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data import Dataset 3 | import numpy as np 4 | 5 | from utils.ply import ply2dict 6 | from sklearn.neighbors import KDTree 7 | 8 | 9 | class AerialPointDataset(Dataset): 10 | def __init__(self, input_file, features, n_neighbors, all_labels=False): 11 | "Initialization" 12 | data = ply2dict(input_file) 13 | try: 14 | all_features = ["x", "y", "z"] + features 15 | X = np.vstack([data[f] for f in all_features]).T 16 | except KeyError: 17 | print(f"ERROR: Input features {features} not recognized") 18 | return 19 | labels = data["labels"] 20 | 21 | self.index = np.arange(X.shape[0]) 22 | if not all_labels: 23 | X, labels = self.filter_labels(X, labels) 24 | 25 | self.X = torch.from_numpy(X) 26 | self.labels = torch.from_numpy(labels) 27 | self.n_samples = self.labels.shape[0] 28 | tree = KDTree(self.X[:, :3]) 29 | _, self.neighbors_idx = tree.query( 30 | self.X[:, :3], k=n_neighbors, sort_results=True 31 | ) 32 | 33 | def filter_labels(self, X, labels): 34 | new_labels = convert_labels(labels) 35 | mask = new_labels >= 0 36 | self.index = self.index[mask] 37 | return X[mask], new_labels[mask] 38 | 39 | def __getitem__(self, index): 40 | point = self.X[index].view(1, -1) 41 | neighbors = self.X[self.neighbors_idx[index]] 42 | sequence = torch.cat((point, neighbors), 0) 43 | 44 | return sequence, self.labels[index] 45 | 46 | def __len__(self): 47 | return self.n_samples 48 | 49 | 50 | def convert_labels(labels): 51 | """Convert 9-labels to 4-labels as follows: 52 | 0 Powerline -> -1 Other 53 | 1 Low vegetation -> 0 GLO 54 | 2 Impervious surfaces -> 0 GLO 55 | 3 Car -> -1 Other 56 | 4 Fence/Hedge -> -1 Other 57 | 5 Roof -> 1 Roof 58 | 6 Facade -> 2 Facade 59 | 7 Shrub -> 3 Vegetation 60 | 8 Tree -> 3 Vegetation 61 | """ 62 | LABELS_MAP = {0: -1, 1: 0, 2: 0, 3: -1, 4: -1, 5: 1, 6: 2, 7: 3, 8: 3} 63 | return np.vectorize(LABELS_MAP.get)(labels) 64 | -------------------------------------------------------------------------------- /features/region_growing.py: -------------------------------------------------------------------------------- 1 | from sklearn.neighbors import KDTree 2 | import numpy as np 3 | 4 | 5 | def geometry_criterion(p1, p2, n1, n2, thresh_height, thresh_angle): 6 | 7 | criterion_height = (p2 - p1) @ n1 < thresh_height 8 | 9 | # dot product 10 | criterion_angle = np.abs(n2 @ n1) > np.cos(thresh_angle) 11 | # abs to avoid issues with normals in different directions 12 | 13 | return np.logical_and(criterion_height, criterion_angle) 14 | 15 | 16 | def descriptor_criterion(descriptor, minimize, thresh_descriptor): 17 | if minimize: 18 | return descriptor < thresh_descriptor 19 | return descriptor > thresh_descriptor 20 | 21 | 22 | def region_growing( 23 | coords, normals, radius, descriptor_vals, minimize, thresholds 24 | ): 25 | 26 | N = len(coords) 27 | tree = KDTree(coords) 28 | region_mask = np.zeros(N, dtype=bool) 29 | 30 | Q_idx = [ 31 | np.argmin(descriptor_vals) if minimize else np.argmax(descriptor_vals) 32 | ] 33 | seen_idx = np.zeros(N, dtype=bool) 34 | i = 0 35 | 36 | while len(Q_idx) > 0: 37 | print(f" * N processed neighborhoods : {i}", end="\r") 38 | i += 1 39 | seed_idx = Q_idx.pop(0) 40 | 41 | seed_coords = coords[seed_idx] 42 | seed_normal = normals[seed_idx] 43 | 44 | neighbors_idx = tree.query_radius(seed_coords.reshape(1, -1), radius) 45 | neighbors_idx = neighbors_idx[0] 46 | 47 | # discard neighbors that have already been processed 48 | neighbors_idx = neighbors_idx[~seen_idx[neighbors_idx]] 49 | 50 | neighbors_points = coords[neighbors_idx] 51 | neighbors_normals = normals[neighbors_idx] 52 | 53 | # select neighbors 54 | geometry_mask = geometry_criterion( 55 | seed_coords, 56 | neighbors_points, 57 | seed_normal, 58 | neighbors_normals, 59 | thresholds["height"], 60 | thresholds["angle"], 61 | ) 62 | selected_idx = neighbors_idx[geometry_mask] 63 | 64 | # add them to the region 65 | region_mask[selected_idx] = 1 66 | 67 | # add some of the to the queue 68 | selected_planarities = descriptor_vals[selected_idx] 69 | descriptor_mask = descriptor_criterion( 70 | selected_planarities, minimize, thresholds["descriptor"] 71 | ) 72 | 73 | queue_idx = selected_idx[descriptor_mask] 74 | 75 | # add processed indexes to the seen array 76 | seen_idx[neighbors_idx] = 1 77 | 78 | Q_idx += list(queue_idx) 79 | if i > 500000: 80 | print("Region growing stopped early : n_points > 500.000") 81 | break 82 | 83 | print(f" * Total number of points in region : {np.sum(region_mask)}") 84 | 85 | return region_mask 86 | 87 | 88 | def multi_region_growing( 89 | coords, normals, descriptor_vals, radius, n_regions, minimize, thresholds 90 | ): 91 | 92 | N = len(coords) 93 | is_region = np.zeros(N, dtype=bool) 94 | region_labels = -np.ones(N, dtype=np.int32) 95 | indexes = np.arange(N) 96 | 97 | for i in range(n_regions): 98 | print(f"* Region {i + 1}/{n_regions}") 99 | label_i = i + 1 100 | region_mask = region_growing( 101 | coords, normals, radius, descriptor_vals, minimize, thresholds 102 | ) 103 | 104 | idx_region = indexes[region_mask] 105 | region_labels[idx_region] = label_i 106 | is_region[idx_region] = 1 107 | 108 | coords = coords[~region_mask] 109 | normals = normals[~region_mask] 110 | descriptor_vals = descriptor_vals[~region_mask] 111 | 112 | indexes = indexes[~region_mask] 113 | n_regions_grown = len(np.unique(region_labels) - 1) 114 | print(f"* Number of valid regions grown : {n_regions_grown}") 115 | 116 | return region_labels 117 | -------------------------------------------------------------------------------- /models/bilstm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | 5 | SIZE_RELATION_TYPE = {0: 0, 1: 1, 2: 3, 3: 4, 4: 3} 6 | 7 | 8 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 9 | 10 | 11 | class BiLSTM(nn.Module): 12 | def __init__( 13 | self, n_features, num_classes, hidden_size, num_layers, relation_type=1 14 | ): 15 | super(BiLSTM, self).__init__() 16 | self.relation_type = relation_type 17 | 18 | self.hidden_size = hidden_size 19 | self.num_layers = num_layers 20 | 21 | try: 22 | size_relation_vector = SIZE_RELATION_TYPE[relation_type] 23 | except KeyError: 24 | print(f"Relation type '{self.relation_type}' not recognized") 25 | return 26 | 27 | input_size = n_features + size_relation_vector 28 | self.lstm = nn.LSTM( 29 | input_size=input_size, 30 | hidden_size=hidden_size, 31 | num_layers=num_layers, 32 | bidirectional=True, 33 | batch_first=True, 34 | ) 35 | 36 | self.fc = nn.Linear(hidden_size * 2, num_classes) # 2 for bidirection 37 | 38 | def forward(self, x, debug=False): 39 | 40 | if debug: 41 | print("\ninput size") 42 | print(x.shape) 43 | x = self.relation_vectors(x) 44 | 45 | if debug: 46 | print("after transform") 47 | print(x.shape) 48 | 49 | batch_size = x.shape[0] 50 | hidden = self.init_hidden(batch_size) 51 | if debug: 52 | print("batch size") 53 | print(batch_size) 54 | print("hidden") 55 | print(hidden[0].shape) 56 | 57 | # Propagate input through LSTM : 58 | out, hidden = self.lstm(x, hidden) 59 | 60 | # Decode the hidden state of the last time step 61 | out = self.fc(out[:, -1, :]) 62 | 63 | if debug: 64 | print("output size") 65 | print(out.shape) 66 | 67 | return out 68 | 69 | def init_hidden(self, batch_size): 70 | # initialization of hidden states 71 | # shape (num_layers * num_directions, batch, hidden_size) 72 | hidden = ( 73 | Variable( 74 | torch.zeros( 75 | self.num_layers * 2, 76 | batch_size, 77 | self.hidden_size, 78 | device=device, 79 | ) 80 | ), 81 | Variable( 82 | torch.zeros( 83 | self.num_layers * 2, 84 | batch_size, 85 | self.hidden_size, 86 | device=device, 87 | ) 88 | ), 89 | ) 90 | return hidden 91 | 92 | def relation_vectors(self, x): 93 | coords = x[:, :, :3] 94 | # shape : (batch_size, seq_len, 3) 95 | 96 | # no relation 97 | if self.relation_type == 0: 98 | return x[:, :, 3:] 99 | 100 | # distances only 101 | if self.relation_type == 1: 102 | diff = coords - coords[:, 0:1, :] 103 | distances = torch.sum(diff ** 2, dim=2, keepdim=True) 104 | return torch.cat((distances, x[:, :, 3:]), dim=2) 105 | 106 | # centered coords 107 | elif self.relation_type == 2: 108 | diff = coords - coords[:, 0:1, :] 109 | return torch.cat((diff, x[:, :, 3:]), dim=2) 110 | 111 | # centered coords + distances 112 | elif self.relation_type == 3: 113 | diff = coords - coords[:, 0:1, :] 114 | distances = torch.sum(diff ** 2, dim=2, keepdim=True) 115 | return torch.cat((diff, distances, x[:, :, 3:]), dim=2) 116 | 117 | # decentralized coords 118 | elif self.relation_type == 4: 119 | decentralized = coords - torch.min(coords, dim=0, keepdim=True)[0] 120 | return torch.cat((decentralized, x[:, :, 3:]), dim=2) 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Aerial point cloud classification 2 | This repository is our project for the course [Point Cloud for 3D Modeling (NPM3D)](http://caor-mines-paristech.fr/fr/cours-npm3d/) of the [IASD master program](https://www.lamsade.dauphine.fr/wp/iasd/en/). 3 | This implementation is strongly inspired from the paper [classification of aerial point clouds with deep learning ](https://www.int-arch-photogramm-remote-sens-spatial-inf-sci.net/XLII-2-W13/103/2019/isprs-archives-XLII-2-W13-103-2019.pdf) by Emre Özdemir and Fabio Remondino. 4 | 5 | Our project report is available at the following link : https://www.overleaf.com/read/tdjgnxjbcmpg 6 | 7 | ## Installation 8 | This project requires python >= 3.5 9 | 10 | ``` 11 | $ git clone https://github.com/theobdt/aerial_pc_classification.git 12 | $ cd aerial_pc_classification 13 | $ pip3 install -r requirements.txt 14 | ``` 15 | 16 | 17 | ## Getting the data 18 | To get the data, you will first have to request access on the [ISPRS website](http://www2.isprs.org/commissions/comm3/wg4/detection-and-reconstruction.html), it is totally free. 19 | Fill out the questionnaire to receive the credentials. 20 | Then download data using the following script : 21 | 22 | ``` 23 | $ chmod +x fetch_data.sh 24 | $ ./fetch_data.sh 25 | ``` 26 | 27 | 28 | ## 1. Preprocessing 29 | 30 | Convert files to PLY and center/scale the point clouds. 31 | ``` 32 | $ python3 preprocessing.py -f data/vaihingen3D_train.pts data/vaihingen3D_test.pts --centering 33 | ``` 34 | Output files will be saved to `data/preprocessed/`. 35 | 36 | ![intensity train](imgs/intensity_train.jpg)*Vaihingen training intensity* 37 | 38 | 39 | ## 2. Computing features 40 | 41 | Parameters used for features extraction are stored in `cfg/config_features_extraction.yaml`. 42 | Output files will be saved in `data/features/`. 43 | 44 | Available steps are : 45 | 1. Local descriptors (`descriptors`). A few examples : 46 | ![normals train](imgs/normals_train.jpg)*Normal angles to vertical axis* 47 | ![sphericity train](imgs/sphericity_train.jpg)*Sphericity* 48 | ![planarity train](imgs/planarity_train.jpg)*Planarity* 49 | 2. Region growing (`region_growing`) 50 | ![regions train](imgs/regions_train.jpg)*Results of region growing* 51 | 3. Ground extraction (`ground_extraction`) 52 | ![ground train](imgs/ground_train.jpg)*Ground extraction* 53 | 4. Ground rasterization (`ground_rasterization`) 54 | ![ground raterized_train](imgs/ground_rasterized_train.jpg)*Ground rasterized* 55 | 5. Height above ground (`height_above_ground`) 56 | ![height_train](imgs/height_train.jpg)*Height above ground* 57 | 58 | To run all steps at once: 59 | ``` 60 | $ python3 compute_features.py --full_pipeline --files data/preprocessed/vaihingen3D_train.ply data/preprocessed/vaihingen3D_test.ply 61 | ``` 62 | 63 | To run all steps starting from a specific one : 64 | ``` 65 | $ python3 compute_features.py --from_step region_growing --files data/preprocessed/vaihingen3D_train.ply data/preprocessed/vaihingen3D_test.ply 66 | ``` 67 | 68 | To run steps individually : 69 | ``` 70 | $ python3 compute_features.py --steps region_growing ground_extraction --files data/preprocessed/vaihingen3D_train.ply data/preprocessed/vaihingen3D_test.ply 71 | ``` 72 | 73 | 74 | ## 3. Training Bi LSTM network 75 | 76 | Configuration file for training is stored in `cfg/config_bilstm.yaml`. 77 | To train the network, run the following command : 78 | 79 | ``` 80 | $ python3 train.py 81 | ``` 82 | A new checkpoint will be created in `ckpts/[current timestamp]`, along with a plot of accuracies and losses throughout training. 83 | 84 | ## 4. Predict with Bi LSTM network 85 | 86 | To compute predictions on all points of the cloud run : 87 | ``` 88 | $ python3 test.py -f data/features/vaihingen3D_test.ply --ckpt ckpts/2020-04-10_12-25-36 89 | ``` 90 | A new folder will be created at `data/predictions/[model timestamp]`, along with a comprehensive metrics table. 91 | ![labels_test](imgs/labels_test.jpg)*9-classes test set* 92 | ![predictions_test](imgs/predictions_test.jpg)*9-classes predictions on test set* 93 | 94 | 95 | 96 | 97 | 98 | ![confusion_matrix](imgs/confusion_matrix.png)*Confusion matrix and metrics* 99 | -------------------------------------------------------------------------------- /features/descriptors.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.neighbors import KDTree 3 | from tqdm import tqdm 4 | 5 | 6 | ORIENTATIONS = ["+x", "-x", "+y", "-y", "+z", "-z"] 7 | 8 | DESCRIPTORS = [ 9 | "normals", 10 | "verticality", 11 | "linearity", 12 | "planarity", 13 | "sphericity", 14 | "curvature", 15 | "anisotropy", 16 | "surface_variation", 17 | ] 18 | 19 | 20 | def local_PCA(points): 21 | 22 | eigenvalues = None 23 | eigenvectors = None 24 | 25 | n = points.shape[0] 26 | centroids = np.mean(points, axis=0) 27 | centered = points - centroids 28 | 29 | cov = centered.T @ centered / n 30 | 31 | eigenvalues, eigenvectors = np.linalg.eigh(cov) 32 | 33 | return eigenvalues.astype(np.float32), eigenvectors.astype(np.float32) 34 | 35 | 36 | def neighborhood_PCA(query_points, cloud_points, radius): 37 | 38 | # This function needs to compute PCA on the neighborhoods of all 39 | # query_points in cloud_points 40 | tree = KDTree(cloud_points) 41 | 42 | print("* Querying radius..", end=" ", flush=True) 43 | idx_lists = tree.query_radius(query_points, radius) 44 | print("DONE") 45 | 46 | all_eigenvalues = np.zeros((query_points.shape[0], 3), dtype=np.float32) 47 | all_eigenvectors = np.zeros( 48 | (query_points.shape[0], 3, 3), dtype=np.float32 49 | ) 50 | 51 | for i, idx_list in enumerate( 52 | tqdm(idx_lists, desc="* Processing neighborhoods") 53 | ): 54 | if len(idx_list) > 0: 55 | points = cloud_points[idx_list] 56 | eigenvalues, eigenvectors = local_PCA(points) 57 | all_eigenvalues[i, :] = eigenvalues 58 | all_eigenvectors[i, :, :] = eigenvectors 59 | 60 | return all_eigenvalues, all_eigenvectors 61 | 62 | 63 | def orient_normals(normals, preferred_orientation="+z"): 64 | index = ORIENTATIONS.index(preferred_orientation) 65 | sign = 1 if index % 2 == 0 else -1 66 | direction = index // 2 67 | normals[:, direction] = sign * np.abs(normals[:, direction]) 68 | 69 | return normals 70 | 71 | 72 | def compute_descriptors( 73 | coords, radius, descriptors, preferred_orientation, epsilon 74 | ): 75 | 76 | if "all" in descriptors: 77 | descriptors = DESCRIPTORS 78 | assert len(descriptors) > 0 79 | assert np.all([d in DESCRIPTORS for d in descriptors]) 80 | 81 | print(f"* descriptors: {descriptors}") 82 | print(f"* radius: {radius}") 83 | print(f"* epsilon: {epsilon}") 84 | print(f"* preferred normals orientation: {preferred_orientation}") 85 | # Compute the features for all points of the cloud 86 | eigenvalues, eigenvectors = neighborhood_PCA(coords, coords, radius) 87 | 88 | normals = orient_normals(eigenvectors[:, :, 0], preferred_orientation) 89 | 90 | normals_z = normals[:, 2] 91 | 92 | # L_1 >= L_2 >= L_3 93 | L_1 = eigenvalues[:, 2] 94 | L_2 = eigenvalues[:, 1] 95 | L_3 = eigenvalues[:, 0] 96 | 97 | # epsilon = 1e-2 * np.ones(len(normals_z)) 98 | epsilon_array = epsilon * np.ones(len(normals_z), dtype=np.float32) 99 | all_descriptors = {} 100 | 101 | if "normals" in descriptors: 102 | all_descriptors["nx"] = normals[:, 0] 103 | all_descriptors["ny"] = normals[:, 1] 104 | all_descriptors["nz"] = normals[:, 2] 105 | 106 | if "verticality" in descriptors: 107 | verticality = 2 * np.arcsin(normals_z) / np.pi 108 | all_descriptors["verticality"] = verticality 109 | 110 | if "linearity" in descriptors: 111 | linearity = 1 - (L_2 / (L_1 + epsilon_array)) 112 | all_descriptors["linearity"] = linearity 113 | 114 | if "planarity" in descriptors: 115 | planarity = (L_2 - L_3) / (L_1 + epsilon_array) 116 | all_descriptors["planarity"] = planarity 117 | 118 | if "sphericity" in descriptors: 119 | sphericity = L_3 / (L_1 + epsilon_array) 120 | all_descriptors["sphericity"] = sphericity 121 | 122 | if "curvature" in descriptors: 123 | curvature = L_3 / (L_1 + L_2 + L_3 + epsilon_array) 124 | all_descriptors["curvature"] = curvature 125 | 126 | if "anisotropy" in descriptors: 127 | anisotropy = (L_1 - L_3) / (L_1 + epsilon_array) 128 | all_descriptors["anisotropy"] = anisotropy 129 | 130 | if "surface_variation" in descriptors: 131 | surface_variation = L_3 / (L_1 + L_3 + L_3 + epsilon_array) 132 | all_descriptors["surface_variation"] = surface_variation 133 | 134 | return all_descriptors 135 | -------------------------------------------------------------------------------- /features/ground_extraction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.neighbors import KDTree 3 | from scipy import spatial 4 | from tqdm import tqdm 5 | 6 | METHODS_RASTERIZATION = ["closest_neighbor", "delaunay"] 7 | 8 | 9 | def stitch_regions( 10 | coords, region_labels, slope_intra_max, slope_inter_max, percentile_closest 11 | ): 12 | 13 | N = len(coords) 14 | ground_mask = np.zeros(N, dtype=bool) 15 | 16 | all_labels = np.unique(region_labels) 17 | all_labels = all_labels[all_labels > 0] 18 | 19 | spans = np.zeros(len(all_labels), dtype=np.float32) 20 | heights = np.zeros(len(all_labels), dtype=np.float32) 21 | 22 | # computing slopes intra 23 | for i, label in enumerate( 24 | tqdm(all_labels, desc="* Computing intra slopes") 25 | ): 26 | coords_label = coords[region_labels == label] 27 | deltas = np.max(coords_label, axis=0) - np.min(coords_label, axis=0) 28 | spans[i] = np.linalg.norm(deltas[:2]) 29 | heights[i] = deltas[2] 30 | 31 | slopes_intra = heights / spans 32 | 33 | # start with the region with the largest span 34 | init_label = all_labels[np.argmax(spans)] 35 | ground_mask[region_labels == init_label] = 1 36 | 37 | # computing slopes inter 38 | coords_ground = coords[ground_mask] 39 | 40 | tree = KDTree(coords_ground) 41 | for i, label in enumerate( 42 | tqdm(all_labels, desc="* Stitching regions together") 43 | ): 44 | if label == init_label: 45 | continue 46 | 47 | if slopes_intra[i] > slope_intra_max: 48 | continue 49 | 50 | coords_label = coords[region_labels == label] 51 | dist, idx = tree.query(coords_label, k=1) 52 | idx = idx.ravel() 53 | dist = dist.ravel() 54 | 55 | mask_closest = dist < np.percentile(dist, 100 * percentile_closest) 56 | if np.sum(mask_closest) == 0: 57 | continue 58 | coords_label_closest = coords_label[mask_closest] 59 | 60 | idx_ground_closest = idx[mask_closest] 61 | coords_ground_closest = coords_ground[idx_ground_closest] 62 | 63 | deltas = np.abs(coords_label_closest - coords_ground_closest) 64 | 65 | dist_xy = np.linalg.norm(deltas[:, :2], axis=1) 66 | dist_z = deltas[:, 2] 67 | slopes_inter = dist_z / dist_xy 68 | 69 | mean_slopes_inter = np.mean(slopes_inter) 70 | 71 | if mean_slopes_inter < slope_inter_max: 72 | ground_mask[region_labels == label] = 1 73 | coords_ground = coords[ground_mask] 74 | tree = KDTree(coords_ground) 75 | 76 | return ground_mask 77 | 78 | 79 | def interpolate_altitude(coords_ground, coords_queries_xy, method="delaunay"): 80 | 81 | if method == "closest_neighbor": 82 | # create a KD tree on xy coordinates 83 | tree = KDTree(coords_ground[:, :2]) 84 | 85 | # find closest neighbor on the ground 86 | _, idx_neighbor = tree.query(coords_queries_xy, k=1) 87 | idx_neighbor = idx_neighbor.flatten() 88 | 89 | z_ground = coords_ground[:, -1] 90 | z_queries = z_ground[idx_neighbor] 91 | grid_3d = np.hstack((coords_queries_xy, z_queries.reshape(-1, 1))) 92 | 93 | elif method == "delaunay": 94 | # create 2D triangulation of ground coordinates 95 | tri = spatial.Delaunay(coords_ground[:, :2]) 96 | 97 | # Find simplex of each query point 98 | idx_simplices = tri.find_simplex(coords_queries_xy) 99 | convex_hull_mask = idx_simplices >= 0 100 | 101 | # keep only query points inside convex hull 102 | idx_simplices = idx_simplices[convex_hull_mask] 103 | coords_queries_hull = coords_queries_xy[convex_hull_mask] 104 | 105 | # compute weights 106 | trans = tri.transform[idx_simplices] 107 | inv_T = trans[:, :-1, :] 108 | r = trans[:, -1, :] 109 | diff = (coords_queries_hull - r)[:, :, np.newaxis] 110 | barycent = (inv_T @ diff).squeeze() 111 | weights = np.c_[barycent, 1 - barycent.sum(axis=1)] 112 | 113 | # interpolate z values of vertices 114 | z_vertices = coords_ground[:, -1][tri.simplices][idx_simplices] 115 | z_queries = np.sum(weights * z_vertices, axis=1) 116 | grid_3d = np.hstack((coords_queries_hull, z_queries.reshape(-1, 1))) 117 | 118 | else: 119 | raise ValueError(f"Method '{method}' not found") 120 | 121 | return grid_3d 122 | 123 | 124 | def rasterize_ground(coords, ground_mask, step_size, method): 125 | print(f"* method : {method}") 126 | print(f"* step_size : {step_size}") 127 | assert method in METHODS_RASTERIZATION 128 | 129 | mins = np.min(coords[:, :2], axis=0) 130 | maxs = np.max(coords[:, :2], axis=0) 131 | 132 | # Create a grid 133 | grid = np.mgrid[ 134 | mins[0] : maxs[0] : step_size, mins[1] : maxs[1] : step_size 135 | ].T 136 | grid_points = grid.reshape(-1, 2) 137 | 138 | # Interpolate altitudes 139 | grid_3d = interpolate_altitude(coords[ground_mask], grid_points, method) 140 | 141 | return grid_3d 142 | 143 | 144 | def height_above_ground(coords, ground_mask, grid_ground_3d): 145 | heights = np.zeros(len(coords), dtype=np.float32) 146 | 147 | coords_queries = coords[~ground_mask] 148 | 149 | tree = KDTree(grid_ground_3d[:, :2]) 150 | 151 | # find closest neighbor on the rasterized ground 152 | _, idx_neighbor = tree.query(coords_queries[:, :2], k=1) 153 | idx_neighbor = idx_neighbor.flatten() 154 | 155 | # set heights 156 | z_ground = grid_ground_3d[:, -1] 157 | z_queries_ground = z_ground[idx_neighbor] 158 | heights[~ground_mask] = coords_queries[:, -1] - z_queries_ground 159 | 160 | return heights 161 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data import DataLoader 3 | import argparse 4 | import os 5 | import yaml 6 | from tqdm import tqdm 7 | import numpy as np 8 | import pandas as pd 9 | import sklearn.metrics as skm 10 | 11 | from utils.dataloader import AerialPointDataset, convert_labels 12 | from utils.ply import ply2dict, dict2ply 13 | from models import BiLSTM 14 | 15 | NAMES_9 = [ 16 | "Powerline", 17 | "Low veg.", 18 | "Imp. surf.", 19 | "Car", 20 | "Fence", 21 | "Roof", 22 | "Facade", 23 | "Shrub", 24 | "Tree", 25 | ] 26 | 27 | NAMES_4 = ["GLO", "Roof", "Facade", "Vegetation"] 28 | 29 | 30 | parser = argparse.ArgumentParser(description="Training") 31 | 32 | parser.add_argument( 33 | "--files", "-f", type=str, nargs="+", help="Path to point cloud file" 34 | ) 35 | parser.add_argument( 36 | "--ckpt", type=str, help="Path to the checkpoint folder", 37 | ) 38 | parser.add_argument( 39 | "--prefix_path", type=str, default="", help="Path prefix", 40 | ) 41 | parser.add_argument( 42 | "--batch_size", type=int, default=1000, help="Batch size", 43 | ) 44 | parser.add_argument( 45 | "--num_workers", 46 | type=int, 47 | default=4, 48 | help="Number of workers for dataloading", 49 | ) 50 | parser.add_argument( 51 | "--prediction_folder", 52 | type=str, 53 | default="data/predictions", 54 | help="Path to the prediction folder", 55 | ) 56 | 57 | args = parser.parse_args() 58 | 59 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 60 | print(f"Device in use : {device}") 61 | 62 | # Load checkpoint 63 | path_ckpt = os.path.join(args.prefix_path, os.path.normpath(args.ckpt)) 64 | print(f"Loading checkpoint: {path_ckpt}") 65 | path_config = os.path.join(path_ckpt, "config.yaml") 66 | path_ckpt_dict = os.path.join(path_ckpt, "ckpt.pt") 67 | checkpoint = torch.load(path_ckpt_dict, map_location=device) 68 | 69 | # Create prediction folder 70 | ckpt_id = os.path.basename(path_ckpt) 71 | ckpt_prediction_folder = os.path.join( 72 | args.prefix_path, args.prediction_folder, ckpt_id 73 | ) 74 | os.makedirs(ckpt_prediction_folder, exist_ok=True) 75 | 76 | # Load model config 77 | with open(path_config, "r") as f: 78 | config = yaml.safe_load(f) 79 | 80 | # Load model 81 | n_features = len(config["data"]["features"]) 82 | n_classes = 4 83 | if config["data"]["all_labels"]: 84 | n_classes = 9 85 | print(f"Num classes: {n_classes}\n") 86 | 87 | print("Loading model..", end=" ", flush=True) 88 | model = BiLSTM(n_features, n_classes, **config["network"]).to(device) 89 | model.load_state_dict(checkpoint["model_state_dict"]) 90 | model.eval() 91 | print("DONE") 92 | 93 | 94 | def predict(loader, len_dataset): 95 | predictions = torch.empty(len_dataset, dtype=torch.int32, device=device) 96 | with torch.no_grad(): 97 | start = 0 98 | for (sequence, label) in tqdm(loader, desc="* Processing point cloud"): 99 | sequence = sequence.to(device) 100 | label = label.to(device) 101 | 102 | # compute predicted classes 103 | output = model(sequence) 104 | classes = torch.max(output, 1).indices 105 | 106 | # fill predictions 107 | seq_len = sequence.shape[0] 108 | predictions[start : start + seq_len] = classes 109 | start += seq_len 110 | 111 | return predictions.cpu().numpy() 112 | 113 | 114 | def evaluate(y_true, y_pred, names): 115 | labels = np.arange(len(names)) 116 | 117 | cm = skm.confusion_matrix(y_true, y_pred, labels=labels) 118 | totals = np.sum(cm, axis=1) 119 | cm = np.hstack((cm, totals.reshape(-1, 1))) 120 | totals_cols = np.sum(cm, axis=0, keepdims=True) 121 | cm = np.vstack((cm, totals_cols, totals_cols)) 122 | 123 | metrics = skm.precision_recall_fscore_support( 124 | y_true, y_pred, labels=labels 125 | ) 126 | metrics = 100 * np.vstack(metrics[:-1]).T 127 | avg_metrics = np.mean(metrics, axis=0) 128 | weighted_avg_metrics = totals @ metrics / np.sum(totals) 129 | metrics = np.vstack((metrics, avg_metrics, weighted_avg_metrics)) 130 | 131 | all_data = np.hstack((cm, metrics)) 132 | 133 | cols_int = names + ["Total"] 134 | cols_float = ["Precision", "Recall", "F1-score"] 135 | 136 | idx = names + ["Total/Avg", "Total/Weighted Avg"] 137 | df = pd.DataFrame(data=all_data, columns=cols_int + cols_float, index=idx) 138 | df[cols_int] = df[cols_int].astype(int) 139 | return df 140 | 141 | 142 | def write_metrics(path_prediction, filename, df): 143 | filename = filename.split(".")[0] 144 | path_metrics = os.path.join(path_prediction, "metrics") 145 | os.makedirs(path_metrics, exist_ok=True) 146 | 147 | path_tex = os.path.join(path_metrics, f"{filename}.tex") 148 | path_txt = os.path.join(path_metrics, f"{filename}.txt") 149 | print(path_tex) 150 | 151 | # write tex file 152 | column_format = "|l|" + (df.shape[1] - 4) * "r|" + "|r||r|r|r|" 153 | with open(path_tex, "w") as f: 154 | f.write( 155 | df.to_latex( 156 | float_format="{:0.2f}".format, column_format=column_format, 157 | ) 158 | ) 159 | 160 | with open(path_txt, "w") as f: 161 | df.to_string(f) 162 | print(f"* Metrics written to: {path_tex} and {path_tex}") 163 | 164 | 165 | for path_ply in args.files: 166 | path_ply = os.path.join(args.prefix_path, path_ply) 167 | print(f"\nProcessing file: {path_ply}") 168 | print("* Preparing dataloader..", end=" ", flush=True) 169 | dataset = AerialPointDataset(path_ply, **config["data"]) 170 | loader = DataLoader( 171 | dataset=dataset, 172 | batch_size=args.batch_size, 173 | num_workers=args.num_workers, 174 | shuffle=False, 175 | ) 176 | print("DONE") 177 | 178 | # Create and fill point cloud field 179 | data = ply2dict(path_ply) 180 | true_labels = data["labels"] 181 | names = NAMES_9 182 | 183 | # in the 4-labels case 184 | if not config["data"]["all_labels"]: 185 | true_labels = convert_labels(true_labels).astype(np.int32) 186 | names = NAMES_4 187 | 188 | n = len(true_labels) 189 | predictions = -np.ones(n, dtype=np.int32) 190 | raw_predictions = predict(loader, len(dataset)).astype(np.int32) 191 | predictions[dataset.index] = raw_predictions 192 | errors = predictions != true_labels 193 | data["predictions"] = predictions 194 | data["errors"] = errors.astype(np.uint8) 195 | data["labels"] = true_labels 196 | 197 | # Save point cloud 198 | filename = os.path.basename(path_ply) 199 | path_prediction = os.path.join(ckpt_prediction_folder, filename) 200 | if dict2ply(data, path_prediction): 201 | print(f"* Predictions PLY file saved to: {path_prediction}") 202 | 203 | df = evaluate(true_labels[true_labels >= 0], raw_predictions, names) 204 | write_metrics(ckpt_prediction_folder, filename, df) 205 | -------------------------------------------------------------------------------- /compute_features.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import sys 5 | import yaml 6 | 7 | from utils.ply import ply2dict, dict2ply 8 | from features import descriptors, region_growing, ground_extraction 9 | 10 | AVAILABLE_STEPS = [ 11 | "descriptors", 12 | "region_growing", 13 | "ground_extraction", 14 | "ground_rasterization", 15 | "height_above_ground", 16 | ] 17 | 18 | PATH_FEATURES = "data/features" 19 | PATH_GROUND_ONLY = "data/ground_only" 20 | PATH_GROUND_RASTERIZED = "data/ground_rasterized" 21 | 22 | 23 | def compute_features(path_ply, steps_params): 24 | filename = os.path.split(path_ply)[-1] 25 | 26 | data = ply2dict(path_ply) 27 | coords = np.vstack((data["x"], data["y"], data["z"])).T 28 | 29 | grid_ground_3d = None 30 | 31 | for (step, params) in steps_params.items(): 32 | if step == "descriptors": 33 | print("Computing local descriptors..") 34 | all_descriptors = descriptors.compute_descriptors(coords, **params) 35 | data.update(all_descriptors) 36 | 37 | if step == "region_growing": 38 | print("\nComputing regions..") 39 | normals = np.vstack((data["nx"], data["ny"], data["nz"])).T 40 | params_copy = params.copy() 41 | 42 | descriptor_selected = params_copy.pop("descriptor") 43 | print( 44 | "* descriptor selected : " 45 | f"{'min' if params['minimize'] else 'max'} " 46 | f"{descriptor_selected}" 47 | ) 48 | print(f"* thresholds : {params['thresholds']}") 49 | print(f"* radius : {params['radius']}") 50 | try: 51 | descriptor_vals = data[descriptor_selected] 52 | region_labels = region_growing.multi_region_growing( 53 | coords, normals, descriptor_vals, **params_copy 54 | ) 55 | 56 | data["regions"] = region_labels 57 | except KeyError: 58 | print( 59 | f"Descriptor '{descriptor_selected}' has not been computed" 60 | ", run 'python3 compute_features.py --descriptors " 61 | f"{descriptor_selected}'" 62 | ) 63 | sys.exit(-1) 64 | 65 | if step == "ground_extraction": 66 | print("\nExtracting ground from regions..") 67 | region_labels = data["regions"] 68 | ground_mask = ground_extraction.stitch_regions( 69 | coords, region_labels, **params 70 | ) 71 | 72 | ground_only = { 73 | field: data[field][ground_mask] for field in list(data.keys()) 74 | } 75 | 76 | data["ground"] = ground_mask.astype(np.uint8) 77 | 78 | os.makedirs(PATH_GROUND_ONLY, exist_ok=True) 79 | path_ground = os.path.join(PATH_GROUND_ONLY, filename) 80 | if dict2ply(ground_only, path_ground): 81 | print(f"* PLY ground file successfully saved to {path_ground}") 82 | 83 | if step == "ground_rasterization": 84 | print("\nComputing ground rasterization..") 85 | ground_mask = data["ground"].astype(bool) 86 | grid_ground_3d = ground_extraction.rasterize_ground( 87 | coords, ground_mask, **params 88 | ) 89 | 90 | ground_rasterized = { 91 | "x": grid_ground_3d[:, 0], 92 | "y": grid_ground_3d[:, 1], 93 | "z": grid_ground_3d[:, 2], 94 | "ground_altitude": grid_ground_3d[:, 2], 95 | } 96 | 97 | path_rasterized = os.path.join(PATH_GROUND_RASTERIZED, filename) 98 | if dict2ply(ground_rasterized, path_rasterized): 99 | print( 100 | "* PLY ground rasterized file successfully saved to " 101 | f"{path_rasterized}" 102 | ) 103 | 104 | if step == "height_above_ground": 105 | print("\nComputing height above ground..") 106 | if grid_ground_3d is None: 107 | path_rasterized = os.path.join( 108 | PATH_GROUND_RASTERIZED, filename 109 | ) 110 | print(f"* Loading rasterized ground : {path_rasterized}") 111 | ground_rasterized = ply2dict(path_rasterized) 112 | grid_ground_3d = np.vstack( 113 | ( 114 | ground_rasterized["x"], 115 | ground_rasterized["y"], 116 | ground_rasterized["z"], 117 | ) 118 | ).T 119 | 120 | ground_mask = data["ground"].astype(bool) 121 | heights = ground_extraction.height_above_ground( 122 | coords, ground_mask, grid_ground_3d 123 | ) 124 | data["height_above_ground"] = heights 125 | print("DONE") 126 | 127 | # saving data 128 | path_output = os.path.join(PATH_FEATURES, filename) 129 | if dict2ply(data, path_output): 130 | print(f"\nPLY features file successfully saved to {path_output}") 131 | 132 | 133 | if __name__ == "__main__": 134 | parser = argparse.ArgumentParser( 135 | description="Script to extract features form PLY file" 136 | ) 137 | parser.add_argument( 138 | "--prefix_path", type=str, default="", help="Path prefix", 139 | ) 140 | parser.add_argument( 141 | "--files", 142 | "-f", 143 | type=str, 144 | nargs="+", 145 | required=True, 146 | help="Path to point cloud file", 147 | ) 148 | parser.add_argument( 149 | "--config", 150 | type=str, 151 | default="cfg/config_features_extraction.yaml", 152 | help="Path to the config file", 153 | ) 154 | parser.add_argument( 155 | "--full_pipeline", 156 | action="store_true", 157 | help="Run all steps from the beginning", 158 | ) 159 | parser.add_argument( 160 | "--steps", 161 | type=str, 162 | nargs="+", 163 | help=f"List of steps to run, available steps are: {AVAILABLE_STEPS}", 164 | ) 165 | parser.add_argument( 166 | "--from_step", 167 | type=str, 168 | help="Will run the features extraction pipeline from this step", 169 | ) 170 | 171 | args = parser.parse_args() 172 | 173 | # Load config 174 | with open(args.config, "r") as f: 175 | config = yaml.safe_load(f) 176 | 177 | os.makedirs(PATH_FEATURES, exist_ok=True) 178 | 179 | if args.full_pipeline: 180 | steps = AVAILABLE_STEPS 181 | elif args.from_step: 182 | assert args.from_step in AVAILABLE_STEPS 183 | steps = AVAILABLE_STEPS[AVAILABLE_STEPS.index(args.from_step) :] 184 | elif args.steps: 185 | assert np.all([s in AVAILABLE_STEPS for s in args.steps]) 186 | steps = args.steps 187 | else: 188 | raise ValueError("No input step") 189 | steps_params = {step: config[step] for step in steps} 190 | 191 | # update path with prefix 192 | PATH_FEATURES = os.path.join(args.prefix_path, PATH_FEATURES) 193 | PATH_GROUND_ONLY = os.path.join(args.prefix_path, PATH_GROUND_ONLY) 194 | PATH_GROUND_RASTERIZED = os.path.join( 195 | args.prefix_path, PATH_GROUND_RASTERIZED 196 | ) 197 | 198 | for path_file in args.files: 199 | path_ply = os.path.join(args.prefix_path, path_file) 200 | print(f"\nComputing features of file {path_ply}") 201 | 202 | data = ply2dict(path_ply) 203 | compute_features(path_ply, steps_params) 204 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.utils.data import DataLoader 4 | import argparse 5 | import os 6 | import matplotlib.pyplot as plt 7 | import yaml 8 | from datetime import datetime 9 | import shutil 10 | 11 | from utils.dataloader import AerialPointDataset 12 | from models import BiLSTM 13 | 14 | 15 | parser = argparse.ArgumentParser(description="Training") 16 | parser.add_argument( 17 | "--config", 18 | type=str, 19 | default="cfg/config_bilstm.yaml", 20 | help="Path to the config file", 21 | ) 22 | parser.add_argument( 23 | "--debug", action="store_true", help="Debug mode", 24 | ) 25 | parser.add_argument( 26 | "--log_interval", type=int, default=10, help="Log interval for training", 27 | ) 28 | parser.add_argument( 29 | "--path_ckpts", 30 | type=str, 31 | default="ckpts", 32 | help="Path to the checkpoint folder", 33 | ) 34 | parser.add_argument( 35 | "--prefix_path", type=str, default='', help="Path prefix", 36 | ) 37 | parser.add_argument( 38 | "--resume", "-r", type=str, help="Name of the checkpoint to resume", 39 | ) 40 | parser.add_argument( 41 | "--data_train", 42 | type=str, 43 | default="data/features/vaihingen3D_train.ply", 44 | help="Path to training data", 45 | ) 46 | parser.add_argument( 47 | "--data_test", 48 | type=str, 49 | default="data/features/vaihingen3D_test.ply", 50 | help="Path to test data", 51 | ) 52 | 53 | args = parser.parse_args() 54 | 55 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 56 | print(f"Device in use : {device}") 57 | 58 | 59 | def init_ckpt(): 60 | checkpoint = { 61 | "epoch": 0, 62 | "model_state_dict": None, 63 | "optimizer_state_dict": None, 64 | "losses": [], 65 | "accuracies": [], 66 | "best_train_loss": float("inf"), 67 | } 68 | return checkpoint 69 | 70 | 71 | if args.resume: 72 | path_ckpt = os.path.join(args.prefix_path, args.path_ckpts, args.resume) 73 | print(f"Loading checkpoint {path_ckpt}") 74 | path_config = os.path.join(path_ckpt, "config.yaml") 75 | path_ckpt_dict = os.path.join(path_ckpt, "ckpt.pt") 76 | checkpoint = torch.load(path_ckpt_dict, map_location=device) 77 | else: 78 | ckpt_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") 79 | path_ckpt = os.path.join(args.prefix_path, args.path_ckpts, ckpt_name) 80 | os.makedirs(path_ckpt, exist_ok=True) 81 | path_config = os.path.join(path_ckpt, "config.yaml") 82 | shutil.copy2(args.config, path_config) 83 | path_ckpt_dict = os.path.join(path_ckpt, "ckpt.pt") 84 | checkpoint = init_ckpt() 85 | print(f"Initialized checkpoint {path_ckpt}") 86 | 87 | print(f"\nConfig file: {path_config}") 88 | 89 | with open(path_config, "r") as f: 90 | config = yaml.safe_load(f) 91 | 92 | # get number of epochs 93 | epoch_start = checkpoint["epoch"] 94 | epoch_end = config["training"].pop("epoch_end") 95 | print(f"Epoch start: {epoch_start}, Epoch end: {epoch_end}") 96 | 97 | max_batches_train = config["training"].pop("max_batches") 98 | max_batches_test = config["test"].pop("max_batches") 99 | 100 | path_train_ply = os.path.join(args.prefix_path, args.data_train) 101 | path_test_ply = os.path.join(args.prefix_path, args.data_test) 102 | print(f'\nTraining file: {path_train_ply}') 103 | print(f'Test file: {path_test_ply}') 104 | 105 | dataset_train = AerialPointDataset(path_train_ply, **config["data"]) 106 | dataset_test = AerialPointDataset(path_test_ply, **config["data"]) 107 | 108 | train_loader = DataLoader(dataset=dataset_train, **config["training"]) 109 | test_loader = DataLoader(dataset=dataset_test, **config["test"]) 110 | 111 | 112 | print( 113 | f"Total samples train: {len(dataset_train)}, " 114 | f"Number of batches: {len(train_loader)}" 115 | f"\nTotal samples test: {len(dataset_test)}, " 116 | f"Number of batches: {len(test_loader)}" 117 | ) 118 | 119 | # Define model 120 | n_features = len(config["data"]["features"]) 121 | n_classes = 4 122 | if config["data"]["all_labels"]: 123 | n_classes = 9 124 | print(f"Num classes: {n_classes}\n") 125 | model = BiLSTM(n_features, n_classes, **config["network"]).to(device) 126 | 127 | # Define loss and optimizer 128 | criterion = nn.CrossEntropyLoss() 129 | lr = config["optimizer"]["learning_rate"] 130 | optimizer = torch.optim.Adam(model.parameters(), lr=lr) 131 | 132 | if args.resume: 133 | model.load_state_dict(checkpoint["model_state_dict"]) 134 | optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) 135 | 136 | 137 | def decentralized_coordinate(coords): 138 | decentralized_coords = coords - torch.min(coords, axis=0).values 139 | return decentralized_coords 140 | 141 | 142 | def train(loader, log_interval, max_batches=None): 143 | model.train() 144 | if max_batches is None: 145 | max_batches = len(loader) 146 | history_acc_train = [] 147 | history_loss_train = [] 148 | for i, (sequence, label) in enumerate(train_loader): 149 | sequence = sequence.to(device) 150 | label = label.to(device) 151 | 152 | label = label.type(dtype=torch.long) 153 | 154 | # Forward pass 155 | output = model(sequence, debug=args.debug) 156 | train_loss = criterion(output, label) 157 | 158 | # Backward and optimize 159 | optimizer.zero_grad() 160 | train_loss.backward() 161 | optimizer.step() 162 | 163 | # calculate accuracy of predictions in the current batch 164 | n_correct = (torch.max(output, 1).indices == label).sum().item() 165 | train_acc = 100.0 * n_correct / len(label) 166 | 167 | history_loss_train.append(train_loss.item()) 168 | history_acc_train.append(train_acc) 169 | 170 | if (i + 1) % log_interval == 0: 171 | print( 172 | "Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, " 173 | "Acc: {:.2f} %".format( 174 | epoch + 1, 175 | epoch_end, 176 | i + 1, 177 | max_batches, 178 | train_loss.item(), 179 | sum(history_acc_train[-log_interval:]) / log_interval, 180 | ) 181 | ) 182 | if i + 1 > max_batches: 183 | break 184 | return history_loss_train, history_acc_train 185 | 186 | 187 | def evaluate(loader, max_batches=None): 188 | model.eval() 189 | if max_batches is None: 190 | max_batches = len(loader) 191 | total_loss = 0 192 | total_correct = 0 193 | with torch.no_grad(): 194 | for i, (sequence, label) in enumerate(loader): 195 | sequence = sequence.to(device) 196 | label = label.to(device) 197 | label = label.type(dtype=torch.long) 198 | output = model(sequence) 199 | 200 | total_loss += criterion(output, label) 201 | 202 | n_correct = (torch.max(output, 1).indices == label).sum().item() 203 | total_correct += n_correct / len(label) 204 | if i + 1 > max_batches: 205 | break 206 | avg_loss = total_loss / max_batches 207 | avg_acc = 100 * total_correct / max_batches 208 | print(f"Test Loss : {avg_loss:.4f}, Test Acc : {avg_acc:.2f} %\n") 209 | return avg_loss, avg_acc 210 | 211 | 212 | # Training loop 213 | for epoch in range(epoch_start, epoch_end): 214 | hist_loss_train, hist_acc_train = train( 215 | train_loader, 216 | log_interval=args.log_interval, 217 | max_batches=max_batches_train, 218 | ) 219 | test_loss, test_acc = evaluate(test_loader, max_batches=max_batches_test) 220 | 221 | # update checkpoint 222 | checkpoint["losses"].append((hist_loss_train, test_loss)) 223 | checkpoint["accuracies"].append((hist_acc_train, test_acc)) 224 | 225 | avg_train_loss = sum(hist_loss_train) / len(hist_loss_train) 226 | if avg_train_loss < checkpoint["best_train_loss"]: 227 | checkpoint["best_train_loss"] = avg_train_loss 228 | checkpoint["model_state_dict"] = model.state_dict() 229 | checkpoint["epoch"] = epoch 230 | checkpoint["optimizer_state_dict"] = optimizer.state_dict() 231 | torch.save(checkpoint, path_ckpt_dict) 232 | 233 | 234 | def plot_metric(ax, metric, label): 235 | train_values = [] 236 | x_test = [] 237 | y_test = [] 238 | for (hist_train, test_value) in metric: 239 | train_values += hist_train 240 | x_test.append(len(train_values)) 241 | y_test.append(test_value) 242 | ax.plot(train_values, label="train_" + label) 243 | ax.plot(x_test, y_test, label="test_" + label) 244 | ax.set_xlabel("Batches") 245 | ax.legend() 246 | 247 | 248 | _, axes = plt.subplots(nrows=2, sharex=True) 249 | axes[0].set_title("Loss") 250 | plot_metric(axes[0], checkpoint["losses"], "loss") 251 | axes[0].set_ylim([0, 2]) 252 | 253 | axes[1].set_title("Accuracy") 254 | plot_metric(axes[1], checkpoint["accuracies"], "acc") 255 | axes[1].set_ylim([0, 100]) 256 | plt.tight_layout() 257 | path_fig = os.path.join(path_ckpt, "figure.png") 258 | plt.savefig(path_fig) 259 | print(f"Figure saved to {path_fig}") 260 | plt.show() 261 | -------------------------------------------------------------------------------- /utils/ply.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0===============================0 4 | # | PLY files reader/writer | 5 | # 0===============================0 6 | # 7 | # 8 | # ------------------------------------------------------------------------------------------ 9 | # 10 | # function to read/write .ply files 11 | # 12 | # ------------------------------------------------------------------------------------------ 13 | # 14 | # Hugues THOMAS - 10/02/2017 15 | # 16 | 17 | 18 | # ------------------------------------------------------------------------------------------ 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Basic libs 26 | import numpy as np 27 | import sys 28 | 29 | 30 | # Define PLY types 31 | ply_dtypes = dict( 32 | [ 33 | (b"int8", "i1"), 34 | (b"char", "i1"), 35 | (b"uint8", "u1"), 36 | (b"uchar", "b1"), 37 | (b"uchar", "u1"), 38 | (b"int16", "i2"), 39 | (b"short", "i2"), 40 | (b"uint16", "u2"), 41 | (b"ushort", "u2"), 42 | (b"int32", "i4"), 43 | (b"int", "i4"), 44 | (b"uint32", "u4"), 45 | (b"uint", "u4"), 46 | (b"float32", "f4"), 47 | (b"float", "f4"), 48 | (b"float64", "f8"), 49 | (b"double", "f8"), 50 | ] 51 | ) 52 | 53 | # Numpy reader format 54 | valid_formats = { 55 | "ascii": "", 56 | "binary_big_endian": ">", 57 | "binary_little_endian": "<", 58 | } 59 | 60 | 61 | # ------------------------------------------------------------------------------------------ 62 | # 63 | # Functions 64 | # \***************/ 65 | # 66 | 67 | 68 | def parse_header(plyfile, ext): 69 | 70 | # Variables 71 | line = [] 72 | properties = [] 73 | num_points = None 74 | 75 | while b"end_header" not in line and line != b"": 76 | line = plyfile.readline() 77 | if b"element" in line: 78 | line = line.split() 79 | num_points = int(line[2]) 80 | 81 | elif b"property" in line: 82 | line = line.split() 83 | properties.append((line[2].decode(), ext + ply_dtypes[line[1]])) 84 | 85 | return num_points, properties 86 | 87 | 88 | def read_ply(filename): 89 | """ 90 | Read ".ply" files 91 | 92 | Parameters 93 | ---------- 94 | filename : string 95 | the name of the file to read. 96 | 97 | Returns 98 | ------- 99 | result : array 100 | data stored in the file 101 | 102 | Examples 103 | -------- 104 | Store data in file 105 | 106 | >>> points = np.random.rand(5, 3) 107 | >>> values = np.random.randint(2, size=10) 108 | >>> write_ply('example.ply', [points, values], ['x', 'y', 'z', 'values']) 109 | 110 | Read the file 111 | 112 | >>> data = read_ply('example.ply') 113 | >>> values = data['values'] 114 | array([0, 0, 1, 1, 0]) 115 | 116 | >>> points = np.vstack((data['x'], data['y'], data['z'])).T 117 | array([[ 0.466 0.595 0.324] 118 | [ 0.538 0.407 0.654] 119 | [ 0.850 0.018 0.988] 120 | [ 0.395 0.394 0.363] 121 | [ 0.873 0.996 0.092]]) 122 | 123 | """ 124 | 125 | with open(filename, "rb") as plyfile: 126 | 127 | # Check if the file start with ply 128 | if b"ply" not in plyfile.readline(): 129 | raise ValueError("The file does not start whith the word ply") 130 | 131 | # get binary_little/big or ascii 132 | fmt = plyfile.readline().split()[1].decode() 133 | if fmt == "ascii": 134 | raise ValueError("The file is not binary") 135 | 136 | # get extension for building the numpy dtypes 137 | ext = valid_formats[fmt] 138 | 139 | # Parse header 140 | num_points, properties = parse_header(plyfile, ext) 141 | 142 | # Get data 143 | data = np.fromfile(plyfile, dtype=properties, count=num_points) 144 | 145 | return data 146 | 147 | 148 | def header_properties(field_list, field_names): 149 | 150 | # List of lines to write 151 | lines = [] 152 | 153 | # First line describing element vertex 154 | lines.append("element vertex %d" % field_list[0].shape[0]) 155 | 156 | # Properties lines 157 | i = 0 158 | for fields in field_list: 159 | for field in fields.T: 160 | lines.append("property %s %s" % (field.dtype.name, field_names[i])) 161 | i += 1 162 | 163 | return lines 164 | 165 | 166 | def write_ply(filename, field_list, field_names): 167 | """ 168 | Write ".ply" files 169 | 170 | Parameters 171 | ---------- 172 | filename : string 173 | the name of the file to which the data is saved. A '.ply' extension 174 | will be appended to the file name if it does no already have one. 175 | 176 | field_list : list, tuple, numpy array 177 | the fields to be saved in the ply file. Either a numpy array, a list of 178 | numpy arrays or a tuple of numpy arrays. Each 1D numpy array and each 179 | column of 2D numpy arrays are considered as one field. 180 | 181 | field_names : list 182 | the name of each fields as a list of strings. Has to be the same length 183 | as the number of fields. 184 | 185 | Examples 186 | -------- 187 | >>> points = np.random.rand(10, 3) 188 | >>> write_ply('example1.ply', points, ['x', 'y', 'z']) 189 | 190 | >>> values = np.random.randint(2, size=10) 191 | >>> write_ply('example2.ply', [points, values], ['x', 'y', 'z', 'values']) 192 | 193 | >>> colors = np.random.randint(255, size=(10,3), dtype=np.uint8) 194 | >>> field_names = ['x', 'y', 'z', 'red', 'green', 'blue', values'] 195 | >>> write_ply('example3.ply', [points, colors, values], field_names) 196 | 197 | """ 198 | 199 | # Format list input to the right form 200 | field_list = ( 201 | list(field_list) 202 | if (type(field_list) == list or type(field_list) == tuple) 203 | else list((field_list,)) 204 | ) 205 | for i, field in enumerate(field_list): 206 | if field is None: 207 | print("WRITE_PLY ERROR: a field is None") 208 | return False 209 | elif field.ndim > 2: 210 | print("WRITE_PLY ERROR: a field have more than 2 dimensions") 211 | return False 212 | elif field.ndim < 2: 213 | field_list[i] = field.reshape(-1, 1) 214 | 215 | # check all fields have the same number of data 216 | n_points = [field.shape[0] for field in field_list] 217 | if not np.all(np.equal(n_points, n_points[0])): 218 | print("wrong field dimensions") 219 | return False 220 | 221 | # Check if field_names and field_list have same nb of column 222 | n_fields = np.sum([field.shape[1] for field in field_list]) 223 | if n_fields != len(field_names): 224 | print("wrong number of field names") 225 | return False 226 | 227 | # Add extension if not there 228 | if not filename.endswith(".ply"): 229 | filename += ".ply" 230 | 231 | # open in text mode to write the header 232 | with open(filename, "w") as plyfile: 233 | 234 | # First magical word 235 | header = ["ply"] 236 | 237 | # Encoding format 238 | header.append("format binary_" + sys.byteorder + "_endian 1.0") 239 | 240 | # Points properties description 241 | header.extend(header_properties(field_list, field_names)) 242 | 243 | # End of header 244 | header.append("end_header") 245 | 246 | # Write all lines 247 | for line in header: 248 | plyfile.write("%s\n" % line) 249 | 250 | # open in binary/append to use tofile 251 | with open(filename, "ab") as plyfile: 252 | 253 | # Create a structured array 254 | i = 0 255 | type_list = [] 256 | for fields in field_list: 257 | for field in fields.T: 258 | type_list += [(field_names[i], field.dtype.str)] 259 | i += 1 260 | data = np.empty(field_list[0].shape[0], dtype=type_list) 261 | i = 0 262 | for fields in field_list: 263 | for field in fields.T: 264 | data[field_names[i]] = field 265 | i += 1 266 | 267 | data.tofile(plyfile) 268 | 269 | return True 270 | 271 | 272 | def describe_element(name, df): 273 | """ Takes the columns of the dataframe and builds a ply-like description 274 | 275 | Parameters 276 | ---------- 277 | name: str 278 | df: pandas DataFrame 279 | 280 | Returns 281 | ------- 282 | element: list[str] 283 | """ 284 | property_formats = {"f": "float", "u": "uchar", "i": "int"} 285 | element = ["element " + name + " " + str(len(df))] 286 | 287 | if name == "face": 288 | element.append("property list uchar int points_indices") 289 | 290 | else: 291 | for i in range(len(df.columns)): 292 | # get first letter of dtype to infer format 293 | f = property_formats[str(df.dtypes[i])[0]] 294 | element.append("property " + f + " " + df.columns.values[i]) 295 | 296 | return element 297 | 298 | 299 | def ply2dict(filename): 300 | data = read_ply(filename) 301 | dictionary = {} 302 | for name in data.dtype.names: 303 | dictionary[name] = data[name] 304 | return dictionary 305 | 306 | 307 | def dict2ply(dictionary, filename): 308 | # fields = [f.reshape(-1, 1) for f in list(dictionary.values())] 309 | fields = list(dictionary.values()) 310 | names = list(dictionary.keys()) 311 | 312 | return write_ply(filename, fields, names) 313 | --------------------------------------------------------------------------------