├── readme ├── mesh.jpg ├── distant.png ├── flavour │ ├── label │ │ ├── mask.png │ │ └── image.jpg │ ├── mesh_models │ │ ├── ring.jpg │ │ ├── nm_grid.jpg │ │ ├── xm_grid.jpg │ │ └── xy_grid.jpg │ ├── view.md │ ├── example.md │ └── orientation.md ├── testing.md ├── architecture.md ├── setup.md ├── dataset.md └── quickstart.md ├── pyproject.toml ├── example ├── images │ ├── image0000000.jpg │ ├── image0000001.jpg │ ├── image0000002.jpg │ ├── image0000003.jpg │ ├── image0000004.jpg │ ├── image0000005.jpg │ ├── image0000006.jpg │ ├── image0000007.jpg │ ├── image0000008.jpg │ ├── image0000009.jpg │ ├── image0000010.jpg │ ├── image0000011.jpg │ ├── image0000012.jpg │ ├── image0000013.jpg │ ├── image0000014.jpg │ ├── image0000015.jpg │ ├── image0000016.jpg │ ├── image0000017.jpg │ ├── image0000018.jpg │ ├── image0000019.jpg │ ├── image0000020.jpg │ ├── image0000021.jpg │ ├── image0000022.jpg │ ├── image0000023.jpg │ ├── image0000024.jpg │ ├── image0000025.jpg │ ├── image0000026.jpg │ ├── image0000027.jpg │ ├── image0000028.jpg │ ├── image0000029.jpg │ ├── lens0000001.yaml │ ├── lens0000005.yaml │ ├── lens0000009.yaml │ ├── lens0000011.yaml │ ├── lens0000012.yaml │ ├── lens0000017.yaml │ ├── lens0000020.yaml │ ├── lens0000021.yaml │ ├── lens0000023.yaml │ ├── lens0000029.yaml │ ├── lens0000000.yaml │ ├── lens0000002.yaml │ ├── lens0000003.yaml │ ├── lens0000006.yaml │ ├── lens0000008.yaml │ ├── lens0000010.yaml │ ├── lens0000014.yaml │ ├── lens0000015.yaml │ ├── lens0000016.yaml │ ├── lens0000018.yaml │ ├── lens0000019.yaml │ ├── lens0000025.yaml │ ├── lens0000026.yaml │ ├── lens0000027.yaml │ ├── lens0000028.yaml │ ├── lens0000004.yaml │ ├── lens0000007.yaml │ ├── lens0000013.yaml │ ├── lens0000022.yaml │ └── lens0000024.yaml ├── Timer.hpp ├── CMakeLists.txt ├── load_model.hpp ├── ArrayPrint.hpp └── draw.hpp ├── .cmake-format.py ├── .dockerignore ├── cmake ├── VisualMeshConfig.cmake.in ├── Scripts │ ├── clang-tidy.py │ └── wrap_opencl.py └── Modules │ ├── FindSPIRV.cmake │ └── FindTensorFlow.cmake ├── .vscode ├── extensions.json └── settings.json ├── training ├── dataset │ ├── projection │ │ └── __init__.py │ ├── orientation │ │ ├── __init__.py │ │ ├── ground.py │ │ └── random_rotation.py │ ├── view │ │ ├── __init__.py │ │ └── monoscopic.py │ ├── label │ │ ├── __init__.py │ │ └── classification.py │ ├── example │ │ └── __init__.py │ └── __init__.py ├── flavour │ ├── dataset.py │ ├── __init__.py │ ├── loss.py │ ├── merge_configuration.py │ ├── metrics.py │ └── image_callback.py ├── op │ └── __init__.py ├── __init__.py ├── model │ └── __init__.py ├── loss │ ├── __init__.py │ └── focal_loss.py ├── layer │ ├── __init__.py │ ├── graph_convolution.py │ └── depthwise_seperable_graph_convolution.py ├── callbacks │ ├── __init__.py │ └── one_cycle.py ├── metrics │ ├── test │ │ ├── __init__.py │ │ ├── bucket.py │ │ └── confusion.py │ ├── __init__.py │ ├── average_recall.py │ ├── class_recall.py │ ├── average_precision.py │ ├── class_precision.py │ ├── seeker_recall.py │ ├── seeker_precision.py │ ├── confusion_base.py │ ├── seeker_confusion_base.py │ └── seeker_stddev.py ├── projection │ └── __init__.py ├── testing.py └── unexport.py ├── .gitignore ├── tensorflow ├── CMakeLists.txt └── shape_op_base.hpp ├── LICENSE ├── .clang-tidy ├── cpp ├── visualmesh │ ├── model │ │ ├── nmgrid4.hpp │ │ ├── nmgrid6.hpp │ │ ├── nmgrid8.hpp │ │ ├── xmgrid4.hpp │ │ ├── xmgrid6.hpp │ │ ├── xmgrid8.hpp │ │ ├── xygrid4.hpp │ │ ├── xygrid6.hpp │ │ ├── xygrid8.hpp │ │ ├── ring4.hpp │ │ ├── ring6.hpp │ │ ├── ring8.hpp │ │ └── xygrid_map.hpp │ ├── projected_mesh.hpp │ ├── utility │ │ ├── static_if.hpp │ │ └── fourcc.hpp │ ├── network_structure.hpp │ ├── node.hpp │ ├── lens.hpp │ ├── engine │ │ └── opencl │ │ │ └── operation │ │ │ ├── make_queue.hpp │ │ │ ├── scalar_defines.hpp │ │ │ └── wrapper.hpp │ └── classified_mesh.hpp └── CMakeLists.txt ├── Dockerfile ├── .clang-format └── CMakeLists.txt /readme/mesh.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/mesh.jpg -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length=120 3 | 4 | [tool.isort] 5 | line_length=120 6 | -------------------------------------------------------------------------------- /readme/distant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/distant.png -------------------------------------------------------------------------------- /readme/flavour/label/mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/flavour/label/mask.png -------------------------------------------------------------------------------- /example/images/image0000000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000000.jpg -------------------------------------------------------------------------------- /example/images/image0000001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000001.jpg -------------------------------------------------------------------------------- /example/images/image0000002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000002.jpg -------------------------------------------------------------------------------- /example/images/image0000003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000003.jpg -------------------------------------------------------------------------------- /example/images/image0000004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000004.jpg -------------------------------------------------------------------------------- /example/images/image0000005.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000005.jpg -------------------------------------------------------------------------------- /example/images/image0000006.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000006.jpg -------------------------------------------------------------------------------- /example/images/image0000007.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000007.jpg -------------------------------------------------------------------------------- /example/images/image0000008.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000008.jpg -------------------------------------------------------------------------------- /example/images/image0000009.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000009.jpg -------------------------------------------------------------------------------- /example/images/image0000010.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000010.jpg -------------------------------------------------------------------------------- /example/images/image0000011.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000011.jpg -------------------------------------------------------------------------------- /example/images/image0000012.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000012.jpg -------------------------------------------------------------------------------- /example/images/image0000013.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000013.jpg -------------------------------------------------------------------------------- /example/images/image0000014.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000014.jpg -------------------------------------------------------------------------------- /example/images/image0000015.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000015.jpg -------------------------------------------------------------------------------- /example/images/image0000016.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000016.jpg -------------------------------------------------------------------------------- /example/images/image0000017.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000017.jpg -------------------------------------------------------------------------------- /example/images/image0000018.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000018.jpg -------------------------------------------------------------------------------- /example/images/image0000019.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000019.jpg -------------------------------------------------------------------------------- /example/images/image0000020.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000020.jpg -------------------------------------------------------------------------------- /example/images/image0000021.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000021.jpg -------------------------------------------------------------------------------- /example/images/image0000022.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000022.jpg -------------------------------------------------------------------------------- /example/images/image0000023.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000023.jpg -------------------------------------------------------------------------------- /example/images/image0000024.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000024.jpg -------------------------------------------------------------------------------- /example/images/image0000025.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000025.jpg -------------------------------------------------------------------------------- /example/images/image0000026.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000026.jpg -------------------------------------------------------------------------------- /example/images/image0000027.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000027.jpg -------------------------------------------------------------------------------- /example/images/image0000028.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000028.jpg -------------------------------------------------------------------------------- /example/images/image0000029.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/example/images/image0000029.jpg -------------------------------------------------------------------------------- /readme/flavour/label/image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/flavour/label/image.jpg -------------------------------------------------------------------------------- /readme/flavour/mesh_models/ring.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/flavour/mesh_models/ring.jpg -------------------------------------------------------------------------------- /readme/flavour/mesh_models/nm_grid.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/flavour/mesh_models/nm_grid.jpg -------------------------------------------------------------------------------- /readme/flavour/mesh_models/xm_grid.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/flavour/mesh_models/xm_grid.jpg -------------------------------------------------------------------------------- /readme/flavour/mesh_models/xy_grid.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fastcode/VisualMesh/HEAD/readme/flavour/mesh_models/xy_grid.jpg -------------------------------------------------------------------------------- /.cmake-format.py: -------------------------------------------------------------------------------- 1 | # How wide to allow formatted cmake files 2 | line_width = 120 3 | 4 | # How many spaces to tab for indent 5 | tab_size = 4 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | * 3 | 4 | # Except for the files related to building the tensorflow op 5 | !CMakeLists.txt 6 | !cpp/* 7 | !tensorflow/* 8 | !cmake/* 9 | -------------------------------------------------------------------------------- /example/images/lens0000001.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.80248 6 | Hoc: 7 | - [-0.244225, -0.905877, 0.346036, 0] 8 | - [-0.0304556, 0.363831, 0.930967, 0] 9 | - [-0.96924, 0.216827, -0.116446, 0.8] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000005.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 807.17 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.70174 6 | Hoc: 7 | - [0.3651, -0.695642, 0.618695, 0] 8 | - [0.643041, 0.669002, 0.372738, 0] 9 | - [-0.673201, 0.26176, 0.69158, 0.85] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000009.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.29277 6 | Hoc: 7 | - [-0.0943642, -0.912074, 0.399019, 0] 8 | - [0.765975, 0.189501, 0.614305, 0] 9 | - [-0.635906, 0.363607, 0.680744, 0.79] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000011.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.59286 6 | Hoc: 7 | - [-0.308872, 0.945477, 0.103304, 0] 8 | - [-0.84664, -0.223831, -0.482805, 0] 9 | - [-0.433359, -0.236586, 0.869613, 0.9] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000012.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.8691 6 | Hoc: 7 | - [0.430391, -0.108123, 0.896143, 0] 8 | - [0.734872, 0.618468, -0.278316, 0] 9 | - [-0.524144, 0.778336, 0.34564, 1.04] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000017.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.77693 6 | Hoc: 7 | - [-0.119489, 0.665166, -0.737073, 0] 8 | - [-0.171253, -0.745073, -0.644623, 0] 9 | - [-0.977954, 0.0492007, 0.20294, 1] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000020.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.43933 6 | Hoc: 7 | - [0.202951, -0.627579, 0.751635, 0] 8 | - [-0.503845, 0.591263, 0.629721, 0] 9 | - [-0.839614, -0.50651, -0.196205, 1.18] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000021.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.80718 6 | Hoc: 7 | - [-0.072171, 0.39044, 0.917795, 0] 8 | - [-0.471497, 0.797527, -0.376353, 0] 9 | - [-0.878909, -0.459899, 0.126533, 0.8] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000023.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.47912 6 | Hoc: 7 | - [0.812205, 0.518554, 0.267256, 0] 8 | - [-0.501786, 0.854656, -0.133324, 0] 9 | - [-0.297547, -0.025819, 0.954358, 1.15] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000029.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.15361 6 | Hoc: 7 | - [-0.546473, -0.360091, -0.75611, 0] 8 | - [0.442768, -0.890571, 0.104119, 0] 9 | - [-0.710861, -0.277883, 0.64611, 1.18] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000000.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 868.041 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.66562 6 | Hoc: 7 | - [-0.316188, -0.144188, 0.937675, 0] 8 | - [0.181584, 0.960911, 0.208992, 0] 9 | - [-0.931157, 0.236348, -0.277646, 1.2] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000002.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 876.805 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.66062 6 | Hoc: 7 | - [0.355455, 0.913602, 0.197446, 0] 8 | - [-0.886057, 0.396609, -0.24001, 0] 9 | - [-0.297582, -0.0896359, 0.95048, 0.9] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000003.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 968.415 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.61104 6 | Hoc: 7 | - [0.102792, 0.934841, 0.339862, 0] 8 | - [0.525315, 0.23912, -0.816618, 0] 9 | - [-0.844676, 0.262477, -0.466506, 0.91] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000006.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 861.603 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.66932 6 | Hoc: 7 | - [0.745791, -0.435725, 0.503924, 0] 8 | - [-0.373565, 0.352781, 0.857901, 0] 9 | - [-0.551584, -0.828063, 0.100329, 0.8] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000008.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.62449 6 | Hoc: 7 | - [0.192518, 0.537159, 0.821217, 0] 8 | - [-0.13231, 0.843439, -0.520677, 0] 9 | - [-0.972333, -0.00841571, 0.233448, 0.85] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000010.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 666.323 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.79606 6 | Hoc: 7 | - [0.691674, -0.700795, -0.174567, 0] 8 | - [0.65984, 0.51494, 0.54722, 0] 9 | - [-0.293598, -0.493683, 0.818583, 1.07] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000014.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 1303.65 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.46263 6 | Hoc: 7 | - [0.38568, 0.266973, 0.883163, 0] 8 | - [-0.426309, 0.900477, -0.0860361, 0] 9 | - [-0.818236, -0.343318, 0.461109, 1] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000015.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.53731 6 | Hoc: 7 | - [-0.243588, -0.334629, -0.910323, 0] 8 | - [0.231476, -0.93153, 0.280485, 0] 9 | - [-0.941851, -0.142395, 0.304368, 1.155] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000016.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.42449 6 | Hoc: 7 | - [-0.49091, 0.732683, -0.471363, 0] 8 | - [-0.740414, -0.635998, -0.217473, 0] 9 | - [-0.459125, 0.242244, 0.854706, 1.05] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000018.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 641.168 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.81478 6 | Hoc: 7 | - [-0.847717, 0.17231, -0.501682, 0] 8 | - [-0.415604, -0.803451, 0.426311, 0] 9 | - [-0.329619, 0.569892, 0.752711, 1.2] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000019.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.68629 6 | Hoc: 7 | - [-0.257506, 0.442693, -0.858902, 0] 8 | - [-0.757062, -0.644791, -0.105363, 0] 9 | - [-0.600455, 0.623111, 0.501184, 0.78] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000025.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 2.72 6 | Hoc: 7 | - [-0.351826, -0.0251788, -0.935727, 0] 8 | - [0.367305, -0.923179, -0.113263, 0] 9 | - [-0.860991, -0.383546, 0.334047, 1.18] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000026.yaml: -------------------------------------------------------------------------------- 1 | projection: EQUISOLID 2 | focal_length: 420 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 3.06337 6 | Hoc: 7 | - [0.515385, 0.854798, -0.0608212, 0] 8 | - [-0.829754, 0.480023, -0.284757, 0] 9 | - [-0.214214, 0.197226, 0.956669, 0.91] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000027.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 702.465 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.77025 6 | Hoc: 7 | - [0.467327, -0.518072, 0.716385, 0] 8 | - [0.357837, 0.851814, 0.382578, 0] 9 | - [-0.808429, 0.07756, 0.583461, 0.85] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000028.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 1358.06 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.44233 6 | Hoc: 7 | - [0.446279, -0.888994, 0.102591, 0] 8 | - [0.854602, 0.457388, 0.24587, 0] 9 | - [-0.2655, -0.0220524, 0.963858, 0.85] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000004.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 990.316 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.59986 6 | Hoc: 7 | - [0.614168, 0.784489, 0.0858825, 0] 8 | - [-0.674782, 0.578455, -0.458323, 0] 9 | - [-0.409228, 0.223535, 0.884627, 1.15] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000007.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 644.708 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.8121 6 | Hoc: 7 | - [0.952649, -0.0339823, 0.302167, 0] 8 | - [-0.077588, 0.933675, 0.349616, 0] 9 | - [-0.294007, -0.356506, 0.886828, 0.78] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000013.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 894.642 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.65058 6 | Hoc: 7 | - [0.0845476, -0.967505, -0.238297, 0] 8 | - [0.952746, 0.0084689, 0.303649, 0] 9 | - [-0.291764, -0.252709, 0.922503, 1.07] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000022.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 1367.32 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.43896 6 | Hoc: 7 | - [-0.470696, -0.882268, -0.00701099, 0] 8 | - [0.119312, -0.0715228, 0.990277, 0] 9 | - [-0.874191, 0.465283, 0.13893, 1.2] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /example/images/lens0000024.yaml: -------------------------------------------------------------------------------- 1 | projection: RECTILINEAR 2 | focal_length: 672.103 3 | centre: [0, 0] 4 | k: [0, 0] 5 | fov: 1.79185 6 | Hoc: 7 | - [0.274291, 0.669558, -0.690259, 0] 8 | - [0.286749, -0.742082, -0.60588, 0] 9 | - [-0.9179, -0.0317439, -0.395541, 1.07] 10 | - [0, 0, 0, 1] 11 | -------------------------------------------------------------------------------- /cmake/VisualMeshConfig.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include("${CMAKE_CURRENT_LIST_DIR}/VisualMeshTargets.cmake") 4 | 5 | set_and_check(VisualMesh_INCLUDE_DIR "@PACKAGE_INSTALL_INCLUDE_DIR@") 6 | set_and_check(VisualMesh_INCLUDE_DIRS "@PACKAGE_INSTALL_INCLUDE_DIR@") 7 | set(VisualMesh_LIBRARIES visualmesh::visualmesh) 8 | 9 | check_required_components("@PROJECT_NAME@") 10 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-vscode.cpptools", 4 | "twxs.cmake", 5 | "ms-vscode.cmake-tools", 6 | "cheshirekow.cmake-format", 7 | "ms-azuretools.vscode-docker", 8 | "galarius.vscode-opencl", 9 | "ms-python.python", 10 | "redhat.vscode-yaml", 11 | "streetsidesoftware.code-spell-checker", 12 | "cschlosser.doxdocgen" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.formatting.provider": "black", 3 | "editor.formatOnSave": true, 4 | "editor.tabSize": 4, 5 | "editor.insertSpaces": true, 6 | "files.insertFinalNewline": true, 7 | "files.trimFinalNewlines": true, 8 | "files.trimTrailingWhitespace": true, 9 | "files.eol": "\n", 10 | "editor.rulers": [ 11 | 120 12 | ], 13 | "cSpell.language": "en-GB", 14 | "yaml.format.printWidth": 200 15 | } 16 | -------------------------------------------------------------------------------- /training/dataset/projection/__init__.py: -------------------------------------------------------------------------------- 1 | # Projection functions should take an orientation and other properties and return projected coordinates 2 | # The returned data should contain at least the following 3 | # { 4 | # 'G': The graph used to connect the results with -1 used as the off screen point 5 | # 'C': The pixel coordinates of each the visual mesh nodes 6 | # 'V': The unit vectors from the camera in observation plane space the pixels represent 7 | # } 8 | 9 | from .visual_mesh import VisualMesh 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | 6 | # Compiled Dynamic libraries 7 | *.so 8 | *.dylib 9 | 10 | # Compiled Static libraries 11 | *.lai 12 | *.la 13 | *.a 14 | 15 | # Build & CMake files 16 | build/ 17 | CMakeCache.txt 18 | CMakeFiles 19 | Makefile 20 | cmake_install.cmake 21 | install_manifest.txt 22 | 23 | .exe 24 | tags 25 | *.swp 26 | .idea 27 | docs/doxygen 28 | docs/doxygen_sqlite3.db 29 | *.sublime-workspace 30 | 31 | # Compiled python 32 | *.pyc 33 | 34 | # Default output location for data 35 | /output/ 36 | 37 | # Location for raw input data 38 | /dataset/ 39 | 40 | # Miscellaneous 41 | .DS_Store 42 | \.vscode/ipch/ 43 | -------------------------------------------------------------------------------- /training/flavour/dataset.py: -------------------------------------------------------------------------------- 1 | import training.dataset as ds 2 | 3 | from .merge_configuration import merge_configuration 4 | 5 | 6 | def Dataset(config, split, batch_size=None): 7 | 8 | dataset_config = config["dataset"][split] 9 | specific_config = merge_configuration(config, dataset_config.get("config", {})) 10 | 11 | # Datasets work out the flavour themselves provided they are given the correct configuration 12 | return ds.Dataset( 13 | paths=dataset_config["paths"], 14 | batch_size=batch_size if batch_size is not None else dataset_config["batch_size"], 15 | keys=dataset_config.get("keys", {}), 16 | **specific_config, 17 | ) 18 | -------------------------------------------------------------------------------- /tensorflow/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Add this directory so we can find the FindTensorflow.cmake 2 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake/Modules/") 3 | find_package(TensorFlow REQUIRED) 4 | 5 | add_library(tf_op SHARED "map.cpp" "unmap.cpp" "lookup.cpp" "difference.cpp" ${hdr}) 6 | target_compile_options(tf_op PRIVATE -march=native -mtune=native) 7 | set_target_properties( 8 | tf_op 9 | PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${PROJECT_SOURCE_DIR}/training/op" 10 | PREFIX "" 11 | OUTPUT_NAME visualmesh_op 12 | SUFFIX ".so") 13 | target_include_directories(tf_op SYSTEM PRIVATE ${TENSORFLOW_INCLUDE_DIRS}) 14 | target_link_libraries(tf_op visualmesh ${TENSORFLOW_LIBRARIES}) 15 | -------------------------------------------------------------------------------- /cmake/Scripts/clang-tidy.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import hashlib 4 | import os 5 | import subprocess 6 | import sys 7 | 8 | # Get the output folder from the arguments 9 | output_folder = sys.argv[1] 10 | clang_tidy_binary = sys.argv[2] 11 | 12 | os.makedirs(output_folder, exist_ok=True) 13 | 14 | # Hash the remaining arguments as the output file name 15 | output_file = "{}.yaml".format(hashlib.sha256(" ".join(sys.argv[2:]).encode("utf-8")).hexdigest()) 16 | 17 | # Work out our arguments to call the clang-tidy binary with 18 | args = [clang_tidy_binary, "--export-fixes={}".format(os.path.join(output_folder, output_file)), *sys.argv[3:]] 19 | result = subprocess.run(args) 20 | 21 | # Pass the return code back to the caller 22 | exit(result.returncode) 23 | -------------------------------------------------------------------------------- /readme/testing.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | Testing in the Visual Mesh code is run after training in a similar way to training. 3 | 4 | For example run the following using docker 5 | ```sh 6 | ./docker run --gpus all -u $(id -u):$(id -g) -it --rm --volume $(pwd):/workspace visualmesh:latest ./mesh.py test 7 | ``` 8 | 9 | The output of the tests will be placed into the output directory. 10 | This includes several graph images as well as some statistics. 11 | 12 | Due to the way these graphs are produced, they will be much more accurate than if a simple histogram were used and can be used to make much more complex shapes. 13 | However, they unfortunately can suffer from an issue at the ends of the plot. 14 | The easiest way to fix this is to delete the start/end of the data in the CSV file to remove the erroneous sections. 15 | -------------------------------------------------------------------------------- /readme/flavour/view.md: -------------------------------------------------------------------------------- 1 | # View 2 | The view controls how multiple cameras can be used together to make a multi-view visual mesh. 3 | It is able to add a prefix to all the dataset elements used by other flavours and provide a combine function at the end to apply the multi-view transformations. 4 | Often the expanded keys will have some overlap and different views will have the same data. 5 | In this case, you can use the keys field in the dataset to remap the features from multiple views back to a single input. 6 | 7 | ## Monoscopic 8 | The monoscopic view is the traditional single-camera visual mesh mode with no fancy multi-camera options. 9 | There is no configuration for this view and it does not impact the keys required by other flavour components. 10 | 11 | ### Dataset Keys 12 | No dataset keys are required for Monoscopic view 13 | ``` 14 | None 15 | ``` 16 | 17 | ### Configuration 18 | ```yaml 19 | view: 20 | type: Monoscopic 21 | config: {} 22 | ``` 23 | -------------------------------------------------------------------------------- /training/op/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import tensorflow as tf 4 | 5 | # If we are in docker look for the visual mesh op in /visualmesh/training/visualmesh_op.so 6 | if ( 7 | os.path.exists("/.dockerenv") 8 | or os.path.isfile("/proc/self/cgroup") 9 | and any("docker" in line for line in open("/proc/self/cgroup")) 10 | ) and os.path.isfile("/visualmesh/training/op/visualmesh_op.so"): 11 | _library = tf.load_op_library("/visualmesh/training/op/visualmesh_op.so") 12 | # Otherwise check to see if we built it and it should be right next to this file 13 | elif os.path.isfile(os.path.join(os.path.dirname(__file__), "visualmesh_op.so")): 14 | _library = tf.load_op_library(os.path.join(os.path.dirname(__file__), "visualmesh_op.so")) 15 | else: 16 | raise Exception("Please build the tensorflow visual mesh op before running") 17 | 18 | lookup_visual_mesh = _library.lookup_visual_mesh 19 | map_visual_mesh = _library.map_visual_mesh 20 | unmap_visual_mesh = _library.unmap_visual_mesh 21 | difference_visual_mesh = _library.difference_visual_mesh 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2017-2020 Trent Houliston 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the Software), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /training/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | -------------------------------------------------------------------------------- /training/model/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from .visual_mesh_model import VisualMeshModel 17 | -------------------------------------------------------------------------------- /training/loss/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from .focal_loss import FocalLoss 17 | from .seeker_loss import SeekerLoss 18 | -------------------------------------------------------------------------------- /training/layer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from .graph_convolution import GraphConvolution 17 | from .depthwise_seperable_graph_convolution import DepthwiseSeparableGraphConvolution 18 | -------------------------------------------------------------------------------- /training/flavour/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from .dataset import Dataset 17 | from .image_callback import ImageCallback 18 | from .loss import Loss 19 | from .metrics import Metrics 20 | from .test_metrics import TestMetrics 21 | -------------------------------------------------------------------------------- /training/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from .classification_images import ClassificationImages 17 | from .one_cycle import OneCycle 18 | from .seeker_images import SeekerImages 19 | from .image_tensorboard import ImageTensorBoard 20 | -------------------------------------------------------------------------------- /training/metrics/test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .confusion import Confusion 19 | from .confusion_curve import ConfusionCurve 20 | from .curve import Curve 21 | from .seeker_hourglass import SeekerHourglass 22 | -------------------------------------------------------------------------------- /training/dataset/orientation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | # These operations should at least a single output as follows 17 | # { 18 | # "Hoc": The homogenous transform from the camera space to the observation plane 19 | # } 20 | 21 | from .ground import Ground 22 | from .spotlight import Spotlight 23 | -------------------------------------------------------------------------------- /training/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from .average_precision import AveragePrecision 17 | from .average_recall import AverageRecall 18 | from .class_precision import ClassPrecision 19 | from .class_recall import ClassRecall 20 | from .seeker_precision import SeekerPrecision 21 | from .seeker_recall import SeekerRecall 22 | from .seeker_stddev import SeekerStdDev 23 | -------------------------------------------------------------------------------- /training/dataset/view/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | # Views define how to handle training data that is taken from multiple cameras 17 | # It provides a prefix function that will identify each of the camera inputs 18 | # and a merge function that will take the results from the individual stages and merge them into a single training example 19 | 20 | from .monoscopic import Monoscopic 21 | -------------------------------------------------------------------------------- /training/dataset/label/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | # The labels should return at least two values in the dictionary 17 | # { 18 | # "Y": The labels that the network will be trained against 19 | # "W": The weighting for each point with 1.0 being full, and 0.0 being ignore the point for training 20 | # } 21 | 22 | from .classification import Classification 23 | from .seeker import Seeker 24 | -------------------------------------------------------------------------------- /training/metrics/test/bucket.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | def curve_bucket(x, y): 20 | return tf.math.sqrt(tf.add(tf.math.squared_difference(x[:-1], x[1:]), tf.math.squared_difference(y[:-1], y[1:]),)) 21 | 22 | 23 | def x_bucket(x, y): 24 | return tf.math.abs(x[1:] - x[:-1]) 25 | 26 | 27 | def y_bucket(x, y): 28 | return tf.math.abs(x[1:] - x[:-1]) 29 | -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | Checks: > 2 | -*, 3 | bugprone-*, 4 | cert-*, 5 | clang-diagnostic-*, 6 | clang-analyzer-*, 7 | cppcoreguidelines-*, 8 | -cppcoreguidelines-avoid-magic-numbers, 9 | -cppcoreguidelines-non-private-member-variables-in-classes, 10 | -cppcoreguidelines-pro-bounds-array-to-pointer-decay, 11 | -cppcoreguidelines-pro-bounds-constant-array-index, 12 | -cppcoreguidelines-pro-bounds-pointer-arithmetic, 13 | -cppcoreguidelines-pro-type-reinterpret-cast, 14 | -cppcoreguidelines-pro-type-union-access, 15 | -cppcoreguidelines-pro-type-vararg, 16 | google-*, 17 | -google-explicit-constructor, 18 | -google-runtime-references, 19 | misc-*, 20 | -misc-non-private-member-variables-in-classes, 21 | -misc-no-recursion, 22 | performance-*, 23 | readability-*, 24 | -readability-uppercase-literal-suffix, 25 | -readability-magic-numbers, 26 | modernize-*, 27 | -modernize-use-trailing-return-type 28 | WarningsAsErrors: "" 29 | HeaderFilterRegex: ".*" 30 | AnalyzeTemporaryDtors: false 31 | FormatStyle: file 32 | CheckOptions: 33 | - key: cppcoreguidelines-avoid-magic-numbers.IgnoredFloatingPointValues 34 | value: "0.5;1.0;2.0;3.0;4.0;100.0;" 35 | - key: readability-magic-numbers.IgnoredFloatingPointValues 36 | value: "0.5;1.0;2.0;3.0;4.0;100.0;" 37 | - key: cppcoreguidelines-avoid-magic-numbers.IgnoredIntegerValues 38 | value: "1;2;3;4;" 39 | - key: readability-magic-numbers.IgnoredIntegerValues 40 | value: "1;2;3;4;" 41 | - key: modernize-loop-convert.NamingStyle 42 | value: lower_case 43 | -------------------------------------------------------------------------------- /training/flavour/loss.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from ..loss import * 17 | 18 | 19 | def Loss(config): 20 | 21 | if config["label"]["type"] == "Classification": 22 | return FocalLoss() 23 | 24 | elif config["label"]["type"] == "Seeker": 25 | return SeekerLoss() 26 | 27 | else: 28 | raise RuntimeError("Cannot create loss function, {} is not a supported type".format(config["label"]["type"])) 29 | -------------------------------------------------------------------------------- /training/dataset/example/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | # These functions should take a feature to act as the X value for the network 17 | # They should return at least two things for proper function of the training 18 | # { 19 | # 'X': # The value for each of the projected visual mesh points 20 | # 'image': # The raw compressed image for use in tensorboard display records 21 | # } 22 | # {} 23 | 24 | from .image import Image 25 | -------------------------------------------------------------------------------- /training/layer/graph_convolution.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | class GraphConvolution(tf.keras.layers.Layer): 20 | def __init__(self, **kwargs): 21 | super(GraphConvolution, self).__init__() 22 | self.dense = tf.keras.layers.Dense(**kwargs) 23 | 24 | def call(self, X, G): 25 | # Call the dense layer with the gathered data 26 | return self.dense(tf.reshape(tf.gather(X, G, name="NetworkGather"), shape=[-1, X.shape[-1] * G.shape[-1]])) 27 | -------------------------------------------------------------------------------- /readme/architecture.md: -------------------------------------------------------------------------------- 1 | # Flavours 2 | The Visual Mesh code is designed to allow the easy implementation of new use cases. 3 | It allows this by providing several different components that can be combined to make a vision system. 4 | In the code, these components are referred to as flavours. 5 | There are five different categories for these that are each discussed on their own pages. 6 | 7 | [View](flavour/view.md) describes how the system is set up in terms of cameras. 8 | It can be used to implement features like Stereoscopic or Multi-Camera systems. 9 | 10 | [Orientation](flavour/orientation.md) describes how the visual mesh's observation plane is set up from the perspective of the camera. 11 | For example, the traditional position for the observation plane is on the ground below the camera. 12 | However, if you were to attach the plane to another flat surface this flavour can provide the tools to do that. 13 | 14 | [Projection](flavour/projection.md) describes how the visual mesh itself is designed and projected into pixel coordinates. 15 | It includes details about which visual mesh model to use and also contains the logic that uses lens parameters to project to image space. 16 | 17 | [Example](flavour/example.md) describes how the training input examples are provided. 18 | For example, the Image method will load images from the dataset and grab the pixels projected by the visual mesh. 19 | If you find some other wonderful way to use the visual mesh besides images put it here! 20 | 21 | [Label](flavour/label.md) describes how the training labels are provided. 22 | For a classification network this would be the mask image that the labels are gathered from. 23 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/nmgrid4.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_NMGRID4_HPP 19 | #define VISUALMESH_MODEL_NMGRID4_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "nmgrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct NMGrid4 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 4; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_NMGRID4_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/nmgrid6.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_NMGRID6_HPP 19 | #define VISUALMESH_MODEL_NMGRID6_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "nmgrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct NMGrid6 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 6; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_NMGRID6_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/nmgrid8.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_NMGRID8_HPP 19 | #define VISUALMESH_MODEL_NMGRID8_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "nmgrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct NMGrid8 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 8; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_NMGRID8_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/xmgrid4.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_XMGRID4_HPP 19 | #define VISUALMESH_MODEL_XMGRID4_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "xmgrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct XMGrid4 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 4; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_XMGRID4_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/xmgrid6.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_XMGRID6_HPP 19 | #define VISUALMESH_MODEL_XMGRID6_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "xmgrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct XMGrid6 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 6; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_XMGRID6_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/xmgrid8.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_XMGRID8_HPP 19 | #define VISUALMESH_MODEL_XMGRID8_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "xmgrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct XMGrid8 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 8; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_XMGRID8_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/xygrid4.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_XYGRID4_HPP 19 | #define VISUALMESH_MODEL_XYGRID4_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "xygrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct XYGrid4 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 4; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_XYGRID4_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/xygrid6.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_XYGRID6_HPP 19 | #define VISUALMESH_MODEL_XYGRID6_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "xygrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct XYGrid6 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 6; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_XYGRID6_HPP 37 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/xygrid8.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_XYGRID8_HPP 19 | #define VISUALMESH_MODEL_XYGRID8_HPP 20 | 21 | #include "grid_base.hpp" 22 | #include "xygrid_map.hpp" 23 | 24 | namespace visualmesh { 25 | namespace model { 26 | 27 | template 28 | struct XYGrid8 : public GridBase { 29 | public: 30 | static constexpr int N_NEIGHBOURS = 8; 31 | }; 32 | 33 | } // namespace model 34 | } // namespace visualmesh 35 | 36 | #endif // VISUALMESH_MODEL_XYGRID8_HPP 37 | -------------------------------------------------------------------------------- /readme/flavour/example.md: -------------------------------------------------------------------------------- 1 | # Example 2 | Example flavours describe how the input image data will be provided to the network. 3 | They provide two essential functions. 4 | Firstly, via their `input` function, they describe how to generate the initial input data loaded from the dataset. 5 | Secondly, they take the projection from the visual mesh that describes which pixels are targeted and load those specific pixels from the input image. 6 | 7 | ## Image 8 | Image flavour takes an image that is stored in the dataset and provides it to the visual mesh. 9 | It can accept a wide variety of compressed image formats (anything that is accepted by [tf.image.decode_image](https://www.tensorflow.org/api_docs/python/tf/io/decode_image)) 10 | 11 | ### Dataset Keys 12 | ```python 13 | "image": bytes[1] # a compressed image (png, jpg, gif, etc) 14 | ``` 15 | 16 | ### Configuration 17 | Any or all of the augmentations can be left out if you do not wish to apply image augmentations when running. 18 | All augmentations are done with pixel values as floats which means that they go from 0->1, not 0->255. 19 | 20 | ```yaml 21 | example: 22 | type: Image 23 | config: 24 | augmentations: 25 | # Adjust the brightness `x + delta` 26 | brightness: { mean: 0, stddev: 0.05 } 27 | # Adjust the contrast `(x - mean) * delta + mean` 28 | contrast: { mean: 1, stddev: 0.05 } 29 | # Convert to hsv, adjust the hue by a value from [-1 -> 1] and back to rgb 30 | hue: { mean: 0, stddev: 0.05 } 31 | # Convert to hsv, multiply saturation by value and convert back to rgb 32 | saturation: { mean: 0, stddev: 0.05 } 33 | # Adjust the gamma `gain * x**gamma` 34 | gamma: 35 | gamma: { mean: 1, stddev: 0.05 } 36 | gain: { mean: 1, stddev: 0.05 } 37 | ``` 38 | -------------------------------------------------------------------------------- /training/metrics/average_recall.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .confusion_base import ConfusionBase 19 | 20 | 21 | class AverageRecall(ConfusionBase): 22 | def __init__(self, name, size, **kwargs): 23 | super(AverageRecall, self).__init__(name, size, **kwargs) 24 | 25 | def result(self): 26 | # True positives (predicted and labelled true) 27 | tp = tf.cast(tf.linalg.diag_part(self.confusion), self.dtype) 28 | # For all predictions where idx was labelled 29 | tp_fn = tf.cast(tf.reduce_sum(self.confusion, axis=1), self.dtype) 30 | 31 | return tf.math.reduce_mean(tp / tp_fn) 32 | -------------------------------------------------------------------------------- /training/dataset/view/monoscopic.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | class Monoscopic: 20 | def __init__(self, **config): 21 | pass 22 | 23 | def prefixes(self): 24 | return ("",) 25 | 26 | def merge(self, views): 27 | 28 | # We have only one view, so just choose it 29 | view = views[""] 30 | 31 | return { 32 | **{k: v for k, v in view.items() if len(v.shape) > 0 and v.shape[0] == None}, 33 | **{k: tf.expand_dims(v, 0) for k, v in view.items() if len(v.shape) == 0 or v.shape[0] != None}, 34 | "n": tf.expand_dims(tf.shape(view["G"])[0], 0), 35 | } 36 | -------------------------------------------------------------------------------- /training/metrics/class_recall.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .confusion_base import ConfusionBase 19 | 20 | 21 | class ClassRecall(ConfusionBase): 22 | def __init__(self, name, idx, size, **kwargs): 23 | super(ClassRecall, self).__init__(name, size, **kwargs) 24 | self.idx = idx 25 | 26 | def result(self): 27 | # True positives (predicted and labelled true) 28 | tp = tf.cast(self.confusion[self.idx, self.idx], self.dtype) 29 | # For all predictions where idx was labelled 30 | tp_fn = tf.cast(tf.reduce_sum(self.confusion[self.idx, :]), self.dtype) 31 | 32 | return tp / tp_fn 33 | -------------------------------------------------------------------------------- /training/metrics/average_precision.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .confusion_base import ConfusionBase 19 | 20 | 21 | class AveragePrecision(ConfusionBase): 22 | def __init__(self, name, size, **kwargs): 23 | super(AveragePrecision, self).__init__(name, size, **kwargs) 24 | 25 | def result(self): 26 | # True positives (predicted and labelled true) 27 | tp = tf.cast(tf.linalg.diag_part(self.confusion), self.dtype) 28 | # For all labels where idx was predicted (all positives) 29 | p = tf.cast(tf.reduce_sum(self.confusion, axis=0), self.dtype) 30 | 31 | return tf.math.reduce_mean(tp / p) 32 | -------------------------------------------------------------------------------- /training/metrics/class_precision.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .confusion_base import ConfusionBase 19 | 20 | 21 | class ClassPrecision(ConfusionBase): 22 | def __init__(self, name, idx, size, **kwargs): 23 | super(ClassPrecision, self).__init__(name, size, **kwargs) 24 | self.idx = idx 25 | 26 | def result(self): 27 | # True positives (predicted and labelled true) 28 | tp = tf.cast(self.confusion[self.idx, self.idx], self.dtype) 29 | # For all labels where idx was predicted (all positives) 30 | p = tf.cast(tf.reduce_sum(self.confusion[:, self.idx]), self.dtype) 31 | 32 | return tp / p 33 | -------------------------------------------------------------------------------- /training/metrics/seeker_recall.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .seeker_confusion_base import SeekerConfusionBase 19 | 20 | 21 | class SeekerRecall(SeekerConfusionBase): 22 | def __init__(self, name, threshold, **kwargs): 23 | super(SeekerRecall, self).__init__(name, threshold, **kwargs) 24 | 25 | def result(self): 26 | # True positives (predicted and labelled true) 27 | tp = tf.cast(tf.linalg.diag_part(self.confusion), self.dtype) 28 | # For all predictions where idx was labelled 29 | tp_fn = tf.cast(tf.reduce_sum(self.confusion, axis=1), self.dtype) 30 | 31 | return tf.math.reduce_mean(tp / tp_fn) 32 | -------------------------------------------------------------------------------- /training/metrics/seeker_precision.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .seeker_confusion_base import SeekerConfusionBase 19 | 20 | 21 | class SeekerPrecision(SeekerConfusionBase): 22 | def __init__(self, name, threshold, **kwargs): 23 | super(SeekerPrecision, self).__init__(name, threshold, **kwargs) 24 | 25 | def result(self): 26 | # True positives (predicted and labelled true) 27 | tp = tf.cast(tf.linalg.diag_part(self.confusion), self.dtype) 28 | # For all labels where idx was predicted (all positives) 29 | p = tf.cast(tf.reduce_sum(self.confusion, axis=0), self.dtype) 30 | 31 | return tf.math.reduce_mean(tp / p) 32 | -------------------------------------------------------------------------------- /cmake/Scripts/wrap_opencl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (C) 2017-2020 Trent Houliston 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 6 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 7 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 11 | # Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 14 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 15 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 16 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | import os 19 | import sys 20 | 21 | # Get our input and output from the list 22 | input_path = sys.argv[1] 23 | output_path = sys.argv[2] 24 | 25 | # Make our output directory if it doesn't exist 26 | try: 27 | os.makedirs(os.path.dirname(output_path)) 28 | except: 29 | pass 30 | 31 | with open(input_path, "r") as input_file: 32 | with open(output_path, "w") as output_file: 33 | 34 | # Work out what our define will be 35 | define = os.path.basename(input_path)[:-3].replace(os.sep, "_").upper() 36 | 37 | output_file.write('constexpr const char* {}_CL = R"({})";'.format(define, input_file.read())) 38 | output_file.write("\n") 39 | -------------------------------------------------------------------------------- /training/projection/__init__.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import tensorflow as tf 4 | 5 | 6 | def _inverse_coefficents(k): 7 | return [ 8 | -k[0], 9 | 3.0 * (k[0] * k[0]) - k[1], 10 | -12.0 * (k[0] * k[0]) * k[0] + 8.0 * k[0] * k[1], 11 | 55.0 * (k[0] * k[0]) * (k[0] * k[0]) - 55.0 * (k[0] * k[0]) * k[1] + 5.0 * (k[1] * k[1]), 12 | ] 13 | 14 | 15 | def _distort(r, k): 16 | ik = _inverse_coefficents(k) 17 | return r * ( 18 | 1.0 19 | + ik[0] * (r * r) 20 | + ik[1] * ((r * r) * (r * r)) 21 | + ik[2] * ((r * r) * (r * r)) * (r * r) 22 | + ik[3] * ((r * r) * (r * r)) * ((r * r) * (r * r)) 23 | ) 24 | 25 | 26 | def _equidistant_r(theta, f): 27 | return f * theta 28 | 29 | 30 | def _rectilinear_r(theta, f): 31 | return f * tf.math.tan(tf.clip_by_value(theta, 0.0, math.pi * 0.5)) 32 | 33 | 34 | def _equisolid_r(theta, f): 35 | return 2.0 * f * tf.math.sin(theta * 0.5) 36 | 37 | 38 | def project(V, dimensions, projection, f, centre, k): 39 | 40 | # Perform the projection math 41 | theta = tf.math.acos(V[:, 0]) 42 | rsin_theta = tf.math.rsqrt(1.0 - tf.square(V[:, 0])) 43 | if projection == "RECTILINEAR": 44 | r_u = _rectilinear_r(theta, f) 45 | elif projection == "EQUISOLID": 46 | r_u = _equisolid_r(theta, f) 47 | elif projection == "EQUIDISTANT": 48 | r_u = _equidistant_r(theta, f) 49 | else: 50 | r_u = tf.zeros_like(theta) 51 | 52 | r_d = _distort(r_u, k) 53 | 54 | # Screen as y,x 55 | screen = tf.stack([r_d * V[:, 2] * rsin_theta, r_d * V[:, 1] * rsin_theta], axis=1) 56 | 57 | # Sometimes floating point error makes x > 1.0 58 | # In this case we are basically the centre of the screen anway 59 | screen = tf.where(tf.math.is_finite(screen), screen, 0.0) 60 | 61 | # Convert to pixel coordinates 62 | return (tf.cast(dimensions, tf.float32) * 0.5) - screen - centre 63 | -------------------------------------------------------------------------------- /training/flavour/merge_configuration.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | 17 | def merge_configuration(base, detail): 18 | def _merge(a, b): 19 | 20 | # If they're both dictionaries we start with a and then overwrite with b 21 | if type(a) == dict and type(b) == dict: 22 | v = {**a} 23 | for k in b: 24 | if k not in v: 25 | v[k] = b[k] 26 | else: 27 | v[k] = _merge(a[k], b[k]) 28 | return v 29 | 30 | # Otherwise b always wins 31 | return b 32 | 33 | config = {} 34 | for k in ["view", "example", "orientation", "label", "projection"]: 35 | if k not in detail: 36 | config[k] = base[k] 37 | else: 38 | config[k] = _merge(base[k], detail[k]) 39 | 40 | return config 41 | -------------------------------------------------------------------------------- /training/loss/focal_loss.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | def FocalLoss(gamma=2.0): 20 | def focal_loss(y_true, y_pred, sample_weight=None): 21 | 22 | # Trim down the indexes to only those that have a class label 23 | idx = tf.squeeze(tf.where(tf.reduce_any(tf.greater(y_true, 0.0), axis=-1)), axis=-1) 24 | y_true = tf.gather(y_true, idx) 25 | y_pred = tf.gather(y_pred, idx) 26 | 27 | # Calculate the class weights required to balance the output 28 | C = tf.math.reduce_sum(y_true, axis=0, keepdims=True) 29 | C = tf.math.divide_no_nan(tf.math.reduce_max(C), C) 30 | 31 | # Calculate focal loss 32 | p_t = tf.where(tf.equal(y_true, 1.0), y_pred, 1.0 - y_pred) 33 | loss = tf.reduce_sum(tf.multiply(C, -tf.math.pow((1.0 - p_t), gamma) * tf.math.log(p_t)), axis=-1) 34 | 35 | return loss 36 | 37 | return focal_loss 38 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | FROM tensorflow/tensorflow:2.7.3-gpu 17 | 18 | # Need cmake to build the op 19 | RUN apt-get update && apt-get -y install \ 20 | cmake \ 21 | libtcmalloc-minimal4 22 | 23 | # Need these libraries for training 24 | RUN pip3 install \ 25 | pyyaml \ 26 | opencv-contrib-python-headless \ 27 | matplotlib \ 28 | tensorflow-addons \ 29 | tqdm 30 | 31 | # Matplotlib wants /.{config,cache}/matplotlib to be writable 32 | RUN install -d -m 0777 /.config/matplotlib 33 | RUN install -d -m 0777 /.cache/matplotlib 34 | 35 | # Build the tensorflow op and put it in /visualmesh 36 | RUN mkdir visualmesh 37 | COPY . visualmesh/ 38 | ENV CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0 39 | RUN mkdir visualmesh/build && cd visualmesh/build \ 40 | && cmake .. \ 41 | -DCMAKE_BUILD_TYPE=Release \ 42 | -DBUILD_EXAMPLES=Off \ 43 | -DBUILD_TENSORFLOW_OP=On \ 44 | && make 45 | 46 | ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4 47 | 48 | # Make tensorflow only print out info and above logs 49 | ENV TF_CPP_MIN_LOG_LEVEL 1 50 | 51 | RUN mkdir /workspace 52 | WORKDIR /workspace 53 | -------------------------------------------------------------------------------- /cpp/visualmesh/projected_mesh.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_PROJECTED_MESH_HPP 19 | #define VISUALMESH_PROJECTED_MESH_HPP 20 | 21 | #include 22 | #include 23 | 24 | namespace visualmesh { 25 | 26 | /** 27 | * @brief A projected mesh is a Visual Mesh that has been projected onto a camera image 28 | * 29 | * @tparam Scalar the scalar type used for calculations and storage (normally one of float or double) 30 | * @tparam N_NEIGHBOURS the number of neighbours that each point has 31 | */ 32 | template 33 | struct ProjectedMesh { 34 | 35 | /// The pixel coordinates (x,y) of the points projected from the visual mesh 36 | std::vector> pixel_coordinates; 37 | /// The index graph giving the locations of the neighbours of each point 38 | std::vector> neighbourhood; 39 | /// The original indicies of these points in the visual mesh 40 | std::vector global_indices; 41 | }; 42 | 43 | } // namespace visualmesh 44 | 45 | #endif // VISUALMESH_PROJECTED_MESH_HPP 46 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | # C++ files 2 | Language: Cpp 3 | BasedOnStyle: Google 4 | AccessModifierOffset: -4 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveAssignments: true 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlines: Left 9 | AlignOperands: true 10 | AlignTrailingComments: true 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: true 13 | AllowShortCaseLabelsOnASingleLine: true 14 | AllowShortFunctionsOnASingleLine: Empty 15 | AllowShortIfStatementsOnASingleLine: true 16 | AllowShortLoopsOnASingleLine: false 17 | AlwaysBreakTemplateDeclarations: true 18 | BinPackArguments: false 19 | BinPackParameters: false 20 | BreakBeforeBinaryOperators: NonAssignment 21 | BreakBeforeBraces: Custom 22 | BraceWrapping: 23 | AfterClass: false 24 | AfterControlStatement: false 25 | AfterEnum: false 26 | AfterFunction: false 27 | AfterNamespace: false 28 | AfterStruct: false 29 | AfterUnion: false 30 | AfterExternBlock: false 31 | BeforeCatch: true 32 | BeforeElse: true 33 | IndentBraces: false 34 | BreakBeforeInheritanceComma: true 35 | BreakBeforeTernaryOperators: true 36 | BreakConstructorInitializers: BeforeComma 37 | BreakStringLiterals: true 38 | ColumnLimit: 120 39 | CompactNamespaces: false 40 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 41 | ConstructorInitializerIndentWidth: 2 42 | ContinuationIndentWidth: 2 43 | Cpp11BracedListStyle: true 44 | DerivePointerAlignment: false 45 | FixNamespaceComments: true 46 | IncludeBlocks: Regroup 47 | IndentCaseLabels: true 48 | IndentPPDirectives: None 49 | IndentWrappedFunctionNames: true 50 | IndentWidth: 4 51 | KeepEmptyLinesAtTheStartOfBlocks: true 52 | MaxEmptyLinesToKeep: 2 53 | NamespaceIndentation: Inner 54 | PointerAlignment: Left 55 | ReflowComments: true 56 | SortIncludes: true 57 | SortUsingDeclarations: true 58 | SpaceAfterCStyleCast: true 59 | SpaceAfterTemplateKeyword: true 60 | SpaceBeforeAssignmentOperators: true 61 | SpaceBeforeParens: ControlStatements 62 | SpaceInEmptyParentheses: false 63 | SpacesBeforeTrailingComments: 2 64 | SpacesInAngles: false 65 | SpacesInCStyleCastParentheses: false 66 | SpacesInContainerLiterals: false 67 | SpacesInParentheses: false 68 | SpacesInSquareBrackets: false 69 | Standard: Cpp11 70 | TabWidth: 4 71 | UseTab: Never 72 | -------------------------------------------------------------------------------- /cpp/visualmesh/utility/static_if.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_UTILITY_STATIC_IF_HPP 19 | #define VISUALMESH_UTILITY_STATIC_IF_HPP 20 | 21 | // https://baptiste-wicht.com/posts/2015/07/simulate-static_if-with-c11c14.html 22 | namespace static_if_detail { 23 | 24 | struct identity { 25 | template 26 | T operator()(T&& x) const { 27 | return std::forward(x); 28 | } 29 | }; 30 | 31 | template 32 | struct statement { 33 | template 34 | void then(const F& f) { 35 | f(identity()); 36 | } 37 | 38 | template 39 | void else_(const F&) {} 40 | }; 41 | 42 | template <> 43 | struct statement { 44 | template 45 | void then(const F&) {} 46 | 47 | template 48 | void else_(const F& f) { 49 | f(identity()); 50 | } 51 | }; 52 | 53 | } // namespace static_if_detail 54 | 55 | template 56 | static_if_detail::statement static_if(F const& f) { 57 | static_if_detail::statement if_; 58 | if_.then(f); 59 | return if_; 60 | } 61 | 62 | #endif // VISUALMESH_UTILITY_STATIC_IF_HPP 63 | -------------------------------------------------------------------------------- /cmake/Modules/FindSPIRV.cmake: -------------------------------------------------------------------------------- 1 | #[=======================================================================[.rst: 2 | FindSPIRV 3 | --------- 4 | 5 | Find SPIRV, which is a simple binary intermediate language for graphical 6 | shaders and compute kernels. 7 | 8 | IMPORTED Targets 9 | ^^^^^^^^^^^^^^^^ 10 | 11 | This module defines :prop_tgt:`IMPORTED` target ``SPIRV::SPIRV``, if 12 | SPIRV has been found. 13 | 14 | Result Variables 15 | ^^^^^^^^^^^^^^^^ 16 | 17 | This module defines the following variables:: 18 | 19 | SPIRV_FOUND - "True" if SPIRV was found 20 | SPIRV_INCLUDE_DIRS - include directories for SPIRV 21 | SPIRV_LIBRARIES - link against this library to use SPIRV 22 | 23 | The module will also define three cache variables:: 24 | 25 | SPIRV_INCLUDE_DIR - the SPIRV include directory 26 | SPIRV_LIBRARY_DEBUG - the path to the SPIRV debug library 27 | SPIRV_LIBRARY_RELEASE - the path to the SPIRV optimised library 28 | 29 | #]=======================================================================] 30 | 31 | find_path(SPIRV_INCLUDE_DIR NAMES "spirv/unified1/spirv.h") 32 | find_library( 33 | SPIRV_LIBRARY_DEBUG 34 | NAMES SPIRVd SPIRV 35 | PATH_SUFFIXES lib) 36 | find_library( 37 | SPIRV_LIBRARY_RELEASE 38 | NAMES SPIRV 39 | PATH_SUFFIXES lib) 40 | set(SPIRV_LIBRARY debug ${SPIRV_LIBRARY_DEBUG} optimized ${SPIRV_LIBRARY_RELEASE}) 41 | 42 | set(SPIRV_LIBRARIES ${SPIRV_LIBRARY}) 43 | set(SPIRV_INCLUDE_DIRS ${SPIRV_INCLUDE_DIR}) 44 | 45 | include(FindPackageHandleStandardArgs) 46 | find_package_handle_standard_args(SPIRV REQUIRED_VARS SPIRV_LIBRARY SPIRV_INCLUDE_DIR) 47 | 48 | mark_as_advanced(SPIRV_INCLUDE_DIR SPIRV_LIBRARY_DEBUG SPIRV_LIBRARY_RELEASE) 49 | 50 | if(SPIRV_FOUND AND NOT TARGET SPIRV::SPIRV) 51 | add_library(SPIRV::SPIRV UNKNOWN IMPORTED) 52 | set_target_properties(SPIRV::SPIRV PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${SPIRV_INCLUDE_DIRS}") 53 | 54 | if(SPIRV_LIBRARY_DEBUG AND SPIRV_LIBRARY_RELEASE) 55 | set_target_properties(SPIRV::SPIRV PROPERTIES IMPORTED_LOCATION_DEBUG "${SPIRV_LIBRARY_DEBUG}" 56 | IMPORTED_LOCATION_RELEASE "${SPIRV_LIBRARY_RELEASE}") 57 | else() 58 | set_target_properties(SPIRV::SPIRV PROPERTIES IMPORTED_LOCATION "${SPIRV_LIBRARY_RELEASE}") 59 | endif() 60 | endif() 61 | -------------------------------------------------------------------------------- /cpp/visualmesh/network_structure.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_NETWORKSTRUCTURE_HPP 19 | #define VISUALMESH_NETWORKSTRUCTURE_HPP 20 | 21 | #include 22 | 23 | namespace visualmesh { 24 | 25 | /// Weights are a matrix (vector of vectors) 26 | template 27 | using Weights = std::vector>; 28 | /// Biases are a vector 29 | template 30 | using Biases = std::vector; 31 | 32 | enum ActivationFunction { 33 | SELU, 34 | RELU, 35 | SOFTMAX, 36 | TANH, 37 | }; 38 | 39 | /// A layer is made up of weights biases and activation function 40 | template 41 | struct Layer { 42 | Weights weights; 43 | Biases biases; 44 | ActivationFunction activation; 45 | }; 46 | 47 | /// A convolutional layer is made up of a list of network layers 48 | template 49 | using ConvolutionalGroup = std::vector>; 50 | /// A network is a list of convolutional layers 51 | template 52 | using NetworkStructure = std::vector>; 53 | 54 | } // namespace visualmesh 55 | 56 | #endif // VISUALMESH_NETWORKSTRUCTURE_HPP 57 | -------------------------------------------------------------------------------- /training/dataset/orientation/ground.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .random_rotation import random_rotation 19 | 20 | 21 | class Ground: 22 | def __init__(self, **config): 23 | self.augmentations = {} if "augmentations" not in config else config["augmentations"] 24 | 25 | def features(self): 26 | return { 27 | "Hoc": tf.io.FixedLenFeature([4, 4], tf.float32), 28 | } 29 | 30 | def __call__(self, Hoc, **features): 31 | 32 | # If we have a height augmentation, apply it 33 | if "height" in self.augmentations: 34 | v = self.augmentations["height"] 35 | 36 | Hoc = tf.tensor_scatter_nd_add( 37 | Hoc, 38 | [[3, 2]], 39 | tf.expand_dims(tf.random.truncated_normal(shape=(), mean=v["mean"], stddev=v["stddev"]), 0), 40 | ) 41 | 42 | if "rotation" in self.augmentations: 43 | v = self.augmentations["rotation"] 44 | 45 | # Apply a random axis angle rotation 46 | Hoc = tf.matmul(random_rotation(v["mean"], v["stddev"]), Hoc) 47 | 48 | # We can just read the cameras ground orientation directly from camera/Hoc 49 | return {"Hoc": Hoc} 50 | -------------------------------------------------------------------------------- /training/metrics/confusion_base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | class ConfusionBase(tf.keras.metrics.Metric): 20 | def __init__(self, name, size, **kwargs): 21 | super(ConfusionBase, self).__init__(name=name, **kwargs) 22 | 23 | self.confusion = self.add_weight(name="confusion", shape=(size, size), initializer="zeros", dtype=tf.int32) 24 | 25 | def update_state(self, y_true, y_pred, sample_weight=None): 26 | 27 | # Build up an index list that maps each class 28 | idx = tf.stack( 29 | [ 30 | tf.argmax(input=y_true, axis=-1, output_type=self.confusion.dtype), 31 | tf.argmax(input=y_pred, axis=-1, output_type=self.confusion.dtype), 32 | ], 33 | axis=-1, 34 | ) 35 | 36 | # Trim down the indexes to only those that have a class label 37 | idx = tf.gather(idx, tf.squeeze(tf.where(tf.reduce_any(tf.greater(y_true, 0.0), axis=-1)), axis=-1)) 38 | 39 | # Add them into the corresponding locations 40 | self.confusion.scatter_nd_add(idx, tf.ones_like(idx[:, 0], dtype=self.confusion.dtype)) 41 | 42 | def reset_states(self): 43 | self.confusion.assign(tf.zeros_like(self.confusion)) 44 | -------------------------------------------------------------------------------- /cpp/visualmesh/utility/fourcc.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_UTILITY_FOURCC_HPP 19 | #define VISUALMESH_UTILITY_FOURCC_HPP 20 | 21 | #include 22 | 23 | namespace visualmesh { 24 | 25 | /** 26 | * @brief Given a fourcc (four character code), in string form convert it into it's uint32_t representation 27 | * 28 | * @param code the four characters to convert 29 | * 30 | * @return the uint32_t representing this four character code 31 | */ 32 | // We allow c arrays here as it lets us pass in string literals with no copying 33 | // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) 34 | inline constexpr uint32_t fourcc(const char (&code)[5]) { 35 | return uint32_t(code[0] | (code[1] << 8) | (code[2] << 16) | (code[3] << 24)); 36 | } 37 | 38 | /** 39 | * @brief Given a fourcc (four character code), in uint32_t form convert it into it's four character representation 40 | * 41 | * @param code 42 | * @return std::string 43 | */ 44 | inline std::string fourcc_text(const uint32_t& code) { 45 | return std::string({char(code & 0xFF), char(code >> 8 & 0xFF), char(code >> 16 & 0xFF), char(code >> 24 & 0xFF)}); 46 | } 47 | 48 | } // namespace visualmesh 49 | 50 | #endif // VISUALMESH_UTILITY_FOURCC_HPP 51 | -------------------------------------------------------------------------------- /cpp/visualmesh/node.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_NODE_HPP 19 | #define VISUALMESH_NODE_HPP 20 | 21 | #include 22 | 23 | #include "utility/math.hpp" 24 | 25 | namespace visualmesh { 26 | 27 | /** 28 | * @brief This represents a single node in the visual mesh. 29 | * 30 | * @details 31 | * A single node in the visual mesh is a single point, it is made up of a vector in observation plane space that points 32 | * to the position, as well as a list of neighbours that are connected to this point. A collection of these with each 33 | * having neighbours pointing to indices of other points in the list make up a Visual Mesh. The neighbours are ordered 34 | * in a clockwise fashion. 35 | * 36 | * @tparam Scalar the scalar type used for calculations and storage (normally one of float or double) 37 | * @tparam N_NEIGHBOURS the number of neighbours that each point has 38 | */ 39 | template 40 | struct Node { 41 | /// The unit vector in the direction for this node 42 | vec3 ray; 43 | /// Absolute indices to the linked nodes ordered L, TL, TR, R, BR, BL (clockwise) 44 | std::array neighbours; 45 | }; 46 | 47 | } // namespace visualmesh 48 | 49 | #endif // VISUALMESH_NODE_HPP 50 | -------------------------------------------------------------------------------- /example/Timer.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef TIMER_HPP 19 | #define TIMER_HPP 20 | 21 | #include 22 | #include 23 | 24 | /** 25 | * @brief Easily time events while removing the influence of the timer as much as possible 26 | * 27 | */ 28 | class Timer { 29 | public: 30 | std::chrono::steady_clock::time_point t; 31 | 32 | Timer() : t(std::chrono::steady_clock::now()) {} 33 | 34 | template 35 | // We allow c arrays here as it lets us pass in string literals with no copying 36 | // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) 37 | inline void measure(const char (&c)[N]) { 38 | 39 | // Work out how long it took 40 | auto end = std::chrono::steady_clock::now(); 41 | auto val = end - t; 42 | auto v = std::chrono::duration_cast>(val).count(); 43 | 44 | // Print out how many microseconds 45 | std::cout << c << " " << v << "µs" << std::endl; 46 | 47 | // Restart the timer 48 | t = std::chrono::steady_clock::now(); 49 | } 50 | 51 | inline void reset() { 52 | // Restart the timer 53 | t = std::chrono::steady_clock::now(); 54 | } 55 | }; 56 | 57 | #endif // TIMER_HPP 58 | -------------------------------------------------------------------------------- /cmake/Modules/FindTensorFlow.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit 6 | # persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | find_package(PythonInterp 3 REQUIRED) 17 | 18 | execute_process( 19 | COMMAND ${PYTHON_EXECUTABLE} -c "import tensorflow; print(tensorflow.__version__)" 20 | OUTPUT_VARIABLE TENSORFLOW_VERSION 21 | OUTPUT_STRIP_TRAILING_WHITESPACE) 22 | execute_process( 23 | COMMAND ${PYTHON_EXECUTABLE} -c "import tensorflow; print(tensorflow.sysconfig.get_include())" 24 | OUTPUT_VARIABLE tf_inc_dir 25 | OUTPUT_STRIP_TRAILING_WHITESPACE) 26 | execute_process( 27 | COMMAND ${PYTHON_EXECUTABLE} -c "import tensorflow; print(tensorflow.sysconfig.get_lib())" 28 | OUTPUT_VARIABLE tf_lib_dir 29 | OUTPUT_STRIP_TRAILING_WHITESPACE) 30 | 31 | find_path( 32 | TENSORFLOW_INCLUDE_DIRS 33 | NAMES tensorflow/core/framework/op.h 34 | HINTS ${tf_inc_dir} 35 | DOC "TensorFlow include directory") 36 | 37 | find_library( 38 | TENSORFLOW_LIBRARIES 39 | NAMES tensorflow_framework libtensorflow_framework.so.2 libtensorflow_framework.so.1 40 | HINTS ${tf_lib_dir} 41 | DOC "TensorFlow library") 42 | 43 | include(FindPackageHandleStandardArgs) 44 | find_package_handle_standard_args( 45 | TensorFlow 46 | FOUND_VAR TensorFlow_FOUND 47 | REQUIRED_VARS TENSORFLOW_INCLUDE_DIRS TENSORFLOW_LIBRARIES TENSORFLOW_VERSION 48 | VERSION_VAR TENSORFLOW_VERSION) 49 | -------------------------------------------------------------------------------- /training/layer/depthwise_seperable_graph_convolution.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | class Depthwise(tf.keras.layers.Layer): 20 | def __init__(self, **kwargs): 21 | super(Depthwise, self).__init__() 22 | self.pointwise = tf.keras.layers.Dense(**kwargs) 23 | 24 | def build(self, input_shape): 25 | # Copy whatever we have on our pointwise kernel 26 | self.depthwise_weights = self.add_weight( 27 | "depthwise_kernel", 28 | input_shape[1:], 29 | dtype=self.dtype, 30 | initializer=self.pointwise.kernel_initializer, 31 | regularizer=self.pointwise.kernel_regularizer, 32 | constraint=self.pointwise.kernel_constraint, 33 | ) 34 | 35 | def call(self, X): 36 | depthwise = tf.einsum("ijk,jk->ik", X, self.depthwise_weights) 37 | return self.pointwise(depthwise) 38 | 39 | 40 | class DepthwiseSeparableGraphConvolution(tf.keras.layers.Layer): 41 | def __init__(self, **kwargs): 42 | super(DepthwiseSeparableGraphConvolution, self).__init__() 43 | self.depthwise = Depthwise(**kwargs) 44 | 45 | def call(self, X, G): 46 | convolved = tf.reshape(tf.gather(X, G, name="NetworkGather"), shape=[-1, G.shape[-1], X.shape[-1]]) 47 | return self.depthwise(convolved) 48 | -------------------------------------------------------------------------------- /training/flavour/metrics.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from ..metrics import * 17 | from ..metrics.test import SeekerHourglass 18 | 19 | 20 | def Metrics(config): 21 | 22 | if config["label"]["type"] == "Classification": 23 | 24 | classes = config["label"]["config"]["classes"] 25 | metrics = [ 26 | AveragePrecision("metrics/average_precision", len(classes)), 27 | AverageRecall("metrics/average_recall", len(classes)), 28 | ] 29 | for i, c in enumerate(classes): 30 | metrics.append(ClassPrecision("metrics/{}_precision".format(c["name"]), i, len(classes))) 31 | metrics.append(ClassRecall("metrics/{}_recall".format(c["name"]), i, len(classes))) 32 | 33 | return metrics 34 | 35 | elif config["label"]["type"] == "Seeker": 36 | return [ 37 | SeekerPrecision("metrics/precision75", 0.75), 38 | SeekerRecall("metrics/recall75", 0.75), 39 | SeekerStdDev("metrics/stddev75", 0.75), 40 | SeekerPrecision("metrics/precision50", 0.5), 41 | SeekerRecall("metrics/recall50", 0.5), 42 | SeekerStdDev("metrics/stddev50", 0.5), 43 | SeekerHourglass("metrics/hourglass"), 44 | ] 45 | 46 | else: 47 | raise RuntimeError("Cannot create metrics, {} is not a supported type".format(config["label"]["type"])) 48 | -------------------------------------------------------------------------------- /training/dataset/orientation/random_rotation.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import math 17 | 18 | import tensorflow as tf 19 | 20 | 21 | def random_axis(): 22 | # Get two random values between 0 and 1 23 | uv = tf.random.uniform([2], 0, 1) 24 | 25 | # Math to make a uniform distribution on the unit sphere 26 | # https://mathworld.wolfram.com/SpherePointPicking.html 27 | theta = 2 * math.pi * uv[0] 28 | phi = tf.math.acos(2.0 * uv[1] - 1.0) 29 | 30 | # Convert this into an axis 31 | return [tf.math.sin(theta) * tf.math.cos(phi), tf.math.sin(theta) * tf.math.sin(phi), tf.math.cos(theta)] 32 | 33 | 34 | def random_rotation(mean, stddev): 35 | 36 | # Get a random axis to perform the rotation around 37 | ux, uy, uz = random_axis() 38 | 39 | # Get a random angle within our specifications 40 | angle = tf.random.truncated_normal((), mean, stddev) 41 | ca = tf.math.cos(angle) 42 | ia = 1.0 - ca 43 | sa = tf.math.sin(angle) 44 | 45 | # Axis angle to rotation matrix 46 | return tf.convert_to_tensor( 47 | [ 48 | [ux * ux * ia + ca, uy * ux * ia - uz * sa, uz * ux * ia + uy * sa, 0.0], 49 | [ux * uy * ia + uz * sa, uy * uy * ia + ca, uz * uy * ia - ux * sa, 0.0], 50 | [ux * uz * ia - uy * sa, uy * uz * ia + ux * sa, uz * uz * ia + ca, 0.0], 51 | [0.0, 0.0, 0.0, 1.0], 52 | ] 53 | ) 54 | -------------------------------------------------------------------------------- /training/testing.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import os 17 | 18 | import numpy as np 19 | 20 | import tensorflow as tf 21 | 22 | from .dataset import keras_dataset 23 | from .flavour import Dataset, Loss, Metrics, TestMetrics 24 | from .model import VisualMeshModel 25 | 26 | 27 | def test(config, output_path): 28 | 29 | # Get the testing dataset 30 | testing_dataset = ( 31 | Dataset(config, "testing").map(keras_dataset, num_parallel_calls=tf.data.AUTOTUNE).prefetch(tf.data.AUTOTUNE) 32 | ) 33 | 34 | # Get the dimensionality of the Y part of the dataset 35 | output_dims = testing_dataset.element_spec[1].shape[-1] 36 | 37 | # Define the model 38 | model = VisualMeshModel(structure=config["network"]["structure"], output_dims=output_dims) 39 | 40 | # Find the latest checkpoint file and load it 41 | checkpoint_file = tf.train.latest_checkpoint(output_path) 42 | if checkpoint_file is not None: 43 | model.load_weights(checkpoint_file) 44 | else: 45 | raise RuntimeError("Could not find weights to load into the network") 46 | 47 | # Get the metrics and curves we will be building 48 | metrics = TestMetrics(config) 49 | model.compile(loss=Loss(config), metrics=metrics) 50 | 51 | # Run the evaluation step for each of the batches to build up our metrics 52 | model.evaluate(testing_dataset) 53 | 54 | # Save all the metric data 55 | for m in metrics: 56 | m.save(output_path) 57 | -------------------------------------------------------------------------------- /example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Build an program that classifies using the visual mesh 2 | option(BUILD_EXAMPLES "Build the program that executes the mesh" OFF) 3 | if(BUILD_EXAMPLES) 4 | 5 | # Global compiler flags 6 | if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") 7 | list(APPEND compile_options /W4 /WX) 8 | elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") 9 | list( 10 | APPEND 11 | compile_options 12 | -Wall 13 | -Wextra 14 | -Wpedantic 15 | -Werror 16 | -march=native 17 | -mtune=native) 18 | 19 | # Always have the colours otherwise using ninja makes them go away 20 | if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") 21 | list(APPEND compile_options -fcolor-diagnostics) 22 | elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") 23 | list(APPEND compile_options -fdiagnostics-color=always) 24 | endif() 25 | endif() 26 | 27 | # Find packages 28 | set(CMAKE_THREAD_PREFER_PTHREAD TRUE) 29 | set(THREADS_PREFER_PTHREAD_FLAG TRUE) 30 | find_package(Threads REQUIRED) 31 | find_package(OpenCV COMPONENTS core highgui imgproc) 32 | find_package(YAML-CPP) 33 | find_package(fmt) 34 | 35 | # We need OpenCV to build all the GUI examples 36 | if(OpenCV_FOUND) 37 | add_executable(classified "classified.cpp") 38 | target_compile_options(classified PRIVATE ${compile_options}) 39 | target_link_libraries(classified visualmesh ${OpenCV_LIBS} ${fmt_LIBRARIES} ${YAML_CPP_LIBRARIES} 40 | Threads::Threads) 41 | 42 | add_executable(mesh_types "mesh_types.cpp") 43 | target_compile_options(mesh_types PRIVATE ${compile_options}) 44 | target_link_libraries(mesh_types visualmesh ${OpenCV_LIBS} ${fmt_LIBRARIES} ${YAML_CPP_LIBRARIES} 45 | Threads::Threads) 46 | 47 | add_executable(benchmark "benchmark.cpp") 48 | target_compile_options(benchmark PRIVATE ${compile_options}) 49 | target_include_directories(benchmark SYSTEM PRIVATE ${OpenCV_INCLUDE_DIRS} ${YAML_CPP_INCLUDE_DIR}) 50 | target_link_libraries(benchmark visualmesh ${OpenCV_LIBS} ${fmt_LIBRARIES} ${YAML_CPP_LIBRARIES} 51 | Threads::Threads) 52 | endif(OpenCV_FOUND) 53 | 54 | add_executable(mesh_quality "mesh_quality.cpp") 55 | target_compile_options(mesh_quality PRIVATE ${compile_options}) 56 | target_link_libraries(mesh_quality visualmesh) 57 | 58 | endif(BUILD_EXAMPLES) 59 | -------------------------------------------------------------------------------- /example/load_model.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef LOAD_MODEL_HPP 19 | #define LOAD_MODEL_HPP 20 | 21 | #include 22 | 23 | #include "visualmesh/network_structure.hpp" 24 | 25 | inline visualmesh::ActivationFunction activation_function(const std::string& name) { 26 | 27 | if (name == "selu") { return visualmesh::ActivationFunction::SELU; } 28 | if (name == "softmax") { return visualmesh::ActivationFunction::SOFTMAX; } 29 | if (name == "relu") { return visualmesh::ActivationFunction::RELU; } 30 | if (name == "tanh") { return visualmesh::ActivationFunction::TANH; } 31 | throw std::runtime_error("Unknown activation function " + name); 32 | } 33 | 34 | template 35 | visualmesh::NetworkStructure load_model(const std::string& path) { 36 | 37 | visualmesh::NetworkStructure model; 38 | YAML::Node config = YAML::LoadFile(path); 39 | for (const auto& conv : config["network"]) { 40 | model.emplace_back(); 41 | auto& net_conv = model.back(); 42 | 43 | for (const auto& layer : conv) { 44 | net_conv.emplace_back(visualmesh::Layer{ 45 | layer["weights"].as>>(), 46 | layer["biases"].as>(), 47 | activation_function(layer["activation"].as()), 48 | }); 49 | } 50 | } 51 | return model; 52 | } 53 | 54 | #endif // LOAD_MODEL_HPP 55 | -------------------------------------------------------------------------------- /cpp/visualmesh/lens.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_LENS_HPP 19 | #define VISUALMESH_LENS_HPP 20 | 21 | #include 22 | 23 | namespace visualmesh { 24 | 25 | /** 26 | * @brief An enum that describes the lens projection type 27 | */ 28 | enum LensProjection { RECTILINEAR, EQUISOLID, EQUIDISTANT }; 29 | 30 | /** 31 | * @brief A description of a lens that will be used for projection of unit vectors into camera space. 32 | * 33 | * @tparam Scalar the scalar type used for calculations and storage (normally one of float or double) 34 | */ 35 | template 36 | struct Lens { 37 | /// The dimensions of the image 38 | std::array dimensions; 39 | /// The projection that this image is using 40 | LensProjection projection; 41 | /// The focal length of the camera in pixels 42 | Scalar focal_length; 43 | /// The offset required to move the centre of the lens to the centre of the image 44 | std::array centre; 45 | /// The distortion parameters for the camera model 46 | std::array k; 47 | /// The field of view of the camera measured in radians 48 | /// This field of view is used to cut off sections of the image that the lens does not project for (think the black 49 | /// sections on a fisheye camera). If there are no black sections on the image this should be set to the diagonal 50 | /// field of view of the lens. 51 | Scalar fov; 52 | }; 53 | 54 | } // namespace visualmesh 55 | 56 | #endif // VISUALMESH_LENS_HPP 57 | -------------------------------------------------------------------------------- /cpp/visualmesh/engine/opencl/operation/make_queue.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_OPENCL_OPERATION_MAKE_QUEUE_HPP 19 | #define VISUALMESH_OPENCL_OPERATION_MAKE_QUEUE_HPP 20 | 21 | #include "wrapper.hpp" 22 | 23 | namespace visualmesh { 24 | namespace engine { 25 | namespace opencl { 26 | namespace operation { 27 | 28 | /** 29 | * @brief Make an OpenCL command queue 30 | * 31 | * @param context the context to make the queue for 32 | * @param device the device to make the queue for 33 | * 34 | * @return a reference counted tracker of a command queue 35 | */ 36 | inline cl::command_queue make_queue(cl_context context, cl_device_id device) { 37 | cl_command_queue queue = nullptr; 38 | cl_int error = 0; 39 | // Use out of order execution if we can 40 | queue = ::clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &error); 41 | if (error == CL_INVALID_VALUE) { queue = ::clCreateCommandQueue(context, device, 0, &error); } 42 | throw_cl_error(error, "Error creating the OpenCL command queue"); 43 | return cl::command_queue(queue, ::clReleaseCommandQueue); 44 | } 45 | 46 | } // namespace operation 47 | } // namespace opencl 48 | } // namespace engine 49 | } // namespace visualmesh 50 | 51 | #endif // VISUALMESH_OPENCL_OPERATION_MAKE_QUEUE_HPP 52 | -------------------------------------------------------------------------------- /cpp/visualmesh/classified_mesh.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_CLASSIFIED_MESH_HPP 19 | #define VISUALMESH_CLASSIFIED_MESH_HPP 20 | 21 | #include 22 | #include 23 | 24 | namespace visualmesh { 25 | 26 | /** 27 | * @brief Holds a classified visual mesh segment 28 | * 29 | * @details 30 | * This holds the result of a classification operation in the visual mesh. It holds a subset of a total visual mesh, 31 | * that subset being the components that were on the screen at the time of projection. the global_indices can be used 32 | * to map back into the original mesh if needed (for example to get the world space unit vectors). 33 | * 34 | * @tparam Scalar the scalar type used for calculations and storage (normally one of float or double) 35 | * @tparam N_NEIGHBOURS the number of neighbours that each point has 36 | */ 37 | template 38 | struct ClassifiedMesh { 39 | 40 | /// The pixel coordinates (x,y) of the points projected from the visual mesh 41 | std::vector> pixel_coordinates; 42 | /// The index graph giving the locations of the neighbours of each point 43 | std::vector> neighbourhood; 44 | /// The original indicies of these points in the visual mesh 45 | std::vector global_indices; 46 | /// The final output of classification in the visual mesh 47 | std::vector classifications; 48 | }; 49 | 50 | } // namespace visualmesh 51 | 52 | #endif // VISUALMESH_MESH_CLASSIFIED_MESH_HPP 53 | -------------------------------------------------------------------------------- /readme/setup.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | In order to setup the training and inference code, you will need to provide the required libraries to build the code. 3 | 4 | ### Training/Testing 5 | In order to train a network using the visual mesh, you need to have TensorFlow 2.4 and have built the custom op. 6 | TensorFlow 2.4 is needed as currently how ragged batching is done is experimental and visual mesh needs updating to match that progress. 7 | You have two options for how you go about this, you can either install these on your host system and train from there, or you can use the preferred method and use the provided `Dockerfile` to build a docker image you can use. 8 | 9 | ### Host System 10 | In order to install directly on the host system, you need to have CMake 3.1.0 or higher and be running in a Unix-like environment (Linux/OSX/WSL/etc). 11 | You can't build the custom op directly on windows since TensorFlow does not provide their library on windows systems. 12 | The dependencies for building the custom op are 13 | - Python 3 14 | - TensorFlow 2.4 15 | - C++ compiler that can build c++14 code 16 | 17 | ```sh 18 | mkdir build 19 | cd build 20 | cmake .. 21 | make 22 | ``` 23 | 24 | You also need some python libraries installed using whatever your favourite method of installing python libraries may be (e.g. pip). 25 | ```yaml 26 | matplotlib 27 | numpy 28 | opencv-python 29 | pyyaml 30 | tensorflow 31 | tensorflow-addons # For using the Ranger optimiser 32 | tqdm 33 | 34 | ``` 35 | 36 | Once this is done you can run training code by using 37 | ```sh 38 | ./mesh.py train 39 | ``` 40 | 41 | ### Docker 42 | The preferred way to build and run the training code is to use docker. 43 | Using docker will prevent having a different version of TensorFlow etc from impacting on the running of the code. 44 | If you are running on Linux you can also forward GPUs through to docker by using `nvidia-container-runtime`. 45 | 46 | ```sh 47 | # Build the docker image 48 | docker build . --pull -t visualmesh:latest 49 | ``` 50 | 51 | This will build a docker image that can be used to run the visual mesh training and testing code. 52 | You can then run training code by using the following command, forwarding all GPUs using the nvidia-container-runtime. 53 | 54 | ```sh 55 | # To only use the CPU 56 | docker run -u $(id -u):$(id -g) -it --rm --volume $(pwd):/workspace visualmesh:latest ./mesh.py train 57 | 58 | # To use all available GPUs 59 | docker run --gpus all -u $(id -u):$(id -g) -it --rm --volume $(pwd):/workspace visualmesh:latest ./mesh.py train 60 | 61 | # To only use GPU0 62 | docker run --gpus device=0 -u $(id -u):$(id -g) -it --rm --volume $(pwd):/workspace visualmesh:latest ./mesh.py train 63 | -------------------------------------------------------------------------------- /training/dataset/label/classification.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | class Classification: 20 | def __init__(self, classes, **config): 21 | self.classes = classes 22 | 23 | def features(self): 24 | return { 25 | "mask": tf.io.FixedLenFeature([], tf.string), 26 | } 27 | 28 | def __call__(self, mask, C, **features): 29 | 30 | # Use the nearest neighbour pixel to get the classification from the mask making sure the point is on the screen 31 | mask = tf.image.decode_png(mask, channels=4) 32 | Y = tf.gather_nd(mask, tf.clip_by_value(tf.cast(tf.round(C), tf.int32), [[0, 0]], [tf.shape(mask)[:2] - 1])) 33 | 34 | # Expand the classes from colours into individual columns 35 | W = tf.image.convert_image_dtype(Y[:, 3], tf.float32) # Alpha channel 36 | cs = [] 37 | for c in self.classes: 38 | cs.append( 39 | tf.where( 40 | tf.logical_and( 41 | tf.reduce_any( 42 | tf.stack( 43 | [tf.reduce_all(input_tensor=tf.equal(Y[:, :3], [v]), axis=-1) for v in c["colours"]], 44 | axis=-1, 45 | ), 46 | axis=-1, 47 | ), 48 | tf.greater(W, 0.0), 49 | ), 50 | 1.0, 51 | 0.0, 52 | ) 53 | ) 54 | Y = tf.stack(cs, axis=-1) 55 | 56 | return {"Y": Y} 57 | -------------------------------------------------------------------------------- /training/flavour/image_callback.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from ..callbacks import ClassificationImages, SeekerImages 17 | from .dataset import Dataset 18 | from .merge_configuration import merge_configuration 19 | 20 | 21 | def ImageCallback(config, output_path): 22 | 23 | validation_config = merge_configuration(config, config["dataset"].get("config", {})) 24 | 25 | if config["label"]["type"] == "Classification": 26 | 27 | n_images = config["training"]["validation"]["progress_images"] 28 | classes = config["label"]["config"]["classes"] 29 | 30 | return ClassificationImages( 31 | output_path=output_path, 32 | dataset=Dataset(config, "validation", batch_size=n_images).take(1), 33 | # Draw using the first colour in the list for each class 34 | colours=[c["colours"][0] for c in classes], 35 | ) 36 | 37 | elif config["label"]["type"] == "Seeker": 38 | 39 | n_images = config["training"]["validation"]["progress_images"] 40 | 41 | return SeekerImages( 42 | output_path=output_path, 43 | dataset=Dataset(config, "validation", batch_size=n_images).take(1), 44 | model=validation_config["projection"]["config"]["mesh"]["model"], 45 | max_distance=validation_config["projection"]["config"]["mesh"]["max_distance"], 46 | geometry=validation_config["projection"]["config"]["geometry"]["shape"], 47 | radius=validation_config["projection"]["config"]["geometry"]["radius"], 48 | scale=validation_config["label"]["config"]["scale"], 49 | ) 50 | 51 | else: 52 | raise RuntimeError("Cannot create images callback, {} is not a supported type".format(config["label"]["type"])) 53 | -------------------------------------------------------------------------------- /training/metrics/seeker_confusion_base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | from .confusion_base import ConfusionBase 19 | 20 | 21 | class SeekerConfusionBase(ConfusionBase): 22 | def __init__(self, name, threshold, **kwargs): 23 | super(SeekerConfusionBase, self).__init__(name, size=2, **kwargs) 24 | self.threshold = threshold 25 | 26 | def update_state(self, y_true, y_pred, sample_weight=None): 27 | 28 | # Y true contains a list of all the possible answers, we are ranked on how close we are to the closest one 29 | # This code works out the closest point to our prediction for each case so we can use that for training 30 | y_true = tf.gather_nd( 31 | y_true, 32 | tf.stack( 33 | [ 34 | tf.range(tf.shape(y_true)[0], dtype=tf.int32), 35 | tf.math.argmin( 36 | tf.reduce_sum(tf.math.squared_difference(tf.expand_dims(y_pred, axis=-2), y_true), axis=-1), 37 | axis=-1, 38 | output_type=tf.int32, 39 | ), 40 | ], 41 | axis=1, 42 | ), 43 | ) 44 | 45 | # Our labels for this confusion matrix are based on if the points are near enough 46 | y_true = tf.where(tf.reduce_all(tf.math.abs(y_true) <= self.threshold, axis=-1), 1.0, 0.0) 47 | y_pred = tf.where(tf.reduce_all(tf.math.abs(y_pred) <= self.threshold, axis=-1), 1.0, 0.0) 48 | 49 | super(SeekerConfusionBase, self).update_state( 50 | tf.stack([y_true, 1.0 - y_true], axis=1), 51 | tf.stack([y_pred, 1.0 - y_pred], axis=1), 52 | sample_weight=sample_weight, 53 | ) 54 | -------------------------------------------------------------------------------- /example/ArrayPrint.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef ARRAY_PRINT_HPP 19 | #define ARRAY_PRINT_HPP 20 | 21 | #include 22 | #include 23 | 24 | template 25 | struct Printer; 26 | 27 | // Prints matrices 28 | template 29 | struct Printer, m>> { 30 | static inline void print(std::ostream& out, const std::array, m>& s) { 31 | for (std::size_t j = 0; j < m; ++j) { 32 | out << "["; 33 | for (std::size_t i = 0; i < n - 1; ++i) { 34 | out << s[j][i] << ", "; 35 | } 36 | if (n > 0) { out << s[j][n - 1]; } 37 | out << "]"; 38 | 39 | if (j < m - 1) { out << std::endl; } 40 | } 41 | } 42 | }; 43 | 44 | // Prints vectors 45 | template 46 | struct Printer> { 47 | static inline void print(std::ostream& out, const std::array& s) { 48 | out << "["; 49 | for (std::size_t i = 0; i < n - 1; ++i) { 50 | out << s[i] << ", "; 51 | } 52 | if (n > 0) { out << s[n - 1]; } 53 | out << "]"; 54 | } 55 | }; 56 | 57 | /** 58 | * @brief Provides an operator that prints out a matrix or vector to a stream 59 | * 60 | * @tparam T the type of the data in the array 61 | * @tparam n the size of the array 62 | * 63 | * @param out the stream to output the data to 64 | * @param s the array to print 65 | * 66 | * @return std::ostream& the ostream out that was put in 67 | */ 68 | template 69 | std::ostream& operator<<(std::ostream& out, const std::array& s) { 70 | Printer>::print(out, s); 71 | return out; 72 | } 73 | 74 | #endif // ARRAY_PRINT_HPP 75 | -------------------------------------------------------------------------------- /training/callbacks/one_cycle.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Alex Biddulph 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import os 17 | 18 | import numpy as np 19 | 20 | import tensorflow as tf 21 | 22 | 23 | class OneCycle(tf.keras.callbacks.LearningRateScheduler): 24 | def __init__(self, config, **kwargs): 25 | super(OneCycle, self).__init__(self.calc_lr, **kwargs) 26 | 27 | # lr ranges 28 | self.min_lr = float(config["training"]["learning_rate"]["min_learning_rate"]) 29 | self.max_lr = float(config["training"]["learning_rate"]["max_learning_rate"]) 30 | self.decay_lr = float(config["training"]["learning_rate"]["decay_learning_rate"]) 31 | 32 | # Cycle size 33 | self.cycle_epochs = int(config["training"]["learning_rate"]["cycle_epochs"]) 34 | self.decay_epochs = int(config["training"]["epochs"]) - self.cycle_epochs 35 | self.start_step = None if config["training"]["learning_rate"].get("hot_start", False) else 0 36 | 37 | def calc_lr(self, epoch, lr): 38 | 39 | # Update our start step if we haven't run yet 40 | self.start_step = epoch if self.start_step is None else self.start_step 41 | 42 | # While we are in the one cycle, cycle our learning rate 43 | cycle_phase = (epoch - self.start_step) / self.cycle_epochs 44 | if cycle_phase < 0.5: # Going up 45 | p = cycle_phase * 2 # Value from 0-1 46 | return self.min_lr + (self.max_lr - self.min_lr) * p 47 | elif cycle_phase < 1.0: # Going down 48 | p = 1.0 - (cycle_phase - 0.5) * 2 # Value from 0-1 49 | return self.min_lr + (self.max_lr - self.min_lr) * p 50 | 51 | # After the one cycle, we just decay our learning rate slowly down to nothing 52 | else: 53 | decay_phase = (epoch - self.start_step - self.cycle_epochs) / self.decay_epochs 54 | return self.min_lr * (1.0 - decay_phase) + self.decay_lr * decay_phase 55 | -------------------------------------------------------------------------------- /cpp/visualmesh/engine/opencl/operation/scalar_defines.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_OPENCL_OPERATION_SCALAR_DEFINES_HPP 19 | #define VISUALMESH_OPENCL_OPERATION_SCALAR_DEFINES_HPP 20 | 21 | #include "wrapper.hpp" 22 | 23 | namespace visualmesh { 24 | namespace engine { 25 | namespace opencl { 26 | namespace operation { 27 | 28 | /** 29 | * @brief Get the scalar defines for single precision floating point 30 | * 31 | * @return a string containing the defines that are needed if the Scalar type is float 32 | */ 33 | inline constexpr auto get_scalar_defines(float /*scalar_type*/) { 34 | return "#define Scalar float\n" 35 | "#define Scalar2 float2\n" 36 | "#define Scalar3 float3\n" 37 | "#define Scalar4 float4\n" 38 | "#define Scalar8 float8\n" 39 | "#define Scalar16 float16\n"; 40 | } 41 | 42 | /** 43 | * @brief Get the scalar defines for double precision floating point 44 | * 45 | * @return a string containing the defines that are needed if the Scalar type is double 46 | */ 47 | inline constexpr auto get_scalar_defines(double /*scalar_type*/) { 48 | return "#define Scalar double\n" 49 | "#define Scalar2 double2\n" 50 | "#define Scalar3 double3\n" 51 | "#define Scalar4 double4\n" 52 | "#define Scalar8 double8\n" 53 | "#define Scalar16 double16\n"; 54 | } 55 | 56 | } // namespace operation 57 | } // namespace opencl 58 | } // namespace engine 59 | } // namespace visualmesh 60 | 61 | #endif // VISUALMESH_OPENCL_OPERATION_SCALAR_DEFINES_HPP 62 | -------------------------------------------------------------------------------- /readme/dataset.md: -------------------------------------------------------------------------------- 1 | # Dataset 2 | The dataset for the visual mesh should be provided as at least three TFRecord files, these being one or more for each of training, validation and testing. 3 | 4 | Each of the flavours used in the network will add requirements for the data that needs to be provided. 5 | You can look these up in each of the individual flavours and how they influence the dataset on the [Architecture Page](./architecture.md). 6 | For example, if you were to make a dataset for a Monoscopic Image Ground Classification Mesh it would need to have the following keys in it: 7 | 8 | ```python 9 | "Hoc": float[4, 4] # 3D Affine transformation from camera space to observation space 10 | "image": bytes[1] # Compressed image 11 | "mask": bytes[1] # png image 12 | "lens/projection": bytes[1] # 'RECTILINEAR' | 'EQUISOLID' | 'EQUIDISTANT' 13 | "lens/focal_length": float[1] # pixels 14 | "lens/fov": float[1] # radians 15 | "lens/centre": float[2] # pixels 16 | "lens/k": float[2] # pixels 17 | ``` 18 | 19 | If you have a TFRecord dataset which has the correct data types in it, but the keys are incorrect you are able to use the keys field in the configuration file in order to map the keys across. 20 | For example, if you had a field in your dataset `camera/field_of_view` which contained the data required by `lens/fov` you would be able to set up your configuration file like so. 21 | ```yaml 22 | dataset: 23 | training: 24 | paths: 25 | - dataset/training.tfrecord 26 | keys: 27 | lens/fov: camera/field_of_view 28 | ``` 29 | This would allow the dataset to look for the data needed by `lens/fov` in the field `camera/field_of_view` 30 | 31 | You are also able to override any of the settings for the flavours for only a specific dataset using a config tag. 32 | This is typically used if you want to provide augmentations to your training data, but not apply them when validating or testing the network. 33 | 34 | For example, to manipulate the ground plane from the Ground orientation flavour only in training: 35 | 36 | ```yaml 37 | dataset: 38 | training: 39 | paths: 40 | - dataset/training.tfrecord 41 | config: 42 | orientation: 43 | augmentations: 44 | height: { mean: 0, stddev: 0.05 } 45 | rotation: { mean: 0, stddev: 0.0872665 } 46 | ``` 47 | 48 | To make a simple dataset you can follow the instructions in [Quick Start Guide](quickstart.md) or read the code in [training/dataset.py](../training/make_dataset.py) 49 | 50 | ## Batching 51 | For the visual mesh, the method used for batching is very different from how most batching systems work. 52 | Because of the projection, each image that is loaded into the network will likely have a different number of pixels that are projected onto the image. 53 | This results in a variable-length that would go into the batch. 54 | Therefore instead of adding a new dimension and batching over that dimension we instead concatenate all the samples together. 55 | Then to fix the network we update their graph indices by offsetting them based on their position in the concatenation. 56 | This way the graph will ensure that the ragged length batches will continue to interact properly once concatenated. 57 | -------------------------------------------------------------------------------- /cpp/visualmesh/engine/opencl/operation/wrapper.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_ENGINE_OPENCL_OPERATION_WRAPPER_HPP 19 | #define VISUALMESH_ENGINE_OPENCL_OPERATION_WRAPPER_HPP 20 | 21 | #define CL_USE_DEPRECATED_OPENCL_1_2_APIS 22 | #if defined(__APPLE__) || defined(__MACOSX) 23 | #include 24 | #else 25 | #include 26 | #endif // !__APPLE__ 27 | 28 | #include 29 | #include 30 | #include 31 | 32 | #include "opencl_error_category.hpp" 33 | 34 | namespace visualmesh { 35 | namespace engine { 36 | namespace opencl { 37 | 38 | /** 39 | * @brief A shorthand function to throw an OpenCL system error if the error code is not success 40 | * 41 | * @param code the error code to check and throw 42 | * @param msg the message to attach to the exception if it is thrown 43 | */ 44 | inline void throw_cl_error(const cl_int& code, const std::string& msg) { 45 | if (code != CL_SUCCESS) { throw std::system_error(code, operation::opencl_error_category(), msg); } 46 | } 47 | 48 | namespace cl { 49 | template 50 | struct opencl_wrapper : public std::shared_ptr> { 51 | using std::shared_ptr>::shared_ptr; 52 | 53 | operator T() const { 54 | return this->get(); 55 | } 56 | }; 57 | 58 | using command_queue = opencl_wrapper<::cl_command_queue>; 59 | using context = opencl_wrapper<::cl_context>; 60 | using event = opencl_wrapper<::cl_event>; 61 | using kernel = opencl_wrapper<::cl_kernel>; 62 | using mem = opencl_wrapper<::cl_mem>; 63 | using program = opencl_wrapper<::cl_program>; 64 | } // namespace cl 65 | 66 | } // namespace opencl 67 | } // namespace engine 68 | } // namespace visualmesh 69 | 70 | #endif // VISUALMESH_ENGINE_OPENCL_OPERATION_WRAPPER_HPP 71 | -------------------------------------------------------------------------------- /tensorflow/shape_op_base.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_TENSORFLOW_SHAPE_OP_BASE_HPP 19 | #define VISUALMESH_TENSORFLOW_SHAPE_OP_BASE_HPP 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include "visualmesh/geometry/Circle.hpp" 26 | #include "visualmesh/geometry/Sphere.hpp" 27 | 28 | template 29 | class ShapeOpBase : public tensorflow::OpKernel { 30 | public: 31 | explicit ShapeOpBase(tensorflow::OpKernelConstruction* context) : OpKernel(context) {} 32 | 33 | void Compute(tensorflow::OpKernelContext* context) override { 34 | 35 | // Check the geometry type and radius 36 | OP_REQUIRES(context, 37 | tensorflow::TensorShapeUtils::IsScalar(context->input(GEOMETRY).shape()), 38 | tensorflow::errors::InvalidArgument("Geometry must be a single string value")); 39 | OP_REQUIRES(context, 40 | tensorflow::TensorShapeUtils::IsScalar(context->input(RADIUS).shape()), 41 | tensorflow::errors::InvalidArgument("The radius must be a scalar")); 42 | 43 | std::string geometry = *context->input(GEOMETRY).flat().data(); 44 | T radius = context->input(RADIUS).scalar()(0); 45 | 46 | OP_REQUIRES(context, 47 | geometry == "SPHERE" || geometry == "CIRCLE", 48 | tensorflow::errors::InvalidArgument("Geometry must be one of SPHERE or CIRCLE")); 49 | 50 | // clang-format off 51 | using namespace visualmesh::geometry; // NOLINT(google-build-using-namespace) function scope is fine 52 | if (geometry == "SPHERE") { static_cast(this)->DoCompute(context, Sphere(radius)); } 53 | else if (geometry == "CIRCLE") { static_cast(this)->DoCompute(context, Circle(radius)); } 54 | // clang-format on 55 | } 56 | }; 57 | 58 | #endif // VISUALMESH_TENSORFLOW_SHAPE_OP_BASE_HPP 59 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/ring4.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_RING4_HPP 19 | #define VISUALMESH_MODEL_RING4_HPP 20 | 21 | #include 22 | 23 | #include "ring_base.hpp" 24 | #include "visualmesh/node.hpp" 25 | 26 | namespace visualmesh { 27 | namespace model { 28 | 29 | /** 30 | * @brief Model utilising the Ring4 method 31 | * 32 | * @tparam Scalar the scalar type used for calculations and storage (normally one of float or double) 33 | */ 34 | template 35 | struct Ring4 : public RingBase { 36 | 37 | /// The number of neighbours that each node in the graph has 38 | static constexpr int N_NEIGHBOURS = 4; 39 | 40 | /** 41 | * @brief Calculates the neighbour indicies for this ring form 42 | * 43 | * @param i the index of the current point in the current ring 44 | * @param p the number of points in the previous ring 45 | * @param c the number of points in the current ring 46 | * @param n the number of points in the next ring 47 | * 48 | * @return the list of neighbours of this point in a clockwise order 49 | */ 50 | static std::array neighbours(const int& i, const int& p, const int& c, const int& n) { 51 | 52 | // Get how far we are through this ring as a value between 0 and 1 53 | const Scalar f = Scalar(i) / Scalar(c); 54 | 55 | // Left and right is just our index += 1 with wraparound 56 | const int l = static_cast(i > 0 ? i - 1 : c - 1); 57 | const int r = static_cast(i + 1 < c ? i + 1 : 0); 58 | 59 | // Top is closest above 60 | const int t = (static_cast(std::round(f * n)) % n) + c; 61 | 62 | // Bottom is closest below 63 | const int b = (static_cast(std::round(f * p)) % p) - p; 64 | 65 | return std::array{{l, t, r, b}}; 66 | } 67 | }; 68 | 69 | } // namespace model 70 | } // namespace visualmesh 71 | 72 | 73 | #endif // VISUALMESH_MODEL_RING4_HPP 74 | -------------------------------------------------------------------------------- /readme/flavour/orientation.md: -------------------------------------------------------------------------------- 1 | # Orientation 2 | A systems orientation describes where the observation plane is as related to the camera. 3 | The expected output from this flavour is a key `Hoc: float[4, 4]`. 4 | This key is a homogenous transformation matrix which if used like `Hoc * (a,b,c,1)` would convert a vector in the cameras coordinate system into one measured in the observation planes coordinate system. 5 | In this matrix, the visual mesh only cares about rotation and the z component of the translation for height. 6 | ``` 7 | ┌ ┐ 8 | │ 0 │ 9 | │ R 0 │ 10 | │ z │ <- Only Z is important for the visual mesh 11 | │ 0 0 0 1 │ 12 | └ ┘ 13 | ``` 14 | 15 | For the purposes of the visual mesh, the cameras coordinate system is defined as a right-handed coordinate system with the X axis travelling along the optical axis (direction of view) the Y axis pointing to the left of the camera, and the Z axis pointing upward from the camera. 16 | 17 | ## Ground 18 | The ground orientation is the most typical use case for the visual mesh. 19 | In this set up the Hoc provided by the dataset is forwarded on as `Hoc` for the output. 20 | 21 | ### Dataset Keys 22 | The only dataset key required by the Ground flavour is `Hoc` which will be forwarded on as Hoc for the network. 23 | ```python 24 | "Hoc": float[4, 4] 25 | ``` 26 | 27 | ### Configuration 28 | ```yaml 29 | orientation: 30 | type: Ground 31 | config: 32 | augmentations: 33 | # Adjust the height above the observation plane. 34 | height: { mean: 0, stddev: 0.05 } 35 | # Rotate around a random axis by this angle 36 | rotation: { mean: 0, stddev: 0.0872665 } 37 | ``` 38 | 39 | ## Spotlight 40 | The spotlight mesh is based on the idea of projecting a visual mesh plane at some specific point relative to the camera. 41 | It is given a list of targets and will choose one of these targets at random. 42 | When training over multiple epochs this random selection is done each time the example is shown. 43 | This object is projected and used as the origin for a visual mesh observation plane that is tangential to the location of the object. 44 | When this projection is done, the X-axis of the visual mesh will always be aligned with the Z-axis of the observation plane. 45 | This ensures consistent results regardless of where the target is looking 46 | 47 | ### Dataset Keys 48 | The spotlight network requires a `Hoc` which describes the orientation of the camera relative to the observation plane, the same as in Ground. 49 | It additionally requires targets to points that the mesh will be projected to. 50 | The targets are 3D vectors measured in the cameras coordinate system. 51 | ```python 52 | "Hoc": float[4, 4] 53 | "spotlight/targets": float[n, 3] # n >= 1 54 | ``` 55 | 56 | ### Configuration 57 | ```yaml 58 | orientation: 59 | type: Spotlight 60 | config: 61 | augmentations: 62 | # Rotate around a random axis by this angle 63 | rotation: { mean: 0, stddev: 0.0872665 } 64 | # Adjust the position of the spotlight target 65 | # Make sure to use min and max to limit objects min distance 66 | # Otherwise you can end up with an invalid mesh 67 | # Typically set it to the diameter of the object you are looking for 68 | position: { mean: 0, stddev: 0.5, min: 0.5, max: 50.0 } 69 | ``` 70 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit 6 | # persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | cmake_minimum_required(VERSION 3.7.0) 17 | project(VisualMesh VERSION 2.0.0) 18 | 19 | # Default to Release build 20 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 21 | message(STATUS "Setting build type to 'Release' as none was specified.") 22 | set(CMAKE_BUILD_TYPE 23 | Release 24 | CACHE STRING "Choose the type of build." FORCE) 25 | # Set the possible values of build type for cmake-gui 26 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") 27 | endif() 28 | 29 | # If this option is set we are building using continous integration 30 | option(CI_BUILD "Enable build options for building in the CI server" OFF) 31 | 32 | # Default not to run the clang-tidy checks, default to whatever our CI_BUILD is 33 | option(ENABLE_CLANG_TIDY "Enable building with clang-tidy checks." OFF) 34 | if(ENABLE_CLANG_TIDY OR CI_BUILD) 35 | find_package(PythonInterp 3 REQUIRED) 36 | set(CMAKE_CXX_CLANG_TIDY "${PYTHON_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/Scripts/clang-tidy.py" 37 | "${PROJECT_BINARY_DIR}/clang-tidy-fixes" clang-tidy) 38 | set(CMAKE_C_CLANG_TIDY ${CMAKE_CXX_CLANG_TIDY}) 39 | 40 | # Create a target that will apply clang-tidy fixes to the codebase 41 | add_custom_target( 42 | apply-clang-tidy 43 | COMMAND clang-apply-replacements --format --style=file --style-config="${PROJECT_SOURCE_DIR}" 44 | --remove-change-desc-files "${PROJECT_BINARY_DIR}/clang-tidy-fixes" 45 | COMMENT "Applying fixes from clang-tidy to the codebase.") 46 | endif(ENABLE_CLANG_TIDY OR CI_BUILD) 47 | 48 | # Add the cmake module path for custom modules 49 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake/Modules/") 50 | 51 | # Configure the c++ header only library and build generated files 52 | add_subdirectory("cpp") 53 | 54 | # Build the tensorflow op for training 55 | option(BUILD_TENSORFLOW_OP "Bu ild the tensorflow op used for training" ON) 56 | if(BUILD_TENSORFLOW_OP) 57 | add_subdirectory("tensorflow") 58 | endif(BUILD_TENSORFLOW_OP) 59 | 60 | # Build the c++ examples 61 | option(BUILD_EXAMPLES "Build the c++ examples" OFF) 62 | if(BUILD_EXAMPLES) 63 | add_subdirectory("example") 64 | endif(BUILD_EXAMPLES) 65 | -------------------------------------------------------------------------------- /training/metrics/seeker_stddev.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import tensorflow as tf 17 | 18 | 19 | class SeekerStdDev(tf.keras.metrics.Metric): 20 | def __init__(self, name, threshold, **kwargs): 21 | super(SeekerStdDev, self).__init__(name=name, **kwargs) 22 | 23 | self.threshold = threshold 24 | 25 | self.n = self.add_weight(name="n", shape=(), initializer="zeros", dtype=tf.int64) 26 | self.s = self.add_weight(name="s", shape=(), initializer="zeros", dtype=tf.float64) 27 | 28 | def update_state(self, y_true, y_pred, sample_weight=None): 29 | 30 | # Y true contains a list of all the possible answers, we are ranked on how close we are to the closest one 31 | # This code works out the closest point to our prediction for each case so we can use that for training 32 | y_true = tf.gather_nd( 33 | y_true, 34 | tf.stack( 35 | [ 36 | tf.range(tf.shape(y_true)[0], dtype=tf.int32), 37 | tf.math.argmin( 38 | tf.reduce_sum(tf.math.squared_difference(tf.expand_dims(y_pred, axis=-2), y_true), axis=-1), 39 | axis=-1, 40 | output_type=tf.int32, 41 | ), 42 | ], 43 | axis=1, 44 | ), 45 | ) 46 | 47 | y_true = tf.clip_by_value(y_true, -1.0, 1.0) 48 | 49 | # For standard deviation we only want to look at how we went when we were predicting that we were close 50 | idx = tf.squeeze(tf.where(tf.reduce_all(tf.abs(y_pred) <= self.threshold, axis=-1)), axis=-1) 51 | y_true = tf.cast(tf.gather(y_true, idx), dtype=tf.float64) 52 | y_pred = tf.cast(tf.gather(y_pred, idx), dtype=tf.float64) 53 | 54 | # Get the properties we care about from the sample we just received 55 | n = tf.shape(y_true, out_type=tf.int64)[0] 56 | s = tf.math.reduce_sum(tf.math.squared_difference(y_pred, y_true)) 57 | 58 | # First time we just assign the b numbers 59 | self.s.assign_add(s) 60 | self.n.assign_add(n) 61 | 62 | def result(self): 63 | return tf.math.sqrt(self.s / tf.cast(self.n - 1, tf.float64)) 64 | 65 | def reset_states(self): 66 | self.n.assign(tf.zeros_like(self.n)) 67 | self.s.assign(tf.zeros_like(self.s)) 68 | -------------------------------------------------------------------------------- /training/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | from .example import Image 17 | from .label import Classification, Seeker 18 | from .orientation import Ground, Spotlight 19 | from .projection import VisualMesh 20 | from .view import Monoscopic 21 | from .visual_mesh_dataset import VisualMeshDataset 22 | 23 | 24 | def Dataset(paths, batch_size, view, example, orientation, label, projection, keys): 25 | 26 | # Find the correct class to handle our view 27 | if view["type"] == "Monoscopic": 28 | view = Monoscopic(**view["config"]) 29 | else: 30 | raise RuntimeError("Unknown view type '{}'".format(view["type"])) 31 | 32 | # Find the correct class to handle our view 33 | if example["type"] == "Image": 34 | example = Image(**example["config"]) 35 | else: 36 | raise RuntimeError("Unknown example type '{}'".format(example["type"])) 37 | 38 | # Find the correct class to handle our mesh orientation 39 | if orientation["type"] == "Ground": 40 | orientation = Ground(**orientation["config"]) 41 | elif orientation["type"] == "Spotlight": 42 | orientation = Spotlight(**orientation["config"]) 43 | else: 44 | raise RuntimeError("Unknown orientation type '{}'".format(orientation["type"])) 45 | 46 | # Find the correct class to handle our mesh orientation 47 | if projection["type"] == "VisualMesh": 48 | projection = VisualMesh(**projection["config"]) 49 | else: 50 | raise RuntimeError("Unknown projection type '{}'".format(projection["type"])) 51 | 52 | # Find the correct dataset labelling class 53 | if label["type"] == "Classification": 54 | label = Classification(**label["config"]) 55 | elif label["type"] == "Seeker": 56 | label = Seeker(**label["config"]) 57 | else: 58 | raise RuntimeError("Unknown data labelling scheme '{}'".format(label["type"])) 59 | 60 | # Create our dataset with these specific components 61 | return VisualMeshDataset( 62 | paths=paths, 63 | batch_size=batch_size, 64 | view=view, 65 | example=example, 66 | orientation=orientation, 67 | projection=projection, 68 | label=label, 69 | keys=keys, 70 | ).build() 71 | 72 | 73 | # Convert a dataset into a format that will be accepted by keras fit 74 | def keras_dataset(args): 75 | # Return in the format (x, y, weights) 76 | return ((args["X"], args["G"]), args["Y"]) 77 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/ring6.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_RING6_HPP 19 | #define VISUALMESH_MODEL_RING6_HPP 20 | 21 | #include 22 | 23 | #include "ring_base.hpp" 24 | #include "visualmesh/node.hpp" 25 | 26 | namespace visualmesh { 27 | namespace model { 28 | 29 | /** 30 | * @brief Model utilising the Ring6 method 31 | * 32 | * @tparam Scalar the scalar type used for calculations and storage (normally one of float or double) 33 | */ 34 | template 35 | struct Ring6 : public RingBase { 36 | 37 | /// The number of neighbours that each node in the graph has 38 | static constexpr int N_NEIGHBOURS = 6; 39 | 40 | /** 41 | * @brief Calculates the neighbour indicies for this ring form 42 | * 43 | * @param i the index of the current point in the current ring 44 | * @param p the number of points in the previous ring 45 | * @param c the number of points in the current ring 46 | * @param n the number of points in the next ring 47 | * 48 | * @return the list of neighbours of this point in a clockwise order 49 | */ 50 | static std::array neighbours(const int& i, const int& p, const int& c, const int& n) { 51 | 52 | // Get how far we are through this ring as a value between 0 and 1 53 | const Scalar f = Scalar(i) / Scalar(c); 54 | 55 | // Left and right is just our index += 1 with wraparound 56 | const int l = static_cast(i > 0 ? i - 1 : c - 1); 57 | const int r = static_cast(i + 1 < c ? i + 1 : 0); 58 | 59 | // Top left and top right are the next ring around nearest left and right with wraparound 60 | const int tl = (static_cast(f * n) % n) + c; 61 | const int tr = ((static_cast(f * n) + 1) % n) + c; 62 | 63 | // Bottom left and bottom right are the next ring around nearest left and right with wraparound 64 | const int bl = (static_cast(f * p) % p) - p; 65 | const int br = ((static_cast(f * p) + 1) % p) - p; 66 | 67 | // The indices of our neighbours presented in a clockwise arrangement relative to the start of our row 68 | return std::array{{l, tl, tr, r, br, bl}}; 69 | } 70 | }; 71 | 72 | } // namespace model 73 | } // namespace visualmesh 74 | 75 | 76 | #endif // VISUALMESH_MODEL_RING6_HPP 77 | -------------------------------------------------------------------------------- /training/unexport.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2024 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import os 17 | 18 | import numpy as np 19 | import yaml 20 | 21 | import tensorflow as tf 22 | 23 | from .flavour import Dataset 24 | from .layer.graph_convolution import GraphConvolution 25 | from .model import VisualMeshModel 26 | 27 | 28 | def unexport(config, output_path): 29 | 30 | with open(os.path.join(output_path, "model.yaml"), "r") as f: 31 | network = yaml.safe_load(f) 32 | 33 | # Get the training dataset so we know the output size 34 | training_dataset = Dataset(config, "training") 35 | 36 | # Get the dimensionality of the Y part of the dataset 37 | output_dims = training_dataset.element_spec["Y"].shape[-1] 38 | 39 | # Define the model 40 | model = VisualMeshModel(structure=config["network"]["structure"], output_dims=output_dims) 41 | 42 | # We have to run a predict step so that everything is loaded properly 43 | for v in training_dataset.take(1): 44 | model(v["X"], v["G"], training=False) 45 | 46 | stages = [] 47 | for m in model.stages: 48 | op = model.ops[m] 49 | 50 | if type(op[0]) is GraphConvolution: 51 | op = op[0] 52 | stages.append( 53 | [ 54 | { 55 | "weights": op.dense.weights[0], 56 | "biases": op.dense.weights[1], 57 | "activation": op.dense.activation.__name__, 58 | } 59 | ] 60 | ) 61 | elif type(op[0]) is tf.keras.layers.Dense: 62 | op = op[0] 63 | stages[-1].append( 64 | { 65 | "weights": op.weights[0], 66 | "biases": op.weights[1], 67 | "activation": op.activation.__name__, 68 | } 69 | ) 70 | else: 71 | print("Error: currently we can only import GraphConvolution and Dense layers") 72 | exit(1) 73 | 74 | for stage, conf in zip(stages, network["network"]): 75 | for w, c in zip(stage, conf): 76 | 77 | weight_shape = w["weights"].shape 78 | bias_shape = w["biases"].shape 79 | 80 | in_weights = tf.constant(c["weights"], w["weights"].dtype) 81 | in_biases = tf.constant(c["biases"], w["biases"].dtype) 82 | 83 | w["weights"].assign(in_weights[0 : weight_shape[0], 0 : weight_shape[1]]) 84 | w["biases"].assign(in_biases[0 : bias_shape[0]]) 85 | 86 | model.save_weights(os.path.join(output_path, "checkpoint")) 87 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/ring8.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_RING8_HPP 19 | #define VISUALMESH_MODEL_RING8_HPP 20 | 21 | #include 22 | 23 | #include "ring_base.hpp" 24 | #include "visualmesh/node.hpp" 25 | 26 | namespace visualmesh { 27 | namespace model { 28 | 29 | /** 30 | * @brief Model utilising the Ring8 method 31 | * 32 | * @tparam Scalar the scalar type used for calculations and storage (normally one of float or double) 33 | */ 34 | template 35 | struct Ring8 : public RingBase { 36 | 37 | /// The number of neighbours that each node in the graph has 38 | static constexpr int N_NEIGHBOURS = 8; 39 | 40 | /** 41 | * @brief Calculates the neighbour indicies for this ring form 42 | * 43 | * @param i the index of the current point in the current ring 44 | * @param p the number of points in the previous ring 45 | * @param c the number of points in the current ring 46 | * @param n the number of points in the next ring 47 | * 48 | * @return the list of neighbours of this point in a clockwise order 49 | */ 50 | static std::array neighbours(const int& i, const int& p, const int& c, const int& n) { 51 | 52 | // Get how far we are through this ring as a value between 0 and 1 53 | const Scalar f = Scalar(i) / Scalar(c); 54 | 55 | // Left and right is just our index += 1 with wraparound 56 | const int l = static_cast(i > 0 ? i - 1 : c - 1); 57 | const int r = static_cast(i + 1 < c ? i + 1 : 0); 58 | 59 | // Top is closest to us above, and tl/tr are the two around that with wraparound 60 | const int t = static_cast(std::round(f * n)) % n; 61 | const int tl = (t + n - 1) % n; 62 | const int tr = (t + 1) % n; 63 | 64 | // Bottom is closest to us below, and bl/br are the two around that with wraparound 65 | const int b = static_cast(std::round(f * p)) % p; 66 | const int bl = (b + p - 1) % p; 67 | const int br = (b + 1) % p; 68 | 69 | // The indices of our neighbours presented in a clockwise arrangement relative to the start of our row 70 | return std::array{{l, tl + c, t + c, tr + c, r, br - p, b - p, bl - p}}; 71 | } 72 | }; 73 | 74 | } // namespace model 75 | } // namespace visualmesh 76 | 77 | 78 | #endif // VISUALMESH_MODEL_RING8_HPP 79 | -------------------------------------------------------------------------------- /example/draw.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef EXAMPLE_DRAW_HPP 19 | #define EXAMPLE_DRAW_HPP 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include "visualmesh/classified_mesh.hpp" 26 | #include "visualmesh/projected_mesh.hpp" 27 | 28 | template 29 | void draw(const std::string& window, 30 | const cv::Mat& image, 31 | const visualmesh::ClassifiedMesh& mesh, 32 | const std::vector& colours) { 33 | 34 | const auto& classifications = mesh.classifications; 35 | const auto& pixel_coordinates = mesh.pixel_coordinates; 36 | const auto& neighbourhood = mesh.neighbourhood; 37 | const size_t num_classes = colours.size(); 38 | 39 | cv::Mat scratch = image.clone(); 40 | 41 | for (unsigned int i = 0; i < pixel_coordinates.size(); ++i) { 42 | cv::Point p1(pixel_coordinates[i][0], pixel_coordinates[i][1]); 43 | 44 | // Work out what colour based on mixing 45 | const Scalar* cl = classifications.data() + (i * num_classes); 46 | cv::Scalar colour(0, 0, 0); 47 | 48 | for (unsigned int i = 0; i < colours.size(); ++i) { 49 | colour += colours[i] * cl[i]; 50 | } 51 | 52 | for (const auto& n : neighbourhood[i]) { 53 | if (n < static_cast(pixel_coordinates.size())) { 54 | cv::Point p2(pixel_coordinates[n][0], pixel_coordinates[n][1]); 55 | cv::Point p2x = p1 + ((p2 - p1) * 0.5); 56 | cv::line(scratch, p1, p2x, colour, 1, cv::LINE_AA); 57 | } 58 | } 59 | } 60 | 61 | cv::imshow(window, scratch); 62 | } 63 | 64 | 65 | template 66 | void draw(const std::string& window, 67 | const cv::Mat& image, 68 | const visualmesh::ProjectedMesh& mesh, 69 | const cv::Scalar& colour) { 70 | 71 | const auto& pixel_coordinates = mesh.pixel_coordinates; 72 | const auto& neighbourhood = mesh.neighbourhood; 73 | 74 | cv::Mat scratch = image.clone(); 75 | 76 | for (unsigned int i = 0; i < pixel_coordinates.size(); ++i) { 77 | cv::Point p1(pixel_coordinates[i][0], pixel_coordinates[i][1]); 78 | 79 | for (const auto& n : neighbourhood[i]) { 80 | if (n < static_cast(pixel_coordinates.size())) { 81 | cv::Point p2(pixel_coordinates[n][0], pixel_coordinates[n][1]); 82 | cv::Point p2x = p1 + ((p2 - p1) * 0.5); 83 | cv::line(scratch, p1, p2x, colour, 1, cv::LINE_AA); 84 | } 85 | } 86 | } 87 | 88 | cv::imshow(window, scratch); 89 | } 90 | 91 | #endif // EXAMPLE_DRAW_HPP 92 | -------------------------------------------------------------------------------- /readme/quickstart.md: -------------------------------------------------------------------------------- 1 | # Quick Start Guide 2 | This guide is designed to get a classification network training for your dataset ASAP. 3 | If you need something more specific read through the rest of these readme files and feel free to ask on Gitter [![Join the chat at https://gitter.im/Fastcode/VisualMesh](https://badges.gitter.im/Fastcode/VisualMesh.svg)](https://gitter.im/Fastcode/VisualMesh?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge). 4 | 5 | This guide assumes you're using docker and have already installed it. 6 | 7 | ## Setting up 8 | To setup with docker once you have docker installed all you need to do is build the docker image. 9 | ```sh 10 | docker build . --pull -t visualmesh:latest 11 | ``` 12 | 13 | ## Making a dataset 14 | 15 | ### Gathering the data 16 | Create a folder that contains your source training data. 17 | You will need three files per image: 18 | 19 | |`imageX.jpg`|`maskX.png`| 20 | |:-:|:-:| 21 | |![Image](flavour/label/image.jpg)| ![Mask](flavour/label/mask.png)| 22 | 23 | **Lens.yaml** 24 | ```yaml 25 | projection: EQUISOLID 26 | focal_length: 420 27 | centre: [0, 0] 28 | k: [0, 0] 29 | fov: 1.80248 30 | Hoc: 31 | - [-0.244225, -0.905877, 0.346036, 0] 32 | - [-0.0304556, 0.363831, 0.930967, 0] 33 | - [-0.96924, 0.216827, -0.116446, 0.8] 34 | - [0, 0, 0, 1] 35 | ``` 36 | 37 | - **`Hoc`** is a homogenous transformation matrix which if used like `Hoc * (a,b,c,1)` would convert a vector in the cameras coordinate system into one from the ground. 38 | In this matrix, only the z component of translation matters (the height of the camera above the ground). 39 | In this example, the units are measured in meters (m). 40 | However, the unit that you use for this measurement don't matter, so long as the units you use for the radius of your geometry are the same. 41 | ``` 42 | ┌ ┐ 43 | │ 0 │ 44 | │ R 0 │ 45 | │ z │ <- Only Z is important for the visual mesh 46 | │ 0 0 0 1 │ 47 | └ ┘ 48 | ``` 49 | - **`projection`** is either `RECTILINEAR`, `EQUISOLID` or `EQUIDISTANT` depending on the lens used in the camera. 50 | 51 | - **`focal_length`** is the focal length of the camera measured in pixels. 52 | 53 | - **`centre`** is the offset from the centre of the lens, to the centre of the image in pixels. 54 | If you don't know set it to `[0, 0]` 55 | 56 | - **`k`** is the two component radial distortion values for the lens as per a polynomial series. 57 | If you don't know set it to `[0, 0]` 58 | 59 | - **`fov`** is the field of view of the camera, for many rectilinear lenses it will be the full diagonal field of view, but for lenses with black circles (like above) it will be the field of view of that black circle. 60 | 61 | ### Building the dataset 62 | Once you have these in a folder make sure they are named `imageX.jpg`, `maskX.jpg` and `lensX.yaml` where `X` is replaced with a unique number per example. 63 | 64 | You can then run 65 | ```sh 66 | ./training/build_dataset.py 67 | ``` 68 | and it will create a training, validation and testing TFRecord file containing the data. 69 | 70 | ## Training 71 | In order to train, you must create a configuration file. 72 | To start with you can copy `example_net.yaml`. 73 | You must place this configuration file in the output folder under the name `config.yaml`. 74 | For example, copy over `example_net.yaml` to `output/quickstart/v1/config.yaml` 75 | 76 | Running the following command will then begin training the network. 77 | ```sh 78 | ./docker run --gpus all -u $(id -u):$(id -g) -it --rm --volume $(pwd):/workspace visualmesh:latest ./mesh.py train 79 | ``` 80 | 81 | ## Testing 82 | Once your network has trained you are able to run the following command to test the network. 83 | ```sh 84 | ./docker run --gpus all -u $(id -u):$(id -g) -it --rm --volume $(pwd):/workspace visualmesh:latest ./mesh.py test 85 | ``` 86 | It will measure several properties of the network and output them to the output directory. 87 | -------------------------------------------------------------------------------- /training/metrics/test/confusion.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017-2020 Trent Houliston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | # documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 6 | # permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | # 8 | # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 9 | # Software. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 12 | # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 13 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 14 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | import os 17 | 18 | import tensorflow as tf 19 | 20 | 21 | class Confusion(tf.keras.metrics.Metric): 22 | def __init__(self, name, classes, **kwargs): 23 | super(Confusion, self).__init__(name=name, **kwargs) 24 | 25 | self.classes = classes 26 | 27 | self.confusion = self.add_weight( 28 | name="confusion", shape=(len(classes), len(classes)), initializer="zeros", dtype=tf.int32 29 | ) 30 | 31 | def update_state(self, y_true, y_pred, sample_weight=None): 32 | 33 | # Build up an index list that maps each class 34 | idx = tf.stack( 35 | [ 36 | tf.argmax(input=y_true, axis=-1, output_type=self.confusion.dtype), 37 | tf.argmax(input=y_pred, axis=-1, output_type=self.confusion.dtype), 38 | ], 39 | axis=-1, 40 | ) 41 | 42 | # Trim down the indexes to only those that have a class label 43 | idx = tf.gather(idx, tf.squeeze(tf.where(tf.reduce_any(tf.greater(y_true, 0.0), axis=-1)), axis=-1)) 44 | 45 | # Add them into the corresponding locations 46 | self.confusion.scatter_nd_add(idx, tf.ones_like(idx[:, 0], dtype=self.confusion.dtype)) 47 | 48 | def reset_states(self): 49 | self.confusion.assign(tf.zeros_like(self.confusion)) 50 | 51 | def result(self): 52 | return 0 53 | 54 | def _write(self, f, txt): 55 | print("{}".format(txt)) 56 | f.write("{}\n".format(txt)) 57 | 58 | def save(self, output_path): 59 | 60 | base_path = os.path.join(output_path, "test", self.name) 61 | os.makedirs(os.path.dirname(base_path), exist_ok=True) 62 | 63 | with open("{}.txt".format(base_path), "w") as f: 64 | 65 | for i, c in enumerate(self.classes): 66 | 67 | # For all labels where idx was predicted (all positives) 68 | p = tf.reduce_sum(self.confusion[:, i]) 69 | # For all predictions where idx was labelled 70 | tp_fn = tf.reduce_sum(self.confusion[i, :]) 71 | 72 | # Save the metrics 73 | name = c["name"] 74 | self._write(f, "{}".format(name.title())) 75 | self._write(f, "\tPrecision: {}".format(self.confusion[i, i] / p)) 76 | self._write(f, "\tRecall: {}".format(self.confusion[i, i] / tp_fn)) 77 | 78 | self._write(f, "\tPredicted {} samples are really:".format(name.title())) 79 | for j, k in enumerate(self.classes): 80 | self._write(f, "\t\t{}: {:.3f}%".format(k["name"].title(), 100 * (self.confusion[j, i] / p))) 81 | self._write(f, "\tReal {} samples are predicted as:".format(name.title())) 82 | for j, k in enumerate(self.classes): 83 | self._write(f, "\t\t{}: {:.3f}%".format(k["name"].title(), 100 * (self.confusion[i, j] / tp_fn))) 84 | -------------------------------------------------------------------------------- /cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # All the header files for this library 2 | file(GLOB hdr "**/*.hpp") 3 | 4 | # Convert our opencl files into header files with their contents and put them in the build folder 5 | file(GLOB cls "visualmesh/engine/opencl/kernels/**.cl") 6 | foreach(cl IN LISTS cls) 7 | file(RELATIVE_PATH cl_path ${CMAKE_CURRENT_SOURCE_DIR} ${cl}) 8 | add_custom_command( 9 | OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${cl_path}.hpp" 10 | COMMAND ${PYTHON_EXECUTABLE} ARGS "${PROJECT_SOURCE_DIR}/cmake/Scripts/wrap_opencl.py" "${cl}" 11 | "${CMAKE_CURRENT_BINARY_DIR}/${cl_path}.hpp" 12 | DEPENDS "${cl}" "${PROJECT_SOURCE_DIR}/cmake/Scripts/wrap_opencl.py" 13 | COMMENT "Wrapping OpenCL file ${cl_path} in a header") 14 | list(APPEND hdr "${CMAKE_CURRENT_BINARY_DIR}/${cl_path}.hpp") 15 | endforeach(cl ${cls}) 16 | 17 | # Add to ALL target so that the wrapped header files are always generated, even if nothing else is being built 18 | add_custom_target(visualmesh_sources ALL DEPENDS ${hdr}) 19 | 20 | # Define the VisualMesh library as header only but setup its links 21 | add_library(visualmesh INTERFACE) 22 | target_include_directories(visualmesh INTERFACE $ 23 | $) 24 | target_include_directories(visualmesh INTERFACE $ 25 | $) 26 | add_dependencies(visualmesh visualmesh_sources) 27 | target_compile_features(visualmesh INTERFACE cxx_std_14) 28 | 29 | # Find engine libraries so we can link to them 30 | option(BUILD_OPENCL_ENGINE "Should we build the OpenCL engine" ON) 31 | if(BUILD_OPENCL_ENGINE) 32 | find_package(OpenCL) 33 | if(OpenCL_FOUND) 34 | target_link_libraries(visualmesh INTERFACE OpenCL::OpenCL) 35 | else() 36 | target_compile_definitions(visualmesh INTERFACE VISUALMESH_DISABLE_OPENCL) 37 | endif(OpenCL_FOUND) 38 | else() 39 | target_compile_definitions(visualmesh INTERFACE VISUALMESH_DISABLE_OPENCL) 40 | endif(BUILD_OPENCL_ENGINE) 41 | 42 | option(BUILD_VULKAN_ENGINE "Should we build the Vulkan engine" OFF) 43 | if(BUILD_VULKAN_ENGINE) 44 | find_package(SPIRV) 45 | find_package(Vulkan) 46 | if(SPIRV_FOUND AND Vulkan_FOUND) 47 | target_link_libraries(visualmesh INTERFACE Vulkan::Vulkan SPIRV::SPIRV) 48 | else() 49 | target_compile_definitions(visualmesh INTERFACE VISUALMESH_DISABLE_VULKAN) 50 | endif(SPIRV_FOUND AND Vulkan_FOUND) 51 | else() 52 | target_compile_definitions(visualmesh INTERFACE VISUALMESH_DISABLE_VULKAN) 53 | endif(BUILD_VULKAN_ENGINE) 54 | 55 | # Create the VisualMeshConfig files 56 | include(GNUInstallDirs) 57 | include(CMakePackageConfigHelpers) 58 | set(INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_FULL_INCLUDEDIR}) 59 | write_basic_package_version_file(${PROJECT_BINARY_DIR}/VisualMeshConfigVersion.cmake COMPATIBILITY SameMajorVersion) 60 | configure_package_config_file( 61 | "${PROJECT_SOURCE_DIR}/cmake/VisualMeshConfig.cmake.in" "${PROJECT_BINARY_DIR}/VisualMeshConfig.cmake" 62 | INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/VisualMesh" 63 | PATH_VARS INSTALL_INCLUDE_DIR) 64 | 65 | # Install version, config and target files. 66 | install(FILES "${PROJECT_BINARY_DIR}/VisualMeshConfigVersion.cmake" "${PROJECT_BINARY_DIR}/VisualMeshConfig.cmake" 67 | DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/VisualMesh") 68 | install( 69 | EXPORT VisualMeshTargets 70 | DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/VisualMesh" 71 | NAMESPACE visualmesh::) 72 | 73 | # Install headers and targets 74 | install( 75 | TARGETS visualmesh 76 | EXPORT VisualMeshTargets 77 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) 78 | install( 79 | DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/visualmesh" 80 | DESTINATION include 81 | FILES_MATCHING 82 | PATTERN "*.hpp") 83 | install( 84 | DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/visualmesh" 85 | DESTINATION include 86 | FILES_MATCHING 87 | PATTERN "*.hpp") 88 | -------------------------------------------------------------------------------- /cpp/visualmesh/model/xygrid_map.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Trent Houliston 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 5 | * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 6 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | * permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | * 9 | * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the 10 | * Software. 11 | * 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 15 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | 18 | #ifndef VISUALMESH_MODEL_XYGRID_MAP_HPP 19 | #define VISUALMESH_MODEL_XYGRID_MAP_HPP 20 | 21 | #include "visualmesh/utility/math.hpp" 22 | 23 | namespace visualmesh { 24 | namespace model { 25 | 26 | template 27 | struct XYGridMap { 28 | 29 | /** 30 | * @brief Takes a point in n, m space (jumps along x and jumps along y) and converts it into a vector to the 31 | * centre of the object using the x grid method 32 | * 33 | * @param shape the shape object used to calculate the angles 34 | * @param h the height of the camera above the observation plane 35 | * @param nm the coordinates in the nm space (object space) 36 | * 37 | * @return a vector that points to the centre of the object at these coordinates 38 | */ 39 | template 40 | static vec3 map(const Shape& shape, const Scalar& h, const vec2& nm) { 41 | const Scalar phi_x = shape.phi(nm[0], h); 42 | const Scalar x = (h - shape.c()) * std::tan(phi_x); 43 | 44 | const Scalar phi_y = shape.phi(nm[1], h); 45 | const Scalar y = (h - shape.c()) * std::tan(phi_y); 46 | 47 | return vec3{x, y, shape.c() - h}; 48 | } 49 | 50 | /** 51 | * @brief Takes a unit vector that points to a location and maps it to object coordinates as xy space 52 | * 53 | * @tparam Shape the shape of the object we are mapping for 54 | * 55 | * @param shape the shape object used to calculate the angles 56 | * @param h the height of the camera above the observation plane 57 | * @param u the unit vector that points towards the centre of the object 58 | * 59 | * @return a vector that points to the centre of the object at these coordinates 60 | */ 61 | template 62 | static vec2 unmap(const Shape& shape, const Scalar& h, const vec3& u) { 63 | 64 | // Height of the object above the observation plane so we can get planes from it's centre 65 | const Scalar& c = shape.c(); 66 | 67 | // Extend out vec to the ground (divide by z and multiply by h-c) 68 | vec3 v = multiply(u, (c - h) / u[2]); 69 | 70 | // Work out what phi would have been 71 | const Scalar phi_n = std::atan(-std::abs(v[0]) / std::abs(c - h)); 72 | const Scalar phi_m = std::atan(-std::abs(v[1]) / std::abs(c - h)); 73 | 74 | vec2 nm = {{shape.n(phi_n, h) * (u[0] >= 0 ? -1 : 1), shape.n(phi_m, h) * (u[1] >= 0 ? -1 : 1)}}; 75 | 76 | return nm; 77 | } 78 | }; 79 | 80 | } // namespace model 81 | } // namespace visualmesh 82 | 83 | #endif // VISUALMESH_MODEL_XYGRID_MAP_HPP 84 | --------------------------------------------------------------------------------