├── experiments ├── model │ ├── __init__.py │ └── configs.py ├── sim2real │ ├── __init__.py │ ├── util │ │ ├── __init__.py │ │ ├── readme.md │ │ ├── confusion_matrix_cb.py │ │ └── acc_loss_cb.py │ ├── plot_confusion_matrix.py │ ├── nn.py │ ├── eval.py │ ├── plot_all.py │ └── train_nn.py ├── fine_tuning │ ├── __init__.py │ ├── readme.md │ ├── losses.py │ ├── grid_search.py │ ├── random_search.py │ └── bayesian_optimization.py ├── preparation │ ├── __init__.py │ ├── split.py │ ├── generate_sim.py │ ├── loss_eval.py │ ├── align_globally.py │ └── align_per_object.py └── data_loader.py ├── fdm_printer_description ├── meshes │ ├── thanks.txt │ ├── bed.stl │ ├── bed2.stl │ ├── x_axis.stl │ ├── external_frame.stl │ └── CR-10_platform2.stl ├── materials │ ├── smartlab_logo.png │ └── smartlab_logo.material ├── models │ ├── hardware.yaml │ ├── fdm_printer_control.yaml │ ├── fdm_printer.xacro │ └── macros.xacro ├── package.xml └── CMakeLists.txt ├── gelsight_description ├── meshes │ ├── glass.stl │ ├── gelsight2014_back.stl │ └── gelsight2014_front.stl ├── urdf │ ├── gelsight2014.xacro │ ├── gelsight2014_spectre.xacro │ ├── gelsight2014_case.xacro │ ├── gelsight_camera.xacro │ └── gelsight2014_macro.xacro ├── launch │ ├── rviz.launch │ └── config.rviz ├── package.xml └── CMakeLists.txt ├── gelsight_gazebo ├── assets │ ├── background.png │ └── background_gelsight2017.jpg ├── launch │ └── launch.launch ├── package.xml ├── CMakeLists.txt └── src │ └── gelsight_driver.py ├── gelsight_simulation ├── solids │ ├── stl │ │ ├── cone.stl │ │ ├── dots.stl │ │ ├── line.stl │ │ ├── moon.stl │ │ ├── dot_in.stl │ │ ├── hexagon.stl │ │ ├── pacman.stl │ │ ├── prism.stl │ │ ├── random.stl │ │ ├── sphere.stl │ │ ├── sphere2.stl │ │ ├── torus.stl │ │ ├── wave1.stl │ │ ├── cylinder.stl │ │ ├── flat_slab.stl │ │ ├── triangle.stl │ │ ├── cross_lines.stl │ │ ├── curved_surface.stl │ │ ├── cylinder_shell.stl │ │ ├── cylinder_side.stl │ │ └── parallel_lines.stl │ └── cad │ │ ├── cone.fcstd │ │ ├── cone.fcstd1 │ │ ├── dots.FCStd │ │ ├── dots.FCStd1 │ │ ├── line.fcstd │ │ ├── line.fcstd1 │ │ ├── moon.fcstd │ │ ├── moon.fcstd1 │ │ ├── prism.FCStd │ │ ├── torus.fcstd │ │ ├── wave1.FCStd │ │ ├── dot_in.FCStd │ │ ├── dot_in.FCStd1 │ │ ├── hexagon.fcstd │ │ ├── pacman.FCStd │ │ ├── pacman.FCStd1 │ │ ├── prism.FCStd1 │ │ ├── random1.fcstd │ │ ├── sphere.fcstd │ │ ├── sphere.fcstd1 │ │ ├── sphere2.fcstd │ │ ├── torus.fcstd1 │ │ ├── wave1.FCStd1 │ │ ├── cylinder.fcstd │ │ ├── cylinder.fcstd1 │ │ ├── flat_slab.fcstd │ │ ├── flat_slab.fcstd1 │ │ ├── hexagon.fcstd1 │ │ ├── random1.fcstd1 │ │ ├── sphere2.fcstd1 │ │ ├── triangle.fcstd │ │ ├── triangle.fcstd1 │ │ ├── cross_lines.FCStd │ │ ├── cross_lines.FCStd1 │ │ ├── curved_surface.FCStd │ │ ├── curved_surface.FCStd1 │ │ ├── cylinder_shell.fcstd │ │ ├── cylinder_shell.fcstd1 │ │ ├── cylinder_side.FCStd │ │ ├── cylinder_side.FCStd1 │ │ ├── parallel_lines.FCStd │ │ ├── parallel_lines.FCStd1 │ │ ├── tactile_dataset_mount.fcstd │ │ └── tactile_dataset_mount.fcstd1 ├── models │ ├── setup.xacro │ └── solid.xacro ├── scripts │ ├── gelsight_driver.py │ └── data_collection.py ├── package.xml ├── launch │ └── dc.launch └── CMakeLists.txt ├── fdm_printer ├── controller_plugins.xml ├── src │ ├── position_controller.cpp │ ├── fdm_printer_driver.py │ └── fdm_printer_hardware_interface.cpp ├── package.xml └── CMakeLists.txt ├── fdm_printer_bringup ├── launch │ └── bringup.launch ├── package.xml └── CMakeLists.txt └── README.md /experiments/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/sim2real/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/fine_tuning/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/sim2real/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/preparation/__init__.py: -------------------------------------------------------------------------------- 1 | import src.experiments.preparation.align -------------------------------------------------------------------------------- /experiments/sim2real/util/readme.md: -------------------------------------------------------------------------------- 1 | copied and adapted from 2 | https://github.com/chasingbob/keras-visuals -------------------------------------------------------------------------------- /fdm_printer_description/meshes/thanks.txt: -------------------------------------------------------------------------------- 1 | https://www.youtube.com/watch?v=jkudSgv_uTY 2 | 3 | Jantzen Day 4 | -------------------------------------------------------------------------------- /gelsight_description/meshes/glass.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_description/meshes/glass.stl -------------------------------------------------------------------------------- /gelsight_gazebo/assets/background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_gazebo/assets/background.png -------------------------------------------------------------------------------- /fdm_printer_description/meshes/bed.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/fdm_printer_description/meshes/bed.stl -------------------------------------------------------------------------------- /fdm_printer_description/meshes/bed2.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/fdm_printer_description/meshes/bed2.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/cone.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/cone.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/dots.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/dots.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/line.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/line.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/moon.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/moon.stl -------------------------------------------------------------------------------- /fdm_printer_description/meshes/x_axis.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/fdm_printer_description/meshes/x_axis.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cone.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cone.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cone.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cone.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/dots.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/dots.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/dots.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/dots.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/line.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/line.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/line.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/line.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/moon.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/moon.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/moon.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/moon.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/prism.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/prism.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/torus.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/torus.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/wave1.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/wave1.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/dot_in.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/dot_in.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/hexagon.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/hexagon.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/pacman.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/pacman.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/prism.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/prism.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/random.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/random.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/sphere.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/sphere.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/sphere2.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/sphere2.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/torus.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/torus.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/wave1.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/wave1.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/dot_in.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/dot_in.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/dot_in.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/dot_in.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/hexagon.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/hexagon.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/pacman.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/pacman.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/pacman.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/pacman.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/prism.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/prism.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/random1.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/random1.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/sphere.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/sphere.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/sphere.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/sphere.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/sphere2.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/sphere2.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/torus.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/torus.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/wave1.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/wave1.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/cylinder.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/cylinder.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/flat_slab.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/flat_slab.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/triangle.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/triangle.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cylinder.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cylinder.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cylinder.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cylinder.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/flat_slab.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/flat_slab.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/flat_slab.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/flat_slab.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/hexagon.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/hexagon.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/random1.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/random1.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/sphere2.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/sphere2.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/triangle.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/triangle.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/triangle.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/triangle.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/cross_lines.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/cross_lines.stl -------------------------------------------------------------------------------- /fdm_printer_description/meshes/external_frame.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/fdm_printer_description/meshes/external_frame.stl -------------------------------------------------------------------------------- /gelsight_description/meshes/gelsight2014_back.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_description/meshes/gelsight2014_back.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cross_lines.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cross_lines.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cross_lines.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cross_lines.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/curved_surface.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/curved_surface.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/cylinder_shell.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/cylinder_shell.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/cylinder_side.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/cylinder_side.stl -------------------------------------------------------------------------------- /gelsight_simulation/solids/stl/parallel_lines.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/stl/parallel_lines.stl -------------------------------------------------------------------------------- /fdm_printer_description/materials/smartlab_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/fdm_printer_description/materials/smartlab_logo.png -------------------------------------------------------------------------------- /fdm_printer_description/meshes/CR-10_platform2.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/fdm_printer_description/meshes/CR-10_platform2.stl -------------------------------------------------------------------------------- /gelsight_description/meshes/gelsight2014_front.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_description/meshes/gelsight2014_front.stl -------------------------------------------------------------------------------- /gelsight_gazebo/assets/background_gelsight2017.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_gazebo/assets/background_gelsight2017.jpg -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/curved_surface.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/curved_surface.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/curved_surface.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/curved_surface.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cylinder_shell.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cylinder_shell.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cylinder_shell.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cylinder_shell.fcstd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cylinder_side.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cylinder_side.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/cylinder_side.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/cylinder_side.FCStd1 -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/parallel_lines.FCStd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/parallel_lines.FCStd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/parallel_lines.FCStd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/parallel_lines.FCStd1 -------------------------------------------------------------------------------- /fdm_printer_description/models/hardware.yaml: -------------------------------------------------------------------------------- 1 | fdm_printer: 2 | hardware_interface: 3 | loop_hz: 10 # hz 4 | joints: 5 | - x_axis 6 | - y_axis 7 | - z_axis -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/tactile_dataset_mount.fcstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/tactile_dataset_mount.fcstd -------------------------------------------------------------------------------- /gelsight_simulation/solids/cad/tactile_dataset_mount.fcstd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danfergo/gelsight_simulation/HEAD/gelsight_simulation/solids/cad/tactile_dataset_mount.fcstd1 -------------------------------------------------------------------------------- /fdm_printer/controller_plugins.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | -------------------------------------------------------------------------------- /gelsight_description/urdf/gelsight2014.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /experiments/fine_tuning/readme.md: -------------------------------------------------------------------------------- 1 | These experiments were an attempt of further optimizing the loss between real and synthetic samples, however didn't 2 | yield any successful results, probably due to the difficulty in comparing real-sim images, and thus are not reported 3 | inthe paper. -------------------------------------------------------------------------------- /fdm_printer_description/materials/smartlab_logo.material: -------------------------------------------------------------------------------- 1 | material fdm_printer_description/smartlab_logo { 2 | technique 3 | { 4 | pass 5 | { 6 | ambient 1 1 1 1 7 | diffuse 1 1 1 1 8 | specular 0.1 0.1 0.1 1 9 | emissive 0 0 0 0 10 | 11 | 12 | texture_unit 13 | { 14 | texture smartlab_logo.png 15 | scale 1 1 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /gelsight_description/launch/rviz.launch: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 | 10 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /fdm_printer_description/models/fdm_printer_control.yaml: -------------------------------------------------------------------------------- 1 | fdm_printer: 2 | joint_state_controller: 3 | type: joint_state_controller/JointStateController 4 | publish_rate: 50 5 | 6 | x_axis: 7 | pid: {p: 100.0, i: 0.01, d: 10.0} 8 | y_axis: 9 | pid: {p: 100.0, i: 0.01, d: 10.0} 10 | z_axis: 11 | pid: {p: 100.0, i: 0.01, d: 10.0} 12 | 13 | xyz_controller: 14 | type: position_controllers/JointGroupPositionController 15 | joints: 16 | - x_axis 17 | - y_axis 18 | - z_axis 19 | x_axis: 20 | pid: {p: 100.0, i: 0.01, d: 10.0} 21 | y_axis: 22 | pid: {p: 100.0, i: 0.01, d: 10.0} 23 | z_axis: 24 | pid: {p: 100.0, i: 0.01, d: 10.0} -------------------------------------------------------------------------------- /gelsight_gazebo/launch/launch.launch: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | 12 | 16 | 17 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /experiments/sim2real/plot_confusion_matrix.py: -------------------------------------------------------------------------------- 1 | from src.experiments.sim2real.nn import nn 2 | from src.experiments.data_loader import DataGenerator, OBJECT_SET_CLASSES 3 | from src.experiments import ConfusionMatrixPlotter 4 | 5 | import numpy as np 6 | n_classes = 21 7 | 8 | model = nn(n_classes, fc_d=128) 9 | exp_name = 'sim2real' 10 | logs_path = 'logs/' + exp_name + '/' 11 | 12 | validation_sim_generator = DataGenerator( 13 | path='aligned/real/', 14 | splits_file='aligned/index.yaml', 15 | output_paths=False, 16 | split='validation', 17 | batch_size=-1, 18 | ) 19 | 20 | model.compile() 21 | 22 | model.load_weights(logs_path) 23 | 24 | x, y = next(validation_sim_generator) 25 | cm = ConfusionMatrixPlotter(x, y, OBJECT_SET_CLASSES, interactive=False) 26 | cm.model = model 27 | cm.on_epoch_end() 28 | -------------------------------------------------------------------------------- /experiments/preparation/split.py: -------------------------------------------------------------------------------- 1 | from itertools import groupby 2 | from random import sample 3 | from yaml import dump 4 | 5 | from ..data_loader import DataGenerator 6 | 7 | generator = DataGenerator( 8 | 'aligned/real/', 9 | shuffle=False, 10 | batch_size=1, 11 | ) 12 | 13 | splits = { 14 | 'train': [], 15 | 'validation': [], 16 | 'test': [] 17 | } 18 | 19 | for k, g in groupby(generator.files, lambda f: generator.filename_data(f)[0]): 20 | files = list(g) 21 | 22 | train_split = sample(files, 70) 23 | validation_split = sample([x for x in files if x not in train_split], 20) 24 | test_split = sample([x for x in files if x not in train_split + validation_split], 8) 25 | 26 | splits['train'] += train_split 27 | splits['validation'] += validation_split 28 | splits['test'] += test_split 29 | 30 | stream = open('aligned/index.yaml', 'w') 31 | dump(splits, stream) -------------------------------------------------------------------------------- /experiments/sim2real/nn.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | config = tf.compat.v1.ConfigProto() 4 | config.gpu_options.allow_growth = True 5 | session = tf.compat.v1.InteractiveSession(config=config) 6 | 7 | from keras.models import Sequential 8 | from keras.layers import Flatten, Dense, BatchNormalization, ReLU, LeakyReLU, Softmax 9 | from keras.applications.resnet_v2 import ResNet50V2 10 | 11 | 12 | def dense_norm_relu(model, size): 13 | model.add(Dense(size)) 14 | model.add(BatchNormalization()) 15 | model.add(ReLU()) 16 | 17 | 18 | def nn(n_classes, fc_d=512): 19 | model = Sequential() 20 | 21 | model.add(ResNet50V2( 22 | input_shape=(224, 224, 3), 23 | include_top=False, 24 | weights='imagenet')) 25 | 26 | model.add(Flatten()) 27 | dense_norm_relu(model, fc_d) 28 | dense_norm_relu(model, fc_d) 29 | 30 | model.add(Dense(n_classes)) 31 | model.add(Softmax()) 32 | return model 33 | -------------------------------------------------------------------------------- /gelsight_simulation/models/setup.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /gelsight_simulation/scripts/gelsight_driver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import rospy 3 | 4 | import numpy as np 5 | import cv2 6 | from cv_bridge import CvBridge 7 | 8 | from sensor_msgs.msg import Image 9 | 10 | 11 | def main_loop(): 12 | # rate = rospy.Rate(30) 13 | cap = cv2.VideoCapture(0) 14 | 15 | bridge = CvBridge() 16 | 17 | while not rospy.is_shutdown() \ 18 | and cap.isOpened(): 19 | ret, frame = cap.read() 20 | 21 | # cv2.imshow('frame', frame) 22 | 23 | image_message = bridge.cv2_to_imgmsg(frame, encoding="bgr8") 24 | pub.publish(image_message) 25 | cv2.waitKey(1) 26 | # if cv2.waitKey(1) & 0xFF == ord('q'): 27 | # break 28 | # rate.sleep() 29 | cap.release() 30 | cv2.destroyAllWindows() 31 | 32 | 33 | if __name__ == '__main__': 34 | 35 | pub = rospy.Publisher('/gelsight/tactile_image', Image, queue_size=10) 36 | rospy.init_node('gelsight_driver', anonymous=True) 37 | print('DRIVER RUNNING') 38 | main_loop() 39 | -------------------------------------------------------------------------------- /experiments/preparation/generate_sim.py: -------------------------------------------------------------------------------- 1 | from ..data_loader import data_generator, DataGenerator 2 | from ..model.configs import smartlab_gelsight2014_config 3 | from ..model.sim_model import SimulationApproach, SimulationApproachLegacy 4 | 5 | import cv2 6 | import numpy as np 7 | 8 | generator = DataGenerator( 9 | '/aligned/real', 10 | depth_path='aligned/depth', 11 | shuffle=False, 12 | batch_size=1, 13 | resize=False) 14 | 15 | # smartlab_gelsight2014_config['background_img'] = cv2.imread('aligned/background.png') 16 | 17 | for (_, depth, cls, path) in generator: 18 | cls_name = generator.classes[np.argmax(cls)] 19 | 20 | smartlab_gelsight2014_config['background_img'] = cv2.imread('aligned/background_' + cls_name + '.png') 21 | 22 | simulation2014_approach = SimulationApproachLegacy( 23 | **smartlab_gelsight2014_config 24 | ) 25 | 26 | sim = simulation2014_approach.generate(depth[0]) 27 | cv2.imwrite('aligned/' + path[0], sim) 28 | cv2.imshow('sim', sim) 29 | cv2.waitKey(1) 30 | 31 | print(path) 32 | -------------------------------------------------------------------------------- /experiments/preparation/loss_eval.py: -------------------------------------------------------------------------------- 1 | from ..data_loader import data_generator, DataGenerator 2 | from ..fine_tuning.losses import rectified_mae_loss, psnr_loss, ssim_loss 3 | 4 | import numpy as np 5 | 6 | generator = DataGenerator( 7 | 'aligned/real', 8 | sim_path='aligned/legacy', 9 | shuffle=False, 10 | batch_size=1, 11 | resize=False) 12 | 13 | maes = [] 14 | psnrs = [] 15 | ssims = [] 16 | 17 | n = 0 18 | 19 | print('GENERATOR SIZE: ', generator.size()) 20 | for (real, sim, cls, path) in generator: 21 | real = np.array(real) 22 | sim = np.array(sim) 23 | 24 | c_mae = rectified_mae_loss(real, sim) 25 | c_psnr = psnr_loss(real, sim) 26 | c_ssim = ssim_loss(real, sim) 27 | 28 | maes.append(c_mae) 29 | psnrs.append(c_psnr) 30 | ssims.append(c_ssim) 31 | 32 | n += 1 33 | 34 | if len(maes) % 50 == 0: 35 | print(str(int(n/generator.size() * 100)) + '%') 36 | 37 | 38 | 39 | print('mean MAE:', np.mean(maes), np.var(maes)) 40 | print('mean PSNR:', np.mean(psnrs), np.var(psnrs)) 41 | print('mean SSIM:', np.mean(ssims), np.var(ssims)) 42 | -------------------------------------------------------------------------------- /experiments/sim2real/eval.py: -------------------------------------------------------------------------------- 1 | from src.experiments.sim2real.nn import nn 2 | from src.experiments.data_loader import DataGenerator, OBJECT_SET_CLASSES, to_categorical 3 | from src.experiments import ConfusionMatrixPlotter 4 | 5 | import numpy as np 6 | from sklearn.metrics import accuracy_score 7 | 8 | validation_sim_generator = DataGenerator( 9 | path='aligned/real/', 10 | splits_file='/aligned/index.yaml', 11 | output_paths=False, 12 | split='test', 13 | batch_size=-1, 14 | ) 15 | 16 | n_classes = 21 17 | 18 | model = nn(n_classes, fc_d=128) 19 | exp_name = 'dynamic_train_sim2real_300it_128fc_lr0x1' 20 | logs_path = 'logs/' + exp_name + '/' 21 | 22 | model.compile() 23 | 24 | model.load_weights(logs_path) 25 | 26 | x, y = next(validation_sim_generator) 27 | 28 | cm = ConfusionMatrixPlotter(x, y, OBJECT_SET_CLASSES, interactive=False) 29 | cm.model = model 30 | cm.on_epoch_end() 31 | 32 | pred = model.predict(x) 33 | 34 | max_pred = np.argmax(pred, axis=1) 35 | max_y = np.argmax(y, axis=1) 36 | score = accuracy_score(max_y, max_pred) 37 | 38 | print('Accuracy: ', score) 39 | print('N classes: ', len(OBJECT_SET_CLASSES)) 40 | -------------------------------------------------------------------------------- /gelsight_simulation/models/solid.xacro: -------------------------------------------------------------------------------- 1 | > 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | 19 | 20 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | Gazebo/DarkGrey 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /gelsight_description/urdf/gelsight2014_spectre.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | ${color} 0.9 19 | ${color} 0.9 20 | 0.1 0.1 0.1 0 21 | ${color} 0.01 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /experiments/fine_tuning/losses.py: -------------------------------------------------------------------------------- 1 | from skimage.metrics import structural_similarity as ssim 2 | from skimage.metrics import peak_signal_noise_ratio as psnr 3 | 4 | import numpy as np 5 | import cv2 6 | 7 | 8 | def rectified_mae_loss(true, test): 9 | assert true.shape[0] == test.shape[0], \ 10 | "True and Test batches must have the same shape." 11 | assert true.dtype == test.dtype and (true.dtype == np.uint8 or true.dtype == np.float32), \ 12 | "That's a wrong dtype" + str(true.dtype) + " " + str(test.dtype) 13 | 14 | pixel_range = 255 if test.dtype == np.uint8 else 1 15 | area = true.shape[1] * true.shape[2] * true.shape[3] 16 | 17 | losses = [np.sum(cv2.absdiff(true[i] / pixel_range, test[i] / pixel_range)) / area for i in range(true.shape[0])] 18 | 19 | return sum(losses) / len(losses) 20 | 21 | 22 | # -1 to 1, 1 means the images are identical 23 | def ssim_loss(true, test): 24 | assert true.shape[0] == test.shape[0], "True and Test batches must have the same shape." 25 | losses = [ssim(true[i], test[i], multichannel=True) for i in range(true.shape[0])] 26 | return sum(losses) / true.shape[0] 27 | 28 | 29 | def psnr_loss(true, test): 30 | assert true.shape[0] == test.shape[0], "True and Test batches must have the same shape." 31 | losses = [psnr(true, test) for i in range(true.shape[0])] 32 | return sum(losses) / true.shape[0] 33 | 34 | def combined_loss(true, test): 35 | return mse(true, test) + (1 - ssim_loss(true, test)) + psnr_loss(true, test) 36 | 37 | -------------------------------------------------------------------------------- /experiments/model/configs.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | 3 | PKG_PATH = 'gelsight_simulation' 4 | 5 | # {'position': [0, 1, 0.25], 'color': (240, 240, 240)}, 6 | # {'position': [-1, 0, 0.25], 'color': (255, 139, 78)}, 7 | # {'position': [0, -1, 0.25], 'color': (108, 82, 255)}, 8 | # {'position': [1, 0, 0.25], 'color': (100, 240, 150)}, 9 | smartlab_gelsight2014_config = { 10 | 'light_sources': [ 11 | {'position': [0, 1, 0.25], 'color': (255, 255, 255), 'kd': 0.6, 'ks': 0.5}, # white, top 12 | {'position': [-1, 0, 0.25], 'color': (255, 130, 115), 'kd': 0.5, 'ks': 0.3}, # blue, right 13 | {'position': [0, -1, 0.25], 'color': (108, 82, 255), 'kd': 0.6, 'ks': 0.4}, # red, bottom 14 | {'position': [1, 0, 0.25], 'color': (120, 255, 153), 'kd': 0.1, 'ks': 0.1} # green, left 15 | 16 | # {'position': [-1, 0, 0.25], 'color': (255, 255, 255), 'kd': 0.5, 'ks': 0.3}, # white, top 17 | # {'position': [0, 0, 1], 'color': (255, 255, 255), 'kd': 0.5, 'ks': 0.3}, # blue, right 18 | 19 | ], 20 | 'background_img': cv2.imread(PKG_PATH + '/experiments/fine_tuning/background.png'), 21 | 'ka': 0.8, 22 | 'px2m_ratio': 5.4347826087e-05, 23 | 'elastomer_thickness': 0.004, 24 | 'min_depth': 0.026, 25 | 'texture_sigma': 0.000002 26 | } 27 | 28 | mit_gelsight2017_config = { 29 | 'light_sources': [ 30 | {'position': [-1, 0, 0.25], 'color': (108, 82, 255), 'kd': 0.6, 'ks': 0.4}, # red, bottom 31 | {'position': [0.50, -0.866, 0.25], 'color': (120, 255, 153), 'kd': 0.1, 'ks': 0.4}, # green, left 32 | {'position': [0.50, 0.866, 0.25], 'color': (255, 130, 115), 'kd': 0.1, 'ks': 0.4}, # blue, left 33 | ], 34 | 'background_img': cv2.imread(PKG_PATH + '/experiments/fine_tuning/background_gelsight2017.jpg'), 35 | 'ka': 0.8, 36 | 'px2m_ratio': 5.4347826087e-05, 37 | 'elastomer_thickness': 0.004, 38 | 'min_depth': 0.026, 39 | 'texture_sigma': 0.000002 40 | } 41 | -------------------------------------------------------------------------------- /fdm_printer/src/position_controller.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | namespace fdm_printer_ns { 7 | 8 | class PositionController : public controller_interface::Controller { 9 | 10 | bool init(hardware_interface::EffortJointInterface *hw, ros::NodeHandle &n){ 11 | 12 | std::string my_joint; 13 | if(!n.getParam("joint", my_joint)) 14 | { 15 | ROS_ERROR(("Could not find joint name")); 16 | return false; 17 | } 18 | 19 | joint_ = hw->getHandle(my_joint); // throws on failure 20 | command_ = joint_.getPosition(); 21 | 22 | // Load gain using gains set on parameter server 23 | if(!n.getParam("gain", gain_)) 24 | { 25 | ROS_ERROR("Could not find the gain parameter value"); 26 | return false; 27 | } 28 | 29 | // Start command subscriber 30 | sub_command_ = n.subscribe("command", 1, &PositionController::setCommandCB, this); 31 | } 32 | 33 | void setCommandCB(const std_msgs::Float64ConstPtr & msg){ 34 | command_ = msg->data; 35 | } 36 | 37 | void update(const ros::Time & time, const ros::Duration & period){ 38 | double error = command_ - joint_.getPosition(); 39 | double commanded_effort = error*gain_; 40 | joint_.setCommand(commanded_effort); 41 | } 42 | void starting(const ros::Time & time){ 43 | } 44 | 45 | void stopping(const ros::Time & time){ 46 | } 47 | 48 | private: 49 | hardware_interface::JointHandle joint_; 50 | double gain_; 51 | double command_; 52 | ros::Subscriber sub_command_; 53 | 54 | }; 55 | 56 | PLUGINLIB_EXPORT_CLASS(fdm_printer_ns::PositionController, controller_interface::ControllerBase); 57 | } 58 | -------------------------------------------------------------------------------- /fdm_printer_bringup/launch/bringup.launch: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 14 | 15 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 32 | 33 | 34 | 35 | 40 | 41 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /experiments/fine_tuning/grid_search.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from time import time 4 | import scipy.stats as stats 5 | from sklearn.utils.fixes import loguniform 6 | 7 | from sklearn.model_selection import GridSearchCV, RandomizedSearchCV 8 | from sklearn.datasets import load_digits 9 | from sklearn.linear_model import SGDClassifier 10 | 11 | # get some data 12 | X, y = load_digits(return_X_y=True) 13 | 14 | # build a classifier 15 | clf = SGDClassifier(loss='hinge', penalty='elasticnet', 16 | fit_intercept=True) 17 | 18 | 19 | # Utility function to report best scores 20 | def report(results, n_top=3): 21 | for i in range(1, n_top + 1): 22 | candidates = np.flatnonzero(results['rank_test_score'] == i) 23 | for candidate in candidates: 24 | print("Model with rank: {0}".format(i)) 25 | print("Mean validation score: {0:.3f} (std: {1:.3f})" 26 | .format(results['mean_test_score'][candidate], 27 | results['std_test_score'][candidate])) 28 | print("Parameters: {0}".format(results['params'][candidate])) 29 | print("") 30 | 31 | 32 | # specify parameters and distributions to sample from 33 | param_dist = { 34 | 'average': [True, False], 35 | 'l1_ratio': stats.uniform(0, 1), 36 | 'alpha': loguniform(1e-4, 1e0) 37 | } 38 | 39 | # run randomized search 40 | n_iter_search = 20 41 | random_search = RandomizedSearchCV(clf, param_distributions=param_dist, 42 | n_iter=n_iter_search) 43 | 44 | start = time() 45 | random_search.fit(X, y) 46 | print("RandomizedSearchCV took %.2f seconds for %d candidates" 47 | " parameter settings." % ((time() - start), n_iter_search)) 48 | report(random_search.cv_results_) 49 | 50 | # use a full grid over all parameters 51 | param_grid = {'average': [True, False], 52 | 'l1_ratio': np.linspace(0, 1, num=10), 53 | 'alpha': np.power(10, np.arange(-4, 1, dtype=float))} 54 | 55 | # run grid search 56 | grid_search = GridSearchCV(clf, param_grid=param_grid) 57 | start = time() 58 | grid_search.fit(X, y) 59 | 60 | print("GridSearchCV took %.2f seconds for %d candidate parameter settings." 61 | % (time() - start, len(grid_search.cv_results_['params']))) 62 | report(grid_search.cv_results_) 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GelSight Simulation 2 | 3 | This repository contains the necessary code for executing the Simulated Environment and experiments as in our [GelSight Simulation paper](https://arxiv.org/abs/2101.07169). These packages contain the drivers for running a GelSight sensor (in real world and simulation) and a FDM-Printer for carrying the described experiments (also in real world and simulation). The experiments includes the final experiments for the dataset alignment and the *Sim2Real* classification task, and should be executed using Python 3 (outside ROS). Visit [danfergo.github.io/gelsight-simulation](https://danfergo.github.io/gelsight-simulation/) for more information about the work and links for downloading the datasets, 4 | 5 | ### Index of Packages 6 | 7 | | Package | Description | 8 | | ------------- | ------------------| 9 | | experiments | Python3 (outside ROS) scripts with the experiments described in the paper. 10 | | fdm_printer | Contains two drivers for running the FDM printer in the real world: a standard ROS subscriber/publisher and a ROS Control Harware Interface. The drivers work by issuing g-code commands to the printer. | 11 | | fdm_printer_bringup | Includes the file for launching the printer either in simulation or the real world (sim:=true for simulation) | 12 | | fdm_printer_description | The URDF files and STL meshes describing the printer. | 13 | | gelsight_description | The URDF files and STL meshes describing the GelSight sensor. The modeling is based on sensor proposed [here](https://ieeexplore.ieee.org/document/6943123). | 14 | | **gelsight_gazebo** | This package contains the [driver](gelsight_gazebo/src/gelsight_driver.py) that implements the proposed approach, to be used in simulation. | 15 | | gelsight_simulation | Scripts and Materials used to carry the data collection process. 16 | 17 | To run the collection of tactile images using the simulated setup. 18 | ``` 19 | roscore 20 | roslaunch gelsight_simulation dc.launch sim:=true 21 | rosrun gelsight_simulation data_collection.py 22 | ``` 23 | 24 | To run the experiments scripts, e.g., 25 | ``` 26 | python -m experiments.sim2real.train_nn 27 | ``` 28 | 29 | 30 | ---- 31 | A big thanks to [keras-visuals](https://github.com/chasingbob/keras-visuals), for providing some helpful [Keras](https://keras.io/) callbacks for assessing our NN optimization. -------------------------------------------------------------------------------- /gelsight_description/urdf/gelsight2014_case.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 34 | 35 | 36 | 37 | 38 | 39 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | Gazebo/DarkGrey 55 | 56 | 57 | 58 | Gazebo/DarkGrey 59 | 60 | 61 | -------------------------------------------------------------------------------- /gelsight_description/urdf/gelsight_camera.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 1 8 | 30.0 9 | 1 10 | /depth_camera 11 | 12 | 13 | 0.5 14 | 15 | 640 16 | 480 17 | R8G8B8 18 | 19 | 20 | 0.0001 21 | 300 22 | 23 | 24 | depths 25 | 26 | 27 | 28 | 0.2 29 | true 30 | 30.0 31 | gelsight_ir 32 | /gelsight/image/image_raw 33 | /gelsight/depth/camera_info 34 | /gelsight/depth/image_raw 35 | /gelsight/depth/camera_info 36 | /gelsight/depth/points 37 | gelsight_link 38 | 0.0 39 | 0 40 | 0 41 | 0 42 | 0 43 | 0 44 | 0 45 | 0 46 | 0 47 | 0 48 | 0 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /gelsight_gazebo/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | gelsight_gazebo 4 | 0.0.0 5 | The gelsight_gazebo package 6 | 7 | 8 | 9 | 10 | danfergo 11 | 12 | 13 | 14 | 15 | 16 | TODO 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | catkin 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /fdm_printer_bringup/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | fdm_printer_bringup 4 | 0.0.0 5 | The fdm_printer_bringup package 6 | 7 | 8 | 9 | 10 | danfergo 11 | 12 | 13 | 14 | 15 | 16 | TODO 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | catkin 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /gelsight_simulation/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | gelsight_simulation 4 | 0.0.0 5 | The gelsight_simulation package 6 | 7 | 8 | 9 | 10 | danfergo 11 | 12 | 13 | 14 | 15 | 16 | TODO 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | catkin 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /gelsight_description/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | gelsight_description 4 | 0.0.0 5 | The gelsight_description package 6 | 7 | 8 | 9 | 10 | danfergo 11 | 12 | 13 | 14 | 15 | 16 | TODO 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | catkin 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /fdm_printer_description/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | fdm_printer_description 4 | 0.0.0 5 | The fdm_printer_description package 6 | 7 | 8 | 9 | 10 | danfergo 11 | 12 | 13 | 14 | 15 | 16 | TODO 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | catkin 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /experiments/sim2real/util/confusion_matrix_cb.py: -------------------------------------------------------------------------------- 1 | from keras.callbacks import Callback 2 | import matplotlib.pyplot as plt 3 | from sklearn.metrics import confusion_matrix 4 | import itertools 5 | import numpy as np 6 | 7 | 8 | class ConfusionMatrixPlotter(Callback): 9 | """Plot the confusion matrix on a graph and update after each epoch 10 | # Arguments 11 | X_val: The input values 12 | Y_val: The expected output values 13 | classes: The categories as a list of string names 14 | normalize: True - normalize to [0,1], False - keep as is 15 | cmap: Specify matplotlib colour map 16 | title: Graph Title 17 | """ 18 | 19 | def __init__(self, X_val, Y_val, classes, interactive=True, normalize=False, to_file=None, cmap=plt.cm.Blues, 20 | title='Confusion Matrix', remap_classes=None): 21 | self.X_val = X_val 22 | self.Y_val = Y_val 23 | self.title = title 24 | self.classes = classes 25 | self.normalize = normalize 26 | self.cmap = cmap 27 | self.interactive = interactive 28 | self.to_file = to_file 29 | self.remap_classes = remap_classes 30 | 31 | if interactive: 32 | plt.ion() 33 | 34 | plt.figure(figsize=[6.4 * 2, 4.8 * 2]) 35 | 36 | if to_file is None: 37 | plt.title(self.title) 38 | 39 | def on_train_begin(self, logs={}): 40 | pass 41 | 42 | def on_epoch_end(self, epoch=None, logs={}): 43 | plt.clf() 44 | 45 | pred = self.model.predict(self.X_val) 46 | 47 | if self.remap_classes is not None: 48 | pred = self.remap_classes(pred) 49 | 50 | max_pred = np.argmax(pred, axis=1) 51 | max_y = np.argmax(self.Y_val, axis=1) 52 | cnf_mat = confusion_matrix(max_y, max_pred) 53 | 54 | if self.normalize: 55 | cnf_mat = cnf_mat.astype('float') / cnf_mat.sum(axis=1)[:, np.newaxis] 56 | 57 | thresh = cnf_mat.max() / 2. 58 | for i, j in itertools.product(range(cnf_mat.shape[0]), range(cnf_mat.shape[1])): 59 | plt.text(j, i, cnf_mat[i, j], 60 | horizontalalignment="center", 61 | color="white" if cnf_mat[i, j] > thresh else "black") 62 | 63 | plt.imshow(cnf_mat, interpolation='nearest', cmap=self.cmap) 64 | 65 | # Labels 66 | tick_marks = np.arange(len(self.classes)) 67 | plt.xticks(tick_marks, self.classes, rotation=45) 68 | plt.yticks(tick_marks, self.classes) 69 | 70 | plt.colorbar() 71 | 72 | # plt.tight_layout() 73 | plt.ylabel('True label') 74 | plt.xlabel('Predicted label') 75 | # plt.draw() 76 | 77 | if self.to_file is not None: 78 | plt.savefig(self.to_file, dpi=150) 79 | else: 80 | plt.show() 81 | 82 | if self.interactive: 83 | plt.pause(0.001) 84 | -------------------------------------------------------------------------------- /experiments/sim2real/plot_all.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import matplotlib.pyplot as plt 3 | import argparse 4 | 5 | def parse(epoch, accuracy, loss, val_accuracy, val_loss): 6 | print(epoch) 7 | epoch = [int(i) for i in epoch] 8 | accuracy = [float(i) for i in accuracy] 9 | loss = [float(i) for i in loss] 10 | val_accuracy = [float(i) for i in val_accuracy] 11 | val_loss = [float(i) for i in val_loss] 12 | return epoch, accuracy, loss, val_accuracy, val_loss 13 | 14 | 15 | def read_log(filename): 16 | return parse(*list(map(list, zip(*list(csv.reader(open(filename, 'r')))[1:])))) 17 | 18 | 19 | from scipy.ndimage.filters import gaussian_filter1d 20 | 21 | max_epochs = 300 22 | 23 | fig, (ax1, ax2_) = plt.subplots(1, 2) 24 | ax2 = ax1.twinx() 25 | ax2.tick_params(labelright='off') 26 | 27 | epoch, accuracy, loss, val_accuracy, val_loss = read_log('logs/sim2sim_300it_128fc_lr0x1/log.log') 28 | epoch = epoch[0:max_epochs] 29 | accuracy, loss, val_accuracy, val_loss = gaussian_filter1d(accuracy, sigma=4)[0:max_epochs], \ 30 | gaussian_filter1d(loss, sigma=4)[0:max_epochs], \ 31 | gaussian_filter1d(val_accuracy, sigma=4)[0:max_epochs], \ 32 | gaussian_filter1d(val_loss, sigma=4)[0:max_epochs] 33 | ax1.plot(epoch, val_accuracy, label='Sim2Sim') 34 | ax2.plot(epoch, val_loss, alpha=0.3) 35 | 36 | # ================== 37 | epoch, accuracy, loss, val_accuracy, val_loss = read_log('logs/sim2real_300it_128fc_lr0x1/log.log') 38 | epoch = epoch[0:max_epochs] 39 | accuracy, loss, val_accuracy, val_loss = gaussian_filter1d(accuracy, sigma=4)[0:max_epochs], \ 40 | gaussian_filter1d(loss, sigma=4)[0:max_epochs], \ 41 | gaussian_filter1d(val_accuracy, sigma=4)[0:max_epochs], \ 42 | gaussian_filter1d(val_loss, sigma=4)[0:max_epochs] 43 | ax1.plot(epoch, val_accuracy, label='Sim2Real') 44 | ax2.plot(epoch, val_loss, alpha=0.3) 45 | # ================== 46 | 47 | epoch, accuracy, loss, val_accuracy, val_loss = read_log('logs/dynamic_train_sim2real_300it_128fc_lr0x1/log.log') 48 | epoch = epoch[0:max_epochs] 49 | accuracy, loss, val_accuracy, val_loss = gaussian_filter1d(accuracy, sigma=4)[0:max_epochs], \ 50 | gaussian_filter1d(loss, sigma=4)[0:max_epochs], \ 51 | gaussian_filter1d(val_accuracy, sigma=4)[0:max_epochs], \ 52 | gaussian_filter1d(val_loss, sigma=4)[0:max_epochs] 53 | ax1.plot(epoch, val_accuracy, label='Sim2Real (texture augmented)') 54 | ax2.plot(epoch, val_loss, alpha=0.3) 55 | 56 | 57 | # ================== 58 | 59 | ax1.set_title('Sim2Real learning') 60 | ax1.set_ylabel('Accuracy') 61 | ax2.set_ylabel('Loss') 62 | ax1.set_xlabel('Epochs') 63 | 64 | ax1.legend(loc='upper left') 65 | 66 | plt.show() 67 | -------------------------------------------------------------------------------- /experiments/sim2real/util/acc_loss_cb.py: -------------------------------------------------------------------------------- 1 | from keras.callbacks import Callback 2 | import matplotlib.pyplot as plt 3 | import matplotlib.patches as mpatches 4 | from sklearn.metrics import confusion_matrix 5 | import itertools 6 | import numpy as np 7 | 8 | 9 | class AccLossPlotter(Callback): 10 | """Plot training Accuracy and Loss values on a Matplotlib graph. 11 | 12 | The graph is updated by the 'on_epoch_end' event of the Keras Callback class 13 | 14 | # Arguments 15 | graphs: list with some or all of ('acc', 'loss') 16 | save_graph: Save graph as an image on Keras Callback 'on_train_end' event 17 | 18 | """ 19 | 20 | def __init__(self, graphs=['accuracy', 'loss'], to_file=None, save_graph=False): 21 | self.graphs = graphs 22 | self.num_subplots = len(graphs) 23 | self.save_graph = save_graph 24 | self.to_file = to_file 25 | 26 | def on_train_begin(self, logs={}): 27 | self.acc = [] 28 | self.val_acc = [] 29 | self.loss = [] 30 | self.val_loss = [] 31 | self.epoch_count = 0 32 | 33 | if self.to_file is None: 34 | plt.ion() 35 | plt.show() 36 | 37 | def on_epoch_end(self, epoch, logs={}): 38 | self.epoch_count += 1 39 | self.val_acc.append(logs.get('val_accuracy')) 40 | self.acc.append(logs.get('accuracy')) 41 | self.loss.append(logs.get('loss')) 42 | self.val_loss.append(logs.get('val_loss')) 43 | epochs = [x for x in range(self.epoch_count)] 44 | 45 | count_subplots = 0 46 | 47 | if 'accuracy' in self.graphs: 48 | count_subplots += 1 49 | plt.subplot(self.num_subplots, 1, count_subplots) 50 | plt.title('Accuracy') 51 | # plt.axis([0,100,0,1]) 52 | plt.plot(epochs, self.val_acc, color='r') 53 | plt.plot(epochs, self.acc, color='b') 54 | plt.ylabel('accuracy') 55 | 56 | red_patch = mpatches.Patch(color='red', label='Test') 57 | blue_patch = mpatches.Patch(color='blue', label='Train') 58 | 59 | plt.legend(handles=[red_patch, blue_patch], loc=4) 60 | 61 | if 'loss' in self.graphs: 62 | count_subplots += 1 63 | plt.subplot(self.num_subplots, 1, count_subplots) 64 | plt.title('Loss') 65 | # plt.axis([0,100,0,5]) 66 | plt.plot(epochs, self.val_loss, color='r') 67 | plt.plot(epochs, self.loss, color='b') 68 | plt.ylabel('loss') 69 | 70 | red_patch = mpatches.Patch(color='red', label='Test') 71 | blue_patch = mpatches.Patch(color='blue', label='Train') 72 | 73 | plt.legend(handles=[red_patch, blue_patch], loc=4) 74 | 75 | if self.to_file is not None: 76 | plt.savefig(self.to_file, dpi=150) 77 | else: 78 | plt.show() 79 | 80 | if self.interactive: 81 | plt.pause(0.001) 82 | 83 | # def on_train_end(self, logs={}): 84 | # if self.save_graph: 85 | # plt.savefig('training_acc_loss.png') 86 | # 87 | -------------------------------------------------------------------------------- /experiments/fine_tuning/random_search.py: -------------------------------------------------------------------------------- 1 | from random import random 2 | import matplotlib.pyplot as plt 3 | 4 | from src.experiments.data_loader import DataGenerator, load_single_img2 5 | from src.experiments.fine_tuning.losses import ssim_loss 6 | from src.experiments.model import smartlab_gelsight2014_config 7 | 8 | import numpy as np 9 | import cv2 10 | 11 | from src.experiments.model import SimulationApproach 12 | 13 | 14 | def sample(): 15 | return { 16 | 'sigma': int(1 + 10 * random()), 17 | 'kernel_size': int(5 + 25 * random()), 18 | 't': int(1 + 7) 19 | } 20 | 21 | 22 | generator = DataGenerator( 23 | 'aligned/real', 24 | depth_path='aligned/depth', 25 | shuffle=True, 26 | batch_size=32) 27 | 28 | 29 | def plot_loss(res_tmp): 30 | plt.clf() 31 | 32 | plt.plot(res_tmp) 33 | plt.savefig('progress_deformation_random.png') 34 | 35 | bkgs = {} 36 | for cls in generator.classes: 37 | bkgs[cls] = load_single_img2('aligned/background_' + cls + '.png') 38 | 39 | res_tmp = [] 40 | 41 | 42 | def sample_and_generate( 43 | # l1_r, l1_g, l1_b, 44 | # l2_r, l2_g, l2_b, 45 | # l3_r, l3_g, l3_b, 46 | # l4_r, l4_g, l4_b, 47 | # texture_sigma, ka 48 | t, sigma, kernel_size 49 | ): 50 | real, depth, cls, _ = next(generator) 51 | 52 | def config_and_generate(depth_i, cls_i): 53 | cls_i_name = generator.classes[np.argmax(cls_i)] 54 | smartlab_gelsight2014_config['background_img'] = bkgs[cls_i_name] 55 | simulation2014_approach = SimulationApproach( 56 | **{ 57 | **smartlab_gelsight2014_config, 58 | **{ 59 | # 'sigma': int(sigma), 60 | # 't': int(t), 61 | # 'kernel_size': int(kernel_size), 62 | # 'light_sources': [ 63 | # {'position': [0, 1, 0.25], 'color': (l1_r, l1_g, l1_b)}, 64 | # {'position': [-1, 0, 0.25], 'color': (l2_r, l2_g, l2_b)}, 65 | # {'position': [0, -1, 0.25], 'color': (l3_r, l3_g, l3_b)}, 66 | # {'position': [1, 0, 0.25], 'color': (l4_r, l4_g, l4_b)}, 67 | # ], 68 | # 'ka': ka, 69 | # 'texture_sigma': texture_sigma 70 | } 71 | } 72 | ) 73 | return simulation2014_approach.generate(depth_i) 74 | 75 | sim = np.array([config_and_generate(depth[i], cls[i]) for i in range(len(depth))]) 76 | loss = ssim_loss(sim, real) 77 | 78 | res_tmp.append(loss) 79 | plot_loss(res_tmp) 80 | 81 | cv2.imshow('preview', np.concatenate([sim[0], sim[1], sim[2], sim[3], sim[4]], axis=1)) 82 | cv2.waitKey(1) 83 | 84 | return loss 85 | 86 | 87 | n_epochs = 1000 88 | 89 | params = sample() 90 | e = 0 91 | print("|\t".join([''] + [k for k in params] + ['LOSS'])) 92 | for i in range(n_epochs): 93 | params = sample() 94 | loss = sample_and_generate(**params) 95 | e += 0 96 | print("|\t".join([str(i)] + [str(round(params[k], 2)) for k in params] + [str(round(loss, 3))])) 97 | 98 | -------------------------------------------------------------------------------- /fdm_printer_description/models/fdm_printer.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 19 | 20 | 21 | 22 | 26 | 28 | 29 | 30 | 34 | 36 | 37 | 38 | 39 | 43 | 44 | 45 | 46 | 50 | 51 | 52 | 53 | 54 | 58 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 73 | 74 | 81 | 82 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /fdm_printer/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | fdm_printer 4 | 0.0.0 5 | The fdm_printer package 6 | 7 | 8 | 9 | 10 | danfergo 11 | 12 | 13 | 14 | 15 | 16 | TODO 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | catkin 52 | controller_interface 53 | hardware_interface 54 | pluginlib 55 | roscpp 56 | controller_manager 57 | controller_interface 58 | hardware_interface 59 | pluginlib 60 | roscpp 61 | controller_manager 62 | controller_interface 63 | hardware_interface 64 | hardware_interface 65 | pluginlib 66 | roscpp 67 | controller_manager 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /experiments/fine_tuning/bayesian_optimization.py: -------------------------------------------------------------------------------- 1 | from bayes_opt import BayesianOptimization 2 | import numpy as np 3 | 4 | import cv2 5 | import matplotlib.pyplot as plt 6 | 7 | from .losses import ssim_loss 8 | from ..data_loader import data_generator, load_single_img2, DataGenerator 9 | from src.experiments.model import SimulationApproach 10 | from ..model.configs import mit_gelsight2017_config, smartlab_gelsight2014_config 11 | 12 | pbounds = { 13 | 'kernel_size': (3, 51), 14 | 't': (1, 10), 15 | 'sigma': (1, 10) 16 | # 'l1_r': (0, 255), 17 | # 'l1_g': (0, 255), 18 | # 'l1_b': (0, 255), 19 | # 'l2_r': (0, 255), 20 | # 'l2_g': (0, 255), 21 | # 'l2_b': (0, 255), 22 | # 'l3_r': (0, 255), 23 | # 'l3_g': (0, 255), 24 | # 'l3_b': (0, 255), 25 | # 'l4_r': (0, 255), 26 | # 'l4_g': (0, 255), 27 | # 'l4_b': (0, 255), 28 | # 'texture_sigma': (0.0000001, 0.00001), 29 | # 'ka': (0, 1) 30 | } 31 | 32 | generator = DataGenerator( 33 | 'aligned/real', 34 | depth_path='aligned/depth', 35 | shuffle=True, 36 | batch_size=32) 37 | 38 | def plot_loss(res_tmp): 39 | plt.clf() 40 | 41 | plt.plot(res_tmp) 42 | plt.savefig('progress.png') 43 | 44 | 45 | bkgs = {} 46 | for cls in generator.classes: 47 | bkgs[cls] = load_single_img2( 48 | 'aligned/background_' + cls + '.png') 49 | 50 | res_tmp = [] 51 | 52 | 53 | def sample_and_generate( 54 | # l1_r, l1_g, l1_b, 55 | # l2_r, l2_g, l2_b, 56 | # l3_r, l3_g, l3_b, 57 | # l4_r, l4_g, l4_b, 58 | # texture_sigma, ka 59 | t, sigma, kernel_size 60 | ): 61 | real, depth, cls, _ = next(generator) 62 | 63 | def config_and_generate(depth_i, cls_i): 64 | cls_i_name = generator.classes[np.argmax(cls_i)] 65 | smartlab_gelsight2014_config['background_img'] = bkgs[cls_i_name] 66 | simulation2014_approach = SimulationApproach( 67 | **{ 68 | **smartlab_gelsight2014_config, 69 | **{ 70 | 'sigma': int(sigma), 71 | 't': int(t), 72 | 'kernel_size': int(kernel_size), 73 | # 'light_sources': [ 74 | # {'position': [0, 1, 0.25], 'color': (l1_r, l1_g, l1_b)}, 75 | # {'position': [-1, 0, 0.25], 'color': (l2_r, l2_g, l2_b)}, 76 | # {'position': [0, -1, 0.25], 'color': (l3_r, l3_g, l3_b)}, 77 | # {'position': [1, 0, 0.25], 'color': (l4_r, l4_g, l4_b)}, 78 | # ], 79 | # 'ka': ka, 80 | # 'texture_sigma': texture_sigma 81 | } 82 | } 83 | ) 84 | return simulation2014_approach.generate(depth_i) 85 | 86 | sim = np.array([config_and_generate(depth[i], cls[i]) for i in range(len(depth))]) 87 | loss = ssim_loss(sim, real) 88 | 89 | res_tmp.append(loss) 90 | plot_loss(res_tmp) 91 | 92 | cv2.imshow('preview', np.concatenate([sim[0], sim[1], sim[2], sim[3], sim[4]], axis=1)) 93 | cv2.waitKey(1) 94 | 95 | return loss 96 | 97 | 98 | optimizer = BayesianOptimization( 99 | f=sample_and_generate, 100 | pbounds=pbounds, 101 | verbose=2, 102 | random_state=25 103 | ) 104 | 105 | optimizer.maximize( 106 | init_points=5, 107 | n_iter=1000 108 | ) 109 | 110 | import matplotlib.pyplot as plt 111 | 112 | plt.plot([res['target'] for i, res in enumerate(optimizer.res)]) 113 | plt.ylabel('some numbers') 114 | plt.show() 115 | -------------------------------------------------------------------------------- /gelsight_description/urdf/gelsight2014_macro.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 0 0 0 1 39 | 0 0 0 1 40 | 1 1 1 0.5 41 | 0 0 0 0 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | Gazebo/White 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /fdm_printer/src/fdm_printer_driver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import rospy 4 | from geometry_msgs.msg import Point 5 | import serial 6 | import threading 7 | 8 | 9 | def gcode_parse_pos(res): 10 | arr = res.split(' ') 11 | pos_d = {a.split(':')[0]: float(a.split(':')[1]) for a in arr} 12 | return pos_d['X'], pos_d['Y'], pos_d['Z'] 13 | 14 | 15 | class FDMPrinterDriver: 16 | 17 | def __init__(self): 18 | rospy.init_node('fdm_printer', anonymous=True) 19 | rospy.Subscriber("move", Point, self.move_to) 20 | 21 | self.ser_con = serial.Serial("/dev/ttyUSB0", 115200, timeout=1) 22 | 23 | # time.sleep(5) 24 | 25 | self.MAX_X = 320 26 | self.MAX_Y = 320 27 | self.MAX_Z = 420 28 | self.initialized = False 29 | self.position = None 30 | 31 | self.continuously_empty = 0 32 | self.wait_to_be_ready = 5 33 | 34 | self.access_serial = threading.Lock() 35 | self.move_printer = threading.Lock() 36 | 37 | self.wait_until_ready() 38 | 39 | # initialize 40 | self.initialized = True 41 | self.on_init() 42 | 43 | # watch position 44 | self.watch_position() 45 | 46 | def on_init(self): 47 | print('GO HOME.') 48 | self.go_home() 49 | 50 | def wait_until_ready(self): 51 | r = rospy.Rate(10) 52 | while not rospy.is_shutdown(): 53 | ln = self.ser_con.readline().rstrip() 54 | self.continuously_empty = self.continuously_empty + 1 if ln == '' else 0 55 | 56 | if not self.initialized and self.continuously_empty == self.wait_to_be_ready: 57 | break 58 | r.sleep() 59 | 60 | def watch_position(self): 61 | r = rospy.Rate(10) 62 | while not rospy.is_shutdown(): 63 | # self.position = gcode_parse_pos(self.send_cmd('M114')) 64 | # print('self pos', self.position) 65 | r.sleep() 66 | 67 | def move_center(self): 68 | pt = Point() 69 | pt.x = 0 70 | pt.y = 0 71 | pt.z = 0 72 | self.move_to(pt) 73 | 74 | # center_pt = Point(self.MAX_X / 2, 0, 160) 75 | # self.move_to(center_pt) 76 | 77 | def send_cmd(self, cmd): 78 | self.access_serial.acquire() 79 | 80 | self.ser_con.write(str.encode(cmd + ' \n')) 81 | self.ser_con.flush() 82 | rospy.loginfo("G-code sent: " + cmd) 83 | 84 | res = None 85 | r = rospy.Rate(10) 86 | while not rospy.is_shutdown(): 87 | ln = self.ser_con.readline().rstrip() 88 | if ln == 'ok': 89 | self.access_serial.release() 90 | return res 91 | res = ln 92 | r.sleep() 93 | 94 | def move_to(self, position): 95 | if not self.initialized: 96 | return 97 | 98 | # self.move_printer.acquire() 99 | 100 | x, y, z = position.x, position.y, position.z 101 | x = max(0, min(x, self.MAX_X)) 102 | y = max(0, min(y, self.MAX_Y)) 103 | z = max(0, min(z, self.MAX_Z)) 104 | print(x, y, z) 105 | self.send_cmd("G0 X%.2f Y%.2f Z%.2f" % (x, y, z)) 106 | 107 | r = rospy.Rate(10) 108 | while not rospy.is_shutdown(): 109 | print(self.position, (x, y, z)) 110 | if self.position == (x, y, z): 111 | print('TRUE!!!') 112 | # self.move_printer.release() 113 | break 114 | self.position = gcode_parse_pos(self.send_cmd('M114')) 115 | print('->>>>---', self.position) 116 | r.sleep() 117 | 118 | def go_home(self): 119 | self.send_cmd("G28") 120 | 121 | 122 | if __name__ == '__main__': 123 | driver = FDMPrinterDriver() 124 | -------------------------------------------------------------------------------- /experiments/sim2real/train_nn.py: -------------------------------------------------------------------------------- 1 | from keras.optimizers import Adadelta 2 | import imgaug.augmenters as iaa 3 | 4 | from tensorflow.python.keras.callbacks import CSVLogger, ModelCheckpoint 5 | 6 | from src.experiments.data_loader import DataGenerator 7 | from src.experiments.sim2real.nn import nn 8 | from src.experiments import ConfusionMatrixPlotter 9 | from src.experiments.sim2real.util.acc_loss_cb import AccLossPlotter 10 | 11 | import os 12 | import cv2 13 | import numpy as np 14 | 15 | from src.experiments.model import smartlab_gelsight2014_config 16 | from src.experiments.model import SimulationApproach 17 | from random import randint 18 | 19 | smartlab_gelsight2014_config['background_img'] = cv2.resize(cv2.imread('aligned/background.png'), dsize=(224, 224)) 20 | 21 | rnd_bkgs = [] 22 | n_rnd_bkgs = 12 23 | for i in range(n_rnd_bkgs): 24 | rnd_bkgs.append( 25 | cv2.resize(cv2.imread('textures/' + str(i + 1) + '.png', cv2.IMREAD_GRAYSCALE), dsize=(224, 224)) / 225.0) 26 | 27 | simulation2014_approach = SimulationApproach(**smartlab_gelsight2014_config) 28 | 29 | 30 | def texture_augmented_augmentor(x, y): 31 | def config_and_generate(depth_i, cls_i): 32 | rand_idx = randint(0, n_rnd_bkgs - 1) 33 | depth_seq = iaa.Sequential([ 34 | iaa.Affine(scale=(0.5, 1.5), rotate=(-45, 45)) 35 | ]) 36 | depth_i += depth_seq(images=np.array([rnd_bkgs[rand_idx]]))[0] * 0.1 37 | return simulation2014_approach.generate(depth_i) 38 | 39 | sim = np.array([config_and_generate(x[i], y[i]) for i in range(len(x))]) 40 | 41 | seq = iaa.Sequential([ 42 | iaa.RandAugment(n=2, m=9) 43 | ]) 44 | 45 | return seq(images=sim), y 46 | 47 | 48 | def augmentor(x, y): 49 | seq = iaa.Sequential([ 50 | iaa.RandAugment(n=2, m=9) 51 | ]) 52 | 53 | return seq(images=x), y 54 | 55 | 56 | train_real_generator = DataGenerator( 57 | path='aligned/real/', 58 | # path='aligned/depth/', 59 | splits_file='aligned/index.yaml', 60 | output_paths=False, 61 | split='train', 62 | augmentor=augmentor, 63 | # augmentor=texture_augmented_augmentor 64 | ) 65 | 66 | train_sim_generator = DataGenerator( 67 | path='aligned/sim/', 68 | splits_file='aligned/index.yaml', 69 | output_paths=False, 70 | split='train', 71 | augmentor=augmentor 72 | ) 73 | 74 | validation_real_generator = DataGenerator( 75 | path='aligned/real/', 76 | splits_file='aligned/index.yaml', 77 | output_paths=False, 78 | split='validation', 79 | batch_size=1 80 | ) 81 | 82 | validation_sim_generator = DataGenerator( 83 | path='aligned/sim/', 84 | splits_file='aligned/index.yaml', 85 | output_paths=False, 86 | split='validation', 87 | batch_size=1 88 | ) 89 | 90 | epochs = 300 91 | steps_per_epoch = 2 92 | n_classes = 21 93 | fc_d = 128 94 | optimizer = Adadelta(0.1) 95 | exp_name = 'sim2real_300it_128fc_lr0x1' 96 | train_generator = train_sim_generator 97 | validation_generator = validation_real_generator 98 | 99 | model = nn(n_classes, fc_d=fc_d) 100 | 101 | model.compile( 102 | optimizer=optimizer, 103 | loss=['categorical_crossentropy'], 104 | metrics=['accuracy'] 105 | ) 106 | 107 | logs_path = 'logs/' + exp_name + '/' 108 | os.mkdir(logs_path) 109 | csv_logger = CSVLogger(logs_path + 'log.log') 110 | model_checkpoint = ModelCheckpoint( 111 | filepath=logs_path, 112 | save_weights_only=True, 113 | monitor='val_accuracy', 114 | mode='max', 115 | save_best_only=True) 116 | 117 | x, y = next(validation_generator) 118 | model.fit(train_generator, 119 | validation_data=validation_generator, 120 | validation_batch_size=1, 121 | validation_steps=32, 122 | steps_per_epoch=steps_per_epoch, 123 | epochs=epochs, 124 | callbacks=[ 125 | csv_logger, 126 | model_checkpoint, 127 | ConfusionMatrixPlotter(x, y, validation_generator.classes, 128 | to_file=logs_path + 'confusion_matrix.png'), 129 | AccLossPlotter(to_file=logs_path + 'plot_history.png') 130 | ]) 131 | 132 | -------------------------------------------------------------------------------- /gelsight_simulation/launch/dc.launch: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 71 | 72 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 94 | 95 | 96 | 97 | 98 | 99 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /fdm_printer_description/models/macros.xacro: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 40 | 41 | 42 | 43 | ${color} 44 | ${color} 45 | 0.1 0.1 0.1 1 46 | 0 0 0 0 47 | 48 | 49 | 50 | 51 | 52 | 53 | 57 | 59 | 60 | 61 | 62 | 63 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 79 | 89 | 90 | 91 | transmission_interface/SimpleTransmission 92 | 93 | hardware_interface/PositionJointInterface 94 | 95 | 96 | 1 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | /fdm_printer 105 | gazebo_ros_control/DefaultRobotHWSim 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /experiments/preparation/align_globally.py: -------------------------------------------------------------------------------- 1 | from math import sqrt 2 | 3 | import cv2 4 | import numpy as np 5 | 6 | from ..data_loader import data_generator, load_single_img, load_single_img2, DataGenerator 7 | 8 | 9 | def diff(im1, im2): 10 | return cv2.absdiff(im1, im2) 11 | 12 | 13 | def preview(generator): 14 | def collect_points(name, lst, img): 15 | def click_callback(event, x, y, flags, param): 16 | if event == cv2.EVENT_LBUTTONDOWN: 17 | lst.append([x, y]) 18 | cv2.circle(img, (x, y), 3, (255, 0, 0)) 19 | cv2.imshow(name, img) 20 | print([x, y]) 21 | 22 | return click_callback 23 | 24 | def square_four_points(pts): 25 | [x1, y1] = pts[0] 26 | [x2, y2] = pts[1] 27 | 28 | delta_x = abs(x1 - x2) 29 | delta_y = abs(y1 - y2) 30 | 31 | if delta_x > delta_y: 32 | return [[x1, y1], [x1 + delta_x, y1], [x1 + delta_x, y1 + delta_x]] 33 | else: 34 | return [[x1, y1 + delta_y], [x1, y1], [x1 + delta_y, y1]] 35 | 36 | def crop(pts, img): 37 | 38 | img_h, img_w = img.shape[0:2] 39 | 40 | x_min = max(0, pts[0][0]) 41 | x_max = min(img_w, pts[1][0]) 42 | y_min = max(0, pts[0][1]) 43 | y_max = min(img_h, pts[1][1]) 44 | 45 | return img[y_min + 2:y_max - 2, x_min + 2:x_max - 2] 46 | 47 | img, img_sim, path = load_single_img('/real', '/sim', 641, resize=False) 48 | 49 | def findH(img, img_sim): 50 | img_pts = [] 51 | cv2.imshow('img', img) 52 | cv2.setMouseCallback('img', collect_points('img', img_pts, img)) 53 | cv2.waitKey(-1) 54 | 55 | img_pts = square_four_points(img_pts) 56 | print('img pts', img_pts) 57 | 58 | img_sim_pts = [] 59 | cv2.imshow('img_sim', img_sim) 60 | cv2.setMouseCallback('img_sim', collect_points('img_sim', img_sim_pts, img_sim)) 61 | cv2.waitKey(-1) 62 | 63 | img_sim_pts = square_four_points(img_sim_pts) 64 | print('img sim pts', img_sim_pts) 65 | 66 | img_h, img_w, _ = img.shape 67 | 68 | h = cv2.getAffineTransform(np.float32(img_pts), np.float32(img_sim_pts)) 69 | im_dst = cv2.warpAffine(img, h, (img_w, img_h)) 70 | 71 | cv2.imshow('distorted', im_dst) 72 | cv2.waitKey(-1) 73 | 74 | crop_pts = [ 75 | [int(x[0]) for x in h.dot(np.array([[0], [0], [1]])).tolist()], 76 | [int(x[0]) for x in h.dot(np.array([[img_w], [img_h], [1]])).tolist()] 77 | ] 78 | 79 | cv2.waitKey(-1) 80 | background_img = load_single_img2('background.png', resize=False) 81 | 82 | background_img_t = cv2.warpAffine(background_img, h, (img_w, img_h)) 83 | cv2.imshow('bg uncriop img', np.concatenate([background_img_t], axis=1)) 84 | cropped_background_img_t = crop(crop_pts, background_img_t) 85 | cv2.imshow('bg img', np.concatenate([cropped_background_img_t], axis=1)) 86 | cv2.waitKey(0) 87 | 88 | cv2.imwrite('aligned/background.png', cropped_background_img_t) 89 | 90 | return h, crop_pts 91 | 92 | k = 0 93 | 94 | h, crop_pts = findH(img, img_sim) 95 | cv2.destroyAllWindows() 96 | 97 | for (real, fake, depth, cls, path) in generator: 98 | 99 | for i in range(len(real)): 100 | img_h, img_w, _ = real[i].shape 101 | 102 | real_t = cv2.warpAffine(real[i], h, (img_w, img_h)) 103 | 104 | cropped_real_t = crop(crop_pts, real_t) 105 | cropped_fake_t = crop(crop_pts, fake[i]) 106 | cropped_depth_t = crop(crop_pts, depth[i]) 107 | 108 | assert cropped_real_t.shape[0:2] == cropped_depth_t.shape[0:2] == cropped_fake_t.shape[0:2] 109 | 110 | cv2.imshow('depth', np.concatenate([cropped_depth_t], axis=1)) 111 | cv2.imshow('frame', np.concatenate([real[i], fake[i], diff(real[i], fake[i]), 112 | real_t, diff(real_t, fake[i])], 113 | axis=1)) 114 | 115 | cv2.imshow('cropped', np.concatenate([cropped_real_t, cropped_fake_t, 116 | diff(cropped_real_t, cropped_fake_t)], 117 | axis=1)) 118 | 119 | np.save('aligned/depth/' + path[i][:-4] + '.npy', cropped_depth_t) 120 | cv2.imwrite('aligned/real/' + path[i], cropped_real_t) 121 | 122 | if cv2.waitKey(100) & 0xFF == ord('q'): 123 | exit() 124 | k += 1 125 | cv2.destroyAllWindows() 126 | 127 | return h, crop_pts 128 | 129 | 130 | if __name__ == '__main__': 131 | generator = DataGenerator( 132 | 'real', 133 | sim_path='sim', 134 | depth_path='depth', 135 | shuffle=False, 136 | batch_size=99, 137 | resize=False 138 | ) 139 | 140 | h = preview(generator) 141 | -------------------------------------------------------------------------------- /experiments/preparation/align_per_object.py: -------------------------------------------------------------------------------- 1 | from math import sqrt 2 | 3 | import cv2 4 | import numpy as np 5 | 6 | from ..data_loader import data_generator, load_single_img, load_single_img2, DataGenerator 7 | 8 | 9 | def diff(im1, im2): 10 | return cv2.absdiff(im1, im2) 11 | 12 | 13 | def align(generator): 14 | def collect_points(name, lst, img): 15 | def click_callback(event, x, y, flags, param): 16 | if event == cv2.EVENT_LBUTTONDOWN: 17 | lst.append([x, y]) 18 | cv2.circle(img, (x, y), 3, (255, 0, 0)) 19 | cv2.imshow(name, img) 20 | print([x, y]) 21 | 22 | return click_callback 23 | 24 | def square_four_points(pts): 25 | [x1, y1] = pts[0] 26 | [x2, y2] = pts[1] 27 | 28 | delta_x = abs(x1 - x2) 29 | delta_y = abs(y1 - y2) 30 | 31 | if delta_x > delta_y: 32 | return [[x1, y1], [x1 + delta_x, y1], [x1 + delta_x, y1 + delta_x]] 33 | else: 34 | return [[x1, y1 + delta_y], [x1, y1], [x1 + delta_y, y1]] 35 | 36 | def crop(pts, img): 37 | 38 | img_h, img_w = img.shape[0:2] 39 | 40 | x_min = max(0, pts[0][0]) 41 | x_max = min(img_w, pts[1][0]) 42 | y_min = max(0, pts[0][1]) 43 | y_max = min(img_h, pts[1][1]) 44 | 45 | return img[y_min + 2:y_max - 2, x_min + 2:x_max - 2] 46 | 47 | def findH(img, img_sim, cls_name): 48 | img_pts = [] 49 | cv2.imshow('img', img) 50 | cv2.setMouseCallback('img', collect_points('img', img_pts, img)) 51 | cv2.waitKey(-1) 52 | 53 | img_pts = square_four_points(img_pts) 54 | 55 | img_sim_pts = [] 56 | cv2.imshow('img_sim', img_sim) 57 | cv2.setMouseCallback('img_sim', collect_points('img_sim', img_sim_pts, img_sim)) 58 | cv2.waitKey(-1) 59 | 60 | img_sim_pts = square_four_points(img_sim_pts) 61 | print('img sim pts', img_sim_pts) 62 | 63 | img_h, img_w, _ = img.shape 64 | 65 | h = cv2.getAffineTransform(np.float32(img_pts), np.float32(img_sim_pts)) 66 | im_dst = cv2.warpAffine(img, h, (img_w, img_h)) 67 | 68 | cv2.imshow('distorted', im_dst) 69 | cv2.waitKey(-1) 70 | 71 | crop_pts = [ 72 | [int(x[0]) for x in h.dot(np.array([[0], [0], [1]])).tolist()], 73 | [int(x[0]) for x in h.dot(np.array([[img_w], [img_h], [1]])).tolist()] 74 | ] 75 | 76 | cv2.waitKey(-1) 77 | background_img = load_single_img2('background.png', resize=False) 78 | 79 | background_img_t = cv2.warpAffine(background_img, h, (img_w, img_h)) 80 | cv2.imshow('bg uncriop img', np.concatenate([background_img_t], axis=1)) 81 | cropped_background_img_t = crop(crop_pts, background_img_t) 82 | cv2.imshow('bg img', np.concatenate([cropped_background_img_t], axis=1)) 83 | cv2.waitKey(0) 84 | 85 | cv2.imwrite('aligned/background_' + cls_name + '.png', cropped_background_img_t) 86 | 87 | return h, crop_pts 88 | 89 | k = 0 90 | 91 | crop_pts = None 92 | 93 | for (real, fake, depth, cls, path) in generator: 94 | 95 | cls_name = generator.classes[np.argmax(cls)] 96 | h, crop_pts = findH(np.copy(real[52]), np.copy(fake[52]), cls_name) 97 | cv2.destroyAllWindows() 98 | for i in range(len(real)): 99 | img_h, img_w, _ = real[i].shape 100 | 101 | real_t = cv2.warpAffine(real[i], h, (img_w, img_h)) 102 | 103 | cropped_real_t = crop(crop_pts, real_t) 104 | cropped_fake_t = crop(crop_pts, fake[i]) 105 | cropped_depth_t = crop(crop_pts, depth[i]) 106 | 107 | assert cropped_real_t.shape[0:2] == cropped_depth_t.shape[0:2] == cropped_fake_t.shape[0:2] 108 | 109 | cv2.imshow('depth', np.concatenate([cropped_depth_t], axis=1)) 110 | cv2.imshow('frame', np.concatenate([real[i], fake[i], diff(real[i], fake[i]), 111 | real_t, diff(real_t, fake[i])], 112 | axis=1)) 113 | 114 | cv2.imshow('cropped', np.concatenate([cropped_real_t, cropped_fake_t, 115 | diff(cropped_real_t, cropped_fake_t)], 116 | axis=1)) 117 | 118 | np.save('/aligned/depth/' + path[i][:-4] + '.npy', cropped_depth_t) 119 | cv2.imwrite('/aligned/real/' + path[i], cropped_real_t) 120 | 121 | if cv2.waitKey(100) & 0xFF == ord('q'): 122 | exit() 123 | k += 1 124 | cv2.destroyAllWindows() 125 | 126 | return h, crop_pts 127 | 128 | 129 | if __name__ == '__main__': 130 | generator = DataGenerator( 131 | 'unaligned/real', 132 | sim_path='unaligned//sim', 133 | depth_path='unaligned/depth', 134 | shuffle=False, 135 | batch_size=99, 136 | resize=False 137 | ) 138 | 139 | h = align(generator) 140 | -------------------------------------------------------------------------------- /gelsight_description/launch/config.rviz: -------------------------------------------------------------------------------- 1 | Panels: 2 | - Class: rviz/Displays 3 | Help Height: 78 4 | Name: Displays 5 | Property Tree Widget: 6 | Expanded: 7 | - /Global Options1 8 | - /Status1 9 | - /RobotState1 10 | Splitter Ratio: 0.5 11 | Tree Height: 1067 12 | - Class: rviz/Selection 13 | Name: Selection 14 | - Class: rviz/Tool Properties 15 | Expanded: 16 | - /2D Pose Estimate1 17 | - /2D Nav Goal1 18 | - /Publish Point1 19 | Name: Tool Properties 20 | Splitter Ratio: 0.5886790156364441 21 | - Class: rviz/Views 22 | Expanded: 23 | - /Current View1 24 | Name: Views 25 | Splitter Ratio: 0.5 26 | - Class: rviz/Time 27 | Experimental: false 28 | Name: Time 29 | SyncMode: 0 30 | SyncSource: "" 31 | Preferences: 32 | PromptSaveOnExit: true 33 | Toolbars: 34 | toolButtonStyle: 2 35 | Visualization Manager: 36 | Class: "" 37 | Displays: 38 | - Alpha: 0.5 39 | Cell Size: 1 40 | Class: rviz/Grid 41 | Color: 160; 160; 164 42 | Enabled: true 43 | Line Style: 44 | Line Width: 0.029999999329447746 45 | Value: Lines 46 | Name: Grid 47 | Normal Cell Count: 0 48 | Offset: 49 | X: 0 50 | Y: 0 51 | Z: 0 52 | Plane: XY 53 | Plane Cell Count: 10 54 | Reference Frame: 55 | Value: true 56 | - Attached Body Color: 150; 50; 150 57 | Class: moveit_rviz_plugin/RobotState 58 | Collision Enabled: false 59 | Enabled: true 60 | Links: 61 | All Links Enabled: true 62 | Expand Joint Details: false 63 | Expand Link Details: false 64 | Expand Tree: false 65 | Link Tree Style: Links in Alphabetic Order 66 | gelsight_base: 67 | Alpha: 1 68 | Show Axes: false 69 | Show Trail: false 70 | gelsight_cover_back: 71 | Alpha: 1 72 | Show Axes: false 73 | Show Trail: false 74 | Value: true 75 | gelsight_cover_front: 76 | Alpha: 1 77 | Show Axes: false 78 | Show Trail: false 79 | Value: true 80 | gelsight_elastomer: 81 | Alpha: 1 82 | Show Axes: false 83 | Show Trail: false 84 | Value: true 85 | Name: RobotState 86 | Robot Alpha: 1 87 | Robot Description: robot_description 88 | Robot State Topic: display_robot_state 89 | Show All Links: true 90 | Show Highlights: true 91 | Value: true 92 | Visual Enabled: true 93 | Enabled: true 94 | Global Options: 95 | Background Color: 48; 48; 48 96 | Default Light: true 97 | Fixed Frame: gelsight_base 98 | Frame Rate: 30 99 | Name: root 100 | Tools: 101 | - Class: rviz/Interact 102 | Hide Inactive Objects: true 103 | - Class: rviz/MoveCamera 104 | - Class: rviz/Select 105 | - Class: rviz/FocusCamera 106 | - Class: rviz/Measure 107 | - Class: rviz/SetInitialPose 108 | Theta std deviation: 0.2617993950843811 109 | Topic: /initialpose 110 | X std deviation: 0.5 111 | Y std deviation: 0.5 112 | - Class: rviz/SetGoal 113 | Topic: /move_base_simple/goal 114 | - Class: rviz/PublishPoint 115 | Single click: true 116 | Topic: /clicked_point 117 | Value: true 118 | Views: 119 | Current: 120 | Class: rviz/Orbit 121 | Distance: 0.5541285276412964 122 | Enable Stereo Rendering: 123 | Stereo Eye Separation: 0.05999999865889549 124 | Stereo Focal Distance: 1 125 | Swap Stereo Eyes: false 126 | Value: false 127 | Focal Point: 128 | X: 0.005241456441581249 129 | Y: 0.006700105965137482 130 | Z: -0.008443961851298809 131 | Focal Shape Fixed Size: true 132 | Focal Shape Size: 0.05000000074505806 133 | Invert Z Axis: false 134 | Name: Current View 135 | Near Clip Distance: 0.009999999776482582 136 | Pitch: 0.8247954845428467 137 | Target Frame: 138 | Value: Orbit (rviz) 139 | Yaw: 0.5454007983207703 140 | Saved: ~ 141 | Window Geometry: 142 | Displays: 143 | collapsed: false 144 | Height: 1364 145 | Hide Left Dock: false 146 | Hide Right Dock: false 147 | QMainWindow State: 000000ff00000000fd000000040000000000000156000004b6fc0200000008fb0000001200530065006c0065006300740069006f006e00000001e10000009b0000005c00fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000001ed000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c006100790073010000003d000004b6000000c900fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261000000010000010f000004b6fc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a00560069006500770073000000003d000004b6000000a400fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b200000000000000000000000200000490000000a9fc0100000001fb0000000a00560069006500770073030000004e00000080000002e100000197000000030000067d0000003efc0100000002fb0000000800540069006d006501000000000000067d000002eb00fffffffb0000000800540069006d0065010000000000000450000000000000000000000521000004b600000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000 148 | Selection: 149 | collapsed: false 150 | Time: 151 | collapsed: false 152 | Tool Properties: 153 | collapsed: false 154 | Views: 155 | collapsed: false 156 | Width: 1661 157 | X: 60 158 | Y: 23 159 | -------------------------------------------------------------------------------- /fdm_printer/src/fdm_printer_hardware_interface.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | template 21 | std::string string_format( const std::string& format, Args ... args ) 22 | { 23 | size_t size = snprintf( nullptr, 0, format.c_str(), args ... ) + 1; // Extra space for '\0' 24 | if( size <= 0 ){ throw std::runtime_error( "Error during formatting." ); } 25 | std::unique_ptr buf( new char[ size ] ); 26 | snprintf( buf.get(), size, format.c_str(), args ... ); 27 | return std::string( buf.get(), buf.get() + size - 1 ); // We don't want the '\0' inside 28 | } 29 | 30 | class FDMPrinter : public hardware_interface::RobotHW 31 | { 32 | private: 33 | serial::Serial * port; 34 | hardware_interface::JointStateInterface jnt_state_interface; 35 | hardware_interface::PositionJointInterface jnt_pos_interface; 36 | 37 | double cmd[3]; 38 | double pos[3]; 39 | double vel[3]; 40 | double eff[3]; 41 | 42 | ros::Duration elapsed_time_; 43 | boost::shared_ptr controller_manager_; 44 | ros::Timer non_realtime_loop_; 45 | 46 | public: 47 | FDMPrinter(ros::NodeHandle &nh_) 48 | { 49 | serial::Timeout t = serial::Timeout::simpleTimeout(250); 50 | port = new serial::Serial("/dev/ttyUSB0", 115200); 51 | 52 | 53 | cmd[0] = 0; 54 | cmd[1] = 0; 55 | cmd[2] = 0; 56 | 57 | 58 | // connect and register the joint state interface 59 | hardware_interface::JointStateHandle state_handle_x("x_axis", &pos[0], &vel[0], &eff[0]); 60 | jnt_state_interface.registerHandle(state_handle_x); 61 | 62 | hardware_interface::JointStateHandle state_handle_y("y_axis", &pos[1], &vel[1], &eff[1]); 63 | jnt_state_interface.registerHandle(state_handle_y); 64 | 65 | hardware_interface::JointStateHandle state_handle_z("z_axis", &pos[2], &vel[2], &eff[2]); 66 | jnt_state_interface.registerHandle(state_handle_z); 67 | 68 | registerInterface(&jnt_state_interface); 69 | 70 | // connect and register the joint position interface 71 | hardware_interface::JointHandle pos_handle_x(jnt_state_interface.getHandle("x_axis"), &cmd[0]); 72 | jnt_pos_interface.registerHandle(pos_handle_x); 73 | 74 | hardware_interface::JointHandle pos_handle_y(jnt_state_interface.getHandle("y_axis"), &cmd[1]); 75 | jnt_pos_interface.registerHandle(pos_handle_y); 76 | 77 | hardware_interface::JointHandle pos_handle_z(jnt_state_interface.getHandle("z_axis"), &cmd[2]); 78 | jnt_pos_interface.registerHandle(pos_handle_z); 79 | 80 | registerInterface(&jnt_pos_interface); 81 | 82 | 83 | 84 | controller_manager_.reset(new controller_manager::ControllerManager(this, nh_)); 85 | ros::Duration update_freq = ros::Duration(1.0/30); 86 | non_realtime_loop_ = nh_.createTimer(update_freq, &FDMPrinter::update, this); 87 | 88 | 89 | ros::Duration period = ros::Duration(5); 90 | period.sleep(); 91 | this->goHome(); 92 | period.sleep(); 93 | } 94 | 95 | 96 | void goHome() { 97 | std:: cout << "[write] " << "G28 \n"; 98 | port->write("G28 \n"); 99 | port->flush(); 100 | std::cout << "flushed" << std::endl; 101 | 102 | // ros::Duration period = ros::Duration(5); 103 | // period.sleep(); 104 | // period.sleep(); 105 | 106 | } 107 | 108 | 109 | void read(){ 110 | pos[0] = cmd[0]; 111 | pos[1] = cmd[1]; 112 | pos[2] = cmd[2]; 113 | } 114 | 115 | void write(ros::Duration elapsed_time){ 116 | if(cmd[0] < 0 || cmd[1] < 0 || cmd[2] < 0){ 117 | return; 118 | } 119 | 120 | if (cmd[0] != pos[0] || cmd[1] != pos[1] || cmd[2] != pos[2]){ 121 | std::stringstream ss; 122 | ss << string_format("G0 X%.2f Y%.2f Z%.2f \n", cmd[0]*1000, cmd[1]*1000, cmd[2]*1000) << std::endl; 123 | std:: cout << "[write] " << ss.str(); 124 | port->write(ss.str()); 125 | port->flush(); 126 | std::cout << "flushed" << std::endl; 127 | } 128 | } 129 | 130 | void sendGCodeCmd(const std_msgs::String::ConstPtr& msg){ 131 | std::cout << "Send: " << msg->data << std::endl; 132 | port->write(msg->data + "\n"); 133 | port->flush(); 134 | } 135 | 136 | 137 | void update(const ros::TimerEvent& e) { 138 | elapsed_time_ = ros::Duration(e.current_real - e.last_real); 139 | read(); 140 | controller_manager_->update(ros::Time::now(), elapsed_time_); 141 | write(elapsed_time_); 142 | } 143 | 144 | ros::Time get_time() { 145 | return ros::Time::now(); 146 | } 147 | 148 | ros::Duration get_period() { 149 | return ros::Duration(1/(double)2); 150 | } 151 | 152 | 153 | ~FDMPrinter() { 154 | port->close(); 155 | delete port; 156 | } 157 | }; 158 | 159 | 160 | int main(int argc, char** argv) { 161 | 162 | ros::init(argc, argv, "fdm_printer_hardware_interface"); 163 | ros::CallbackQueue ros_queue; 164 | 165 | 166 | ros::NodeHandle nh; 167 | nh.setCallbackQueue(&ros_queue); 168 | FDMPrinter printer(nh); 169 | 170 | // ros::Subscriber sub = nh.subscribe("gcode_script", 10, &FDMPrinter::sendGCodeCmd, &printer); 171 | // std::cout << "subscribed" << sub << std::endl; 172 | 173 | ros::MultiThreadedSpinner spinner(0); 174 | spinner.spin(&ros_queue); 175 | return 0; 176 | 177 | //controller_manager::ControllerManager cm(&printer); 178 | 179 | 180 | 181 | //ros::Duration period = printer.get_period(); 182 | 183 | 184 | 185 | //period.sleep(); 186 | //period.sleep(); 187 | //period.sleep(); 188 | //period.sleep(); 189 | 190 | //printer.goHome(); 191 | //period.sleep(); 192 | 193 | //std::cout << "end" << std::endl; 194 | 195 | //while (ros::ok()) 196 | //{ 197 | // printer.read(); 198 | // cm.update(printer.get_time(), period); 199 | // printer.write(); 200 | // period.sleep(); 201 | // ros::spinOnce(); 202 | //} 203 | //return 0; 204 | } -------------------------------------------------------------------------------- /gelsight_simulation/scripts/data_collection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import rospy 4 | import time 5 | from std_msgs.msg import Float64MultiArray 6 | from sensor_msgs.msg import Image 7 | from cv_bridge import CvBridge, CvBridgeError 8 | import cv2 9 | import math 10 | import numpy as np 11 | # import rospack 12 | # PKG_PATH = rospack.get_path('gelsight_gazebo') 13 | 14 | 15 | bridge = CvBridge() 16 | pub = None 17 | rate = None 18 | previous_position = (0, 0, 0) 19 | # printer_speed = 0.004 20 | printer_speed = 0.001 21 | 22 | WS_MAX = (0.32, 0.32, 0.42) 23 | # WS_MIN = (0, 0, 0.06) 24 | WS_MIN = (0, 0, 0.01) 25 | gelsight_img = None 26 | gelsight_depth = None 27 | 28 | 29 | def show_normalized_img(name, img): 30 | draw = img.copy() 31 | draw -= np.min(draw) 32 | draw = draw / np.max(draw) 33 | cv2.imshow(name, draw) 34 | return draw 35 | 36 | 37 | def euclidean_dist(t1, t2): 38 | return math.sqrt(math.pow(t1[0] - t2[0], 2) + math.pow(t1[1] - t2[1], 2) + math.pow(t1[2] - t2[2], 2)) 39 | 40 | 41 | def gelsight_callback(img_msg): 42 | global gelsight_img 43 | 44 | camera_img = bridge.imgmsg_to_cv2(img_msg, desired_encoding="bgr8") 45 | gelsight_img = camera_img 46 | cv2.imshow('tactile_img', camera_img) 47 | cv2.waitKey(1) 48 | 49 | 50 | def depth_callback(depth_msg): 51 | global gelsight_depth 52 | 53 | img = bridge.imgmsg_to_cv2(depth_msg, desired_encoding="32FC1") 54 | img[np.isnan(img)] = np.inf 55 | gelsight_depth = img 56 | show_normalized_img('depth_map', gelsight_depth) 57 | cv2.waitKey(1) 58 | 59 | 60 | def move(x, y, z, force=False, wait=None): 61 | if rospy.is_shutdown(): 62 | exit(2) 63 | 64 | if not force and ( 65 | x > WS_MAX[0] or y > WS_MAX[1] or z > WS_MAX[2] or x < WS_MIN[0] or y < WS_MIN[1] or z < WS_MIN[2]): 66 | print('ERROR. Attempted to move to invalid position.', (x, y, z)) 67 | exit(2) 68 | 69 | global previous_position 70 | print('Move to:', (x, y, z)) 71 | pos = Float64MultiArray() 72 | pos.data = [x, y, z] 73 | pub.publish(pos) 74 | 75 | if wait is None: 76 | s = 3 + euclidean_dist(previous_position, (x, y, z)) / printer_speed 77 | else: 78 | s = wait 79 | print('Waiting seconds: ', s) 80 | time.sleep(s) 81 | previous_position = (x, y, z) 82 | 83 | 84 | def collect_data(): 85 | global previous_position 86 | 87 | x_steps = 3 88 | y_steps = 3 89 | z_steps = 10 90 | h_step_size = 0.001 91 | z_step_size = 0.0001 92 | 93 | # SIM 94 | start_x = 0.185 - 0.045 95 | start_y = 0.165 + 0.007 96 | start_z = 0.022 + 0.027 97 | 98 | previous_position = (0.0, 0.0, 0.0) 99 | 100 | # REAL 101 | # start_x = 0.1625 102 | # start_y = 0.16 103 | # # start_z = 0.0641 104 | # start_z = 0.065 105 | 106 | starting_position = previous_position 107 | # starting_position = (0.1, 0.1, start_z + 0.01) 108 | # previous_position = starting_position 109 | # previous_position = (start_x, start_y, start_z + 0.01) 110 | print('START.') 111 | move(0, 0, start_z, force=True, wait=20) 112 | 113 | # TEST 114 | # move(start_x, start_y, start_z + 2*z_step_size) 115 | # move(start_x, start_y, start_z + 1*z_step_size) 116 | # move(start_x, start_y, start_z) 117 | # move(start_x, start_y, start_z - 1 * z_step_size) 118 | move(start_x, start_y, start_z + 3 * z_step_size, wait=20) # 20 for sim, 60 for real 119 | 120 | # return 121 | # 122 | # move(*starting_position) 123 | 124 | k = 0 125 | BASE = '/home/danfergo/Projects/PhD/gelsight_simulation/dataset/sim/exp3' 126 | BASE_DEPTH = '/home/danfergo/Projects/PhD/gelsight_simulation/dataset/sim/depth3' 127 | # BASE = '/home/danfergo/Projects/gelsight_simulation/dataset/demo/sim_test' 128 | solid = 'wave1' 129 | # solid = 'dots' 130 | # solid = 'cross_lines' 131 | # solid = 'flat_slab' 132 | # solid = 'curved_surface' 133 | # solid = 'parallel_lines' 134 | # solid = 'pacman' 135 | # solid = 'torus' 136 | # solid = 'cylinder_shell' 137 | # solid = 'sphere2' 138 | # solid = 'line' 139 | # solid = 'cylinder_side' 140 | # solid = 'moon' 141 | # solid = 'random' 142 | # solid = 'prism' 143 | # solid = 'dot_in' 144 | # solid = 'triangle' 145 | # solid = 'sphere' 146 | # solid = 'hexagon' 147 | # solid = 'cylinder' 148 | # solid = 'cone' 149 | 150 | for x in range(-(x_steps // 2), (x_steps // 2) + 1): 151 | for y in range(-(y_steps // 2), (y_steps // 2) + 1): 152 | p = start_x + x * h_step_size, start_y + y * h_step_size 153 | move(*(p + (start_z + z_step_size,))) 154 | print('========================================>') 155 | for z in range(z_steps + 1): 156 | print('--START ---->') 157 | print('POINT:', (x, y, z)) 158 | 159 | pp = p + (start_z - z * z_step_size,) 160 | move(*pp) 161 | 162 | k += 1 163 | if gelsight_img is not None: 164 | cv2.imwrite( 165 | BASE + '/' + solid + '__' + str(k) + '__' + str(x) + '_' + str(y) + '_' + str(z) + '.png', 166 | gelsight_img) 167 | print('...>', np.max(gelsight_depth), np.min(gelsight_depth), gelsight_depth.dtype, 168 | np.shape(gelsight_depth), np.shape(gelsight_depth)) 169 | # cv2.imwrite( 170 | # BASE_DEPTH + '/' + solid + '__' + str(k) + '__' + str(x) + '_' + str(y) + '_' + str(z) + '.bmp', 171 | # gelsight_depth) 172 | np.save(BASE_DEPTH + '/' + solid + '__' + str(k) + '__' + str(x) + '_' + str(y) + '_' + str(z) + '.npy', gelsight_depth) 173 | 174 | 175 | 176 | else: 177 | print('warn. tactile img not received') 178 | 179 | print('--END ---->') 180 | time.sleep(1) 181 | 182 | print('========================================>') 183 | move(*(p + (start_z + z_step_size,))) 184 | 185 | # move(*starting_position) 186 | 187 | 188 | if __name__ == '__main__': 189 | rospy.init_node('gelsight_simulation_dc') 190 | 191 | pub = rospy.Publisher('/fdm_printer/xyz_controller/command', Float64MultiArray, queue_size=10) 192 | rospy.Subscriber("/gelsight/tactile_image", Image, gelsight_callback) 193 | rospy.Subscriber("/gelsight/depth/image_raw", Image, depth_callback) 194 | 195 | rate = rospy.Rate(1) 196 | # rate.sleep() 197 | # rate.sleep() 198 | # rate.sleep() 199 | rate.sleep() 200 | rate.sleep() 201 | print('--------------------_>>> Data Collection start.') 202 | collect_data() 203 | -------------------------------------------------------------------------------- /experiments/data_loader.py: -------------------------------------------------------------------------------- 1 | from os import listdir 2 | from os.path import isfile, join 3 | import random 4 | import cv2 5 | import numpy as np 6 | import yaml 7 | 8 | OBJECT_SET_CLASSES = [ 9 | 'wave1', 10 | 'dots', 11 | 'cross_lines', 12 | 'flat_slab', 13 | 'curved_surface', 14 | 'parallel_lines', 15 | 'pacman', 16 | 'torus', 17 | 'cylinder_shell', 18 | 'sphere2', 19 | 'line', 20 | 'cylinder_side', 21 | 'moon', 22 | 'random', 23 | 'prism', 24 | 'dot_in', 25 | 'triangle', 26 | 'sphere', 27 | 'hexagon', 28 | 'cylinder', 29 | 'cone' 30 | ] 31 | 32 | 33 | def to_categorical(y, num_classes=None, dtype='float32'): 34 | """ retrieved from keras """ 35 | y = np.array(y, dtype='int') 36 | input_shape = y.shape 37 | if input_shape and input_shape[-1] == 1 and len(input_shape) > 1: 38 | input_shape = tuple(input_shape[:-1]) 39 | y = y.ravel() 40 | if not num_classes: 41 | num_classes = np.max(y) + 1 42 | n = y.shape[0] 43 | categorical = np.zeros((n, num_classes), dtype=dtype) 44 | categorical[np.arange(n), y] = 1 45 | output_shape = input_shape + (num_classes,) 46 | categorical = np.reshape(categorical, output_shape) 47 | return categorical 48 | 49 | 50 | def crop_center(img, cropx, cropy): 51 | y, x = img.shape[:2] 52 | startx = x // 2 - (cropx // 2) 53 | starty = y // 2 - (cropy // 2) 54 | return img[starty:starty + cropy, startx:startx + cropx] 55 | 56 | 57 | class DataGenerator: 58 | 59 | def __init__(self, 60 | path=None, 61 | sim_path=None, 62 | depth_path=None, 63 | batch_size=32, 64 | shuffle=True, 65 | output_shape=(224, 224, 3), 66 | classes=None, 67 | resize=True, 68 | split=None, 69 | splits_file=None, 70 | output_paths=True, 71 | augmentor=None): 72 | 73 | if classes is None: 74 | classes = OBJECT_SET_CLASSES 75 | self.c_idx = {c: classes.index(c) for c in classes} 76 | self.classes = classes 77 | 78 | if split is None: 79 | files_path = path if path else depth_path 80 | self.files = [f for f in listdir(files_path) if 81 | isfile(join(files_path, f)) and f.split('__')[0] in classes] 82 | else: 83 | print(splits_file, split) 84 | try: 85 | self.files = yaml.load(open(splits_file, 'r'))[split] 86 | except e: 87 | print(e) 88 | # print('----------------->', yaml.load(open(splits_file, 'r'))['train']) 89 | self.files.sort(key=self.compare_files) 90 | 91 | self.batch_size = len(self.files) if batch_size < 0 else batch_size 92 | self.files_sent = 0 93 | self.shuffle = shuffle 94 | self.sim_path = sim_path 95 | self.depth_path = depth_path 96 | self.path = path 97 | self.resize = resize 98 | self.output_shape = output_shape 99 | self.output_paths = output_paths 100 | self.augmentor = augmentor 101 | 102 | def filename_data(self, file): 103 | parts = file[:-4].split('__') 104 | 105 | return self.c_idx[parts[0]], int(parts[1]) 106 | 107 | def compare_files(self, file): 108 | cls, k = self.filename_data(file) 109 | return cls * 1000 + k 110 | 111 | def read_img(self, base, im_path): 112 | im = cv2.imread(join(base, im_path)) 113 | if self.resize: 114 | shape = np.shape(im) 115 | min_side = min(shape[0], shape[1]) 116 | im = crop_center(im, min_side, min_side) 117 | return cv2.resize(im, (self.output_shape[1], self.output_shape[0])) 118 | return im 119 | 120 | def read_depth_img(self, base, im_path): 121 | im = np.load(join(base, im_path[:-4] + '.npy')) 122 | if self.resize: 123 | shape = np.shape(im) 124 | min_side = min(shape[0], shape[1]) 125 | im = crop_center(im, min_side, min_side) 126 | return cv2.resize(im, (self.output_shape[1], self.output_shape[0])) 127 | return im 128 | 129 | def size(self): 130 | return len(self.files) 131 | 132 | def __iter__(self): 133 | return self 134 | 135 | def next(self): 136 | return self.__next__() 137 | 138 | def __next__(self): 139 | shuffle = self.shuffle 140 | files = self.files 141 | batch_size = self.batch_size 142 | c_idx = self.c_idx 143 | classes = self.classes 144 | sim_path = self.sim_path 145 | depth_path = self.depth_path 146 | path = self.path 147 | 148 | if shuffle or self.files_sent < len(files): 149 | if shuffle: 150 | random.shuffle(files) 151 | 152 | batch_paths = files[0:batch_size] if shuffle else files[self.files_sent:self.files_sent + batch_size] 153 | 154 | ys = [y[:-4].split('__') for y in batch_paths] 155 | cls = [to_categorical(c_idx[y[0]], len(classes)) for y in ys] 156 | 157 | ret = [] 158 | if path is not None: 159 | imgs = [self.read_img(path, x) for x in batch_paths] 160 | ret.append(np.array(imgs)) 161 | 162 | if sim_path is not None: 163 | sim_batch = [self.read_img(sim_path, x) for x in batch_paths] 164 | ret.append(sim_batch) 165 | 166 | if depth_path is not None: 167 | depth_batch = [self.read_depth_img(depth_path, x) for x in batch_paths] 168 | ret.append(depth_batch) 169 | 170 | ret.append(np.array(cls)) 171 | 172 | if self.output_paths: 173 | ret.append(batch_paths) 174 | 175 | self.files_sent += batch_size 176 | 177 | return self.augmentor(*ret) if (self.augmentor is not None) else tuple(ret) 178 | 179 | raise StopIteration 180 | 181 | 182 | def load_single_img(path, clones_path=None, ith=0, output_shape=(224, 224, 3), resize=True): 183 | files = [f for f in listdir(path) if isfile(join(path, f))] 184 | files.sort() 185 | 186 | def read_img(base, im_path): 187 | im = cv2.imread(join(base, im_path)) 188 | if resize: 189 | shape = np.shape(im) 190 | min_side = min(shape[0], shape[1]) 191 | im = crop_center(im, min_side, min_side) 192 | return cv2.resize(im, (output_shape[1], output_shape[0])) 193 | return im 194 | 195 | return read_img(path, files[ith]), read_img(clones_path, files[ith]), files[ith] 196 | 197 | 198 | def load_single_img2(path, output_shape=(224, 224, 3), resize=True): 199 | im = cv2.imread(path) 200 | if resize: 201 | shape = np.shape(im) 202 | min_side = min(shape[0], shape[1]) 203 | im = crop_center(im, min_side, min_side) 204 | return cv2.resize(im, (output_shape[1], output_shape[0])) 205 | return im 206 | 207 | 208 | def preview(generator): 209 | img, cls = next(generator) 210 | k = 0 211 | 212 | print('Batch Shapes:', np.shape(img), np.shape(cls)) 213 | for i in range(32): 214 | cv2.imshow('frame', np.concatenate([img[i]], axis=1)) 215 | print(cls[i]) 216 | if cv2.waitKey(0) & 0xFF == ord('q'): 217 | break 218 | k += 1 219 | 220 | 221 | if __name__ == '__main__': 222 | r_generator = data_generator( 223 | '/home/danfergo/Projects/PhD/gelsight_simulation/dataset/real', 224 | # '/home/danfergo/Projects/gelsight_simulation/dataset' 225 | ) 226 | preview(r_generator) 227 | -------------------------------------------------------------------------------- /gelsight_gazebo/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.3) 2 | project(gelsight_gazebo) 3 | 4 | ## Compile as C++11, supported in ROS Kinetic and newer 5 | # add_compile_options(-std=c++11) 6 | 7 | ## Find catkin macros and libraries 8 | ## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) 9 | ## is used, also find other catkin packages 10 | find_package(catkin REQUIRED) 11 | 12 | ## System dependencies are found with CMake's conventions 13 | # find_package(Boost REQUIRED COMPONENTS system) 14 | 15 | 16 | ## Uncomment this if the package has a setup.py. This macro ensures 17 | ## modules and global scripts declared therein get installed 18 | ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html 19 | # catkin_python_setup() 20 | 21 | ################################################ 22 | ## Declare ROS messages, services and actions ## 23 | ################################################ 24 | 25 | ## To declare and build messages, services or actions from within this 26 | ## package, follow these steps: 27 | ## * Let MSG_DEP_SET be the set of packages whose message types you use in 28 | ## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). 29 | ## * In the file package.xml: 30 | ## * add a build_depend tag for "message_generation" 31 | ## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET 32 | ## * If MSG_DEP_SET isn't empty the following dependency has been pulled in 33 | ## but can be declared for certainty nonetheless: 34 | ## * add a exec_depend tag for "message_runtime" 35 | ## * In this file (CMakeLists.txt): 36 | ## * add "message_generation" and every package in MSG_DEP_SET to 37 | ## find_package(catkin REQUIRED COMPONENTS ...) 38 | ## * add "message_runtime" and every package in MSG_DEP_SET to 39 | ## catkin_package(CATKIN_DEPENDS ...) 40 | ## * uncomment the add_*_files sections below as needed 41 | ## and list every .msg/.srv/.action file to be processed 42 | ## * uncomment the generate_messages entry below 43 | ## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) 44 | 45 | ## Generate messages in the 'msg' folder 46 | # add_message_files( 47 | # FILES 48 | # Message1.msg 49 | # Message2.msg 50 | # ) 51 | 52 | ## Generate services in the 'srv' folder 53 | # add_service_files( 54 | # FILES 55 | # Service1.srv 56 | # Service2.srv 57 | # ) 58 | 59 | ## Generate actions in the 'action' folder 60 | # add_action_files( 61 | # FILES 62 | # Action1.action 63 | # Action2.action 64 | # ) 65 | 66 | ## Generate added messages and services with any dependencies listed here 67 | # generate_messages( 68 | # DEPENDENCIES 69 | # std_msgs # Or other packages containing msgs 70 | # ) 71 | 72 | ################################################ 73 | ## Declare ROS dynamic reconfigure parameters ## 74 | ################################################ 75 | 76 | ## To declare and build dynamic reconfigure parameters within this 77 | ## package, follow these steps: 78 | ## * In the file package.xml: 79 | ## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" 80 | ## * In this file (CMakeLists.txt): 81 | ## * add "dynamic_reconfigure" to 82 | ## find_package(catkin REQUIRED COMPONENTS ...) 83 | ## * uncomment the "generate_dynamic_reconfigure_options" section below 84 | ## and list every .cfg file to be processed 85 | 86 | ## Generate dynamic reconfigure parameters in the 'cfg' folder 87 | # generate_dynamic_reconfigure_options( 88 | # cfg/DynReconf1.cfg 89 | # cfg/DynReconf2.cfg 90 | # ) 91 | 92 | ################################### 93 | ## catkin specific configuration ## 94 | ################################### 95 | ## The catkin_package macro generates cmake config files for your package 96 | ## Declare things to be passed to dependent projects 97 | ## INCLUDE_DIRS: uncomment this if your package contains header files 98 | ## LIBRARIES: libraries you create in this project that dependent projects also need 99 | ## CATKIN_DEPENDS: catkin_packages dependent projects also need 100 | ## DEPENDS: system dependencies of this project that dependent projects also need 101 | catkin_package( 102 | # INCLUDE_DIRS include 103 | # LIBRARIES gelsight_gazebo 104 | # CATKIN_DEPENDS other_catkin_pkg 105 | # DEPENDS system_lib 106 | ) 107 | 108 | ########### 109 | ## Build ## 110 | ########### 111 | 112 | ## Specify additional locations of header files 113 | ## Your package locations should be listed before other locations 114 | include_directories( 115 | # include 116 | # ${catkin_INCLUDE_DIRS} 117 | ) 118 | 119 | ## Declare a C++ library 120 | # add_library(${PROJECT_NAME} 121 | # src/${PROJECT_NAME}/gelsight_gazebo.cpp 122 | # ) 123 | 124 | ## Add cmake target dependencies of the library 125 | ## as an example, code may need to be generated before libraries 126 | ## either from message generation or dynamic reconfigure 127 | # add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 128 | 129 | ## Declare a C++ executable 130 | ## With catkin_make all packages are built within a single CMake context 131 | ## The recommended prefix ensures that target names across packages don't collide 132 | # add_executable(${PROJECT_NAME}_node src/gelsight_gazebo_node.cpp) 133 | 134 | ## Rename C++ executable without prefix 135 | ## The above recommended prefix causes long target names, the following renames the 136 | ## target back to the shorter version for ease of user use 137 | ## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" 138 | # set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") 139 | 140 | ## Add cmake target dependencies of the executable 141 | ## same as for the library above 142 | # add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 143 | 144 | ## Specify libraries to link a library or executable target against 145 | # target_link_libraries(${PROJECT_NAME}_node 146 | # ${catkin_LIBRARIES} 147 | # ) 148 | 149 | ############# 150 | ## Install ## 151 | ############# 152 | 153 | # all install targets should use catkin DESTINATION variables 154 | # See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html 155 | 156 | ## Mark executable scripts (Python etc.) for installation 157 | ## in contrast to setup.py, you can choose the destination 158 | # install(PROGRAMS 159 | # scripts/my_python_script 160 | # DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 161 | # ) 162 | 163 | ## Mark executables for installation 164 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html 165 | # install(TARGETS ${PROJECT_NAME}_node 166 | # RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 167 | # ) 168 | 169 | ## Mark libraries for installation 170 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html 171 | # install(TARGETS ${PROJECT_NAME} 172 | # ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 173 | # LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 174 | # RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} 175 | # ) 176 | 177 | ## Mark cpp header files for installation 178 | # install(DIRECTORY include/${PROJECT_NAME}/ 179 | # DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 180 | # FILES_MATCHING PATTERN "*.h" 181 | # PATTERN ".svn" EXCLUDE 182 | # ) 183 | 184 | ## Mark other files for installation (e.g. launch and bag files, etc.) 185 | # install(FILES 186 | # # myfile1 187 | # # myfile2 188 | # DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} 189 | # ) 190 | 191 | ############# 192 | ## Testing ## 193 | ############# 194 | 195 | ## Add gtest based cpp test target and link libraries 196 | # catkin_add_gtest(${PROJECT_NAME}-test test/test_gelsight_gazebo.cpp) 197 | # if(TARGET ${PROJECT_NAME}-test) 198 | # target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) 199 | # endif() 200 | 201 | ## Add folders to be run by python nosetests 202 | # catkin_add_nosetests(test) 203 | -------------------------------------------------------------------------------- /fdm_printer_bringup/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.3) 2 | project(fdm_printer_bringup) 3 | 4 | ## Compile as C++11, supported in ROS Kinetic and newer 5 | # add_compile_options(-std=c++11) 6 | 7 | ## Find catkin macros and libraries 8 | ## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) 9 | ## is used, also find other catkin packages 10 | find_package(catkin REQUIRED) 11 | 12 | ## System dependencies are found with CMake's conventions 13 | # find_package(Boost REQUIRED COMPONENTS system) 14 | 15 | 16 | ## Uncomment this if the package has a setup.py. This macro ensures 17 | ## modules and global scripts declared therein get installed 18 | ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html 19 | # catkin_python_setup() 20 | 21 | ################################################ 22 | ## Declare ROS messages, services and actions ## 23 | ################################################ 24 | 25 | ## To declare and build messages, services or actions from within this 26 | ## package, follow these steps: 27 | ## * Let MSG_DEP_SET be the set of packages whose message types you use in 28 | ## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). 29 | ## * In the file package.xml: 30 | ## * add a build_depend tag for "message_generation" 31 | ## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET 32 | ## * If MSG_DEP_SET isn't empty the following dependency has been pulled in 33 | ## but can be declared for certainty nonetheless: 34 | ## * add a exec_depend tag for "message_runtime" 35 | ## * In this file (CMakeLists.txt): 36 | ## * add "message_generation" and every package in MSG_DEP_SET to 37 | ## find_package(catkin REQUIRED COMPONENTS ...) 38 | ## * add "message_runtime" and every package in MSG_DEP_SET to 39 | ## catkin_package(CATKIN_DEPENDS ...) 40 | ## * uncomment the add_*_files sections below as needed 41 | ## and list every .msg/.srv/.action file to be processed 42 | ## * uncomment the generate_messages entry below 43 | ## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) 44 | 45 | ## Generate messages in the 'msg' folder 46 | # add_message_files( 47 | # FILES 48 | # Message1.msg 49 | # Message2.msg 50 | # ) 51 | 52 | ## Generate services in the 'srv' folder 53 | # add_service_files( 54 | # FILES 55 | # Service1.srv 56 | # Service2.srv 57 | # ) 58 | 59 | ## Generate actions in the 'action' folder 60 | # add_action_files( 61 | # FILES 62 | # Action1.action 63 | # Action2.action 64 | # ) 65 | 66 | ## Generate added messages and services with any dependencies listed here 67 | # generate_messages( 68 | # DEPENDENCIES 69 | # std_msgs # Or other packages containing msgs 70 | # ) 71 | 72 | ################################################ 73 | ## Declare ROS dynamic reconfigure parameters ## 74 | ################################################ 75 | 76 | ## To declare and build dynamic reconfigure parameters within this 77 | ## package, follow these steps: 78 | ## * In the file package.xml: 79 | ## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" 80 | ## * In this file (CMakeLists.txt): 81 | ## * add "dynamic_reconfigure" to 82 | ## find_package(catkin REQUIRED COMPONENTS ...) 83 | ## * uncomment the "generate_dynamic_reconfigure_options" section below 84 | ## and list every .cfg file to be processed 85 | 86 | ## Generate dynamic reconfigure parameters in the 'cfg' folder 87 | # generate_dynamic_reconfigure_options( 88 | # cfg/DynReconf1.cfg 89 | # cfg/DynReconf2.cfg 90 | # ) 91 | 92 | ################################### 93 | ## catkin specific configuration ## 94 | ################################### 95 | ## The catkin_package macro generates cmake config files for your package 96 | ## Declare things to be passed to dependent projects 97 | ## INCLUDE_DIRS: uncomment this if your package contains header files 98 | ## LIBRARIES: libraries you create in this project that dependent projects also need 99 | ## CATKIN_DEPENDS: catkin_packages dependent projects also need 100 | ## DEPENDS: system dependencies of this project that dependent projects also need 101 | catkin_package( 102 | # INCLUDE_DIRS include 103 | # LIBRARIES fdm_printer_bringup 104 | # CATKIN_DEPENDS other_catkin_pkg 105 | # DEPENDS system_lib 106 | ) 107 | 108 | ########### 109 | ## Build ## 110 | ########### 111 | 112 | ## Specify additional locations of header files 113 | ## Your package locations should be listed before other locations 114 | include_directories( 115 | # include 116 | # ${catkin_INCLUDE_DIRS} 117 | ) 118 | 119 | ## Declare a C++ library 120 | # add_library(${PROJECT_NAME} 121 | # src/${PROJECT_NAME}/fdm_printer_bringup.cpp 122 | # ) 123 | 124 | ## Add cmake target dependencies of the library 125 | ## as an example, code may need to be generated before libraries 126 | ## either from message generation or dynamic reconfigure 127 | # add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 128 | 129 | ## Declare a C++ executable 130 | ## With catkin_make all packages are built within a single CMake context 131 | ## The recommended prefix ensures that target names across packages don't collide 132 | # add_executable(${PROJECT_NAME}_node src/fdm_printer_bringup_node.cpp) 133 | 134 | ## Rename C++ executable without prefix 135 | ## The above recommended prefix causes long target names, the following renames the 136 | ## target back to the shorter version for ease of user use 137 | ## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" 138 | # set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") 139 | 140 | ## Add cmake target dependencies of the executable 141 | ## same as for the library above 142 | # add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 143 | 144 | ## Specify libraries to link a library or executable target against 145 | # target_link_libraries(${PROJECT_NAME}_node 146 | # ${catkin_LIBRARIES} 147 | # ) 148 | 149 | ############# 150 | ## Install ## 151 | ############# 152 | 153 | # all install targets should use catkin DESTINATION variables 154 | # See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html 155 | 156 | ## Mark executable scripts (Python etc.) for installation 157 | ## in contrast to setup.py, you can choose the destination 158 | # install(PROGRAMS 159 | # scripts/my_python_script 160 | # DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 161 | # ) 162 | 163 | ## Mark executables for installation 164 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html 165 | # install(TARGETS ${PROJECT_NAME}_node 166 | # RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 167 | # ) 168 | 169 | ## Mark libraries for installation 170 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html 171 | # install(TARGETS ${PROJECT_NAME} 172 | # ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 173 | # LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 174 | # RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} 175 | # ) 176 | 177 | ## Mark cpp header files for installation 178 | # install(DIRECTORY include/${PROJECT_NAME}/ 179 | # DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 180 | # FILES_MATCHING PATTERN "*.h" 181 | # PATTERN ".svn" EXCLUDE 182 | # ) 183 | 184 | ## Mark other files for installation (e.g. launch and bag files, etc.) 185 | # install(FILES 186 | # # myfile1 187 | # # myfile2 188 | # DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} 189 | # ) 190 | 191 | ############# 192 | ## Testing ## 193 | ############# 194 | 195 | ## Add gtest based cpp test target and link libraries 196 | # catkin_add_gtest(${PROJECT_NAME}-test test/test_fdm_printer_bringup.cpp) 197 | # if(TARGET ${PROJECT_NAME}-test) 198 | # target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) 199 | # endif() 200 | 201 | ## Add folders to be run by python nosetests 202 | # catkin_add_nosetests(test) 203 | -------------------------------------------------------------------------------- /gelsight_simulation/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.3) 2 | project(gelsight_simulation) 3 | 4 | ## Compile as C++11, supported in ROS Kinetic and newer 5 | # add_compile_options(-std=c++11) 6 | 7 | ## Find catkin macros and libraries 8 | ## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) 9 | ## is used, also find other catkin packages 10 | find_package(catkin REQUIRED) 11 | 12 | ## System dependencies are found with CMake's conventions 13 | # find_package(Boost REQUIRED COMPONENTS system) 14 | 15 | 16 | ## Uncomment this if the package has a setup.py. This macro ensures 17 | ## modules and global scripts declared therein get installed 18 | ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html 19 | # catkin_python_setup() 20 | 21 | ################################################ 22 | ## Declare ROS messages, services and actions ## 23 | ################################################ 24 | 25 | ## To declare and build messages, services or actions from within this 26 | ## package, follow these steps: 27 | ## * Let MSG_DEP_SET be the set of packages whose message types you use in 28 | ## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). 29 | ## * In the file package.xml: 30 | ## * add a build_depend tag for "message_generation" 31 | ## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET 32 | ## * If MSG_DEP_SET isn't empty the following dependency has been pulled in 33 | ## but can be declared for certainty nonetheless: 34 | ## * add a exec_depend tag for "message_runtime" 35 | ## * In this file (CMakeLists.txt): 36 | ## * add "message_generation" and every package in MSG_DEP_SET to 37 | ## find_package(catkin REQUIRED COMPONENTS ...) 38 | ## * add "message_runtime" and every package in MSG_DEP_SET to 39 | ## catkin_package(CATKIN_DEPENDS ...) 40 | ## * uncomment the add_*_files sections below as needed 41 | ## and list every .msg/.srv/.action file to be processed 42 | ## * uncomment the generate_messages entry below 43 | ## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) 44 | 45 | ## Generate messages in the 'msg' folder 46 | # add_message_files( 47 | # FILES 48 | # Message1.msg 49 | # Message2.msg 50 | # ) 51 | 52 | ## Generate services in the 'srv' folder 53 | # add_service_files( 54 | # FILES 55 | # Service1.srv 56 | # Service2.srv 57 | # ) 58 | 59 | ## Generate actions in the 'action' folder 60 | # add_action_files( 61 | # FILES 62 | # Action1.action 63 | # Action2.action 64 | # ) 65 | 66 | ## Generate added messages and services with any dependencies listed here 67 | # generate_messages( 68 | # DEPENDENCIES 69 | # std_msgs # Or other packages containing msgs 70 | # ) 71 | 72 | ################################################ 73 | ## Declare ROS dynamic reconfigure parameters ## 74 | ################################################ 75 | 76 | ## To declare and build dynamic reconfigure parameters within this 77 | ## package, follow these steps: 78 | ## * In the file package.xml: 79 | ## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" 80 | ## * In this file (CMakeLists.txt): 81 | ## * add "dynamic_reconfigure" to 82 | ## find_package(catkin REQUIRED COMPONENTS ...) 83 | ## * uncomment the "generate_dynamic_reconfigure_options" section below 84 | ## and list every .cfg file to be processed 85 | 86 | ## Generate dynamic reconfigure parameters in the 'cfg' folder 87 | # generate_dynamic_reconfigure_options( 88 | # cfg/DynReconf1.cfg 89 | # cfg/DynReconf2.cfg 90 | # ) 91 | 92 | ################################### 93 | ## catkin specific configuration ## 94 | ################################### 95 | ## The catkin_package macro generates cmake config files for your package 96 | ## Declare things to be passed to dependent projects 97 | ## INCLUDE_DIRS: uncomment this if your package contains header files 98 | ## LIBRARIES: libraries you create in this project that dependent projects also need 99 | ## CATKIN_DEPENDS: catkin_packages dependent projects also need 100 | ## DEPENDS: system dependencies of this project that dependent projects also need 101 | catkin_package( 102 | # INCLUDE_DIRS include 103 | # LIBRARIES gelsight_simulation 104 | # CATKIN_DEPENDS other_catkin_pkg 105 | # DEPENDS system_lib 106 | ) 107 | 108 | ########### 109 | ## Build ## 110 | ########### 111 | 112 | ## Specify additional locations of header files 113 | ## Your package locations should be listed before other locations 114 | include_directories( 115 | # include 116 | # ${catkin_INCLUDE_DIRS} 117 | ) 118 | 119 | ## Declare a C++ library 120 | # add_library(${PROJECT_NAME} 121 | # src/${PROJECT_NAME}/gelsight_simulation.cpp 122 | # ) 123 | 124 | ## Add cmake target dependencies of the library 125 | ## as an example, code may need to be generated before libraries 126 | ## either from message generation or dynamic reconfigure 127 | # add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 128 | 129 | ## Declare a C++ executable 130 | ## With catkin_make all packages are built within a single CMake context 131 | ## The recommended prefix ensures that target names across packages don't collide 132 | # add_executable(${PROJECT_NAME}_node src/gelsight_simulation_node.cpp) 133 | 134 | ## Rename C++ executable without prefix 135 | ## The above recommended prefix causes long target names, the following renames the 136 | ## target back to the shorter version for ease of user use 137 | ## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" 138 | # set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") 139 | 140 | ## Add cmake target dependencies of the executable 141 | ## same as for the library above 142 | # add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 143 | 144 | ## Specify libraries to link a library or executable target against 145 | # target_link_libraries(${PROJECT_NAME}_node 146 | # ${catkin_LIBRARIES} 147 | # ) 148 | 149 | ############# 150 | ## Install ## 151 | ############# 152 | 153 | # all install targets should use catkin DESTINATION variables 154 | # See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html 155 | 156 | ## Mark executable scripts (Python etc.) for installation 157 | ## in contrast to setup.py, you can choose the destination 158 | # install(PROGRAMS 159 | # scripts/my_python_script 160 | # DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 161 | # ) 162 | 163 | ## Mark executables for installation 164 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html 165 | # install(TARGETS ${PROJECT_NAME}_node 166 | # RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 167 | # ) 168 | 169 | ## Mark libraries for installation 170 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html 171 | # install(TARGETS ${PROJECT_NAME} 172 | # ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 173 | # LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 174 | # RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} 175 | # ) 176 | 177 | ## Mark cpp header files for installation 178 | # install(DIRECTORY include/${PROJECT_NAME}/ 179 | # DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 180 | # FILES_MATCHING PATTERN "*.h" 181 | # PATTERN ".svn" EXCLUDE 182 | # ) 183 | 184 | ## Mark other files for installation (e.g. launch and bag files, etc.) 185 | # install(FILES 186 | # # myfile1 187 | # # myfile2 188 | # DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} 189 | # ) 190 | 191 | ############# 192 | ## Testing ## 193 | ############# 194 | 195 | ## Add gtest based cpp test target and link libraries 196 | # catkin_add_gtest(${PROJECT_NAME}-test test/test_gelsight_simulation.cpp) 197 | # if(TARGET ${PROJECT_NAME}-test) 198 | # target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) 199 | # endif() 200 | 201 | ## Add folders to be run by python nosetests 202 | # catkin_add_nosetests(test) 203 | -------------------------------------------------------------------------------- /gelsight_description/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.3) 2 | project(gelsight_description) 3 | 4 | ## Compile as C++11, supported in ROS Kinetic and newer 5 | # add_compile_options(-std=c++11) 6 | 7 | ## Find catkin macros and libraries 8 | ## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) 9 | ## is used, also find other catkin packages 10 | find_package(catkin REQUIRED) 11 | 12 | ## System dependencies are found with CMake's conventions 13 | # find_package(Boost REQUIRED COMPONENTS system) 14 | 15 | 16 | ## Uncomment this if the package has a setup.py. This macro ensures 17 | ## modules and global scripts declared therein get installed 18 | ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html 19 | # catkin_python_setup() 20 | 21 | ################################################ 22 | ## Declare ROS messages, services and actions ## 23 | ################################################ 24 | 25 | ## To declare and build messages, services or actions from within this 26 | ## package, follow these steps: 27 | ## * Let MSG_DEP_SET be the set of packages whose message types you use in 28 | ## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). 29 | ## * In the file package.xml: 30 | ## * add a build_depend tag for "message_generation" 31 | ## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET 32 | ## * If MSG_DEP_SET isn't empty the following dependency has been pulled in 33 | ## but can be declared for certainty nonetheless: 34 | ## * add a exec_depend tag for "message_runtime" 35 | ## * In this file (CMakeLists.txt): 36 | ## * add "message_generation" and every package in MSG_DEP_SET to 37 | ## find_package(catkin REQUIRED COMPONENTS ...) 38 | ## * add "message_runtime" and every package in MSG_DEP_SET to 39 | ## catkin_package(CATKIN_DEPENDS ...) 40 | ## * uncomment the add_*_files sections below as needed 41 | ## and list every .msg/.srv/.action file to be processed 42 | ## * uncomment the generate_messages entry below 43 | ## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) 44 | 45 | ## Generate messages in the 'msg' folder 46 | # add_message_files( 47 | # FILES 48 | # Message1.msg 49 | # Message2.msg 50 | # ) 51 | 52 | ## Generate services in the 'srv' folder 53 | # add_service_files( 54 | # FILES 55 | # Service1.srv 56 | # Service2.srv 57 | # ) 58 | 59 | ## Generate actions in the 'action' folder 60 | # add_action_files( 61 | # FILES 62 | # Action1.action 63 | # Action2.action 64 | # ) 65 | 66 | ## Generate added messages and services with any dependencies listed here 67 | # generate_messages( 68 | # DEPENDENCIES 69 | # std_msgs # Or other packages containing msgs 70 | # ) 71 | 72 | ################################################ 73 | ## Declare ROS dynamic reconfigure parameters ## 74 | ################################################ 75 | 76 | ## To declare and build dynamic reconfigure parameters within this 77 | ## package, follow these steps: 78 | ## * In the file package.xml: 79 | ## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" 80 | ## * In this file (CMakeLists.txt): 81 | ## * add "dynamic_reconfigure" to 82 | ## find_package(catkin REQUIRED COMPONENTS ...) 83 | ## * uncomment the "generate_dynamic_reconfigure_options" section below 84 | ## and list every .cfg file to be processed 85 | 86 | ## Generate dynamic reconfigure parameters in the 'cfg' folder 87 | # generate_dynamic_reconfigure_options( 88 | # cfg/DynReconf1.cfg 89 | # cfg/DynReconf2.cfg 90 | # ) 91 | 92 | ################################### 93 | ## catkin specific configuration ## 94 | ################################### 95 | ## The catkin_package macro generates cmake config files for your package 96 | ## Declare things to be passed to dependent projects 97 | ## INCLUDE_DIRS: uncomment this if your package contains header files 98 | ## LIBRARIES: libraries you create in this project that dependent projects also need 99 | ## CATKIN_DEPENDS: catkin_packages dependent projects also need 100 | ## DEPENDS: system dependencies of this project that dependent projects also need 101 | catkin_package( 102 | # INCLUDE_DIRS include 103 | # LIBRARIES gelsight_description 104 | # CATKIN_DEPENDS other_catkin_pkg 105 | # DEPENDS system_lib 106 | ) 107 | 108 | ########### 109 | ## Build ## 110 | ########### 111 | 112 | ## Specify additional locations of header files 113 | ## Your package locations should be listed before other locations 114 | include_directories( 115 | # include 116 | # ${catkin_INCLUDE_DIRS} 117 | ) 118 | 119 | ## Declare a C++ library 120 | # add_library(${PROJECT_NAME} 121 | # src/${PROJECT_NAME}/gelsight_description.cpp 122 | # ) 123 | 124 | ## Add cmake target dependencies of the library 125 | ## as an example, code may need to be generated before libraries 126 | ## either from message generation or dynamic reconfigure 127 | # add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 128 | 129 | ## Declare a C++ executable 130 | ## With catkin_make all packages are built within a single CMake context 131 | ## The recommended prefix ensures that target names across packages don't collide 132 | # add_executable(${PROJECT_NAME}_node src/gelsight_description_node.cpp) 133 | 134 | ## Rename C++ executable without prefix 135 | ## The above recommended prefix causes long target names, the following renames the 136 | ## target back to the shorter version for ease of user use 137 | ## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" 138 | # set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") 139 | 140 | ## Add cmake target dependencies of the executable 141 | ## same as for the library above 142 | # add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 143 | 144 | ## Specify libraries to link a library or executable target against 145 | # target_link_libraries(${PROJECT_NAME}_node 146 | # ${catkin_LIBRARIES} 147 | # ) 148 | 149 | ############# 150 | ## Install ## 151 | ############# 152 | 153 | # all install targets should use catkin DESTINATION variables 154 | # See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html 155 | 156 | ## Mark executable scripts (Python etc.) for installation 157 | ## in contrast to setup.py, you can choose the destination 158 | # install(PROGRAMS 159 | # scripts/my_python_script 160 | # DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 161 | # ) 162 | 163 | ## Mark executables for installation 164 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html 165 | # install(TARGETS ${PROJECT_NAME}_node 166 | # RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 167 | # ) 168 | 169 | ## Mark libraries for installation 170 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html 171 | # install(TARGETS ${PROJECT_NAME} 172 | # ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 173 | # LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 174 | # RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} 175 | # ) 176 | 177 | ## Mark cpp header files for installation 178 | # install(DIRECTORY include/${PROJECT_NAME}/ 179 | # DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 180 | # FILES_MATCHING PATTERN "*.h" 181 | # PATTERN ".svn" EXCLUDE 182 | # ) 183 | 184 | ## Mark other files for installation (e.g. launch and bag files, etc.) 185 | # install(FILES 186 | # # myfile1 187 | # # myfile2 188 | # DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} 189 | # ) 190 | 191 | ############# 192 | ## Testing ## 193 | ############# 194 | 195 | ## Add gtest based cpp test target and link libraries 196 | # catkin_add_gtest(${PROJECT_NAME}-test test/test_gelsight_description.cpp) 197 | # if(TARGET ${PROJECT_NAME}-test) 198 | # target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) 199 | # endif() 200 | 201 | ## Add folders to be run by python nosetests 202 | # catkin_add_nosetests(test) 203 | -------------------------------------------------------------------------------- /fdm_printer_description/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.3) 2 | project(fdm_printer_description) 3 | 4 | ## Compile as C++11, supported in ROS Kinetic and newer 5 | # add_compile_options(-std=c++11) 6 | 7 | ## Find catkin macros and libraries 8 | ## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) 9 | ## is used, also find other catkin packages 10 | find_package(catkin REQUIRED) 11 | 12 | ## System dependencies are found with CMake's conventions 13 | # find_package(Boost REQUIRED COMPONENTS system) 14 | 15 | 16 | ## Uncomment this if the package has a setup.py. This macro ensures 17 | ## modules and global scripts declared therein get installed 18 | ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html 19 | # catkin_python_setup() 20 | 21 | ################################################ 22 | ## Declare ROS messages, services and actions ## 23 | ################################################ 24 | 25 | ## To declare and build messages, services or actions from within this 26 | ## package, follow these steps: 27 | ## * Let MSG_DEP_SET be the set of packages whose message types you use in 28 | ## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). 29 | ## * In the file package.xml: 30 | ## * add a build_depend tag for "message_generation" 31 | ## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET 32 | ## * If MSG_DEP_SET isn't empty the following dependency has been pulled in 33 | ## but can be declared for certainty nonetheless: 34 | ## * add a exec_depend tag for "message_runtime" 35 | ## * In this file (CMakeLists.txt): 36 | ## * add "message_generation" and every package in MSG_DEP_SET to 37 | ## find_package(catkin REQUIRED COMPONENTS ...) 38 | ## * add "message_runtime" and every package in MSG_DEP_SET to 39 | ## catkin_package(CATKIN_DEPENDS ...) 40 | ## * uncomment the add_*_files sections below as needed 41 | ## and list every .msg/.srv/.action file to be processed 42 | ## * uncomment the generate_messages entry below 43 | ## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) 44 | 45 | ## Generate messages in the 'msg' folder 46 | # add_message_files( 47 | # FILES 48 | # Message1.msg 49 | # Message2.msg 50 | # ) 51 | 52 | ## Generate services in the 'srv' folder 53 | # add_service_files( 54 | # FILES 55 | # Service1.srv 56 | # Service2.srv 57 | # ) 58 | 59 | ## Generate actions in the 'action' folder 60 | # add_action_files( 61 | # FILES 62 | # Action1.action 63 | # Action2.action 64 | # ) 65 | 66 | ## Generate added messages and services with any dependencies listed here 67 | # generate_messages( 68 | # DEPENDENCIES 69 | # std_msgs # Or other packages containing msgs 70 | # ) 71 | 72 | ################################################ 73 | ## Declare ROS dynamic reconfigure parameters ## 74 | ################################################ 75 | 76 | ## To declare and build dynamic reconfigure parameters within this 77 | ## package, follow these steps: 78 | ## * In the file package.xml: 79 | ## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" 80 | ## * In this file (CMakeLists.txt): 81 | ## * add "dynamic_reconfigure" to 82 | ## find_package(catkin REQUIRED COMPONENTS ...) 83 | ## * uncomment the "generate_dynamic_reconfigure_options" section below 84 | ## and list every .cfg file to be processed 85 | 86 | ## Generate dynamic reconfigure parameters in the 'cfg' folder 87 | # generate_dynamic_reconfigure_options( 88 | # cfg/DynReconf1.cfg 89 | # cfg/DynReconf2.cfg 90 | # ) 91 | 92 | ################################### 93 | ## catkin specific configuration ## 94 | ################################### 95 | ## The catkin_package macro generates cmake config files for your package 96 | ## Declare things to be passed to dependent projects 97 | ## INCLUDE_DIRS: uncomment this if your package contains header files 98 | ## LIBRARIES: libraries you create in this project that dependent projects also need 99 | ## CATKIN_DEPENDS: catkin_packages dependent projects also need 100 | ## DEPENDS: system dependencies of this project that dependent projects also need 101 | catkin_package( 102 | # INCLUDE_DIRS include 103 | # LIBRARIES fdm_printer_description 104 | # CATKIN_DEPENDS other_catkin_pkg 105 | # DEPENDS system_lib 106 | ) 107 | 108 | ########### 109 | ## Build ## 110 | ########### 111 | 112 | ## Specify additional locations of header files 113 | ## Your package locations should be listed before other locations 114 | include_directories( 115 | # include 116 | # ${catkin_INCLUDE_DIRS} 117 | ) 118 | 119 | ## Declare a C++ library 120 | # add_library(${PROJECT_NAME} 121 | # src/${PROJECT_NAME}/fdm_printer_description.cpp 122 | # ) 123 | 124 | ## Add cmake target dependencies of the library 125 | ## as an example, code may need to be generated before libraries 126 | ## either from message generation or dynamic reconfigure 127 | # add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 128 | 129 | ## Declare a C++ executable 130 | ## With catkin_make all packages are built within a single CMake context 131 | ## The recommended prefix ensures that target names across packages don't collide 132 | # add_executable(${PROJECT_NAME}_node src/fdm_printer_description_node.cpp) 133 | 134 | ## Rename C++ executable without prefix 135 | ## The above recommended prefix causes long target names, the following renames the 136 | ## target back to the shorter version for ease of user use 137 | ## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" 138 | # set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") 139 | 140 | ## Add cmake target dependencies of the executable 141 | ## same as for the library above 142 | # add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 143 | 144 | ## Specify libraries to link a library or executable target against 145 | # target_link_libraries(${PROJECT_NAME}_node 146 | # ${catkin_LIBRARIES} 147 | # ) 148 | 149 | ############# 150 | ## Install ## 151 | ############# 152 | 153 | # all install targets should use catkin DESTINATION variables 154 | # See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html 155 | 156 | ## Mark executable scripts (Python etc.) for installation 157 | ## in contrast to setup.py, you can choose the destination 158 | # install(PROGRAMS 159 | # scripts/my_python_script 160 | # DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 161 | # ) 162 | 163 | ## Mark executables for installation 164 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html 165 | # install(TARGETS ${PROJECT_NAME}_node 166 | # RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 167 | # ) 168 | 169 | ## Mark libraries for installation 170 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html 171 | # install(TARGETS ${PROJECT_NAME} 172 | # ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 173 | # LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 174 | # RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} 175 | # ) 176 | 177 | ## Mark cpp header files for installation 178 | # install(DIRECTORY include/${PROJECT_NAME}/ 179 | # DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 180 | # FILES_MATCHING PATTERN "*.h" 181 | # PATTERN ".svn" EXCLUDE 182 | # ) 183 | 184 | ## Mark other files for installation (e.g. launch and bag files, etc.) 185 | # install(FILES 186 | # # myfile1 187 | # # myfile2 188 | # DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} 189 | # ) 190 | 191 | ############# 192 | ## Testing ## 193 | ############# 194 | 195 | ## Add gtest based cpp test target and link libraries 196 | # catkin_add_gtest(${PROJECT_NAME}-test test/test_fdm_printer_description.cpp) 197 | # if(TARGET ${PROJECT_NAME}-test) 198 | # target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) 199 | # endif() 200 | 201 | ## Add folders to be run by python nosetests 202 | # catkin_add_nosetests(test) 203 | -------------------------------------------------------------------------------- /fdm_printer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.3) 2 | project(fdm_printer) 3 | 4 | ## Compile as C++11, supported in ROS Kinetic and newer 5 | # add_compile_options(-std=c++11) 6 | 7 | ## Find catkin macros and libraries 8 | ## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) 9 | ## is used, also find other catkin packages 10 | find_package(catkin REQUIRED COMPONENTS 11 | controller_interface 12 | hardware_interface 13 | effort_controllers 14 | position_controllers 15 | controller_manager 16 | pluginlib 17 | roscpp 18 | serial 19 | ) 20 | 21 | ## System dependencies are found with CMake's conventions 22 | # find_package(Boost REQUIRED COMPONENTS system) 23 | 24 | 25 | ## Uncomment this if the package has a setup.py. This macro ensures 26 | ## modules and global scripts declared therein get installed 27 | ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html 28 | # catkin_python_setup() 29 | 30 | ################################################ 31 | ## Declare ROS messages, services and actions ## 32 | ################################################ 33 | 34 | ## To declare and build messages, services or actions from within this 35 | ## package, follow these steps: 36 | ## * Let MSG_DEP_SET be the set of packages whose message types you use in 37 | ## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). 38 | ## * In the file package.xml: 39 | ## * add a build_depend tag for "message_generation" 40 | ## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET 41 | ## * If MSG_DEP_SET isn't empty the following dependency has been pulled in 42 | ## but can be declared for certainty nonetheless: 43 | ## * add a exec_depend tag for "message_runtime" 44 | ## * In this file (CMakeLists.txt): 45 | ## * add "message_generation" and every package in MSG_DEP_SET to 46 | ## find_package(catkin REQUIRED COMPONENTS ...) 47 | ## * add "message_runtime" and every package in MSG_DEP_SET to 48 | ## catkin_package(CATKIN_DEPENDS ...) 49 | ## * uncomment the add_*_files sections below as needed 50 | ## and list every .msg/.srv/.action file to be processed 51 | ## * uncomment the generate_messages entry below 52 | ## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) 53 | 54 | ## Generate messages in the 'msg' folder 55 | # add_message_files( 56 | # FILES 57 | # Message1.msg 58 | # Message2.msg 59 | # ) 60 | 61 | ## Generate services in the 'srv' folder 62 | # add_service_files( 63 | # FILES 64 | # Service1.srv 65 | # Service2.srv 66 | # ) 67 | 68 | ## Generate actions in the 'action' folder 69 | # add_action_files( 70 | # FILES 71 | # Action1.action 72 | # Action2.action 73 | # ) 74 | 75 | ## Generate added messages and services with any dependencies listed here 76 | # generate_messages( 77 | # DEPENDENCIES 78 | # std_msgs # Or other packages containing msgs 79 | # ) 80 | 81 | ################################################ 82 | ## Declare ROS dynamic reconfigure parameters ## 83 | ################################################ 84 | 85 | ## To declare and build dynamic reconfigure parameters within this 86 | ## package, follow these steps: 87 | ## * In the file package.xml: 88 | ## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" 89 | ## * In this file (CMakeLists.txt): 90 | ## * add "dynamic_reconfigure" to 91 | ## find_package(catkin REQUIRED COMPONENTS ...) 92 | ## * uncomment the "generate_dynamic_reconfigure_options" section below 93 | ## and list every .cfg file to be processed 94 | 95 | ## Generate dynamic reconfigure parameters in the 'cfg' folder 96 | # generate_dynamic_reconfigure_options( 97 | # cfg/DynReconf1.cfg 98 | # cfg/DynReconf2.cfg 99 | # ) 100 | 101 | ################################### 102 | ## catkin specific configuration ## 103 | ################################### 104 | ## The catkin_package macro generates cmake config files for your package 105 | ## Declare things to be passed to dependent projects 106 | ## INCLUDE_DIRS: uncomment this if your package contains header files 107 | ## LIBRARIES: libraries you create in this project that dependent projects also need 108 | ## CATKIN_DEPENDS: catkin_packages dependent projects also need 109 | ## DEPENDS: system dependencies of this project that dependent projects also need 110 | catkin_package( 111 | # INCLUDE_DIRS include 112 | # LIBRARIES fdm_printer 113 | # CATKIN_DEPENDS other_catkin_pkg 114 | # DEPENDS system_lib 115 | ) 116 | 117 | ########### 118 | ## Build ## 119 | ########### 120 | 121 | ## Specify additional locations of header files 122 | ## Your package locations should be listed before other locations 123 | include_directories( 124 | # include 125 | ${catkin_INCLUDE_DIRS} 126 | ) 127 | 128 | ## Declare a C++ library 129 | # add_library(${PROJECT_NAME} 130 | # src/${PROJECT_NAME}/fdm_printer.cpp 131 | # ) 132 | add_library(position_controller_lib src/position_controller.cpp) 133 | 134 | 135 | ## Add cmake target dependencies of the library 136 | ## as an example, code may need to be generated before libraries 137 | ## either from message generation or dynamic reconfigure 138 | # add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 139 | 140 | ## Declare a C++ executable 141 | ## With catkin_make all packages are built within a single CMake context 142 | ## The recommended prefix ensures that target names across packages don't collide 143 | # add_executable(${PROJECT_NAME}_node src/fdm_printer_node.cpp) 144 | add_executable(${PROJECT_NAME}_hardware_interface src/fdm_printer_hardware_interface.cpp) 145 | add_dependencies(${PROJECT_NAME}_hardware_interface ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 146 | target_link_libraries(${PROJECT_NAME}_hardware_interface ${catkin_LIBRARIES}) 147 | 148 | ## Rename C++ executable without prefix 149 | ## The above recommended prefix causes long target names, the following renames the 150 | ## target back to the shorter version for ease of user use 151 | ## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" 152 | # set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") 153 | 154 | ## Add cmake target dependencies of the executable 155 | ## same as for the library above 156 | # add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) 157 | 158 | 159 | ## Specify libraries to link a library or executable target against 160 | # target_link_libraries(${PROJECT_NAME}_node 161 | # ${catkin_LIBRARIES} 162 | # ) 163 | target_link_libraries(position_controller_lib ${catkin_LIBRARIES}) 164 | 165 | ############# 166 | ## Install ## 167 | ############# 168 | 169 | # all install targets should use catkin DEST0 INATION variables 170 | # See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html 171 | 172 | ## Mark executable scripts (Python etc.) for installation 173 | ## in contrast to setup.py, you can choose the destination 174 | # install(PROGRAMS 175 | # scripts/my_python_script 176 | # DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 177 | # ) 178 | 179 | ## Mark executables for installation 180 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html 181 | # install(TARGETS ${PROJECT_NAME}_node 182 | # RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} 183 | # ) 184 | 185 | ## Mark libraries for installation 186 | ## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html 187 | # install(TARGETS ${PROJECT_NAME} 188 | # ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 189 | # LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} 190 | # RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} 191 | # ) 192 | 193 | ## Mark cpp header files for installation 194 | # install(DIRECTORY include/${PROJECT_NAME}/ 195 | # DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 196 | # FILES_MATCHING PATTERN "*.h" 197 | # PATTERN ".svn" EXCLUDE 198 | # ) 199 | 200 | ## Mark other files for installation (e.g. launch and bag files, etc.) 201 | # install(FILES 202 | # # myfile1 203 | # # myfile2 204 | # DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} 205 | # ) 206 | 207 | ############# 208 | ## Testing ## 209 | ############# 210 | 211 | ## Add gtest based cpp test target and link libraries 212 | # catkin_add_gtest(${PROJECT_NAME}-test test/test_fdm_printer.cpp) 213 | # if(TARGET ${PROJECT_NAME}-test) 214 | # target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) 215 | # endif() 216 | 217 | ## Add folders to be run by python nosetests 218 | # catkin_add_nosetests(test) 219 | -------------------------------------------------------------------------------- /gelsight_gazebo/src/gelsight_driver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import rospy 3 | import rospkg 4 | 5 | from sensor_msgs.msg import Image 6 | from cv_bridge import CvBridge, CvBridgeError 7 | 8 | bridge = CvBridge() 9 | rospack = rospkg.RosPack() 10 | PKG_PATH = rospack.get_path('gelsight_gazebo') 11 | import matplotlib.pyplot as plt 12 | 13 | import cv2 14 | import numpy as np 15 | 16 | import scipy.ndimage.filters as fi 17 | 18 | """ 19 | Utils section 20 | """ 21 | 22 | 23 | def show_normalized_img(name, img): 24 | draw = img.copy() 25 | draw -= np.min(draw) 26 | draw = draw / np.max(draw) 27 | cv2.imshow(name, draw) 28 | return draw 29 | 30 | 31 | def gkern2(kernlen=21, nsig=3): 32 | """Returns a 2D Gaussian kernel array.""" 33 | 34 | # create nxn zeros 35 | inp = np.zeros((kernlen, kernlen)) 36 | # set element at the middle to one, a dirac delta 37 | inp[kernlen // 2, kernlen // 2] = 1 38 | # gaussian-smooth the dirac, resulting in a gaussian filter mask 39 | return fi.gaussian_filter(inp, nsig) 40 | 41 | 42 | def gaus_noise(image, sigma): 43 | row, col = image.shape 44 | mean = 0 45 | gauss = np.random.normal(mean, sigma, (row, col)) 46 | gauss = gauss.reshape(row, col) 47 | noisy = image + gauss 48 | return noisy 49 | 50 | 51 | def derivative(mat, direction): 52 | assert (direction == 'x' or direction == 'y'), "The derivative direction must be 'x' or 'y'" 53 | kernel = None 54 | if direction == 'x': 55 | kernel = [[-1.0, 0.0, 1.0]] 56 | elif direction == 'y': 57 | kernel = [[-1.0], [0.0], [1.0]] 58 | kernel = np.array(kernel, dtype=np.float64) 59 | return cv2.filter2D(mat, -1, kernel) / 2.0 60 | 61 | 62 | def tangent(mat): 63 | dx = derivative(mat, 'x') 64 | dy = derivative(mat, 'y') 65 | img_shape = np.shape(mat) 66 | _1 = np.repeat([1.0], img_shape[0] * img_shape[1]).reshape(img_shape).astype(dx.dtype) 67 | unormalized = cv2.merge((-dx, -dy, _1)) 68 | norms = np.linalg.norm(unormalized, axis=2) 69 | return (unormalized / np.repeat(norms[:, :, np.newaxis], 3, axis=2)) 70 | 71 | 72 | def solid_color_img(color, size): 73 | image = np.zeros(size + (3,), np.float64) 74 | image[:] = color 75 | return image 76 | 77 | 78 | def add_overlay(rgb, alpha, color): 79 | s = np.shape(alpha) 80 | 81 | opacity3 = np.repeat(alpha, 3).reshape((s[0], s[1], 3)) # * 10.0 82 | 83 | overlay = solid_color_img(color, s) 84 | 85 | foreground = opacity3 * overlay 86 | background = (1.0 - opacity3) * rgb.astype(np.float64) 87 | res = background + foreground 88 | 89 | res[res > 255.0] = 255.0 90 | res[res < 0.0] = 0.0 91 | res = res.astype(np.uint8) 92 | 93 | return res 94 | 95 | 96 | """ 97 | GelSight Simulation 98 | """ 99 | 100 | 101 | class SimulationApproach: 102 | 103 | def __init__(self, **config): 104 | self.light_sources = config['light_sources'] 105 | self.background = config['background_img'] 106 | self.px2m_ratio = config['px2m_ratio'] 107 | self.elastomer_thickness = config['elastomer_thickness'] 108 | self.min_depth = config['min_depth'] 109 | 110 | self.default_ks = 0.15 111 | self.default_kd = 0.5 112 | self.default_alpha = 5 113 | 114 | self.ka = config['ka'] or 0.8 115 | 116 | self.texture_sigma = config['texture_sigma'] or 0.00001 117 | self.t = config['t'] if 't' in config else 3 118 | self.sigma = config['sigma'] if 'sigma' in config else 7 119 | self.kernel_size = config['sigma'] if 'sigma' in config else 21 120 | 121 | self.max_depth = self.min_depth + self.elastomer_thickness 122 | 123 | def protrusion_map(self, original, not_in_touch): 124 | protrusion_map = np.copy(original) 125 | protrusion_map[not_in_touch >= self.max_depth] = self.max_depth 126 | return protrusion_map 127 | 128 | def segments(self, depth_map): 129 | not_in_touch = np.copy(depth_map) 130 | not_in_touch[not_in_touch < self.max_depth] = 0.0 131 | not_in_touch[not_in_touch >= self.max_depth] = 1.0 132 | 133 | in_touch = 1 - not_in_touch 134 | 135 | return not_in_touch, in_touch 136 | 137 | def internal_shadow(self, elastomer_depth): 138 | elastomer_depth_inv = self.max_depth - elastomer_depth 139 | elastomer_depth_inv = np.interp(elastomer_depth_inv, (0, self.elastomer_thickness), (0.0, 1.0)) 140 | return elastomer_depth_inv 141 | 142 | def apply_elastic_deformation_v1(self, protrusion_depth, not_in_touch, in_touch): 143 | kernel = gkern2(15, 7) 144 | deformation = self.max_depth - protrusion_depth 145 | 146 | for i in range(5): 147 | # # cv2.waitKey(10) 148 | deformation = cv2.filter2D(deformation, -1, kernel) 149 | # # show_normalized_img('deformation', deformation) 150 | # return deformation 151 | return 30 * -deformation * not_in_touch + (protrusion_depth * in_touch) 152 | 153 | def apply_elastic_deformation(self, protrusion_depth, not_in_touch, in_touch): 154 | protrusion_depth = - (protrusion_depth - self.max_depth) 155 | 156 | kernel = gkern2(self.kernel_size, self.sigma) 157 | deformation = protrusion_depth 158 | 159 | deformation2 = protrusion_depth 160 | kernel2 = gkern2(52, 9) 161 | 162 | for i in range(self.t): 163 | deformation_ = cv2.filter2D(deformation, -1, kernel) 164 | r = np.max(protrusion_depth) / np.max(deformation_) if np.max(deformation_) > 0 else 1 165 | deformation = np.maximum(r * deformation_, protrusion_depth) 166 | 167 | deformation2_ = cv2.filter2D(deformation2, -1, kernel2) 168 | r = np.max(protrusion_depth) / np.max(deformation2_) if np.max(deformation2_) > 0 else 1 169 | deformation2 = np.maximum(r * deformation2_, protrusion_depth) 170 | 171 | deformation_v1 = self.apply_elastic_deformation_v1(protrusion_depth, not_in_touch, in_touch) 172 | 173 | # deformation2 = protrusion_depth 174 | # 175 | for i in range(self.t): 176 | deformation_ = cv2.filter2D(deformation2, -1, kernel) 177 | r = np.max(protrusion_depth) / np.max(deformation_) if np.max(deformation_) > 0 else 1 178 | deformation2 = np.maximum(r * deformation_, protrusion_depth) 179 | 180 | # 181 | 182 | # for i in range(3): 183 | # deformation3 = protrusion_depth 184 | # kernel3 = gkern2(21, 7) 185 | # for i in range(3): 186 | # deformation3_ = cv2.filter2D(deformation3, -1, kernel3) 187 | # r = np.max(protrusion_depth) / np.max(deformation3_) if np.max(deformation3_) > 0 else 1 188 | # deformation3 = np.maximum(r * deformation3_, protrusion_depth) 189 | 190 | # 191 | # # r = np.max(protrusion_depth) / np.max(deformation) if np.max(deformation) > 0 else 1 192 | # # deformation = np.maximum(r * deformation, protrusion_depth) 193 | # # plt.axis('off') 194 | # 195 | # 196 | # plt.plot(list(range(len(protrusion_depth[150]))), -1 * protrusion_depth[240], color="gray", 197 | # label='Before Smoothing') 198 | # plt.plot(list(range(len(deformation[150]))), -1 * deformation[240], color="limegreen", linestyle='dashed', 199 | # label='Single Gaussian') 200 | # # 201 | # # plt.plot(list(range(len(deformation2[150]))), -1 * deformation[150] + deformation2[150], color='red', 202 | # # linestyle='dashed', 203 | # # label='with ratioxxxxx') 204 | # # deformation_x = -1 * deformation[150] + deformation2[150] - deformation[150] 205 | deformation_x = 2 * deformation - deformation2 206 | # 207 | # plt.plot(list(range(len(deformation[150]))), - deformation_x[240], color="darkorange", linestyle='dashed', 208 | # label='Difference of Gaussians') 209 | 210 | # plt.plot(list(range(len(deformation2[150]))), -deformation_v1[150], 211 | # color='black', 212 | # # linestyle='do', 213 | # label='Previous ') 214 | 215 | # plt.plot(list(range(len(deformation2[150]))), deformation_x[150], 216 | # color='red', 217 | # linestyle='dashed', 218 | # label='with ratioxxxxx') 219 | 220 | # tangent = lambda arr: np.array([abs(arr[i + 1] - arr[i - 1]) / 2 if i > 0 and i < len(arr) - 2 else 0 for i in 221 | # range(len(arr))]) 222 | # 223 | # t = tangent(deformation2[150]) 224 | # plt.plot(list(range(len(deformation2[240]))), 225 | # deformation[150] + (np.max(deformation2[150]) / np.max(t)) * t, 226 | # color='red', 227 | # label='After Filtering') 228 | 229 | # plt.xticks([]) 230 | # plt.yticks([]) 231 | # plt.legend() 232 | # plt.show() 233 | # plt.clf() 234 | # plt.cla() 235 | 236 | # 237 | # cv2.imwrite('protrusion.png', show_normalized_img('protrusion', protrusion_depth) * 255) 238 | # cv2.imwrite('deformation.png', show_normalized_img('deformation', deformation) * 255) 239 | 240 | return self.max_depth - deformation_x 241 | 242 | def phong_illumination(self, T, source_dir, kd, ks, alpha): 243 | dot = np.dot(T, np.array(source_dir)).astype(np.float64) 244 | difuse_l = dot * kd 245 | difuse_l[difuse_l < 0] = 0.0 246 | 247 | dot3 = np.repeat(dot[:, :, np.newaxis], 3, axis=2) 248 | 249 | R = 2.0 * dot3 * T - source_dir 250 | V = [0.0, 0.0, 1.0] 251 | 252 | spec_l = np.power(np.dot(R, V), alpha) * ks 253 | 254 | return difuse_l + spec_l 255 | 256 | def generate(self, obj_depth, return_depth=False): 257 | # print('-----------> ', np.shape(obj_depth)) 258 | # cv2.imwrite('object_depth.png', obj_depth) 259 | not_in_touch, in_touch = self.segments(obj_depth) 260 | protrusion_depth = self.protrusion_map(obj_depth, not_in_touch) 261 | elastomer_depth = self.apply_elastic_deformation(protrusion_depth, not_in_touch, in_touch) 262 | 263 | textured_elastomer_depth = gaus_noise(elastomer_depth, self.texture_sigma) 264 | 265 | out = self.ka * self.background 266 | out = add_overlay(out, self.internal_shadow(protrusion_depth), (0.0, 0.0, 0.0)) 267 | 268 | T = tangent(textured_elastomer_depth / self.px2m_ratio) 269 | # show_normalized_img('tangent', T) 270 | for light in self.light_sources: 271 | ks = light['ks'] if 'ks' in light else self.default_ks 272 | kd = light['kd'] if 'kd' in light else self.default_kd 273 | alpha = light['alpha'] if 'alpha' in light else self.default_alpha 274 | out = add_overlay(out, self.phong_illumination(T, light['position'], kd, ks, alpha), light['color']) 275 | 276 | kernel = gkern2(3, 1) 277 | out = cv2.filter2D(out, -1, kernel) 278 | 279 | # cv2.imshow('tactile img', out) 280 | # cv2.imwrite('tactile_img.png', out) 281 | # 282 | if return_depth: 283 | return out, elastomer_depth 284 | return out 285 | 286 | 287 | """ 288 | ROS Driver 289 | """ 290 | 291 | 292 | class GelSightDriver: 293 | 294 | def __init__(self, name, sim_approach): 295 | self.simulation_approach = sim_approach 296 | self.depth_img = None 297 | self.visual_img = None 298 | 299 | rospy.init_node(name) 300 | rospy.Subscriber("/gelsight/depth/image_raw", Image, self.on_depth_img) 301 | rospy.Subscriber("/gelsight/image/image_raw", Image, self.on_rgb_img) 302 | self.publisher = rospy.Publisher("/gelsight/tactile_image", Image, queue_size=1) 303 | 304 | self.rate = rospy.Rate(30) 305 | 306 | def on_depth_img(self, img_msg): 307 | img = bridge.imgmsg_to_cv2(img_msg, desired_encoding="32FC1") 308 | img[np.isnan(img)] = np.inf 309 | self.depth_img = img 310 | 311 | def on_rgb_img(self, img_msg): 312 | self.visual_img = bridge.imgmsg_to_cv2(img_msg, desired_encoding="bgr8") 313 | 314 | def publish(self, tactile_img): 315 | self.publisher.publish(bridge.cv2_to_imgmsg(tactile_img, "bgr8")) 316 | 317 | def run(self): 318 | while not rospy.is_shutdown(): 319 | if self.depth_img is None or self.visual_img is None: 320 | continue 321 | 322 | tactile_img = self.simulation_approach.generate(self.depth_img) 323 | self.publish(tactile_img) 324 | self.rate.sleep() 325 | cv2.waitKey(1) 326 | 327 | 328 | def main(): 329 | # light position: x,y,z, color BGR 330 | light_sources_mit2014 = [ 331 | {'position': [0, 1, 0.25], 'color': (240, 240, 240)}, 332 | {'position': [-1, 0, 0.25], 'color': (255, 139, 78)}, 333 | {'position': [0, -1, 0.25], 'color': (108, 82, 255)}, 334 | {'position': [1, 0, 0.25], 'color': (100, 240, 150)}, 335 | ] 336 | 337 | light_sources_smartlab2014 = [ 338 | {'position': [0, 1, 0.25], 'color': (255, 255, 255), 'kd': 0.6, 'ks': 0.5}, # white, top 339 | {'position': [-1, 0, 0.25], 'color': (255, 130, 115), 'kd': 0.5, 'ks': 0.3}, # blue, right 340 | {'position': [0, -1, 0.25], 'color': (108, 82, 255), 'kd': 0.6, 'ks': 0.4}, # red, bottom 341 | {'position': [1, 0, 0.25], 'color': (120, 255, 153), 'kd': 0.1, 'ks': 0.1}, # green, left 342 | ] 343 | 344 | background_img = cv2.imread(PKG_PATH + '/assets/background.png') 345 | ka = 0.8 346 | 347 | px2m_ratio = 5.4347826087e-05 348 | elastomer_thickness = 0.004 349 | min_depth = 0.026 # distance from the image sensor to the rigid glass outer surface 350 | texture_sigma = 0.00001 351 | 352 | simulation_approach = SimulationApproach( 353 | light_sources=light_sources_smartlab2014, 354 | background_img=background_img, 355 | ka=ka, 356 | texture_sigma=texture_sigma, 357 | px2m_ratio=px2m_ratio, 358 | elastomer_thickness=elastomer_thickness, 359 | min_depth=min_depth 360 | ) 361 | 362 | driver = GelSightDriver('gelsight_node', simulation_approach) 363 | driver.run() 364 | 365 | 366 | def test(): 367 | # light position: x,y,z, color BGR 368 | # light_sources_mit2014 = [ 369 | # {'position': [0, 1, 0.25], 'color': (240, 240, 240)}, 370 | # {'position': [-1, 0, 0.25], 'color': (255, 139, 78)}, 371 | # {'position': [0, -1, 0.25], 'color': (108, 82, 255)}, 372 | # {'position': [1, 0, 0.25], 'color': (100, 240, 150)}, 373 | # ] 374 | 375 | light_sources_smartlab2014 = [ 376 | # {'position': [0, 1, 0.25], 'color': (255, 255, 255), 'kd': 0.6, 'ks': 0.5}, # white, top 377 | # {'position': [-1, 0, 0.25], 'color': (255, 130, 115), 'kd': 0.5, 'ks': 0.3}, # blue, right 378 | {'position': [-1, 0, 0.25], 'color': (108, 82, 255), 'kd': 0.6, 'ks': 0.4}, # red, bottom 379 | {'position': [0.50, -0.866, 0.25], 'color': (120, 255, 153), 'kd': 0.1, 'ks': 0.4}, # green, left 380 | {'position': [0.50, 0.866, 0.25], 'color': (255, 130, 115), 'kd': 0.1, 'ks': 0.4}, # blue, left 381 | ] 382 | 383 | # light_sources_mit2017 = [ 384 | # {'position': [0, 1, 0.25], 'color': (240, 240, 240)}, 385 | # {'position': [-1, 0, 0.25], 'color': (255, 139, 78)}, 386 | # {'position': [0, -1, 0.25], 'color': (108, 82, 255)}, 387 | # {'position': [1, 0, 0.25], 'color': (100, 240, 150)}, 388 | # ] 389 | 390 | background_img = cv2.imread(PKG_PATH + '/assets/background_gelsight2017.jpg') 391 | ka = 0.8 392 | 393 | px2m_ratio = 5.4347826087e-05 394 | elastomer_thickness = 0.004 395 | min_depth = 0.026 # distance from the image sensor to the rigid glass outer surface 396 | texture_sigma = 0.000002 397 | 398 | simulation_approach = SimulationApproach( 399 | light_sources=light_sources_smartlab2014, 400 | background_img=background_img, 401 | ka=ka, 402 | texture_sigma=texture_sigma, 403 | px2m_ratio=px2m_ratio, 404 | elastomer_thickness=elastomer_thickness, 405 | min_depth=min_depth 406 | ) 407 | 408 | from PIL import Image 409 | # pil_depth = Image.open("/home/danfergo/Projects/gelsight_simulation/dataset/sim/depth2/random__6__-1_-1_5.bmp") 410 | # depth = np.array(pil_depth.getdata()).reshape((640, 480, 3)) 411 | 412 | # print('---------_> ', cv2.CV_32F1) 413 | depth = np.load("/home/danfergo/Projects/gelsight_simulation/dataset/sim/depth2/random__6__-1_-1_5.npy") 414 | print(np.shape(depth), np.max(depth), np.min(depth), depth.dtype) 415 | out = simulation_approach.generate(depth) 416 | cv2.imshow('test 2017', out) 417 | cv2.waitKey(0) 418 | 419 | 420 | if __name__ == '__main__': 421 | main() 422 | --------------------------------------------------------------------------------