├── images ├── vaig.pdf └── vaig.png ├── data └── collisions.pickle ├── teapot_vertices.txt ├── export_groundtruth.py ├── .gitignore ├── clean_collada.py ├── process_mesh.py ├── rsync models.txt ├── extract.py ├── README.md ├── teapots.txt ├── zernike.py ├── torch_nn.py ├── geometry.py ├── save_exr_images.py ├── shape_model.py ├── var_inf.py ├── diffrender_experiment.py ├── densecrf_model.py ├── recognition_models.py ├── export_collisions.py ├── collision.py ├── lasagne_visualize.py ├── diffrender_opt.py ├── teapots.py ├── azimuth_test.py ├── export_occlusions.py ├── image_processing.py ├── differentiable_renderer.py ├── render.py ├── occlusion_viewpoints.txt └── probLineSearch.py /images/vaig.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/polmorenoc/inversegraphics/HEAD/images/vaig.pdf -------------------------------------------------------------------------------- /images/vaig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/polmorenoc/inversegraphics/HEAD/images/vaig.png -------------------------------------------------------------------------------- /data/collisions.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/polmorenoc/inversegraphics/HEAD/data/collisions.pickle -------------------------------------------------------------------------------- /teapot_vertices.txt: -------------------------------------------------------------------------------- 1 | Teapot Vertices 2 | teapots/fa1fa0818738e932924ed4f13e49b59d/Teapot N300912_cleaned 3 | Spout: -0.61921 -0.01568 0.62589 4 | Handle: 0.6 0.02 0.56735 5 | Tip: 0 0 0.8 -------------------------------------------------------------------------------- /export_groundtruth.py: -------------------------------------------------------------------------------- 1 | import save_exr_images 2 | from save_exr_images import exportExrImages 3 | import os 4 | print ("Reading xml ") 5 | 6 | 7 | outputDir = '../data/output/' 8 | imgDir = outputDir + "images/" 9 | 10 | lines = [line.strip() for line in open(outputDir + 'groundtruth.txt')] 11 | 12 | if not os.path.exists(imgDir): 13 | os.makedirs(imgDir) 14 | 15 | for instance in lines: 16 | 17 | parts = instance.split(' ') 18 | teapot = int(parts[3]) 19 | frame = int(parts[4]) 20 | 21 | sceneNum = int(parts[5]) 22 | 23 | targetIndex = int(parts[6]) 24 | 25 | prefix = '' 26 | if len(parts) == 17: 27 | prefix = parts[16] 28 | try: 29 | exportExrImages(outputDir, imgDir, teapot, frame, sceneNum, targetIndex, prefix) 30 | except Exception as e: 31 | print(e) 32 | 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | *.so 57 | *.o 58 | *.os 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | .idea/ 64 | -------------------------------------------------------------------------------- /clean_collada.py: -------------------------------------------------------------------------------- 1 | import pyassimp 2 | from pyassimp.postprocess import * 3 | 4 | modelPath = '../databaseFull/models/teapots/78ed1a0383cd2eab7552b099aebcb24e/Teapot_fixed.dae' 5 | 6 | # aiProcessPreset_TargetRealtime_Quality = ( \ 7 | # aiProcess_CalcTangentSpace | \ 8 | # aiProcess_GenSmoothNormals | \ 9 | # aiProcess_JoinIdenticalVertices | \ 10 | # aiProcess_ImproveCacheLocality | \ 11 | # aiProcess_LimitBoneWeights | \ 12 | # aiProcess_RemoveRedundantMaterials | \ 13 | # aiProcess_SplitLargeMeshes | \ 14 | # aiProcess_Triangulate | \ 15 | # aiProcess_GenUVCoords | \ 16 | # aiProcess_SortByPType | \ 17 | # aiProcess_FindDegenerates | \ 18 | # aiProcess_FindInvalidData | \ 19 | # 0 ) 20 | 21 | postprocess = aiProcessPreset_TargetRealtime_Quality 22 | 23 | scene = pyassimp.load(modelPath, postprocess) 24 | -------------------------------------------------------------------------------- /process_mesh.py: -------------------------------------------------------------------------------- 1 | import meshtool.filters.simplify_filters.sander_simplify 2 | from meshtool.filters.base_filters import MetaFilter 3 | from meshtool.filters import factory 4 | from itertools import chain, izip, combinations 5 | import collada 6 | import numpy 7 | 8 | 9 | def process(meshPath): 10 | 11 | mesh = collada.Collada(meshPath) 12 | 13 | # 'triangulate', 14 | optimize_filters = [ 15 | 'combine_primitives', 16 | 'optimize_sources', 17 | 'strip_unused_sources', 18 | 'normalize_indices' 19 | ] 20 | 21 | for f in optimize_filters: 22 | inst = factory.getInstance(f) 23 | mesh = inst.apply(mesh) 24 | 25 | # f = 'sander_simplify' 26 | # pmout = open('pm_file', 'w') 27 | # inst = factory.getInstance(f) 28 | # mesh = inst.apply(mesh, pmout) 29 | 30 | # s = meshtool.filters.simplify_filters.sander_simplify.SanderSimplify(mesh, pmout) 31 | 32 | # meshsimple = s.simplify() 33 | 34 | return mesh 35 | 36 | 37 | -------------------------------------------------------------------------------- /rsync models.txt: -------------------------------------------------------------------------------- 1 | rsync -a --ignore-existing ../databaseFull/models/teapots/ s1251804@salmon:/disk/data3/s1251804/databaseFull/models/teapots/ 2 | 3 | rsync -a --ignore-existing ../COLLADA/1308f3ff2e55eae4f1783a44a88d6274* /home/pol/MEGA/scene\ update/COLLADA 4 | 5 | newModel 0 room09 6 | newModel 1 26ebcfb2ebd8345f14b86d5282eb8301 7 | newModel 2 d5030e978cceae91f11c3739edd52fa3 8 | newModel 3 2997f21fa426e18a6ab1a25d0e8f3590 9 | newModel 4 6372c4d2e046d0fd7d3aa3bae1f7b494 10 | newModel 5 6970d21f2c2146656ab04bd3af593c95 11 | newModel 6 38c843d1075bd351e652fa812161367 12 | newModel 7 63a1164c4dcf8874b6c61406205e1df7 13 | newModel 8 a28f3699bdb6176614513156cf2b8d0d 14 | newModel 9 56ac77092883c683eeff745b3f73bf77 15 | newModel 10 c9fe86bef85fd1d0caeedf6b101df8f6 16 | newModel 11 8824ac5d1d34309794247f03639516c 17 | newModel 12 4a1cc007d6e0754d18b14db3b83de9ff 18 | newModel 13 cd82398f7e6b64b08c9441777325d0fd 19 | newModel 14 fca9fcb710592311d291861d5bc3e7c8 20 | newModel 15 1308f3ff2e55eae4f1783a44a88d6274 21 | newModel 16 1308f3ff2e55eae4f1783a44a88d6274 22 | newModel 17 1308f3ff2e55eae4f1783a44a88d6274 23 | newModel 18 1308f3ff2e55eae4f1783a44a88d6274 -------------------------------------------------------------------------------- /extract.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | # matplotlib.use('Agg') 3 | import bpy 4 | import numpy 5 | import matplotlib.pyplot as plt 6 | 7 | width = 110 8 | 9 | height = 110 10 | 11 | 12 | 13 | scene = bpy.data.scenes[0] 14 | scene.render.resolution_x = width #perhaps set resolution in code 15 | scene.render.resolution_y = height 16 | scene.render.resolution_percentage = 100 17 | 18 | bpy.data.objects['Cube'].pass_index = 1 19 | 20 | scene.render.layers[0].use_pass_object_index = True 21 | 22 | bpy.context.scene.use_nodes = True 23 | tree = bpy.context.scene.node_tree 24 | 25 | rl = tree.nodes[1] 26 | links = tree.links 27 | 28 | v = tree.nodes.new('CompositorNodeViewer') 29 | v.location = 750,210 30 | v.use_alpha = False 31 | links.new(rl.outputs['Image'], v.inputs['Image']) # link Image output to Viewer input 32 | 33 | outnode = tree.nodes.new('CompositorNodeOutputFile') 34 | outnode.base_path = 'indexob.png' 35 | links.new(rl.outputs['IndexOB'], outnode.inputs['Image']) 36 | 37 | bpy.data.objects['Cube'].pass_index = 1 38 | 39 | bpy.ops.render.render( write_still=False ) 40 | 41 | blendImage = bpy.data.images['Render Result'] 42 | 43 | image = numpy.flipud(numpy.array(blendImage.extract_render(scene=scene)).reshape([height,width,4]))[:,:,0:3] 44 | # 45 | # blendImage2 = bpy.data.images['Viewer Node'] 46 | # 47 | # image2 = numpy.flipud(numpy.array(blendImage2.extract_render(scene=scene)).reshape([256,256,4]))[:,:,0:3] 48 | 49 | print("DOne") -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overcoming Occlusion with Inverse Graphics 2 | This is the code for our [ECCV16 paper on the Geometry meets Deep Learning workshop](http://homepages.inf.ed.ac.uk/ckiw/postscript/eccv_gmdl16.pdf). 3 | 4 | ![](images/vaig.png) 5 | 6 | Here you'll find the scene generator as well as the code for the models. The main libraries needed are: 7 | 8 | - My updated versions of OpenDR and Chumpy you can [find here](https://github.com/polmorenoc/opendr). 9 | - Blender 2.6+ (installed as a python module), so you can do "import bpy". Please follow [these instructions](https://wiki.blender.org/index.php/User:Ideasman42/BlenderAsPyModule). 10 | - OpenCV 3.0+ as Python module. 11 | 12 | 13 | Python 3.4+ is a requirement. 14 | 15 | The main script files to run are the following: 16 | 17 | * diffrender_demo.py: Code to interactively run and fit the generative models. 18 | * diffrender_groundtruth.py: Main code to generate ground-truth for synthetic scenes (both for OpenGL and Photorealistic (cycles) types of rendering! 19 | * diffrender_groundtruth_multi.p: As above, but extended to multiple objects. 20 | * diffrender_experiment.py: Generate experiment train/test splits. 21 | * diffrender_train.py: Train Lasagne neural networks (used to train our recognition models). 22 | * diffrender_test.py: Main code to evaluate and fit different models. 23 | * diffrender_analyze.py: Extract statistics and plots of experimental evaluation. 24 | 25 | For the stocastic generation of synthetic images with ground-truth, you'll need additional CAD data and other files. Please get in touch with me (polmorenoc@gmail.com) for it. 26 | -------------------------------------------------------------------------------- /teapots.txt: -------------------------------------------------------------------------------- 1 | teapots/8e6a162e707ecdf323c90f8b869f2ce9/TeapotN280912_cleaned 2 | teapots/c7549b28656181c91bff71a472da9153/TeapotN311012_cleaned 3 | teapots/fa1fa0818738e932924ed4f13e49b59d/TeapotN300912_cleaned 4 | teapots/4ed8e98d46c6ca0f31503d620f32f381/Teapot_cleaned 5 | teapots/1c43a79bd6d814c84a0fee00d66a5e35/Teapot_cleaned 6 | teapots/def97faaa62364c85f1ceb98f92e8da8/teapot_gold_cleaned 7 | teapots/5553cc5e9a831c13674b6111068c99d5/teapot_fixed_cleaned 8 | teapots/760324f8bd1d289aaaadf77877ab4cc9/teapot_BS_fixed_cleaned 9 | teapots/d8ec1b644628228fedec191a3310877e/Teapot_fixed_cleaned 10 | teapots/3f6e96960050a234bae0ed33e799e8ca/Teapot_fixed_cleaned 11 | teapots/457cf2b21b8ebab5f0ec18866e1116e4/Teapot_fixed_cleaned 12 | teapots/64107a74f9e41fe477b38c8e2744b5b3/Teapot_fixed_cleaned 13 | teapots/78ed1a0383cd2eab7552b099aebcb24e/Teapot_fixed_cleaned 14 | teapots/a7fa82f5982edfd033da2d90df7af046/Teapot_fixed_cleaned 15 | teapots/c22b3f6099236942b4b54176aedb7c4b/Teapot_fixed_cleaned 16 | teapots/8396cb51b2caa10c630f6a9f35f0722c/Teapotbig_fixed_cleaned 17 | teapots/12b81ec72a967dc1714fc48a3b0c961a/TeapotN260113_fixed_cleaned 18 | teapots/1945c4f0c7bd15c95ae1dc6be392df2b/TeapotN041012_fixed_cleaned 19 | teapots/36efdf1adb40ddcb0c838d8d90833c17/TeapotN190710_fixed_cleaned 20 | teapots/6e2d299fcd4c7f5b69770b0f89610efb/TeapotN190511_fixed_cleaned 21 | teapots/6f4cc645ea0c70961106df21f9631c5f/TeapotN230113_fixed_cleaned 22 | teapots/a0e9679824623104706f9677db83a1b1/TeapotN271107_fixed_cleaned 23 | teapots/c97035a3b82808ab4a9ba8f3de9c3e24/TeapotN100608_fixed_cleaned 24 | teapots/d71943d06e680f62d568a4546220fa43/TeapotN121112_fixed_cleaned 25 | teapots/e41b46d4249bf5cd81709c7a26423382/TeapotN240608_fixed_cleaned 26 | teapots/2593256ba73fe3d5e698ad3c1c3af9d4/TeapotRZN280612_fixed_cleaned 27 | teapots/5b9f9bcded23e15244b1563e66d7bd64/TeapotRZN230612_fixed_cleaned 28 | teapots/teapot27 -------------------------------------------------------------------------------- /zernike.py: -------------------------------------------------------------------------------- 1 | """ 2 | @file py102-example2-zernike.py 3 | @brief Fitting a surface in Python example for Python 102 lecture 4 | @author Tim van Werkhoven (t.i.m.vanwerkhoven@gmail.com) 5 | @url http://python101.vanwerkhoven.org 6 | @date 20111012 7 | Created by Tim van Werkhoven (t.i.m.vanwerkhoven@xs4all.nl) on 2011-10-12 8 | Copyright (c) 2011 Tim van Werkhoven. All rights reserved. 9 | This file is licensed under the Creative Commons Attribution-Share Alike 10 | license versions 3.0 or higher, see 11 | http://creativecommons.org/licenses/by-sa/3.0/ 12 | """ 13 | 14 | ### Libraries 15 | 16 | import numpy as N 17 | from scipy.misc import factorial as fac 18 | 19 | ### Init functions 20 | def zernike_rad(m, n, rho): 21 | """ 22 | Calculate the radial component of Zernike polynomial (m, n) 23 | given a grid of radial coordinates rho. 24 | 25 | """ 26 | 27 | if (n < 0 or m < 0 or abs(m) > n): 28 | raise ValueError 29 | 30 | if ((n-m) % 2): 31 | return rho*0.0 32 | 33 | pre_fac = lambda k: (-1.0)**k * fac(n-k) / ( fac(k) * fac( (n+m)/2.0 - k ) * fac( (n-m)/2.0 - k ) ) 34 | 35 | return sum(pre_fac(k) * rho**(n-2.0*k) for k in range(int((n-m)/2+1))) 36 | 37 | def zernike(m, n, rho, phi): 38 | """ 39 | Calculate Zernike polynomial (m, n) given a grid of radial 40 | coordinates rho and azimuthal coordinates phi. 41 | 42 | """ 43 | if (m > 0): return zernike_rad(m, n, rho) * N.cos(m * phi) 44 | if (m < 0): return zernike_rad(-m, n, rho) * N.sin(-m * phi) 45 | return zernike_rad(0, n, rho) 46 | 47 | def zernikel(j, rho, phi): 48 | """ 49 | Calculate Zernike polynomial with Noll coordinate j given a grid of radial 50 | coordinates rho and azimuthal coordinates phi. 51 | """ 52 | n = 0 53 | while (j > n): 54 | n += 1 55 | j -= n 56 | 57 | m = -n+2*j 58 | return zernike(m, n, rho, phi) -------------------------------------------------------------------------------- /torch_nn.py: -------------------------------------------------------------------------------- 1 | import ipdb 2 | import PyTorchAug 3 | import PyTorch 4 | nn = PyTorch.Nn() 5 | 6 | lua = PyTorchAug.lua 7 | lua.getGlobal("require") 8 | lua.pushString('modules/LinearCR') 9 | lua.call(1, 0) 10 | 11 | lua = PyTorchAug.lua 12 | lua.getGlobal("require") 13 | lua.pushString('modules/Reparametrize') 14 | lua.call(1, 0) 15 | 16 | lua = PyTorchAug.lua 17 | lua.getGlobal("require") 18 | lua.pushString('modules/SelectiveOutputClamp') 19 | lua.call(1, 0) 20 | 21 | lua = PyTorchAug.lua 22 | lua.getGlobal("require") 23 | lua.pushString('modules/SelectiveGradientFilter') 24 | lua.call(1, 0) 25 | 26 | 27 | dim_hidden = 200 28 | feature_maps = 96 29 | 30 | filter_size = 5 31 | colorchaPyTorchAugels = 1 32 | 33 | # ipdb.set_trace() 34 | 35 | encoder = PyTorchAug.Sequential() 36 | 37 | 38 | encoder.add(PyTorchAug.SpatialConvolution(colorchaPyTorchAugels,feature_maps,filter_size,filter_size)) 39 | 40 | encoder.add(PyTorchAug.SpatialMaxPooling(2,2,2,2)) 41 | encoder.add(PyTorchAug.Threshold(0,1e-6)) 42 | 43 | encoder.add(PyTorchAug.SpatialConvolution(feature_maps,feature_maps/2,filter_size,filter_size)) 44 | encoder.add(PyTorchAug.SpatialMaxPooling(2,2,2,2)) 45 | encoder.add(PyTorchAug.Threshold(0,1e-6)) 46 | 47 | 48 | 49 | encoder.add(PyTorchAug.SpatialConvolution(feature_maps/2,feature_maps/4,filter_size,filter_size)) 50 | encoder.add(PyTorchAug.SpatialMaxPooling(2,2,2,2)) 51 | encoder.add(PyTorchAug.Threshold(0,1e-6)) 52 | 53 | encoder.add(PyTorchAug.Reshape((feature_maps/4)*15*15)) 54 | 55 | z = PyTorchAug.ConcatTable() 56 | 57 | mu = PyTorchAug.Sequential() 58 | 59 | mu.add(PyTorchAug.LinearCR((feature_maps/4)*15*15, dim_hidden)) 60 | mu.add(PyTorchAug.SelectiveGradientFilter()) 61 | mu.add(PyTorchAug.SelectiveOutputClamp()) 62 | z.add(mu) 63 | 64 | sigma = PyTorchAug.Sequential() 65 | sigma.add(PyTorchAug.LinearCR((feature_maps/4)*15*15, dim_hidden)) 66 | sigma.add(PyTorchAug.SelectiveGradientFilter()) 67 | sigma.add(PyTorchAug.SelectiveOutputClamp()) 68 | z.add(sigma) 69 | 70 | 71 | encoder.add(z) 72 | 73 | decoder = PyTorchAug.Sequential() 74 | decoder.add(PyTorchAug.LinearCR(dim_hidden, (feature_maps/4)*15*15 )) 75 | decoder.add(PyTorchAug.Threshold(0,1e-6)) 76 | 77 | decoder.add(PyTorchAug.Reshape((feature_maps/4),15,15)) 78 | 79 | decoder.add(PyTorchAug.SpatialUpSamplingNearest(2)) 80 | decoder.add(PyTorchAug.SpatialConvolution(feature_maps/4,feature_maps/2, 7, 7)) 81 | decoder.add(PyTorchAug.Threshold(0,1e-6)) 82 | 83 | decoder.add(PyTorchAug.SpatialUpSamplingNearest(2)) 84 | decoder.add(PyTorchAug.SpatialConvolution(feature_maps/2,feature_maps,7,7)) 85 | decoder.add(PyTorchAug.Threshold(0,1e-6)) 86 | 87 | decoder.add(PyTorchAug.SpatialUpSamplingNearest(2)) 88 | decoder.add(PyTorchAug.SpatialConvolution(feature_maps,feature_maps,7,7)) 89 | decoder.add(PyTorchAug.Threshold(0,1e-6)) 90 | 91 | decoder.add(PyTorchAug.SpatialUpSamplingNearest(2)) 92 | decoder.add(PyTorchAug.SpatialConvolution(feature_maps,1,7,7)) 93 | decoder.add(PyTorchAug.Sigmoid()) 94 | 95 | model = PyTorchAug.Sequential() 96 | model.add(encoder) 97 | model.add(PyTorchAug.Reparametrize(dim_hidden)) 98 | model.add(decoder) 99 | 100 | model.cuda() 101 | nn.collectgarbage() -------------------------------------------------------------------------------- /geometry.py: -------------------------------------------------------------------------------- 1 | import chumpy as ch 2 | from chumpy import depends_on, Ch 3 | import cv2 4 | import numpy as np 5 | import scipy.sparse as sp 6 | from chumpy.utils import row, col 7 | from opendr.geometry import Rodrigues 8 | 9 | 10 | 11 | class RotateZ(Ch): 12 | dterms = 'a' 13 | 14 | def compute_r(self): 15 | return np.array([[np.cos(self.a.r), -np.sin(self.a.r), 0, 0], [np.sin(self.a.r), np.cos(self.a.r), 0, 0], [0, 0, 1, 0], [0,0,0,1]]) 16 | 17 | def compute_dr_wrt(self, wrt): 18 | 19 | if wrt is not self.a: 20 | return 21 | 22 | if wrt is self.a: 23 | return np.array([[-np.sin(self.a.r)[0], -np.cos(self.a.r)[0], 0, 0], [np.cos(self.a.r)[0], -np.sin(self.a.r)[0], 0, 0], [0, 0, 0, 0], [0,0,0,0]]).reshape(16,1) 24 | 25 | 26 | class RotateX(Ch): 27 | dterms = 'a' 28 | 29 | def compute_r(self): 30 | return np.array([[1, 0, 0, 0],[0, np.cos(self.a.r), -np.sin(self.a.r), 0], [0, np.sin(self.a.r), np.cos(self.a.r),0],[0,0,0,1]]) 31 | 32 | def compute_dr_wrt(self, wrt): 33 | 34 | # if wrt is not self.a: 35 | # return 36 | 37 | if wrt is self.a: 38 | return np.array([[0, 0, 0, 0],[0, -np.sin(self.a.r)[0], -np.cos(self.a.r)[0], 0], [0, np.cos(self.a.r)[0], -np.sin(self.a.r)[0],0],[0,0,0,0]]).reshape(16,1) 39 | 40 | class Scale(Ch): 41 | dterms = 'x', 'y','z' 42 | 43 | def compute_r(self): 44 | return np.array([[self.x.r, 0, 0, 0],[0, self.y.r, 0, 0], [0, 0, self.z.r,0],[0,0,0,1]]) 45 | 46 | def compute_dr_wrt(self, wrt): 47 | return 48 | 49 | def compute_dr_wrt(self, wrt): 50 | if wrt is not self.x and wrt is not self.y and wrt is not self.z: 51 | return 52 | if wrt is self.x: 53 | return np.array([[1, 0, 0, 0],[0, 0, 0, 0], [0, 0, 0,0],[0,0,0,0]]).reshape(16,1) 54 | if wrt is self.y: 55 | return np.array([[0, 0, 0, 0],[0, 1, 0, 0], [0, 0, 0,0],[0,0,0,0]]).reshape(16,1) 56 | if wrt is self.z: 57 | return np.array([[0, 0, 0, 0],[0, 0, 0, 0], [0, 0, 1, 0],[0,0,0,0]]).reshape(16,1) 58 | 59 | 60 | class Translate(Ch): 61 | dterms = 'x', 'y', 'z' 62 | 63 | def compute_r(self): 64 | return np.array([[1, 0, 0, self.x.r],[0, 1, 0, self.y.r], [0, 0, 1,self.z.r],[0,0,0,1]]) 65 | 66 | def compute_dr_wrt(self, wrt): 67 | 68 | if wrt is not self.x and wrt is not self.y and wrt is not self.z: 69 | return 70 | 71 | if wrt is self.x: 72 | return np.array([[0, 0, 0, 1],[0, 0, 0, 0], [0, 0, 0,0],[0,0,0,0]]).reshape(16,1) 73 | if wrt is self.y: 74 | return np.array([[0, 0, 0, 0],[0, 0, 0, 1], [0, 0, 0,0],[0,0,0,0]]).reshape(16,1) 75 | if wrt is self.z: 76 | return np.array([[0, 0, 0, 0],[0, 0, 0, 0], [0, 0, 0, 1],[0,0,0,0]]).reshape(16,1) 77 | 78 | 79 | def getNormals(vertices, faces): 80 | norm = np.zeros( vertices.shape, dtype=vertices.dtype ) 81 | tris = vertices[faces] 82 | n = np.cross( tris[::,1 ] - tris[::,0] , tris[::,2 ] - tris[::,0] ) 83 | normalize_v3(n) 84 | norm[ faces[:,0] ] += n 85 | norm[ faces[:,1] ] += n 86 | norm[ faces[:,2] ] += n 87 | normalize_v3(norm) 88 | return norm 89 | 90 | def chGetNormals(vertices, faces): 91 | import opendr.geometry 92 | return opendr.geometry.VertNormals(vertices, faces).reshape((-1,3)) 93 | 94 | 95 | -------------------------------------------------------------------------------- /save_exr_images.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import OpenEXR 4 | import Imath 5 | from PIL import Image 6 | import sys 7 | import numpy as np 8 | 9 | def exportExrImages(annotationdir, imgdir, numTeapot, frame, sceneNum, target, prefix): 10 | 11 | framestr = '{0:04d}'.format(frame) 12 | 13 | outfilename = "render" + prefix + "_obj" + str(numTeapot) + "_scene" + str(sceneNum) + "_target" + str(target) + "_" + framestr 14 | exrfile = OpenEXR.InputFile(annotationdir + outfilename + ".exr") 15 | pt = Imath.PixelType(Imath.PixelType.FLOAT) 16 | dw = exrfile.header()['dataWindow'] 17 | size = (dw.max.x - dw.min.x + 1, dw.max.y - dw.min.y + 1) 18 | 19 | quality_val = 100 20 | 21 | # rgbf = [Image.fromstring("F", size, exrfile.channel("shNormal." + c, pt)) for c in "RGB"] 22 | # extrema = [im.getextrema() for im in rgbf] 23 | # darkest = min([lo for (lo,hi) in extrema]) 24 | # lighest = max([hi for (lo,hi) in extrema]) 25 | # scale = 255 / (lighest - darkest) 26 | # def normalize_0_255(v): 27 | # return (v * scale) + darkest 28 | # rgb8 = [im.point(normalize_0_255).convert("L") for im in rgbf] 29 | # Image.merge("RGB", rgb8).save(jpgfilename + "_normal.jpg", "JPEG") 30 | 31 | rgbf = [Image.fromstring("F", size, exrfile.channel("RenderLayer.Combined." + c, pt)) for c in "RGB"] 32 | pix = [np.array(im) for im in rgbf] 33 | pix[0][pix[0]>1] = 1 34 | pix[1][pix[1]>1] = 1 35 | pix[2][pix[2]>1] = 1 36 | pix[0] = pix[0]*255 37 | pix[1] = pix[1]*255 38 | pix[2] = pix[2]*255 39 | imgr = Image.fromarray(pix[0].astype('uint8')) 40 | imgg = Image.fromarray(pix[1].astype('uint8')) 41 | imgb = Image.fromarray(pix[2].astype('uint8')) 42 | finalimg = Image.merge("RGB", (imgr, imgg, imgb)) 43 | # finalimg.save(imgdir + outfilename + ".png", "PNG", quality=quality_val) 44 | 45 | distancestr = exrfile.channel('RenderLayer.IndexOB.X', pt) 46 | distance = Image.fromstring("F", size, distancestr) 47 | 48 | 49 | shapeIndexstr = exrfile.channel('RenderLayer.IndexOB.X', pt) 50 | shapeIndex = Image.fromstring("F", size, shapeIndexstr) 51 | segment = np.array(shapeIndex) 52 | sumComplete = np.sum(segment) 53 | segmentimg = Image.fromarray(segment.astype('uint8')*255) 54 | segmentimg.save(imgdir + outfilename + "_segment.png", "PNG", quality=quality_val) 55 | 56 | 57 | singlefile = "render" + prefix + "_obj" + str(numTeapot) + "_scene" + str(sceneNum) + "_target" + str(target) + "_single_" + framestr 58 | exrfile = OpenEXR.InputFile(annotationdir + singlefile + ".exr") 59 | 60 | rgbf = [Image.fromstring("F", size, exrfile.channel("RenderLayer.001.Combined." + c, pt)) for c in "RGB"] 61 | pix = [np.array(im) for im in rgbf] 62 | pix[0][pix[0]>1] = 1 63 | pix[1][pix[1]>1] = 1 64 | pix[2][pix[2]>1] = 1 65 | pix[0] = pix[0]*255 66 | pix[1] = pix[1]*255 67 | pix[2] = pix[2]*255 68 | imgr = Image.fromarray(pix[0].astype('uint8')) 69 | imgg = Image.fromarray(pix[1].astype('uint8')) 70 | imgb = Image.fromarray(pix[2].astype('uint8')) 71 | finalimg = Image.merge("RGB", (imgr, imgg, imgb)) 72 | finalimg.save(imgdir + singlefile + ".png", "PNG", quality=quality_val) 73 | 74 | shapeIndexstrsingle = exrfile.channel('RenderLayer.001.IndexOB.X', pt) 75 | shapeIndexsingle = Image.fromstring("F", size, shapeIndexstrsingle) 76 | segmentsingle = np.array(shapeIndexsingle) 77 | 78 | segmentimgsingle = Image.fromarray(segmentsingle.astype('uint8')*255) 79 | segmentimgsingle.save(imgdir + singlefile + "_segment.png", "PNG", quality=quality_val) 80 | sumSingle = np.sum(segmentsingle) 81 | 82 | print "Sum Complete " + str(sumComplete) 83 | print "Sum Single " + str(sumSingle) 84 | with open(annotationdir + 'occlusions' + ".txt", "a") as myfile: 85 | myfile.write(str(numTeapot) + ' ' + str(frame) + ' ' + str(sceneNum) + " " + str(target) + " " + str(sumComplete/sumSingle) + ' ' + prefix + "\n") 86 | 87 | return 88 | -------------------------------------------------------------------------------- /shape_model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | import chumpy as ch 4 | import ipdb 5 | from chumpy import depends_on, Ch 6 | import scipy.sparse as sp 7 | 8 | #%% Helper functions 9 | def longToPoints3D(pointsLong): 10 | nPointsLong = np.size(pointsLong) 11 | return np.reshape(pointsLong, [nPointsLong/3, 3]) 12 | 13 | 14 | def shapeParamsToVerts(shapeParams, teapotModel): 15 | landmarksLong = shapeParams.dot(teapotModel['ppcaW'].T) + teapotModel['ppcaB'] 16 | landmarks = longToPoints3D(landmarksLong) 17 | vertices = teapotModel['meshLinearTransform'].dot(landmarks) 18 | return vertices 19 | 20 | def chShapeParamsToVerts(landmarks, meshLinearTransform): 21 | vertices = ch.dot(meshLinearTransform,landmarks) 22 | return vertices 23 | 24 | class VerticesModel(Ch): 25 | terms = 'meshLinearTransform', 'W', 'b' 26 | dterms = 'chShapeParams' 27 | 28 | def init(self): 29 | self.jac = self.meshLinearTransform.dot(self.W.reshape([self.meshLinearTransform.shape[1], -1, len(self.chShapeParams)]).transpose((1,0,2))).reshape([-1,len(self.chShapeParams)]) 30 | 31 | def compute_r(self): 32 | landmarks = np.dot(self.chShapeParams.r,self.W.T) + self.b 33 | landmarks = landmarks.reshape([-1,3]) 34 | return np.dot(self.meshLinearTransform, landmarks) 35 | 36 | 37 | def compute_dr_wrt(self,wrt): 38 | if self.chShapeParams is wrt: 39 | # ipdb.set_trace() 40 | return self.jac 41 | return None 42 | 43 | def chShapeParamsToNormals(N, landmarks, linT): 44 | T = ch.dot(linT,landmarks) 45 | invT = [] 46 | nLandmarks = landmarks.r.shape[0] 47 | for i in range(nLandmarks): 48 | R = T[4*i:4*i+3,:3].T 49 | invR = ch.linalg.inv(R.T) 50 | invT = invT + [invR] 51 | 52 | invT = ch.vstack(invT) 53 | newNormals = ch.dot(N, invT) 54 | 55 | import opendr.geometry 56 | n = opendr.geometry.NormalizedNx3(newNormals) 57 | 58 | return newNormals 59 | 60 | def getT(targetPoints, linT): 61 | T = linT.dot(targetPoints) 62 | return T 63 | 64 | def shapeParamsToNormals(shapeParams, teapotModel): 65 | landmarksLong = shapeParams.dot(teapotModel['ppcaW'].T) + teapotModel['ppcaB'] 66 | landmarks = longToPoints3D(landmarksLong) 67 | T = getT(landmarks, teapotModel['linT']) 68 | nLandmarks = np.shape(landmarks)[0] 69 | invT = np.empty([3*nLandmarks, 3]) 70 | for i in range(nLandmarks): 71 | R = T[4*i:4*i+3,:3].T 72 | invR = np.linalg.inv(R) 73 | invT[3*i:3*(i+1),:] = invR 74 | newNormals = np.array(teapotModel['N'].dot(invT)) 75 | normalize_v3(newNormals) 76 | 77 | return newNormals 78 | 79 | def saveObj(vertices, faces, normals, filePath): 80 | with open(filePath, 'w') as f: 81 | f.write("# OBJ file\n") 82 | for v in vertices: 83 | f.write("v %.4f %.4f %.4f\n" % (v[0], v[1], v[2])) 84 | for n in normals: 85 | f.write("vn %.4f %.4f %.4f\n" % (n[0], n[1], n[2])) 86 | for p in faces: 87 | f.write("f") 88 | for i in p: 89 | f.write(" %d" % (i + 1)) 90 | f.write("\n") 91 | 92 | def loadObject(fileName): 93 | with open(fileName, 'rb') as inpt: 94 | return pickle.load(inpt) 95 | 96 | def normalize_v3(arr): 97 | lens = np.sqrt( arr[:,0]**2 + arr[:,1]**2 + arr[:,2]**2 ) 98 | arr[:,0] /= lens 99 | arr[:,1] /= lens 100 | arr[:,2] /= lens 101 | return arr 102 | 103 | 104 | def getNormals(vertices, faces): 105 | norm = np.zeros( vertices.shape, dtype=vertices.dtype ) 106 | tris = vertices[faces] 107 | n = np.cross( tris[::,1 ] - tris[::,0] , tris[::,2 ] - tris[::,0] ) 108 | normalize_v3(n) 109 | norm[ faces[:,0] ] += n 110 | norm[ faces[:,1] ] += n 111 | norm[ faces[:,2] ] += n 112 | normalize_v3(norm) 113 | return norm 114 | 115 | def chGetNormals(vertices, faces): 116 | import opendr.geometry 117 | return opendr.geometry.VertNormals(vertices, faces).reshape((-1,3)) 118 | 119 | 120 | -------------------------------------------------------------------------------- /var_inf.py: -------------------------------------------------------------------------------- 1 | __author__ = 'pol' 2 | import ipdb 3 | import matplotlib 4 | matplotlib.use('Qt4Agg') 5 | from math import radians 6 | import chumpy as ch 7 | import numpy as np 8 | import cv2 9 | import matplotlib.pyplot as plt 10 | from sklearn import mixture 11 | from numpy.random import choice 12 | 13 | plt.ion() 14 | 15 | image = cv2.imread('opendr_GT.png') 16 | image = np.float64(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))/255.0 17 | 18 | nComps = 5 19 | nRecComps = 4 20 | gmm = mixture.GMM(n_components=nComps, covariance_type='spherical') 21 | win = 40 22 | colors = image[image.shape[0]/2-win:image.shape[0]/2+win,image.shape[1]/2-win:image.shape[1]/2+win,:].reshape([4*win*win,3]) 23 | gmm.fit(colors) 24 | 25 | imshape = [win*2,win*2,3] 26 | 27 | numPixels = win*2 * win*2 28 | chInput = ch.Ch(colors) 29 | numVars = chInput.size 30 | 31 | 32 | recSoftmaxW = ch.Ch(np.random.uniform(0,1, [nRecComps,numVars])/numVars) 33 | 34 | chRecLogistic = ch.exp(ch.dot(recSoftmaxW,chInput.reshape([numVars,1]))) 35 | chRecSoftmax = chRecLogistic.ravel()/ch.sum(chRecLogistic) 36 | 37 | chZRecComps = ch.zeros([numVars, nRecComps]) 38 | 39 | chZ = ch.zeros([numVars]) 40 | 41 | recMeans = ch.Ch(np.random.uniform(0,1, [3,nRecComps])) 42 | recCovars = 0.2 43 | chRecLogLikelihoods = - 0.5*(chZ.reshape([numPixels,3, 1]) - ch.tile(recMeans, [numPixels, 1,1]))**2 - ch.log((2 * recCovars) * (1/(ch.sqrt(recCovars) * np.sqrt(2 * np.pi)))) 44 | 45 | genZCovars = 0.2 46 | chGenComponentsProbs = ch.Ch(gmm.weights_) 47 | chCompMeans = ch.zeros([nComps, 3]) 48 | 49 | for comp in range(nComps): 50 | chCompMeans[comp, :] = gmm.means_[comp] 51 | 52 | chPZComp = ch.exp( - (ch.tile(chZ.reshape([numPixels,3,1]), [1, 1, nComps]) - chCompMeans.reshape([1,3, nComps]))**2 / (2 * genZCovars)) * (1/(ch.sqrt(genZCovars) * np.sqrt(2 * np.pi))) 53 | 54 | chPZ = ch.dot(chGenComponentsProbs.reshape([1,nComps]), chPZComp.reshape([5, numVars])) 55 | 56 | prec = 0.5 57 | 58 | covars = np.eye(colors.size, colors.size) 59 | # for row in np.arange(covars.shape[0]): 60 | # cols = [max(row-1, 0), min(row+1,colors.size-1)] 61 | # covars[row, cols[0]] = prec 62 | # covars[row, cols[1]] = prec 63 | # covars[cols[0], row] = prec 64 | # covars[cols[1], row] = prec 65 | 66 | # covars = np.linalg.inv(covars) 67 | detCov = 1 68 | # detCov = np.linalg.det(covars) 69 | # covars = [ch.Ch([covar] for covar in gmm.covars_)] 70 | chResiduals = chInput.ravel() - chZ.ravel() 71 | 72 | covar = 0.2 73 | 74 | ipdb.set_trace() 75 | 76 | chLogJoint = ch.log(chPZ.ravel()) - 0.5*covar*ch.dot(chResiduals,chResiduals) - 0.5*(ch.log(detCov) + numVars*ch.log((2 * np.pi))) 77 | # chGenMarginal = ch.prod(chLikelihoods) 78 | 79 | ipdb.set_trace() 80 | 81 | # likelihoodsZ = [chGenComponentsProbs[comp]*ch.exp( - (chInput - chZ)**2 / (2 * covars)) * (1/(ch.sqrt(covars) * np.sqrt(2 * np.pi))) for comp in range(nComps)] 82 | # chLikelihoodsZ = ch.concatenate(likelihoods) 83 | # chGenMarginalZ = ch.exp(ch.sum(ch.log(chLikelihoodsZ))) 84 | 85 | gmmRec = mixture.GMM(n_components=nRecComps, covariance_type='spherical') 86 | gmmRec.covars_=gmm.covars_.copy() 87 | 88 | 89 | #Update the mean of the gaussians and update the mixing weights. 90 | methods=['dogleg', 'minimize', 'BFGS', 'L-BFGS-B', 'Nelder-Mead'] 91 | free_vars = [recMeans.ravel(), recSoftmaxW] 92 | 93 | print("Beginning optimization.") 94 | while True: 95 | 96 | gmmRec.weights_=np.array(chRecSoftmax.r) 97 | gmmRec.means_=np.array(ch.concatenate(recMeans)) 98 | epsilon = np.random.randn(numVars) 99 | u = choice(nRecComps, size=1, p=chRecSoftmax.r) 100 | chZ[:] = chZRecComps[:,u].r.ravel() + recCovars*epsilon.ravel() 101 | pu = chRecSoftmax 102 | L = ch.log(pu[u]) + ch.sum(chLogJoint.ravel()) - ch.sum(chRecLogLikelihoods[:,:,u].ravel()) 103 | drL = L.dr_wrt(recMeans)/numPixels 104 | alpha = 0.1 105 | 106 | recSoftmaxW[:] = recSoftmaxW.r[:] + alpha*L.dr_wrt(recSoftmaxW).reshape(recSoftmaxW.shape)/numPixels 107 | ipdb.set_trace() 108 | chZ[:] = chZ.r[:] + alpha*L.dr_wrt(chZ).reshape(chZ.r.shape)/numPixels 109 | chZRecComps[:,u] = chZ.r[:] 110 | # ch.minimize({'raw': -L}, bounds=None, method=methods[1], x0=free_vars, callback=None, options={'disp':False, 'maxiter':1}) 111 | -------------------------------------------------------------------------------- /diffrender_experiment.py: -------------------------------------------------------------------------------- 1 | __author__ = 'pol' 2 | 3 | import matplotlib 4 | matplotlib.use('Qt4Agg') 5 | from math import radians 6 | import timeit 7 | import time 8 | import numpy as np 9 | from utils import * 10 | import matplotlib.pyplot as plt 11 | plt.ion() 12 | import h5py 13 | import ipdb 14 | import pickle 15 | 16 | ######################################### 17 | # Initialization ends here 18 | ######################################### 19 | 20 | seed = 1 21 | np.random.seed(seed) 22 | 23 | gtPrefix = 'objectnet3d_teapots' 24 | experimentPrefix = 'objectnet3d_teapots' 25 | experimentDescr = 'The real teapots' 26 | gtDir = 'groundtruth/' + gtPrefix + '/' 27 | experimentDir = 'experiments/' + experimentPrefix + '/' 28 | 29 | # groundTruthFilename = gtDir + 'groundTruth.h5' 30 | # gtDataFile = h5py.File(groundTruthFilename, 'r') 31 | # 32 | # onlySynthetic = False 33 | # 34 | # 35 | # print("Reading experiment data.") 36 | # 37 | # shapeGT = gtDataFile[gtPrefix].shape 38 | # 39 | # groundTruth = gtDataFile[gtPrefix] 40 | # 41 | # dataAzsGT = groundTruth['trainAzsGT'] 42 | # dataObjAzsGT = groundTruth['trainObjAzsGT'] 43 | # dataElevsGT = groundTruth['trainElevsGT'] 44 | # dataLightAzsGT = groundTruth['trainLightAzsGT'] 45 | # dataLightElevsGT = groundTruth['trainLightElevsGT'] 46 | # dataLightIntensitiesGT = groundTruth['trainLightIntensitiesGT'] 47 | # dataVColorGT = groundTruth['trainVColorGT'] 48 | # dataScenes = groundTruth['trainScenes'] 49 | # dataTeapotIds = groundTruth['trainTeapotIds'] 50 | # dataEnvMaps = groundTruth['trainEnvMaps'] 51 | # dataOcclusions = groundTruth['trainOcclusions'] 52 | # dataTargetIndices = groundTruth['trainTargetIndices'] 53 | # dataLightCoefficientsGT = groundTruth['trainLightCoefficientsGT'] 54 | # dataLightCoefficientsGTRel = groundTruth['trainLightCoefficientsGTRel'] 55 | # dataAmbientIntensityGT = groundTruth['trainAmbientIntensityGT'] 56 | # dataIds = groundTruth['trainIds'] 57 | # 58 | # gtDtype = groundTruth.dtype 59 | # 60 | # allDataIds = gtDataFile[gtPrefix]['trainIds'] 61 | 62 | 63 | ########## Check if there is anything wrong with the renders: 64 | 65 | # print("Reading images.") 66 | # # images = readImages(imagesDir, trainSet, False, loadFromHdf5) 67 | # writeHdf5 = False 68 | # writeGray = False 69 | # if writeHdf5: 70 | # writeImagesHdf5(gtDir, gtDir, allDataIds, writeGray) 71 | # if onlySynthetic: 72 | # imagesDir = gtDir + 'images_opendr/' 73 | # else: 74 | # imagesDir = gtDir + 'images/' 75 | # 76 | # loadGray = True 77 | # imagesAreH5 = False 78 | # loadGrayFromHdf5 = False 79 | # 80 | # if not imagesAreH5: 81 | # grayImages = readImages(imagesDir, allDataIds, loadGray, loadGrayFromHdf5) 82 | # else: 83 | # grayImages = h5py.File(imagesDir + 'images_gray.h5', 'r')["images"] 84 | # 85 | # badImages = np.where(np.mean(grayImages, (1,2)) < 0.01)[0] 86 | # 87 | # for id, badImage in enumerate(grayImages[badImages]): 88 | # print("There are bad images!") 89 | # plt.imsave('tmp/check/badImage' + str(badImages[id]) + '.png', np.tile(badImage[:,:,None], [1,1,3])) 90 | # 91 | 92 | size = 375 93 | # if not os.path.isfile(experimentDir + 'train.npy'): 94 | np.random.seed(seed) 95 | 96 | 97 | data = np.arange(size) 98 | np.random.shuffle(data) 99 | 100 | generateExperiment(size, experimentDir, 0, 1) 101 | 102 | # if not os.path.exists(experimentDir): 103 | # os.makedirs(experimentDir) 104 | # 105 | # np.save(experimentDir + 'train.npy', train) 106 | # np.save(experimentDir + 'test.npy', test) 107 | 108 | 109 | ########## Out of sample selections. 110 | 111 | # testSamplesIds= [2,4] 112 | # trainSamplesIds = [0,14,20,25,26,1] 113 | # 114 | # dataIdx = np.arange(shapeGT[0]) 115 | # train = np.array([],dtype=np.uint16) 116 | # test = np.array([],dtype=np.uint16) 117 | # for testId in testSamplesIds: 118 | # test = np.append(test, np.where(dataTeapotIds == testId)) 119 | # 120 | # for trainId in trainSamplesIds: 121 | # train = np.append(train, np.where(dataTeapotIds == trainId)) 122 | # 123 | # # boolTrainSet = np.ones(shapeGT[0]).astype(np.bool) 124 | # # boolTrainSet[test] = False 125 | # # train = dataIdx[boolTrainSet] 126 | # 127 | # np.random.shuffle(train) 128 | # np.random.shuffle(test) 129 | # 130 | # if not os.path.exists(experimentDir): 131 | # os.makedirs(experimentDir) 132 | # 133 | # np.save(experimentDir + 'train.npy', train) 134 | # np.save(experimentDir + 'test.npy', test) 135 | 136 | with open(experimentDir + 'description.txt', 'w') as expfile: 137 | expfile.write(experimentDescr) 138 | 139 | -------------------------------------------------------------------------------- /densecrf_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: python util_inference_example.py image annotations 3 | Adapted from the dense_inference.py to demonstate the usage of the util 4 | functions. 5 | """ 6 | 7 | import sys 8 | import numpy as np 9 | import cv2 10 | import pydensecrf.densecrf as dcrf 11 | import matplotlib.pylab as plt 12 | from skimage.segmentation import relabel_sequential 13 | import skimage 14 | from skimage.segmentation import slic 15 | import skimage.segmentation 16 | from skimage.segmentation import mark_boundaries 17 | from skimage.util import img_as_float 18 | import ipdb 19 | 20 | from pydensecrf.utils import compute_unary, create_pairwise_bilateral, \ 21 | create_pairwise_gaussian 22 | 23 | def unaryOcclusionModel(img, fgMask, probs): 24 | 25 | """ 26 | Simple classifier that is 50% certain that the annotation is correct 27 | (same as in the inference example). 28 | """ 29 | fg_energy = -np.log(probs[0]) 30 | occ_energy = -np.log(probs[1]) 31 | 32 | probBackground = 0.6 33 | 34 | U = np.zeros((3, fgMask.size), dtype='float32') 35 | 36 | U[0, :] = -np.log(0.0001) 37 | 38 | U[1, :] = -np.log((1-probBackground)) 39 | 40 | U[2, :] = -np.log(probBackground) 41 | 42 | U[0, fgMask] = fg_energy 43 | U[1, fgMask] = occ_energy 44 | U[2, fgMask] = -np.log(0.0001) 45 | 46 | return U 47 | 48 | def superpixelUnary(img, U, fgMask, iouProb): 49 | 50 | segments = skimage.segmentation.quickshift(img, ratio=1, max_dist=10, convert2lab=True) 51 | 52 | # plt.imshow(segments) 53 | 54 | fgMaskReshaped = fgMask.reshape([img.shape[0], img.shape[1]]) 55 | 56 | for seg_i in np.arange(np.max(segments)): 57 | currentSegment = segments == seg_i 58 | masksCat = np.concatenate([fgMaskReshaped[:,:,None], currentSegment[:,:,None]], axis=2) 59 | segmentationIOU = np.sum(np.all(masksCat, axis=2)) / np.sum(currentSegment) 60 | 61 | if segmentationIOU > 0.05 and segmentationIOU < 0.95: 62 | U[0, currentSegment.ravel()*fgMask] = -np.log(1 - iouProb) 63 | U[1, currentSegment.ravel()*fgMask] = -np.log(iouProb) 64 | 65 | U[2, currentSegment.ravel()*(~fgMask)] = -np.log(1 - iouProb) 66 | U[1, currentSegment.ravel()*(~fgMask)] = -np.log(iouProb) 67 | 68 | # # Edge pixels inside the mask (and far from boundary) are occluder pixels. How to choose the right one? (interior pixels) 69 | 70 | # if segmentationIOU >= 0.95 : 71 | # 72 | # U[0, currentSegment.ravel()*fgMask] = -np.log(iouProb) 73 | # U[1, currentSegment.ravel()*fgMask] = -np.log(1-iouProb) 74 | 75 | 76 | # # Segments with edges that match with boundary of the mask are fg. 77 | # if segmentationIOU >= 0.99: 78 | # U[0, currentSegment*fgMask] = 1 - iouProb 79 | # U[1, currentSegment*fgMask] = iouProb 80 | 81 | # U[2, currentSegment*(~fgMask)] = 1 - iouProb 82 | # U[1, currentSegment*(~fgMask)] = iouProb 83 | 84 | return U 85 | 86 | def boundaryUnary(img, U, fgMask, boundProb): 87 | 88 | from damascene import damascene 89 | 90 | #img must be uint8 0-255 91 | borders, textons, orientations = damascene(np.uint8(img[:,:,0:3]*255), device_num=0) 92 | 93 | 94 | 95 | 96 | return U 97 | 98 | def crfInference(imageGT, fgMask, probs): 99 | ################################## 100 | ### Read images and annotation ### 101 | ################################## 102 | # img = np.uint8(img*255) 103 | img = skimage.color.rgb2lab(imageGT) 104 | 105 | M = 3 # forground, background, occluding object. 106 | 107 | ########################### 108 | ### Setup the CRF model ### 109 | ########################### 110 | 111 | # Example using the DenseCRF class and the util functions 112 | crfmodel = dcrf.DenseCRF(img.shape[0] * img.shape[1], M) 113 | 114 | # get unary potentials (neg log probability) 115 | # U = compute_unary(labels, M) 116 | U = unaryOcclusionModel(img, fgMask, probs) 117 | 118 | U = superpixelUnary(imageGT, U, fgMask, 0.8) 119 | 120 | crfmodel.setUnaryEnergy(U) 121 | 122 | # This creates the color-independent features and then add them to the CRF 123 | feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2]) 124 | crfmodel.addPairwiseEnergy(feats, compat=3, 125 | kernel=dcrf.DIAG_KERNEL, 126 | normalization=dcrf.NORMALIZE_SYMMETRIC) 127 | 128 | # This creates the color-dependent features and then add them to the CRF 129 | feats = create_pairwise_bilateral(sdims=(30, 30), schan=(13, 13, 13), 130 | img=img, chdim=2) 131 | crfmodel.addPairwiseEnergy(feats, compat=10, 132 | kernel=dcrf.DIAG_KERNEL, 133 | normalization=dcrf.NORMALIZE_SYMMETRIC) 134 | 135 | #################################### 136 | ### Do inference and compute map ### 137 | #################################### 138 | Q = crfmodel.inference(5) 139 | mapseg = np.argmax(Q, axis=0).reshape(img.shape[:2]) 140 | 141 | # res = map.astype('float32') * 255 / map.max() 142 | # plt.imshow(res) 143 | # plt.show() 144 | 145 | # # Manually inference 146 | # Q, tmp1, tmp2 = crfmodel.startInference() 147 | # for i in range(5): 148 | # print("KL-divergence at {}: {}".format(i, crfmodel.klDivergence(Q))) 149 | # crfmodel.stepInference(Q, tmp1, tmp2) 150 | 151 | return mapseg, np.array(Q) -------------------------------------------------------------------------------- /recognition_models.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from sklearn import datasets, linear_model 4 | from sklearn.metrics import r2_score 5 | from collections import defaultdict 6 | from sklearn.ensemble import RandomForestRegressor 7 | from sklearn import mixture 8 | import ipdb 9 | 10 | from chumpy.ch import MatVecMult, Ch 11 | 12 | class segmentCRFModel(Ch): 13 | dterms = ['renderer', 'groundtruth', 'priorProbs'] 14 | 15 | def compute_r(self): 16 | return self.segmentation() 17 | 18 | def compute_dr_wrt(self, wrt): 19 | return None 20 | # if wrt is self.renderer: 21 | # return self.segmentation().dr_wrt(self.renderer) 22 | 23 | def segmentation(self): 24 | import densecrf_model 25 | 26 | vis_im = np.array(self.renderer.indices_image == 1).copy().astype(np.bool) 27 | bound_im = self.renderer.boundarybool_image.astype(np.bool) 28 | 29 | # 30 | segmentation, Q = densecrf_model.crfInference(self.groundtruth.r, vis_im, bound_im, [self.priorProbs[0], self.priorProbs[1], self.priorProbs[2]], 31 | self.resultDir + 'imgs/crf/Q_' + str(self.test_i)) 32 | return Q 33 | 34 | 35 | 36 | 37 | def evaluatePrediction(azsGT, elevsGT, azsPred, elevsPred): 38 | errazs = np.arctan2(np.sin(azsGT - azsPred), np.cos(azsGT - azsPred))*180/np.pi 39 | errelevs = np.arctan2(np.sin(elevsGT-elevsPred), np.cos(elevsGT-elevsPred))*180/np.pi 40 | return errazs, errelevs 41 | 42 | def trainRandomForest(xtrain, ytrain): 43 | randForest = RandomForestRegressor(n_estimators=400, n_jobs=-1) 44 | rf = randForest.fit(xtrain, ytrain) 45 | return rf 46 | 47 | def testRandomForest(randForest, xtest): 48 | return randForest.predict(xtest) 49 | 50 | def filteredMean(image, win): 51 | pixels = image[image.shape[0]/2-win:image.shape[0]/2+win,image.shape[1]/2-win:image.shape[1]/2+win,:] 52 | pixels = pixels.reshape([-1,3]) 53 | gray = 0.3*pixels[:,0] + 0.59*pixels[:,1] + 0.11*pixels[:,2] 54 | stdM = 2 55 | pixels[np.abs(gray - np.mean(gray)) < stdM * np.std(gray),:] 56 | color = np.mean(image) 57 | return color 58 | 59 | def meanColor(image, win): 60 | image = np.mean(image[image.shape[0]/2-win:image.shape[0]/2+win,image.shape[1]/2-win:image.shape[1]/2+win,:], axis=0) 61 | color = np.mean(image, axis=0) 62 | return color 63 | 64 | def medianColor(image, win): 65 | imageWin = image[image.shape[0]/2-win:image.shape[0]/2+win,image.shape[1]/2-win:image.shape[1]/2+win,:] 66 | color = np.median(imageWin.reshape([-1,3]), axis=0) 67 | return color 68 | 69 | def midColor(image): 70 | color = image[image.shape[0]/2,image.shape[1]/2,:] 71 | return color 72 | 73 | def colorGMM(image, win): 74 | np.random.seed(1) 75 | gmm = mixture.GMM(n_components=8, covariance_type='spherical') 76 | colors = image[image.shape[0]/2-win:image.shape[0]/2+win,image.shape[1]/2-win:image.shape[1]/2+win,:][:,3] 77 | gmm.fit(colors) 78 | gmm._weights=np.array([0.6,0.3,0.1,0,0,0,0,0]) 79 | return gmm 80 | 81 | from scipy.stats import vonmises 82 | 83 | def poseGMM(azimuth, elevation): 84 | np.random.seed(1) 85 | components = [0.7,0.05,0.05,0.05,0.05,0.05,0.05] 86 | azs = np.random.uniform(0,2*np.pi, 6) 87 | elevs = np.random.uniform(0,np.pi/2, 6) 88 | kappa = 50 89 | vmParamsAz = [(azs[i],kappa) for i in azs] 90 | vmParamsEl = [(elevs[i], kappa) for i in elevs] 91 | vmParamsAz = [(azimuth, kappa)] + vmParamsAz 92 | vmParamsEl = [(elevation, kappa)] + vmParamsEl 93 | return components, vmParamsAz, vmParamsEl 94 | 95 | 96 | def trainLinearRegression(xtrain, ytrain): 97 | # Create linear regression object 98 | regr = linear_model.LinearRegression(n_jobs=-1) 99 | # Train the model using the training sets 100 | regr.fit(xtrain, ytrain) 101 | # print('Coefficients: \n', regr.coef_) 102 | # 103 | # % (regr.predict(diabetes_X_test) - diabetes_y_test) ** 2)) 104 | # Explained variance score: 1 is perfect prediction 105 | # print('Variance score: %.2f' % regr.score(diabetes_X_test, diabetes_y_test)) 106 | 107 | # Plot outputs 108 | # plt.scatter(xtest, ytest, color='black') 109 | return regr 110 | 111 | 112 | def testLinearRegression(lrmodel, xtest): 113 | 114 | return lrmodel.predict(xtest) 115 | 116 | class sphericalHarmonicsModel(): 117 | 118 | def __init__(self, image=None, barycentric=None, visibility=None, SHComponents=None, f=None, vc=None, vn=None): 119 | 120 | self.SHComponents = SHComponents 121 | self.vn = vn 122 | self.visibility = visibility 123 | self.f = f 124 | self.vc = vc 125 | self.vn = vn 126 | self.barycentric = None 127 | visible = np.nonzero(visibility.ravel() != 4294967295)[0] 128 | vertexpix = np.where(self.barycentric[visible].ravel() <= 0.01) 129 | fvis = f[visibility.ravel()[visible]].ravel()[vertexpix] 130 | vcvis = vc[fvis] 131 | vnvis = vn[fvis] 132 | 133 | imvis = image[visible] 134 | 135 | evis = imvis/(vcvis+1e-5) 136 | 137 | self.X = vnvis 138 | self.y = evis 139 | 140 | # def fit(self, X, y): 141 | # 142 | # 143 | # 144 | # def score(self, X, y): 145 | # 146 | # def predict(self, X): 147 | 148 | def solveSHCoefficients(groundtruth, visibility, f, vn, vc): 149 | 150 | #RANSAC solution. 151 | 152 | 153 | #1 Select nine vertices. 154 | 155 | #2 Get E by dividing by vc. 156 | 157 | #3 We know the normalizing constant and transfer function. What is A_l? Probably wrt vertex normals. 158 | 159 | #4 Solve the coefficients. 160 | bestVertices = None 161 | return bestVertices 162 | 163 | -------------------------------------------------------------------------------- /export_collisions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4m 2 | 3 | import scene_io_utils 4 | import re 5 | from blender_utils import * 6 | from collision import * 7 | 8 | 9 | numpy.random.seed(1) 10 | 11 | inchToMeter = 0.0254 12 | outputDir = 'data/' 13 | 14 | width = 150 15 | height = 150 16 | 17 | numSamples = 100 18 | useCycles = False 19 | distance = 0.75 20 | 21 | scene_io_utils.loadTargetsBlendData() 22 | 23 | sceneCollisions = {} 24 | replaceableScenesFile = '../databaseFull/fields/scene_replaceables_backup_new.txt' 25 | sceneLines = [line.strip() for line in open(replaceableScenesFile)] 26 | teapots = [line.strip() for line in open('teapots.txt')] 27 | renderTeapotsList = np.arange(len(teapots)) 28 | 29 | scaleZ = 0.265 30 | scaleY = 0.18 31 | scaleX = 0.35 32 | cubeScale = mathutils.Matrix([[scaleX/2, 0,0,0], [0,scaleY/2,0 ,0] ,[0,0,scaleZ/2,0],[0,0,0,1]]) 33 | 34 | for sceneIdx in range(len(sceneLines))[:]: 35 | sceneLine = sceneLines[sceneIdx] 36 | sceneParts = sceneLine.split(' ') 37 | sceneFile = sceneParts[0] 38 | sceneNumber = int(re.search('.+?scene([0-9]+)\.txt', sceneFile, re.IGNORECASE).groups()[0]) 39 | scene_io_utils.loadTargetsBlendData() 40 | bpy.ops.wm.read_factory_settings() 41 | scene_io_utils.loadSceneBlendData(sceneIdx, replaceableScenesFile) 42 | scene = bpy.data.scenes['Main Scene'] 43 | 44 | scene.render.resolution_x = width #perhaps set resolution in code 45 | scene.render.resolution_y = height 46 | scene.render.tile_x = height/2 47 | scene.render.tile_y = width/2 48 | scene.cycles.samples = 10 49 | sceneNumber, sceneFileName, instances, roomName, roomInstanceNum, targetIndices, targetPositions = scene_io_utils.getSceneInformation(sceneIdx, replaceableScenesFile) 50 | 51 | roomName = '' 52 | for model in scene.objects: 53 | reg = re.compile('(room[0-9]+)') 54 | res = reg.match(model.name) 55 | if res: 56 | roomName = res.groups()[0] 57 | 58 | cubeTarget = bpy.data.scenes['Scene'].objects['Cube'] 59 | try: 60 | bpy.data.scenes['Main Scene'].objects.unlink(bpy.data.scenes['Scene'].objects['Cube']) 61 | except: 62 | pass 63 | 64 | scene.objects.link(cubeTarget) 65 | targetCollisions = {} 66 | for targetIdx in range(len(targetIndices)): 67 | targetParentIndex = targetIndices[targetIdx] 68 | targetParentPosition = np.array(targetPositions[targetIdx]) 69 | 70 | 71 | scene.objects.unlink(scene.objects[str(targetParentIndex)]) 72 | 73 | director = outputDir 74 | cubeTarget.layers[1] = True 75 | # objAzimuths = numpy.arange(0,360, 5) # Map it to non colliding rotations. 76 | objAzimuths = numpy.array([]) 77 | intersections = [] 78 | translationMat = mathutils.Matrix.Translation(targetParentPosition) 79 | 80 | azimuthRot = mathutils.Matrix.Rotation(radians(0), 4, 'Z') 81 | cubeTarget.matrix_world = translationMat * azimuthRot * cubeScale * mathutils.Matrix.Translation(mathutils.Vector((0,0,1))) 82 | scene.render.filepath = 'scenes/' + str(sceneNumber) + '/collisionCubeExample_' + str(targetIdx) + '.png' 83 | placeCamera(scene.camera, -90, 25, 0.75, targetParentPosition) 84 | bpy.ops.render.render(write_still=True) 85 | 86 | collisionInterval = 5 87 | for objAzimuth in numpy.arange(0,360,collisionInterval): 88 | azimuthRot = mathutils.Matrix.Rotation(radians(-objAzimuth), 4, 'Z') 89 | cubeTarget.matrix_world = translationMat * azimuthRot * cubeScale * mathutils.Matrix.Translation(mathutils.Vector((0,0,1))) 90 | # scene.render.filepath = 'scenes/' + str(sceneNumber) + '/' + str(objAzimuth) + 'collisionCubeExample_' + str(targetIdx) + '.png' 91 | # bpy.ops.render.render(write_still=True) 92 | intersect = False 93 | 94 | for sceneInstanceIdx, sceneInstance in enumerate(scene.objects): 95 | if sceneInstance.type == 'EMPTY' and sceneInstance != cubeTarget and sceneInstance.name != str(roomInstanceNum) and sceneInstance.name != str(instances[targetParentIndex][1]): 96 | intersect = instancesIntersect(mathutils.Matrix.Identity(4), [cubeTarget], sceneInstance.matrix_world, sceneInstance.dupli_group.objects) 97 | if intersect: 98 | print("Intersects with " + sceneInstance.name) 99 | break 100 | 101 | intersections = intersections + [[objAzimuth, intersect]] 102 | 103 | startInterval = True 104 | intervals = [] 105 | initInterval = 0 106 | endInterval = 0 107 | 108 | for idx, intersection in enumerate(intersections): 109 | if not intersection[1]: 110 | if startInterval: 111 | initInterval = intersection[0] 112 | startInterval = False 113 | else: 114 | if not startInterval: 115 | if idx >= 1 and intersections[idx-1][0] != initInterval: 116 | endInterval = intersection[0] - collisionInterval 117 | intervals = intervals + [[initInterval, endInterval]] 118 | startInterval = True 119 | 120 | if not intersection[1]: 121 | endInterval = intersection[0] 122 | if not intersections[0][1]: 123 | intervals = intervals + [[initInterval, endInterval+collisionInterval]] 124 | else: 125 | intervals = intervals + [[initInterval, endInterval]] 126 | 127 | targetCollisions[targetParentIndex] = (targetParentPosition, intervals) 128 | 129 | sceneCollisions[sceneNumber] = targetCollisions 130 | with open('data/collisions_new/collisionScene' + str(sceneNumber) + '.pickle', 'wb') as pfile: 131 | pickle.dump(targetCollisions, pfile) 132 | 133 | 134 | print("Collision detection ended.") 135 | # with open('data/collisions/collisions.pickle', 'wb') as pfile: 136 | # pickle.dump(sceneCollisions, pfile) 137 | # Cleanup 138 | 139 | # for scene in bpy.data.scenes: 140 | # # for objnum, obji in enumerate(scene.objects): 141 | # # 142 | # # obji.user_clear() 143 | # # bpy.data.objects.remove(obji) 144 | # # scene = bpy.data.scenes['Main Scene'] 145 | # if scene.name != 'Scene': 146 | # scene.user_clear() 147 | # bpy.data.scenes.remove(scene) 148 | # bpy.ops.scene.delete() 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /collision.py: -------------------------------------------------------------------------------- 1 | __author__ = 'pol' 2 | #From http://blender.stackexchange.com/a/9080 3 | 4 | import bpy 5 | import bmesh 6 | from blender_utils import * 7 | import blender_utils 8 | 9 | def bmesh_copy_from_object(obj, objTransf, transform=True, triangulate=True, apply_modifiers=False): 10 | assert (obj.type == 'MESH') 11 | 12 | if apply_modifiers and obj.modifiers: 13 | me = obj.to_mesh(bpy.context.scene, True, 'PREVIEW', calc_tessface=False) 14 | bm = bmesh.new() 15 | bm.from_mesh(me) 16 | bpy.data.meshes.remove(me) 17 | else: 18 | me = obj.data 19 | if obj.mode == 'EDIT': 20 | bm_orig = bmesh.from_edit_mesh(me) 21 | bm = bm_orig.copy() 22 | else: 23 | bm = bmesh.new() 24 | bm.from_mesh(me) 25 | 26 | # Remove custom data layers to save memory 27 | for elem in (bm.faces, bm.edges, bm.verts, bm.loops): 28 | for layers_name in dir(elem.layers): 29 | if not layers_name.startswith("_"): 30 | layers = getattr(elem.layers, layers_name) 31 | for layer_name, layer in layers.items(): 32 | layers.remove(layer) 33 | 34 | if transform: 35 | bm.transform(objTransf * obj.matrix_world) 36 | 37 | if triangulate: 38 | bmesh.ops.triangulate(bm, faces=bm.faces) 39 | 40 | return bm 41 | 42 | def aabb_intersect(matrix_world1, instanceObjs1, matrix_world2, instanceObjs2): 43 | 44 | minX1, maxX1 = blender_utils.modelWidth(instanceObjs1, matrix_world1) 45 | minY1, maxY1 = blender_utils.modelDepth(instanceObjs1, matrix_world1) 46 | minZ1, maxZ1 = blender_utils.modelHeight(instanceObjs1, matrix_world1) 47 | 48 | minX2, maxX2 = blender_utils.modelWidth(instanceObjs2, matrix_world2) 49 | minY2, maxY2 = blender_utils.modelDepth(instanceObjs2, matrix_world2) 50 | minZ2, maxZ2 = blender_utils.modelHeight(instanceObjs2, matrix_world2) 51 | 52 | return ((maxX1 > minX2) and (minX1 < maxX2) and (maxY1 > minY2) and (minY1 < maxY2) and (maxZ1 > minZ2) and (minZ1 < maxZ2)) 53 | 54 | 55 | def bmesh_check_intersect_objects(obj, objTransf, obj2, obj2Transf, selectface=False): 56 | """ 57 | Check if any faces intersect with the other object 58 | 59 | returns a boolean 60 | """ 61 | from mathutils.bvhtree import BVHTree 62 | assert(obj != obj2) 63 | 64 | # Triangulate 65 | bm = bmesh_copy_from_object(obj, objTransf, transform=True, triangulate=True) 66 | bm2 = bmesh_copy_from_object(obj2, obj2Transf, transform=True, triangulate=True) 67 | 68 | intersect = False 69 | BMT1 = BVHTree.FromBMesh(bm) 70 | BMT2 = BVHTree.FromBMesh(bm2) 71 | overlap_pairs = BMT1.overlap(BMT2) 72 | 73 | if len(overlap_pairs) > 0: 74 | intersect = True 75 | 76 | if selectface: 77 | # deselect everything for both objects 78 | bpy.context.scene.objects.active = obj 79 | bpy.ops.object.mode_set(mode='EDIT', toggle=False) 80 | bpy.ops.mesh.select_all(action='DESELECT') 81 | bpy.ops.object.mode_set(mode='OBJECT', toggle=False) 82 | bpy.context.scene.objects.active = obj2 83 | bpy.ops.object.mode_set(mode='EDIT', toggle=False) 84 | bpy.ops.mesh.select_all(action='DESELECT') 85 | bpy.ops.object.mode_set(mode='OBJECT', toggle=False) 86 | 87 | for each in overlap_pairs: 88 | obj.data.polygons[each[0]].select = True 89 | obj.update_from_editmode() 90 | obj2.data.polygons[each[1]].select = True 91 | obj2.update_from_editmode() 92 | 93 | bm.free() 94 | bm2.free() 95 | 96 | return intersect 97 | 98 | def instancesIntersect(matrix_world1, instanceObjs1, matrix_world2, instanceObjs2): 99 | 100 | if aabb_intersect(matrix_world1, instanceObjs1, matrix_world2, instanceObjs2): 101 | # print ("AABB intersection!") 102 | for mesh1 in instanceObjs1: 103 | for mesh2 in instanceObjs2: 104 | if bmesh_check_intersect_objects(mesh1, matrix_world1, mesh2, matrix_world2): 105 | # print ("There's a MESH intersection!") 106 | return True 107 | # else: 108 | # print ("There's NO intersection!") 109 | # print("There's NO intersection!") 110 | 111 | return False 112 | 113 | def targetSceneCollision(target, scene, roomName, targetParentInstance): 114 | # bpy.ops.mesh.primitive_cube_add 115 | for sceneInstance in scene.objects: 116 | if sceneInstance.type == 'EMPTY' and sceneInstance != target and sceneInstance.name != roomName and sceneInstance != targetParentInstance: 117 | if instancesIntersect(target.matrix_world, target.dupli_group.objects, sceneInstance.matrix_world, sceneInstance.dupli_group.objects): 118 | return True 119 | 120 | return False 121 | 122 | 123 | def targetCubeSceneCollision(target, scene, roomName, targetParentInstance): 124 | # bpy.ops.mesh.primitive_cube_add 125 | for sceneInstance in scene.objects: 126 | if sceneInstance.type == 'MESH' and sceneInstance != target and sceneInstance.name != roomName and sceneInstance != targetParentInstance: 127 | 128 | if instancesIntersect(mathutils.Matrix.Identity(4), [target], mathutils.Matrix.Identity(4), [sceneInstance]): 129 | return True 130 | 131 | return False 132 | 133 | def parseSceneCollisions(gtDir, scene_i, target_i, target, scene, targetPosOffset, chObjDistGT, chObjRotationGT, targetParentInstance, roomObj, distRange, rotationRange, distInterval, rotationInterval): 134 | 135 | scene.cycles.samples = 500 136 | 137 | 138 | original_matrix_world = target.matrix_world.copy() 139 | distBins = np.linspace(distInterval, distRange, distRange/distInterval) 140 | rotBins = np.linspace(0, rotationRange, rotationRange / rotationInterval) 141 | 142 | totalBins = np.meshgrid(distBins, rotBins) 143 | totalBins[0] = np.append([0], totalBins[0].ravel()) 144 | totalBins[1] = np.append([0], totalBins[1].ravel()) 145 | 146 | boolBins = np.zeros(len(totalBins[0].ravel())).astype(np.bool) 147 | 148 | scene.update() 149 | 150 | for bin_i in range(len(totalBins[0].ravel())): 151 | dist = totalBins[0].ravel()[bin_i] 152 | rot = totalBins[1].ravel()[bin_i] 153 | chObjDistGT[:]= dist 154 | chObjRotationGT[:]= rot 155 | 156 | ignore = False 157 | 158 | azimuthRot = mathutils.Matrix.Rotation(0, 4, 'Z') 159 | 160 | target.matrix_world = mathutils.Matrix.Translation(original_matrix_world.to_translation() + mathutils.Vector(targetPosOffset.r)) * azimuthRot * (mathutils.Matrix.Translation(-original_matrix_world.to_translation())) * original_matrix_world 161 | 162 | if targetCubeSceneCollision(target, scene, roomObj.name, targetParentInstance): 163 | # print("Teapot intersects with an object.") 164 | ignore = True 165 | # pass 166 | 167 | if not instancesIntersect(mathutils.Matrix.Translation(mathutils.Vector((0, 0, -0.01))), [target], mathutils.Matrix.Identity(4), [targetParentInstance]): 168 | # print("Teapot not on table.") 169 | ignore = True 170 | 171 | if instancesIntersect(mathutils.Matrix.Translation(mathutils.Vector((0, 0, +0.02))), [target], mathutils.Matrix.Identity(4), [targetParentInstance]): 172 | # print("Teapot interesects supporting object.") 173 | ignore = True 174 | 175 | # ipdb.set_trace() 176 | if instancesIntersect(mathutils.Matrix.Identity(4), [target], mathutils.Matrix.Identity(4), [roomObj]): 177 | # print("Teapot intersects room") 178 | ignore = True 179 | 180 | boolBins[bin_i] = not ignore 181 | 182 | # bpy.ops.render.render(write_still=True) 183 | # 184 | # import imageio 185 | # 186 | # image = np.array(imageio.imread(scene.render.filepath))[:, :, 0:3] 187 | # 188 | # from blender_utils import lin2srgb 189 | # blender_utils.lin2srgb(image) 190 | # 191 | # cv2.imwrite(gtDir + 'images/scene' + str(scene_i) + '_' + 'targetidx' + str(target_i) + 'bin' + str(bin_i) + '_ignore' + str(ignore) + '.jpeg', 255 * image[:, :, [2, 1, 0]], [int(cv2.IMWRITE_JPEG_QUALITY), 100]) 192 | 193 | return boolBins, totalBins 194 | -------------------------------------------------------------------------------- /lasagne_visualize.py: -------------------------------------------------------------------------------- 1 | from itertools import product 2 | 3 | from lasagne.layers import get_output 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import theano 7 | import theano.tensor as T 8 | 9 | 10 | def plot_loss(net): 11 | train_loss = [row['train_loss'] for row in net.train_history_] 12 | valid_loss = [row['valid_loss'] for row in net.train_history_] 13 | plt.plot(train_loss, label='train loss') 14 | plt.plot(valid_loss, label='valid loss') 15 | plt.xlabel('epoch') 16 | plt.ylabel('loss') 17 | plt.legend(loc='best') 18 | 19 | 20 | def plot_conv_weights(layer, figsize=(6, 6)): 21 | """Plot the weights of a specific layer. 22 | 23 | Only really makes sense with convolutional layers. 24 | 25 | Parameters 26 | ---------- 27 | layer : lasagne.layers.Layer 28 | 29 | """ 30 | W = layer.W.get_value() 31 | shape = W.shape 32 | nrows = np.ceil(np.sqrt(shape[0])).astype(int) 33 | ncols = nrows 34 | 35 | for feature_map in range(shape[1]): 36 | figs, axes = plt.subplots(nrows, ncols, figsize=figsize) 37 | 38 | for ax in axes.flatten(): 39 | ax.set_xticks([]) 40 | ax.set_yticks([]) 41 | ax.axis('off') 42 | 43 | for i, (r, c) in enumerate(product(range(nrows), range(ncols))): 44 | if i >= shape[0]: 45 | break 46 | axes[r, c].imshow(W[i, feature_map], cmap='gray', 47 | interpolation='nearest') 48 | 49 | 50 | def plot_conv_activity(layer, x, figsize=(6, 8)): 51 | """Plot the acitivities of a specific layer. 52 | 53 | Only really makes sense with layers that work 2D data (2D 54 | convolutional layers, 2D pooling layers ...). 55 | 56 | Parameters 57 | ---------- 58 | layer : lasagne.layers.Layer 59 | 60 | x : numpy.ndarray 61 | Only takes one sample at a time, i.e. x.shape[0] == 1. 62 | 63 | """ 64 | if x.shape[0] != 1: 65 | raise ValueError("Only one sample can be plotted at a time.") 66 | 67 | # compile theano function 68 | xs = T.tensor4('xs').astype(theano.config.floatX) 69 | get_activity = theano.function([xs], get_output(layer, xs)) 70 | 71 | activity = get_activity(x) 72 | shape = activity.shape 73 | nrows = np.ceil(np.sqrt(shape[1])).astype(int) 74 | ncols = nrows 75 | 76 | figs, axes = plt.subplots(nrows + 1, ncols, figsize=figsize) 77 | axes[0, ncols // 2].imshow(1 - x[0][0], cmap='gray', 78 | interpolation='nearest') 79 | axes[0, ncols // 2].set_title('original') 80 | 81 | for ax in axes.flatten(): 82 | ax.set_xticks([]) 83 | ax.set_yticks([]) 84 | ax.axis('off') 85 | 86 | for i, (r, c) in enumerate(product(range(nrows), range(ncols))): 87 | if i >= shape[1]: 88 | break 89 | ndim = activity[0][i].ndim 90 | if ndim != 2: 91 | raise ValueError("Wrong number of dimensions, image data should " 92 | "have 2, instead got {}".format(ndim)) 93 | axes[r + 1, c].imshow(-activity[0][i], cmap='gray', 94 | interpolation='nearest') 95 | 96 | 97 | def occlusion_heatmap(net, x, target, square_length=7): 98 | """An occlusion test that checks an image for its critical parts. 99 | 100 | In this function, a square part of the image is occluded (i.e. set 101 | to 0) and then the net is tested for its propensity to predict the 102 | correct label. One should expect that this propensity shrinks of 103 | critical parts of the image are occluded. If not, this indicates 104 | overfitting. 105 | 106 | Depending on the depth of the net and the size of the image, this 107 | function may take awhile to finish, since one prediction for each 108 | pixel of the image is made. 109 | 110 | Currently, all color channels are occluded at the same time. Also, 111 | this does not really work if images are randomly distorted by the 112 | batch iterator. 113 | 114 | See paper: Zeiler, Fergus 2013 115 | 116 | Parameters 117 | ---------- 118 | net : NeuralNet instance 119 | The neural net to test. 120 | 121 | x : np.array 122 | The input data, should be of shape (1, c, x, y). Only makes 123 | sense with image data. 124 | 125 | target : int 126 | The true value of the image. If the net makes several 127 | predictions, say 10 classes, this indicates which one to look 128 | at. 129 | 130 | square_length : int (default=7) 131 | The length of the side of the square that occludes the image. 132 | Must be an odd number. 133 | 134 | Results 135 | ------- 136 | heat_array : np.array (with same size as image) 137 | An 2D np.array that at each point (i, j) contains the predicted 138 | probability of the correct class if the image is occluded by a 139 | square with center (i, j). 140 | 141 | """ 142 | if (x.ndim != 4) or x.shape[0] != 1: 143 | raise ValueError("This function requires the input data to be of " 144 | "shape (1, c, x, y), instead got {}".format(x.shape)) 145 | if square_length % 2 == 0: 146 | raise ValueError("Square length has to be an odd number, instead " 147 | "got {}.".format(square_length)) 148 | 149 | num_classes = net.layers_[-1].num_units 150 | img = x[0].copy() 151 | bs, col, s0, s1 = x.shape 152 | 153 | heat_array = np.zeros((s0, s1)) 154 | pad = square_length // 2 + 1 155 | x_occluded = np.zeros((s1, col, s0, s1), dtype=img.dtype) 156 | probs = np.zeros((s0, s1, num_classes)) 157 | 158 | # generate occluded images 159 | for i in range(s0): 160 | # batch s1 occluded images for faster prediction 161 | for j in range(s1): 162 | x_pad = np.pad(img, ((0, 0), (pad, pad), (pad, pad)), 'constant') 163 | x_pad[:, i:i + square_length, j:j + square_length] = 0. 164 | x_occluded[j] = x_pad[:, pad:-pad, pad:-pad] 165 | y_proba = net.predict_proba(x_occluded) 166 | probs[i] = y_proba.reshape(s1, num_classes) 167 | 168 | # from predicted probabilities, pick only those of target class 169 | for i in range(s0): 170 | for j in range(s1): 171 | heat_array[i, j] = probs[i, j, target] 172 | return heat_array 173 | 174 | 175 | def plot_occlusion(net, X, target, square_length=7, figsize=(9, None)): 176 | """Plot which parts of an image are particularly import for the 177 | net to classify the image correctly. 178 | 179 | See paper: Zeiler, Fergus 2013 180 | 181 | Parameters 182 | ---------- 183 | net : NeuralNet instance 184 | The neural net to test. 185 | 186 | X : numpy.array 187 | The input data, should be of shape (b, c, 0, 1). Only makes 188 | sense with image data. 189 | 190 | target : list or numpy.array of ints 191 | The true values of the image. If the net makes several 192 | predictions, say 10 classes, this indicates which one to look 193 | at. If more than one sample is passed to X, each of them needs 194 | its own target. 195 | 196 | square_length : int (default=7) 197 | The length of the side of the square that occludes the image. 198 | Must be an odd number. 199 | 200 | figsize : tuple (int, int) 201 | Size of the figure. 202 | 203 | Plots 204 | ----- 205 | Figure with 3 subplots: the original image, the occlusion heatmap, 206 | and both images super-imposed. 207 | 208 | """ 209 | if (X.ndim != 4): 210 | raise ValueError("This function requires the input data to be of " 211 | "shape (b, c, x, y), instead got {}".format(X.shape)) 212 | 213 | num_images = X.shape[0] 214 | if figsize[1] is None: 215 | figsize = (figsize[0], num_images * figsize[0] / 3) 216 | figs, axes = plt.subplots(num_images, 3, figsize=figsize) 217 | 218 | for ax in axes.flatten(): 219 | ax.set_xticks([]) 220 | ax.set_yticks([]) 221 | ax.axis('off') 222 | 223 | for n in range(num_images): 224 | heat_img = occlusion_heatmap( 225 | net, X[n:n + 1, :, :, :], target[n], square_length 226 | ) 227 | 228 | ax = axes if num_images == 1 else axes[n] 229 | img = X[n, :, :, :].mean(0) 230 | ax[0].imshow(-img, interpolation='nearest', cmap='gray') 231 | ax[0].set_title('image') 232 | ax[1].imshow(-heat_img, interpolation='nearest', cmap='Reds') 233 | ax[1].set_title('critical parts') 234 | ax[2].imshow(-img, interpolation='nearest', cmap='gray') 235 | ax[2].imshow(-heat_img, interpolation='nearest', cmap='Reds', 236 | alpha=0.6) 237 | ax[2].set_title('super-imposed') 238 | -------------------------------------------------------------------------------- /diffrender_opt.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Javier Gonzalez 2 | # Copyright (c) 2015, the GPy Authors (see GPy AUTHORS.txt) 3 | # Licensed under the BSD 3-clause license (see LICENSE.txt) 4 | 5 | import numpy as np 6 | import GPyOpt 7 | import GPy 8 | import ipdb 9 | from numpy.random import seed 10 | import chumpy as ch 11 | 12 | """ 13 | This is a simple demo to demonstrate the use of Bayesian optimization with GPyOpt with some simple options. Run the example by writing: 14 | 15 | import GPyOpt 16 | BO_demo_2d = GPyOpt.demos.advanced_optimization_2d() 17 | 18 | As a result you should see: 19 | 20 | - A plot with the model and the current acquisition function 21 | - A plot with the diagnostic plots of the optimization. 22 | - An object call BO_demo_auto that contains the results of the optimization process (see reference manual for details). Among the available results you have access to the GP model via 23 | 24 | >> BO_demo_2d.model 25 | 26 | and to the location of the best found location writing. 27 | 28 | BO_demo_2d.x_opt 29 | 30 | """ 31 | 32 | 33 | 34 | 35 | def opendrObjectiveFunction(obj, free_variables): 36 | 37 | def changevars(vs, obj, free_variables): 38 | vs = vs.ravel() 39 | cur = 0 40 | changed = False 41 | for idx, freevar in enumerate(free_variables): 42 | sz = freevar.r.size 43 | 44 | newvals = vs[cur:cur+sz].copy().reshape(free_variables[idx].shape) 45 | if np.max(np.abs(newvals-free_variables[idx]).ravel()) > 0: 46 | free_variables[idx][:] = newvals 47 | changed = True 48 | 49 | cur += sz 50 | 51 | return changed 52 | 53 | def objFun(vs): 54 | vs = np.array(vs) 55 | if vs.shape[0] == 1: 56 | changevars(vs, obj, free_variables) 57 | return obj.r.reshape([1,1]) 58 | else: 59 | res = [] 60 | for vs_i in vs: 61 | changevars(vs_i, obj, free_variables) 62 | res = res + [obj.r.reshape([1,1])] 63 | 64 | return np.vstack(res) 65 | 66 | return objFun 67 | 68 | def opendrObjectiveFunctionCRF(free_variables, rendererGT, renderer, color, chVColors, chSHLightCoeffs, lightCoeffs, free_variables_app_light, resultDir, test_i, stds, method, updateColor=False,minAppLight=False): 69 | 70 | def changevars(vs, free_variables): 71 | vs = vs.ravel() 72 | cur = 0 73 | changed = False 74 | for idx, freevar in enumerate(free_variables): 75 | sz = freevar.r.size 76 | 77 | newvals = vs[cur:cur+sz].copy().reshape(free_variables[idx].shape) 78 | if np.max(np.abs(newvals-free_variables[idx]).ravel()) > 0: 79 | free_variables[idx][:] = newvals 80 | changed = True 81 | 82 | cur += sz 83 | 84 | return changed 85 | 86 | def objFun(vs): 87 | 88 | vs = np.array(vs) 89 | res = [] 90 | for vs_it, vs_i in enumerate(vs): 91 | changevars(vs_i, free_variables) 92 | 93 | import densecrf_model 94 | 95 | vis_im = np.array(renderer.indices_image == 1).copy().astype(np.bool) 96 | bound_im = renderer.boundarybool_image.astype(np.bool) 97 | 98 | segmentation, Q = densecrf_model.crfInference(rendererGT.r, vis_im, bound_im, [0.75,0.25,0.01], resultDir + 'imgs/crf/Q_' + str(test_i) + '_it' + str(vs_it)) 99 | vColor = color 100 | if updateColor: 101 | if np.sum(segmentation == 0) > 5: 102 | segmentRegion = segmentation == 0 103 | vColor = np.median(rendererGT.reshape([-1, 3])[segmentRegion.ravel()], axis=0) * 1.4 104 | vColor = vColor / max(np.max(vColor), 1.) 105 | 106 | chVColors[:] = vColor 107 | chSHLightCoeffs[:] = lightCoeffs 108 | 109 | variances = stds**2 110 | 111 | fgProb = ch.exp( - (renderer - rendererGT)**2 / (2 * variances)) * (1./(stds * np.sqrt(2 * np.pi))) 112 | 113 | 114 | h = renderer.r.shape[0] 115 | w = renderer.r.shape[1] 116 | 117 | occProb = np.ones([h,w]) 118 | bgProb = np.ones([h,w]) 119 | 120 | errorFun = -ch.sum(ch.log(vis_im[:, :, None]*((Q[0].reshape([h, w, 1]) * fgProb) + (Q[1].reshape([h, w]) * occProb + Q[2].reshape([h, w]) * bgProb)[:, :, None]) + (1- vis_im[:, :, None])))/(h*w) 121 | 122 | if minAppLight: 123 | options = {'disp': False, 'maxiter': 10} 124 | 125 | def cb(_): 126 | print("Error: " + str(errorFun.r)) 127 | 128 | ch.minimize({'raw': errorFun}, bounds=None, method=method, x0=free_variables_app_light, callback=cb, options=options) 129 | 130 | res = res + [errorFun.r.reshape([1,1])] 131 | 132 | return np.vstack(res) 133 | 134 | return objFun 135 | 136 | def bayesOpt(objFun, initX, initF, bounds): 137 | 138 | seed(12345) 139 | 140 | input_dim = len(bounds) 141 | 142 | # Select an specific kernel from GPy 143 | kernel = GPy.kern.RBF(input_dim, variance=.1, lengthscale=1) + GPy.kern.Bias(input_dim) # we add a bias kernel 144 | 145 | # --- Problem definition and optimization 146 | BO_model = GPyOpt.methods.BayesianOptimization(f=objFun, # function to optimize 147 | kernel = kernel, # pre-specified model 148 | X = initX, 149 | Y = initF, 150 | bounds=bounds, # box-constrains of the problem 151 | acquisition='EI', # Selects the Expected improvement 152 | numdata_initial_design=len(initX), 153 | type_initial_design='random', # latin desing of the initial points 154 | normalize = True) # normalized y 155 | 156 | # Run the optimization 157 | max_iter = 10 158 | 159 | # --- Run the optimization # evaluation budget 160 | ipdb.set_trace() 161 | BO_model.run_optimization(max_iter, n_inbatch=10, n_procs=10, # Number of iterations 162 | acqu_optimize_method = 'DIRECT', # method to optimize the acq. function 163 | acqu_optimize_restarts = 1, # number of local optimizers 164 | eps=10e-2, # secondary stop criteria (apart from the number of iterations) 165 | true_gradients = True) # The gradients of the acquisition function are approximated (faster) 166 | 167 | # # --- Plots 168 | # if plots: 169 | # objective_true.plot() 170 | # BO_demo_2d.plot_acquisition() 171 | # BO_demo_2d.plot_convergence() 172 | 173 | return BO_model 174 | 175 | def advanced_optimization_2d(plots=True): 176 | import GPyOpt 177 | import GPy 178 | from numpy.random import seed 179 | seed(12345) 180 | 181 | # --- Objective function 182 | objective_true = GPyOpt.fmodels.experiments2d.sixhumpcamel() # true function 183 | objective_noisy = GPyOpt.fmodels.experiments2d.sixhumpcamel(sd = 0.1) # noisy version 184 | bounds = objective_noisy.bounds # problem constrains 185 | input_dim = len(bounds) 186 | 187 | 188 | # Select an specific kernel from GPy 189 | kernel = GPy.kern.RBF(input_dim, variance=.1, lengthscale=.1) + GPy.kern.Bias(input_dim) # we add a bias kernel 190 | 191 | 192 | # --- Problem definition and optimization 193 | BO_demo_2d = GPyOpt.methods.BayesianOptimization(f=objective_noisy.f, # function to optimize 194 | kernel = kernel, # pre-specified model 195 | bounds=bounds, # box-constrains of the problem 196 | acquisition='EI', # Selects the Expected improvement 197 | acquisition_par = 2, # parameter of the acquisition function 198 | numdata_initial_design = 15, # 15 initial points 199 | type_initial_design='random', # latin desing of the initial points 200 | model_optimize_interval= 2, # The model is updated every two points are collected 201 | normalize = True) # normalized y 202 | 203 | 204 | # Run the optimization 205 | max_iter = 20 206 | 207 | # --- Run the optimization # evaluation budget 208 | BO_demo_2d.run_optimization(max_iter, # Number of iterations 209 | acqu_optimize_method = 'DIRECT', # method to optimize the acq. function 210 | acqu_optimize_restarts = 30, # number of local optimizers 211 | eps=10e-6, # secondary stop criteria (apart from the number of iterations) 212 | true_gradients = True) # The gradients of the acquisition function are approximated (faster) 213 | 214 | 215 | # --- Plots 216 | if plots: 217 | objective_true.plot() 218 | BO_demo_2d.plot_acquisition() 219 | BO_demo_2d.plot_convergence() 220 | 221 | 222 | return BO_demo_2d -------------------------------------------------------------------------------- /teapots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4m 2 | 3 | from blender_utils import * 4 | 5 | baseDir = '../databaseFull/models/' 6 | 7 | lines = [line.strip() for line in open('teapots.txt')] 8 | 9 | for object in bpy.data.scenes['Scene'].objects: print(object.name) 10 | 11 | lamp = bpy.data.scenes['Scene'].objects[1] 12 | lamp.location = (0,0.0,1.5) 13 | 14 | camera = bpy.data.scenes['Scene'].objects[2] 15 | 16 | camera.data.angle = 60 * 180 / numpy.pi 17 | 18 | distance = 0.5 19 | originalLoc = mathutils.Vector((0,-distance,0.0)) 20 | 21 | elevation = 0.0 22 | azimuth = 0.0 23 | elevationRot = mathutils.Matrix.Rotation(radians(-elevation), 4, 'X') 24 | azimuthRot = mathutils.Matrix.Rotation(radians(-azimuth), 4, 'Z') 25 | location = azimuthRot * elevationRot * originalLoc 26 | camera.location = location 27 | 28 | look_at(camera, mathutils.Vector((0,0,0))) 29 | 30 | 31 | world = bpy.context.scene.world 32 | 33 | 34 | # Environment lighting 35 | world.light_settings.use_environment_light = True 36 | world.light_settings.environment_energy = 0.2 37 | world.horizon_color = mathutils.Color((0.0,0.0,0.0)) 38 | 39 | width = 200 40 | height = 200 41 | 42 | 43 | data, images, experiments = loadData() 44 | 45 | groundTruthEls = data['azimuths'][0][0][0] 46 | groundTruthAzs = data['altitudes'][0][0][0] 47 | 48 | filenames = [name[0] for name in data['filenames'][0][0][0][:]] 49 | 50 | 51 | # images["images"][i] 52 | 53 | labels = numpy.column_stack((numpy.cos(groundTruthAzs*numpy.pi/180), numpy.sin(groundTruthAzs*numpy.pi/180), numpy.cos(groundTruthAzs*numpy.pi/180.0), numpy.sin(groundTruthAzs*numpy.pi/180.0))) 54 | 55 | 56 | expi = 3 57 | 58 | experiment = experiments['experiments'][0][0][0][expi] 59 | 60 | selTrain = experiment['selTrain'][0][0][0] 61 | 62 | selTest = experiment['selTest'][0][0][0] 63 | 64 | output = scipy.io.loadmat('../data/crossval6div2-hog8-alldataexperiments.mat')['output_data'] 65 | 66 | idx = output['idx'][0][0][expi][0] 67 | 68 | # filenames = [u''.join(chr(c[0]) for c in fdata[name[0]]) for name in numpy.array(data["filenames"])[:][:]] 69 | 70 | nnpredazs = output['nnpredradazs'][0][0][expi][0]*180.0/numpy.pi 71 | nnpredalts = output['nnpredradalts'][0][0][expi][0]*180.0/numpy.pi 72 | # rtpredazs = output['rtpredradazs'][0][0][expi][0]*180.0/numpy.pi 73 | # rtpredalts = output['rtpredradalts'][0][0][expi][0]*180.0/numpy.pi 74 | 75 | predazs =nnpredazs.squeeze() 76 | predalts=nnpredalts.squeeze() 77 | 78 | numTests = selTest.size 79 | 80 | bestModels= [""]*numTests 81 | bestScores = numpy.ones(numTests)*999999 82 | bestAzimuths = numpy.zeros(numTests) 83 | bestElevations = numpy.zeros(numTests) 84 | 85 | predi = 0 86 | 87 | # selTest[[10384,10397,10408,10440,10442,10446,10458,10469,10478,10492]]: 88 | for selTestNum in [10384, 10397, 10408]: 89 | 90 | test = selTest[selTestNum] 91 | rgbTestImage = numpy.transpose(images["images"][test]) 92 | testImage = cv2.cvtColor(numpy.float32(rgbTestImage*255), cv2.COLOR_RGB2BGR)/255.0 93 | 94 | testImageEdges = cv2.Canny(numpy.uint8(testImage*255), 50,150) 95 | cv2.imwrite("canny_" + str(test) + ".png" , testImageEdges) 96 | cv2.imwrite("image_" + str(test) + ".png" , numpy.uint8(testImage*255)) 97 | 98 | score = 9999999 99 | 100 | for teapot in lines[0:5]: 101 | 102 | 103 | fullTeapot = baseDir + teapot 104 | 105 | print("Reading " + fullTeapot + '.dae') 106 | 107 | bpy.ops.scene.new() 108 | bpy.context.scene.name = teapot 109 | scene = bpy.context.scene 110 | 111 | scene.objects.link(lamp) 112 | 113 | scene.camera = camera 114 | 115 | # scene.render.use_raytrace = True 116 | # scene.render.antialiasing_samples = '16' 117 | 118 | 119 | scene.render.resolution_x = width #perhaps set resolution in code 120 | scene.render.resolution_y = height 121 | scene.world = world 122 | 123 | scene.render.filepath = teapot + '.png' 124 | 125 | bpy.utils.collada_import(fullTeapot + '.dae') 126 | 127 | minZ, maxZ = modelHeight(scene) 128 | 129 | minY, maxY = modelWidth(scene) 130 | 131 | 132 | scaleZ = 0.254/(maxZ-minZ) 133 | scaleY = 0.1778/(maxY-minY) 134 | 135 | scale = min(scaleZ, scaleY) 136 | 137 | for mesh in scene.objects: 138 | if mesh.type == 'MESH': 139 | scaleMat = mathutils.Matrix.Scale(scale, 4) 140 | mesh.matrix_world = scaleMat * mesh.matrix_world 141 | 142 | minZ, maxZ = modelHeight(scene) 143 | 144 | center = centerOfGeometry(scene) 145 | for mesh in scene.objects: 146 | if mesh.type == 'MESH': 147 | mesh.matrix_world = mathutils.Matrix.Translation(-center) * mesh.matrix_world 148 | 149 | #Rotate the object to the azimuth angle we define as 0. 150 | rot = mathutils.Matrix.Rotation(radians(90), 4, 'Z') 151 | rotateMatrixWorld(scene, rot) 152 | 153 | camera.data.angle = 60 * 180 / numpy.pi 154 | 155 | stopSearchEl = False 156 | stopSearchAz = False 157 | dirEl = 1 158 | dirAz = 1 159 | 160 | elevation = predalts[selTestNum] 161 | azimuth = predazs[selTestNum] 162 | center = centerOfGeometry(scene) 163 | elevationRot = mathutils.Matrix.Rotation(radians(-elevation), 4, 'X') 164 | azimuthRot = mathutils.Matrix.Rotation(radians(azimuth), 4, 'Z') 165 | location = azimuthRot * elevationRot * (center + originalLoc) 166 | camera.location = location 167 | scene.update() 168 | look_at(camera, center) 169 | scene.update() 170 | 171 | 172 | bpy.ops.render.render( write_still=False ) 173 | 174 | 175 | blendImage = bpy.data.images['Render Result'] 176 | image = numpy.flipud(numpy.array(blendImage.extract_render(scene=scene)).reshape([height/2,width/2,4])) 177 | image[numpy.where(image > 1)] = 1 178 | 179 | # image = cv2.imread(teapot + '.png', cv2.IMREAD_ANYDEPTH) 180 | # image = numpy.float16(image)/255.0 181 | 182 | distance = getChamferDistance(testImage, image) 183 | 184 | 185 | if distance < score: 186 | score = distance 187 | bestModels[predi] = teapot 188 | bestScores[predi] = score 189 | bestElevations[predi] = elevation 190 | bestAzimuths[predi] = azimuth 191 | 192 | 193 | # while not stopSearchEl: 194 | # elevation = (elevation + dirEl*2) % 90 195 | 196 | # elevationRot = mathutils.Matrix.Rotation(radians(-elevation), 4, 'X') 197 | # location = azimuthRot * elevationRot * (center + originalLoc) 198 | 199 | # camera.location = location 200 | # scene.update() 201 | 202 | # look_at(camera, center) 203 | 204 | # bpy.ops.render.render( write_still=False ) 205 | 206 | # blendImage = bpy.data.images['Render Result'] 207 | 208 | # image = numpy.flipud(numpy.array(blendImage.extract_render(scene=scene)).reshape([height/2,width/2,4])) 209 | 210 | # # # Truncate intensities larger than 1. 211 | # image[numpy.where(image > 1)] = 1 212 | 213 | ## image = cv2.imread(teapot + '.png', cv2.IMREAD_ANYDEPTH) 214 | 215 | ## image = numpy.float16(image)/255.0 216 | 217 | # distance = getChamferDistance(image, testImage) 218 | 219 | # if distance < score: 220 | # score = distance 221 | # bestModels[predi] = teapot 222 | # bestScores[predi] = score 223 | # bestElevations[predi] = elevation 224 | 225 | # elif dirEl > 0: 226 | # elevation = predalts[selTestNum] 227 | # dirEl = -1 228 | # else: 229 | # stopSearchEl = True 230 | 231 | # iaz = 0 232 | # azimuth = 0 233 | # while not stopSearchAz: 234 | # azimuth = (azimuth + dirAz*5) % 360 235 | 236 | # azimuthRot = mathutils.Matrix.Rotation(radians(azimuth), 4, 'Z') 237 | # location = azimuthRot * elevationRot * (center + originalLoc) 238 | # camera.location = location 239 | # scene.update() 240 | # look_at(camera, center) 241 | # scene.update() 242 | 243 | 244 | # bpy.ops.render.render( write_still=False ) 245 | 246 | 247 | # blendImage = bpy.data.images['Render Result'] 248 | 249 | # # image = numpy.flipud(numpy.array(blendImage.extract_render(scene=scene)).reshape([height/2,width/2,4]))[:,:,0:3] 250 | 251 | # # # Truncate intensities larger than 1. 252 | # # image[numpy.where(image > 1)] = 1 253 | 254 | # image = cv2.imread(teapot + '.png', cv2.IMREAD_ANYDEPTH) 255 | # image = numpy.float32(image)/255.0 256 | 257 | # distance = getChamferDistance(testImage, image) 258 | 259 | # if distance < score: 260 | # score = distance 261 | # bestModels[predi] = teapot 262 | # bestScores[predi] = score 263 | # bestAzimuths[predi] = azimuth 264 | # imageEdges = cv2.Canny(numpy.uint8(image*255.0), 25,225) 265 | # cv2.imwrite(teapot + "_canny_" + str(test) + ".png" , imageEdges) 266 | # # elif dirAz > 0: 267 | # # azimuth = predazs[selTestNum] 268 | # # dirAz = -1 269 | # # else: 270 | # # stopSearchAz = True 271 | # if azimuth >= 355: 272 | # stopSearchAz = True 273 | 274 | 275 | 276 | 277 | # # Save best image. 278 | # # im = Image.fromarray(numpy.uint8(image*255)) 279 | 280 | # # im.save(teapot + '.png') 281 | 282 | 283 | # # Cleanup 284 | 285 | # for obji in scene.objects: 286 | # if obji.type == 'MESH': 287 | # obji.user_clear() 288 | # bpy.data.objects.remove(obji) 289 | 290 | # scene.user_clear() 291 | # bpy.ops.scene.delete() 292 | 293 | predi = predi + 1 294 | 295 | -------------------------------------------------------------------------------- /azimuth_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4m 2 | 3 | from blender_utils import * 4 | 5 | # bpy.ops.render.render( write_still=True ) 6 | 7 | 8 | lines = [line.strip() for line in open('teapots.txt')] 9 | 10 | # lamp = bpy.data.scenes['Scene'].objects[1] 11 | # lamp.location = (0,0.0,1.5) 12 | 13 | lamp_data = bpy.data.lamps.new(name="LampTopData", type='AREA') 14 | lamp = bpy.data.objects.new(name="LampTop", object_data=lamp_data) 15 | lamp.location = (0,0.0,2) 16 | lamp.data.energy = 0.004 17 | lamp.data.size = 0.5 18 | lamp.data.use_diffuse = True 19 | # lamp.data.use_nodes = True 20 | 21 | 22 | lamp_data2 = bpy.data.lamps.new(name="LampBotData", type='POINT') 23 | lamp2 = bpy.data.objects.new(name="LampBot", object_data=lamp_data2) 24 | lamp2.location = (0,0.0,-1.0) 25 | lamp2.data.energy = 0.2 26 | # lamp.data.size = 0.25 27 | lamp2.data.use_diffuse = True 28 | lamp2.data.use_specular = False 29 | # lamp2.data.use_nodes = True 30 | 31 | 32 | camera = bpy.data.scenes['Scene'].objects[2] 33 | 34 | camera.data.angle = 60 * 180 / numpy.pi 35 | 36 | distance = 0.5 37 | originalLoc = mathutils.Vector((0,-distance,0.0)) 38 | 39 | elevation = 0.0 40 | azimuth = 0.0 41 | elevationRot = mathutils.Matrix.Rotation(radians(-elevation), 4, 'X') 42 | azimuthRot = mathutils.Matrix.Rotation(radians(-azimuth), 4, 'Z') 43 | location = azimuthRot * elevationRot * originalLoc 44 | camera.location = location 45 | 46 | look_at(camera, mathutils.Vector((0,0,0))) 47 | 48 | world = bpy.context.scene.world 49 | 50 | # Environment lighting 51 | world.light_settings.use_environment_light = True 52 | world.light_settings.environment_energy = 0.15 53 | world.horizon_color = mathutils.Color((0.0,0.0,0.0)) 54 | 55 | width = 230 56 | height = 230 57 | 58 | data, images, experiments = loadData() 59 | 60 | groundTruthEls = data['altitudes'][0][0][0] 61 | groundTruthAzs = data['azimuths'][0][0][0] 62 | 63 | filenames = [name[0] for name in data['filenames'][0][0][0][:]] 64 | ids = [name[0] for name in data['ids'][0][0][0][:]] 65 | 66 | labels = numpy.column_stack((numpy.cos(groundTruthAzs*numpy.pi/180), numpy.sin(groundTruthAzs*numpy.pi/180), numpy.cos(groundTruthAzs*numpy.pi/180.0), numpy.sin(groundTruthAzs*numpy.pi/180.0))) 67 | 68 | output = scipy.io.loadmat('../data/crossval6div2-hog8-alldataexperiments.mat')['output_data'] 69 | 70 | numpy.random.seed(1) 71 | 72 | minThresTemplate = 10 73 | maxThresTemplate = 100 74 | minThresImage = 50 75 | maxThresImage = 150 76 | 77 | baseDir = '../databaseFull/models/' 78 | 79 | experimentTeapots = ['teapots/fa1fa0818738e932924ed4f13e49b59d/Teapot N300912','teapots/c7549b28656181c91bff71a472da9153/Teapot N311012', 'teapots/1c43a79bd6d814c84a0fee00d66a5e35/Teapot', 'teapots/a7fa82f5982edfd033da2d90df7af046/Teapot_fixed', 'teapots/8e6a162e707ecdf323c90f8b869f2ce9/Teapot N280912', 'teapots/12b81ec72a967dc1714fc48a3b0c961a/Teapot N260113_fixed'] 80 | # experimentTeapots = ['teapots/fa1fa0818738e932924ed4f13e49b59d/Teapot N300912','teapots/c7549b28656181c91bff71a472da9153/Teapot N311012', 'teapots/1c43a79bd6d814c84a0fee00d66a5e35/Teapot'] 81 | 82 | outputExperiments = [] 83 | 84 | # distanceTypes = ['chamferDataToModel', 'robustChamferDataToModel', 'sqDistImages', 'robustSqDistImages'] 85 | distanceTypes = ['chamferDataToModel', 'ignoreSqDistImages', 'sqDistImages', 'chamferModelToData'] 86 | 87 | 88 | for teapotTest in experimentTeapots: 89 | robust = True 90 | robustScale = 0 91 | 92 | for distanceType in distanceTypes: 93 | robust = ~robust 94 | if robust is False: 95 | robustScale = 0 96 | 97 | 98 | experiment = {} 99 | 100 | indices = [i for i, s in enumerate(ids) if teapotTest in s] 101 | 102 | selTest = indices 103 | selTest = numpy.random.permutation(selTest) 104 | numTests = len(selTest) 105 | 106 | 107 | teapot = teapotTest + '_cleaned' 108 | fullTeapot = baseDir + teapot 109 | 110 | print("Reading " + fullTeapot + '.dae') 111 | 112 | bpy.ops.scene.new() 113 | bpy.context.scene.name = teapot 114 | scene = bpy.context.scene 115 | bpy.context.scene.render.engine = 'CYCLES' 116 | # bpy.context.scene.cycles.samples = 128 117 | 118 | scene.camera = camera 119 | 120 | scene.render.resolution_x = width #perhaps set resolution in code 121 | scene.render.resolution_y = height 122 | scene.render.resolution_percentage = 100 123 | scene.world = world 124 | 125 | scene.render.filepath = teapot + '.png' 126 | 127 | bpy.utils.collada_import(fullTeapot + '.dae') 128 | 129 | # modifySpecular(scene, 0.3) 130 | 131 | # ipdb.set_trace() 132 | 133 | minZ, maxZ = modelHeight(scene) 134 | 135 | minY, maxY = modelWidth(scene) 136 | 137 | scaleZ = 0.254/(maxZ-minZ) 138 | scaleY = 0.1778/(maxY-minY) 139 | 140 | scale = min(scaleZ, scaleY) 141 | 142 | for mesh in scene.objects: 143 | if mesh.type == 'MESH': 144 | scaleMat = mathutils.Matrix.Scale(scale, 4) 145 | mesh.matrix_world = scaleMat * mesh.matrix_world 146 | 147 | minZ, maxZ = modelHeight(scene) 148 | scene.objects.link(lamp2) 149 | 150 | scene.objects.link(lamp) 151 | 152 | # lamp2.location = (0,0, 2) 153 | 154 | 155 | center = centerOfGeometry(scene) 156 | for mesh in scene.objects: 157 | if mesh.type == 'MESH': 158 | mesh.matrix_world = mathutils.Matrix.Translation(-center) * mesh.matrix_world 159 | 160 | #Rotate the object to the azimuth angle we define as 0. 161 | rot = mathutils.Matrix.Rotation(radians(90), 4, 'Z') 162 | rotateMatrixWorld(scene, rot) 163 | scene.update() 164 | 165 | camera.data.angle = 60 * 180 / numpy.pi 166 | 167 | 168 | performance = numpy.array([]) 169 | elevations = numpy.array([]) 170 | groundTruthAzimuths = numpy.array([]) 171 | bestAzimuths= numpy.array([]) 172 | 173 | 174 | expSelTest = numpy.arange(0,numTests,int(numTests/100)) 175 | 176 | 177 | for selTestNum in expSelTest: 178 | 179 | test = selTest[selTestNum] 180 | groundTruthAz = groundTruthAzs[test] 181 | groundTruthEl = groundTruthEls[test] 182 | scores = [] 183 | azimuths = [] 184 | directory = 'aztest/' + '_' + teapot.replace("/", "") + '/' + distanceType 185 | if not os.path.exists(directory): 186 | os.makedirs(directory) 187 | 188 | 189 | if not os.path.exists(directory + 'test_samples'): 190 | os.makedirs(directory + 'test_samples') 191 | 192 | 193 | numDir = directory + 'test_samples/num' + str(test) + '_azim' + str(int(groundTruthAz)) + '_elev' + str(int(groundTruthEl)) + '/' 194 | if not os.path.exists(numDir): 195 | os.makedirs(numDir) 196 | 197 | 198 | rgbTestImage = numpy.transpose(images["images"][test]) 199 | testImage = cv2.cvtColor(numpy.float32(rgbTestImage*255), cv2.COLOR_RGB2BGR)/255.0 200 | 201 | testImageEdges = cv2.Canny(numpy.uint8(testImage*255), minThresImage,maxThresImage) 202 | cv2.imwrite(numDir + "image_canny" + ".png" , testImageEdges) 203 | cv2.imwrite(numDir + "image" + ".png" , numpy.uint8(testImage*255)) 204 | 205 | score = numpy.finfo(numpy.float64).max 206 | 207 | 208 | 209 | elevation = groundTruthEls[test] 210 | # elevation = -45 211 | azimuth = 0 212 | center = centerOfGeometry(scene) 213 | elevationRot = mathutils.Matrix.Rotation(radians(-elevation), 4, 'X') 214 | # azimuthRot = mathutils.Matrix.Rotation(radians(azimuth), 4, 'Z') 215 | # location = azimuthRot * elevationRot * (center + originalLoc) 216 | # camera.location = location 217 | # scene.update() 218 | # look_at(camera, center) 219 | # scene.update() 220 | 221 | # bpy.ops.render.render( write_still=False ) 222 | 223 | # blendImage = bpy.data.images['Render Result'] 224 | # image = numpy.flipud(numpy.array(blendImage.extract_render(scene=scene)).reshape([height/2,width/2,4])) 225 | # image[numpy.where(image > 1)] = 1 226 | 227 | # distance = getChamferDistance(testImage, image, minThresImage, maxThresImage, minThresTemplate, maxThresTemplate) 228 | 229 | for azimuth in numpy.arange(0,360,5): 230 | 231 | azimuthRot = mathutils.Matrix.Rotation(radians(azimuth), 4, 'Z') 232 | location = azimuthRot * elevationRot * (center + originalLoc) 233 | camera.location = location 234 | scene.update() 235 | look_at(camera, center) 236 | scene.update() 237 | 238 | 239 | scene.render.filepath = directory + teapot.replace("/", "") + "blender_" + '_' + str(test) + "_az" + '%.1f' % azimuth + '_dist' + '%.1f' % distance + '.png' 240 | 241 | bpy.ops.render.render( write_still=False ) 242 | 243 | # image = cv2.imread(scene.render.filepath, cv2.IMREAD_ANYDEPTH) 244 | 245 | blendImage = bpy.data.images['Render Result'] 246 | 247 | image = numpy.flipud(numpy.array(blendImage.extract_render(scene=scene)).reshape([height/scene.render.resolution_percentage/100,width/scene.render.resolution_percentage/100,4]))[7:107,7:107,0:3] 248 | 249 | # Truncate intensities larger than 1. 250 | image[numpy.where(image > 1)] = 1 251 | # ipdb.set_trace() 252 | image[0:20, 75:100, :] = 0 253 | 254 | image = cv2.cvtColor(numpy.float32(image*255), cv2.COLOR_RGB2BGR)/255.0 255 | 256 | methodParams = {'scale': robustScale, 'minThresImage': minThresImage, 'maxThresImage': maxThresImage, 'minThresTemplate': minThresTemplate, 'maxThresTemplate': maxThresTemplate} 257 | 258 | 259 | distance = scoreImage(testImage, image, distanceType, methodParams) 260 | cv2.imwrite(numDir + 'image' + "_az" + '%.1f' % azimuth + '_dist' + '%.1f' % distance + '.png', numpy.uint8(image*255.0)) 261 | 262 | if distance <= score: 263 | imageEdges = cv2.Canny(numpy.uint8(image*255.0), minThresTemplate,maxThresTemplate) 264 | bestImageEdges = imageEdges 265 | bestImage = image 266 | score = distance 267 | 268 | 269 | scores.append(distance) 270 | azimuths.append(azimuth) 271 | 272 | 273 | 274 | bestAzimuth = azimuths[numpy.argmin(scores)] 275 | if robust is False: 276 | robustScale = 1.4826 * numpy.sqrt(numpy.median(scores)) 277 | 278 | error = numpy.arctan2(numpy.sin((groundTruthAz-bestAzimuth)*numpy.pi/180), numpy.cos((groundTruthAz-bestAzimuth)*numpy.pi/180))*180/numpy.pi 279 | performance = numpy.append(performance, error) 280 | elevations = numpy.append(elevations, elevation) 281 | bestAzimuths = numpy.append(bestAzimuths, bestAzimuth) 282 | groundTruthAzimuths = numpy.append(groundTruthAzimuths, groundTruthAz) 283 | 284 | cv2.imwrite(numDir + 'bestImage' + "_canny_az" + '%.1f' % bestAzimuth + '_dist' + '%.1f' % score + '.png' , bestImageEdges) 285 | cv2.imwrite(numDir + 'bestImage' + "_az" + '%.1f' % bestAzimuth + '_dist' + '%.1f' % score + '.png', numpy.uint8(bestImage*255.0)) 286 | 287 | imgEdges = cv2.Canny(numpy.uint8(testImage*255), minThresImage,maxThresImage) 288 | bwEdges1 = cv2.distanceTransform(~imgEdges, cv2.DIST_L2, 5) 289 | disp = cv2.normalize(bwEdges1, bwEdges1, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U) 290 | cv2.imwrite(numDir + 'dist_transform' + '.png', disp) 291 | 292 | plt.plot(azimuths, numpy.array(scores)) 293 | plt.xlabel('Azimuth (degrees)') 294 | plt.ylabel('Distance') 295 | plt.title('Chamfer distance') 296 | plt.axvline(x=bestAzimuth, linewidth=2, color='b', label='Minimum distance azimuth') 297 | plt.axvline(x=groundTruthAz, linewidth=2, color='g', label='Ground truth azimuth') 298 | plt.axvline(x=(bestAzimuth + 180) % 360, linewidth=1, color='b', ls='--', label='Minimum distance azimuth + 180') 299 | fontP = FontProperties() 300 | fontP.set_size('small') 301 | x1,x2,y1,y2 = plt.axis() 302 | plt.axis((0,360,0,y2)) 303 | # plt.legend() 304 | plt.savefig(numDir + 'performance.png') 305 | plt.clf() 306 | 307 | 308 | experiment = {'methodParams': methodParams, 'distanceType': distanceType, 'teapot':teapot, 'bestAzimuths':bestAzimuths, 'performance': performance, 'elevations':elevations, 'groundTruthAzimuths': groundTruthAzimuths, 'selTest':selTest, 'expSelTest':expSelTest} 309 | outputExperiments.append(experiment) 310 | with open(directory + 'experiment.pickle', 'wb') as pfile: 311 | pickle.dump(experiment, pfile) 312 | 313 | plt.scatter(elevations, performance) 314 | plt.xlabel('Elevation (degrees)') 315 | plt.ylabel('Angular error') 316 | x1,x2,y1,y2 = plt.axis() 317 | plt.axis((0,90,-180,180)) 318 | plt.title('Performance scatter plot') 319 | plt.savefig(directory + '_elev-performance-scatter.png') 320 | plt.clf() 321 | 322 | plt.scatter(groundTruthAzimuths, performance) 323 | plt.xlabel('Azimuth (degrees)') 324 | plt.ylabel('Angular error') 325 | x1,x2,y1,y2 = plt.axis() 326 | plt.axis((0,360,-180,180)) 327 | plt.title('Performance scatter plot') 328 | plt.savefig(directory + '_azimuth-performance-scatter.png') 329 | plt.clf() 330 | 331 | 332 | plt.hist(performance, bins=36) 333 | plt.xlabel('Angular error') 334 | plt.ylabel('Counts') 335 | x1,x2,y1,y2 = plt.axis() 336 | plt.axis((-180,180,0, y2)) 337 | plt.title('Performance histogram') 338 | plt.savefig(directory + '_performance-histogram.png') 339 | plt.clf() 340 | # experimentFile = 'aztest/teapotsc7549b28656181c91bff71a472da9153Teapot N311012_cleaned.pickle' 341 | # with open(experimentFile, 'rb') as pfile: 342 | # experiment = pickle.load( pfile) 343 | 344 | headers=["Best global fit", ""] 345 | table = [["Mean angular error", numpy.mean(numpy.abs(performance))],["Median angualar error",numpy.median(numpy.abs(performance))]] 346 | performanceTable = tabulate(table, tablefmt="latex", floatfmt=".1f") 347 | 348 | with open(directory + 'performance.tex', 'w') as expfile: 349 | expfile.write(performanceTable) 350 | 351 | # Cleanup 352 | # for obji in scene.objects: 353 | # if obji.type == 'MESH': 354 | # obji.user_clear() 355 | # bpy.data.objects.remove(obji) 356 | 357 | # scene.user_clear() 358 | # bpy.ops.scene.delete() 359 | 360 | print("Finished the experiment") 361 | 362 | 363 | 364 | -------------------------------------------------------------------------------- /export_occlusions.py: -------------------------------------------------------------------------------- 1 | __author__ = 'pol' 2 | 3 | import matplotlib 4 | matplotlib.use('Qt4Agg') 5 | import bpy 6 | import scene_io_utils 7 | import mathutils 8 | from math import radians 9 | import timeit 10 | import time 11 | import opendr 12 | import chumpy as ch 13 | import geometry 14 | import image_processing 15 | import numpy as np 16 | import cv2 17 | from blender_utils import * 18 | import glfw 19 | import generative_models 20 | import matplotlib.pyplot as plt 21 | from opendr_utils import * 22 | import OpenGL.GL as GL 23 | import light_probes 24 | import imageio 25 | from OpenGL import contextdata 26 | 27 | plt.ion() 28 | 29 | ######################################### 30 | # Initialization starts here 31 | ######################################### 32 | 33 | #Main script options: 34 | useBlender = False 35 | loadBlenderSceneFile = True 36 | groundTruthBlender = False 37 | useCycles = True 38 | unpackModelsFromBlender = False 39 | unpackSceneFromBlender = False 40 | loadSavedSH = False 41 | glModes = ['glfw','mesa'] 42 | glMode = glModes[0] 43 | 44 | width, height = (150, 150) 45 | win = -1 46 | 47 | if glMode == 'glfw': 48 | #Initialize base GLFW context for the Demo and to share context among all renderers. 49 | glfw.init() 50 | glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 3) 51 | glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 3) 52 | # glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, GL.GL_TRUE) 53 | glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) 54 | glfw.window_hint(glfw.DEPTH_BITS,32) 55 | glfw.window_hint(glfw.VISIBLE, GL.GL_FALSE) 56 | # win = glfw.create_window(width, height, "Demo", None, None) 57 | # glfw.make_context_current(win) 58 | 59 | angle = 60 * 180 / numpy.pi 60 | clip_start = 0.01 61 | clip_end = 10 62 | frustum = {'near': clip_start, 'far': clip_end, 'width': width, 'height': height} 63 | camDistance = 0.4 64 | 65 | teapots = [line.strip() for line in open('teapots.txt')] 66 | renderTeapotsList = np.arange(len(teapots)) 67 | sceneIdx = 0 68 | replaceableScenesFile = '../databaseFull/fields/scene_replaceables_backup_new.txt' 69 | sceneNumber, sceneFileName, instances, roomName, roomInstanceNum, targetIndices, targetPositions = scene_io_utils.getSceneInformation(sceneIdx, replaceableScenesFile) 70 | sceneDicFile = 'data/scene' + str(sceneNumber) + '.pickle' 71 | targetParentIdx = 0 72 | targetIndex = targetIndices[targetParentIdx] 73 | targetParentPosition = targetPositions[targetParentIdx] 74 | targetPosition = targetParentPosition 75 | 76 | tex_srgb2lin = True 77 | 78 | v, f_list, vc, vn, uv, haveTextures_list, textures_list = scene_io_utils.loadSavedScene(sceneDicFile, tex_srgb2lin) 79 | 80 | removeObjectData(int(targetIndex), v, f_list, vc, vn, uv, haveTextures_list, textures_list) 81 | 82 | targetModels = [] 83 | blender_teapots = [] 84 | teapots = [line.strip() for line in open('teapots.txt')] 85 | selection = [ teapots[i] for i in renderTeapotsList] 86 | scene_io_utils.loadTargetsBlendData() 87 | for teapotIdx, teapotName in enumerate(selection): 88 | teapot = bpy.data.scenes[teapotName[0:63]].objects['teapotInstance' + str(renderTeapotsList[teapotIdx])] 89 | teapot.layers[1] = True 90 | teapot.layers[2] = True 91 | targetModels = targetModels + [teapot] 92 | blender_teapots = blender_teapots + [teapot] 93 | 94 | 95 | v_teapots, f_list_teapots, vc_teapots, vn_teapots, uv_teapots, haveTextures_list_teapots, textures_list_teapots, vflat, varray, center_teapots = scene_io_utils.loadTeapotsOpenDRData(renderTeapotsList, useBlender, unpackModelsFromBlender, targetModels) 96 | 97 | azimuth = np.pi 98 | chCosAz = ch.Ch([np.cos(azimuth)]) 99 | chSinAz = ch.Ch([np.sin(azimuth)]) 100 | 101 | chAz = 2*ch.arctan(chSinAz/(ch.sqrt(chCosAz**2 + chSinAz**2) + chCosAz)) 102 | chAz = ch.Ch([np.pi/4]) 103 | chObjAz = ch.Ch([np.pi/4]) 104 | chAzRel = chAz - chObjAz 105 | 106 | elevation = 0 107 | chLogCosEl = ch.Ch(np.log(np.cos(elevation))) 108 | chLogSinEl = ch.Ch(np.log(np.sin(elevation))) 109 | chEl = 2*ch.arctan(ch.exp(chLogSinEl)/(ch.sqrt(ch.exp(chLogCosEl)**2 + ch.exp(chLogSinEl)**2) + ch.exp(chLogCosEl))) 110 | chEl = ch.Ch([0.95993109]) 111 | chDist = ch.Ch([camDistance]) 112 | 113 | chObjAzGT = ch.Ch([np.pi*3/2]) 114 | chAzGT = ch.Ch([np.pi*3/2]) 115 | chAzRelGT = chAzGT - chObjAzGT 116 | chElGT = ch.Ch(chEl.r[0]) 117 | chDistGT = ch.Ch([camDistance]) 118 | chComponentGT = ch.Ch(np.array([2, 0.25, 0.25, 0.12,-0.17,0.36,0.1,0.,0.])) 119 | chComponent = ch.Ch(np.array([2, 0.25, 0.25, 0.12,-0.17,0.36,0.1,0.,0.])) 120 | 121 | chPointLightIntensity = ch.Ch([1]) 122 | chPointLightIntensityGT = ch.Ch([1]) 123 | chLightAz = ch.Ch([0.0]) 124 | chLightEl = ch.Ch([np.pi/2]) 125 | chLightDist = ch.Ch([0.5]) 126 | chLightDistGT = ch.Ch([0.5]) 127 | chLightAzGT = ch.Ch([0.0]) 128 | chLightElGT = ch.Ch([np.pi/4]) 129 | 130 | ligthTransf = computeHemisphereTransformation(chLightAz, chLightEl, chLightDist, targetPosition) 131 | ligthTransfGT = computeHemisphereTransformation(chLightAzGT, chLightElGT, chLightDistGT, targetPosition) 132 | 133 | lightPos = ch.dot(ligthTransf, ch.Ch([0.,0.,0.,1.]))[0:3] 134 | lightPos = ch.Ch([targetPosition[0]+0.5,targetPosition[1],targetPosition[2] + 0.5]) 135 | lightPosGT = ch.dot(ligthTransfGT, ch.Ch([0.,0.,0.,1.]))[0:3] 136 | 137 | chGlobalConstant = ch.Ch([0.5]) 138 | chGlobalConstantGT = ch.Ch([0.5]) 139 | light_color = ch.ones(3)*chPointLightIntensity 140 | light_colorGT = ch.ones(3)*chPointLightIntensityGT 141 | chVColors = ch.Ch([0.8,0.8,0.8]) 142 | chVColorsGT = ch.Ch([0.8,0.8,0.8]) 143 | 144 | shCoefficientsFile = 'data/sceneSH' + str(sceneIdx) + '.pickle' 145 | 146 | chAmbientIntensityGT = ch.Ch([0.025]) 147 | clampedCosCoeffs = clampedCosineCoefficients() 148 | chAmbientSHGT = ch.zeros([9]) 149 | 150 | envMapDic = {} 151 | SHFilename = 'data/LightSHCoefficients.pickle' 152 | 153 | with open(SHFilename, 'rb') as pfile: 154 | envMapDic = pickle.load(pfile) 155 | 156 | phiOffset = ch.Ch([0]) 157 | totalOffset = phiOffset + chObjAzGT 158 | envMapCoeffs = ch.Ch(list(envMapDic.items())[0][1][1]) 159 | 160 | envMapCoeffsRotated = ch.Ch(np.dot(light_probes.chSphericalHarmonicsZRotation(totalOffset), envMapCoeffs[[0,3,2,1,4,5,6,7,8]])[[0,3,2,1,4,5,6,7,8]]) 161 | envMapCoeffsRotatedRel = ch.Ch(np.dot(light_probes.chSphericalHarmonicsZRotation(phiOffset), envMapCoeffs[[0,3,2,1,4,5,6,7,8]])[[0,3,2,1,4,5,6,7,8]]) 162 | 163 | shCoeffsRGB = envMapCoeffsRotated 164 | shCoeffsRGBRel = envMapCoeffsRotatedRel 165 | chShCoeffs = 0.3*shCoeffsRGB[:,0] + 0.59*shCoeffsRGB[:,1] + 0.11*shCoeffsRGB[:,2] 166 | chShCoeffsRel = 0.3*shCoeffsRGBRel[:,0] + 0.59*shCoeffsRGBRel[:,1] + 0.11*shCoeffsRGBRel[:,2] 167 | chAmbientSHGT = chShCoeffs.ravel() * chAmbientIntensityGT * clampedCosCoeffs * 10 168 | chAmbientSHGTRel = chShCoeffsRel.ravel() * chAmbientIntensityGT * clampedCosCoeffs 169 | 170 | chLightRadGT = ch.Ch([0.1]) 171 | chLightDistGT = ch.Ch([0.5]) 172 | chLightIntensityGT = ch.Ch([0]) 173 | chLightAzGT = ch.Ch([np.pi*3/2]) 174 | chLightElGT = ch.Ch([np.pi/4]) 175 | angle = ch.arcsin(chLightRadGT/chLightDistGT) 176 | zGT = chZonalHarmonics(angle) 177 | shDirLightGT = chZonalToSphericalHarmonics(zGT, np.pi/2 - chLightElGT, chLightAzGT + chObjAzGT - np.pi/2) * clampedCosCoeffs 178 | shDirLightGTRel = chZonalToSphericalHarmonics(zGT, np.pi/2 - chLightElGT, chLightAzGT - np.pi/2) * clampedCosCoeffs 179 | chComponentGT = chAmbientSHGT 180 | # chComponentGT = ch.Ch(chAmbientSHGT.r[:].copy()) 181 | # + shDirLightGT*chLightIntensityGT 182 | chComponentGTRel = chAmbientSHGTRel 183 | # chComponentGTRel = ch.Ch(chAmbientSHGTRel.r[:].copy()) 184 | # chComponentGT = chAmbientSHGT.r[:] + shDirLightGT.r[:]*chLightIntensityGT.r[:] 185 | 186 | chDisplacement = ch.Ch([0.0, 0.0,0.0]) 187 | chDisplacementGT = ch.Ch([0.0,0.0,0.0]) 188 | chScale = ch.Ch([1.0,1.0,1.0]) 189 | chScaleGT = ch.Ch([1, 1.,1.]) 190 | 191 | currentTeapotModel = 0 192 | 193 | addObjectData(v, f_list, vc, vn, uv, haveTextures_list, textures_list, v_teapots[currentTeapotModel][0], f_list_teapots[currentTeapotModel][0], vc_teapots[currentTeapotModel][0], vn_teapots[currentTeapotModel][0], uv_teapots[currentTeapotModel][0], haveTextures_list_teapots[currentTeapotModel][0], textures_list_teapots[currentTeapotModel][0]) 194 | 195 | center = center_teapots[currentTeapotModel] 196 | rendererGT = createRendererGT(glMode, chAzGT, chObjAzGT, chElGT, chDistGT, center, v, vc, f_list, vn, light_colorGT, chComponentGT, chVColorsGT, targetPosition[:].copy(), chDisplacementGT, chScaleGT, width,height, uv, haveTextures_list, textures_list, frustum, None ) 197 | 198 | vis_gt = np.array(rendererGT.indices_image!=1).copy().astype(np.bool) 199 | vis_mask = np.array(rendererGT.indices_image==1).copy().astype(np.bool) 200 | 201 | shapeIm = vis_gt.shape 202 | numPixels = shapeIm[0] * shapeIm[1] 203 | 204 | def imageGT(): 205 | global groundTruthBlender 206 | global rendererGT 207 | global blenderRender 208 | 209 | if groundTruthBlender: 210 | return blenderRender 211 | else: 212 | return np.copy(np.array(rendererGT.r)).astype(np.float64) 213 | 214 | 215 | ######################################### 216 | # Initialization ends here 217 | ######################################### 218 | 219 | replaceableScenesFile = '../databaseFull/fields/scene_replaceables_backup_new.txt' 220 | sceneLines = [line.strip() for line in open(replaceableScenesFile)] 221 | scenesToRender = range(len(sceneLines))[:] 222 | lenScenes = 0 223 | for sceneIdx in scenesToRender: 224 | sceneNumber, sceneFileName, instances, roomName, roomInstanceNum, targetIndices, targetPositions = scene_io_utils.getSceneInformation(sceneIdx, replaceableScenesFile) 225 | sceneDicFile = 'data/scene' + str(sceneNumber) + '.pickle' 226 | 227 | lenScenes += len(targetIndices) 228 | collisionSceneFile = 'data/collisions_new/collisionScene' + str(sceneNumber) + '.pickle' 229 | with open(collisionSceneFile, 'rb') as pfile: 230 | collisions = pickle.load(pfile) 231 | 232 | for targetidx, targetIndex in enumerate(targetIndices): 233 | if not collisions[targetIndex][1]: 234 | print("Scene idx " + str(sceneIdx) + " at index " + str(targetIndex) + " collides everywhere.") 235 | 236 | sceneOcclusions = {} 237 | 238 | for sceneIdx in scenesToRender[:]: 239 | 240 | print("Rendering scene: " + str(sceneIdx)) 241 | sceneNumber, sceneFileName, instances, roomName, roomInstanceNum, targetIndices, targetPositions = scene_io_utils.getSceneInformation(sceneIdx, replaceableScenesFile) 242 | 243 | sceneDicFile = 'data/scene' + str(sceneNumber) + '.pickle' 244 | # v, f_list, vc, vn, uv, haveTextures_list, textures_list = sceneimport.loadSavedScene(sceneDicFile) 245 | import copy 246 | v2, f_list2, vc2, vn2, uv2, haveTextures_list2, textures_list2 = scene_io_utils.loadSavedScene(sceneDicFile, tex_srgb2lin) 247 | 248 | collisionSceneFile = 'data/collisions_new/collisionScene' + str(sceneNumber) + '.pickle' 249 | with open(collisionSceneFile, 'rb') as pfile: 250 | collisions = pickle.load(pfile) 251 | 252 | 253 | targetOcclusions = {} 254 | for targetidx, targetIndex in enumerate(targetIndices): 255 | targetPosition = targetPositions[targetidx] 256 | 257 | if collisions[targetIndex][1]: 258 | 259 | collisionProbs = np.zeros(len(collisions[targetIndex][1])) 260 | import copy 261 | 262 | rendererGT.makeCurrentContext() 263 | rendererGT.clear() 264 | contextdata.cleanupContext(contextdata.getContext()) 265 | glfw.destroy_window(rendererGT.win) 266 | del rendererGT 267 | 268 | v, f_list, vc, vn, uv, haveTextures_list, textures_list = copy.deepcopy(v2), copy.deepcopy(f_list2), copy.deepcopy(vc2), copy.deepcopy(vn2), copy.deepcopy(uv2), copy.deepcopy(haveTextures_list2), copy.deepcopy(textures_list2) 269 | 270 | removeObjectData(len(v) -1 - targetIndex, v, f_list, vc, vn, uv, haveTextures_list, textures_list) 271 | 272 | addObjectData(v, f_list, vc, vn, uv, haveTextures_list, textures_list, v_teapots[currentTeapotModel][0], f_list_teapots[currentTeapotModel][0], vc_teapots[currentTeapotModel][0], vn_teapots[currentTeapotModel][0], uv_teapots[currentTeapotModel][0], haveTextures_list_teapots[currentTeapotModel][0], textures_list_teapots[currentTeapotModel][0]) 273 | 274 | rendererGT = createRendererGT(glMode, chAzGT, chObjAzGT, chElGT, chDistGT, center, v, vc, f_list, vn, light_colorGT, chComponentGT, chVColorsGT, targetPosition.copy(), chDisplacementGT, chScaleGT, width,height, uv, haveTextures_list, textures_list, frustum, None ) 275 | # removeObjectData(int(targetIndex-1), v, f_list, vc, vn, uv, haveTextures_list, textures_list) 276 | 277 | for intervalIdx, interval in enumerate(collisions[targetIndex][1]): 278 | collisionProbs[intervalIdx] = collisions[targetIndex][1][intervalIdx][1] - collisions[targetIndex][1][intervalIdx][0] 279 | 280 | collisionsProbs = collisionProbs / np.sum(collisionProbs) 281 | 282 | teapot = None 283 | 284 | intersections = [] 285 | 286 | cameraInterval = 5 287 | for azimuth in np.mod(numpy.arange(270,270+180,cameraInterval), 360): 288 | 289 | occludes = False 290 | 291 | from numpy.random import choice 292 | 293 | objAzInterval = choice(len(collisionsProbs), size=1, p=collisionsProbs) 294 | 295 | chAzGT[:] = azimuth*np.pi/180 296 | 297 | for elevation in numpy.arange(0,90,cameraInterval): 298 | chElGT[:] = elevation*np.pi/180 299 | #occludes = 300 | occlusion = getOcclusionFraction(rendererGT) 301 | if occlusion > 0.01 and occlusion < 0.95: 302 | occludes = True 303 | print("Found occlusion!") 304 | # cv2.imwrite('tmp/imOcclusion_scene' + str(sceneNumber) + '_tgIndex' + str(targetIndex) + '_az' + str(int(azimuth)) + '.jpeg' , 255*rendererGT.r[:,:,[2,1,0]], [int(cv2.IMWRITE_JPEG_QUALITY), 100]) 305 | break 306 | 307 | intersections = intersections + [[azimuth, occludes]] 308 | 309 | startInterval = True 310 | intervals = [] 311 | initInterval = 0 312 | endInterval = 0 313 | 314 | for idx, intersection in enumerate(intersections): 315 | if intersection[1]: 316 | if startInterval: 317 | initInterval = intersection[0] 318 | startInterval = False 319 | else: 320 | if not startInterval: 321 | if idx >= 1 and intersections[idx-1][0] != initInterval: 322 | endInterval = intersection[0] - cameraInterval 323 | intervals = intervals + [[initInterval, endInterval]] 324 | startInterval = True 325 | 326 | if intersection[1]: 327 | endInterval = intersection[0] 328 | if intersections[0][1]: 329 | intervals = intervals + [[initInterval, endInterval+cameraInterval]] 330 | else: 331 | intervals = intervals + [[initInterval, endInterval]] 332 | 333 | targetOcclusions[targetIndex] = (targetParentPosition, intervals) 334 | 335 | 336 | sceneOcclusions[sceneNumber] = targetOcclusions 337 | with open('data/occlusions_new/occlusionScene' + str(sceneNumber) + '.pickle', 'wb') as pfile: 338 | pickle.dump(targetOcclusions, pfile) 339 | 340 | 341 | print("Occlusion detection ended.") -------------------------------------------------------------------------------- /image_processing.py: -------------------------------------------------------------------------------- 1 | __author__ = 'pol' 2 | 3 | from skimage.feature import hog 4 | from skimage import data, color, exposure 5 | import numpy as np 6 | import ipdb 7 | 8 | import skimage.color 9 | from numpy.core.umath_tests import matrix_multiply 10 | 11 | # def xyz2labCh(xyz, illuminant="D65", observer="2"): 12 | # """XYZ to CIE-LAB color space conversion. 13 | # Parameters 14 | # ---------- 15 | # xyz : array_like 16 | # The image in XYZ format, in a 3- or 4-D array of shape 17 | # ``(.., ..,[ ..,] 3)``. 18 | # illuminant : {"A", "D50", "D55", "D65", "D75", "E"}, optional 19 | # The name of the illuminant (the function is NOT case sensitive). 20 | # observer : {"2", "10"}, optional 21 | # The aperture angle of the observer. 22 | # Returns 23 | # ------- 24 | # out : ndarray 25 | # The image in CIE-LAB format, in a 3- or 4-D array of shape 26 | # ``(.., ..,[ ..,] 3)``. 27 | # Raises 28 | # ------ 29 | # ValueError 30 | # If `xyz` is not a 3-D array of shape ``(.., ..,[ ..,] 3)``. 31 | # ValueError 32 | # If either the illuminant or the observer angle is unsupported or 33 | # unknown. 34 | # Notes 35 | # ----- 36 | # By default Observer= 2A, Illuminant= D65. CIE XYZ tristimulus values 37 | # x_ref=95.047, y_ref=100., z_ref=108.883. See function `get_xyz_coords` for 38 | # a list of supported illuminants. 39 | # References 40 | # ---------- 41 | # .. [1] http://www.easyrgb.com/index.php?X=MATH&H=07#text7 42 | # .. [2] http://en.wikipedia.org/wiki/Lab_color_space 43 | # Examples 44 | # -------- 45 | # 46 | # """ 47 | # arr = xyz 48 | # 49 | # xyz_ref_white = skimage.color.get_xyz_coords(illuminant, observer) 50 | # 51 | # # scale by CIE XYZ tristimulus values of the reference white point 52 | # arr = arr / xyz_ref_white 53 | # 54 | # # Nonlinear distortion and linear transformation 55 | # mask = arr > 0.008856 56 | # notmask = arr <= 0.008856 57 | # arr[mask] = ch.power(arr[mask], 1. / 3.) 58 | # arr[notmask] = 7.787 * arr[notmask] + 16. / 116. 59 | # 60 | # x, y, z = arr[..., 0], arr[..., 1], arr[..., 2] 61 | # 62 | # # Vector scaling 63 | # L = (116. * y) - 16. 64 | # a = 500.0 * (x - y) 65 | # b = 200.0 * (y - z) 66 | # 67 | # return ch.concatenate([x[..., np.newaxis] for x in [L, a, b]], axis=-1) 68 | # 69 | # def rgb2xyz(rgb): 70 | # """RGB to XYZ color space conversion. 71 | # Parameters 72 | # ---------- 73 | # rgb : array_like 74 | # The image in RGB format, in a 3- or 4-D array of shape 75 | # ``(.., ..,[ ..,] 3)``. 76 | # Returns 77 | # ------- 78 | # out : ndarray 79 | # The image in XYZ format, in a 3- or 4-D array of shape 80 | # ``(.., ..,[ ..,] 3)``. 81 | # Raises 82 | # ------ 83 | # ValueError 84 | # If `rgb` is not a 3- or 4-D array of shape ``(.., ..,[ ..,] 3)``. 85 | # Notes 86 | # ----- 87 | # The CIE XYZ color space is derived from the CIE RGB color space. Note 88 | # however that this function converts from sRGB. 89 | # References 90 | # ---------- 91 | # .. [1] http://en.wikipedia.org/wiki/CIE_1931_color_space 92 | # Examples 93 | # -------- 94 | # 95 | # """ 96 | # # Follow the algorithm from http://www.easyrgb.com/index.php 97 | # # except we don't multiply/divide by 100 in the conversion 98 | # arr = rgb 99 | # mask = arr > 0.04045 100 | # notmask = arr <= 0.04045 101 | # arr[mask] = ch.power((arr[mask] + 0.055) / 1.055, 2.4) 102 | # arr[notmask] /= 12.92 103 | # return _convert(xyz_from_rgb, arr) 104 | def scaleInvariantMSECoeff(x_pred, x_target): 105 | #Rows: test samples 106 | #Cols: target variables 107 | scales = (matrix_multiply(x_pred[:,None,:],x_target[:,:,None])/matrix_multiply(x_pred[:,None,:],x_pred[:,:,None])).ravel() 108 | 109 | return scales 110 | 111 | def scaleInvariantColourDifference(rgbGT, rgbPred): 112 | scaleRGB = scaleInvariantMSECoeff(rgbPred, rgbGT) 113 | dist = np.sqrt(np.sum((rgbGT - scaleRGB[:,None]*rgbPred)**2, axis=1)) 114 | return dist 115 | 116 | 117 | def cColourDifference(rgb1, rgb2): 118 | lab1 = cv2.cvtColor(np.uint8(rgb1.reshape([-1,1,3])*255), cv2.COLOR_RGB2LAB)/255 119 | lab2 = cv2.cvtColor(np.uint8(rgb2.reshape([-1,1,3])*255), cv2.COLOR_RGB2LAB)/255 120 | dist = np.sqrt(np.sum((lab1[:,0,1:] - lab2[:,0,1:])**2, axis=1)) 121 | return dist 122 | 123 | def eColourDifference(rgb1, rgb2): 124 | lab1 = cv2.cvtColor(np.uint8(rgb1.reshape([-1,1,3])*255), cv2.COLOR_RGB2LAB)/255 125 | lab2 = cv2.cvtColor(np.uint8(rgb2.reshape([-1,1,3])*255), cv2.COLOR_RGB2LAB)/255 126 | dist = np.sqrt(np.sum((lab1[:,0] - lab2[:,0])**2, axis=1)) 127 | return dist 128 | 129 | 130 | # conf.cellSize = cellSize; 131 | # conf.numOrientations = 9; 132 | def computeHoG(image): 133 | 134 | image = color.rgb2gray(image) 135 | 136 | hog_descr, hog_image = hog(image, orientations=9, pixels_per_cell=(8, 8),cells_per_block=(1, 1), visualise=True) 137 | 138 | # fig, ( ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4)) 139 | # 140 | # ax1.axis('off') 141 | # ax1.imshow(image, cmap=plt.cm.gray) 142 | # ax1.set_title('Input image') 143 | 144 | # # Rescale histogram for better display 145 | # hog_image_rescaled = exposure.rescale_intensity(hog_image, in_range=(0, 0.02)) 146 | # 147 | # ax2.axis('off') 148 | # ax2.imshow(hog_image_rescaled, cmap=plt.cm.gray) 149 | # ax2.set_title('Histogram of Oriented Gradients') 150 | # plt.show() 151 | 152 | return hog_descr 153 | 154 | def computeHoGFeatures(images): 155 | hogs = [] 156 | 157 | for image in images: 158 | features = computeHoG(image) 159 | hogs = hogs + [features[None,:] ] 160 | 161 | hogfeats = np.vstack(hogs) 162 | return hogfeats 163 | 164 | def computeIllumFeatures(images, numFreq): 165 | illum = [] 166 | win = 40 167 | for image in images: 168 | features = featuresIlluminationDirection(image, win)[:numFreq,:].ravel() 169 | illum = illum + [features[None,:]] 170 | 171 | illumfeats = np.vstack(illum) 172 | return illumfeats 173 | 174 | 175 | def featuresIlluminationDirection(image,win): 176 | image = color.rgb2gray(image) 177 | coeffs = np.fft.fft2(image[image.shape[0]/2-win:image.shape[0]/2+win,image.shape[1]/2-win:image.shape[1]/2+win]) 178 | magnitudes = np.sqrt(coeffs.real**2 + coeffs.imag**2) 179 | 180 | phases = np.angle(coeffs) 181 | 182 | return np.hstack([magnitudes.ravel()[:,None], phases.ravel()[:,None]]) 183 | 184 | from chumpy import depends_on, Ch 185 | import chumpy as ch 186 | from math import radians 187 | import cv2 188 | import scipy 189 | import matplotlib.pyplot as plt 190 | 191 | 192 | def dr_wrt_convolution(x, filter): 193 | print("Computing convolution gradients") 194 | widthRolls = x.shape[1] 195 | heightRolls = x.shape[0] 196 | tmpShape = [x.shape[0]+filter.shape[0], x.shape[1]+filter.shape[1]] 197 | template = np.zeros(tmpShape) 198 | template[0:filter.shape[0], 0:filter.shape[1]] = filter 199 | jacs = [] 200 | for i in range(heightRolls): 201 | for j in range(widthRolls): 202 | templateRolled = np.roll(template, shift=i, axis=0) 203 | templateRolled = np.roll(templateRolled, shift=j, axis=1) 204 | 205 | templateGrad = templateRolled[tmpShape[0] - x.shape[0] - np.int(filter.shape[0]/2): tmpShape[0] - np.int(filter.shape[0]/2), tmpShape[1] - x.shape[1] - np.int(filter.shape[1]/2): tmpShape[1] - np.int(filter.shape[1]/2)] 206 | jacs = jacs + [scipy.sparse.coo_matrix(templateGrad.ravel())] 207 | 208 | return scipy.sparse.vstack(jacs).tocsc() 209 | 210 | 211 | 212 | class convolve2D(Ch): 213 | terms = 'filter' 214 | dterms = 'x' 215 | 216 | def compute_r(self): 217 | # convolved = scipy.signal.convolve2d(self.x, self.filter, mode='same') 218 | convolved = scipy.ndimage.convolve(self.x, self.filter, mode='reflect') 219 | # return convolved[np.int((convolved.shape[0]-self.x.shape[0])/2):np.int((convolved.shape[0]-self.x.shape[0])/2) + self.x.shape[1], np.int((convolved.shape[1]-self.x.shape[1])/2):np.int((convolved.shape[1]-self.x.shape[1])/2) + self.x.shape[1]] 220 | return convolved 221 | 222 | def compute_dr_wrt(self, wrt): 223 | if wrt is self.x: 224 | return self.convolve2DDr 225 | else: 226 | return None 227 | 228 | class HogImage(Ch): 229 | terms = 'numOrient', 'cwidth', 'cheight' 230 | dterms = 'image', 'hog' 231 | def compute_r(self): 232 | from skimage import draw 233 | sy,sx, _ = self.image.shape 234 | radius = min(self.cwidth, self.cheight) // 2 - 1 235 | orientations_arr = np.arange(self.numOrient) 236 | dx_arr = radius * np.cos(orientations_arr / self.numOrient * np.pi) 237 | dy_arr = radius * np.sin(orientations_arr / self.numOrient * np.pi) 238 | cr2 = self.cheight + self.cheight 239 | cc2 = self.cwidth + self.cwidth 240 | hog_image = np.zeros((sy, sx), dtype=float) 241 | n_cellsx = int(np.floor(sx // self.cwidth)) # number of cells in x 242 | n_cellsy = int(np.floor(sy // self.cheight)) # number of cells in y 243 | for x in range(n_cellsx): 244 | for y in range(n_cellsy): 245 | for o, dx, dy in zip(orientations_arr, dx_arr, dy_arr): 246 | centre = tuple([y * cr2 // 2, x * cc2 // 2]) 247 | rr, cc = draw.line(int(centre[0] + dy), 248 | int(centre[1] + dx), 249 | int(centre[0] - dy), 250 | int(centre[1] - dx)) 251 | hog_image[rr, cc] += self.hog[y, x, o] 252 | return hog_image 253 | 254 | def compute_dr_wrt(self, wrt): 255 | return None 256 | 257 | import skimage 258 | def diffHog(image, drconv=None, numOrient = 9, cwidth=8, cheight=8): 259 | imagegray = 0.3*image[:,:,0] + 0.59*image[:,:,1] + 0.11*image[:,:,2] 260 | sy,sx = imagegray.shape 261 | 262 | # gx = ch.empty(imagegray.shape, dtype=np.double) 263 | gx = imagegray[:, 2:] - imagegray[:, :-2] 264 | gx = ch.hstack([np.zeros([sy,1]), gx, np.zeros([sy,1])]) 265 | 266 | gy = imagegray[2:, :] - imagegray[:-2, :] 267 | # gy = imagegray[:, 2:] - imagegray[:, :-2] 268 | gy = ch.vstack([np.zeros([1,sx]), gy, np.zeros([1,sx])]) 269 | 270 | gx += 1e-5 271 | # gy = imagegray[:-2,1:-1] - imagegray[2:,1:-1] + 0.00001 272 | # gx = imagegray[1:-1,:-2] - imagegray[1:-1, 2:] + 0.00001 273 | 274 | distFilter = np.ones([2*cheight,2*cwidth], dtype=np.uint8) 275 | distFilter[np.int(2*cheight/2), np.int(2*cwidth/2)] = 0 276 | distFilter = (cv2.distanceTransform(distFilter, cv2.DIST_L2, 3)- np.max(cv2.distanceTransform(distFilter, cv2.DIST_L2, 3)))/(-np.max(cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) 277 | 278 | magn = ch.sqrt(gy**2 + gx**2)*180/np.sqrt(2) 279 | 280 | angles = ch.arctan(gy/gx)*180/np.pi + 90 281 | 282 | # meanOrient = np.linspace(0, 180, numOrient) 283 | 284 | orientations_arr = np.arange(numOrient) 285 | 286 | meanOrient = orientations_arr / numOrient * 180 287 | 288 | fb_resttmp = 1 - ch.abs(ch.expand_dims(angles[:,:],2) - meanOrient[1:].reshape([1,1,numOrient-1]))*numOrient/180 289 | zeros_rest = np.zeros([sy,sx, numOrient-1, 1]) 290 | fb_rest = ch.max(ch.concatenate([fb_resttmp[:,:,:,None], zeros_rest],axis=3), axis=3) 291 | 292 | chMinOrient0 = ch.min(ch.concatenate([ch.abs(ch.expand_dims(angles[:,:],2) - meanOrient[0].reshape([1,1,1]))[:,:,:,None], ch.abs(180 - ch.expand_dims(angles[:,:],2) - meanOrient[0].reshape([1,1,1]))[:,:,:,None]], axis=3), axis=3) 293 | 294 | zeros_fb0 = np.zeros([sy,sx, 1]) 295 | fb0_tmp = ch.concatenate([1 - chMinOrient0[:,:]*numOrient/180, zeros_fb0],axis=2) 296 | fb_0 = ch.max(fb0_tmp,axis=2) 297 | 298 | fb = ch.concatenate([fb_0[:,:,None], fb_rest],axis=2) 299 | 300 | # fb[:,:,0] = ch.max(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180,0) 301 | 302 | # fb = 1./(1. + ch.exp(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180)) 303 | 304 | Fb = ch.expand_dims(magn,2)*fb 305 | 306 | if drconv is None: 307 | drconv = dr_wrt_convolution(Fb[:,:,0], distFilter) 308 | 309 | 310 | Fs_list = [convolve2D(x=Fb[:,:,Fbi], filter=distFilter, convolve2DDr=drconv).reshape([Fb.shape[0], Fb.shape[1],1]) for Fbi in range(numOrient)] 311 | 312 | # Fs_list = [scipy.signal.convolve2d(Fb[:,:,Fbi], distFilter).reshape([Fb.shape[0], Fb.shape[1],1]) for Fbi in range(numOrient)] 313 | Fs = ch.concatenate(Fs_list, axis=2) 314 | 315 | # cellCols = np.arange(start=cwidth/2, stop=Fs.shape[1]-cwidth/2 , step=cwidth) 316 | # cellRows = np.arange(start=cheight/2, stop=Fs.shape[0]-cheight/2 , step=cheight) 317 | 318 | Fcells = Fs[0:Fs.shape[0] :cheight,0:Fs.shape[1] :cwidth,:] 319 | 320 | epsilon = 1e-5 321 | 322 | v = Fcells/ch.sqrt(ch.sum(Fcells**2) + epsilon) 323 | # v = Fcells 324 | 325 | # hog, hogim = skimage.feature.hog(imagegray, orientations=numOrient, pixels_per_cell=(cheight, cwidth), visualise=True) 326 | hog_image = HogImage(image=image, hog=Fcells, numOrient=numOrient, cwidth=cwidth, cheight=cheight) 327 | 328 | # plt.imshow(hog_image) 329 | # plt.figure() 330 | # plt.imshow(hogim) 331 | # ipdb.set_trace() 332 | 333 | return v, hog_image, drconv 334 | 335 | import zernike 336 | def zernikeProjection(images, numCoeffs, win=50): 337 | 338 | croppedImages = images[:,images.shape[1]/2-win:images.shape[1]/2+win,images.shape[2]/2-win:images.shape[2]/2+win, :] 339 | 340 | zpolys = zernikePolynomials(imageSize=(croppedImages.shape[1],croppedImages.shape[2]), numCoeffs=numCoeffs) 341 | 342 | coeffs = np.sum(np.sum(croppedImages[:,:,:,:,None]*zpolys.reshape([1,zpolys.shape[0], zpolys.shape[1], 1, -1]), axis=2), axis=1) 343 | return coeffs 344 | 345 | def zernikeProjectionGray(images, numCoeffs, win=50): 346 | 347 | if images.shape[3] == 3: 348 | images = 0.3*images[:,:,:,0] + 0.59*images[:,:,:,1] + 0.11*images[:,:,:,2] 349 | croppedImages = images[:,images.shape[1]/2-win:images.shape[1]/2+win,images.shape[2]/2-win:images.shape[2]/2+win] 350 | 351 | zpolys = zernikePolynomials(imageSize=(croppedImages.shape[1],croppedImages.shape[2]), numCoeffs=numCoeffs) 352 | 353 | coeffs = np.sum(np.sum(croppedImages[:,:,:,None]*zpolys.reshape([1,zpolys.shape[0], zpolys.shape[1], -1]), axis=2), axis=1) 354 | return coeffs 355 | 356 | def chZernikeProjection(image, numCoeffs=20, win=50): 357 | croppedImage = image[image.shape[0]/2-win:image.shape[0]/2+win,image.shape[1]/2-win:image.shape[1]/2+win, :] 358 | zpolys = zernikePolynomials(imageSize=(croppedImage.shape[0],croppedImage.shape[1]), numCoeffs=numCoeffs) 359 | imageProjections = croppedImage[:,:,:,None]*zpolys.reshape([zpolys.shape[0], zpolys.shape[1], 1, -1]) 360 | coeffs = ch.sum(ch.sum(imageProjections, axis=0), axis=0) 361 | return coeffs, imageProjections 362 | 363 | def zernikePolynomials(imageSize=(100,100), numCoeffs=20): 364 | 365 | sy,sx = imageSize 366 | 367 | # distFilter = np.ones([sy,sx], dtype=np.uint8) 368 | # distFilter[np.int(sy/2), np.int(sx/2)] = 0 369 | # distFilter = cv2.distanceTransform(distFilter, cv2.DIST_L2, 3) 370 | # distFilter /= np.max(distFilter) 371 | # np.arange() 372 | 373 | ones = np.ones([sy,sx], dtype=np.bool) 374 | imgind = np.where(ones) 375 | 376 | dy = imgind[0] - int(sy/2) 377 | dx = imgind[1] - int(sx/2) 378 | 379 | pixaz = np.arctan2(dy,dx) 380 | pixrad = np.sqrt(dy**2 + dx**2) 381 | 382 | imaz = np.zeros([sy,sx]) 383 | imrad = np.zeros([sy,sx]) 384 | imaz[imgind] = pixaz 385 | imrad[imgind] = pixrad 386 | 387 | outcircle = imrad>=sy/2 388 | imrad[outcircle] = 0 389 | imaz[outcircle] = 0 390 | 391 | imrad/=np.max(imrad) 392 | 393 | zpolys = [zernike.zernikel(j, imrad, imaz)[:,:,None] for j in range(numCoeffs)] 394 | 395 | zpolys = np.concatenate(zpolys, axis=2) 396 | 397 | return zpolys -------------------------------------------------------------------------------- /differentiable_renderer.py: -------------------------------------------------------------------------------- 1 | import chumpy as ch 2 | from chumpy import depends_on, Ch 3 | import cv2 4 | import numpy as np 5 | import scipy.sparse as sp 6 | from chumpy.utils import row, col 7 | from opendr.geometry import Rodrigues 8 | import warnings 9 | 10 | #Make simple experiment. 11 | def nanmean(a, axis): 12 | # don't call nan_to_num in here, unless you check that 13 | # occlusion_test.py still works after you do it! 14 | result = np.nanmean(a, axis=axis) 15 | return result 16 | 17 | def nangradients(arr): 18 | 19 | with warnings.catch_warnings(): 20 | warnings.simplefilter("ignore", category=RuntimeWarning) 21 | dy = np.expand_dims(arr[:-1,:,:] - arr[1:,:,:], axis=3) 22 | dx = np.expand_dims(arr[:,:-1,:] - arr[:, 1:, :], axis=3) 23 | 24 | dy = np.concatenate((dy[1:,:,:], dy[:-1,:,:]), axis=3) 25 | dy = np.nanmean(dy, axis=3) 26 | dx = np.concatenate((dx[:,1:,:], dx[:,:-1,:]), axis=3) 27 | dx = np.nanmean(dx, axis=3) 28 | 29 | if arr.shape[2] > 1: 30 | gy, gx, _ = np.gradient(arr) 31 | else: 32 | gy, gx = np.gradient(arr.squeeze()) 33 | gy = np.atleast_3d(gy) 34 | gx = np.atleast_3d(gx) 35 | gy[1:-1,:,:] = -dy 36 | gx[:,1:-1,:] = -dx 37 | 38 | return gy, gx 39 | 40 | 41 | class SQErrorRenderer(Ch): 42 | terms = ['renderer', 'params_list'] 43 | dterms = ['params'] 44 | 45 | def compute_r(self): 46 | return self.renderer.errors 47 | 48 | def compute_dr_wrt(self, wrt): 49 | import ipdb 50 | # ipdb.set_trace() 51 | inParamsList = False 52 | if wrt is self.params: 53 | return None 54 | 55 | if not inParamsList: 56 | for param in self.params_list: 57 | if wrt is param: 58 | inParamsList = True 59 | 60 | if inParamsList: 61 | return self.gradient_pred(wrt) 62 | else: 63 | errJac = sp.csc_matrix((2*self.renderer.r - 2*self.imageGT).ravel()[:,None]) 64 | return errJac.multiply(self.renderer.dr_wrt(wrt)) 65 | 66 | def gradient_pred(self, paramWrt): 67 | 68 | observed = self.renderer.color_image 69 | boundaryid_image = self.renderer.boundaryid_image 70 | barycentric = self.renderer.barycentric_image 71 | visibility = self.renderer.visibility_image 72 | visible = np.nonzero(visibility.ravel() != 4294967295)[0] 73 | jacIm = self.dImage_wrt_2dVerts_predict(observed, paramWrt, visible, visibility, barycentric, observed.shape[0], observed.shape[1], self.renderer.v.shape[0], self.renderer.f, boundaryid_image != 4294967295) 74 | return jacIm.dot(self.renderer.camera.dr_wrt(paramWrt)) 75 | 76 | def dImage_wrt_2dVerts_predict(self, observed, paramWrt, visible, visibility, barycentric, image_width, image_height, num_verts, f, bnd_bool): 77 | """Construct a sparse jacobian that relates 2D projected vertex positions 78 | (in the columns) to pixel values (in the rows). This can be done 79 | in two steps.""" 80 | bnd_bool = np.logical_and(bnd_bool, self.renderer.visibility_image != 4294967295) 81 | 82 | camJac = self.renderer.camera.dr_wrt(paramWrt) 83 | 84 | xdiff = self.renderer.dEdx 85 | ydiff = self.renderer.dEdy 86 | 87 | # visible.ravel()[bidxs_out[bidx]] = True 88 | 89 | # import ipdb 90 | # ipdb.set_trace() 91 | 92 | visible = np.nonzero(visibility.ravel() != 4294967295)[0] 93 | 94 | n_channels = np.atleast_3d(observed).shape[2] 95 | shape = visibility.shape 96 | 97 | #2: Take the data and copy the corresponding dxs and dys to these new pixels. 98 | 99 | # Step 1: get the structure ready, ie the IS and the JS 100 | IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel() 101 | JS = col(f[visibility.ravel()[visible]].ravel()) 102 | JS = np.hstack((JS*2, JS*2+1)).ravel() 103 | 104 | pxs = np.asarray(visible % shape[1], np.int32) 105 | pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32) 106 | 107 | if n_channels > 1: 108 | IS = np.concatenate([IS*n_channels+i for i in range(n_channels)]) 109 | JS = np.concatenate([JS for i in range(n_channels)]) 110 | 111 | datas = [] 112 | 113 | # The data is weighted according to barycentric coordinates 114 | bc0 = col(barycentric[pys, pxs, 0]) 115 | bc1 = col(barycentric[pys, pxs, 1]) 116 | bc2 = col(barycentric[pys, pxs, 2]) 117 | for k in range(n_channels): 118 | dxs = xdiff[pys, pxs, k] 119 | dys = ydiff[pys, pxs, k] 120 | if f.shape[1] == 3: 121 | datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel()) 122 | else: 123 | datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel()) 124 | 125 | data = np.concatenate(datas) 126 | 127 | ij = np.vstack((IS.ravel(), JS.ravel())) 128 | 129 | result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2)) 130 | 131 | return result 132 | 133 | 134 | class DifferentiableRenderer(Ch): 135 | terms = ['renderer', 'params_list'] 136 | dterms = ['params'] 137 | 138 | def compute_r(self): 139 | return self.renderer.r 140 | 141 | def compute_dr_wrt(self, wrt): 142 | import ipdb 143 | 144 | for param in self.params_list: 145 | if wrt is param: 146 | return self.gradient_pred(wrt) 147 | 148 | return self.renderer.dr_wrt(wrt) 149 | 150 | 151 | def gradients(self): 152 | # self._call_on_changed() 153 | 154 | observed = self.renderer.color_image 155 | boundaryid_image = self.renderer.boundaryid_image 156 | barycentric = self.renderer.barycentric_image 157 | visibility = self.renderer.visibility_image 158 | visible = np.nonzero(visibility.ravel() != 4294967295)[0] 159 | return self.dImage_wrt_2dVerts_bnd_gradient(observed, barycentric, observed.shape[0], observed.shape[1], boundaryid_image != 4294967295) 160 | 161 | def dImage_wrt_2dVerts_bnd_gradient(self, observed, barycentric, image_width, image_height, bnd_bool): 162 | 163 | bnd_bool = np.logical_and(self.renderer.visibility_image != 4294967295, bnd_bool) 164 | 165 | n_channels = np.atleast_3d(observed).shape[2] 166 | shape = [image_height, image_width] 167 | 168 | bndf = bnd_bool.astype(np.float64) 169 | 170 | bnd_nan = bndf.reshape((observed.shape[0], observed.shape[1], -1)).copy() 171 | bnd_nan.ravel()[bnd_nan.ravel()>0] = np.nan 172 | bnd_nan += 1 173 | obs_nonbnd = np.atleast_3d(observed) * bnd_nan 174 | 175 | ydiffnb, xdiffnb = nangradients(obs_nonbnd) 176 | 177 | observed = np.atleast_3d(observed) 178 | 179 | if observed.shape[2] > 1: 180 | ydiffbnd, xdiffbnd, _ = np.gradient(observed) 181 | else: 182 | ydiffbnd, xdiffbnd = np.gradient(observed.squeeze()) 183 | ydiffbnd = np.atleast_3d(ydiffbnd) 184 | xdiffbnd = np.atleast_3d(xdiffbnd) 185 | 186 | # This corrects for a bias imposed boundary differences begin spread over two pixels 187 | # (by np.gradients or similar) but only counted once (since OpenGL's line 188 | # drawing spans 1 pixel) 189 | xdiffbnd *= 2.0 190 | ydiffbnd *= 2.0 191 | 192 | xdiffnb = -xdiffnb 193 | ydiffnb = -ydiffnb 194 | xdiffbnd = -xdiffbnd 195 | ydiffbnd = -ydiffbnd 196 | 197 | idxs = np.isnan(xdiffnb.ravel()) 198 | xdiffnb.ravel()[idxs] = xdiffbnd.ravel()[idxs] 199 | 200 | idxs = np.isnan(ydiffnb.ravel()) 201 | ydiffnb.ravel()[idxs] = ydiffbnd.ravel()[idxs] 202 | 203 | xdiff = xdiffnb 204 | ydiff = ydiffnb 205 | 206 | dybt = -np.vstack([np.diff(observed, n=1, axis=0), np.zeros([1,observed.shape[1],3])]) 207 | dytb = np.vstack([np.zeros([1,observed.shape[1],3]), np.flipud(np.diff(np.flipud(observed), n=1, axis=0))]) 208 | 209 | dxrl = -np.hstack([np.diff(observed, n=1, axis=1), np.zeros([observed.shape[0],1,3])]) 210 | dxlr = np.hstack([np.zeros([observed.shape[0],1,3]),np.fliplr(np.diff(np.fliplr(observed), n=1, axis=1))]) 211 | 212 | bary_sl = np.roll(barycentric , shift=-1, axis=1) 213 | bary_sr = np.roll(barycentric , shift=1, axis=1) 214 | bary_st = np.roll(barycentric , shift=-1, axis=0) 215 | bary_sb = np.roll(barycentric , shift=1, axis=0) 216 | 217 | return xdiff, ydiff, dybt, dxrl, dytb, dxlr, bary_sl, bary_sr, bary_st, bary_sb 218 | 219 | def gradient_pred(self, paramWrt): 220 | observed = self.renderer.color_image 221 | boundaryid_image = self.renderer.boundaryid_image 222 | barycentric = self.renderer.barycentric_image 223 | visibility = self.renderer.visibility_image 224 | visible = np.nonzero(visibility.ravel() != 4294967295)[0] 225 | jacIm = self.dImage_wrt_2dVerts_predict(observed, paramWrt, visible, visibility, barycentric, observed.shape[0], observed.shape[1], self.renderer.v.shape[0], self.renderer.f, boundaryid_image != 4294967295) 226 | return jacIm.dot(self.renderer.camera.dr_wrt(paramWrt)) 227 | 228 | def boundary_neighborhood(self): 229 | boundary = self.renderer.boundaryid_image != 4294967295 230 | visibility = self.renderer.visibility_image != 4294967295 231 | 232 | boundary = np.logical_and(visibility, boundary) 233 | shape = boundary.shape 234 | 235 | notboundary = np.logical_not(boundary) 236 | horizontal = np.hstack((np.diff(boundary.astype(np.int8),axis=1), np.zeros((shape[0],1), dtype=np.int8))) 237 | # horizontal = np.hstack((np.diff(boundary.astype(np.int8),axis=1), np.zeros((shape[0],1), dtype=np.int8))) 238 | vertical = np.vstack((np.diff(boundary.astype(np.int8), axis=0), np.zeros((1,shape[1]), dtype=np.int8))) 239 | # vertical = np.vstack((np.diff(boundary.astype(np.int8), axis=0), np.zeros((1,shape[1]), dtype=np.int8))) 240 | 241 | pixl = (horizontal == 1) 242 | pixr = (horizontal == -1) 243 | pixt = (vertical == 1) 244 | pixb = (vertical == -1) 245 | 246 | # plt.imshow((pixrl | pixlr | pixtb | pixbt)) 247 | 248 | #Quicker, convolve (FFT) and take mask * etc. 249 | 250 | lidxs_out = np.where(pixl.ravel())[0] 251 | ridxs_out = np.where(pixr.ravel())[0] + 1 252 | tidxs_out = np.where(pixt.ravel())[0] 253 | bidxs_out = np.where(pixb.ravel())[0] + shape[1] 254 | lidxs_int = np.where(pixl.ravel())[0] + 1 255 | ridxs_int = np.where(pixr.ravel())[0] 256 | tidxs_int = np.where(pixt.ravel())[0] + shape[1] 257 | bidxs_int = np.where(pixb.ravel())[0] 258 | 259 | 260 | return pixr, pixl, pixt, pixb, lidxs_out, ridxs_out, tidxs_out, bidxs_out, lidxs_int, ridxs_int, tidxs_int, bidxs_int 261 | 262 | def dImage_wrt_2dVerts_predict(self, observed, paramWrt, visible, visibility, barycentric, image_width, image_height, num_verts, f, bnd_bool): 263 | """Construct a sparse jacobian that relates 2D projected vertex positions 264 | (in the columns) to pixel values (in the rows). This can be done 265 | in two steps.""" 266 | bnd_bool = np.logical_and(bnd_bool, self.renderer.visibility_image != 4294967295) 267 | 268 | camJac = self.renderer.camera.dr_wrt(paramWrt) 269 | 270 | xdiff, ydiff, dybt, dxrl, dytb, dxlr, bary_sl, bary_sr, bary_st, bary_sb = self.gradients() 271 | 272 | pixr, pixl, pixt, pixb, lidxs_out, ridxs_out, tidxs_out, bidxs_out, lidxs_int, ridxs_int, tidxs_int, bidxs_int = self.boundary_neighborhood() 273 | 274 | lidxs_out = np.where(bnd_bool.ravel())[0]-1 275 | ridxs_out = np.where(bnd_bool.ravel())[0]+1 276 | tidxs_out = np.where(bnd_bool.ravel())[0]-bnd_bool.shape[1] 277 | bidxs_out = np.where(bnd_bool.ravel())[0]+bnd_bool.shape[1] 278 | 279 | lidxs_int = np.where(bnd_bool.ravel())[0] 280 | ridxs_int= np.where(bnd_bool.ravel())[0] 281 | tidxs_int= np.where(bnd_bool.ravel())[0] 282 | bidxs_int = np.where(bnd_bool.ravel())[0] 283 | 284 | #Where are triangles moving wrt to the image coordinates at the boundaries? 285 | lintGrad = camJac[f[visibility.ravel()[lidxs_int]]*2] 286 | rintGrad = camJac[f[visibility.ravel()[ridxs_int]]*2] 287 | tintGrad = camJac[f[visibility.ravel()[tidxs_int]]*2+1] 288 | bintGrad = camJac[f[visibility.ravel()[bidxs_int]]*2+1] 289 | 290 | lidx = lintGrad[:,0,0] < -0.0001 291 | xdiff.reshape([-1,3])[lidxs_out[lidx]] = xdiff.reshape([-1,3])[lidxs_int[lidx]] 292 | barycentric.reshape([-1,3])[lidxs_out[lidx]] = barycentric.reshape([-1,3])[lidxs_int[lidx]] 293 | visibility.ravel()[lidxs_out[lidx]] = visibility.ravel()[lidxs_int[lidx]] 294 | xdiff.reshape([-1,3])[lidxs_int[lidx]] = dxrl.reshape([-1,3])[lidxs_int[lidx]] 295 | 296 | # visible.ravel()[lidxs_out[lidx]] = True 297 | 298 | ridx = rintGrad[:,0,0] > 0.0001 299 | xdiff.reshape([-1,3])[ridxs_out[ridx]] = xdiff.reshape([-1,3])[ridxs_int[ridx]] 300 | barycentric.reshape([-1,3])[ridxs_out[ridx]] = barycentric.reshape([-1,3])[ridxs_int[ridx]] 301 | visibility.ravel()[ridxs_out[ridx]] = visibility.ravel()[ridxs_int[ridx]] 302 | xdiff.reshape([-1,3])[ridxs_int[ridx]] = dxlr.reshape([-1,3])[ridxs_int[ridx]] 303 | # visible.ravel()[ridxs_out[ridx]] = True 304 | 305 | tidx = tintGrad[:,0,0] > 0.0001 306 | ydiff.reshape([-1,3])[tidxs_out[tidx]] = ydiff.reshape([-1,3])[tidxs_int[tidx]] 307 | barycentric.reshape([-1,3])[tidxs_out[tidx]] = barycentric.reshape([-1,3])[tidxs_int[tidx]] 308 | visibility.ravel()[tidxs_out[tidx]] = visibility.ravel()[tidxs_int[tidx]] 309 | ydiff.reshape([-1,3])[tidxs_int[tidx]] = dybt.reshape([-1,3])[tidxs_int[tidx]] 310 | # visible.ravel()[tidxs_out[tidx]] = True 311 | 312 | bidx = bintGrad[:,0,0] < -0.0001 313 | ydiff.reshape([-1,3])[bidxs_out[bidx]] = ydiff.reshape([-1,3])[bidxs_int[bidx]] 314 | barycentric.reshape([-1,3])[bidxs_out[bidx]] = barycentric.reshape([-1,3])[bidxs_int[bidx]] 315 | visibility.ravel()[bidxs_out[bidx]] = visibility.ravel()[bidxs_int[bidx]] 316 | ydiff.reshape([-1,3])[bidxs_int[bidx]] = dytb.reshape([-1,3])[bidxs_int[bidx]] 317 | # visible.ravel()[bidxs_out[bidx]] = True 318 | 319 | # import ipdb 320 | # ipdb.set_trace() 321 | 322 | visible = np.nonzero(visibility.ravel() != 4294967295)[0] 323 | 324 | n_channels = np.atleast_3d(observed).shape[2] 325 | shape = visibility.shape 326 | 327 | #2: Take the data and copy the corresponding dxs and dys to these new pixels. 328 | 329 | # Step 1: get the structure ready, ie the IS and the JS 330 | IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel() 331 | JS = col(f[visibility.ravel()[visible]].ravel()) 332 | JS = np.hstack((JS*2, JS*2+1)).ravel() 333 | 334 | pxs = np.asarray(visible % shape[1], np.int32) 335 | pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32) 336 | 337 | if n_channels > 1: 338 | IS = np.concatenate([IS*n_channels+i for i in range(n_channels)]) 339 | JS = np.concatenate([JS for i in range(n_channels)]) 340 | 341 | datas = [] 342 | 343 | # The data is weighted according to barycentric coordinates 344 | bc0 = col(barycentric[pys, pxs, 0]) 345 | bc1 = col(barycentric[pys, pxs, 1]) 346 | bc2 = col(barycentric[pys, pxs, 2]) 347 | for k in range(n_channels): 348 | dxs = xdiff[pys, pxs, k] 349 | dys = ydiff[pys, pxs, k] 350 | if f.shape[1] == 3: 351 | datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel()) 352 | else: 353 | datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel()) 354 | 355 | data = np.concatenate(datas) 356 | 357 | ij = np.vstack((IS.ravel(), JS.ravel())) 358 | 359 | result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2)) 360 | 361 | return result 362 | -------------------------------------------------------------------------------- /render.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4m 2 | import matplotlib 3 | # matplotlib.use('Agg') 4 | import scene_io_utils 5 | import re 6 | from blender_utils import * 7 | from collision import * 8 | 9 | import matplotlib.pyplot as plt 10 | 11 | numpy.random.seed(1) 12 | 13 | inchToMeter = 0.0254 14 | outputDir = '../data/output/' 15 | if not os.path.exists(outputDir): 16 | os.makedirs(outputDir) 17 | 18 | prefix = '_tmp' 19 | 20 | width = 110 21 | height = 110 22 | 23 | numSamples = 1024 24 | useCycles = False 25 | useGPU = False 26 | distance = 0.45 27 | numFrames = 200 28 | batchSize = 10 29 | 30 | completeScene = True 31 | 32 | cam = bpy.data.cameras.new("MainCamera") 33 | camera = bpy.data.objects.new("MainCamera", cam) 34 | world = bpy.data.worlds.new("MainWorld") 35 | renderTeapotsList = [2] 36 | 37 | [targetScenes, targetModels, transformations] = scene_io_utils.loadTargetModels(renderTeapotsList) 38 | 39 | replaceableScenesFile = '../databaseFull/fields/scene_replaceables.txt' 40 | 41 | sceneLines = [line.strip() for line in open(replaceableScenesFile)] 42 | sceneLineNums = numpy.arange(len(sceneLines)) 43 | 44 | 45 | spout = mathutils.Vector((-6.2, -0.16, 6.25)) 46 | handle = mathutils.Vector((6.2, 0.2, 5.7)) 47 | tip = mathutils.Vector((0, 0, 8)) 48 | 49 | occludeHandle = True 50 | 51 | for sceneNum in sceneLineNums[0:1]: 52 | sceneLine = sceneLines[sceneNum] 53 | sceneParts = sceneLine.split(' ') 54 | 55 | sceneFile = sceneParts[0] 56 | 57 | sceneNumber = int(re.search('.+?scene([0-9]+)\.txt', sceneFile, re.IGNORECASE).groups()[0]) 58 | 59 | sceneFileName = re.search('.+?(scene[0-9]+\.txt)', sceneFile, re.IGNORECASE).groups()[0] 60 | 61 | targetIndex = int(sceneParts[1]) 62 | 63 | instances = scene_io_utils.getSceneInstancesInfo('../databaseFull/scenes/' + sceneFileName) 64 | 65 | targetParentPosition = instances[targetIndex][2] 66 | targetParentIndex = instances[targetIndex][1] 67 | 68 | [blenderScenes, modelInstances] = scene_io_utils.importBlenderScenes(instances, completeScene, targetIndex) 69 | targetParentInstance = modelInstances[targetParentIndex] 70 | targetParentInstance.layers[2] = True 71 | 72 | roomName = '' 73 | for model in modelInstances: 74 | reg = re.compile('(room[0-9]+)') 75 | res = reg.match(model.name) 76 | if res: 77 | roomName = res.groups()[0] 78 | 79 | occludingObjName = 'c9fe86bef85fd1d0caeedf6b101df8f6' 80 | for model in modelInstances: 81 | reg = re.compile('(.*?' + occludingObjName + '.*?)') 82 | res = reg.match(model.name) 83 | if res: 84 | occludingObjName = res.groups()[0] 85 | 86 | scene = scene_io_utils.composeScene(modelInstances, targetIndex) 87 | roomInstance = scene.objects[roomName] 88 | 89 | ipdb.set_trace() 90 | roomInstance.layers[2] = True 91 | targetParentInstance.layers[2] = True 92 | 93 | occludingObjInstance = scene.objects[occludingObjName] 94 | newOccludingObjInstance = bpy.data.objects.new(occludingObjName + '_2', None) 95 | newOccludingObjInstance.dupli_type = 'GROUP' 96 | newOccludingObjInstance.dupli_group = occludingObjInstance.dupli_group 97 | newOccludingObjInstance.matrix_world = occludingObjInstance.matrix_world 98 | newOccludingObjInstance.pass_index = 0 99 | 100 | scene.update() 101 | scene.render.threads = 20 102 | scene.render.threads_mode = 'AUTO' 103 | bpy.context.screen.scene = scene 104 | 105 | cycles = bpy.context.scene.cycles 106 | scene.render.tile_x = 25 107 | scene.render.tile_y = 25 108 | 109 | originalLoc = mathutils.Vector((0,-distance , 0)) 110 | 111 | setupScene(scene, targetIndex,roomName, world, distance, camera, width, height, numSamples, useCycles, useGPU) 112 | 113 | bpy.context.user_preferences.system.prefetch_frames = batchSize 114 | bpy.context.user_preferences.system.memory_cache_limit = 1000 115 | 116 | totalAzimuths = [] 117 | totalObjAzimuths = [] 118 | totalElevations = [] 119 | totalObjectIds = [] 120 | 121 | frameStart = 0 122 | frameEnd = frameStart + numFrames 123 | 124 | for teapotIdx, teapotNum in enumerate(renderTeapotsList): 125 | 126 | director = outputDir 127 | 128 | teapot = targetModels[teapotIdx] 129 | teapot.layers[1] = True 130 | teapot.layers[2] = True 131 | 132 | transformation = transformations[teapotIdx] 133 | spout = transformation * spout 134 | handle = transformation * handle 135 | tip = transformation * tip 136 | 137 | azimuths = numpy.mod(numpy.random.uniform(270,450, numFrames), 360) # Avoid looking outside the room 138 | 139 | # azimuths = numpy.array([]) 140 | # while len(azimuths) < numFrames: 141 | # num = numpy.random.uniform(0,360, 1) 142 | # numpy.arccos(mathutils.Vector((-0.6548619270324707, 0.6106656193733215, -0.4452454447746277)) * mathutils.Vector((0.0, -1.0, 0.0))) 143 | # objAzimuths = numpy.append(azimuths, num) 144 | 145 | # objAzimuths = numpy.arange(0,360, 5) # Map it to non colliding rotations. 146 | objAzimuths = numpy.array([]) 147 | while len(objAzimuths) < numFrames: 148 | num = numpy.random.uniform(0,360, 1) 149 | # if not(num>= 250 and num<290) and not(num>= 80 and num<110): 150 | objAzimuths = numpy.append(objAzimuths, num) 151 | 152 | elevations = numpy.random.uniform(0,90, numFrames) 153 | 154 | spoutProjectionsX = numpy.zeros(numFrames) 155 | spoutProjectionsY = numpy.zeros(numFrames) 156 | handleProjectionsX = numpy.zeros(numFrames) 157 | handleProjectionsY = numpy.zeros(numFrames) 158 | tipProjectionsX = numpy.zeros(numFrames) 159 | tipProjectionsY = numpy.zeros(numFrames) 160 | 161 | spoutsOccluded = numpy.zeros(numFrames) 162 | handlesOccluded = numpy.zeros(numFrames) 163 | tipsOccluded = numpy.zeros(numFrames) 164 | 165 | scene.objects.link(teapot) 166 | teapot.layers[1] = True 167 | teapot.layers[2] = True 168 | teapot.matrix_world = mathutils.Matrix.Translation(targetParentPosition) 169 | 170 | center = centerOfGeometry(teapot.dupli_group.objects, teapot.matrix_world) 171 | 172 | original_matrix_world = teapot.matrix_world.copy() 173 | 174 | # ipdb.set_trace() 175 | for frame in range(frameStart, frameEnd): 176 | 177 | azimuth = azimuths[frame - frameStart] 178 | objAzimuth = objAzimuths[frame - frameStart] 179 | elevation = elevations[frame - frameStart] 180 | 181 | bpy.context.scene.frame_set(frame) 182 | azimuthRot = mathutils.Matrix.Rotation(radians(-azimuth), 4, 'Z') 183 | elevationRot = mathutils.Matrix.Rotation(radians(-elevation), 4, 'X') 184 | location = center + azimuthRot * elevationRot * originalLoc 185 | camera.location = location 186 | 187 | azimuthRot = mathutils.Matrix.Rotation(radians(-objAzimuth), 4, 'Z') 188 | teapot.matrix_world = mathutils.Matrix.Translation(original_matrix_world.to_translation()) * azimuthRot * (mathutils.Matrix.Translation(-original_matrix_world.to_translation())) * original_matrix_world 189 | 190 | scene.update() 191 | look_at(camera, center) 192 | scene.update() 193 | teapot.keyframe_insert(data_path="rotation_euler", frame=frame, index=-1) 194 | camera.keyframe_insert(data_path="location", frame=frame, index=-1) 195 | camera.keyframe_insert(data_path="rotation_euler", frame=frame, index=-1) 196 | 197 | scene.frame_set(frame) 198 | scene.update() 199 | sphereSpout_matrix_world = teapot.matrix_world * mathutils.Matrix.Translation(spout) 200 | sphereHandle_matrix_world = teapot.matrix_world * mathutils.Matrix.Translation(handle) 201 | sphereTip_matrix_world = teapot.matrix_world * mathutils.Matrix.Translation(tip) 202 | 203 | spoutlocation = image_project(scene, camera, sphereSpout_matrix_world.to_translation()) 204 | spoutProjectionsX[frame - frameStart] = spoutlocation[0] 205 | spoutProjectionsY[frame - frameStart] = spoutlocation[1] 206 | handlelocation = image_project(scene, camera,sphereHandle_matrix_world.to_translation()) 207 | handleProjectionsX[frame - frameStart] = handlelocation[0] 208 | handleProjectionsY[frame - frameStart] = handlelocation[1] 209 | tiplocation = image_project(scene, camera, sphereTip_matrix_world.to_translation()) 210 | tipProjectionsX[frame - frameStart] = tiplocation[0] 211 | tipProjectionsY[frame - frameStart] = tiplocation[1] 212 | 213 | #Occlude Handle 214 | 215 | result, object, matrix, location, normal = scene.ray_cast(scene.camera.location, sphereSpout_matrix_world.to_translation()) 216 | spoutOccluded = False 217 | if result and object.users_group[0].name != teapot.name: 218 | spoutOccluded = True 219 | spoutsOccluded[frame - frameStart] = int(spoutOccluded) 220 | 221 | result, object, matrix, location, normal = scene.ray_cast(scene.camera.location, sphereHandle_matrix_world.to_translation()) 222 | handleOccluded = False 223 | if result and object.users_group[0].name != teapot.name: 224 | handleOccluded = True 225 | handlesOccluded[frame - frameStart] = int(handleOccluded) 226 | #Occlude Handle: 227 | if scene.objects.find(newOccludingObjInstance.name) != -1: 228 | scene.objects.unlink(newOccludingObjInstance) 229 | if (not result) and occludeHandle and elevation < 35: 230 | scene.objects.link(newOccludingObjInstance) 231 | initLoc = newOccludingObjInstance.location 232 | handleLoc = sphereHandle_matrix_world.to_translation() 233 | initLocA = numpy.array(initLoc) 234 | handleLocA = numpy.array(handleLoc) 235 | 236 | newX = ((handleLocA - camera.location)/4 + camera.location)[0] 237 | newY = ((handleLocA - camera.location)/4 + camera.location)[1] 238 | newOccludingObjInstance.location = mathutils.Vector((newX, newY, initLoc.z)) 239 | 240 | newOccludingObjInstance.keyframe_insert(data_path="location", frame=frame, index=-1) 241 | 242 | scene.frame_set(frame) 243 | scene.update() 244 | result, object, matrix, location, normal = scene.ray_cast(scene.camera.location, sphereHandle_matrix_world.to_translation()) 245 | handleOccluded = False 246 | if result and object.users_group[0].name != teapot.name: 247 | handleOccluded = True 248 | handlesOccluded[frame - frameStart] = int(handleOccluded) 249 | 250 | result, object, matrix, location, normal = scene.ray_cast(scene.camera.location, sphereTip_matrix_world.to_translation()) 251 | tipOccluded = False 252 | if result and object.users_group[0].name != teapot.name: 253 | tipOccluded = True 254 | tipsOccluded[frame - frameStart] = int(tipOccluded) 255 | 256 | 257 | numBatches = int(numFrames / batchSize) 258 | for batch in range(numBatches): 259 | with open(director + 'groundtruth.txt', "a") as groundtruth: 260 | for batch_i in range(batchSize): 261 | 262 | print(str(azimuths[batch * batchSize + batch_i]) + ' ' + str(objAzimuths[batch * batchSize + batch_i]) + ' ' + str(elevations[batch * batchSize + batch_i]) + ' ' + str(teapotNum) + ' ' + str(batch * batchSize + batch_i + frameStart) + ' ' + str(sceneNumber) + ' ' + str(targetIndex) \ 263 | + ' ' + str(spoutProjectionsX[batch * batchSize + batch_i]) + ' ' + str(spoutProjectionsY[batch * batchSize + batch_i]) \ 264 | + ' ' + str(handleProjectionsX[batch * batchSize + batch_i]) + ' ' + str(handleProjectionsY[batch * batchSize + batch_i]) \ 265 | + ' ' + str(tipProjectionsX[batch * batchSize + batch_i]) + ' ' + str(tipProjectionsY[batch * batchSize + batch_i]) \ 266 | + ' ' + str(int(spoutsOccluded[batch * batchSize + batch_i])) \ 267 | + ' ' + str(int(handlesOccluded[batch * batchSize + batch_i])) \ 268 | + ' ' + str(int(tipsOccluded[batch * batchSize + batch_i])) \ 269 | + ' ' + prefix, file = groundtruth) 270 | 271 | scene.frame_start = frameStart + batch * batchSize 272 | scene.frame_end = min(frameStart + batch * batchSize + batchSize - 1, frameEnd) 273 | 274 | scene.layers[1] = True 275 | scene.layers[0] = False 276 | scene.layers[2] = False 277 | scene.render.layers[0].use = False 278 | scene.render.layers[2].use = False 279 | scene.render.layers[1].use = True 280 | 281 | cycles.samples = 1 282 | scene.render.engine = 'CYCLES' 283 | 284 | scene.render.image_settings.file_format = 'OPEN_EXR_MULTILAYER' 285 | # scene.render.image_settings.file_format = 'PNG' 286 | scene.render.filepath = director + 'render' + prefix + '_obj' + str(teapotNum) + '_' + 'scene' + str(sceneNumber) + '_target' + str(targetIndex) + '_' + 'single_' 287 | scene.update() 288 | bpy.ops.render.render( animation=True ) 289 | 290 | scene.layers[1] = False 291 | scene.layers[2] = False 292 | scene.layers[0] = True 293 | scene.render.layers[0].use = True 294 | scene.render.layers[1].use = False 295 | scene.render.layers[2].use = False 296 | 297 | # scene.render.image_settings.file_format = 'OPEN_EXR_MULTILAYER' 298 | #scene.render.image_settings.file_format = 'PNG' 299 | scene.render.filepath = director + 'render' + prefix + '_obj' + str(teapotNum) + '_' + 'scene' + str(sceneNumber) + '_target' + str(targetIndex) + '_' 300 | scene.update() 301 | bpy.ops.render.render( animation=True ) 302 | 303 | if useCycles: 304 | cycles.samples = numSamples 305 | 306 | if not useCycles: 307 | scene.render.engine = 'BLENDER_RENDER' 308 | 309 | scene.render.image_settings.file_format = 'PNG' 310 | scene.render.filepath = director + 'images/' + 'render' + prefix + '_obj' + str(teapotNum) + '_' + 'scene' + str(sceneNumber) + '_target' + str(targetIndex) + '_' 311 | scene.update() 312 | bpy.ops.render.render( animation=True ) 313 | 314 | scene.layers[1] = False 315 | scene.layers[2] = True 316 | scene.layers[0] = False 317 | scene.render.layers[0].use = False 318 | scene.render.layers[1].use = False 319 | scene.render.layers[2].use = True 320 | 321 | scene.render.image_settings.file_format = 'PNG' 322 | scene.render.filepath = director + 'images/' + 'render' + prefix + '_obj' + str(teapotNum) + '_' + 'scene' + str(sceneNumber) + '_target' + str(targetIndex) + '_unoccluded' 323 | scene.update() 324 | bpy.ops.render.render( animation=True ) 325 | 326 | scene.objects.unlink(teapot) 327 | 328 | objectIds = [teapotNum]*numFrames 329 | 330 | # with open(director + 'groundtruth' + str(teapotNum) + '.txt', mode='wt', encoding='utf-8') as groundtruth: 331 | # print(str(azimuths.tolist())[1:-1], file = groundtruth) 332 | # print(str(objAzimuths.tolist())[1:-1], file = groundtruth) 333 | # print(str(elevations.tolist())[1:-1], file = groundtruth) 334 | # print(str(objectIds)[1:-1], file = groundtruth) 335 | 336 | totalAzimuths = totalAzimuths + azimuths.tolist() 337 | totalObjAzimuths = totalObjAzimuths + objAzimuths.tolist() 338 | totalElevations = totalElevations + elevations.tolist() 339 | totalObjectIds = totalObjectIds + objectIds 340 | 341 | # Cleanup 342 | for objnum, obji in enumerate(scene.objects): 343 | if obji.name != teapot.name: 344 | obji.user_clear() 345 | bpy.data.objects.remove(obji) 346 | 347 | scene.user_clear() 348 | bpy.data.scenes.remove(scene) 349 | # bpy.ops.scene.delete() 350 | 351 | 352 | print("Renders ended.") 353 | 354 | # with open(director + 'groundtruth_total.txt', mode='wt', encoding='utf-8') as groundtruth: 355 | # print(str(totalAzimuths)[1:-1], file = groundtruth) 356 | # print(str(totalObjAzimuths)[1:-1], file = groundtruth) 357 | # print(str(totalElevations)[1:-1], file = groundtruth) 358 | # print(str(totalObjectIds)[1:-1], file = groundtruth) 359 | # 360 | # with open(director + 'scene.pickle', 'wb') as pfile: 361 | # pickle.dump(scene, pfile) -------------------------------------------------------------------------------- /occlusion_viewpoints.txt: -------------------------------------------------------------------------------- 1 | GT Azimuth: [ 1.48352986] 2 | GT Elevation: [ 0.52359878] 3 | Occlusion is 79.1333613799 % 4 | 5 | ** 6 | GT Azimuth: [ 1.3962634] 7 | GT Elevation: [ 0.52359878] 8 | Occlusion is 85.1288550908 % 9 | Gaussian 10 | Avg Eucl. distance. approx gradients - finite differenes: 81.6997861029 11 | Avg Angle. approx gradients - finite differenes: 0.912647172648 12 | No overdraw: 13 | Avg Eucl. distance. approx gradients - finite differenes: 76.5651198717 14 | Avg Angle. approx gradients - finite differenes: 0.908498848194 15 | Multisample 1 with overdraw: 16 | Avg Eucl. distance. approx gradients - finite differenes: 79.2377949074 17 | Avg Angle. approx gradients - finite differenes: 0.845691624812 18 | 19 | * Robust 20 | ** Approx gradients - finite differenes. 21 | Avg Eucl. distance :: 0.686739770773 22 | Avg Angle.: 1.55582059879 23 | Num opposite (red) gradients: 48 24 | * No overdraw: 25 | Avg Eucl. distance. approx gradients - finite differenes: 0.718853103976 26 | Avg Angle. approx gradients - finite differenes: 1.62214865732 27 | * Multisample 1 with overdraw: 28 | Avg Eucl. distance. approx gradients - finite differenes: 0.76216046262 29 | Avg Angle. approx gradients - finite differenes: 1.71472681868 30 | 31 | * Normal Gaussian large std 125/255 32 | ** Approx gradients - finite differenes. 33 | Avg Eucl. distance :: 0.130718722478 34 | Avg Angle.: 0.9125591491 35 | Num opposite (red) gradients: 15 36 | * Normal Gaussian small std 5/255 37 | ** Approx gradients - finite differenes. 38 | Avg Eucl. distance :: 81.6992015486 39 | Avg Angle.: 0.9125591491 40 | Num opposite (red) gradients: 15 41 | 42 | * Normal Gaussian small std 250/255 43 | ** Approx gradients - finite differenes. 44 | Avg Eucl. distance :: 0.0510620009679 45 | Avg Angle.: 0.9125591491 46 | Num opposite (red) gradients: 15 47 | 48 | 49 | * Normal Gaussian small std 250/255 50 | ** Approx gradients - finite differenes. 51 | Avg Eucl. distance :: 3.29262861693 52 | Avg Angle.: 0.930982816577 53 | Num opposite (red) gradients: 15 54 | 55 | 56 | * Robust large variance 125/255 57 | ** Approx gradients - finite differenes. 58 | Avg Eucl. distance :: 0.0417428151941 59 | Avg Angle.: 0.980122043236 60 | Num opposite (red) gradients: 19 61 | 62 | * Robust large variance 125/255 and 0.5 global outlier prior 63 | ** Approx gradients - finite differenes. 64 | Avg Eucl. distance :: 0.0167998916809 65 | Avg Angle.: 0.983492134726 66 | Num opposite (red) gradients: 20 67 | 68 | * Robust large variance 25/255 and 0.5 global outlier prior 69 | ** Approx gradients - finite differenes. 70 | ** Approx gradients - finite differenes. 71 | Avg Eucl. distance :: 0.506774782969 72 | Avg Angle.: 1.50484196247 73 | Num opposite (red) gradients: 48 74 | 75 | 76 | 77 | * 78 | 79 | [ 0.52359878] 80 | # chAzGT = ch.Ch([1.30899694]) 81 | 60% 82 | 83 | 84 | **** Statistics **** 85 | GT Azimuth: [ 1.13446401] 86 | GT Elevation: [ 0.34906585] 87 | ** Approx gradients - finite differenes. 88 | *Gaussian (multisample 1) 89 | Avg Eucl. distance :: 99.0476029579 90 | Avg Angle.: 0.581261294902 91 | Num opposite (red) gradients: 5 92 | *Robust (multisample 1) 93 | Avg Eucl. distance :: 1.88375479698 94 | Avg Angle.: 0.845430194414 95 | Num opposite (red) gradients: 21 96 | *Robust No Multisample 97 | Avg Eucl. distance :: 2.6245997208 98 | Avg Angle.: 0.898467528049 99 | Num opposite (red) gradients: 26 100 | *Gaussian No Multisample 101 | Avg Eucl. distance :: 139.932469293 102 | Avg Angle.: 0.82792289203 103 | Num opposite (red) gradients: 17 104 | *Gaussian No Multisample and overdraw line 105 | ** Approx gradients - finite differenes. 106 | Avg Eucl. distance :: 139.932469293 107 | Avg Angle.: 0.82792289203 108 | Num opposite (red) gradients: 17 109 | *Robust no multisample and overdraw line 110 | ** Approx gradients - finite differenes. 111 | Avg Eucl. distance :: 2.6245997208 112 | Avg Angle.: 0.898467528049 113 | Num opposite (red) gradients: 26 114 | *Normal Gaussian 115 | ** Approx gradients - finite differenes. 116 | Avg Eucl. distance :: 101.353784177 117 | Avg Angle.: 0.617828016023 118 | Num opposite (red) gradients: 4 119 | *Normal Robust 120 | ** Approx gradients - finite differenes. 121 | Avg Eucl. distance :: 1.79766912185 122 | Avg Angle.: 0.767355114417 123 | Num opposite (red) gradients: 17 124 | * Normal gaussian Without boundary 125 | Avg Eucl. distance :: 114.034854996 126 | Avg Angle.: 0.727746715691 127 | Num opposite (red) gradients: 3 128 | * Normal Robust without boundary 129 | ** Approx gradients - finite differenes. 130 | Avg Eucl. distance :: 1.78135420088 131 | Avg Angle.: 0.818473626778 132 | Num opposite (red) gradients: 23 133 | * Normal Gaussian with std: 125/255 134 | ** Approx gradients - finite differenes. 135 | Avg Eucl. distance :: 0.162166054683 136 | Avg Angle.: 0.617828016023 137 | Num opposite (red) gradients: 4 138 | 139 | * Robust Gaussian with std: 125/255 140 | ** Approx gradients - finite differenes. 141 | Avg Eucl. distance :: 0.0433517896201 142 | Avg Angle.: 0.405682328751!!! 143 | Num opposite (red) gradients: 3!!! 144 | 145 | 146 | 147 | First notices: Seems like in general robust gives a more accute basin of attraction surface which makes it work better especially with instantiation parameters that are far away from the ground truth. However, so far the approximate gradients with the occlusion model are generally worse in the presence of occlusions. Why? 148 | 149 | Also, multisamples is important for a much better gradient approximatino which show the importance of properly drawn edges. If edges are important, e.g. more so than texture / lighting variations, what happens if we increase its factors? 150 | 151 | Thoughts: 152 | - If something is occluding we want to explain in (give an adequate probability) it so as to ignore it wrt the gradients as they give no information as to what pose the object is in. What really matters in terms of the gradients are the actual object's boundaries. Hence: 153 | - How do we ignore large differences in occlusions vs large differenecs due to different pose (or other params.)? 154 | - Does focusing more on edges help? What happens with texture and lighting? 155 | - Feels like the shading and approx gradient produces an arbitrary estimate of the gradient. 156 | - So... we want to ignore two things: the approximated gradients on ligthing differences (hence ignoring small differences, e.g. large variance) and at the same time ignoring large differences created by occlusion. While we want the cost function to be sensitive to boundaries differences and strong appearance differences due to pose. 157 | - So... we need to "ignore occlusion (esp around edges)" while "factoring in strong appearance difference"... and ignore 158 | 159 | Solutions 160 | - E.g. Knowing the 2D shape of the masks can certainly help. 161 | - E.g. Focusing on edges. 162 | - E.g. Statstical model of colors. 163 | - E.g. Downsampled renders. 164 | - That is, multiple Differentiable renderers: Complete appearance, edges and mask (2D shape). 165 | - Mask: isntead of global prior, use prior on shape as per the SBM. This might be more important when we don't know shape and appearance, but can help a little to ignore certain approx. gradients. 166 | - Edges: Make a differentiable distance transform as a function of the image gradients of GT image and the renderer (or use a BoundaryRenderer instead). 167 | - Appearance: the current model. Maybe something a bit more fancy in terms of mixtures (a-la remembering colors) 168 | 169 | Second notices: Seems like the approx gradeients fail more often when the azimuth has been shifted towards one specific side. Simple examples can show that. 170 | 171 | **** Statistics **** 172 | GT Azimuth: [ 1.22173047] 173 | GT Elevation: [ 0.52359878] 174 | Occlusion is 33.0038676407 % 175 | 176 | Robust 177 | ** Approx gradients - finite differenes. 178 | Avg Eucl. distance :: 0.00328405815871 179 | Avg Angle.: 0.607137740148 180 | Num opposite (red) gradients: 4 181 | 182 | 183 | 184 | **** Statistics **** Good cost function but wrong approximations. OCCLUDED SPOUT. 185 | 186 | GT Azimuth: [ 1.55334304] 187 | Azimuth: [ 1.51843645] 188 | GT Elevation: [ 0.82173048] 189 | Elevation: [ 0.82173048] 190 | 191 | 192 | ****** no occlusion. 200x200 193 | 194 | **** Statistics **** 195 | GT Azimuth: [5.742895587179587] 196 | Azimuth: [-0.52283642] 197 | GT Elevation: [ 0.82120681] 198 | Elevation: [ 0.82120681] 199 | Dr wrt Azimuth: (0, 0) 0.422214380249 200 | Dr wrt Elevation: (0, 0) -2.08153283791 201 | Occlusion is 0.0 % 202 | Current Azimuth difference of [-0.99994924] 203 | Current Elevation difference of [ 0.] 204 | 205 | 206 | with my changes: 207 | 208 | small variance 209 | 210 | Gaussian 211 | 212 | Avg Eucl. distance :: 0.725755140192 213 | Avg Angle.: 0.682183073116 214 | Num opposite (red) gradients: 12 215 | Minimum Azimuth difference of [ 17.24396264] 216 | Minimum Elevation difference of [ 2.49920771] 217 | Current Azimuth difference of [ 0.] 218 | Current Elevation difference of [-0.03000408] 219 | Reading keys... 220 | 221 | robust 222 | 223 | Avg Eucl. distance :: 0.277988329113 224 | Avg Angle.: 0.743337203132 225 | Num opposite (red) gradients: 14 226 | Minimum Azimuth difference of [ 2.49920771] 227 | Minimum Elevation difference of [-2.49920771] 228 | Current Azimuth difference of [ 0.] 229 | Current Elevation difference of [-0.03000408] 230 | 231 | Large variance 232 | 233 | gaussian 234 | 235 | Avg Eucl. distance :: 0.0637151289058 236 | Avg Angle.: 0.682183073116 237 | Num opposite (red) gradients: 12 238 | Minimum Azimuth difference of [ 17.24396264] 239 | Minimum Elevation difference of [ 2.49920771] 240 | Current Azimuth difference of [ 0.] 241 | Current Elevation difference of [-0.03000408] 242 | Reading keys... 243 | 244 | robust 245 | 246 | Avg Eucl. distance :: 0.0368706622509 247 | Avg Angle.: 0.885016683711 248 | Num opposite (red) gradients: 20 249 | Minimum Azimuth difference of [ 17.24396264] 250 | Minimum Elevation difference of [ 2.49920771] 251 | Current Azimuth difference of [ 0.] 252 | Current Elevation difference of [-0.03000408] 253 | 254 | WITHout my changes. 255 | 256 | little variance 257 | 258 | gaussian 259 | 260 | Avg Eucl. distance :: 0.334667634087 261 | Avg Angle.: 0.419201731552 262 | Num opposite (red) gradients: 3 263 | Minimum Azimuth difference of [ 17.24396264] 264 | Minimum Elevation difference of [ 2.49920771] 265 | Current Azimuth difference of [ 0.] 266 | Current Elevation difference of [-0.03000408] 267 | Reading keys... 268 | 269 | robust 270 | 271 | Avg Eucl. distance :: 0.305621414784 272 | Avg Angle.: 0.802735423959 273 | Num opposite (red) gradients: 21 274 | Minimum Azimuth difference of [ 2.49920771] 275 | Minimum Elevation difference of [-2.49920771] 276 | Current Azimuth difference of [ 0.] 277 | Current Elevation difference of [-0.03000408] 278 | 279 | 280 | 281 | 282 | 283 | 284 | ==== without -x 285 | **** Statistics **** 286 | GT Azimuth: [ 5.74289559] 287 | Azimuth: [ 5.74289559] 288 | GT Elevation: [ 0.82120681] 289 | Elevation: [ 0.82173048] 290 | Dr wrt Azimuth: (0, 0) 0.120777622982 291 | Dr wrt Elevation: (0, 0) 0.0346005703334 292 | Occlusion is 0.0 % 293 | ** Approx gradients - finite differenes. 294 | Avg Eucl. distance :: 0.593209073934 295 | Avg Angle.: 0.734524454497 296 | Num opposite (red) gradients: 16 297 | Minimum Azimuth difference of [ 17.24396264] 298 | Minimum Elevation difference of [ 2.49920771] 299 | Current Azimuth difference of [ 0.] 300 | Current Elevation difference of [-0.03000408] 301 | Reading keys... 302 | Reading keys... 303 | Using Outlier model 304 | Sq Error: [ 0.43121879] 305 | Reading keys... 306 | Reading keys... 307 | **** Statistics **** 308 | GT Azimuth: [ 5.74289559] 309 | Azimuth: [ 5.74289559] 310 | GT Elevation: [ 0.82120681] 311 | Elevation: [ 0.82173048] 312 | Dr wrt Azimuth: (0, 0) 0.064235664126 313 | Dr wrt Elevation: (0, 0) 0.0377447423815 314 | Occlusion is 0.0 % 315 | ** Approx gradients - finite differenes. 316 | Avg Eucl. distance :: 0.296382569213 317 | Avg Angle.: 0.65180490566 318 | Num opposite (red) gradients: 9 319 | Minimum Azimuth difference of [ 2.49920771] 320 | Minimum Elevation difference of [-2.49920771] 321 | Current Azimuth difference of [ 0.] 322 | Current Elevation difference of [-0.03000408] 323 | 324 | 325 | with -x (both -) 326 | **** Statistics **** 327 | GT Azimuth: [ 5.74289559] 328 | Azimuth: [ 5.74289559] 329 | GT Elevation: [ 0.82120681] 330 | Elevation: [ 0.82173048] 331 | Dr wrt Azimuth: (0, 0) -0.0353451789405 332 | Dr wrt Elevation: (0, 0) 0.0197232103231 333 | Occlusion is 0.0 % 334 | ** Approx gradients - finite differenes. 335 | Avg Eucl. distance :: 0.417941896706 336 | Avg Angle.: 0.584765084995 337 | Num opposite (red) gradients: 9 338 | Minimum Azimuth difference of [ 17.24396264] 339 | Minimum Elevation difference of [ 2.49920771] 340 | Current Azimuth difference of [ 0.] 341 | Current Elevation difference of [-0.03000408] 342 | Reading keys... 343 | Reading keys... 344 | Using Outlier model 345 | Sq Error: [ 0.43121879] 346 | Reading keys... 347 | Reading keys... 348 | **** Statistics **** 349 | GT Azimuth: [ 5.74289559] 350 | Azimuth: [ 5.74289559] 351 | GT Elevation: [ 0.82120681] 352 | Elevation: [ 0.82173048] 353 | Dr wrt Azimuth: (0, 0) -0.0184159505845 354 | Dr wrt Elevation: (0, 0) 0.026453000725 355 | Occlusion is 0.0 % 356 | ** Approx gradients - finite differenes. 357 | Avg Eucl. distance :: 0.286299368328 358 | Avg Angle.: 0.487362103679 359 | Num opposite (red) gradients: 4 360 | Minimum Azimuth difference of [ 2.49920771] 361 | Minimum Elevation difference of [-2.49920771] 362 | Current Azimuth difference of [ 0.] 363 | Current Elevation difference of [-0.03000408] 364 | 365 | 366 | == With no edge fix 367 | 368 | **** Statistics **** 369 | GT Azimuth: [ 5.74289559] 370 | Azimuth: [ 5.74289559] 371 | GT Elevation: [ 0.82120681] 372 | Elevation: [ 0.82173048] 373 | Dr wrt Azimuth: (0, 0) -0.0445468842116 374 | Dr wrt Elevation: (0, 0) -0.101599735735 375 | Occlusion is 0.0 % 376 | ** Approx gradients - finite differenes. 377 | Avg Eucl. distance :: 0.299423965318 378 | Avg Angle.: 0.460194065748 379 | Num opposite (red) gradients: 6 380 | Minimum Azimuth difference of [ 17.24396264] 381 | Minimum Elevation difference of [ 2.49920771] 382 | Current Azimuth difference of [ 0.] 383 | Current Elevation difference of [-0.03000408] 384 | Reading keys... 385 | Reading keys... 386 | Using Outlier model 387 | Sq Error: [ 0.43121879] 388 | Reading keys... 389 | Reading keys... 390 | **** Statistics **** 391 | GT Azimuth: [ 5.74289559] 392 | Azimuth: [ 5.74289559] 393 | GT Elevation: [ 0.82120681] 394 | Elevation: [ 0.82173048] 395 | Dr wrt Azimuth: (0, 0) -0.00343098986534 396 | Dr wrt Elevation: (0, 0) -0.0381466876599 397 | Occlusion is 0.0 % 398 | ** Approx gradients - finite differenes. 399 | Avg Eucl. distance :: 0.299700611813 400 | Avg Angle.: 0.646335919541 401 | Num opposite (red) gradients: 12 402 | Minimum Azimuth difference of [ 2.49920771] 403 | Minimum Elevation difference of [-2.49920771] 404 | Current Azimuth difference of [ 0.] 405 | Current Elevation difference of [-0.03000408] 406 | 407 | == with edges dvs = 0 408 | 409 | **** Statistics **** 410 | GT Azimuth: [ 5.74289559] 411 | Azimuth: [ 5.74289559] 412 | GT Elevation: [ 0.82120681] 413 | Elevation: [ 0.82173048] 414 | Dr wrt Azimuth: (0, 0) -0.0375992593648 415 | Dr wrt Elevation: (0, 0) 0.0210751637229 416 | Occlusion is 0.0 % 417 | ** Approx gradients - finite differenes. 418 | Avg Eucl. distance :: 0.44750201777 419 | Avg Angle.: 0.610552758524 420 | Num opposite (red) gradients: 9 421 | Minimum Azimuth difference of [ 17.24396264] 422 | Minimum Elevation difference of [ 2.49920771] 423 | Current Azimuth difference of [ 0.] 424 | Current Elevation difference of [-0.03000408] 425 | Reading keys... 426 | Reading keys... 427 | Reading keys... 428 | Using Outlier model 429 | Sq Error: [ 0.43121879] 430 | Reading keys... 431 | Reading keys... 432 | **** Statistics **** 433 | GT Azimuth: [ 5.74289559] 434 | Azimuth: [ 5.74289559] 435 | GT Elevation: [ 0.82120681] 436 | Elevation: [ 0.82173048] 437 | Dr wrt Azimuth: (0, 0) -0.034150987609 438 | Dr wrt Elevation: (0, 0) 0.0225951365728 439 | Occlusion is 0.0 % 440 | ** Approx gradients - finite differenes. 441 | Avg Eucl. distance :: 0.279888039212 442 | Avg Angle.: 0.438374009091 443 | Num opposite (red) gradients: 3 444 | Minimum Azimuth difference of [ 2.49920771] 445 | Minimum Elevation difference of [-2.49920771] 446 | Current Azimuth difference of [ 0.] 447 | Current Elevation difference of [-0.03000408] 448 | 449 | == with edges but only -1 or +1 450 | 451 | **** Statistics **** 452 | GT Azimuth: [ 5.74289559] 453 | Azimuth: [ 5.74289559] 454 | GT Elevation: [ 0.82120681] 455 | Elevation: [ 0.82173048] 456 | Dr wrt Azimuth: (0, 0) -0.0846243245997 457 | Dr wrt Elevation: (0, 0) -0.1352997865 458 | Occlusion is 0.0 % 459 | ** Approx gradients - finite differenes. 460 | Avg Eucl. distance :: 0.335971818889 461 | Avg Angle.: 0.503541274018 462 | Num opposite (red) gradients: 6 463 | Minimum Azimuth difference of [ 17.24396264] 464 | Minimum Elevation difference of [ 2.49920771] 465 | Current Azimuth difference of [ 0.] 466 | Current Elevation difference of [-0.03000408] 467 | Reading keys... 468 | Reading keys... 469 | Reading keys... 470 | Using Outlier model 471 | Sq Error: [ 0.43121879] 472 | Reading keys... 473 | Reading keys... 474 | **** Statistics **** 475 | GT Azimuth: [ 5.74289559] 476 | Azimuth: [ 5.74289559] 477 | GT Elevation: [ 0.82120681] 478 | Elevation: [ 0.82173048] 479 | Dr wrt Azimuth: (0, 0) -0.0146806350145 480 | Dr wrt Elevation: (0, 0) -0.0480049335377 481 | Occlusion is 0.0 % 482 | ** Approx gradients - finite differenes. 483 | Avg Eucl. distance :: 0.269571084568 484 | Avg Angle.: 0.448151815832 485 | Num opposite (red) gradients: 5 486 | Minimum Azimuth difference of [ 2.49920771] 487 | Minimum Elevation difference of [-2.49920771] 488 | Current Azimuth difference of [ 0.] 489 | Current Elevation difference of [-0.03000408] 490 | 491 | 492 | 493 | ******* 494 | 495 | **** Statistics **** 496 | GT Azimuth: [ 4.74289559] 497 | Azimuth: [ 4.72544229] 498 | GT Elevation: [ 0.22120681] 499 | Elevation: [ 0.22173048] 500 | Dr wrt Azimuth: (0, 0) 0.0879849265413 501 | Dr wrt Elevation: (0, 0) -0.139759052748 502 | Occlusion is 65.6031288596 % 503 | ** Approx gradients - finite differenes. 504 | Avg Eucl. distance :: 0.367562688842 505 | Avg Angle.: 0.246419115294 506 | Num opposite (red) gradients: 3 507 | Minimum Azimuth difference of [-12.40386435] 508 | Minimum Elevation difference of [-2.95745579] 509 | Current Azimuth difference of [ 0.99994924] 510 | Current Elevation difference of [-0.03000408] 511 | Reading keys... 512 | 513 | **** Statistics **** 514 | GT Azimuth: [ 4.74289559] 515 | Azimuth: [ 4.72544229] 516 | GT Elevation: [ 0.22120681] 517 | Elevation: [ 0.22173048] 518 | Dr wrt Azimuth: (0, 0) 0.0120324484456 519 | Dr wrt Elevation: (0, 0) -0.114002024369 520 | Occlusion is 65.6031288596 % 521 | ** Approx gradients - finite differenes. 522 | Avg Eucl. distance :: 0.389171829955 523 | Avg Angle.: 0.8990086174 524 | Num opposite (red) gradients: 16 525 | Minimum Azimuth difference of [ 2.49920771] 526 | Minimum Elevation difference of [ 0.94943429] 527 | Current Azimuth difference of [ 0.99994924] 528 | Current Elevation difference of [-0.03000408] 529 | Reading keys... 530 | 531 | vs (dr edges = 0) 532 | 533 | **** Statistics **** 534 | GT Azimuth: [ 4.74289559] 535 | Azimuth: [ 4.74289559] 536 | GT Elevation: [ 0.22120681] 537 | Elevation: [ 0.22173048] 538 | Dr wrt Azimuth: (0, 0) 0.112401212381 539 | Dr wrt Elevation: (0, 0) -0.159960011618 540 | Occlusion is 65.6031288596 % 541 | ** Approx gradients - finite differenes. 542 | Avg Eucl. distance :: 0.402964660415 543 | Avg Angle.: 0.945531618922 544 | Num opposite (red) gradients: 20 545 | Minimum Azimuth difference of [ 2.49920771] 546 | Minimum Elevation difference of [ 0.94943429] 547 | Current Azimuth difference of [ 0.] 548 | Current Elevation difference of [-0.03000408] 549 | 550 | vs (dr edges * 2) 551 | **** Statistics **** 552 | GT Azimuth: [ 4.74289559] 553 | Azimuth: [ 4.74289559] 554 | GT Elevation: [ 0.22120681] 555 | Elevation: [ 0.22173048] 556 | Dr wrt Azimuth: (0, 0) 0.201084344835 557 | Dr wrt Elevation: (0, 0) -0.120651895831 558 | Occlusion is 65.6031288596 % 559 | ** Approx gradients - finite differenes. 560 | Avg Eucl. distance :: 0.388320515702 561 | Avg Angle.: 0.888102277972 562 | Num opposite (red) gradients: 11 563 | Minimum Azimuth difference of [ 2.49920771] 564 | Minimum Elevation difference of [ 0.94943429] 565 | Current Azimuth difference of [ 0.] 566 | Current Elevation difference of [-0.03000408] 567 | 568 | 569 | LARGE VARIANCE 570 | 571 | **** Statistics **** 572 | GT Azimuth: [ 4.74289559] 573 | Azimuth: [ 4.91742851] 574 | GT Elevation: [ 0.22120681] 575 | Elevation: [ 0.22120681] 576 | Dr wrt Azimuth: (0, 0) 0.038489005312 577 | Dr wrt Elevation: (0, 0) -0.0691267356372 578 | Occlusion is 65.6031288596 % 579 | ** Approx gradients - finite differenes. 580 | Avg Eucl. distance :: 0.0322688780328 581 | Avg Angle.: 0.246419115294 582 | Num opposite (red) gradients: 3 583 | Minimum Azimuth difference of [-12.40386435] 584 | Minimum Elevation difference of [-2.95745579] 585 | Current Azimuth difference of [-9.95022626] 586 | Current Elevation difference of [ 0.] 587 | Reading keys... 588 | 589 | **** Statistics **** 590 | GT Azimuth: [ 4.74289559] 591 | Azimuth: [ 4.91742851] 592 | GT Elevation: [ 0.22120681] 593 | Elevation: [ 0.22120681] 594 | Dr wrt Azimuth: (0, 0) 0.0233711737569 595 | Dr wrt Elevation: (0, 0) -0.0420540304559 596 | Occlusion is 65.6031288596 % 597 | ** Approx gradients - finite differenes. 598 | Avg Eucl. distance :: 0.0257575855627 599 | Avg Angle.: 0.195586213869 600 | Num opposite (red) gradients: 3 601 | Minimum Azimuth difference of [ 12.40386435] 602 | Minimum Elevation difference of [ 0.94943429] 603 | Current Azimuth difference of [-9.95022626] 604 | Current Elevation difference of [ 0.] 605 | 606 | 607 | **** Statistics **** 608 | GT Azimuth: [ 4.74289559] 609 | Azimuth: [ 4.64147166] 610 | GT Elevation: [ 0.22120681] 611 | Elevation: [ 0.22081677] 612 | Dr wrt Azimuth: (0, 0) -0.593674497532 613 | Dr wrt Elevation: (0, 0) -0.150886592675 614 | Occlusion is 65.6005444029 % 615 | Current Azimuth difference of [ 3.95584613] 616 | Current Elevation difference of [ 0.01423049] 617 | 618 | 619 | 620 | -------------------------------------------------------------------------------- /probLineSearch.py: -------------------------------------------------------------------------------- 1 | from scipy.special import erf 2 | import numpy as np 3 | from scipy.stats import mvn 4 | import ipdb 5 | 6 | def probLineSearch(func, x0, f0, df0, search_direction, alpha0, 7 | verbosity, outs, paras, var_f0, var_df0): 8 | # probLineSearch.m -- A probabilistic line search algorithm for nonlinear 9 | # optimization problems with noisy gradients. 10 | # 11 | # == INPUTS =============================================================== 12 | # [f,f', var_f, var_df] = func(x) -- function handle 13 | # input: 14 | # x -- column vectors (positions) (Dx1) 15 | # output: 16 | # f -- scalar function values 17 | # df -- vector gradients (Dx1) 18 | # var_f -- estimated noise for function values (scalar) 19 | # var_df -- estimated noise for gradients (Dx1) 20 | # x0 -- current position of optimizer (Dx1) 21 | # f0 -- function value at x0 (scalar, previous output y_tt) 22 | # df0 -- gradient at x0 ((Dx1), previous output dy_tt) 23 | # search_direction -- (- df(x0) does not need to be normalized) 24 | # alpha0: initial step size (scalar, previous output alpha0_out) 25 | # var_f0 -- variance of function values at x0. (scalar, previous output var_f_tt) 26 | # var_df0 -- variance of gradients at x0. ((Dx1), previous output var_df_tt) 27 | # verbosity -- level of stdout output. 28 | # 0 -- no output 29 | # 1 -- steps, function values, state printed to stdout 30 | # 2 -- plots with only function values 31 | # 3 -- plots including Gaussian process and Wolfe condition beliefs. 32 | # paras -- possible parameters that func needs. 33 | # outs -- struct with collected statistics 34 | # 35 | # == OUTPUTS ============================================================== 36 | # outs -- struct including counters and statistics 37 | # alpha0_out -- accepted stepsize * 1.3 (initial step size for next step) 38 | # x_tt -- accepted position 39 | # y_tt -- functin value at x_tt 40 | # dy_tt -- gradient at x_tt 41 | # var_f_tt -- variance of function values at x_tt 42 | # var_df_tt -- variance of gradients values at x_tt 43 | # 44 | 45 | # outs = {} 46 | 47 | # -- setup fixed parameters ----------------------------------------------- 48 | # if ~isfield(outs, 'counter') 49 | # outs['counter'] = 1 50 | # end 51 | 52 | 53 | outs['alpha_stats'] = alpha0 # running average over accepted step sizes 54 | 55 | limit = 7 # maximum #function evaluations in one line search (+1) 56 | 57 | # constants for Wolfe conditions (must be chosen 0 < c1 < c2 < 1) 58 | c1 = 0.05 # <---- DECIDED FIXED 0.05 59 | c2 = 0.8 # <---- DECIDED FIXED 0.8 60 | # c2 = 0 extends until ascend location reached: lots of extrapolation 61 | # c2 = 1 accepts any point of increased gradient: almost no extrapolation 62 | 63 | WolfeThreshold = 0.3 # <---- DECIDED FIXED (0.3) 64 | # the new free parameter of this method relative to sgd: 65 | # search is terminated when probability(Wolfe conditions hold) > WolfeThreshold 66 | # not sensitive between 0.1 and 0.8 (0 = accept everyting, 1= accept nothing) 67 | 68 | offset = 10 # off-set, for numerical stability. 69 | 70 | EXT = 1 # extrapolation factor 71 | tt = 1 # initial step size in scaled space 72 | 73 | # -- set up GP ------------------------------------------------------------ 74 | # create variables with shared scope. Ugly, but necessary because 75 | # matlab does not discover shared variables if they are first created in a 76 | # nested function. 77 | 78 | d2m = np.array([]); d3m = np.array([]); V = np.array([]); Vd = np.array([]); dVd = np.array([]) 79 | m0 = np.array([]); dm0 = np.array([]); V0= np.array([]);Vd0 = np.array([]); dVd0= np.array([]) 80 | V0f = np.array([]); Vd0f= np.array([]); V0df=np.array([]); Vd0df = np.array([]) 81 | 82 | 83 | # kernel: 84 | k = lambda a,b: ((np.minimum(a+offset,b+offset)**3)/3 + 0.5 * np.abs(a-b) * np.minimum(a+offset,b+offset)**2) 85 | kd = lambda a,b: np.int32(a=b) * (np.dot(a+offset,b+offset) - 0.5 * (b+offset)**2) 86 | dk = lambda a,b: np.int32(a>b) * ((b+offset)**2)/2 + np.int32(a<=b) * (np.dot(a+offset,b+offset) - 0.5 * (a+offset)**2) 87 | dkd= lambda a,b: np.minimum(a+offset,b+offset) 88 | 89 | # further derivatives 90 | ddk = lambda a,b: np.int32(a<=b) * (b-a) 91 | ddkd= lambda a,b: np.int32(a<=b) 92 | dddk= lambda a,b: -np.int32(a<=b) 93 | 94 | # -- helper functions ----------------------------------------------------- 95 | GaussCDF = lambda z: 0.5 * (1 + erf(z/np.sqrt(2))) 96 | GaussPDF = lambda z: np.exp( - 0.5 * z**2 ) / np.sqrt(2*np.pi) 97 | EI = lambda m,s,eta: (eta - m) * GaussCDF((eta-m)/s) + s * GaussPDF((eta-m)/s) 98 | 99 | # -- scale ---------------------------------------------------------------- 100 | beta = np.abs(np.dot(search_direction.T,df0)) 101 | # scale f and df according to 1/(beta*alpha0) 102 | 103 | # -- scaled noise --------------------------------------------------------- 104 | sigmaf = np.sqrt(var_f0)/(alpha0*beta) 105 | 106 | sigmadf = np.sqrt(np.dot((search_direction**2).T,var_df0))/beta 107 | 108 | # -- initiate data storage ------------------------------------------------ 109 | T = np.array([0]) 110 | Y = np.array([0]) 111 | dY = np.array(df0)[:,None] 112 | dY_projected = np.array([np.dot(df0.T,search_direction)/beta]) 113 | Sigmaf = np.array([var_f0]) 114 | Sigmadf = np.array(var_df0)[:,None] 115 | N = 1 116 | 117 | m = [] 118 | d1m = [] 119 | d2m = [] 120 | d3m = [] 121 | V = [] 122 | Vd = [] 123 | dVd = [] 124 | m0 = [] 125 | dm0 = [] 126 | V0 = [] 127 | Vd0 = [] 128 | dVd0 = [] 129 | V0f = [] 130 | Vd0f = [] 131 | V0df = [] 132 | d0df = [] 133 | # -- helper functions ----------------------------------------------------- 134 | def updateGP(): # using multiscope variables to construct GP 135 | 136 | nonlocal m 137 | nonlocal d1m 138 | nonlocal d2m 139 | nonlocal d3m 140 | nonlocal V 141 | nonlocal Vd 142 | nonlocal dVd 143 | nonlocal m0 144 | nonlocal dm0 145 | nonlocal V0 146 | nonlocal Vd0 147 | nonlocal dVd0 148 | nonlocal V0f 149 | nonlocal Vd0f 150 | nonlocal V0df 151 | nonlocal Vd0df 152 | 153 | # build Gram matrix 154 | kTT = np.zeros([N,N]); 155 | kdTT = np.zeros([N,N]); 156 | dkdTT = np.zeros([N,N]); 157 | for i in range(N): 158 | for j in range(N): 159 | kTT[i,j] = k(T[i], T[j]) 160 | kdTT[i,j] = kd(T[i], T[j]) 161 | dkdTT[i,j] = dkd(T[i],T[j]) 162 | 163 | # build noise matrix 164 | Sig = sigmaf**2 * np.ones([2*N, 1]); Sig[N::] = sigmadf**2 165 | 166 | # build Gram matrix 167 | G = np.diag(Sig.ravel()) + np.r_[np.c_[kTT, kdTT], np.c_[kdTT.T, dkdTT]] 168 | 169 | 170 | A = np.linalg.solve(G, np.append(Y, dY_projected)) 171 | 172 | # posterior mean function and all its derivatives 173 | m = lambda t: np.dot(np.concatenate([k(t, T.T) , kd(t, T.T)]), A) 174 | d1m = lambda t: np.dot(np.concatenate([dk(t, T.T) , dkd(t, T.T)]), A) 175 | d2m = lambda t: np.dot(np.concatenate([ddk(t, T.T) ,ddkd(t, T.T)]), A) 176 | d3m = lambda t: np.dot(np.concatenate([dddk(t, T.T), np.zeros([N])]), A) 177 | 178 | # posterior marginal covariance between function and first derivative 179 | V = lambda t: k(t,t) - np.dot(np.concatenate([k(t, T.T) , kd(t, T.T)]), np.linalg.solve(G , np.concatenate([k(t, T.T) , kd(t, T.T)]).T)) 180 | Vd = lambda t: kd(t,t) - np.dot(np.concatenate([k(t, T.T) , kd(t, T.T)]), np.linalg.solve(G , np.concatenate([dk(t, T.T),dkd(t, T.T)]).T)) 181 | dVd = lambda t: dkd(t,t) - np.dot(np.concatenate([dk(t, T.T), dkd(t, T.T)]), np.linalg.solve(G , np.concatenate([dk(t, T.T),dkd(t, T.T)]).T)) 182 | 183 | # belief at starting point, used for Wolfe conditions 184 | m0 = m(0) 185 | dm0 = d1m(0) 186 | V0 = V(0) 187 | Vd0 = Vd(0) 188 | dVd0 = dVd(0) 189 | 190 | # covariance terms with function (derivative) values at origin 191 | V0f = lambda t: k(0,t) - np.dot(np.concatenate([k(0, T.T) , kd(0, T.T)]) , np.linalg.solve(G, np.concatenate([k(t, T.T) , kd(t, T.T)]).T)) 192 | Vd0f = lambda t: dk(0,t) - np.dot(np.concatenate([dk(0, T.T), dkd(0, T.T)]) , np.linalg.solve(G , np.concatenate([k(t, T.T) , kd(t, T.T)]).T)) 193 | V0df = lambda t: kd(0,t) - np.dot(np.concatenate([k(0, T.T), kd(0, T.T)]) , np.linalg.solve(G , np.concatenate([dk(t, T.T),dkd(t, T.T)]).T)) 194 | Vd0df = lambda t: dkd(0,t)- np.dot(np.concatenate([dk(0, T.T), dkd(0, T.T)]) , np.linalg.solve(G , np.concatenate([dk(t, T.T),dkd(t, T.T)]).T)) 195 | 196 | # -- update GP with new datapoint ----------------------------------------- 197 | updateGP() 198 | 199 | x_tt = [] 200 | y_tt = [] 201 | dy_tt = [] 202 | var_f_tt = [] 203 | var_df_tt = [] 204 | alpha0_out = [] 205 | 206 | def make_outs(y, dy, var_f, var_df): 207 | 208 | nonlocal x_tt 209 | nonlocal y_tt 210 | nonlocal dy_tt 211 | nonlocal var_f_tt 212 | nonlocal var_df_tt 213 | nonlocal alpha0_out 214 | nonlocal outs 215 | x_tt = x0 + tt*alpha0*search_direction # accepted position 216 | y_tt = y*(alpha0*beta) + f0 # function value at accepted position 217 | dy_tt = dy # gradient at accepted position 218 | var_f_tt = var_f # variance of function value at accepted position 219 | var_df_tt = var_df # variance of gradients at accepted position 220 | 221 | # set new set size 222 | # next initial step size is 1.3 times larger than last accepted step size 223 | alpha0_out = tt*alpha0 * 1.3 224 | 225 | # running average for reset in case the step size becomes very small 226 | # this is a saveguard 227 | gamma = 0.9 228 | outs['alpha_stats'] = gamma*outs['alpha_stats'] + (1-gamma)*tt*alpha0; 229 | 230 | # reset NEXT initial step size to average step size if accepted step 231 | # size is 100 times smaller or larger than average step size 232 | if (alpha0_out > 1e2*outs['alpha_stats']) or (alpha0_out < 1e-2*outs['alpha_stats']): 233 | if verbosity > 0: 234 | print('making a very small step, resetting alpha0') 235 | alpha0_out = outs['alpha_stats'] # reset step size 236 | 237 | 238 | def probWolfe(t): # probability for Wolfe conditions to be fulfilled 239 | 240 | # marginal for Armijo condition 241 | ma = m0 - m(t) + c1 * t * dm0 242 | Vaa = V0 + (c1 * t)**2 * dVd0 + V(t) + 2 * (c1 * t * (Vd0 - Vd0f(t)) - V0f(t)) 243 | 244 | # marginal for curvature condition 245 | mb = d1m(t) - c2 * dm0 246 | Vbb = c2**2 * dVd0 - 2 * c2 * Vd0df(t) + dVd(t) 247 | 248 | # covariance between conditions 249 | Vab = -c2 * (Vd0 + c1 * t * dVd0) + V0df(t) + c2 * Vd0f(t) + c1 * t * Vd0df(t) - Vd(t) 250 | 251 | if (Vaa < 1e-9) and (Vbb < 1e-9): # deterministic evaluations 252 | p = np.int32(ma >= 0) * np.int32(mb >= 0) 253 | return p, None 254 | 255 | # joint probability 256 | if Vaa <= 0 or Vbb <= 0: 257 | p = 0 258 | p12 = np.array([0,0,0]) 259 | return p,p12 260 | 261 | rho = Vab / np.sqrt(Vaa * Vbb) 262 | 263 | upper = 2 * c2 * ((np.abs(dm0)+2*np.sqrt(dVd0)-mb)/np.sqrt(Vbb) ) 264 | # p = bvn(-ma / np.sqrt(Vaa), np.inf, -mb / np.sqrt(Vbb), upper, rho) 265 | _, p, _ = mvn.mvndst(np.array([-ma / np.sqrt(Vaa),-mb / np.sqrt(Vbb)]), np.array([np.inf,upper]), np.array([1, 2]), rho) 266 | 267 | # if nargout > 1: 268 | # individual marginal probabilities for each condition 269 | # (for debugging) 270 | 271 | p12 = np.array([1 - GaussCDF(-ma/np.sqrt(Vaa)), GaussCDF(upper)- GaussCDF(-mb/np.sqrt(Vbb)), Vab / np.sqrt(Vaa * Vbb)]) 272 | 273 | return p, p12 274 | 275 | def cubicMinimum(ts): 276 | nonlocal d1m 277 | nonlocal d2m 278 | nonlocal d3m 279 | # mean belief at ts is a cubic function. It is defined up to a constant by 280 | d1mt = d1m(ts) 281 | d2mt = d2m(ts) 282 | d3mt = d3m(ts) 283 | 284 | a = 0.5 * d3mt 285 | b = d2mt - ts * d3mt 286 | c = d1mt - d2mt * ts + 0.5 * d3mt * ts**2 287 | 288 | if abs(d3mt) < 1e-9: # essentially a quadratic. Single extremum 289 | tm = - (d1mt - ts * d2mt) / d2mt 290 | return tm 291 | 292 | 293 | # compute the two possible roots: 294 | detmnt = b**2 - 4*a*c 295 | if detmnt < 0: # no roots 296 | tm = np.inf 297 | return tm 298 | 299 | LR = (-b - np.sign(a) * np.sqrt(detmnt)) / (2*a) # left root 300 | RR = (-b + np.sign(a) * np.sqrt(detmnt)) / (2*a) # right root 301 | 302 | # and the two values of the cubic at those points (up to constant) 303 | Ldt = LR - ts # delta t for left root 304 | Rdt = RR - ts # delta t for right root 305 | LCV = d1mt * Ldt + 0.5 * d2mt * Ldt**2 + (d3mt * Ldt**3)/6 # left cubic value 306 | RCV = d1mt * Rdt + 0.5 * d2mt * Rdt**2 + (d3mt * Rdt**3)/6 # right cubic value 307 | 308 | if LCV < RCV: 309 | tm = LR 310 | else: 311 | tm = RR 312 | 313 | return tm 314 | 315 | # -- search (until budget used or converged) ------------------------------ 316 | for N in range(2,limit+1): 317 | 318 | # -- evaluate function (change minibatch!) ---------------------------- 319 | outs['counter'] = outs['counter'] + 1 320 | 321 | [y, dy, var_f, var_df] = func(x0 + tt*alpha0*search_direction) # y: function value at tt 322 | 323 | 324 | # # Test with matlab function TESTFUNCTION=3 325 | # if N==2: 326 | # y = 140.9659 327 | # dy = np.array([16.5853, 22.3174]) 328 | # var_f = 1 329 | # var_df = np.array([1, 1]) 330 | # 331 | # if N == 3: 332 | # y = 133.2719 333 | # dy = np.array([18.4566,22.1113]) 334 | # var_f = 1 335 | # var_df = np.array([1,1]) 336 | # 337 | # if N == 4: 338 | # y = 118.0413 339 | # dy = np.array([18.0786,21.8977]) 340 | # var_f = 1 341 | # var_df = np.array([1,1]) 342 | # 343 | # if N == 5: 344 | # y = 85.3557 345 | # dy = np.array([17.3779,18.2448]) 346 | # var_f = 1 347 | # var_df = np.array([1,1]) 348 | # 349 | # if N == 6: 350 | # y = 43.5753 351 | # dy = np.array([3.1085,11.2455]) 352 | # var_f = 1 353 | # var_df = np.array([1,1]) 354 | 355 | 356 | if np.isinf(y) or np.isnan(y): 357 | # this does not happen often, but still needs a fix 358 | # e.g. if function return inf or nan (probably too large step), 359 | # evaluate again at 0.1*tt 360 | print('function values is inf or nan.') 361 | print("N " + str(N)) 362 | # -- scale function output -------------------------------------------- 363 | y = (y - f0)/(alpha0*beta) # substract offset 364 | dy_projected = np.dot(dy.T,search_direction)/beta # projected gradient 365 | 366 | # -- store output ----------------------------------------------------- 367 | T = np.append(T,tt) 368 | Y = np.append(Y,y) 369 | 370 | dY = np.hstack([dY,dy.reshape([-1,1])]) 371 | 372 | dY_projected = np.append(dY_projected, dy_projected) 373 | 374 | Sigmaf = np.append(Sigmaf, var_f) 375 | 376 | Sigmadf = np.hstack([Sigmadf,var_df.reshape([-1,1])]) 377 | 378 | # if N == 7: 379 | # ipdb.set_trace() 380 | # -- update GP with new datapoint ------------------------------------- 381 | updateGP() # store, update belief 382 | 383 | # -- check last evaluated point for acceptance ------------------------ 384 | (probWolfeVal, _) = probWolfe(tt) 385 | 386 | 387 | if probWolfeVal > WolfeThreshold: # are we done? 388 | if verbosity > 0: 389 | print('found acceptable point.') 390 | 391 | make_outs(y, dy, var_f, var_df) 392 | 393 | return outs, alpha0_out, y_tt, dy_tt, x_tt, var_f_tt, var_df_tt 394 | 395 | # -- find candidates (set up for EI) ---------------------------------- 396 | # decide where to probe next: evaluate expected improvement and prob 397 | # Wolfe conditions at a number of candidate points, take the most promising one. 398 | 399 | # lowest mean evaluation, and its gradient (needed for EI): 400 | M = np.zeros([N,1]) 401 | dM = np.zeros([N,1]) 402 | for l in range(N): 403 | M[l] = m(T[l]) 404 | dM[l] = d1m(T[l]) 405 | 406 | minj = np.argmin(M) # minm: lowest GP mean, minj: index in candidate list 407 | minm = M[minj] 408 | 409 | tmin = T[minj] # tt of point with lowest GP mean of function values 410 | dmin = dM[minj] # GP mean of gradient at lowest point 411 | 412 | # -- check this point as well for acceptance -------------------------- 413 | if np.abs(dmin) < 1e-5 and Vd(tmin) < 1e-4: # nearly deterministic 414 | tt = tmin; dy = dY[:, minj]; y = Y[minj]; var_f = Sigmaf[minj]; var_df = Sigmadf[:, minj]; 415 | print('found a point with almost zero gradient. Stopping, although Wolfe conditions not guaranteed.') 416 | make_outs(y, dy, var_f, var_df) 417 | 418 | return outs, alpha0_out, y_tt, dy_tt, x_tt, var_f_tt, var_df_tt 419 | 420 | # -- find candidates -------------------------------------------------- 421 | # CANDIDATES 1: minimal means between all evaluations: 422 | # iterate through all `cells' (O[N]), check minimum mean locations. 423 | Tcand = np.array([]) # positions of candidate points 424 | Mcand = np.array([]) # means of candidate points 425 | Scand = np.array([]) # standard deviation of candidate points 426 | Tsort = np.sort(T) 427 | Wolfes = np.array([]) # list of acceptable points. 428 | reeval = False 429 | 430 | for cel in range(N-1): # loop over cells 431 | Trep = Tsort[cel] + 1e-6 * (Tsort[cel+1] - Tsort[cel]) 432 | cc = cubicMinimum(Trep) 433 | 434 | # add point to candidate list if minimum lies in between T(cel) and T(cel+1) 435 | if cc > Tsort[cel] and cc < Tsort[cel+1]: 436 | Tcand = np.append(Tcand,cc) 437 | Mcand = np.append(Mcand,m(cc)) 438 | Scand = np.append(Scand, np.sqrt(V(cc))) 439 | 440 | else: # no minimum, just take half-way 441 | if cel==1 and d1m(0) > 0: # only in first cell 442 | if verbosity > 0: 443 | print('function seems very steep, reevaluating close to start.') 444 | reeval = True 445 | Tcand = np.array([0.1 * (Tsort[cel] + Tsort[cel+1])]) 446 | Mcand = np.append(Mcand, m(0.1 * (Tsort[cel] + Tsort[cel+1]))) 447 | Scand = np.append(Scand, np.sqrt(V(0.1 * (Tsort[cel] + Tsort[cel+1])))) 448 | break 449 | 450 | 451 | # check whether there is an acceptable point among already 452 | # evaluated points (since this changes each time the GP gets updated) 453 | probWolfeVal, _ = probWolfe(Tsort[cel]) 454 | if cel > 1 and (probWolfeVal > WolfeThreshold): 455 | Wolfes = np.append(Wolfes,Tsort[cel]) # list of acceptable points. 456 | 457 | # -- check if at least on point is acceptable ------------------------- 458 | if len(Wolfes) > 0: 459 | if verbosity > 0: 460 | # makePlot() 461 | print('found acceptable point.') 462 | 463 | # -- chose best point among Wolfes, return. ----------------------- 464 | MWolfes = 0 * Wolfes; 465 | for o in range(len(Wolfes)): 466 | MWolfes[o] = m(Wolfes[o]) # compute GP means of acceptable points 467 | 468 | tt = Wolfes[MWolfes == np.min(MWolfes)] 469 | 470 | # find corresponding gradient and variances 471 | dy = dY[:, T == tt].ravel(); y = Y[T == tt].ravel(); var_f = Sigmaf[T == tt].ravel(); var_df = Sigmadf[:, T==tt].ravel(); 472 | make_outs(y, dy, var_f, var_df) 473 | # if dy.shape == (2,1): 474 | # ipdb.set_trace() 475 | return outs, alpha0_out, y_tt, dy_tt, x_tt, var_f_tt, var_df_tt 476 | 477 | 478 | # Candidate 2: one extrapolation step 479 | 480 | if not reeval: 481 | Tcand = np.append(Tcand, np.max(T) + EXT) 482 | Mcand = np.append(Mcand, m(np.max(T) + EXT)) 483 | 484 | Scand = np.append(Scand, np.sqrt(V(np.max(T)+EXT))) 485 | 486 | # -- discriminate candidates through EI and prob Wolfe ---------------- 487 | EIcand = EI(Mcand, Scand, minm) # minm: lowest GP mean of collected evaluations (not candidates) 488 | PPcand = np.zeros(EIcand.shape) 489 | for ti in range(len(EIcand)): 490 | PPcand[ti], _ = probWolfe(Tcand[ti]) 491 | 492 | 493 | idx_best = np.argmax(EIcand * PPcand) # find best candidate 494 | 495 | if Tcand[idx_best] == tt + EXT: # extrapolating. Extend extrapolation step 496 | EXT = 2 * EXT 497 | 498 | tt = Tcand[idx_best] 499 | 500 | # makePlot() 501 | 502 | # -- algorithm reached limit without finding acceptable point. ------------ 503 | # Evaluate a final time, return "best" point (one with lowest function value) 504 | 505 | outs['counter'] = outs['counter'] + 1 506 | 507 | 508 | [y, dy, var_f, var_df] = func(x0 + tt*alpha0*search_direction) # y: function value at tt 509 | 510 | if np.isinf(y) or np.isnan(y): 511 | # this does not happen often, but still needs a fix 512 | # e.g. if function return inf or nan (probably too large step), 513 | # evaluate again at 0.1*tt 514 | print('function values is inf or nan.') 515 | 516 | # -- scale function output ------------------------------------------------ 517 | y = (y - f0)/(alpha0*beta) # substract offset 518 | dy_projected = np.dot(dy.T,search_direction)/beta # projected gradient at tt; 519 | 520 | # -- store output ----------------------------------------------------- 521 | T = np.append(T,tt) 522 | Y = np.append(Y,y) 523 | dY = np.hstack([dY,dy.reshape([-1,1])]) 524 | dY_projected = np.append(dY_projected, dy_projected) 525 | N = limit + 1 526 | Sigmaf = np.append(Sigmaf, var_f) 527 | Sigmadf = np.hstack([Sigmadf,var_df.reshape([-1,1])]) 528 | 529 | # -- update GP with new datapoint ----------------------------------------- 530 | updateGP() 531 | 532 | # -- check last evaluated point for acceptance ---------------------------- 533 | probWolfeVal, _ = probWolfe(tt) 534 | if probWolfeVal > WolfeThreshold: # are we done? 535 | if verbosity > 0: 536 | print('found acceptable point right at end of budget. Phew!') 537 | make_outs(y, dy, var_f, var_df) 538 | 539 | return outs, alpha0_out, y_tt, dy_tt, x_tt, var_f_tt, var_df_tt 540 | 541 | 542 | # -- return point with lowest mean ---------------------------------------- 543 | M = np.ones([N,1])*np.inf 544 | for ii in range(N): 545 | M[ii] = m(T[ii]) # compute all GP means of all evaluated locations 546 | 547 | minj = np.argmin(M) # index of minimal GP mean 548 | if verbosity > 0: 549 | print('reached evaluation limit. Returning ''best'' known point.'); 550 | 551 | 552 | # find corresponding tt, gradient and noise 553 | tt = T[minj]; dy = dY[:, minj]; y = Y[minj]; var_f = Sigmaf[minj]; var_df = Sigmadf[:, minj] 554 | make_outs(y, dy, var_f, var_df) 555 | 556 | return outs, alpha0_out, y_tt, dy_tt, x_tt, var_f_tt, var_df_tt --------------------------------------------------------------------------------