├── .gitignore
├── LICENSE
├── README.md
├── RTL
├── dataloader.py
├── floor
│ ├── carpet
│ │ ├── carpet.jpg
│ │ ├── carpet.mtl
│ │ └── carpet.obj
│ ├── drum
│ │ ├── drum.jpg
│ │ ├── drum.mtl
│ │ └── drum.obj
│ ├── grass
│ │ ├── data.json
│ │ ├── grass.jpg
│ │ ├── grass.mtl
│ │ └── grass.obj
│ ├── mousemat
│ │ ├── mousemat.jpg
│ │ ├── mousemat.mtl
│ │ └── mousemat.obj
│ └── table
│ │ ├── table.jpg
│ │ ├── table.mtl
│ │ └── table.obj
├── main.py
├── recon.py
├── run_camera.py
├── scene.py
└── templates
│ ├── qrcode.png
│ └── test_flask.html
├── figs
├── algo_comparison.png
├── book_cover.png
├── comp_others.png
├── dance.gif
├── gpu.png
├── icon.png
├── livecap_comparison.png
├── lossless.png
├── ohem.png
├── robustness.png
├── rtl.jpg
├── shift.gif
├── suppl_results1.png
├── suppl_results2.png
├── twoside.png
└── zeng.gif
├── monoport
├── __init__.py
└── lib
│ ├── common
│ ├── config.py
│ ├── logger.py
│ └── trainer.py
│ ├── dataset
│ ├── ppl_dynamic.py
│ ├── ppl_static.py
│ └── utils.py
│ ├── mesh_util.py
│ ├── modeling
│ ├── MonoPortNet.py
│ ├── __init__.py
│ ├── backbones
│ │ ├── HGFilters.py
│ │ ├── HRNetFilters.py
│ │ ├── ResBlkFilters.py
│ │ ├── Yolov4Filters.py
│ │ └── __init__.py
│ ├── geometry.py
│ ├── heads
│ │ ├── SurfaceClassifier.py
│ │ └── __init__.py
│ └── normalizers
│ │ ├── DepthNormalizer.py
│ │ └── __init__.py
│ └── render
│ ├── BaseCamera.py
│ ├── CameraPose.py
│ ├── PespectiveCamera.py
│ ├── __init__.py
│ └── gl
│ ├── AlbedoRender.py
│ ├── GLObject.py
│ ├── NormalRender.py
│ ├── PrtRender.py
│ ├── Render.py
│ ├── ShRender.py
│ ├── Shader.py
│ ├── __init__.py
│ ├── glcontext.py
│ └── shaders
│ ├── albedo.fs
│ ├── albedo.vs
│ ├── color.fs
│ ├── color.vs
│ ├── normal.fs
│ ├── normal.vs
│ ├── prt.fs
│ ├── prt.vs
│ ├── prt_uv.fs
│ ├── prt_uv.vs
│ ├── quad.fs
│ ├── quad.vs
│ ├── sh.fs
│ ├── sh.vs
│ ├── simple.fs
│ └── simple.vs
├── requirements.txt
├── scripts
└── download_model.sh
└── setup.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | # lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | MANIFEST
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 | .pytest_cache/
49 |
50 | # Translations
51 | *.mo
52 | *.pot
53 |
54 | # Django stuff:
55 | *.log
56 | local_settings.py
57 | db.sqlite3
58 |
59 | # Flask stuff:
60 | instance/
61 | .webassets-cache
62 |
63 | # Scrapy stuff:
64 | .scrapy
65 |
66 | # Sphinx documentation
67 | docs/_build/
68 |
69 | # PyBuilder
70 | target/
71 |
72 | # Jupyter Notebook
73 | .ipynb_checkpoints
74 |
75 | # pyenv
76 | .python-version
77 |
78 | # celery beat schedule file
79 | celerybeat-schedule
80 |
81 | # SageMath parsed files
82 | *.sage.py
83 |
84 | # Environments
85 | .env
86 | .venv
87 | env/
88 | venv/
89 | ENV/
90 | env.bak/
91 | venv.bak/
92 |
93 | # Spyder project settings
94 | .spyderproject
95 | .spyproject
96 |
97 | # Rope project settings
98 | .ropeproject
99 |
100 | # mkdocs documentation
101 | /site
102 |
103 | # mypy
104 | .mypy_cache/
105 |
106 | data/
107 |
108 | .DS_Store
109 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | This software is Copyright © 2021 Ruilong Li, The University of Southern California. All Rights Reserved.
2 |
3 | Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice, this paragraph and the following three paragraphs appear in all copies.
4 |
5 | Permission to make commercial use of this software may be obtained by contacting:
6 | USC Stevens Center for Innovation
7 | University of Southern California
8 | 1150 S. Olive Street, Suite 2300
9 | Los Angeles, CA 90115, USA
10 |
11 | This software program and documentation are copyrighted by The University of Southern California. The software program and documentation are supplied "as is", without any accompanying services from USC. USC does not warrant that the operation of the program will be uninterrupted or error-free. The end-user understands that the program was developed for research purposes and is advised not to rely exclusively on the program for any reason.
12 |
13 | IN NO EVENT SHALL THE UNIVERSITY OF SOUTHERN CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF SOUTHERN CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF SOUTHERN CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF SOUTHERN CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
14 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # [Monoport: Monocular Volumetric Human Teleportation (SIGGRAPH 2020 Real-Time Live)](http://xiuyuliang.cn/monoport/)
2 |
3 | ### Time: Tuesday, 25 August 2020 (Pacific Time Zone)
4 | [](https://arxiv.org/abs/2007.13988) [](https://project-splinter.github.io/) [](https://youtu.be/fQDsYVE7GtQ)
5 |
6 |
7 |
9 |
10 | Our volumetric capture system captures a completely clothed human body (including the back) using **a single RGB webcam** and in real time.
11 |
12 | ## Requirements
13 | - Python 3.7
14 | - PyOpenGL 3.1.5 (need X server in Ubuntu)
15 | - [PyTorch](https://pytorch.org/) tested on 1.4.0
16 | - [ImplicitSegCUDA](https://github.com/Project-Splinter/ImplicitSegCUDA)
17 | - [human_inst_seg](https://github.com/Project-Splinter/human_inst_seg)
18 | - [streamer_pytorch](https://github.com/Project-Splinter/streamer_pytorch)
19 | - [human_det](https://github.com/Project-Splinter/human_det)
20 |
21 | We run the demo with 2 GeForce RTX 2080Ti GPUs, the memory usage is as follows (~3.4GB at GPU1, ~9.7GB at GPU2):
22 |
23 |
24 |
26 |
27 | **Note**: The last four dependencies are also developed by our team, and are all in active maintenance. If you meet any installation problems specifically regarding those tools, we recommend you to file the issue in the corresponding repo. (You don't need to install them manually here as they are included in the requirements.txt)
28 |
29 | ## How to run our Siggraph RTL Demo
30 |
31 | #### 1. Setup the repo
32 | First you need to download the model:
33 | ```
34 | sh scripts/download_model.sh
35 | ```
36 |
37 | Then install all the dependencies:
38 | ```
39 | pip install -r requirements.txt
40 | ```
41 |
42 | #### 2. Start the main process as a server.
43 | ```
44 | # if you want to use the input from a webcam:
45 | python RTL/main.py --use_server --ip --port 5555 --camera -- netG.ckpt_path ./data/PIFu/net_G netC.ckpt_path ./data/PIFu/net_C
46 |
47 | # or if you want to use the input from a image folder:
48 | python RTL/main.py --use_server --ip --port 5555 --image_folder -- netG.ckpt_path ./data/PIFu/net_G netC.ckpt_path ./data/PIFu/net_C
49 |
50 | # or if you want to use the input from a video:
51 | python RTL/main.py --use_server --ip --port 5555 --videos -- netG.ckpt_path ./data/PIFu/net_G netC.ckpt_path ./data/PIFu/net_C
52 | ```
53 |
54 | If everything goes well, you should be able to see those logs after waiting for a few seconds:
55 |
56 | loading networkG from ./data/PIFu/net_G ...
57 | loading networkC from ./data/PIFu/net_C ...
58 | initialize data streamer ...
59 | Using cache found in /home/rui/.cache/torch/hub/NVIDIA_DeepLearningExamples_torchhub
60 | Using cache found in /home/rui/.cache/torch/hub/NVIDIA_DeepLearningExamples_torchhub
61 | * Serving Flask app "main" (lazy loading)
62 | * Environment: production
63 | WARNING: This is a development server. Do not use it in a production deployment.
64 | Use a production WSGI server instead.
65 | * Debug mode: on
66 | * Running on http://:5555/ (Press CTRL+C to quit)
67 |
68 | #### 2. Access the server to start.
69 | Open the page `http://:5555/` on a web browser from any device (Desktop/IPad/IPhone), You should be able to see the **MonoPort VR Demo** page on that device, and at the same time you should be able to see the a screen poping up on your desktop, showing the reconstructed normal and texture image.
70 |
71 |
72 |
74 |
75 | ## Contributors
76 |
77 | MonoPort is based on [Monocular Real-Time Volumetric Performance Capture(ECCV'20)](https://project-splinter.github.io/), authored by Ruilong Li*([@liruilong940607](https://github.com/liruilong940607)), Yuliang Xiu*([@yuliangxiu](https://github.com/YuliangXiu)), Shunsuke Saito([@shunsukesaito](https://github.com/shunsukesaito)), Zeng Huang([@ImaginationZ](https://github.com/ImaginationZ)) and Kyle Olszewski([@kyleolsz](https://github.com/kyleolsz)), [Hao Li](https://www.hao-li.com/) is the corresponding author.
78 |
79 |
80 | ## Citation
81 |
82 | ```
83 | @inproceedings{li2020monoport,
84 | title={Monocular Real-Time Volumetric Performance Capture},
85 | author={Li, Ruilong and Xiu, Yuliang and Saito, Shunsuke and Huang, Zeng and Olszewski, Kyle and Li, Hao},
86 | booktitle={European Conference on Computer Vision},
87 | pages={49--67},
88 | year={2020},
89 | organization={Springer}
90 | }
91 |
92 | @incollection{li2020monoportRTL,
93 | title={Volumetric human teleportation},
94 | author={Li, Ruilong and Olszewski, Kyle and Xiu, Yuliang and Saito, Shunsuke and Huang, Zeng and Li, Hao},
95 | booktitle={ACM SIGGRAPH 2020 Real-Time Live},
96 | pages={1--1},
97 | year={2020}
98 | }
99 | ```
100 |
101 | ## Relevant Works
102 |
103 | **[PIFu: Pixel-Aligned Implicit Function for High-Resolution Clothed Human Digitization (ICCV 2019)](https://shunsukesaito.github.io/PIFu/)**
104 | *Shunsuke Saito\*, Zeng Huang\*, Ryota Natsume\*, Shigeo Morishima, Angjoo Kanazawa, Hao Li*
105 |
106 | The original work of Pixel-Aligned Implicit Function for geometry and texture reconstruction, unifying sigle-view and multi-view methods.
107 |
108 | **[PIFuHD: Multi-Level Pixel-Aligned Implicit Function for High-Resolution 3D Human Digitization (CVPR 2020)](https://shunsukesaito.github.io/PIFuHD/)**
109 | *Shunsuke Saito, Tomas Simon, Jason Saragih, Hanbyul Joo*
110 |
111 | They further improve the quality of reconstruction by leveraging multi-level approach!
112 |
113 | **[ARCH: Animatable Reconstruction of Clothed Humans (CVPR 2020)](https://arxiv.org/pdf/2004.04572.pdf)**
114 | *Zeng Huang, Yuanlu Xu, Christoph Lassner, Hao Li, Tony Tung*
115 |
116 | Learning PIFu in canonical space for animatable avatar generation!
117 |
118 | **[Robust 3D Self-portraits in Seconds (CVPR 2020)](http://www.liuyebin.com/portrait/portrait.html)**
119 | *Zhe Li, Tao Yu, Chuanyu Pan, Zerong Zheng, Yebin Liu*
120 |
121 | They extend PIFu to RGBD + introduce "PIFusion" utilizing PIFu reconstruction for non-rigid fusion.
122 |
123 | ## Relavant applications
124 |
125 | **[Real-time VR PhD Defense](https://www.youtube.com/watch?v=RhWTqjVekVU&feature=youtu.be)**
126 | Dr. [Zeng Huang](https://zeng.science/) defensed his PhD virtually using our system. [(Media in Chinese)](https://mp.weixin.qq.com/s/Bl0HohrSVzaVPF0EHzuIWw)
127 |
128 |
129 |
131 |
--------------------------------------------------------------------------------
/RTL/floor/carpet/carpet.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/RTL/floor/carpet/carpet.jpg
--------------------------------------------------------------------------------
/RTL/floor/carpet/carpet.mtl:
--------------------------------------------------------------------------------
1 | #
2 | # Wavefront material file
3 | # Converted by Meshlab Group
4 | #
5 |
6 | newmtl material_0
7 | Ka 0.200000 0.200000 0.200000
8 | Kd 0.584314 0.584314 0.584314
9 | Ks 1.000000 1.000000 1.000000
10 | Tr 0.000000
11 | illum 2
12 | Ns 0.000000
13 | map_Kd carpet.jpg
14 |
15 |
--------------------------------------------------------------------------------
/RTL/floor/carpet/carpet.obj:
--------------------------------------------------------------------------------
1 | ####
2 | #
3 | # OBJ File Generated by Meshlab
4 | #
5 | ####
6 | # Object carpet.obj
7 | #
8 | # Vertices: 24
9 | # Faces: 34
10 | #
11 | ####
12 | mtllib ./carpet.mtl
13 |
14 | vn 2.133503 -2.133503 3.706217
15 | v 22.185629 -22.185631 1.354754 0.749020 0.749020 0.749020
16 | vn 2.133502 2.133502 3.706217
17 | v 22.185629 22.185631 1.354756 0.749020 0.749020 0.749020
18 | vn -2.133502 2.133502 3.706217
19 | v -22.185629 22.185631 1.354756 0.749020 0.749020 0.749020
20 | vn -2.133503 -2.133503 3.706217
21 | v -22.185629 -22.185631 1.354754 0.749020 0.749020 0.749020
22 | vn -2.328125 1.376172 2.073882
23 | v -26.004105 20.084265 1.354756 0.749020 0.749020 0.749020
24 | vn -2.328123 -1.376173 2.073881
25 | v -26.004105 -20.084265 1.354754 0.749020 0.749020 0.749020
26 | vn 1.376173 2.328126 2.073882
27 | v 20.084263 26.004107 1.354756 0.749020 0.749020 0.749020
28 | vn -1.376173 2.328122 2.073881
29 | v -20.084263 26.004107 1.354756 0.749020 0.749020 0.749020
30 | vn 2.328126 -1.376173 2.073882
31 | v 26.004105 -20.084265 1.354754 0.749020 0.749020 0.749020
32 | vn 2.328123 1.376173 2.073881
33 | v 26.004105 20.084265 1.354756 0.749020 0.749020 0.749020
34 | vn -1.376173 -2.328126 2.073882
35 | v -20.084263 -26.004107 1.354754 0.749020 0.749020 0.749020
36 | vn 1.376173 -2.328123 2.073881
37 | v 20.084263 -26.004107 1.354754 0.749020 0.749020 0.749020
38 | vn 2.133503 -2.133503 0.000000
39 | v 22.185629 -22.185631 0.959159 0.749020 0.749020 0.749020
40 | vn 1.376173 -2.328126 0.000000
41 | v 20.084263 -26.004107 0.959159 0.749020 0.749020 0.749020
42 | vn 2.328123 -1.376173 0.000000
43 | v 26.004105 -20.084265 0.959159 0.749020 0.749020 0.749020
44 | vn -1.376173 -2.328123 0.000000
45 | v -20.084263 -26.004107 0.959159 0.749020 0.749020 0.749020
46 | vn 2.328126 1.376173 0.000000
47 | v 26.004105 20.084265 0.959161 0.749020 0.749020 0.749020
48 | vn -1.376173 2.328126 0.000000
49 | v -20.084263 26.004107 0.959161 0.749020 0.749020 0.749020
50 | vn 1.376173 2.328123 0.000000
51 | v 20.084263 26.004107 0.959161 0.749020 0.749020 0.749020
52 | vn -2.328126 -1.376173 0.000000
53 | v -26.004105 -20.084265 0.959159 0.749020 0.749020 0.749020
54 | vn -2.328123 1.376173 0.000000
55 | v -26.004105 20.084265 0.959161 0.749020 0.749020 0.749020
56 | vn -2.133502 2.133502 0.000000
57 | v -22.185629 22.185631 0.959161 0.749020 0.749020 0.749020
58 | vn 2.133503 2.133503 0.000000
59 | v 22.185629 22.185631 0.959161 0.749020 0.749020 0.749020
60 | vn -2.133503 -2.133503 0.000000
61 | v -22.185629 -22.185631 0.959159 0.749020 0.749020 0.749020
62 | # 24 vertices, 0 vertices normals
63 |
64 |
65 | usemtl material_0
66 | vt 0.877100 0.116100
67 | vt 0.877100 0.883900
68 | vt 0.122900 0.883900
69 | f 1/1/1 2/2/2 3/3/3
70 | vt 0.122900 0.116100
71 | f 1/1/1 3/3/3 4/4/4
72 | vt 0.058000 0.847600
73 | vt 0.058000 0.152400
74 | f 3/3/3 5/5/5 6/6/6
75 | f 3/3/3 6/6/6 4/4/4
76 | vt 0.841400 0.950000
77 | vt 0.158600 0.950000
78 | f 2/2/2 7/7/7 8/8/8
79 | f 2/2/2 8/8/8 3/3/3
80 | vt 0.942000 0.152400
81 | vt 0.942000 0.847600
82 | f 1/1/1 9/9/9 10/10/10
83 | f 1/1/1 10/10/10 2/2/2
84 | vt 0.158600 0.050000
85 | vt 0.841400 0.050000
86 | f 4/4/4 11/11/11 12/12/12
87 | f 4/4/4 12/12/12 1/1/1
88 | vt 0.162000 0.054500
89 | vt 0.838000 0.054500
90 | vt 0.817500 0.084300
91 | f 16/13/16 14/14/14 12/15/12
92 | vt 0.188100 0.084300
93 | f 16/13/16 12/15/12 11/16/11
94 | vt 0.873400 0.119900
95 | vt 0.852900 0.155300
96 | f 14/14/14 13/17/13 1/18/1
97 | f 14/14/14 1/18/1 12/15/12
98 | vt 0.937600 0.155900
99 | vt 0.911500 0.174500
100 | f 13/17/13 15/19/15 9/20/9
101 | f 13/17/13 9/20/9 1/18/1
102 | vt 0.937600 0.844100
103 | vt 0.911500 0.810600
104 | f 15/19/15 17/21/17 10/22/10
105 | f 15/19/15 10/22/10 9/20/9
106 | vt 0.873400 0.880100
107 | vt 0.849100 0.848400
108 | f 17/21/17 23/23/23 2/24/2
109 | f 17/21/17 2/24/2 10/22/10
110 | vt 0.838000 0.945500
111 | vt 0.821200 0.910100
112 | f 23/23/23 19/25/19 7/26/7
113 | f 23/23/23 7/26/7 2/24/2
114 | vt 0.162000 0.945500
115 | vt 0.175000 0.910100
116 | f 19/25/19 18/27/18 8/28/8
117 | f 19/25/19 8/28/8 7/26/7
118 | vt 0.113500 0.885700
119 | vt 0.145200 0.855900
120 | f 18/27/18 22/29/22 3/30/3
121 | f 18/27/18 3/30/3 8/28/8
122 | vt 0.049300 0.849700
123 | vt 0.089400 0.829200
124 | f 22/29/22 21/31/21 5/32/5
125 | f 22/29/22 5/32/5 3/30/3
126 | vt 0.053000 0.155900
127 | vt 0.089400 0.187600
128 | f 21/31/21 20/33/20 6/34/6
129 | f 21/31/21 6/34/6 5/32/5
130 | vt 0.126600 0.119900
131 | vt 0.156400 0.144100
132 | f 20/33/20 24/35/24 4/36/4
133 | f 20/33/20 4/36/4 6/34/6
134 | f 24/35/24 16/13/16 11/16/11
135 | f 24/35/24 11/16/11 4/36/4
136 | # 34 faces, 36 coords texture
137 |
138 | # End of File
--------------------------------------------------------------------------------
/RTL/floor/drum/drum.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/RTL/floor/drum/drum.jpg
--------------------------------------------------------------------------------
/RTL/floor/drum/drum.mtl:
--------------------------------------------------------------------------------
1 | #
2 | # Wavefront material file
3 | # Converted by Meshlab Group
4 | #
5 |
6 | newmtl material_0
7 | Ka 0.200000 0.200000 0.200000
8 | Kd 0.639216 0.639216 0.639216
9 | Ks 1.000000 1.000000 1.000000
10 | Tr 1.000000
11 | illum 2
12 | Ns 0.000000
13 | map_Kd drum.jpg
14 |
15 |
--------------------------------------------------------------------------------
/RTL/floor/grass/grass.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/RTL/floor/grass/grass.jpg
--------------------------------------------------------------------------------
/RTL/floor/grass/grass.mtl:
--------------------------------------------------------------------------------
1 | # 3ds Max Wavefront OBJ Exporter v0.97b - (c)2007 guruware
2 | # File Created: 25.04.2011 16:05:12
3 |
4 | newmtl 10438_Circular_Grass_Patch_v1
5 | Ns 10.0000
6 | Ni 1.5000
7 | d 1.0000
8 | Tr 0.0000
9 | Tf 1.0000 1.0000 1.0000
10 | illum 2
11 | Ka 0.5882 0.5882 0.5882
12 | Kd 0.5882 0.5882 0.5882
13 | Ks 0.0450 0.0450 0.0450
14 | Ke 0.0000 0.0000 0.0000
15 | map_Ka grass.jpg
16 | map_Kd grass.jpg
17 |
--------------------------------------------------------------------------------
/RTL/floor/mousemat/mousemat.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/RTL/floor/mousemat/mousemat.jpg
--------------------------------------------------------------------------------
/RTL/floor/mousemat/mousemat.mtl:
--------------------------------------------------------------------------------
1 | #
2 | # Wavefront material file
3 | # Converted by Meshlab Group
4 | #
5 |
6 | newmtl material_0
7 | Ns 10.0000
8 | Ni 1.5000
9 | d 1.0000
10 | Tr 0.0000
11 | Tf 1.0000 1.0000 1.0000
12 | illum 2
13 | Ka 0.5882 0.5882 0.5882
14 | Kd 0.5882 0.5882 0.5882
15 | Ks 0.0450 0.0450 0.0450
16 | Ke 0.0000 0.0000 0.0000
17 | map_Kd mousemat.jpg
18 | map_Ka mousemat.jpg
19 |
20 |
--------------------------------------------------------------------------------
/RTL/floor/table/table.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/RTL/floor/table/table.jpg
--------------------------------------------------------------------------------
/RTL/floor/table/table.mtl:
--------------------------------------------------------------------------------
1 | #
2 | # Wavefront material file
3 | # Converted by Meshlab Group
4 | #
5 |
6 | newmtl material_0
7 | Ka 0.200000 0.200000 0.200000
8 | Kd 0.600000 0.600000 0.894118
9 | Ks 1.000000 1.000000 1.000000
10 | Tr 0.000000
11 | illum 2
12 | Ns 0.000000
13 | map_Kd table.jpg
14 |
15 |
--------------------------------------------------------------------------------
/RTL/main.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import os
3 | import argparse
4 | import glob
5 | import tqdm
6 | import cv2
7 | import math
8 | import numpy as np
9 | from base64 import b64encode
10 | from sys import getsizeof
11 |
12 | from flask import Response
13 | from flask import Flask
14 | from flask import render_template
15 |
16 | import torch
17 | import torch.nn.functional as F
18 |
19 | from monoport.lib.common.config import get_cfg_defaults
20 | from monoport.lib.modeling.MonoPortNet import MonoPortNet
21 | from monoport.lib.modeling.MonoPortNet import PIFuNetG, PIFuNetC
22 | from monoport.lib.modeling.geometry import orthogonal, perspective
23 | from monoport.lib.render.gl.glcontext import create_opengl_context
24 | from monoport.lib.render.gl.AlbedoRender import AlbedoRender
25 |
26 | import streamer_pytorch as streamer
27 | import human_inst_seg
28 | from implicit_seg.functional import Seg3dTopk, Seg3dLossless
29 | from implicit_seg.functional.utils import plot_mask3D
30 |
31 | from dataloader import DataLoader
32 | from scene import MonoPortScene, make_rotate
33 | from recon import pifu_calib, forward_vertices
34 |
35 |
36 | ########################################
37 | ## Global Control
38 | ########################################
39 | DESKTOP_MODE = 'TEXTURE_NORM'
40 | # assert DESKTOP_MODE in ['SEGM', 'NORM', 'TEXTURE', 'TEXTURE_NORM']
41 |
42 | SERVER_MODE = 'TEXTURE'
43 | # assert SERVER_MODE in ['NORM', 'TEXTURE']
44 |
45 | VIEW_MODE = 'AUTO'
46 | # assert VIEW_MODE in ['FRONT', 'BACK', 'LEFT', 'RIGHT', 'AUTO', 'LOAD']
47 |
48 | ########################################
49 | ## load configs
50 | ########################################
51 | parser = argparse.ArgumentParser()
52 | parser.add_argument(
53 | '-cfg', '--config_file', default=None, type=str,
54 | help='path of the yaml config file')
55 | parser.add_argument(
56 | '--camera', action="store_true")
57 | parser.add_argument(
58 | '--images', default="", nargs="*")
59 | parser.add_argument(
60 | '--image_folder', default=None)
61 | parser.add_argument(
62 | '--videos', default="", nargs="*")
63 | parser.add_argument(
64 | '--loop', action="store_true")
65 | parser.add_argument(
66 | '--use_server', action="store_true")
67 | parser.add_argument(
68 | '--ip', type=str, default="192.168.1.53")
69 | parser.add_argument(
70 | '--port', type=str, default="5555")
71 |
72 |
73 | argv = sys.argv[1:sys.argv.index('--')]
74 | args = parser.parse_args(argv)
75 | opts = sys.argv[sys.argv.index('--') + 1:]
76 |
77 | cfg = get_cfg_defaults()
78 | if args.config_file is not None:
79 | cfg.merge_from_file(args.config_file)
80 | cfg.merge_from_list(opts)
81 | cfg.freeze()
82 |
83 |
84 | ########################################
85 | ## access avaiable GPUs
86 | ########################################
87 | device_count = torch.cuda.device_count()
88 | if device_count == 1:
89 | cuda_backbone_G='cuda:0'
90 | cuda_backbone_C='cuda:0'
91 | cuda_recon='cuda:0'
92 | cuda_color='cuda:0'
93 | elif device_count == 2:
94 | cuda_backbone_G='cuda:1'
95 | cuda_backbone_C='cuda:1'
96 | cuda_recon='cuda:0'
97 | cuda_color='cuda:1'
98 | else:
99 | raise NotImplementedError
100 |
101 |
102 | ########################################
103 | ## load networks
104 | ########################################
105 | print (f'loading networkG from {cfg.netG.ckpt_path} ...')
106 | netG = MonoPortNet(cfg.netG)
107 | assert os.path.exists(cfg.netG.ckpt_path), 'we need a ckpt to run RTL demo.'
108 | if 'checkpoints' in cfg.netG.ckpt_path:
109 | ckpt = torch.load(cfg.netG.ckpt_path, map_location="cpu")
110 | netG.load_state_dict(ckpt['net'])
111 | else:
112 | netG.load_legacy_pifu(cfg.netG.ckpt_path)
113 |
114 | netG.image_filter = netG.image_filter.to(cuda_backbone_G)
115 | netG.surface_classifier = netG.surface_classifier.to(cuda_recon)
116 | netG.eval()
117 |
118 | if os.path.exists(cfg.netC.ckpt_path):
119 | print (f'loading networkC from {cfg.netC.ckpt_path} ...')
120 | netC = MonoPortNet(cfg.netC)
121 | netC.load_legacy_pifu(cfg.netC.ckpt_path)
122 |
123 | netC.image_filter = netC.image_filter.to(cuda_backbone_C)
124 | netC.surface_classifier = netC.surface_classifier.to(cuda_color)
125 | netC.eval()
126 | else:
127 | netC = None
128 | print (f'we are not loading netC ...')
129 |
130 |
131 | ########################################
132 | ## initialize data streamer
133 | ########################################
134 | print (f'initialize data streamer ...')
135 | if args.camera:
136 | data_stream = streamer.CaptureStreamer(pad=False)
137 | elif len(args.videos) > 0:
138 | data_stream = streamer.VideoListStreamer(
139 | args.videos * (10 if args.loop else 1))
140 | elif len(args.images) > 0:
141 | data_stream = streamer.ImageListStreamer(
142 | args.images * (10000 if args.loop else 1))
143 | elif args.image_folder is not None:
144 | images = sorted(glob.glob(args.image_folder+'/*.jpg'))
145 | images += sorted(glob.glob(args.image_folder+'/*.png'))
146 | data_stream = streamer.ImageListStreamer(
147 | images * (10 if args.loop else 1))
148 |
149 |
150 | ########################################
151 | ## human segmentation model
152 | ########################################
153 | seg_engine = human_inst_seg.Segmentation(
154 | device=cuda_backbone_G, verbose=False)
155 | seg_engine.eval()
156 |
157 |
158 | ########################################
159 | ## pre-loaded scene for rendering
160 | ########################################
161 | scene = MonoPortScene(size=(256, 256))
162 |
163 |
164 | ########################################
165 | ## variables for hierachy occupancy reconstruction
166 | ########################################
167 | calib_tensor = torch.eye(4).unsqueeze(0).to(cuda_recon)
168 | @torch.no_grad()
169 | def query_func(points, im_feat_list, calib_tensor):
170 | '''
171 | - points: size of (bz, N, 3)
172 | - proj_matrix: size of (bz, 4, 4)
173 | return: size of (bz, 1, N)
174 | '''
175 | assert len(points) == 1
176 | samples = points.repeat(1, 1, 1)
177 | samples = samples.permute(0, 2, 1) # [bz, 3, N]
178 |
179 | preds = netG.query(
180 | im_feat_list,
181 | points=samples,
182 | calibs=calib_tensor)[0]
183 | return preds
184 |
185 | b_min = torch.tensor([-1.0, -1.0, -1.0]).float()
186 | b_max = torch.tensor([ 1.0, 1.0, 1.0]).float()
187 | resolutions = [16+1, 32+1, 64+1, 128+1, 256+1]
188 | reconEngine = Seg3dLossless(
189 | query_func=query_func,
190 | b_min=b_min.unsqueeze(0).numpy(),
191 | b_max=b_max.unsqueeze(0).numpy(),
192 | resolutions=resolutions,
193 | balance_value=0.5,
194 | use_cuda_impl=False,
195 | faster=True).to(cuda_recon)
196 |
197 |
198 | ########################################
199 | ## variables for color inference
200 | ########################################
201 | canvas = torch.ones(
202 | (resolutions[-1], resolutions[-1], 3), dtype=torch.float32
203 | ).to(cuda_color)
204 | mat = torch.eye(4, dtype=torch.float32)
205 | length = b_max - b_min
206 | mat[0, 0] = length[0] / resolutions[-1]
207 | mat[1, 1] = length[1] / resolutions[-1]
208 | mat[2, 2] = length[2] / resolutions[-1]
209 | mat[0:3, 3] = b_min
210 | mat_color = mat.to(cuda_color)
211 |
212 | @torch.no_grad()
213 | def colorization(netC, feat_tensor_C, X, Y, Z, calib_tensor, norm=None):
214 | if X is None:
215 | return None
216 |
217 | device = calib_tensor.device
218 | global canvas
219 | # use normal as color
220 | if norm is not None:
221 | color = (norm + 1) / 2
222 | color = color.clamp(0, 1)
223 | image = canvas.clone()
224 | image[X, Y, :] = color
225 | return image
226 |
227 | # use netC to predict color
228 | else:
229 | feat_tensor_C = [
230 | [feat.to(device) for feat in feats] for feats in feat_tensor_C]
231 | verts = torch.stack([
232 | X.float(), Y.float(), resolutions[-1]-Z.float() # TODO
233 | ], dim=1)
234 |
235 | samples = verts.unsqueeze(0).repeat(1, 1, 1)
236 | samples = samples.permute(0, 2, 1) # [bz, 3, N]
237 | samples = orthogonal(samples, mat_color.unsqueeze(0))
238 |
239 | preds = netC.query(
240 | feat_tensor_C,
241 | points=samples,
242 | calibs=calib_tensor)[0]
243 |
244 | color = preds[0] * 0.5 + 0.5 # FIXME
245 | color = color.t() # [N, 3]
246 |
247 | image = canvas.clone()
248 | image[X, Y, :] = color
249 | return image
250 |
251 |
252 | @torch.no_grad()
253 | def visulization(render_norm, render_tex=None):
254 | if render_norm is None and render_tex is None:
255 | return None, None, None
256 |
257 | render_size = 256
258 |
259 | if render_norm is not None:
260 | render_norm = render_norm.detach() * 255.0
261 | render_norm = torch.rot90(render_norm, 1, [0, 1]).permute(2, 0, 1).unsqueeze(0)
262 | render_norm = F.interpolate(render_norm, size=(render_size, render_size))
263 | render_norm = render_norm[0].cpu().numpy().transpose(1, 2, 0)
264 | reference = render_norm
265 |
266 | if render_tex is not None:
267 | render_tex = render_tex.detach() * 255.0
268 | render_tex = torch.rot90(render_tex, 1, [0, 1]).permute(2, 0, 1).unsqueeze(0)
269 | render_tex = F.interpolate(render_tex, size=(render_size, render_size))
270 | render_tex = render_tex[0].cpu().numpy().transpose(1, 2, 0)
271 | reference = render_tex
272 |
273 | bg = np.logical_and(
274 | np.logical_and(
275 | reference[:, :, 0] == 255,
276 | reference[:, :, 1] == 255),
277 | reference[:, :, 2] == 255,
278 | ).reshape(render_size, render_size, 1)
279 | mask = ~bg
280 |
281 | return render_norm, render_tex, mask
282 |
283 |
284 |
285 | ########################################
286 | ## define async processors
287 | ########################################
288 | mean = torch.tensor(cfg.netG.mean).to(cuda_backbone_G).view(1, 3, 1, 1)
289 | std = torch.tensor(cfg.netG.std).to(cuda_backbone_G).view(1, 3, 1, 1)
290 | scaled_boxes = [torch.Tensor([[ 50.0, 0.0, 450.0, 500.0]]).to(cuda_backbone_G)]
291 |
292 | def update_camera():
293 | extrinsic = np.array([
294 | [1.0, 0.0, 0.0, 0.0],
295 | [0.0, 1.0, 0.0, 0.0],
296 | [0.0, 0.0, 1.0, -2.0],
297 | [0.0, 0.0, 0.0, 1.0],
298 | ], dtype=np.float32)
299 |
300 | if VIEW_MODE == 'FRONT':
301 | yaw, pitch = 20, 0
302 | elif VIEW_MODE == 'BACK':
303 | yaw, pitch = 20, 180
304 | elif VIEW_MODE == 'LEFT':
305 | yaw, pitch = 20, 90
306 | elif VIEW_MODE == 'RIGHT':
307 | yaw, pitch = 20, 270
308 | elif VIEW_MODE == 'AUTO':
309 | extrinsic, intrinsic = scene.update_camera(load=False)
310 | return extrinsic, intrinsic
311 | elif VIEW_MODE == 'LOAD':
312 | extrinsic, intrinsic = scene.update_camera(load=True)
313 | return extrinsic, intrinsic
314 | else:
315 | raise NotImplementedError
316 |
317 | intrinsic = scene.intrinsic
318 | R = np.matmul(
319 | make_rotate(math.radians(yaw), 0, 0),
320 | make_rotate(0, math.radians(pitch), 0)
321 | )
322 | extrinsic[0:3, 0:3] = R
323 | return extrinsic, intrinsic
324 |
325 |
326 | processors=[
327 | lambda data: {"input": data.to(cuda_backbone_G, non_blocking=True)},
328 |
329 | # scene camera updating
330 | lambda data_dict: {
331 | **data_dict,
332 | **dict(zip(
333 | ["extrinsic", "intrinsic"],
334 | update_camera(),
335 | ))},
336 |
337 | # calculate calib tensor
338 | lambda data_dict: {
339 | **data_dict,
340 | "calib_tensor": pifu_calib(
341 | data_dict["extrinsic"], data_dict["intrinsic"], device=cuda_recon)
342 | },
343 |
344 | # instance segmentation:
345 | lambda data_dict: {
346 | **data_dict,
347 | **dict(zip(
348 | ["segm", "bboxes", "probs"],
349 | seg_engine(data_dict["input"], scaled_boxes)
350 | ))},
351 |
352 | # update input by removing bg
353 | lambda data_dict: {
354 | **data_dict,
355 | "input_netG": (
356 | ((data_dict["segm"][:, 0:3] * 0.5 + 0.5) - mean) / std
357 | )*data_dict["segm"][:, 3:4]
358 | },
359 |
360 | # update input by removing bg
361 | lambda data_dict: {
362 | **data_dict,
363 | "input_netC": data_dict["segm"][:, 0:3] * data_dict["segm"][:, 3:4]
364 | },
365 |
366 | # pifu netG feature extraction
367 | lambda data_dict: {
368 | **data_dict,
369 | "feat_tensor_G": netG.filter(data_dict["input_netG"])
370 | },
371 |
372 | # pifu netC feature extraction
373 | lambda data_dict: {
374 | **data_dict,
375 | "feat_tensor_C": netC.filter(
376 | data_dict["input_netC"].to(cuda_backbone_C, non_blocking=True),
377 | feat_prior=data_dict["feat_tensor_G"][-1][-1]) \
378 | if (netC is not None) and ('TEXTURE' in DESKTOP_MODE or 'TEXTURE' in SERVER_MODE) else None
379 | },
380 |
381 | # move feature to cuda_recon device
382 | lambda data_dict: {
383 | **data_dict,
384 | "feat_tensor_G": [
385 | [feat.to(cuda_recon) for feat in feats]
386 | for feats in data_dict["feat_tensor_G"]]
387 | },
388 |
389 | # pifu sdf space recon
390 | lambda data_dict: {
391 | **data_dict,
392 | "sdf": reconEngine(
393 | im_feat_list=data_dict["feat_tensor_G"],
394 | calib_tensor=data_dict["calib_tensor"])
395 | },
396 |
397 | # lambda data_dict: plot_mask3D(
398 | # data_dict["sdf"].to("cpu"), title="sdf"),
399 |
400 | # pifu visible vertices
401 | lambda data_dict: {
402 | **data_dict,
403 | **dict(zip(
404 | ["X", "Y", "Z", "norm"],
405 | forward_vertices(data_dict["sdf"], direction="front")
406 | ))},
407 |
408 | lambda data_dict: {
409 | **data_dict,
410 | "X": data_dict['X'].to(cuda_color) if data_dict['X'] is not None else None,
411 | "Y": data_dict['Y'].to(cuda_color) if data_dict['X'] is not None else None,
412 | "Z": data_dict['Z'].to(cuda_color) if data_dict['X'] is not None else None,
413 | "norm": data_dict['norm'].to(cuda_color) if data_dict['X'] is not None else None,
414 | "calib_tensor": data_dict['calib_tensor'].to(cuda_color) if data_dict['X'] is not None else None,
415 | },
416 |
417 | # pifu render normal
418 | lambda data_dict: {
419 | **data_dict,
420 | "render_norm": colorization(
421 | netC,
422 | None,
423 | data_dict["X"],
424 | data_dict["Y"],
425 | data_dict["Z"],
426 | data_dict["calib_tensor"],
427 | data_dict["norm"]) if ('NORM' in DESKTOP_MODE or 'NORM' in SERVER_MODE) else None
428 | },
429 |
430 | # pifu render texture
431 | lambda data_dict: {
432 | **data_dict,
433 | "render_tex": colorization(
434 | netC,
435 | data_dict["feat_tensor_C"],
436 | data_dict["X"],
437 | data_dict["Y"],
438 | data_dict["Z"],
439 | data_dict["calib_tensor"],
440 | None) if data_dict["feat_tensor_C"] else None
441 | },
442 |
443 | # visualization
444 | lambda data_dict: {
445 | **data_dict,
446 | **dict(zip(
447 | ["render_norm", "render_tex", "mask"],
448 | visulization(
449 | data_dict["render_norm"],
450 | data_dict["render_tex"])
451 | ))},
452 | ]
453 |
454 |
455 | ########################################
456 | ## build async processor
457 | ########################################
458 | loader = DataLoader(
459 | data_stream,
460 | batch_size=1,
461 | num_workers=1,
462 | pin_memory=True,
463 | processors=processors,
464 | )
465 |
466 |
467 | def main_loop():
468 | global DESKTOP_MODE, SERVER_MODE, VIEW_MODE
469 |
470 | window_server = np.ones((256, 256, 3), dtype=np.uint8) * 255
471 | window_desktop = np.ones((512, 1024, 3), dtype=np.uint8) * 255
472 |
473 | create_opengl_context(256, 256)
474 | renderer = AlbedoRender(width=256, height=256, multi_sample_rate=1)
475 | renderer.set_attrib(0, scene.vert_data)
476 | renderer.set_attrib(1, scene.uv_data)
477 | renderer.set_texture('TargetTexture', scene.texture_image)
478 |
479 | def render(extrinsic, intrinsic):
480 | uniform_dict = {'ModelMat': extrinsic, 'PerspMat': intrinsic}
481 | renderer.draw(uniform_dict)
482 | color = (renderer.get_color() * 255).astype(np.uint8)
483 | background = cv2.cvtColor(color, cv2.COLOR_RGB2BGR)
484 | background = cv2.resize(background, (256, 256))
485 | return background
486 |
487 | for data_dict in tqdm.tqdm(loader):
488 | render_norm = data_dict["render_norm"] # [256, 256, 3] RGB
489 | render_tex = data_dict["render_tex"] # [256, 256, 3] RGB
490 | mask = data_dict["mask"]
491 | extrinsic = data_dict["extrinsic"]
492 | intrinsic = data_dict["intrinsic"]
493 |
494 | if DESKTOP_MODE is not None:
495 | input4c = data_dict["segm"].cpu().numpy()[0].transpose(1, 2, 0) # [512, 512, 4]
496 | input = (input4c[:, :, 0:3] * 0.5) + 0.5
497 | if DESKTOP_MODE == 'SEGM':
498 | segmentation = (input4c[:, :, 0:3] * input4c[:, :, 3:4] * 0.5) + 0.5
499 | window_desktop = np.uint8(np.hstack([
500 | input * 255,
501 | segmentation * 255
502 | ])) # RGB
503 | elif DESKTOP_MODE == 'NORM':
504 | if render_norm is None:
505 | render_norm = np.ones((256, 256, 3), dtype=np.float32) * 255
506 | window_desktop = np.uint8(np.hstack([
507 | input * 255,
508 | cv2.resize(render_norm, (512, 512))
509 | ])) # RGB
510 | elif DESKTOP_MODE == 'TEXTURE':
511 | if render_tex is None:
512 | render_tex = np.ones((256, 256, 3), dtype=np.float32) * 255
513 | window_desktop = np.uint8(np.hstack([
514 | input * 255,
515 | cv2.resize(render_tex, (512, 512))
516 | ])) # RGB
517 | elif DESKTOP_MODE == 'TEXTURE_NORM':
518 | if render_tex is None:
519 | render_tex = np.ones((256, 256, 3), dtype=np.float32) * 255
520 | if render_norm is None:
521 | render_norm = np.ones((256, 256, 3), dtype=np.float32) * 255
522 | window_desktop = np.uint8(np.vstack([
523 | render_tex,
524 | render_norm
525 | ])) # RGB
526 | else:
527 | window_desktop = None
528 |
529 | if DESKTOP_MODE is not None:
530 | # window_desktop = cv2.resize(window_desktop, (2400, 1200))
531 | cv2.namedWindow('window_desktop', cv2.WINDOW_NORMAL)
532 | # cv2.setWindowProperty('window_desktop', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
533 | cv2.imshow('window_desktop', window_desktop[:, :, ::-1])
534 |
535 | if args.use_server:
536 | if SERVER_MODE == 'NORM':
537 | background = render(extrinsic, intrinsic)
538 | if mask is None:
539 | window_server = background
540 | else:
541 | window_server = np.uint8(mask * render_norm + (1 - mask) * background)
542 | elif SERVER_MODE == 'TEXTURE':
543 | background = render(extrinsic, intrinsic)
544 | if mask is None:
545 | window_server = background
546 | else:
547 | window_server = np.uint8(mask * render_tex + (1 - mask) * background)
548 | else:
549 | if render_norm is not None:
550 | window_server = np.uint8(render_norm)
551 |
552 | # yield window_desktop, window_server
553 | (flag, encodedImage) = cv2.imencode(".jpg", window_server[:, :, ::-1])
554 | if not flag:
555 | continue
556 | yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' +
557 | bytearray(encodedImage) + b'\r\n')
558 |
559 | key = cv2.waitKey(1)
560 | if key == ord('q'):
561 | DESKTOP_MODE = 'SEGM'
562 | elif key == ord('w'):
563 | DESKTOP_MODE = 'NORM'
564 | elif key == ord('e'):
565 | DESKTOP_MODE = 'TEXTURE'
566 | elif key == ord('r'):
567 | DESKTOP_MODE = 'TEXTURE_NORM'
568 |
569 | # elif key == ord('a'):
570 | # SERVER_MODE = 'SEGM'
571 | elif key == ord('s'):
572 | SERVER_MODE = 'NORM'
573 | elif key == ord('d'):
574 | SERVER_MODE = 'TEXTURE'
575 | elif key == ord('f'):
576 | SERVER_MODE = None
577 |
578 | elif key == ord('z'):
579 | VIEW_MODE = 'FRONT'
580 | elif key == ord('x'):
581 | VIEW_MODE = 'BACK'
582 | elif key == ord('c'):
583 | VIEW_MODE = 'LEFT'
584 | elif key == ord('v'):
585 | VIEW_MODE = 'RIGHT'
586 | elif key == ord('b'):
587 | VIEW_MODE = 'AUTO'
588 | elif key == ord('n'):
589 | VIEW_MODE = 'LOAD'
590 |
591 | elif key == ord('g'):
592 | scene.shift_floor()
593 |
594 |
595 |
596 |
597 | if __name__ == '__main__':
598 | if args.use_server:
599 | ########################################
600 | ## Flask related
601 | ########################################
602 | app = Flask(__name__)
603 |
604 | @app.route("/")
605 | def index():
606 | return render_template("test_flask.html")
607 |
608 | @app.route("/video_feed")
609 | def video_feed():
610 | return Response(main_loop(),
611 | mimetype = "multipart/x-mixed-replace; boundary=frame")
612 |
613 | # start the flask app
614 | app.run(host=args.ip, port=args.port, debug=True,
615 | threaded=True, use_reloader=False)
616 | else:
617 | print('start main_loop.')
618 | for _ in main_loop():
619 | pass
620 |
--------------------------------------------------------------------------------
/RTL/recon.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import numpy as np
3 |
4 | @torch.no_grad()
5 | def pifu_calib(extrinsic, intrinsic, device="cuda:0"):
6 | pifu_matrix = np.array([
7 | [1.0, 0.0, 0.0, 0.0],
8 | [0.0, -1.0, 0.0, 0.0],
9 | [0.0, 0.0, 1.0, 0.0],
10 | [0.0, 0.0, 0.0, 1.0],
11 | ])
12 |
13 | # orthognal
14 | intrinsic = intrinsic.copy()
15 | intrinsic[2, 2] = intrinsic[0, 0]
16 | intrinsic[2, 3] = 0
17 | extrinsic = extrinsic.copy()
18 | extrinsic[2, 3] = 0
19 |
20 | calib = np.linalg.inv(
21 | np.matmul(np.matmul(intrinsic, extrinsic), pifu_matrix))
22 |
23 | calib_tensor = torch.from_numpy(calib).unsqueeze(0).float()
24 | calib_tensor = calib_tensor.to(device)
25 | return calib_tensor
26 |
27 | @torch.no_grad()
28 | def forward_vertices(sdf, direction="front"):
29 | '''
30 | - direction: "front" | "back" | "left" | "right"
31 | '''
32 | if sdf is None:
33 | return None, None, None, None
34 | else:
35 | sdf = sdf[0, 0]
36 |
37 | device = sdf.device
38 | resolution = sdf.size(2)
39 | if direction == "front":
40 | pass
41 | elif direction == "left":
42 | sdf = sdf.permute(2, 1, 0)
43 | elif direction == "back":
44 | inv_idx = torch.arange(sdf.size(2)-1, -1, -1).long()
45 | sdf = sdf[inv_idx, :, :]
46 | elif direction == "right":
47 | inv_idx = torch.arange(sdf.size(2)-1, -1, -1).long()
48 | sdf = sdf[inv_idx, :, :]
49 | sdf = sdf.permute(2, 1, 0)
50 |
51 | inv_idx = torch.arange(sdf.size(2)-1, -1, -1).long()
52 | sdf = sdf[inv_idx, :, :]
53 | sdf_all = sdf.permute(2, 1, 0)
54 |
55 | # shadow
56 | grad_v = (sdf_all>0.5) * torch.linspace(resolution, 1, steps=resolution).to(device)
57 | grad_c = torch.ones_like(sdf_all) * torch.linspace(0, resolution-1, steps=resolution).to(device)
58 | max_v, max_c = grad_v.max(dim=2)
59 | shadow = grad_c > max_c.view(resolution, resolution, 1)
60 | keep = (sdf_all>0.5) & (~shadow)
61 |
62 | p1 = keep.nonzero().t() #[3, N]
63 | p2 = p1.clone() # z
64 | p2[2, :] = (p2[2, :]-2).clamp(0, resolution)
65 | p3 = p1.clone() # y
66 | p3[1, :] = (p3[1, :]-2).clamp(0, resolution)
67 | p4 = p1.clone() # x
68 | p4[0, :] = (p4[0, :]-2).clamp(0, resolution)
69 |
70 | v1 = sdf_all[p1[0, :], p1[1, :], p1[2, :]]
71 | v2 = sdf_all[p2[0, :], p2[1, :], p2[2, :]]
72 | v3 = sdf_all[p3[0, :], p3[1, :], p3[2, :]]
73 | v4 = sdf_all[p4[0, :], p4[1, :], p4[2, :]]
74 |
75 | X = p1[0, :].long() #[N,]
76 | Y = p1[1, :].long() #[N,]
77 | Z = p2[2, :].float() * (0.5 - v1) / (v2 - v1) + p1[2, :].float() * (v2 - 0.5) / (v2 - v1) #[N,]
78 | Z = Z.clamp(0, resolution)
79 |
80 | # normal
81 | norm_z = v2 - v1
82 | norm_y = v3 - v1
83 | norm_x = v4 - v1
84 | # print (v2.min(dim=0)[0], v2.max(dim=0)[0], v3.min(dim=0)[0], v3.max(dim=0)[0])
85 |
86 | norm = torch.stack([norm_x, norm_y, norm_z], dim=1)
87 | norm = norm / torch.norm(norm, p=2, dim=1, keepdim=True)
88 |
89 | return X, Y, Z, norm
90 |
--------------------------------------------------------------------------------
/RTL/run_camera.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import glob
3 | import torch
4 | import tqdm
5 | import numpy as np
6 | import cv2
7 |
8 | import streamer_pytorch as streamer
9 | import human_inst_seg
10 |
11 | from dataloader import DataLoader
12 | from scene import MonoPortScene
13 | from recon import pifu_calib, forward_vertices
14 |
15 |
16 | ########################################
17 | ## load configs
18 | ########################################
19 | parser = argparse.ArgumentParser()
20 | parser.add_argument(
21 | '--camera', action="store_true")
22 | parser.add_argument(
23 | '--images', default="", nargs="*")
24 | parser.add_argument(
25 | '--image_folder', default=None)
26 | parser.add_argument(
27 | '--videos', default="", nargs="*")
28 | parser.add_argument(
29 | '--loop', action="store_true")
30 | parser.add_argument(
31 | '--vis', action="store_true")
32 | parser.add_argument(
33 | '--use_VRweb', action="store_true")
34 | args = parser.parse_args()
35 | device = 'cuda:0'
36 | scaled_boxes = [torch.Tensor([[ 50.0, 0.0, 450.0, 500.0]]).to(device)]
37 |
38 |
39 | ########################################
40 | ## initialize data streamer
41 | ########################################
42 | print (f'initialize data streamer ...')
43 | if args.camera:
44 | data_stream = streamer.CaptureStreamer(pad=False)
45 | elif len(args.videos) > 0:
46 | data_stream = streamer.VideoListStreamer(
47 | args.videos * (10 if args.loop else 1))
48 | elif len(args.images) > 0:
49 | data_stream = streamer.ImageListStreamer(
50 | args.images * (10000 if args.loop else 1))
51 | elif args.image_folder is not None:
52 | images = sorted(glob.glob(args.image_folder+'/*.jpg'))
53 | images += sorted(glob.glob(args.image_folder+'/*.png'))
54 | data_stream = streamer.ImageListStreamer(
55 | images * (10 if args.loop else 1))
56 |
57 |
58 | ########################################
59 | ## human segmentation model
60 | ########################################
61 | seg_engine = human_inst_seg.Segmentation(
62 | device=device, verbose=False)
63 | seg_engine.eval()
64 |
65 |
66 | processors=[
67 | lambda data: {"input": data.to(device, non_blocking=True)},
68 |
69 | # instance segmentation:
70 | lambda data_dict: {
71 | **data_dict,
72 | **dict(zip(
73 | ["segm", "bboxes", "probs"],
74 | seg_engine(data_dict["input"], scaled_boxes)
75 | ))},
76 | ]
77 |
78 |
79 | ########################################
80 | ## build async processor
81 | ########################################
82 | loader = DataLoader(
83 | data_stream,
84 | batch_size=1,
85 | num_workers=1,
86 | pin_memory=True,
87 | processors=processors,
88 | )
89 |
90 |
91 | @torch.no_grad()
92 | def main_loop():
93 | for data_dict in tqdm.tqdm(loader):
94 | segm = data_dict["segm"].cpu().numpy()[0].transpose(1, 2, 0) # [512, 512, 4]
95 | input = (segm[:, :, 0:3] * 0.5) + 0.5
96 | output = (segm[:, :, 0:3] * segm[:, :, 3:4] * 0.5) + 0.5
97 | x1, y1, x2, y2 = scaled_boxes[0].cpu().numpy()[0]
98 |
99 | window = np.hstack([input, output]).astype(np.float32)
100 | window = np.uint8(window[:, :, ::-1] * 255) # To BGR
101 | window = cv2.rectangle(cv2.UMat(window), (x1, y1), (x2, y2), (255, 0, 0), thickness=3)
102 |
103 | window = cv2.resize(window, (0, 0), fx=3, fy=3)
104 | cv2.imshow('segmenation', window)
105 | cv2.waitKey(1)
106 |
107 |
108 | if __name__ == '__main__':
109 | try:
110 | main_loop()
111 | except Exception as e:
112 | print (e)
113 | del data_stream
--------------------------------------------------------------------------------
/RTL/scene.py:
--------------------------------------------------------------------------------
1 | import cv2
2 | import numpy as np
3 | import math
4 | import os
5 | import json
6 |
7 | from monoport.lib.render.gl.glcontext import create_opengl_context
8 | from monoport.lib.render.gl.AlbedoRender import AlbedoRender
9 | from monoport.lib.render.BaseCamera import BaseCamera
10 | from monoport.lib.render.PespectiveCamera import PersPectiveCamera
11 | from monoport.lib.render.CameraPose import CameraPose
12 |
13 | from monoport.lib.mesh_util import load_obj_mesh
14 |
15 |
16 | _RTL_DATA_FOLDER = os.path.join(
17 | os.path.dirname(__file__))
18 |
19 |
20 | def _load_floor(floor_type="grass", floor_size=3.0, floor_center=np.array([0, -0.9, 0])):
21 | mesh_file = os.path.join(
22 | _RTL_DATA_FOLDER,
23 | f'floor/{floor_type}/{floor_type}.obj')
24 | text_file = os.path.join(
25 | _RTL_DATA_FOLDER,
26 | f'floor/{floor_type}/{floor_type}.jpg')
27 | vertices, faces, normals, faces_normals, textures, face_textures = load_obj_mesh(
28 | mesh_file, with_normal=True, with_texture=True)
29 | vertices = vertices[:, [0, 2, 1]]
30 |
31 | # change cm to meter
32 | vertices = vertices / 150 * floor_size
33 | vertices = vertices - vertices.mean(axis=0)
34 | vertices += floor_center
35 |
36 | texture_image = cv2.imread(text_file)
37 | # texture_image = cv2.cvtColor(texture_image, cv2.COLOR_BGR2RGB)
38 |
39 | # Here we pack the vertex data needed for the render
40 | vert_data = vertices[faces.reshape([-1])]
41 | uv_data = textures[face_textures.reshape([-1])]
42 | return vert_data, uv_data, texture_image
43 |
44 |
45 | def _load_intrinsic(near=0.0, far=10.0, scale=2.0):
46 | intrinsic_cam = BaseCamera()
47 | intrinsic_cam.near = near
48 | intrinsic_cam.far = far
49 | intrinsic_cam.set_parameters(scale, scale)
50 | return intrinsic_cam.get_projection_mat()
51 |
52 |
53 | def _load_extrinsic():
54 | path = os.path.join(
55 | _RTL_DATA_FOLDER, 'webxr/modelview.json')
56 | with open(path, 'r') as f:
57 | extrinsic = json.load(f)['data']
58 | extrinsic = np.array(extrinsic).reshape(4, 4).T
59 | return extrinsic
60 |
61 |
62 | def make_rotate(rx, ry, rz):
63 | sinX = np.sin(rx)
64 | sinY = np.sin(ry)
65 | sinZ = np.sin(rz)
66 |
67 | cosX = np.cos(rx)
68 | cosY = np.cos(ry)
69 | cosZ = np.cos(rz)
70 |
71 | Rx = np.zeros((3,3))
72 | Rx[0, 0] = 1.0
73 | Rx[1, 1] = cosX
74 | Rx[1, 2] = -sinX
75 | Rx[2, 1] = sinX
76 | Rx[2, 2] = cosX
77 |
78 | Ry = np.zeros((3,3))
79 | Ry[0, 0] = cosY
80 | Ry[0, 2] = sinY
81 | Ry[1, 1] = 1.0
82 | Ry[2, 0] = -sinY
83 | Ry[2, 2] = cosY
84 |
85 | Rz = np.zeros((3,3))
86 | Rz[0, 0] = cosZ
87 | Rz[0, 1] = -sinZ
88 | Rz[1, 0] = sinZ
89 | Rz[1, 1] = cosZ
90 | Rz[2, 2] = 1.0
91 |
92 | R = np.matmul(np.matmul(Rz,Ry),Rx)
93 | return R
94 |
95 |
96 | class MonoPortScene:
97 | def __init__(self, size=(512, 512), debug=False):
98 | self.floors = ["carpet", "drum", "grass", "mousemat", "table"]
99 | self.vert_data, self.uv_data, self.texture_image = _load_floor()
100 | self.intrinsic = _load_intrinsic()
101 |
102 | create_opengl_context(size[0], size[1])
103 | self.renderer = AlbedoRender(width=size[0], height=size[1], multi_sample_rate=1)
104 | self.renderer.set_attrib(0, self.vert_data)
105 | self.renderer.set_attrib(1, self.uv_data)
106 | self.renderer.set_texture('TargetTexture', self.texture_image)
107 |
108 | self.extrinsic = np.array([
109 | [1.0, 0.0, 0.0, 0.0],
110 | [0.0, 1.0, 0.0, 0.0],
111 | [0.0, 0.0, 1.0, -2.0],
112 | [0.0, 0.0, 0.0, 1.0],
113 | ], dtype=np.float32)
114 | self.step = 0
115 |
116 | def shift_floor(self):
117 | self.vert_data, self.uv_data, self.texture_image = _load_floor(floor_type=np.random.choice(self.floors))
118 | self.renderer.set_attrib(0, self.vert_data)
119 | self.renderer.set_attrib(1, self.uv_data)
120 | self.renderer.set_texture('TargetTexture', self.texture_image)
121 |
122 | def update_camera(self, load=False):
123 | if load == False:
124 | if self.step < 3600000:
125 | yaw = 20
126 | pitch = self.step
127 | else:
128 | yaw = self.step % 180
129 | pitch = 0
130 |
131 | R = np.matmul(
132 | make_rotate(math.radians(yaw), 0, 0),
133 | make_rotate(0, math.radians(pitch), 0)
134 | )
135 | self.extrinsic[0:3, 0:3] = R
136 | self.step += 3
137 | extrinsic = self.extrinsic
138 | else:
139 | while True:
140 | try:
141 | extrinsic = _load_extrinsic()
142 | break
143 | except Exception as e:
144 | print (e)
145 |
146 | return extrinsic, self.intrinsic
147 |
148 | def render(self, extrinsic, intrinsic):
149 | uniform_dict = {
150 | 'ModelMat': extrinsic,
151 | 'PerspMat': intrinsic,
152 | }
153 | self.renderer.draw(
154 | uniform_dict
155 | )
156 |
157 | color = (self.renderer.get_color() * 255).astype(np.uint8)
158 | color = cv2.cvtColor(color, cv2.COLOR_RGB2BGR)
159 | return color
160 |
161 |
162 | if __name__ == '__main__':
163 | import tqdm
164 | scene = MonoPortScene(debug=True)
165 |
166 | for _ in tqdm.tqdm(range(10000)):
167 | extrinsic, intrinsic = scene.update_camera()
168 | background = scene.render(extrinsic, intrinsic)
169 | # print (extrinsic)
170 | cv2.imshow('scene', background)
171 | cv2.waitKey(15)
--------------------------------------------------------------------------------
/RTL/templates/qrcode.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/RTL/templates/qrcode.png
--------------------------------------------------------------------------------
/RTL/templates/test_flask.html:
--------------------------------------------------------------------------------
1 |
2 | MonoPort (SIGGRAPH Real-Time Live 2020)
5 |
6 |
48 |
49 |
56 |
57 |
58 |
MonoPort VR Demo
59 |
SIGGRAPH Real-Time Live 2020
60 |
61 |
62 |
>
39 | """
40 | feats_stages = self.image_filter(images)
41 | if feat_prior is not None: # for netC
42 | feat_prior = F.interpolate(feat_prior, size=(128, 128))
43 | feats_stages = [
44 | [torch.cat([feat_prior, feat_per_lvl], dim=1)
45 | for feat_per_lvl in feats] for feats in feats_stages]
46 | return feats_stages
47 |
48 | def query(self, feats_stages, points, calibs=None, transforms=None):
49 | """Given 3D points, query the network predictions for each point.
50 |
51 | Arguments:
52 | feats_stages {list(list(torch.tensor))} -- image feature lists. First level list
53 | is for multi-stage losses. Second level list is for multi-level features.
54 | points {torch.tensor} -- [B, 3, N] world space coordinates of points.
55 | calibs {torch.tensor} -- [B, 3, 4] calibration matrices for each image.
56 |
57 | Keyword Arguments:
58 | transforms {torch.tensor} -- Optional [B, 2, 3] image space coordinate transforms
59 |
60 | Returns:
61 | list(torch.tensor) -- predictions for each point at each stage. list of [B, Res, N].
62 | """
63 | if not self.training:
64 | feats_stages = [feats_stages[-1]]
65 |
66 | if calibs is None:
67 | xyz = points
68 | else:
69 | xyz = self.projection(points, calibs, transforms)
70 |
71 | xy = xyz[:, :2, :]
72 | z = xyz[:, 2:3, :]
73 |
74 | in_img = (xy[:, 0] >= -1.0) & (xy[:, 0] <= 1.0) & (xy[:, 1] >= -1.0) & (xy[:, 1] <= 1.0)
75 |
76 | z_feat = self.normalizer(z, calibs=calibs)
77 | pred_stages = []
78 | for feats in feats_stages: # each stage
79 | # concatenate feats cross all the levels. [B, Feat_all, N]
80 | # TODO: another option here is to do ADD op if all the levels
81 | # have same feature dimentions.
82 | point_local_feat = torch.cat(
83 | [index(feat_per_lvl, xy) for feat_per_lvl in feats] + [z_feat], 1)
84 |
85 | # [B, Res, N]. Res=1 here.
86 | pred = self.surface_classifier(point_local_feat)
87 |
88 | # out of image plane is always set to 0
89 | preds = in_img[:, None].float() * pred
90 | pred_stages.append(preds)
91 | return pred_stages
92 |
93 | def get_loss(self, pred_stages, labels):
94 | """Calculate loss between predictions and labels
95 |
96 | Arguments:
97 | pred_stages {list(torch.tensor)} -- predictions at each stage. list of [B, Res, N]
98 | labels {torch.tensor} -- labels. typically [B, Res, N]
99 |
100 | Raises:
101 | NotImplementedError:
102 |
103 | Returns:
104 | torch.tensor -- average loss cross stages.
105 | """
106 | if self.opt.loss.IMF == 'MSE':
107 | loss_func = F.mse_loss
108 | elif self.opt.loss.IMF == 'L1':
109 | loss_func = F.l1_loss
110 | else:
111 | raise NotImplementedError
112 |
113 | loss = 0
114 | for pred in pred_stages:
115 | loss += loss_func(pred, labels)
116 | loss /= len(pred_stages)
117 | return loss
118 |
119 |
120 | def forward(self, images, points, calibs, transforms=None, labels=None, feat_prior=None):
121 | """Forward function given points and calibs
122 |
123 | Arguments:
124 | images {torch.tensor} -- shape of [B, C, H, W]
125 | points {torch.tensor} -- shape of [B, 3, N]
126 | calibs {torch.tesnor} -- shape of [B, 3, 4]
127 |
128 | Keyword Arguments:
129 | transforms {torch.tensor} -- shape of [B, 2, 3] (default: {None})
130 | labels {torch.tensor} -- shape of [B, Res, N] (default: {None})
131 |
132 | Returns:
133 | torch.tensor, [torch.scaler] -- return preds at last stages. shape of [B, Res, N]
134 | """
135 | feats_stages = self.filter(images, feat_prior)
136 | pred_stages = self.query(feats_stages, points, calibs, transforms)
137 |
138 | if labels is not None:
139 | loss = self.get_loss(pred_stages, labels)
140 | return pred_stages[-1], loss
141 | else:
142 | return pred_stages[-1]
143 |
144 | # def load_legacy(self, ckpt_path):
145 | # ckpt = torch.load(ckpt_path, map_location="cpu")["netG"]
146 | # backbone_dict = {
147 | # k.replace("image_filter.", ""): v for k, v in ckpt.items() if "image_filter" in k}
148 | # head_dict = {
149 | # k.replace("surface_classifier.", ""): v for k, v in ckpt.items() if "surface_classifier" in k}
150 | # self.image_filter.load_state_dict(backbone_dict)
151 | # self.surface_classifier.load_state_dict(head_dict)
152 |
153 | def load_legacy_pifu(self, ckpt_path):
154 | ckpt = torch.load(ckpt_path, map_location="cpu")
155 | backbone_dict = {
156 | k.replace("image_filter.", ""): v for k, v in ckpt.items() if "image_filter" in k}
157 | head_dict = {
158 | k.replace("surface_classifier.conv", "filters."): v for k, v in ckpt.items() if "surface_classifier" in k}
159 | self.image_filter.load_state_dict(backbone_dict)
160 | self.surface_classifier.load_state_dict(head_dict)
161 |
162 |
163 | def PIFuNetG():
164 | from yacs.config import CfgNode as CN
165 | opt_net = CN()
166 | opt_net.projection = "orthogonal"
167 |
168 | # --- netG:backbone options ---
169 | opt_net.backbone = CN()
170 | opt_net.backbone.IMF = 'PIFuHGFilters'
171 |
172 | # --- netG:normalizer options ---
173 | opt_net.normalizer = CN()
174 | opt_net.normalizer.IMF = 'PIFuNomalizer'
175 |
176 | # --- netG:head options ---
177 | opt_net.head = CN()
178 | opt_net.head.IMF = 'PIFuNetGMLP'
179 |
180 | # --- netG:loss options ---
181 | opt_net.loss = CN()
182 | opt_net.loss.IMF = 'MSE'
183 |
184 | return MonoPortNet(opt_net)
185 |
186 |
187 | def PIFuNetC():
188 | from yacs.config import CfgNode as CN
189 | opt_net = CN()
190 | opt_net.projection = "orthogonal"
191 |
192 | # --- netG:backbone options ---
193 | opt_net.backbone = CN()
194 | opt_net.backbone.IMF = 'PIFuResBlkFilters'
195 |
196 | # --- netG:normalizer options ---
197 | opt_net.normalizer = CN()
198 | opt_net.normalizer.IMF = 'PIFuNomalizer'
199 |
200 | # --- netG:head options ---
201 | opt_net.head = CN()
202 | opt_net.head.IMF = 'PIFuNetCMLP'
203 |
204 | # --- netG:loss options ---
205 | opt_net.loss = CN()
206 | opt_net.loss.IMF = 'L1'
207 |
208 | return MonoPortNet(opt_net)
209 |
210 |
--------------------------------------------------------------------------------
/monoport/lib/modeling/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/monoport/lib/modeling/__init__.py
--------------------------------------------------------------------------------
/monoport/lib/modeling/backbones/HGFilters.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 |
5 |
6 | def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False):
7 | "3x3 convolution with padding"
8 | return nn.Conv2d(in_planes, out_planes, kernel_size=3,
9 | stride=strd, padding=padding, bias=bias)
10 |
11 |
12 | class ConvBlock(nn.Module):
13 | def __init__(self, in_planes, out_planes, norm='batch'):
14 | super(ConvBlock, self).__init__()
15 | self.conv1 = conv3x3(in_planes, int(out_planes / 2))
16 | self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4))
17 | self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4))
18 |
19 | if norm == 'batch':
20 | self.bn1 = nn.BatchNorm2d(in_planes)
21 | self.bn2 = nn.BatchNorm2d(int(out_planes / 2))
22 | self.bn3 = nn.BatchNorm2d(int(out_planes / 4))
23 | self.bn4 = nn.BatchNorm2d(in_planes)
24 | elif norm == 'group':
25 | self.bn1 = nn.GroupNorm(32, in_planes)
26 | self.bn2 = nn.GroupNorm(32, int(out_planes / 2))
27 | self.bn3 = nn.GroupNorm(32, int(out_planes / 4))
28 | self.bn4 = nn.GroupNorm(32, in_planes)
29 |
30 | if in_planes != out_planes:
31 | self.downsample = nn.Sequential(
32 | self.bn4,
33 | nn.ReLU(True),
34 | nn.Conv2d(in_planes, out_planes,
35 | kernel_size=1, stride=1, bias=False),
36 | )
37 | else:
38 | self.downsample = None
39 |
40 | def forward(self, x):
41 | residual = x
42 |
43 | out1 = self.bn1(x)
44 | out1 = F.relu(out1, True)
45 | out1 = self.conv1(out1)
46 |
47 | out2 = self.bn2(out1)
48 | out2 = F.relu(out2, True)
49 | out2 = self.conv2(out2)
50 |
51 | out3 = self.bn3(out2)
52 | out3 = F.relu(out3, True)
53 | out3 = self.conv3(out3)
54 |
55 | out3 = torch.cat((out1, out2, out3), 1)
56 |
57 | if self.downsample is not None:
58 | residual = self.downsample(residual)
59 |
60 | out3 += residual
61 |
62 | return out3
63 |
64 |
65 | class HourGlass(nn.Module):
66 | def __init__(self, num_modules, depth, num_features, norm='batch'):
67 | super(HourGlass, self).__init__()
68 | self.num_modules = num_modules
69 | self.depth = depth
70 | self.features = num_features
71 | self.norm = norm
72 |
73 | self._generate_network(self.depth)
74 |
75 | def _generate_network(self, level):
76 | self.add_module('b1_' + str(level), ConvBlock(self.features, self.features, norm=self.norm))
77 |
78 | self.add_module('b2_' + str(level), ConvBlock(self.features, self.features, norm=self.norm))
79 |
80 | if level > 1:
81 | self._generate_network(level - 1)
82 | else:
83 | self.add_module('b2_plus_' + str(level), ConvBlock(self.features, self.features, norm=self.norm))
84 |
85 | self.add_module('b3_' + str(level), ConvBlock(self.features, self.features, norm=self.norm))
86 |
87 | def _forward(self, level, inp):
88 | # Upper branch
89 | up1 = inp
90 | up1 = self._modules['b1_' + str(level)](up1)
91 |
92 | # Lower branch
93 | low1 = F.avg_pool2d(inp, 2, stride=2)
94 | low1 = self._modules['b2_' + str(level)](low1)
95 |
96 | if level > 1:
97 | low2 = self._forward(level - 1, low1)
98 | else:
99 | low2 = low1
100 | low2 = self._modules['b2_plus_' + str(level)](low2)
101 |
102 | low3 = low2
103 | low3 = self._modules['b3_' + str(level)](low3)
104 |
105 | # NOTE: for newer PyTorch (1.3~), it seems that training results are degraded due to implementation diff in F.grid_sample
106 | # if the pretrained model behaves weirdly, switch with the commented line.
107 | # NOTE: I also found that "bicubic" works better.
108 | up2 = F.interpolate(low3, scale_factor=2, mode='bicubic', align_corners=True)
109 | # up2 = F.interpolate(low3, scale_factor=2, mode='nearest)
110 |
111 | return up1 + up2
112 |
113 | def forward(self, x):
114 | return self._forward(self.depth, x)
115 |
116 |
117 | class HGFilter(nn.Module):
118 | def __init__(self, opt):
119 | super(HGFilter, self).__init__()
120 | self.num_modules = opt.num_stack
121 |
122 | self.opt = opt
123 |
124 | # Base part
125 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)
126 |
127 | if self.opt.norm == 'batch':
128 | self.bn1 = nn.BatchNorm2d(64)
129 | elif self.opt.norm == 'group':
130 | self.bn1 = nn.GroupNorm(32, 64)
131 |
132 | if self.opt.hg_down == 'conv64':
133 | self.conv2 = ConvBlock(64, 64, self.opt.norm)
134 | self.down_conv2 = nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1)
135 | elif self.opt.hg_down == 'conv128':
136 | self.conv2 = ConvBlock(64, 128, self.opt.norm)
137 | self.down_conv2 = nn.Conv2d(128, 128, kernel_size=3, stride=2, padding=1)
138 | elif self.opt.hg_down == 'ave_pool':
139 | self.conv2 = ConvBlock(64, 128, self.opt.norm)
140 | else:
141 | raise NameError('Unknown Fan Filter setting!')
142 |
143 | self.conv3 = ConvBlock(128, 128, self.opt.norm)
144 | self.conv4 = ConvBlock(128, 256, self.opt.norm)
145 |
146 | # Stacking part
147 | for hg_module in range(self.num_modules):
148 | self.add_module('m' + str(hg_module), HourGlass(1, opt.num_hourglass, 256, self.opt.norm))
149 |
150 | self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256, self.opt.norm))
151 | self.add_module('conv_last' + str(hg_module),
152 | nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
153 | if self.opt.norm == 'batch':
154 | self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))
155 | elif self.opt.norm == 'group':
156 | self.add_module('bn_end' + str(hg_module), nn.GroupNorm(32, 256))
157 |
158 | self.add_module('l' + str(hg_module), nn.Conv2d(256,
159 | opt.hourglass_dim, kernel_size=1, stride=1, padding=0))
160 |
161 | if hg_module < self.num_modules - 1:
162 | self.add_module(
163 | 'bl' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
164 | self.add_module('al' + str(hg_module), nn.Conv2d(opt.hourglass_dim,
165 | 256, kernel_size=1, stride=1, padding=0))
166 |
167 | def forward(self, x):
168 | x = F.relu(self.bn1(self.conv1(x)), True)
169 | tmpx = x
170 | if self.opt.hg_down == 'ave_pool':
171 | x = F.avg_pool2d(self.conv2(x), 2, stride=2)
172 | elif self.opt.hg_down in ['conv64', 'conv128']:
173 | x = self.conv2(x)
174 | x = self.down_conv2(x)
175 | else:
176 | raise NameError('Unknown Fan Filter setting!')
177 |
178 | normx = x
179 |
180 | x = self.conv3(x)
181 | x = self.conv4(x)
182 |
183 | previous = x
184 |
185 | outputs = []
186 | for i in range(self.num_modules):
187 | hg = self._modules['m' + str(i)](previous)
188 |
189 | ll = hg
190 | ll = self._modules['top_m_' + str(i)](ll)
191 |
192 | ll = F.relu(self._modules['bn_end' + str(i)]
193 | (self._modules['conv_last' + str(i)](ll)), True)
194 |
195 | # Predict heatmaps
196 | tmp_out = self._modules['l' + str(i)](ll)
197 | outputs.append((tmp_out,))
198 |
199 | if i < self.num_modules - 1:
200 | ll = self._modules['bl' + str(i)](ll)
201 | tmp_out_ = self._modules['al' + str(i)](tmp_out)
202 | previous = previous + ll + tmp_out_
203 |
204 | return outputs
205 |
206 |
207 | def PIFuHGFilters(*args, **kwargs):
208 | from yacs.config import CfgNode as CN
209 | opt = CN()
210 | opt.norm = 'group'
211 | opt.num_stack = 4
212 | opt.num_hourglass = 2
213 | opt.skip_hourglass = False
214 | opt.hg_down = 'ave_pool'
215 | opt.hourglass_dim = 256
216 | return HGFilter(opt)
217 |
218 |
219 | if __name__ == '__main__':
220 | import tqdm
221 |
222 | device = 'cuda:0'
223 | input = torch.randn(1, 3, 512, 512).to(device)
224 | model = PIFuHGFilters().to(device)
225 | model.eval()
226 |
227 | with torch.no_grad():
228 | outputs = model(input)
229 | for stage, output_stage in enumerate(outputs):
230 | for lvl, output_lvl in enumerate(output_stage):
231 | print (f'stage: {stage}, lvl: {lvl}', output_lvl.shape)
232 |
233 | with torch.no_grad(): # 27.42 fps
234 | for _ in tqdm.tqdm(range(1000)):
235 | outputs = model(input)
--------------------------------------------------------------------------------
/monoport/lib/modeling/backbones/ResBlkFilters.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 | import functools
5 |
6 |
7 | def get_norm_layer(norm_type='group'):
8 | """Return a normalization layer
9 | Parameters:
10 | norm_type (str) -- the name of the normalization layer: batch | instance | none
11 | For BatchNorm, we use learnable affine parameters and track running statistics (mean/stddev).
12 | For InstanceNorm, we do not use learnable affine parameters. We do not track running statistics.
13 | """
14 | if norm_type == 'batch':
15 | norm_layer = functools.partial(nn.BatchNorm2d, affine=True, track_running_stats=True)
16 | elif norm_type == 'instance':
17 | norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False)
18 | elif norm_type == 'group':
19 | norm_layer = functools.partial(nn.GroupNorm, 32)
20 | elif norm_type == 'none':
21 | norm_layer = None
22 | else:
23 | raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
24 | return norm_layer
25 |
26 |
27 | class ResnetBlock(nn.Module):
28 | """Define a Resnet block"""
29 |
30 | def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias, last=False):
31 | """Initialize the Resnet block
32 | A resnet block is a conv block with skip connections
33 | We construct a conv block with build_conv_block function,
34 | and implement skip connections in function.
35 | Original Resnet paper: https://arxiv.org/pdf/1512.03385.pdf
36 | """
37 | super(ResnetBlock, self).__init__()
38 | self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, use_dropout, use_bias, last)
39 |
40 | def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, use_bias, last=False):
41 | """Construct a convolutional block.
42 | Parameters:
43 | dim (int) -- the number of channels in the conv layer.
44 | padding_type (str) -- the name of padding layer: reflect | replicate | zero
45 | norm_layer -- normalization layer
46 | use_dropout (bool) -- if use dropout layers.
47 | use_bias (bool) -- if the conv layer uses bias or not
48 | Returns a conv block (with a conv layer, a normalization layer, and a non-linearity layer (ReLU))
49 | """
50 | conv_block = []
51 | p = 0
52 | if padding_type == 'reflect':
53 | conv_block += [nn.ReflectionPad2d(1)]
54 | elif padding_type == 'replicate':
55 | conv_block += [nn.ReplicationPad2d(1)]
56 | elif padding_type == 'zero':
57 | p = 1
58 | else:
59 | raise NotImplementedError('padding [%s] is not implemented' % padding_type)
60 |
61 | conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim), nn.ReLU(True)]
62 | if use_dropout:
63 | conv_block += [nn.Dropout(0.5)]
64 |
65 | p = 0
66 | if padding_type == 'reflect':
67 | conv_block += [nn.ReflectionPad2d(1)]
68 | elif padding_type == 'replicate':
69 | conv_block += [nn.ReplicationPad2d(1)]
70 | elif padding_type == 'zero':
71 | p = 1
72 | else:
73 | raise NotImplementedError('padding [%s] is not implemented' % padding_type)
74 | if last:
75 | conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias)]
76 | else:
77 | conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim)]
78 |
79 | return nn.Sequential(*conv_block)
80 |
81 | def forward(self, x):
82 | """Forward function (with skip connections)"""
83 | out = x + self.conv_block(x) # add skip connections
84 | return out
85 |
86 |
87 | class ResnetFilter(nn.Module):
88 | """Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations.
89 | We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style)
90 | """
91 |
92 | def __init__(self, opt, input_nc=3, output_nc=256, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False,
93 | n_blocks=6, padding_type='reflect'):
94 | """Construct a Resnet-based generator
95 | Parameters:
96 | input_nc (int) -- the number of channels in input images
97 | output_nc (int) -- the number of channels in output images
98 | ngf (int) -- the number of filters in the last conv layer
99 | norm_layer -- normalization layer
100 | use_dropout (bool) -- if use dropout layers
101 | n_blocks (int) -- the number of ResNet blocks
102 | padding_type (str) -- the name of padding layer in conv layers: reflect | replicate | zero
103 | """
104 | assert (n_blocks >= 0)
105 | super(ResnetFilter, self).__init__()
106 | if type(norm_layer) == functools.partial:
107 | use_bias = norm_layer.func == nn.InstanceNorm2d
108 | else:
109 | use_bias = norm_layer == nn.InstanceNorm2d
110 |
111 | model = [nn.ReflectionPad2d(3),
112 | nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias),
113 | norm_layer(ngf),
114 | nn.ReLU(True)]
115 |
116 | n_downsampling = 2
117 | for i in range(n_downsampling): # add downsampling layers
118 | mult = 2 ** i
119 | model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias),
120 | norm_layer(ngf * mult * 2),
121 | nn.ReLU(True)]
122 |
123 | mult = 2 ** n_downsampling
124 | for i in range(n_blocks): # add ResNet blocks
125 | if i == n_blocks - 1:
126 | model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer,
127 | use_dropout=use_dropout, use_bias=use_bias, last=True)]
128 | else:
129 | model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer,
130 | use_dropout=use_dropout, use_bias=use_bias)]
131 |
132 | if opt.use_tanh:
133 | model += [nn.Tanh()]
134 | self.model = nn.Sequential(*model)
135 |
136 | def forward(self, input):
137 | """Standard forward"""
138 | output = self.model(input)
139 | return [(output,)]
140 |
141 |
142 | def PIFuResBlkFilters(*args, **kwargs):
143 | from yacs.config import CfgNode as CN
144 | opt = CN()
145 | opt.use_tanh = False
146 | norm_layer = get_norm_layer('group')
147 | return ResnetFilter(opt, norm_layer=norm_layer)
148 |
149 |
150 | if __name__ == '__main__':
151 | import tqdm
152 |
153 | device = 'cuda:0'
154 | input = torch.randn(1, 3, 512, 512).to(device)
155 | model = PIFuResBlkFilters().to(device)
156 |
157 | with torch.no_grad():
158 | outputs = model(input)
159 | for stage, output_stage in enumerate(outputs):
160 | for lvl, output_lvl in enumerate(output_stage):
161 | print (f'stage: {stage}, lvl: {lvl}', output_lvl.shape)
162 |
163 | with torch.no_grad(): # 39.68 fps
164 | for _ in tqdm.tqdm(range(1000)):
165 | outputs = model(input)
--------------------------------------------------------------------------------
/monoport/lib/modeling/backbones/Yolov4Filters.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 |
5 | class Mish(torch.nn.Module):
6 | def __init__(self):
7 | super().__init__()
8 |
9 | def forward(self, x):
10 | x = x * (torch.tanh(torch.nn.functional.softplus(x)))
11 | return x
12 |
13 |
14 | class Upsample(nn.Module):
15 | def __init__(self):
16 | super(Upsample, self).__init__()
17 |
18 | def forward(self, x, target_size, inference=False):
19 | assert (x.data.dim() == 4)
20 | _, _, tH, tW = target_size
21 |
22 | if inference:
23 | B = x.data.size(0)
24 | C = x.data.size(1)
25 | H = x.data.size(2)
26 | W = x.data.size(3)
27 |
28 | return x.view(B, C, H, 1, W, 1).expand(B, C, H, tH // H, W, tW // W).contiguous().view(B, C, tH, tW)
29 | else:
30 | return F.interpolate(x, size=(tH, tW), mode='nearest')
31 |
32 |
33 | class Conv_Bn_Activation(nn.Module):
34 | def __init__(self, in_channels, out_channels, kernel_size, stride, activation, bn=True, bias=False):
35 | super().__init__()
36 | pad = (kernel_size - 1) // 2
37 |
38 | self.conv = nn.ModuleList()
39 | if bias:
40 | self.conv.append(nn.Conv2d(in_channels, out_channels, kernel_size, stride, pad))
41 | else:
42 | self.conv.append(nn.Conv2d(in_channels, out_channels, kernel_size, stride, pad, bias=False))
43 | if bn:
44 | self.conv.append(nn.BatchNorm2d(out_channels))
45 | if activation == "mish":
46 | self.conv.append(Mish())
47 | elif activation == "relu":
48 | self.conv.append(nn.ReLU(inplace=True))
49 | elif activation == "leaky":
50 | self.conv.append(nn.LeakyReLU(0.1, inplace=True))
51 | elif activation == "linear":
52 | pass
53 | else:
54 | raise NotImplementedError
55 |
56 | def forward(self, x):
57 | for l in self.conv:
58 | x = l(x)
59 | return x
60 |
61 |
62 | class ResBlock(nn.Module):
63 | """
64 | Sequential residual blocks each of which consists of \
65 | two convolution layers.
66 | Args:
67 | ch (int): number of input and output channels.
68 | nblocks (int): number of residual blocks.
69 | shortcut (bool): if True, residual tensor addition is enabled.
70 | """
71 |
72 | def __init__(self, ch, nblocks=1, shortcut=True):
73 | super().__init__()
74 | self.shortcut = shortcut
75 | self.module_list = nn.ModuleList()
76 | for i in range(nblocks):
77 | resblock_one = nn.ModuleList()
78 | resblock_one.append(Conv_Bn_Activation(ch, ch, 1, 1, 'mish'))
79 | resblock_one.append(Conv_Bn_Activation(ch, ch, 3, 1, 'mish'))
80 | self.module_list.append(resblock_one)
81 |
82 | def forward(self, x):
83 | for module in self.module_list:
84 | h = x
85 | for res in module:
86 | h = res(h)
87 | x = x + h if self.shortcut else h
88 | return x
89 |
90 |
91 | class DownSample1(nn.Module):
92 | def __init__(self):
93 | super().__init__()
94 | self.conv1 = Conv_Bn_Activation(3, 32, 3, 1, 'mish')
95 |
96 | self.conv2 = Conv_Bn_Activation(32, 64, 3, 2, 'mish')
97 | self.conv3 = Conv_Bn_Activation(64, 64, 1, 1, 'mish')
98 | # [route]
99 | # layers = -2
100 | self.conv4 = Conv_Bn_Activation(64, 64, 1, 1, 'mish')
101 |
102 | self.conv5 = Conv_Bn_Activation(64, 32, 1, 1, 'mish')
103 | self.conv6 = Conv_Bn_Activation(32, 64, 3, 1, 'mish')
104 | # [shortcut]
105 | # from=-3
106 | # activation = linear
107 |
108 | self.conv7 = Conv_Bn_Activation(64, 64, 1, 1, 'mish')
109 | # [route]
110 | # layers = -1, -7
111 | self.conv8 = Conv_Bn_Activation(128, 64, 1, 1, 'mish')
112 |
113 | def forward(self, input):
114 | x1 = self.conv1(input)
115 | x2 = self.conv2(x1)
116 | x3 = self.conv3(x2)
117 | # route -2
118 | x4 = self.conv4(x2)
119 | x5 = self.conv5(x4)
120 | x6 = self.conv6(x5)
121 | # shortcut -3
122 | x6 = x6 + x4
123 |
124 | x7 = self.conv7(x6)
125 | # [route]
126 | # layers = -1, -7
127 | x7 = torch.cat([x7, x3], dim=1)
128 | x8 = self.conv8(x7)
129 | return x8
130 |
131 |
132 | class DownSample2(nn.Module):
133 | def __init__(self):
134 | super().__init__()
135 | self.conv1 = Conv_Bn_Activation(64, 128, 3, 2, 'mish')
136 | self.conv2 = Conv_Bn_Activation(128, 64, 1, 1, 'mish')
137 | # r -2
138 | self.conv3 = Conv_Bn_Activation(128, 64, 1, 1, 'mish')
139 |
140 | self.resblock = ResBlock(ch=64, nblocks=2)
141 |
142 | # s -3
143 | self.conv4 = Conv_Bn_Activation(64, 64, 1, 1, 'mish')
144 | # r -1 -10
145 | self.conv5 = Conv_Bn_Activation(128, 128, 1, 1, 'mish')
146 |
147 | def forward(self, input):
148 | x1 = self.conv1(input)
149 | x2 = self.conv2(x1)
150 | x3 = self.conv3(x1)
151 |
152 | r = self.resblock(x3)
153 | x4 = self.conv4(r)
154 |
155 | x4 = torch.cat([x4, x2], dim=1)
156 | x5 = self.conv5(x4)
157 | return x5
158 |
159 |
160 | class DownSample3(nn.Module):
161 | def __init__(self):
162 | super().__init__()
163 | self.conv1 = Conv_Bn_Activation(128, 256, 3, 2, 'mish')
164 | self.conv2 = Conv_Bn_Activation(256, 128, 1, 1, 'mish')
165 | self.conv3 = Conv_Bn_Activation(256, 128, 1, 1, 'mish')
166 |
167 | self.resblock = ResBlock(ch=128, nblocks=8)
168 | self.conv4 = Conv_Bn_Activation(128, 128, 1, 1, 'mish')
169 | self.conv5 = Conv_Bn_Activation(256, 256, 1, 1, 'mish')
170 |
171 | def forward(self, input):
172 | x1 = self.conv1(input)
173 | x2 = self.conv2(x1)
174 | x3 = self.conv3(x1)
175 |
176 | r = self.resblock(x3)
177 | x4 = self.conv4(r)
178 |
179 | x4 = torch.cat([x4, x2], dim=1)
180 | x5 = self.conv5(x4)
181 | return x5
182 |
183 |
184 | class DownSample4(nn.Module):
185 | def __init__(self):
186 | super().__init__()
187 | self.conv1 = Conv_Bn_Activation(256, 512, 3, 2, 'mish')
188 | self.conv2 = Conv_Bn_Activation(512, 256, 1, 1, 'mish')
189 | self.conv3 = Conv_Bn_Activation(512, 256, 1, 1, 'mish')
190 |
191 | self.resblock = ResBlock(ch=256, nblocks=8)
192 | self.conv4 = Conv_Bn_Activation(256, 256, 1, 1, 'mish')
193 | self.conv5 = Conv_Bn_Activation(512, 512, 1, 1, 'mish')
194 |
195 | def forward(self, input):
196 | x1 = self.conv1(input)
197 | x2 = self.conv2(x1)
198 | x3 = self.conv3(x1)
199 |
200 | r = self.resblock(x3)
201 | x4 = self.conv4(r)
202 |
203 | x4 = torch.cat([x4, x2], dim=1)
204 | x5 = self.conv5(x4)
205 | return x5
206 |
207 |
208 | class DownSample5(nn.Module):
209 | def __init__(self):
210 | super().__init__()
211 | self.conv1 = Conv_Bn_Activation(512, 1024, 3, 2, 'mish')
212 | self.conv2 = Conv_Bn_Activation(1024, 512, 1, 1, 'mish')
213 | self.conv3 = Conv_Bn_Activation(1024, 512, 1, 1, 'mish')
214 |
215 | self.resblock = ResBlock(ch=512, nblocks=4)
216 | self.conv4 = Conv_Bn_Activation(512, 512, 1, 1, 'mish')
217 | self.conv5 = Conv_Bn_Activation(1024, 1024, 1, 1, 'mish')
218 |
219 | def forward(self, input):
220 | x1 = self.conv1(input)
221 | x2 = self.conv2(x1)
222 | x3 = self.conv3(x1)
223 |
224 | r = self.resblock(x3)
225 | x4 = self.conv4(r)
226 |
227 | x4 = torch.cat([x4, x2], dim=1)
228 | x5 = self.conv5(x4)
229 | return x5
230 |
231 |
232 | class Neck(nn.Module):
233 | def __init__(self, inference=False):
234 | super().__init__()
235 | self.inference = inference
236 |
237 | self.conv1 = Conv_Bn_Activation(1024, 512, 1, 1, 'leaky')
238 | self.conv2 = Conv_Bn_Activation(512, 1024, 3, 1, 'leaky')
239 | self.conv3 = Conv_Bn_Activation(1024, 512, 1, 1, 'leaky')
240 | # SPP
241 | self.maxpool1 = nn.MaxPool2d(kernel_size=5, stride=1, padding=5 // 2)
242 | self.maxpool2 = nn.MaxPool2d(kernel_size=9, stride=1, padding=9 // 2)
243 | self.maxpool3 = nn.MaxPool2d(kernel_size=13, stride=1, padding=13 // 2)
244 |
245 | # R -1 -3 -5 -6
246 | # SPP
247 | self.conv4 = Conv_Bn_Activation(2048, 512, 1, 1, 'leaky')
248 | self.conv5 = Conv_Bn_Activation(512, 1024, 3, 1, 'leaky')
249 | self.conv6 = Conv_Bn_Activation(1024, 512, 1, 1, 'leaky')
250 | self.conv7 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
251 | # UP
252 | self.upsample1 = Upsample()
253 | # R 85
254 | self.conv8 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
255 | # R -1 -3
256 | self.conv9 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
257 | self.conv10 = Conv_Bn_Activation(256, 512, 3, 1, 'leaky')
258 | self.conv11 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
259 | self.conv12 = Conv_Bn_Activation(256, 512, 3, 1, 'leaky')
260 | self.conv13 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
261 | self.conv14 = Conv_Bn_Activation(256, 128, 1, 1, 'leaky')
262 | # UP
263 | self.upsample2 = Upsample()
264 | # R 54
265 | self.conv15 = Conv_Bn_Activation(256, 128, 1, 1, 'leaky')
266 | # R -1 -3
267 | self.conv16 = Conv_Bn_Activation(256, 128, 1, 1, 'leaky')
268 | self.conv17 = Conv_Bn_Activation(128, 256, 3, 1, 'leaky')
269 | self.conv18 = Conv_Bn_Activation(256, 128, 1, 1, 'leaky')
270 | self.conv19 = Conv_Bn_Activation(128, 256, 3, 1, 'leaky')
271 | self.conv20 = Conv_Bn_Activation(256, 128, 1, 1, 'leaky')
272 |
273 | def forward(self, input, downsample4, downsample3, inference=False):
274 | x1 = self.conv1(input)
275 | x2 = self.conv2(x1)
276 | x3 = self.conv3(x2)
277 | # SPP
278 | m1 = self.maxpool1(x3)
279 | m2 = self.maxpool2(x3)
280 | m3 = self.maxpool3(x3)
281 | spp = torch.cat([m3, m2, m1, x3], dim=1)
282 | # SPP end
283 | x4 = self.conv4(spp)
284 | x5 = self.conv5(x4)
285 | x6 = self.conv6(x5)
286 | x7 = self.conv7(x6)
287 | # UP
288 | up = self.upsample1(x7, downsample4.size(), self.inference)
289 | # R 85
290 | x8 = self.conv8(downsample4)
291 | # R -1 -3
292 | x8 = torch.cat([x8, up], dim=1)
293 |
294 | x9 = self.conv9(x8)
295 | x10 = self.conv10(x9)
296 | x11 = self.conv11(x10)
297 | x12 = self.conv12(x11)
298 | x13 = self.conv13(x12)
299 | x14 = self.conv14(x13)
300 |
301 | # UP
302 | up = self.upsample2(x14, downsample3.size(), self.inference)
303 | # R 54
304 | x15 = self.conv15(downsample3)
305 | # R -1 -3
306 | x15 = torch.cat([x15, up], dim=1)
307 |
308 | x16 = self.conv16(x15)
309 | x17 = self.conv17(x16)
310 | x18 = self.conv18(x17)
311 | x19 = self.conv19(x18)
312 | x20 = self.conv20(x19)
313 | return x20, x13, x6
314 |
315 |
316 | class Yolov4Head(nn.Module):
317 | def __init__(self, output_ch, inference=False):
318 | super().__init__()
319 | self.inference = inference
320 |
321 | self.conv1 = Conv_Bn_Activation(128, 256, 3, 1, 'leaky')
322 | self.conv2 = Conv_Bn_Activation(256, output_ch, 1, 1, 'linear', bn=False, bias=True)
323 |
324 | # R -4
325 | self.conv3 = Conv_Bn_Activation(128, 256, 3, 2, 'leaky')
326 |
327 | # R -1 -16
328 | self.conv4 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
329 | self.conv5 = Conv_Bn_Activation(256, 512, 3, 1, 'leaky')
330 | self.conv6 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
331 | self.conv7 = Conv_Bn_Activation(256, 512, 3, 1, 'leaky')
332 | self.conv8 = Conv_Bn_Activation(512, 256, 1, 1, 'leaky')
333 | self.conv9 = Conv_Bn_Activation(256, 512, 3, 1, 'leaky')
334 | self.conv10 = Conv_Bn_Activation(512, output_ch, 1, 1, 'linear', bn=False, bias=True)
335 |
336 | # R -4
337 | self.conv11 = Conv_Bn_Activation(256, 512, 3, 2, 'leaky')
338 |
339 | # R -1 -37
340 | self.conv12 = Conv_Bn_Activation(1024, 512, 1, 1, 'leaky')
341 | self.conv13 = Conv_Bn_Activation(512, 1024, 3, 1, 'leaky')
342 | self.conv14 = Conv_Bn_Activation(1024, 512, 1, 1, 'leaky')
343 | self.conv15 = Conv_Bn_Activation(512, 1024, 3, 1, 'leaky')
344 | self.conv16 = Conv_Bn_Activation(1024, 512, 1, 1, 'leaky')
345 | self.conv17 = Conv_Bn_Activation(512, 1024, 3, 1, 'leaky')
346 | self.conv18 = Conv_Bn_Activation(1024, output_ch, 1, 1, 'linear', bn=False, bias=True)
347 |
348 | def forward(self, input1, input2, input3):
349 | x1 = self.conv1(input1)
350 | x2 = self.conv2(x1)
351 |
352 | x3 = self.conv3(input1)
353 | # R -1 -16
354 | x3 = torch.cat([x3, input2], dim=1)
355 | x4 = self.conv4(x3)
356 | x5 = self.conv5(x4)
357 | x6 = self.conv6(x5)
358 | x7 = self.conv7(x6)
359 | x8 = self.conv8(x7)
360 | x9 = self.conv9(x8)
361 | x10 = self.conv10(x9)
362 |
363 | # R -4
364 | x11 = self.conv11(x8)
365 | # R -1 -37
366 | x11 = torch.cat([x11, input3], dim=1)
367 |
368 | x12 = self.conv12(x11)
369 | x13 = self.conv13(x12)
370 | x14 = self.conv14(x13)
371 | x15 = self.conv15(x14)
372 | x16 = self.conv16(x15)
373 | x17 = self.conv17(x16)
374 | x18 = self.conv18(x17)
375 |
376 | return [x2, x10, x18]
377 |
378 |
379 | class Yolov4(nn.Module):
380 | def __init__(self, yolov4conv137weight=None, output_ch=256, inference=False):
381 | super().__init__()
382 |
383 | # backbone
384 | self.down1 = DownSample1()
385 | self.down2 = DownSample2()
386 | self.down3 = DownSample3()
387 | self.down4 = DownSample4()
388 | self.down5 = DownSample5()
389 | # neck
390 | self.neek = Neck(inference)
391 | # yolov4conv137
392 | if yolov4conv137weight:
393 | _model = nn.Sequential(self.down1, self.down2, self.down3, self.down4, self.down5, self.neek)
394 | pretrained_dict = torch.load(yolov4conv137weight)
395 |
396 | model_dict = _model.state_dict()
397 | # 1. filter out unnecessary keys
398 | pretrained_dict = {k1: v for (k, v), k1 in zip(pretrained_dict.items(), model_dict)}
399 | # 2. overwrite entries in the existing state dict
400 | model_dict.update(pretrained_dict)
401 | _model.load_state_dict(model_dict)
402 |
403 | self.head = Yolov4Head(output_ch, inference)
404 |
405 | def forward(self, input):
406 | d1 = self.down1(input)
407 | d2 = self.down2(d1)
408 | d3 = self.down3(d2)
409 | d4 = self.down4(d3)
410 | d5 = self.down5(d4)
411 |
412 | x20, x13, x6 = self.neek(d5, d4, d3)
413 |
414 | out = self.head(x20, x13, x6)
415 | # return [(out[2],), (out[1],), (out[0],)]
416 | return [(out[0],), (out[1],), (out[2],)]
417 |
418 |
419 | def Yolov4Filters(*args, **kwargs):
420 | ckpt_path = '/media/linux_data/projects/MonoPort-7.9/data/Yolov4/yolov4.conv.137.pth'
421 | return Yolov4(
422 | yolov4conv137weight=ckpt_path,
423 | output_ch=256,
424 | inference=True)
425 |
426 |
427 | if __name__ == '__main__':
428 | import tqdm
429 |
430 | device = 'cuda:0'
431 | input = torch.randn(1, 3, 512, 512).to(device)
432 | model = Yolov4Filters().to(device)
433 | model.eval()
434 |
435 | with torch.no_grad():
436 | outputs = model(input)
437 | for stage, output_stage in enumerate(outputs):
438 | for lvl, output_lvl in enumerate(output_stage):
439 | print (f'stage: {stage}, lvl: {lvl}', output_lvl.shape)
440 |
441 | with torch.no_grad(): # 52.77 fps
442 | for _ in tqdm.tqdm(range(1000)):
443 | outputs = model(input)
--------------------------------------------------------------------------------
/monoport/lib/modeling/backbones/__init__.py:
--------------------------------------------------------------------------------
1 | from .HGFilters import HGFilter, PIFuHGFilters
2 | from .ResBlkFilters import ResnetFilter, PIFuResBlkFilters
3 | from .Yolov4Filters import Yolov4Filters
4 | from .HRNetFilters import HRNetV2_W18_small_v2_balance_last
--------------------------------------------------------------------------------
/monoport/lib/modeling/geometry.py:
--------------------------------------------------------------------------------
1 | import torch
2 |
3 |
4 | def index(feat, uv):
5 | '''
6 |
7 | :param feat: [B, C, H, W] image features
8 | :param uv: [B, 2, N] uv coordinates in the image plane, range [0, 1]
9 | :return: [B, C, N] image features at the uv coordinates
10 | '''
11 | uv = uv.transpose(1, 2) # [B, N, 2]
12 | uv = uv.unsqueeze(2) # [B, N, 1, 2]
13 | # NOTE: for newer PyTorch, it seems that training results are degraded due to implementation diff in F.grid_sample
14 | # for old versions, simply remove the aligned_corners argument.
15 | samples = torch.nn.functional.grid_sample(feat, uv, align_corners=True) # [B, C, N, 1]
16 | return samples[:, :, :, 0] # [B, C, N]
17 |
18 |
19 | def orthogonal(points, calibrations, transforms=None):
20 | '''
21 | Compute the orthogonal projections of 3D points into the image plane by given projection matrix
22 | :param points: [B, 3, N] Tensor of 3D points
23 | :param calibrations: [B, 3, 4] Tensor of projection matrix
24 | :param transforms: [B, 2, 3] Tensor of image transform matrix
25 | :return: xyz: [B, 3, N] Tensor of xyz coordinates in the image plane
26 | '''
27 | rot = calibrations[:, :3, :3]
28 | trans = calibrations[:, :3, 3:4]
29 | pts = torch.baddbmm(trans, rot, points) # [B, 3, N]
30 | if transforms is not None:
31 | scale = transforms[:2, :2]
32 | shift = transforms[:2, 2:3]
33 | pts[:, :2, :] = torch.baddbmm(shift, scale, pts[:, :2, :])
34 | return pts
35 |
36 |
37 | def perspective(points, calibrations, transforms=None):
38 | '''
39 | Compute the perspective projections of 3D points into the image plane by given projection matrix
40 | :param points: [Bx3xN] Tensor of 3D points
41 | :param calibrations: [Bx3x4] Tensor of projection matrix
42 | :param transforms: [Bx2x3] Tensor of image transform matrix
43 | :return: xy: [Bx2xN] Tensor of xy coordinates in the image plane
44 | '''
45 | rot = calibrations[:, :3, :3]
46 | trans = calibrations[:, :3, 3:4]
47 | homo = torch.baddbmm(trans, rot, points) # [B, 3, N]
48 | xy = homo[:, :2, :] / homo[:, 2:3, :]
49 | if transforms is not None:
50 | scale = transforms[:2, :2]
51 | shift = transforms[:2, 2:3]
52 | xy = torch.baddbmm(shift, scale, xy)
53 |
54 | xyz = torch.cat([xy, homo[:, 2:3, :]], 1)
55 | return xyz
56 |
--------------------------------------------------------------------------------
/monoport/lib/modeling/heads/SurfaceClassifier.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 |
5 |
6 | class SurfaceClassifier(nn.Module):
7 | def __init__(self, filter_channels, num_views=1, no_residual=True, last_op=None):
8 | super(SurfaceClassifier, self).__init__()
9 |
10 | self.filters = nn.ModuleList()
11 | self.num_views = num_views
12 | self.no_residual = no_residual
13 | filter_channels = filter_channels
14 | self.last_op = last_op
15 |
16 | if self.no_residual:
17 | for l in range(0, len(filter_channels) - 1):
18 | self.filters.append(nn.Conv1d(
19 | filter_channels[l],
20 | filter_channels[l + 1],
21 | 1))
22 | # self.add_module("conv%d" % l, self.filters[l])
23 | else:
24 | for l in range(0, len(filter_channels) - 1):
25 | if 0 != l:
26 | self.filters.append(
27 | nn.Conv1d(
28 | filter_channels[l] + filter_channels[0],
29 | filter_channels[l + 1],
30 | 1))
31 | else:
32 | self.filters.append(nn.Conv1d(
33 | filter_channels[l],
34 | filter_channels[l + 1],
35 | 1))
36 |
37 | # self.add_module("conv%d" % l, self.filters[l])
38 |
39 | def forward(self, feature):
40 | '''
41 |
42 | :param feature: list of [BxC_inxHxW] tensors of image features
43 | :param xy: [Bx3xN] tensor of (x,y) coodinates in the image plane
44 | :return: [BxC_outxN] tensor of features extracted at the coordinates
45 | '''
46 |
47 | y = feature
48 | tmpy = feature
49 | for i, f in enumerate(self.filters):
50 | if self.no_residual:
51 | y = f(y)
52 | else:
53 | y = f(
54 | y if i == 0
55 | else torch.cat([y, tmpy], 1)
56 | )
57 | if i != len(self.filters) - 1:
58 | y = F.leaky_relu(y)
59 |
60 | if self.num_views > 1 and i == len(self.filters) // 2:
61 | y = y.view(
62 | -1, self.num_views, y.shape[1], y.shape[2]
63 | ).mean(dim=1)
64 | tmpy = feature.view(
65 | -1, self.num_views, feature.shape[1], feature.shape[2]
66 | ).mean(dim=1)
67 |
68 | if self.last_op:
69 | y = self.last_op(y)
70 |
71 | return y
72 |
73 |
74 | def PIFuNetGMLP(*args, **kwargs):
75 | num_views = 1
76 | filter_channels = [257, 1024, 512, 256, 128, 1]
77 | no_residual = False
78 | last_op = nn.Sigmoid()
79 | return SurfaceClassifier(filter_channels, num_views, no_residual, last_op)
80 |
81 |
82 | def PIFuNetCMLP(*args, **kwargs):
83 | num_views = 1
84 | filter_channels = [513, 1024, 512, 256, 128, 3]
85 | no_residual = False
86 | last_op = nn.Tanh()
87 | return SurfaceClassifier(filter_channels, num_views, no_residual, last_op)
88 |
89 |
90 | if __name__ == "__main__":
91 | import tqdm
92 |
93 | device = 'cuda:0'
94 | # netG
95 | input = torch.randn(1, 257, 50000).to(device)
96 | model = PIFuNetGMLP().to(device)
97 |
98 | with torch.no_grad():
99 | outputs = model(input)
100 | print (outputs.shape)
101 |
102 | with torch.no_grad(): # 38.13 fps
103 | for _ in tqdm.tqdm(range(1000)):
104 | outputs = model(input)
105 |
106 | # netC
107 | input = torch.randn(1, 513, 50000).to(device)
108 | model = PIFuNetCMLP().to(device)
109 |
110 | with torch.no_grad():
111 | outputs = model(input)
112 | print (outputs.shape)
113 |
114 | with torch.no_grad(): # 23.71 fps
115 | for _ in tqdm.tqdm(range(1000)):
116 | outputs = model(input)
117 |
--------------------------------------------------------------------------------
/monoport/lib/modeling/heads/__init__.py:
--------------------------------------------------------------------------------
1 | from .SurfaceClassifier import SurfaceClassifier
2 | from .SurfaceClassifier import PIFuNetGMLP
3 | from .SurfaceClassifier import PIFuNetCMLP
4 |
--------------------------------------------------------------------------------
/monoport/lib/modeling/normalizers/DepthNormalizer.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 |
5 |
6 | class DepthNormalizer(nn.Module):
7 | def __init__(self, opt):
8 | super(DepthNormalizer, self).__init__()
9 | self.opt = opt
10 |
11 | def forward(self, z, calibs=None, index_feat=None):
12 | '''
13 | Normalize z_feature
14 | :param z_feat: [B, 1, N] depth value for z in the image coordinate system
15 | :return:
16 | '''
17 | if self.opt.soft_onehot:
18 | soft_dim = self.opt.soft_dim
19 |
20 | # [-1, +1] -> [0, soft_dim-1]
21 | z_feat = torch.zeros(z.size(0), soft_dim, z.size(2)).to(z.device) # [1, 64, 10000]
22 | z_norm = (z.clamp(-1, 1) + 1) / 2.0 * (soft_dim - 1)
23 | z_floor = torch.floor(z_norm) #[1, 1, 10000]
24 | z_ceil = torch.ceil(z_norm)
25 |
26 | z_floor_value = 1 - (z_norm - z_floor) #[1, 1, 10000]
27 | z_ceil_value = 1 - (z_ceil - z_norm)
28 |
29 | z_feat = z_feat.scatter(dim=1, index=z_floor.long(), src=z_floor_value)
30 | z_feat = z_feat.scatter(dim=1, index=z_ceil.long(), src=z_ceil_value)
31 | else:
32 | z_feat = z * self.opt.scale
33 | return z_feat
34 |
35 |
36 | def PIFuNomalizer(*args, **kwargs):
37 | from yacs.config import CfgNode as CN
38 | opt = CN()
39 | opt.soft_onehot = False
40 | opt.scale = 512 // 2 / 200.0
41 | return DepthNormalizer(opt)
--------------------------------------------------------------------------------
/monoport/lib/modeling/normalizers/__init__.py:
--------------------------------------------------------------------------------
1 | from .DepthNormalizer import DepthNormalizer
2 | from .DepthNormalizer import PIFuNomalizer
3 |
--------------------------------------------------------------------------------
/monoport/lib/render/BaseCamera.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 |
4 | class BaseCamera:
5 | def __init__(self, name="BaseCamera"):
6 | self.name = name
7 | #
8 | # The 'magnification' property tells effectively,
9 | # if there is a ruler at one unit away from the camera,
10 | # how much length will the ruler appear in the image the camera will capture.
11 | # This property is useful and transparent for both Perspective and Orthogonal cases.
12 | #
13 | # For instance,
14 | # if the world unit is in meter and magnification_x = 1.8,
15 | # the camera is set at 0.9m height and looking front,
16 | # a 1.8m tall man standing 1m away from the camera will be exactly in the camera view.
17 | #
18 |
19 | # How wide the camera can see from left to right at one unit away
20 | self.magnification_x = 1
21 | # How tall the camera can see from bottom to top at one unit away
22 | self.magnification_y = 1
23 |
24 | # Ratio of camera width to height, e.g. 1:1, 4:3, 16:9.
25 | self.aspect_ratio = 1
26 |
27 | # Close up clamping distance in world unit
28 | self.near = 0.01
29 | # Farthest clamping distance in world unit
30 | self.far = 10000
31 |
32 | def get_name(self):
33 | return self.name
34 |
35 | def set_parameters(self, magnification_x, magnification_y=None):
36 | '''
37 |
38 | :param magnification_x:
39 | :param magnification_y: Optional for y direction; Use the same value as for x direction if None
40 | '''
41 | if magnification_y is None:
42 | magnification_y = magnification_x / self.aspect_ratio
43 |
44 | self.magnification_x = magnification_x
45 | self.magnification_y = magnification_y
46 |
47 | def get_projection_mat(self):
48 | # http://www.songho.ca/opengl/gl_projectionmatrix.html
49 | projection_mat = np.eye(4)
50 | projection_mat[0, 0] = 2 / self.magnification_x
51 | projection_mat[1, 1] = 2 / self.magnification_y
52 | projection_mat[2, 2] = -2 / (self.far - self.near)
53 | projection_mat[2, 3] = -(self.far + self.near) / (self.far - self.near)
54 | return projection_mat
55 |
--------------------------------------------------------------------------------
/monoport/lib/render/CameraPose.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | class CameraPose:
4 | def __init__(self):
5 | # Camera's center in world coordinate
6 | self.center = np.array([0.0, 0.0, 1.0])
7 | # Camera's Z axis direction in world coordinate
8 | self.front = np.array([0.0, 0.0, 1.0])
9 | # Camera's X axis direction in world coordinate
10 | self.right = np.array([1.0, 0.0, 0.0])
11 | # Camera's Y axis direction in world coordinate
12 | self.up = np.array([0.0, 1.0, 0.0])
13 |
14 | self.sanity_check()
15 |
16 | def sanity_check(self):
17 | self.center = np.array(self.center).reshape([-1])
18 |
19 | self.front = np.array(self.front).reshape([-1])
20 | self.front = self.normalize_vector(self.front)
21 |
22 | self.up = np.array(self.up).reshape([-1])
23 | self.right = np.cross(self.up, self.front)
24 | self.right = self.normalize_vector(self.right)
25 |
26 | self.up = np.cross(self.front, self.right)
27 | self.up = self.normalize_vector(self.up)
28 |
29 | assert len(self.center) == 3
30 | assert len(self.front) == 3
31 | assert len(self.right) == 3
32 | assert len(self.up) == 3
33 |
34 | @staticmethod
35 | def normalize_vector(v):
36 | v_norm = np.linalg.norm(v)
37 | return v if v_norm == 0 else v / v_norm
38 |
39 | def get_rotation_matrix(self):
40 | rot_mat = np.eye(3)
41 | rot_mat[0, :] = self.right
42 | rot_mat[1, :] = self.up
43 | rot_mat[2, :] = self.front
44 | return rot_mat
45 |
46 | def get_translation_vector(self):
47 | rot_mat = self.get_rotation_matrix()
48 | trans = -np.dot(rot_mat, self.center)
49 | return trans
50 |
51 | def get_model_view_mat(self):
52 | model_view = np.eye(4)
53 | model_view[:3, :3] = self.get_rotation_matrix()
54 | model_view[:3, 3] = self.get_translation_vector()
55 | return model_view
--------------------------------------------------------------------------------
/monoport/lib/render/PespectiveCamera.py:
--------------------------------------------------------------------------------
1 | from .BaseCamera import BaseCamera
2 | import numpy as np
3 | import math
4 |
5 |
6 | class PersPectiveCamera(BaseCamera):
7 | def __init__(self):
8 | BaseCamera.__init__(self, "PerspectiveCamera")
9 |
10 | def get_projection_mat(self):
11 | # http://www.songho.ca/opengl/gl_projectionmatrix.html
12 | projection_mat = np.eye(4)
13 | projection_mat[0, 0] = 2 / self.magnification_x
14 | projection_mat[1, 1] = 2 / self.magnification_y
15 | projection_mat[2, 2] = -(self.far + self.near) / (self.far - self.near)
16 | projection_mat[2, 3] = -(2 * self.far * self.near) / (self.far - self.near)
17 | projection_mat[3, 2] = -1
18 | projection_mat[3, 3] = 0
19 | return projection_mat
20 |
21 | def set_by_field_of_view(self, fov_x, fov_y=None):
22 | '''
23 | Set the intrinsic by given field of view, in angle degrees
24 | :param fov_x:
25 | :param fov_y: Optional for y direction; Use the same value as for x direction if None
26 | '''
27 | if fov_y is None:
28 | fov_y = fov_x
29 | self.set_parameters(
30 | magnification_x=2 * math.tan(fov_x / 2),
31 | magnification_y=2 * math.tan(fov_y / 2),
32 | )
33 |
34 | def set_by_35mm_equivalent_focal_length(self, focal_x, focal_y=None):
35 | '''
36 | Set the intrinsic by given 35mm equivalent focal lengths.
37 | https://en.wikipedia.org/wiki/35_mm_equivalent_focal_length
38 | :param focal_x:
39 | :param focal_y: Optional for y direction; Use the same value as for x direction if None
40 | '''
41 | if focal_y is None:
42 | focal_y = focal_x
43 | # 35mm equivalent sensor width and height for this camera
44 | film_35mm_height = math.sqrt((36 ** 2 + 24 ** 2) / (1 + self.aspect_ratio ** 2))
45 | film_35mm_width = film_35mm_height * self.aspect_ratio
46 |
47 | self.set_parameters(
48 | magnification_x=film_35mm_width / focal_x,
49 | magnification_y=film_35mm_height / focal_y
50 | )
51 |
52 | def set_by_sensor_and_focal_length(self, sensor_width, sensor_height, focal_x, focal_y=None):
53 | self.aspect_ratio = sensor_width / sensor_height
54 | if focal_y is None:
55 | focal_y = focal_x
56 | # 35mm equivalent sensor width and height for this camera
57 | self.set_parameters(
58 | magnification_x=sensor_width / focal_x,
59 | magnification_y=sensor_height / focal_y
60 | )
61 |
--------------------------------------------------------------------------------
/monoport/lib/render/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Project-Splinter/MonoPort/65551f6422552475b481d295e7a97a5dcec955b3/monoport/lib/render/__init__.py
--------------------------------------------------------------------------------
/monoport/lib/render/gl/AlbedoRender.py:
--------------------------------------------------------------------------------
1 | from .Render import *
2 |
3 |
4 | class AlbedoRender(Render):
5 | def _init_shader(self):
6 | self.shader = Shader(vs_file='albedo.vs', fs_file='albedo.fs', gs_file=None)
7 | # layout (location = 0) in vec3 a_Position;
8 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
9 | # layout (location = 1) in vec2 a_TextureCoord;
10 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
11 |
12 | # Declare all uniform used in the program
13 | self.shader.declare_uniform('ModelMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
14 | self.shader.declare_uniform('PerspMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
15 |
16 | # Declare all textures used in the program
17 | self.texture_dict["TargetTexture"] = Texture()
18 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/GLObject.py:
--------------------------------------------------------------------------------
1 | import OpenGL.GL as gl
2 |
3 |
4 | class GLObject(object):
5 | def __del__(self):
6 | self.release()
7 |
8 | def __enter__(self):
9 | bind_func, const = self._bind
10 | bind_func(const, self)
11 |
12 | def __exit__(self, *args):
13 | bind_func, const = self._bind
14 | bind_func(const, 0)
15 |
16 |
17 | class FBO(GLObject):
18 | _bind = gl.glBindFramebuffer, gl.GL_FRAMEBUFFER
19 |
20 | def __init__(self):
21 | self._as_parameter_ = gl.glGenFramebuffers(1)
22 |
23 | def release(self):
24 | try:
25 | if self._as_parameter_ > 0:
26 | gl.glDeleteFramebuffers(1, [self._as_parameter_])
27 | except Exception:
28 | pass
29 |
30 |
31 | class Texture(GLObject):
32 | _bind = gl.glBindTexture, gl.GL_TEXTURE_2D
33 |
34 | def __init__(self):
35 | self._as_parameter_ = gl.glGenTextures(1)
36 |
37 | def release(self):
38 | try:
39 | if self._as_parameter_ > 0:
40 | gl.glDeleteTextures([self._as_parameter_])
41 | except Exception:
42 | pass
43 |
44 |
45 | class MultiSampleTexture(GLObject):
46 | _bind = gl.glBindTexture, gl.GL_TEXTURE_2D_MULTISAMPLE
47 |
48 | def __init__(self):
49 | self._as_parameter_ = gl.glGenTextures(1)
50 |
51 | def release(self):
52 | try:
53 | if self._as_parameter_ > 0:
54 | gl.glDeleteTextures([self._as_parameter_])
55 | except Exception:
56 | pass
57 |
58 |
59 | class RBO(GLObject):
60 | _bind = gl.glBindRenderbuffer, gl.GL_RENDERBUFFER
61 |
62 | def __init__(self):
63 | self._as_parameter_ = gl.glGenRenderbuffers(1)
64 |
65 | def release(self):
66 | try:
67 | if self._as_parameter_ > 0:
68 | gl.glDeleteRenderbuffers(1, [self._as_parameter_])
69 | except Exception:
70 | pass
71 |
72 |
73 | class VBO(GLObject):
74 | _bind = gl.glBindBuffer, gl.GL_ARRAY_BUFFER
75 |
76 | def __init__(self, type_code, gl_type, dim=3, size=0):
77 | self._as_parameter_ = gl.glGenBuffers(1)
78 | self.type_code = type_code
79 | self.gl_type = gl_type
80 | self.dim = dim
81 | self.size = size
82 |
83 | def release(self):
84 | try:
85 | if self._as_parameter_ > 0:
86 | gl.glDeleteBuffers(1, [self._as_parameter_])
87 | except Exception:
88 | pass
89 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/NormalRender.py:
--------------------------------------------------------------------------------
1 | from .Render import *
2 |
3 |
4 | class NormalRender(Render):
5 | def _init_shader(self):
6 | self.shader = Shader(vs_file='normal.vs', fs_file='normal.fs', gs_file=None)
7 | # layout (location = 0) in vec3 Position;
8 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
9 | # layout (location = 1) in vec3 Normal;
10 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
11 |
12 | # Declare all uniform used in the program
13 | self.shader.declare_uniform('ModelMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
14 | self.shader.declare_uniform('PerspMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
15 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/PrtRender.py:
--------------------------------------------------------------------------------
1 | from .Render import *
2 |
3 |
4 | class PrtRender(Render):
5 | def __init__(self,
6 | width, height,
7 | multi_sample_rate=1
8 | ):
9 | Render.__init__(self, width, height, multi_sample_rate, num_render_target=8)
10 |
11 | def _init_shader(self):
12 | self.shader = Shader(vs_file='prt.vs', fs_file='prt.fs', gs_file=None)
13 |
14 | # Declare all vertex attributes used in the program
15 | # layout (location = 0) in vec3 a_Position;
16 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
17 | # layout (location = 1) in vec3 a_Normal;
18 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
19 | # layout (location = 2) in vec2 a_TextureCoord;
20 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
21 | # layout (location = 5) in vec3 a_PRT1;
22 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
23 | # layout (location = 6) in vec3 a_PRT2;
24 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
25 | # layout (location = 7) in vec3 a_PRT3;
26 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
27 |
28 | # Declare all uniforms used in the program
29 | self.shader.declare_uniform('ModelMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
30 | self.shader.declare_uniform('PerspMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
31 |
32 | self.shader.declare_uniform('SHCoeffs', type_code='f', gl_type=gl.glUniform3fv)
33 |
34 | self.shader.declare_uniform('UVMode', type_code='i', gl_type=gl.glUniform1i)
35 |
36 | # Declare all textures used in the program
37 | self.texture_dict["AlbedoMap"] = Texture()
38 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/Render.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from .Shader import *
3 |
4 |
5 | class Render(object):
6 | def __init__(self,
7 | width, height,
8 | multi_sample_rate=1,
9 | num_render_target=1
10 | ):
11 | self.width = width
12 | self.height = height
13 |
14 | self.vbo_list = []
15 | self.uniform_dict = {}
16 | self.texture_dict = {}
17 |
18 | # Configure frame buffer
19 | # This is an off screen frame buffer holds the rendering results.
20 | # During display, it is drawn onto the screen by a quad program.
21 | self.color_fbo = FBO()
22 | self.color_tex_list = []
23 |
24 | with self.color_fbo:
25 | for i in range(num_render_target):
26 | color_tex = Texture()
27 | self.color_tex_list.append(color_tex)
28 | with color_tex:
29 | gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGBA32F, width, height, 0,
30 | gl.GL_RGBA, gl.GL_FLOAT, None)
31 | gl.glFramebufferTexture2D(gl.GL_FRAMEBUFFER, gl.GL_COLOR_ATTACHMENT0 + i,
32 | gl.GL_TEXTURE_2D, color_tex, 0)
33 | gl.glViewport(0, 0, width, height)
34 | assert gl.glCheckFramebufferStatus(gl.GL_FRAMEBUFFER) == gl.GL_FRAMEBUFFER_COMPLETE
35 |
36 | # This is the actual frame buffer the shader program is rendered to.
37 | # For multi-sampling, it is different than the default fbo.
38 | self._shader_fbo = self.color_fbo
39 | self._shader_color_tex_list = self.color_tex_list
40 |
41 | # However, if multi-sampling is enabled, we need additional render target textures.
42 | if multi_sample_rate > 1:
43 | self._shader_fbo = FBO()
44 | self._shader_color_tex_list = []
45 | with self._shader_fbo:
46 | for i in range(num_render_target):
47 | color_tex = MultiSampleTexture()
48 | self._shader_color_tex_list.append(color_tex)
49 | with color_tex:
50 | gl.glTexImage2DMultisample(gl.GL_TEXTURE_2D_MULTISAMPLE, multi_sample_rate, gl.GL_RGBA32F,
51 | width, height, gl.GL_TRUE)
52 | gl.glFramebufferTexture2D(gl.GL_FRAMEBUFFER, gl.GL_COLOR_ATTACHMENT0 + i,
53 | gl.GL_TEXTURE_2D_MULTISAMPLE, color_tex, 0)
54 |
55 | # Configure depth buffer
56 | self._shader_depth_tex = RBO()
57 | with self._shader_fbo:
58 | with self._shader_depth_tex:
59 | gl.glRenderbufferStorageMultisample(gl.GL_RENDERBUFFER, multi_sample_rate,
60 | gl.GL_DEPTH24_STENCIL8, width, height)
61 | gl.glFramebufferRenderbuffer(gl.GL_FRAMEBUFFER, gl.GL_DEPTH_STENCIL_ATTACHMENT,
62 | gl.GL_RENDERBUFFER, self._shader_depth_tex)
63 |
64 | self._init_shader()
65 | for uniform in self.uniform_dict:
66 | self.uniform_dict[uniform]["handle"] = gl.glGetUniformLocation(self.shader, uniform)
67 |
68 | def _init_shader(self):
69 | self.shader = Shader(vs_file='simple.vs', fs_file='simple.fs', gs_file=None)
70 | # layout (location = 0) in vec3 Position;
71 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
72 |
73 | # Declare all uniform used in the program
74 | self.shader.declare_uniform('ModelMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
75 | self.shader.declare_uniform('PerspMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
76 |
77 | def set_attrib(self, attrib_id, data):
78 | if not 0 <= attrib_id < len(self.vbo_list):
79 | print("Error: Attrib index out if bound.")
80 | return
81 | vbo = self.vbo_list[attrib_id]
82 | with vbo:
83 | data = np.ascontiguousarray(data, vbo.type_code)
84 | vbo.dim = data.shape[-1]
85 | vbo.size = data.shape[0]
86 | gl.glBufferData(gl.GL_ARRAY_BUFFER, data, gl.GL_STATIC_DRAW)
87 |
88 | def set_texture(self, name, texture_image):
89 | if name not in self.texture_dict:
90 | print("Error: Unknown texture name.")
91 | return
92 | width = texture_image.shape[1]
93 | height = texture_image.shape[0]
94 | texture_image = np.flip(texture_image, 0)
95 | img_data = np.fromstring(texture_image.tostring(), np.uint8)
96 | tex = self.texture_dict[name]
97 | with tex:
98 | gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, width, height, 0, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, img_data)
99 | gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAX_LEVEL, 3)
100 | gl.glGenerateMipmap(gl.GL_TEXTURE_2D)
101 |
102 | def draw(self,
103 | uniform_dict,
104 | clear_color=[0, 0, 0, 0],
105 | ):
106 | with self._shader_fbo:
107 | # Clean up
108 | gl.glClearColor(*clear_color)
109 | gl.glClear(gl.GL_COLOR_BUFFER_BIT | gl.GL_DEPTH_BUFFER_BIT)
110 |
111 | with self.shader:
112 | # Setup shader uniforms
113 | for uniform_name in uniform_dict:
114 | self.shader.set_uniform(uniform_name, uniform_dict[uniform_name])
115 |
116 | # Setup up VertexAttrib
117 | for attrib_id in range(len(self.vbo_list)):
118 | vbo = self.vbo_list[attrib_id]
119 | with vbo:
120 | gl.glEnableVertexAttribArray(attrib_id)
121 | gl.glVertexAttribPointer(attrib_id, vbo.dim, vbo.gl_type, gl.GL_FALSE, 0, None)
122 |
123 | # Setup Textures
124 | for i, texture_name in enumerate(self.texture_dict):
125 | gl.glActiveTexture(gl.GL_TEXTURE0 + i)
126 | gl.glBindTexture(gl.GL_TEXTURE_2D, self.texture_dict[texture_name])
127 | gl.glUniform1i(gl.glGetUniformLocation(self.shader, texture_name), i)
128 |
129 | # Setup targets
130 | color_size = len(self.color_tex_list)
131 | attachments = [gl.GL_COLOR_ATTACHMENT0 + i for i in range(color_size)]
132 | gl.glDrawBuffers(color_size, attachments)
133 |
134 | gl.glDrawArrays(gl.GL_TRIANGLES, 0, self.vbo_list[0].size)
135 |
136 | for attrib_id in range(len(self.vbo_list)):
137 | gl.glDisableVertexAttribArray(attrib_id)
138 |
139 | # If render_fbo is not color_fbo, we need to copy data
140 | if self._shader_fbo != self.color_fbo:
141 | for i in range(len(self.color_tex_list)):
142 | gl.glBindFramebuffer(gl.GL_READ_FRAMEBUFFER, self._shader_fbo)
143 | gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + i)
144 | gl.glBindFramebuffer(gl.GL_DRAW_FRAMEBUFFER, self.color_fbo)
145 | gl.glDrawBuffer(gl.GL_COLOR_ATTACHMENT0 + i)
146 | gl.glBlitFramebuffer(0, 0, self.width, self.height, 0, 0, self.width, self.height,
147 | gl.GL_COLOR_BUFFER_BIT, gl.GL_NEAREST)
148 | gl.glBindFramebuffer(gl.GL_READ_FRAMEBUFFER, 0)
149 | gl.glBindFramebuffer(gl.GL_DRAW_FRAMEBUFFER, 0)
150 |
151 | def get_color(self, color_id=0):
152 | with self.color_fbo:
153 | gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + color_id)
154 | data = gl.glReadPixels(0, 0, self.width, self.height, gl.GL_RGBA, gl.GL_FLOAT, outputType=None)
155 | frame = data.reshape(self.height, self.width, -1)
156 | frame = frame[::-1] # vertical flip to match GL convention
157 | return frame
158 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/ShRender.py:
--------------------------------------------------------------------------------
1 | from .Render import *
2 |
3 |
4 | class ShRender(Render):
5 | def __init__(self,
6 | width, height,
7 | multi_sample_rate=1
8 | ):
9 | Render.__init__(self, width, height, multi_sample_rate, num_render_target=8)
10 |
11 | def _init_shader(self):
12 | self.shader = Shader(vs_file='sh.vs', fs_file='sh.fs', gs_file=None)
13 |
14 | # Declare all vertex attributes used in the program
15 | # layout (location = 0) in vec3 a_Position;
16 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
17 | # layout (location = 1) in vec3 a_Normal;
18 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
19 | # layout (location = 2) in vec2 a_TextureCoord;
20 | self.vbo_list.append(VBO(type_code='f', gl_type=gl.GL_FLOAT))
21 |
22 | # Declare all uniforms used in the program
23 | self.shader.declare_uniform('ModelMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
24 | self.shader.declare_uniform('PerspMat', type_code='f', gl_type=gl.glUniformMatrix4fv)
25 |
26 | self.shader.declare_uniform('SHCoeffs', type_code='f', gl_type=gl.glUniform3fv)
27 |
28 | # Declare all textures used in the program
29 | self.texture_dict["AlbedoMap"] = Texture()
30 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/Shader.py:
--------------------------------------------------------------------------------
1 | from .GLObject import *
2 | import numpy as np
3 |
4 |
5 | class Uniform(object):
6 | def __init__(self, program, uniform_name, type_code, gl_type):
7 | self._as_parameter_ = gl.glGetUniformLocation(program, uniform_name)
8 | self.name = uniform_name
9 | self.type_code = type_code
10 | self.gl_type = gl_type
11 |
12 |
13 | class Shader(GLObject):
14 | def __init__(self,
15 | vs_file='simple.vs',
16 | fs_file='simple.fs',
17 | gs_file=None):
18 | # Importing here, when gl context is already present.
19 | # Otherwise get exception on Python3 because of PyOpenGL bug.
20 | from OpenGL.GL import shaders
21 | with open(self._find_shader_file(vs_file), 'r') as f:
22 | vp_code = f.read()
23 | with open(self._find_shader_file(fs_file), 'r') as f:
24 | fp_code = f.read()
25 | if gs_file:
26 | with open(self._find_shader_file(gs_file), 'r') as f:
27 | gp_code = f.read()
28 | self._as_parameter_ = self._shader = shaders.compileProgram(
29 | shaders.compileShader(vp_code, gl.GL_VERTEX_SHADER),
30 | shaders.compileShader(fp_code, gl.GL_FRAGMENT_SHADER),
31 | shaders.compileShader(gp_code, gl.GL_GEOMETRY_SHADER)
32 | )
33 | else:
34 | self._as_parameter_ = self._shader = shaders.compileProgram(
35 | shaders.compileShader(vp_code, gl.GL_VERTEX_SHADER),
36 | shaders.compileShader(fp_code, gl.GL_FRAGMENT_SHADER)
37 | )
38 | self._uniforms = {}
39 |
40 | def declare_uniform(self, uniform_name, type_code, gl_type):
41 | if uniform_name not in self._uniforms:
42 | self._uniforms[uniform_name] = Uniform(self._shader, uniform_name, type_code, gl_type)
43 | else:
44 | self._uniforms[uniform_name].type_code = type_code
45 | self._uniforms[uniform_name].gl_type = gl_type
46 |
47 | def set_uniform(self, uniform_name, data):
48 | if uniform_name not in self._uniforms:
49 | print(
50 | "Error. Unknown uniform variable. "
51 | "You need to declare all uniform variables in YourRender::_init_shader() function.")
52 | return
53 | uniform = self._uniforms[uniform_name]
54 | data = np.ascontiguousarray(data, uniform.type_code)
55 |
56 | if uniform.gl_type is gl.glUniformMatrix4fv:
57 | gl.glUniformMatrix4fv(uniform, 1, gl.GL_TRUE, data)
58 | elif uniform.gl_type is gl.glUniform3fv:
59 | gl.glUniform3fv(uniform, data.shape[0], data)
60 | elif uniform.gl_type is gl.glUniform1i:
61 | gl.glUniform1i(uniform, data)
62 | else:
63 | print(
64 | "Error. Unknown uniform type. "
65 | "You need to declare all uniform types in Shader::set_uniform() function.")
66 | return
67 |
68 | @staticmethod
69 | def _find_shader_file(name):
70 | import os
71 | gl_folder = os.path.dirname(os.path.abspath(__file__))
72 | glsl_file = os.path.join(gl_folder, 'shaders', name)
73 | return glsl_file
74 |
75 | def release(self):
76 | gl.glDeleteProgram(self._as_parameter_)
77 |
78 | def __enter__(self):
79 | return self._shader.__enter__()
80 |
81 | def __exit__(self, *args):
82 | return self._shader.__exit__(*args)
83 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/__init__.py:
--------------------------------------------------------------------------------
1 | from .Render import *
2 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/glcontext.py:
--------------------------------------------------------------------------------
1 | """OpenGL context creation.
2 |
3 | Typical usage:
4 |
5 | # Optional PyOpenGL configuration can be done here.
6 | # import OpenGL
7 | # OpenGL.ERROR_CHECKING = True
8 |
9 | # 'glcontext' must be imported before any OpenGL.* API.
10 | from lib.render.gl.glcontext import create_opengl_context
11 |
12 | # Now it's safe to import OpenGL and EGL functions
13 | import OpenGL.GL as gl
14 |
15 | # create_opengl_context() creates a GL context that is attached to an
16 | # onscreen window of the specified size. Note that rendering to buffers
17 | # of other sizes and formats is still possible with OpenGL Framebuffers.
18 | #
19 | # Users are expected to directly use the GL API in case more advanced
20 | # context management is required.
21 | width, height = 640, 480
22 | create_opengl_context((width, height))
23 |
24 | # OpenGL context is available here.
25 |
26 | """
27 | from OpenGL.GL import *
28 | from OpenGL.GLUT import *
29 |
30 |
31 | def create_opengl_context(width, height, name="My Render"):
32 | '''
33 | Create on screen OpenGL context and make it current.
34 |
35 | Users are expected to directly use GL API in case more advanced
36 | context management is required.
37 |
38 | :param width: window width in pixels
39 | :param height: window height in pixels
40 | :return:
41 | '''
42 |
43 | glutInit()
44 | display_mode = GLUT_DOUBLE | GLUT_RGB | GLUT_DEPTH
45 |
46 | glutInitDisplayMode(display_mode)
47 | glutInitWindowSize(width, height)
48 | glutInitWindowPosition(0, 0)
49 |
50 | glut_window = glutCreateWindow(name)
51 |
52 | glEnable(GL_DEPTH_TEST)
53 |
54 | glClampColor(GL_CLAMP_READ_COLOR, GL_FALSE)
55 | glClampColor(GL_CLAMP_FRAGMENT_COLOR, GL_FALSE)
56 | glClampColor(GL_CLAMP_VERTEX_COLOR, GL_FALSE)
57 |
58 | return glut_window
59 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/albedo.fs:
--------------------------------------------------------------------------------
1 | #version 330 core
2 | out vec4 FragColor;
3 |
4 | in vec2 TextureCoord;
5 |
6 | uniform sampler2D TargetTexture;
7 |
8 | void main()
9 | {
10 | FragColor = texture(TargetTexture, TextureCoord);
11 | //FragColor = vec4(TextureCoord,1,1);
12 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/albedo.vs:
--------------------------------------------------------------------------------
1 | #version 330 core
2 | layout (location = 0) in vec3 a_Position;
3 | layout (location = 1) in vec2 a_TextureCoord;
4 |
5 | out vec2 TextureCoord;
6 |
7 | uniform mat4 ModelMat;
8 | uniform mat4 PerspMat;
9 |
10 | void main()
11 | {
12 | gl_Position = PerspMat * ModelMat * vec4(a_Position, 1.0);
13 | TextureCoord = a_TextureCoord;
14 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/color.fs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | out vec4 FragColor;
4 |
5 | in vec3 PassColor;
6 |
7 | void main()
8 | {
9 | FragColor = vec4(PassColor, 1.0);
10 | }
11 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/color.vs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | layout (location = 0) in vec3 Position;
4 | layout (location = 1) in vec3 Color;
5 |
6 | out vec3 PassColor;
7 |
8 | uniform mat4 ModelMat;
9 | uniform mat4 PerspMat;
10 |
11 | void main()
12 | {
13 | gl_Position = PerspMat * ModelMat * vec4(Position, 1.0);
14 | PassColor = Color;
15 | }
16 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/normal.fs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | out vec4 FragColor;
4 |
5 | in vec3 CamNormal;
6 |
7 | void main()
8 | {
9 | vec3 cam_norm_normalized = normalize(CamNormal);
10 | vec3 rgb = (cam_norm_normalized + 1.0) / 2.0;
11 | FragColor = vec4(rgb, 1.0);
12 | }
13 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/normal.vs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | layout (location = 0) in vec3 Position;
4 | layout (location = 1) in vec3 Normal;
5 |
6 | out vec3 CamNormal;
7 |
8 | uniform mat4 ModelMat;
9 | uniform mat4 PerspMat;
10 |
11 | void main()
12 | {
13 | gl_Position = PerspMat * ModelMat * vec4(Position, 1.0);
14 | CamNormal = (ModelMat * vec4(Normal, 0.0)).xyz;
15 | }
16 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/prt.fs:
--------------------------------------------------------------------------------
1 | #version 330
2 | uniform vec3 SHCoeffs[9];
3 |
4 | uniform sampler2D AlbedoMap;
5 |
6 | in VertexData {
7 | vec3 Position;
8 | vec3 Normal;
9 | vec2 TextureCoord;
10 | vec3 PRT1;
11 | vec3 PRT2;
12 | vec3 PRT3;
13 | } VertexIn;
14 |
15 | layout (location = 0) out vec4 FragColor;
16 | layout (location = 1) out vec4 FragNormal;
17 | layout (location = 2) out vec4 FragPosition;
18 | layout (location = 3) out vec4 FragAlbedo;
19 | layout (location = 4) out vec4 FragShading;
20 | layout (location = 5) out vec4 FragPRT1;
21 | layout (location = 6) out vec4 FragPRT2;
22 | layout (location = 7) out vec4 FragPRT3;
23 |
24 | vec4 gammaCorrection(vec4 vec, float g)
25 | {
26 | return vec4(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g), vec.w);
27 | }
28 |
29 | vec3 gammaCorrection(vec3 vec, float g)
30 | {
31 | return vec3(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g));
32 | }
33 |
34 | void evaluateH(vec3 n, out float H[9])
35 | {
36 | float c1 = 0.429043, c2 = 0.511664,
37 | c3 = 0.743125, c4 = 0.886227, c5 = 0.247708;
38 |
39 | H[0] = c4;
40 | H[1] = 2.0 * c2 * n[1];
41 | H[2] = 2.0 * c2 * n[2];
42 | H[3] = 2.0 * c2 * n[0];
43 | H[4] = 2.0 * c1 * n[0] * n[1];
44 | H[5] = 2.0 * c1 * n[1] * n[2];
45 | H[6] = c3 * n[2] * n[2] - c5;
46 | H[7] = 2.0 * c1 * n[2] * n[0];
47 | H[8] = c1 * (n[0] * n[0] - n[1] * n[1]);
48 | }
49 |
50 | vec3 evaluateLightingModel(vec3 normal)
51 | {
52 | float H[9];
53 | evaluateH(normal, H);
54 | vec3 res = vec3(0.0);
55 | for (int i = 0; i < 9; i++) {
56 | res += H[i] * SHCoeffs[i];
57 | }
58 | return res;
59 | }
60 |
61 | // nC: coarse geometry normal, nH: fine normal from normal map
62 | vec3 evaluateLightingModelHybrid(vec3 nC, vec3 nH, mat3 prt)
63 | {
64 | float HC[9], HH[9];
65 | evaluateH(nC, HC);
66 | evaluateH(nH, HH);
67 |
68 | vec3 res = vec3(0.0);
69 | vec3 shadow = vec3(0.0);
70 | vec3 unshadow = vec3(0.0);
71 | for(int i = 0; i < 3; ++i){
72 | for(int j = 0; j < 3; ++j){
73 | int id = i*3+j;
74 | res += HH[id]* SHCoeffs[id];
75 | shadow += prt[i][j] * SHCoeffs[id];
76 | unshadow += HC[id] * SHCoeffs[id];
77 | }
78 | }
79 | vec3 ratio = clamp(shadow/unshadow,0.0,1.0);
80 | res = ratio * res;
81 |
82 | return res;
83 | }
84 |
85 | vec3 evaluateLightingModelPRT(mat3 prt)
86 | {
87 | vec3 res = vec3(0.0);
88 | for(int i = 0; i < 3; ++i){
89 | for(int j = 0; j < 3; ++j){
90 | res += prt[i][j] * SHCoeffs[i*3+j];
91 | }
92 | }
93 |
94 | return res;
95 | }
96 |
97 | void main()
98 | {
99 | vec2 uv = VertexIn.TextureCoord;
100 | vec3 normal = normalize(VertexIn.Normal);
101 | mat3 prt = mat3(VertexIn.PRT1, VertexIn.PRT2, VertexIn.PRT3);
102 |
103 | FragAlbedo = texture(AlbedoMap, uv);
104 |
105 | vec4 shading = vec4(evaluateLightingModelPRT(prt), 1.0f);
106 | shading = gammaCorrection(shading, 2.2);
107 |
108 | FragColor = clamp(FragAlbedo * shading, 0.0, 1.0);
109 |
110 | FragNormal = vec4(0.5 * (normal + vec3(1.0)), 1.0);
111 |
112 | FragPosition = vec4(VertexIn.Position, 1.0);
113 |
114 | FragShading = vec4(clamp(0.5 * shading.xyz, 0.0, 1.0), 1.0);
115 |
116 | FragPRT1 = vec4(VertexIn.PRT1,1.0);
117 |
118 | FragPRT2 = vec4(VertexIn.PRT2,1.0);
119 |
120 | FragPRT3 = vec4(VertexIn.PRT3,1.0);
121 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/prt.vs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | layout (location = 0) in vec3 a_Position;
4 | layout (location = 1) in vec3 a_Normal;
5 | layout (location = 2) in vec2 a_TextureCoord;
6 | layout (location = 3) in vec3 a_PRT1;
7 | layout (location = 4) in vec3 a_PRT2;
8 | layout (location = 5) in vec3 a_PRT3;
9 |
10 | // All in camera space
11 | out VertexData {
12 | vec3 Position;
13 | vec3 Normal;
14 | vec2 TextureCoord;
15 | vec3 PRT1;
16 | vec3 PRT2;
17 | vec3 PRT3;
18 | } VertexOut;
19 |
20 | uniform mat4 ModelMat;
21 | uniform mat4 PerspMat;
22 |
23 | uniform int UVMode = 0;
24 |
25 |
26 | float s_c3 = 0.94617469575; // (3*sqrt(5))/(4*sqrt(pi))
27 | float s_c4 = -0.31539156525;// (-sqrt(5))/(4*sqrt(pi))
28 | float s_c5 = 0.54627421529; // (sqrt(15))/(4*sqrt(pi))
29 |
30 | float s_c_scale = 1.0/0.91529123286551084;
31 | float s_c_scale_inv = 0.91529123286551084;
32 |
33 | float s_rc2 = 1.5853309190550713*s_c_scale;
34 | float s_c4_div_c3 = s_c4/s_c3;
35 | float s_c4_div_c3_x2 = (s_c4/s_c3)*2.0;
36 |
37 | float s_scale_dst2 = s_c3 * s_c_scale_inv;
38 | float s_scale_dst4 = s_c5 * s_c_scale_inv;
39 |
40 | void OptRotateBand0(float x[1], mat3 R, out float dst[1])
41 | {
42 | dst[0] = x[0];
43 | }
44 |
45 | // 9 multiplies
46 | void OptRotateBand1(float x[3], mat3 R, out float dst[3])
47 | {
48 | // derived from SlowRotateBand1
49 | dst[0] = ( R[1][1])*x[0] + (-R[1][2])*x[1] + ( R[1][0])*x[2];
50 | dst[1] = (-R[2][1])*x[0] + ( R[2][2])*x[1] + (-R[2][0])*x[2];
51 | dst[2] = ( R[0][1])*x[0] + (-R[0][2])*x[1] + ( R[0][0])*x[2];
52 | }
53 |
54 | // 48 multiplies
55 | void OptRotateBand2(float x[5], mat3 R, out float dst[5])
56 | {
57 | // Sparse matrix multiply
58 | float sh0 = x[3] + x[4] + x[4] - x[1];
59 | float sh1 = x[0] + s_rc2*x[2] + x[3] + x[4];
60 | float sh2 = x[0];
61 | float sh3 = -x[3];
62 | float sh4 = -x[1];
63 |
64 | // Rotations. R0 and R1 just use the raw matrix columns
65 | float r2x = R[0][0] + R[0][1];
66 | float r2y = R[1][0] + R[1][1];
67 | float r2z = R[2][0] + R[2][1];
68 |
69 | float r3x = R[0][0] + R[0][2];
70 | float r3y = R[1][0] + R[1][2];
71 | float r3z = R[2][0] + R[2][2];
72 |
73 | float r4x = R[0][1] + R[0][2];
74 | float r4y = R[1][1] + R[1][2];
75 | float r4z = R[2][1] + R[2][2];
76 |
77 | // dense matrix multiplication one column at a time
78 |
79 | // column 0
80 | float sh0_x = sh0 * R[0][0];
81 | float sh0_y = sh0 * R[1][0];
82 | float d0 = sh0_x * R[1][0];
83 | float d1 = sh0_y * R[2][0];
84 | float d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3);
85 | float d3 = sh0_x * R[2][0];
86 | float d4 = sh0_x * R[0][0] - sh0_y * R[1][0];
87 |
88 | // column 1
89 | float sh1_x = sh1 * R[0][2];
90 | float sh1_y = sh1 * R[1][2];
91 | d0 += sh1_x * R[1][2];
92 | d1 += sh1_y * R[2][2];
93 | d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3);
94 | d3 += sh1_x * R[2][2];
95 | d4 += sh1_x * R[0][2] - sh1_y * R[1][2];
96 |
97 | // column 2
98 | float sh2_x = sh2 * r2x;
99 | float sh2_y = sh2 * r2y;
100 | d0 += sh2_x * r2y;
101 | d1 += sh2_y * r2z;
102 | d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2);
103 | d3 += sh2_x * r2z;
104 | d4 += sh2_x * r2x - sh2_y * r2y;
105 |
106 | // column 3
107 | float sh3_x = sh3 * r3x;
108 | float sh3_y = sh3 * r3y;
109 | d0 += sh3_x * r3y;
110 | d1 += sh3_y * r3z;
111 | d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2);
112 | d3 += sh3_x * r3z;
113 | d4 += sh3_x * r3x - sh3_y * r3y;
114 |
115 | // column 4
116 | float sh4_x = sh4 * r4x;
117 | float sh4_y = sh4 * r4y;
118 | d0 += sh4_x * r4y;
119 | d1 += sh4_y * r4z;
120 | d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2);
121 | d3 += sh4_x * r4z;
122 | d4 += sh4_x * r4x - sh4_y * r4y;
123 |
124 | // extra multipliers
125 | dst[0] = d0;
126 | dst[1] = -d1;
127 | dst[2] = d2 * s_scale_dst2;
128 | dst[3] = -d3;
129 | dst[4] = d4 * s_scale_dst4;
130 | }
131 |
132 | void main()
133 | {
134 |
135 | VertexOut.Position = a_Position;
136 | VertexOut.TextureCoord = a_TextureCoord;
137 |
138 | float PRT0, PRT1[3], PRT2[5];
139 | PRT0 = a_PRT1[0];
140 | PRT1[0] = a_PRT1[1];
141 | PRT1[1] = a_PRT1[2];
142 | PRT1[2] = a_PRT2[0];
143 | PRT2[0] = a_PRT2[1];
144 | PRT2[1] = a_PRT2[2];
145 | PRT2[2] = a_PRT3[0];
146 | PRT2[3] = a_PRT3[1];
147 | PRT2[4] = a_PRT3[2];
148 |
149 | mat3 R = mat3(ModelMat);
150 | OptRotateBand1(PRT1, R, PRT1);
151 | OptRotateBand2(PRT2, R, PRT2);
152 |
153 | VertexOut.PRT1 = vec3(PRT0,PRT1[0],PRT1[1]);
154 | VertexOut.PRT2 = vec3(PRT1[2],PRT2[0],PRT2[1]);
155 | VertexOut.PRT3 = vec3(PRT2[2],PRT2[3],PRT2[4]);
156 |
157 | if(UVMode > 0) {
158 | VertexOut.Normal = a_Normal.xyz;
159 | gl_Position = vec4(a_TextureCoord, 0.0, 1.0) - vec4(0.5, 0.5, 0, 0);
160 | gl_Position[0] *= 2.0;
161 | gl_Position[1] *= 2.0;
162 | }
163 | else {
164 | VertexOut.Normal = (ModelMat * vec4(a_Normal, 0.0)).xyz;
165 | gl_Position = PerspMat * ModelMat * vec4(a_Position, 1.0);
166 | }
167 | }
168 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/prt_uv.fs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | uniform vec3 SHCoeffs[9];
4 | uniform uint analytic;
5 |
6 | uniform uint hasNormalMap;
7 | uniform uint hasAlbedoMap;
8 |
9 | uniform sampler2D AlbedoMap;
10 | uniform sampler2D NormalMap;
11 |
12 | in VertexData {
13 | vec3 Position;
14 | vec3 ModelNormal;
15 | vec3 CameraNormal;
16 | vec2 Texcoord;
17 | vec3 Tangent;
18 | vec3 Bitangent;
19 | vec3 PRT1;
20 | vec3 PRT2;
21 | vec3 PRT3;
22 | } VertexIn;
23 |
24 | layout (location = 0) out vec4 FragColor;
25 | layout (location = 1) out vec4 FragPosition;
26 | layout (location = 2) out vec4 FragNormal;
27 |
28 | vec4 gammaCorrection(vec4 vec, float g)
29 | {
30 | return vec4(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g), vec.w);
31 | }
32 |
33 | vec3 gammaCorrection(vec3 vec, float g)
34 | {
35 | return vec3(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g));
36 | }
37 |
38 | void evaluateH(vec3 n, out float H[9])
39 | {
40 | float c1 = 0.429043, c2 = 0.511664,
41 | c3 = 0.743125, c4 = 0.886227, c5 = 0.247708;
42 |
43 | H[0] = c4;
44 | H[1] = 2.0 * c2 * n[1];
45 | H[2] = 2.0 * c2 * n[2];
46 | H[3] = 2.0 * c2 * n[0];
47 | H[4] = 2.0 * c1 * n[0] * n[1];
48 | H[5] = 2.0 * c1 * n[1] * n[2];
49 | H[6] = c3 * n[2] * n[2] - c5;
50 | H[7] = 2.0 * c1 * n[2] * n[0];
51 | H[8] = c1 * (n[0] * n[0] - n[1] * n[1]);
52 | }
53 |
54 | vec3 evaluateLightingModel(vec3 normal)
55 | {
56 | float H[9];
57 | evaluateH(normal, H);
58 | vec3 res = vec3(0.0);
59 | for (int i = 0; i < 9; i++) {
60 | res += H[i] * SHCoeffs[i];
61 | }
62 | return res;
63 | }
64 |
65 | // nC: coarse geometry normal, nH: fine normal from normal map
66 | vec3 evaluateLightingModelHybrid(vec3 nC, vec3 nH, mat3 prt)
67 | {
68 | float HC[9], HH[9];
69 | evaluateH(nC, HC);
70 | evaluateH(nH, HH);
71 |
72 | vec3 res = vec3(0.0);
73 | vec3 shadow = vec3(0.0);
74 | vec3 unshadow = vec3(0.0);
75 | for(int i = 0; i < 3; ++i){
76 | for(int j = 0; j < 3; ++j){
77 | int id = i*3+j;
78 | res += HH[id]* SHCoeffs[id];
79 | shadow += prt[i][j] * SHCoeffs[id];
80 | unshadow += HC[id] * SHCoeffs[id];
81 | }
82 | }
83 | vec3 ratio = clamp(shadow/unshadow,0.0,1.0);
84 | res = ratio * res;
85 |
86 | return res;
87 | }
88 |
89 | vec3 evaluateLightingModelPRT(mat3 prt)
90 | {
91 | vec3 res = vec3(0.0);
92 | for(int i = 0; i < 3; ++i){
93 | for(int j = 0; j < 3; ++j){
94 | res += prt[i][j] * SHCoeffs[i*3+j];
95 | }
96 | }
97 |
98 | return res;
99 | }
100 |
101 | void main()
102 | {
103 | vec2 uv = VertexIn.Texcoord;
104 | vec3 nM = normalize(VertexIn.ModelNormal);
105 | vec3 nC = normalize(VertexIn.CameraNormal);
106 | vec3 nml = nC;
107 | mat3 prt = mat3(VertexIn.PRT1, VertexIn.PRT2, VertexIn.PRT3);
108 |
109 | vec4 albedo, shading;
110 | if(hasAlbedoMap == uint(0))
111 | albedo = vec4(1.0);
112 | else
113 | albedo = texture(AlbedoMap, uv);//gammaCorrection(texture(AlbedoMap, uv), 1.0/2.2);
114 |
115 | if(hasNormalMap == uint(0))
116 | {
117 | if(analytic == uint(0))
118 | shading = vec4(evaluateLightingModelPRT(prt), 1.0f);
119 | else
120 | shading = vec4(evaluateLightingModel(nC), 1.0f);
121 | }
122 | else
123 | {
124 | vec3 n_tan = normalize(texture(NormalMap, uv).rgb*2.0-vec3(1.0));
125 |
126 | mat3 TBN = mat3(normalize(VertexIn.Tangent),normalize(VertexIn.Bitangent),nC);
127 | vec3 nH = normalize(TBN * n_tan);
128 |
129 | if(analytic == uint(0))
130 | shading = vec4(evaluateLightingModelHybrid(nC,nH,prt),1.0f);
131 | else
132 | shading = vec4(evaluateLightingModel(nH), 1.0f);
133 |
134 | nml = nH;
135 | }
136 |
137 | shading = gammaCorrection(shading, 2.2);
138 | FragColor = clamp(albedo * shading, 0.0, 1.0);
139 | FragPosition = vec4(VertexIn.Position,1.0);
140 | FragNormal = vec4(0.5*(nM+vec3(1.0)),1.0);
141 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/prt_uv.vs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | layout (location = 0) in vec3 a_Position;
4 | layout (location = 1) in vec3 a_Normal;
5 | layout (location = 2) in vec2 a_TextureCoord;
6 | layout (location = 3) in vec3 a_Tangent;
7 | layout (location = 4) in vec3 a_Bitangent;
8 | layout (location = 5) in vec3 a_PRT1;
9 | layout (location = 6) in vec3 a_PRT2;
10 | layout (location = 7) in vec3 a_PRT3;
11 |
12 | out VertexData {
13 | vec3 Position;
14 | vec3 ModelNormal;
15 | vec3 CameraNormal;
16 | vec2 Texcoord;
17 | vec3 Tangent;
18 | vec3 Bitangent;
19 | vec3 PRT1;
20 | vec3 PRT2;
21 | vec3 PRT3;
22 | } VertexOut;
23 |
24 | uniform mat3 RotMat;
25 | uniform mat4 NormMat;
26 | uniform mat4 ModelMat;
27 | uniform mat4 PerspMat;
28 |
29 | #define pi 3.1415926535897932384626433832795
30 |
31 | float s_c3 = 0.94617469575; // (3*sqrt(5))/(4*sqrt(pi))
32 | float s_c4 = -0.31539156525;// (-sqrt(5))/(4*sqrt(pi))
33 | float s_c5 = 0.54627421529; // (sqrt(15))/(4*sqrt(pi))
34 |
35 | float s_c_scale = 1.0/0.91529123286551084;
36 | float s_c_scale_inv = 0.91529123286551084;
37 |
38 | float s_rc2 = 1.5853309190550713*s_c_scale;
39 | float s_c4_div_c3 = s_c4/s_c3;
40 | float s_c4_div_c3_x2 = (s_c4/s_c3)*2.0;
41 |
42 | float s_scale_dst2 = s_c3 * s_c_scale_inv;
43 | float s_scale_dst4 = s_c5 * s_c_scale_inv;
44 |
45 | void OptRotateBand0(float x[1], mat3 R, out float dst[1])
46 | {
47 | dst[0] = x[0];
48 | }
49 |
50 | // 9 multiplies
51 | void OptRotateBand1(float x[3], mat3 R, out float dst[3])
52 | {
53 | // derived from SlowRotateBand1
54 | dst[0] = ( R[1][1])*x[0] + (-R[1][2])*x[1] + ( R[1][0])*x[2];
55 | dst[1] = (-R[2][1])*x[0] + ( R[2][2])*x[1] + (-R[2][0])*x[2];
56 | dst[2] = ( R[0][1])*x[0] + (-R[0][2])*x[1] + ( R[0][0])*x[2];
57 | }
58 |
59 | // 48 multiplies
60 | void OptRotateBand2(float x[5], mat3 R, out float dst[5])
61 | {
62 | // Sparse matrix multiply
63 | float sh0 = x[3] + x[4] + x[4] - x[1];
64 | float sh1 = x[0] + s_rc2*x[2] + x[3] + x[4];
65 | float sh2 = x[0];
66 | float sh3 = -x[3];
67 | float sh4 = -x[1];
68 |
69 | // Rotations. R0 and R1 just use the raw matrix columns
70 | float r2x = R[0][0] + R[0][1];
71 | float r2y = R[1][0] + R[1][1];
72 | float r2z = R[2][0] + R[2][1];
73 |
74 | float r3x = R[0][0] + R[0][2];
75 | float r3y = R[1][0] + R[1][2];
76 | float r3z = R[2][0] + R[2][2];
77 |
78 | float r4x = R[0][1] + R[0][2];
79 | float r4y = R[1][1] + R[1][2];
80 | float r4z = R[2][1] + R[2][2];
81 |
82 | // dense matrix multiplication one column at a time
83 |
84 | // column 0
85 | float sh0_x = sh0 * R[0][0];
86 | float sh0_y = sh0 * R[1][0];
87 | float d0 = sh0_x * R[1][0];
88 | float d1 = sh0_y * R[2][0];
89 | float d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3);
90 | float d3 = sh0_x * R[2][0];
91 | float d4 = sh0_x * R[0][0] - sh0_y * R[1][0];
92 |
93 | // column 1
94 | float sh1_x = sh1 * R[0][2];
95 | float sh1_y = sh1 * R[1][2];
96 | d0 += sh1_x * R[1][2];
97 | d1 += sh1_y * R[2][2];
98 | d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3);
99 | d3 += sh1_x * R[2][2];
100 | d4 += sh1_x * R[0][2] - sh1_y * R[1][2];
101 |
102 | // column 2
103 | float sh2_x = sh2 * r2x;
104 | float sh2_y = sh2 * r2y;
105 | d0 += sh2_x * r2y;
106 | d1 += sh2_y * r2z;
107 | d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2);
108 | d3 += sh2_x * r2z;
109 | d4 += sh2_x * r2x - sh2_y * r2y;
110 |
111 | // column 3
112 | float sh3_x = sh3 * r3x;
113 | float sh3_y = sh3 * r3y;
114 | d0 += sh3_x * r3y;
115 | d1 += sh3_y * r3z;
116 | d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2);
117 | d3 += sh3_x * r3z;
118 | d4 += sh3_x * r3x - sh3_y * r3y;
119 |
120 | // column 4
121 | float sh4_x = sh4 * r4x;
122 | float sh4_y = sh4 * r4y;
123 | d0 += sh4_x * r4y;
124 | d1 += sh4_y * r4z;
125 | d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2);
126 | d3 += sh4_x * r4z;
127 | d4 += sh4_x * r4x - sh4_y * r4y;
128 |
129 | // extra multipliers
130 | dst[0] = d0;
131 | dst[1] = -d1;
132 | dst[2] = d2 * s_scale_dst2;
133 | dst[3] = -d3;
134 | dst[4] = d4 * s_scale_dst4;
135 | }
136 |
137 | void main()
138 | {
139 | // normalization
140 | mat3 R = mat3(ModelMat) * RotMat;
141 | VertexOut.ModelNormal = a_Normal;
142 | VertexOut.CameraNormal = (R * a_Normal);
143 | VertexOut.Position = a_Position;
144 | VertexOut.Texcoord = a_TextureCoord;
145 | VertexOut.Tangent = (R * a_Tangent);
146 | VertexOut.Bitangent = (R * a_Bitangent);
147 | float PRT0, PRT1[3], PRT2[5];
148 | PRT0 = a_PRT1[0];
149 | PRT1[0] = a_PRT1[1];
150 | PRT1[1] = a_PRT1[2];
151 | PRT1[2] = a_PRT2[0];
152 | PRT2[0] = a_PRT2[1];
153 | PRT2[1] = a_PRT2[2];
154 | PRT2[2] = a_PRT3[0];
155 | PRT2[3] = a_PRT3[1];
156 | PRT2[4] = a_PRT3[2];
157 |
158 | OptRotateBand1(PRT1, R, PRT1);
159 | OptRotateBand2(PRT2, R, PRT2);
160 |
161 | VertexOut.PRT1 = vec3(PRT0,PRT1[0],PRT1[1]);
162 | VertexOut.PRT2 = vec3(PRT1[2],PRT2[0],PRT2[1]);
163 | VertexOut.PRT3 = vec3(PRT2[2],PRT2[3],PRT2[4]);
164 |
165 | gl_Position = vec4(a_TextureCoord, 0.0, 1.0) - vec4(0.5, 0.5, 0, 0);
166 | gl_Position[0] *= 2.0;
167 | gl_Position[1] *= 2.0;
168 | }
169 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/quad.fs:
--------------------------------------------------------------------------------
1 | #version 330 core
2 | out vec4 FragColor;
3 |
4 | in vec2 TexCoord;
5 |
6 | uniform sampler2D screenTexture;
7 |
8 | void main()
9 | {
10 | FragColor = texture(screenTexture, TexCoord);
11 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/quad.vs:
--------------------------------------------------------------------------------
1 | #version 330 core
2 | layout (location = 0) in vec2 aPos;
3 | layout (location = 1) in vec2 aTexCoord;
4 |
5 | out vec2 TexCoord;
6 |
7 | void main()
8 | {
9 | gl_Position = vec4(aPos.x, aPos.y, 0.0, 1.0);
10 | TexCoord = aTexCoord;
11 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/sh.fs:
--------------------------------------------------------------------------------
1 | #version 330
2 | uniform vec3 SHCoeffs[9];
3 |
4 | uniform sampler2D AlbedoMap;
5 |
6 | in VertexData {
7 | vec3 Position;
8 | vec3 Normal;
9 | vec2 TextureCoord;
10 | } VertexIn;
11 |
12 | layout (location = 0) out vec4 FragColor;
13 | layout (location = 1) out vec4 FragNormal;
14 | layout (location = 2) out vec4 FragPosition;
15 | layout (location = 3) out vec4 FragAlbedo;
16 | layout (location = 4) out vec4 FragShading;
17 |
18 | vec4 gammaCorrection(vec4 vec, float g)
19 | {
20 | return vec4(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g), vec.w);
21 | }
22 |
23 | vec3 gammaCorrection(vec3 vec, float g)
24 | {
25 | return vec3(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g));
26 | }
27 |
28 | void evaluateH(vec3 n, out float H[9])
29 | {
30 | float c1 = 0.429043, c2 = 0.511664,
31 | c3 = 0.743125, c4 = 0.886227, c5 = 0.247708;
32 |
33 | H[0] = c4;
34 | H[1] = 2.0 * c2 * n[1];
35 | H[2] = 2.0 * c2 * n[2];
36 | H[3] = 2.0 * c2 * n[0];
37 | H[4] = 2.0 * c1 * n[0] * n[1];
38 | H[5] = 2.0 * c1 * n[1] * n[2];
39 | H[6] = c3 * n[2] * n[2] - c5;
40 | H[7] = 2.0 * c1 * n[2] * n[0];
41 | H[8] = c1 * (n[0] * n[0] - n[1] * n[1]);
42 | }
43 |
44 | vec3 evaluateLightingModel(vec3 normal)
45 | {
46 | float H[9];
47 | evaluateH(normal, H);
48 | vec3 res = vec3(0.0);
49 | for (int i = 0; i < 9; i++) {
50 | res += H[i] * SHCoeffs[i];
51 | }
52 | return res;
53 | }
54 |
55 |
56 | void main()
57 | {
58 | vec2 uv = VertexIn.TextureCoord;
59 | vec3 normal = normalize(VertexIn.Normal);
60 |
61 | FragAlbedo = texture(AlbedoMap, uv);
62 |
63 | vec4 shading = vec4(evaluateLightingModel(normal), 1.0f);
64 | shading = gammaCorrection(shading, 2.2);
65 |
66 | FragColor = clamp(FragAlbedo * shading, 0.0, 1.0);
67 |
68 | FragNormal = vec4(0.5 * (normal + vec3(1.0)), 1.0);
69 |
70 | FragPosition = vec4(VertexIn.Position, 1.0);
71 |
72 | FragShading = vec4(clamp(0.5 * shading.xyz, 0.0, 1.0), 1.0);
73 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/sh.vs:
--------------------------------------------------------------------------------
1 | #version 330 core
2 | layout (location = 0) in vec3 a_Position;
3 | layout (location = 1) in vec3 a_Normal;
4 | layout (location = 2) in vec2 a_TextureCoord;
5 |
6 | // All in camera space
7 | out VertexData {
8 | vec3 Position;
9 | vec3 Normal;
10 | vec2 TextureCoord;
11 | } VertexOut;
12 |
13 | uniform mat4 ModelMat;
14 | uniform mat4 PerspMat;
15 |
16 | uniform int UVMode = 0;
17 |
18 |
19 | void main()
20 | {
21 |
22 | VertexOut.Position = a_Position;
23 | VertexOut.TextureCoord = a_TextureCoord;
24 | if(UVMode > 0) {
25 | VertexOut.Normal = a_Normal.xyz;
26 | gl_Position = vec4(a_TextureCoord, 0.0, 1.0) - vec4(0.5, 0.5, 0, 0);
27 | gl_Position[0] *= 2.0;
28 | gl_Position[1] *= 2.0;
29 | }
30 | else {
31 | VertexOut.Normal = (ModelMat * vec4(a_Normal, 0.0)).xyz;
32 | gl_Position = PerspMat * ModelMat * vec4(a_Position, 1.0);
33 | }
34 | }
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/simple.fs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | out vec4 FragColor;
4 |
5 | void main()
6 | {
7 | FragColor = vec4(1.0, 1.0, 1.0, 1.0);
8 | }
9 |
--------------------------------------------------------------------------------
/monoport/lib/render/gl/shaders/simple.vs:
--------------------------------------------------------------------------------
1 | #version 330
2 |
3 | layout (location = 0) in vec3 Position;
4 |
5 | uniform mat4 ModelMat;
6 | uniform mat4 PerspMat;
7 |
8 | void main()
9 | {
10 | gl_Position = PerspMat * ModelMat * vec4(Position, 1.0);
11 | }
12 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy
2 | torch==1.4.0
3 | torchvision
4 | Pillow
5 | flask
6 | scikit-image
7 | opencv-python
8 | tqdm
9 | imageio
10 | yacs
11 | pyopengl
12 | imageio-ffmpeg
13 | human-det @ git+https://github.com/Project-Splinter/human_det
14 | human-inst-seg @ git+https://github.com/Project-Splinter/human_inst_seg
15 | implicit-seg @ git+https://github.com/Project-Splinter/ImplicitSegCUDA
16 | streamer-pytorch @ git+https://github.com/Project-Splinter/streamer_pytorch
17 |
--------------------------------------------------------------------------------
/scripts/download_model.sh:
--------------------------------------------------------------------------------
1 | mkdir -p data/PIFu/
2 | cd data/PIFu/
3 | wget "https://drive.google.com/uc?export=download&id=1zEmVXG2VHy0MMzngcRshB4D8Sr_oLHsm" -O net_G
4 | wget "https://drive.google.com/uc?export=download&id=1V83B6GDIjYMfHdpg-KcCSAPgHxpafHgd" -O net_C
5 | cd ../../
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import setuptools
2 |
3 | setuptools.setup(
4 | name='monoport',
5 | url='',
6 | description='',
7 | version='1.0.0',
8 | author='Ruilong Li',
9 | author_email='ruilongl@usc.edu',
10 | license='MIT License',
11 | packages=setuptools.find_packages(),
12 | )
13 |
14 |
--------------------------------------------------------------------------------
58 |  }})
61 | 
8 | 
25 | 
73 | 
130 |