├── .idea
    ├── bisenet.iml
    ├── deployment.xml
    ├── misc.xml
    ├── modules.xml
    ├── vcs.xml
    └── workspace.xml
├── LICENSE.md
├── README.md
├── UnpromptedControl.ipynb
├── __pycache__
    └── scratch_detection.cpython-311.pyc
├── dataset
    ├── CamVid.py
    ├── __init__.py
    └── __pycache__
    │   ├── CamVid.cpython-36.pyc
    │   └── __init__.cpython-36.pyc
├── demo.png
├── demo.py
├── detection_models
    ├── __init__.py
    ├── __pycache__
    │   ├── __init__.cpython-311.pyc
    │   ├── antialiasing.cpython-311.pyc
    │   └── networks.cpython-311.pyc
    ├── antialiasing.py
    ├── networks.py
    └── sync_batchnorm
    │   ├── __init__.py
    │   ├── __pycache__
    │       ├── __init__.cpython-311.pyc
    │       ├── batchnorm.cpython-311.pyc
    │       ├── comm.cpython-311.pyc
    │       └── replicate.cpython-311.pyc
    │   ├── batchnorm.py
    │   ├── batchnorm_reimpl.py
    │   ├── comm.py
    │   ├── replicate.py
    │   └── unittest.py
├── detection_util
    ├── __pycache__
    │   └── util.cpython-311.pyc
    └── util.py
├── eval.py
├── examples
    ├── eg1.jpg
    ├── eg2.jpg
    ├── eg2gif.gif
    ├── obj1.jpg
    ├── obj2.jpg
    └── objgif.gif
├── input_images
    └── input_image.png
├── loss.py
├── model.py
├── obrem.py
├── output_masks
    ├── input
    │   └── input_image.png
    └── mask
    │   └── input_image.png
├── resnet.py
├── rest.py
├── scratch_detection.py
├── src
    ├── __pycache__
    │   └── pipeline_stable_diffusion_controlnet_inpaint.cpython-311.pyc
    └── pipeline_stable_diffusion_controlnet_inpaint.py
├── test.png
├── test_label.png
├── tfboard_loss.jpg
├── tfboard_miou.jpg
├── tfboard_precision.jpg
├── train.py
└── utils.py


/.idea/bisenet.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="jdk" jdkName="Remote Python 3.6.6 (sftp://sqy@192.168.0.243:22/home/sqy/.conda/envs/myenv/bin/python) (1)" jdkType="Python SDK" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="projectConfiguration" value="pytest" />
10 |     <option name="PROJECT_TEST_RUNNER" value="pytest" />
11 |   </component>
12 | </module>


--------------------------------------------------------------------------------
/.idea/deployment.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project version="4">
 3 |   <component name="PublishConfigData" autoUpload="Always" serverName="sqy@192.168.0.243:22">
 4 |     <serverData>
 5 |       <paths name="Copy of project-level server '244'">
 6 |         <serverdata>
 7 |           <mappings>
 8 |             <mapping local="$PROJECT_DIR$" web="/" />
 9 |           </mappings>
10 |         </serverdata>
11 |       </paths>
12 |       <paths name="sqy@192.168.0.242:22">
13 |         <serverdata>
14 |           <mappings>
15 |             <mapping local="$PROJECT_DIR$" web="/" />
16 |           </mappings>
17 |         </serverdata>
18 |       </paths>
19 |       <paths name="sqy@192.168.0.243:22">
20 |         <serverdata>
21 |           <mappings>
22 |             <mapping deploy="/data/sqy/code/bisenet" local="$PROJECT_DIR$" web="/" />
23 |           </mappings>
24 |         </serverdata>
25 |       </paths>
26 |       <paths name="sqy@192.168.0.243:22 (1)">
27 |         <serverdata>
28 |           <mappings>
29 |             <mapping local="$PROJECT_DIR$" web="/" />
30 |           </mappings>
31 |         </serverdata>
32 |       </paths>
33 |       <paths name="sqy@192.168.0.243:22 (2)">
34 |         <serverdata>
35 |           <mappings>
36 |             <mapping local="$PROJECT_DIR$" web="/" />
37 |           </mappings>
38 |         </serverdata>
39 |       </paths>
40 |     </serverData>
41 |     <option name="myAutoUpload" value="ALWAYS" />
42 |   </component>
43 | </project>


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="JavaScriptSettings">
4 |     <option name="languageLevel" value="ES6" />
5 |   </component>
6 |   <component name="ProjectRootManager" version="2" project-jdk-name="Remote Python 3.6.6 (sftp://sqy@192.168.0.243:22/home/sqy/.conda/envs/myenv/bin/python) (1)" project-jdk-type="Python SDK" />
7 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/bisenet.iml" filepath="$PROJECT_DIR$/.idea/bisenet.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project version="4">
  3 |   <component name="ChangeListManager">
  4 |     <list default="true" id="1ac99788-2664-485f-9a80-9e4a872e6c73" name="Default Changelist" comment="" />
  5 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  6 |     <option name="SHOW_DIALOG" value="false" />
  7 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
  8 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
  9 |     <option name="LAST_RESOLUTION" value="IGNORE" />
 10 |   </component>
 11 |   <component name="CoverageDataManager">
 12 |     <SUITE FILE_PATH="coverage/bisenet$utils.coverage" NAME="utils Coverage Results" MODIFIED="1555069226469" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 13 |     <SUITE FILE_PATH="coverage/bisenet$CamVid.coverage" NAME="CamVid Coverage Results" MODIFIED="1542630015326" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/dataset" />
 14 |     <SUITE FILE_PATH="coverage/bisenet$train.coverage" NAME="train Coverage Results" MODIFIED="1559200804017" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 15 |     <SUITE FILE_PATH="coverage/bisenet$eval.coverage" NAME="eval Coverage Results" MODIFIED="1542679606715" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 16 |     <SUITE FILE_PATH="coverage/bisenet$build_contextpath.coverage" NAME="build_contextpath Coverage Results" MODIFIED="1542078357529" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/model" />
 17 |     <SUITE FILE_PATH="coverage/bisenet$pytest_in_test_py.coverage" NAME="pytest in eval.py Coverage Results" MODIFIED="1542462345843" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 18 |     <SUITE FILE_PATH="coverage/bisenet$build_BiSeNet.coverage" NAME="build_BiSeNet Coverage Results" MODIFIED="1559200423133" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/model" />
 19 |     <SUITE FILE_PATH="coverage/bisenet$demo.coverage" NAME="demo Coverage Results" MODIFIED="1542678813901" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 20 |     <SUITE FILE_PATH="coverage/bisenet$transforms.coverage" NAME="transforms Coverage Results" MODIFIED="1542605157204" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torchvision/transforms" />
 21 |     <SUITE FILE_PATH="coverage/bisenet$exp.coverage" NAME="exp Coverage Results" MODIFIED="1555068113328" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 22 |   </component>
 23 |   <component name="FUSProjectUsageTrigger">
 24 |     <session id="730400909">
 25 |       <usages-collector id="statistics.lifecycle.project">
 26 |         <counts>
 27 |           <entry key="project.open.time.0" value="1" />
 28 |           <entry key="project.open.time.12" value="1" />
 29 |           <entry key="project.open.time.18" value="1" />
 30 |           <entry key="project.opened" value="3" />
 31 |         </counts>
 32 |       </usages-collector>
 33 |       <usages-collector id="statistics.file.extensions.open">
 34 |         <counts>
 35 |           <entry key="jpg" value="3" />
 36 |           <entry key="png" value="2" />
 37 |           <entry key="py" value="103" />
 38 |           <entry key="pyi" value="2" />
 39 |         </counts>
 40 |       </usages-collector>
 41 |       <usages-collector id="statistics.file.types.open">
 42 |         <counts>
 43 |           <entry key="Image" value="5" />
 44 |           <entry key="Python" value="103" />
 45 |           <entry key="PythonStub" value="2" />
 46 |         </counts>
 47 |       </usages-collector>
 48 |       <usages-collector id="statistics.file.extensions.edit">
 49 |         <counts>
 50 |           <entry key="dummy" value="62" />
 51 |           <entry key="py" value="25113" />
 52 |         </counts>
 53 |       </usages-collector>
 54 |       <usages-collector id="statistics.file.types.edit">
 55 |         <counts>
 56 |           <entry key="PLAIN_TEXT" value="62" />
 57 |           <entry key="Python" value="25113" />
 58 |         </counts>
 59 |       </usages-collector>
 60 |     </session>
 61 |     <session id="422299589">
 62 |       <usages-collector id="statistics.lifecycle.project">
 63 |         <counts>
 64 |           <entry key="project.open.time.0" value="1" />
 65 |           <entry key="project.opened" value="1" />
 66 |         </counts>
 67 |       </usages-collector>
 68 |     </session>
 69 |     <session id="756654074">
 70 |       <usages-collector id="statistics.lifecycle.project">
 71 |         <counts>
 72 |           <entry key="project.closed" value="10" />
 73 |           <entry key="project.open.time.0" value="6" />
 74 |           <entry key="project.open.time.1" value="3" />
 75 |           <entry key="project.open.time.2" value="1" />
 76 |           <entry key="project.open.time.3" value="1" />
 77 |           <entry key="project.opened" value="11" />
 78 |         </counts>
 79 |       </usages-collector>
 80 |       <usages-collector id="statistics.file.extensions.open">
 81 |         <counts>
 82 |           <entry key="png" value="2" />
 83 |           <entry key="py" value="6" />
 84 |         </counts>
 85 |       </usages-collector>
 86 |       <usages-collector id="statistics.file.types.open">
 87 |         <counts>
 88 |           <entry key="Image" value="2" />
 89 |           <entry key="Python" value="6" />
 90 |         </counts>
 91 |       </usages-collector>
 92 |       <usages-collector id="statistics.file.extensions.edit">
 93 |         <counts>
 94 |           <entry key="py" value="2163" />
 95 |         </counts>
 96 |       </usages-collector>
 97 |       <usages-collector id="statistics.file.types.edit">
 98 |         <counts>
 99 |           <entry key="Python" value="2163" />
100 |         </counts>
101 |       </usages-collector>
102 |     </session>
103 |   </component>
104 |   <component name="FileEditorManager">
105 |     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
106 |       <file pinned="false" current-in-tab="false">
107 |         <entry file="file://$PROJECT_DIR$/model/build_BiSeNet.py">
108 |           <provider selected="true" editor-type-id="text-editor">
109 |             <state relative-caret-position="469">
110 |               <caret line="164" selection-start-line="164" selection-end-line="164" />
111 |               <folding>
112 |                 <element signature="e#0#12#0" expanded="true" />
113 |               </folding>
114 |             </state>
115 |           </provider>
116 |         </entry>
117 |       </file>
118 |       <file pinned="false" current-in-tab="false">
119 |         <entry file="file://$PROJECT_DIR$/model/build_contextpath.py">
120 |           <provider selected="true" editor-type-id="text-editor">
121 |             <state relative-caret-position="412">
122 |               <caret line="67" column="34" selection-start-line="67" selection-start-column="34" selection-end-line="67" selection-end-column="34" />
123 |               <folding>
124 |                 <element signature="e#0#12#0" expanded="true" />
125 |               </folding>
126 |             </state>
127 |           </provider>
128 |         </entry>
129 |       </file>
130 |       <file pinned="false" current-in-tab="false">
131 |         <entry file="file://$PROJECT_DIR$/eval.py">
132 |           <provider selected="true" editor-type-id="text-editor">
133 |             <state relative-caret-position="304">
134 |               <caret line="24" column="41" selection-start-line="24" selection-start-column="41" selection-end-line="24" selection-end-column="41" />
135 |               <folding>
136 |                 <element signature="e#0#33#0" expanded="true" />
137 |               </folding>
138 |             </state>
139 |           </provider>
140 |         </entry>
141 |       </file>
142 |       <file pinned="false" current-in-tab="false">
143 |         <entry file="file://$PROJECT_DIR$/demo.py">
144 |           <provider selected="true" editor-type-id="text-editor">
145 |             <state relative-caret-position="646">
146 |               <caret line="44" column="85" selection-start-line="44" selection-start-column="85" selection-end-line="44" selection-end-column="85" />
147 |               <folding>
148 |                 <element signature="e#0#10#0" expanded="true" />
149 |               </folding>
150 |             </state>
151 |           </provider>
152 |         </entry>
153 |       </file>
154 |       <file pinned="false" current-in-tab="true">
155 |         <entry file="file://$PROJECT_DIR$/train.py">
156 |           <provider selected="true" editor-type-id="text-editor">
157 |             <state relative-caret-position="589">
158 |               <caret line="173" column="30" selection-start-line="173" selection-start-column="30" selection-end-line="173" selection-end-column="30" />
159 |               <folding>
160 |                 <element signature="e#0#15#0" expanded="true" />
161 |               </folding>
162 |             </state>
163 |           </provider>
164 |         </entry>
165 |       </file>
166 |       <file pinned="false" current-in-tab="false">
167 |         <entry file="file://$PROJECT_DIR$/dataset/CamVid.py">
168 |           <provider selected="true" editor-type-id="text-editor">
169 |             <state relative-caret-position="1273">
170 |               <caret line="79" column="43" selection-start-line="79" selection-start-column="43" selection-end-line="79" selection-end-column="43" />
171 |               <folding>
172 |                 <element signature="e#0#12#0" expanded="true" />
173 |               </folding>
174 |             </state>
175 |           </provider>
176 |         </entry>
177 |       </file>
178 |       <file pinned="false" current-in-tab="false">
179 |         <entry file="file://$PROJECT_DIR$/utils.py">
180 |           <provider selected="true" editor-type-id="text-editor">
181 |             <state relative-caret-position="1615">
182 |               <caret line="90" column="24" selection-start-line="90" selection-start-column="24" selection-end-line="90" selection-end-column="24" />
183 |               <folding>
184 |                 <element signature="e#0#21#0" expanded="true" />
185 |               </folding>
186 |             </state>
187 |           </provider>
188 |         </entry>
189 |       </file>
190 |     </leaf>
191 |   </component>
192 |   <component name="FileTemplateManagerImpl">
193 |     <option name="RECENT_TEMPLATES">
194 |       <list>
195 |         <option value="Python Script" />
196 |       </list>
197 |     </option>
198 |   </component>
199 |   <component name="FindInProjectRecents">
200 |     <findStrings>
201 |       <find>bounding</find>
202 |       <find>quokka_segmentation_map</find>
203 |       <find>quokka_heatmap</find>
204 |       <find>SegmentationMapOnImage</find>
205 |       <find>interpolation</find>
206 |       <find>winname</find>
207 |     </findStrings>
208 |   </component>
209 |   <component name="Git.Settings">
210 |     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
211 |   </component>
212 |   <component name="IdeDocumentHistory">
213 |     <option name="CHANGED_PATHS">
214 |       <list>
215 |         <option value="$PROJECT_DIR$/model_tf.py" />
216 |         <option value="$PROJECT_DIR$/build_model.py" />
217 |         <option value="$PROJECT_DIR$/model/__init__.py" />
218 |         <option value="$PROJECT_DIR$/model/model_tf.py" />
219 |         <option value="$PROJECT_DIR$/sample_train.py" />
220 |         <option value="$PROJECT_DIR$/dataset/__init__.py" />
221 |         <option value="$PROJECT_DIR$/test.py" />
222 |         <option value="$PROJECT_DIR$/demo.py" />
223 |         <option value="$PROJECT_DIR$/dataset/CamVid.py" />
224 |         <option value="$PROJECT_DIR$/exp.py" />
225 |         <option value="$PROJECT_DIR$/utils.py" />
226 |         <option value="$PROJECT_DIR$/eval.py" />
227 |         <option value="$PROJECT_DIR$/model/build_contextpath.py" />
228 |         <option value="$PROJECT_DIR$/model/build_BiSeNet.py" />
229 |         <option value="$PROJECT_DIR$/train.py" />
230 |       </list>
231 |     </option>
232 |   </component>
233 |   <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
234 |   <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
235 |   <component name="JsGulpfileManager">
236 |     <detection-done>true</detection-done>
237 |     <sorting>DEFINITION_ORDER</sorting>
238 |   </component>
239 |   <component name="ProjectFrameBounds" extendedState="7">
240 |     <option name="x" value="-8" />
241 |     <option name="y" value="-8" />
242 |     <option name="width" value="1936" />
243 |     <option name="height" value="1056" />
244 |   </component>
245 |   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
246 |   <component name="ProjectView">
247 |     <navigator proportions="" version="1">
248 |       <foldersAlwaysOnTop value="true" />
249 |     </navigator>
250 |     <panes>
251 |       <pane id="Scope" />
252 |       <pane id="ProjectPane">
253 |         <subPane>
254 |           <expand>
255 |             <path>
256 |               <item name="bisenet" type="b2602c69:ProjectViewProjectNode" />
257 |               <item name="bisenet" type="462c0819:PsiDirectoryNode" />
258 |             </path>
259 |             <path>
260 |               <item name="bisenet" type="b2602c69:ProjectViewProjectNode" />
261 |               <item name="bisenet" type="462c0819:PsiDirectoryNode" />
262 |               <item name="dataset" type="462c0819:PsiDirectoryNode" />
263 |             </path>
264 |             <path>
265 |               <item name="bisenet" type="b2602c69:ProjectViewProjectNode" />
266 |               <item name="bisenet" type="462c0819:PsiDirectoryNode" />
267 |               <item name="model" type="462c0819:PsiDirectoryNode" />
268 |             </path>
269 |           </expand>
270 |           <select />
271 |         </subPane>
272 |       </pane>
273 |     </panes>
274 |   </component>
275 |   <component name="PropertiesComponent">
276 |     <property name="WebServerToolWindowFactoryState" value="true" />
277 |     <property name="last_opened_file_path" value="Z:/code/pytorch-mask-rcnn" />
278 |     <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
279 |     <property name="nodejs_npm_path_reset_for_default_project" value="true" />
280 |     <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
281 |   </component>
282 |   <component name="RecentsManager">
283 |     <key name="CopyFile.RECENT_KEYS">
284 |       <recent name="Z:\data\bisenet" />
285 |     </key>
286 |     <key name="MoveFile.RECENT_KEYS">
287 |       <recent name="Z:\data\bisenet\model" />
288 |     </key>
289 |   </component>
290 |   <component name="RunDashboard">
291 |     <option name="ruleStates">
292 |       <list>
293 |         <RuleState>
294 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
295 |         </RuleState>
296 |         <RuleState>
297 |           <option name="name" value="StatusDashboardGroupingRule" />
298 |         </RuleState>
299 |       </list>
300 |     </option>
301 |   </component>
302 |   <component name="RunManager" selected="Python.train">
303 |     <configuration name="build_BiSeNet" type="PythonConfigurationType" factoryName="Python" temporary="true">
304 |       <module name="bisenet" />
305 |       <option name="INTERPRETER_OPTIONS" value="" />
306 |       <option name="PARENT_ENVS" value="true" />
307 |       <envs>
308 |         <env name="PYTHONUNBUFFERED" value="1" />
309 |       </envs>
310 |       <option name="SDK_HOME" value="" />
311 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/model" />
312 |       <option name="IS_MODULE_SDK" value="true" />
313 |       <option name="ADD_CONTENT_ROOTS" value="true" />
314 |       <option name="ADD_SOURCE_ROOTS" value="true" />
315 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
316 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/model/build_BiSeNet.py" />
317 |       <option name="PARAMETERS" value="" />
318 |       <option name="SHOW_COMMAND_LINE" value="false" />
319 |       <option name="EMULATE_TERMINAL" value="false" />
320 |       <option name="MODULE_MODE" value="false" />
321 |       <option name="REDIRECT_INPUT" value="false" />
322 |       <option name="INPUT_FILE" value="" />
323 |       <method v="2" />
324 |     </configuration>
325 |     <configuration name="eval" type="PythonConfigurationType" factoryName="Python" temporary="true">
326 |       <module name="bisenet" />
327 |       <option name="INTERPRETER_OPTIONS" value="" />
328 |       <option name="PARENT_ENVS" value="true" />
329 |       <envs>
330 |         <env name="PYTHONUNBUFFERED" value="1" />
331 |       </envs>
332 |       <option name="SDK_HOME" value="" />
333 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
334 |       <option name="IS_MODULE_SDK" value="true" />
335 |       <option name="ADD_CONTENT_ROOTS" value="true" />
336 |       <option name="ADD_SOURCE_ROOTS" value="true" />
337 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
338 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/eval.py" />
339 |       <option name="PARAMETERS" value="" />
340 |       <option name="SHOW_COMMAND_LINE" value="false" />
341 |       <option name="EMULATE_TERMINAL" value="false" />
342 |       <option name="MODULE_MODE" value="false" />
343 |       <option name="REDIRECT_INPUT" value="false" />
344 |       <option name="INPUT_FILE" value="" />
345 |       <method v="2" />
346 |     </configuration>
347 |     <configuration name="exp" type="PythonConfigurationType" factoryName="Python" temporary="true">
348 |       <module name="bisenet" />
349 |       <option name="INTERPRETER_OPTIONS" value="" />
350 |       <option name="PARENT_ENVS" value="true" />
351 |       <envs>
352 |         <env name="PYTHONUNBUFFERED" value="1" />
353 |       </envs>
354 |       <option name="SDK_HOME" value="" />
355 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
356 |       <option name="IS_MODULE_SDK" value="true" />
357 |       <option name="ADD_CONTENT_ROOTS" value="true" />
358 |       <option name="ADD_SOURCE_ROOTS" value="true" />
359 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
360 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/exp.py" />
361 |       <option name="PARAMETERS" value="" />
362 |       <option name="SHOW_COMMAND_LINE" value="false" />
363 |       <option name="EMULATE_TERMINAL" value="false" />
364 |       <option name="MODULE_MODE" value="false" />
365 |       <option name="REDIRECT_INPUT" value="false" />
366 |       <option name="INPUT_FILE" value="" />
367 |       <method v="2" />
368 |     </configuration>
369 |     <configuration name="train" type="PythonConfigurationType" factoryName="Python" temporary="true">
370 |       <module name="bisenet" />
371 |       <option name="INTERPRETER_OPTIONS" value="" />
372 |       <option name="PARENT_ENVS" value="true" />
373 |       <envs>
374 |         <env name="PYTHONUNBUFFERED" value="1" />
375 |       </envs>
376 |       <option name="SDK_HOME" value="" />
377 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
378 |       <option name="IS_MODULE_SDK" value="true" />
379 |       <option name="ADD_CONTENT_ROOTS" value="true" />
380 |       <option name="ADD_SOURCE_ROOTS" value="true" />
381 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
382 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/train.py" />
383 |       <option name="PARAMETERS" value="" />
384 |       <option name="SHOW_COMMAND_LINE" value="false" />
385 |       <option name="EMULATE_TERMINAL" value="false" />
386 |       <option name="MODULE_MODE" value="false" />
387 |       <option name="REDIRECT_INPUT" value="false" />
388 |       <option name="INPUT_FILE" value="" />
389 |       <method v="2" />
390 |     </configuration>
391 |     <configuration name="utils" type="PythonConfigurationType" factoryName="Python" temporary="true">
392 |       <module name="bisenet" />
393 |       <option name="INTERPRETER_OPTIONS" value="" />
394 |       <option name="PARENT_ENVS" value="true" />
395 |       <envs>
396 |         <env name="PYTHONUNBUFFERED" value="1" />
397 |       </envs>
398 |       <option name="SDK_HOME" value="" />
399 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
400 |       <option name="IS_MODULE_SDK" value="true" />
401 |       <option name="ADD_CONTENT_ROOTS" value="true" />
402 |       <option name="ADD_SOURCE_ROOTS" value="true" />
403 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
404 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/utils.py" />
405 |       <option name="PARAMETERS" value="" />
406 |       <option name="SHOW_COMMAND_LINE" value="false" />
407 |       <option name="EMULATE_TERMINAL" value="false" />
408 |       <option name="MODULE_MODE" value="false" />
409 |       <option name="REDIRECT_INPUT" value="false" />
410 |       <option name="INPUT_FILE" value="" />
411 |       <method v="2" />
412 |     </configuration>
413 |     <list>
414 |       <item itemvalue="Python.train" />
415 |       <item itemvalue="Python.eval" />
416 |       <item itemvalue="Python.exp" />
417 |       <item itemvalue="Python.utils" />
418 |       <item itemvalue="Python.build_BiSeNet" />
419 |     </list>
420 |     <recent_temporary>
421 |       <list>
422 |         <item itemvalue="Python.train" />
423 |         <item itemvalue="Python.build_BiSeNet" />
424 |         <item itemvalue="Python.utils" />
425 |         <item itemvalue="Python.exp" />
426 |         <item itemvalue="Python.eval" />
427 |       </list>
428 |     </recent_temporary>
429 |   </component>
430 |   <component name="SvnConfiguration">
431 |     <configuration />
432 |   </component>
433 |   <component name="TaskManager">
434 |     <task active="true" id="Default" summary="Default task">
435 |       <changelist id="1ac99788-2664-485f-9a80-9e4a872e6c73" name="Default Changelist" comment="" />
436 |       <created>1542001793802</created>
437 |       <option name="number" value="Default" />
438 |       <option name="presentableId" value="Default" />
439 |       <updated>1542001793802</updated>
440 |     </task>
441 |     <servers />
442 |   </component>
443 |   <component name="TestHistory">
444 |     <history-entry file="pytest_in_test_py - 2018.11.17 at 21h 37m 45s.xml">
445 |       <configuration name="pytest in test.py" configurationId="tests" />
446 |     </history-entry>
447 |     <history-entry file="pytest_in_net_test_py - 2018.11.17 at 21h 41m 00s.xml">
448 |       <configuration name="pytest in net_test.py" configurationId="tests" />
449 |     </history-entry>
450 |     <history-entry file="pytest_in_eval_py - 2018.11.17 at 21h 45m 46s.xml">
451 |       <configuration name="pytest in eval.py" configurationId="tests" />
452 |     </history-entry>
453 |   </component>
454 |   <component name="ToolWindowManager">
455 |     <frame x="-8" y="-8" width="1936" height="1056" extended-state="7" />
456 |     <editor active="true" />
457 |     <layout>
458 |       <window_info id="Favorites" order="0" side_tool="true" />
459 |       <window_info content_ui="combo" id="Project" order="1" visible="true" weight="0.25" />
460 |       <window_info id="Structure" order="2" side_tool="true" weight="0.25" />
461 |       <window_info anchor="bottom" id="Database Changes" order="0" show_stripe_button="false" />
462 |       <window_info anchor="bottom" id="Terminal" order="1" />
463 |       <window_info anchor="bottom" id="Event Log" order="2" side_tool="true" />
464 |       <window_info anchor="bottom" id="Version Control" order="3" />
465 |       <window_info anchor="bottom" id="Python Console" order="4" />
466 |       <window_info anchor="bottom" id="Docker" order="5" show_stripe_button="false" />
467 |       <window_info anchor="bottom" id="Message" order="6" />
468 |       <window_info anchor="bottom" id="Find" order="7" weight="0.32944915" />
469 |       <window_info anchor="bottom" id="Run" order="8" weight="0.32873806" />
470 |       <window_info anchor="bottom" id="Debug" order="9" weight="0.39660656" />
471 |       <window_info anchor="bottom" id="Cvs" order="10" weight="0.25" />
472 |       <window_info anchor="bottom" id="Inspection" order="11" weight="0.4" />
473 |       <window_info anchor="bottom" id="TODO" order="12" />
474 |       <window_info anchor="bottom" id="File Transfer" order="13" weight="0.32979852" />
475 |       <window_info anchor="right" id="SciView" order="0" weight="0.3296875" />
476 |       <window_info anchor="right" id="Database" order="1" />
477 |       <window_info anchor="right" id="Remote Host" order="2" />
478 |       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="3" type="SLIDING" weight="0.4" />
479 |       <window_info anchor="right" id="Ant Build" order="4" weight="0.25" />
480 |       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="5" weight="0.25" />
481 |     </layout>
482 |   </component>
483 |   <component name="TypeScriptGeneratedFilesManager">
484 |     <option name="version" value="1" />
485 |   </component>
486 |   <component name="VcsContentAnnotationSettings">
487 |     <option name="myLimit" value="2678400000" />
488 |   </component>
489 |   <component name="debuggerHistoryManager">
490 |     <expressions id="evaluateCodeFragment">
491 |       <expression>
492 |         <expression-string>hist</expression-string>
493 |         <language-id>Python</language-id>
494 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
495 |       </expression>
496 |       <expression>
497 |         <expression-string>per_class_iu(hist)</expression-string>
498 |         <language-id>Python</language-id>
499 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
500 |       </expression>
501 |       <expression>
502 |         <expression-string>np.min(label)</expression-string>
503 |         <language-id>Python</language-id>
504 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
505 |       </expression>
506 |       <expression>
507 |         <expression-string>np.min(predict)</expression-string>
508 |         <language-id>Python</language-id>
509 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
510 |       </expression>
511 |       <expression>
512 |         <expression-string>np.max(predict)</expression-string>
513 |         <language-id>Python</language-id>
514 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
515 |       </expression>
516 |       <expression>
517 |         <expression-string>np.max(label)</expression-string>
518 |         <language-id>Python</language-id>
519 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
520 |       </expression>
521 |       <expression>
522 |         <expression-string>0 /0</expression-string>
523 |         <language-id>Python</language-id>
524 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
525 |       </expression>
526 |       <expression>
527 |         <expression-string>predict</expression-string>
528 |         <language-id>Python</language-id>
529 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
530 |       </expression>
531 |       <expression>
532 |         <expression-string>predict.shape</expression-string>
533 |         <language-id>Python</language-id>
534 |         <evaluation-mode>CODE_FRAGMENT</evaluation-mode>
535 |       </expression>
536 |     </expressions>
537 |   </component>
538 |   <component name="editorHistoryManager">
539 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/tensorflow/contrib/slim/__init__.py" />
540 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/nn/modules/batchnorm.py" />
541 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/nn/modules/pooling.py" />
542 |     <entry file="file://$PROJECT_DIR$/model/__init__.py">
543 |       <provider selected="true" editor-type-id="text-editor" />
544 |     </entry>
545 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/python_stubs/-1473333495/torch/_C/_VariableFunctions.py">
546 |       <provider selected="true" editor-type-id="text-editor">
547 |         <state relative-caret-position="221">
548 |           <caret line="688" column="12" lean-forward="true" selection-start-line="688" selection-start-column="12" selection-end-line="688" selection-end-column="12" />
549 |         </state>
550 |       </provider>
551 |     </entry>
552 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/python_stubs/-1473333495/torch/_C/__init__/_VariableFunctions.py">
553 |       <provider selected="true" editor-type-id="text-editor">
554 |         <state relative-caret-position="-596">
555 |           <caret line="514" column="27" lean-forward="true" selection-start-line="514" selection-start-column="27" selection-end-line="514" selection-end-column="27" />
556 |         </state>
557 |       </provider>
558 |     </entry>
559 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/python_stubs/-1473333495/numpy/__init__.py">
560 |       <provider selected="true" editor-type-id="text-editor" />
561 |     </entry>
562 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/python_stubs/-1473333495/torch/__init__.py">
563 |       <provider selected="true" editor-type-id="text-editor" />
564 |     </entry>
565 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/__init__.py">
566 |       <provider selected="true" editor-type-id="text-editor">
567 |         <state relative-caret-position="-228" />
568 |       </provider>
569 |     </entry>
570 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/tensorflow/__init__.py">
571 |       <provider selected="true" editor-type-id="text-editor" />
572 |     </entry>
573 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/nn/modules/conv.py">
574 |       <provider selected="true" editor-type-id="text-editor">
575 |         <state relative-caret-position="354">
576 |           <caret line="248" column="45" selection-start-line="248" selection-start-column="45" selection-end-line="248" selection-end-column="45" />
577 |         </state>
578 |       </provider>
579 |     </entry>
580 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/tensorflow/contrib/layers/python/layers/layers.py">
581 |       <provider selected="true" editor-type-id="text-editor">
582 |         <state relative-caret-position="-224">
583 |           <caret line="945" column="55" selection-start-line="945" selection-start-column="55" selection-end-line="945" selection-end-column="55" />
584 |         </state>
585 |       </provider>
586 |     </entry>
587 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/imgaug/__init__.py">
588 |       <provider selected="true" editor-type-id="text-editor">
589 |         <state relative-caret-position="19">
590 |           <caret line="1" column="8" selection-start-line="1" selection-start-column="8" selection-end-line="1" selection-end-column="8" />
591 |         </state>
592 |       </provider>
593 |     </entry>
594 |     <entry file="file://$PROJECT_DIR$/dataset/__init__.py">
595 |       <provider selected="true" editor-type-id="text-editor" />
596 |     </entry>
597 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/utils/data/dataloader.py">
598 |       <provider selected="true" editor-type-id="text-editor">
599 |         <state relative-caret-position="240">
600 |           <caret line="449" column="8" selection-start-line="449" selection-start-column="8" selection-end-line="449" selection-end-column="8" />
601 |         </state>
602 |       </provider>
603 |     </entry>
604 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/optim/rmsprop.py">
605 |       <provider selected="true" editor-type-id="text-editor">
606 |         <state relative-caret-position="202">
607 |           <caret line="27" column="8" selection-start-line="27" selection-start-column="8" selection-end-line="27" selection-end-column="8" />
608 |         </state>
609 |       </provider>
610 |     </entry>
611 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/nn/functional.py">
612 |       <provider selected="true" editor-type-id="text-editor">
613 |         <state relative-caret-position="373">
614 |           <caret line="1614" column="46" selection-start-line="1614" selection-start-column="46" selection-end-line="1614" selection-end-column="46" />
615 |         </state>
616 |       </provider>
617 |     </entry>
618 |     <entry file="file://$APPLICATION_HOME_DIR$/helpers/typeshed/stdlib/2and3/argparse.pyi">
619 |       <provider selected="true" editor-type-id="text-editor" />
620 |     </entry>
621 |     <entry file="file://$PROJECT_DIR$/model/model_tf.py" />
622 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/imgaug/imgaug.py">
623 |       <provider selected="true" editor-type-id="text-editor">
624 |         <state relative-caret-position="251">
625 |           <caret line="471" lean-forward="true" selection-start-line="471" selection-end-line="471" />
626 |         </state>
627 |       </provider>
628 |     </entry>
629 |     <entry file="file://$PROJECT_DIR$/test_.jpg" />
630 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/936520103/argparse.py">
631 |       <provider selected="true" editor-type-id="text-editor">
632 |         <state relative-caret-position="696">
633 |           <caret line="1333" column="60" lean-forward="true" selection-start-line="1333" selection-start-column="60" selection-end-line="1333" selection-end-column="60" />
634 |         </state>
635 |       </provider>
636 |     </entry>
637 |     <entry file="file://$PROJECT_DIR$/test.jpg" />
638 |     <entry file="file://$PROJECT_DIR$/xxxxxxxxxxxxxxxxx.py" />
639 |     <entry file="file://$PROJECT_DIR$/sample_train.py" />
640 |     <entry file="file://$PROJECT_DIR$/0001TP_007170.png" />
641 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/functional.py">
642 |       <provider selected="true" editor-type-id="text-editor">
643 |         <state relative-caret-position="392">
644 |           <caret line="364" column="53" lean-forward="true" selection-start-line="364" selection-start-column="53" selection-end-line="364" selection-end-column="53" />
645 |         </state>
646 |       </provider>
647 |     </entry>
648 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/numpy/core/fromnumeric.py">
649 |       <provider selected="true" editor-type-id="text-editor">
650 |         <state relative-caret-position="107">
651 |           <caret line="82" selection-start-line="82" selection-end-line="82" />
652 |         </state>
653 |       </provider>
654 |     </entry>
655 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/PIL/Image.py">
656 |       <provider selected="true" editor-type-id="text-editor">
657 |         <state relative-caret-position="278">
658 |           <caret line="1086" column="23" lean-forward="true" selection-start-line="1086" selection-start-column="23" selection-end-line="1086" selection-end-column="23" />
659 |         </state>
660 |       </provider>
661 |     </entry>
662 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/python_stubs/-1473333495/cv2/cv2/__init__.py">
663 |       <provider selected="true" editor-type-id="text-editor">
664 |         <state relative-caret-position="139">
665 |           <caret line="9162" column="30" lean-forward="true" selection-start-line="9162" selection-start-column="30" selection-end-line="9162" selection-end-column="30" />
666 |         </state>
667 |       </provider>
668 |     </entry>
669 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torch/nn/modules/module.py">
670 |       <provider selected="true" editor-type-id="text-editor">
671 |         <state relative-caret-position="19">
672 |           <caret line="476" selection-start-line="476" selection-end-line="476" />
673 |         </state>
674 |       </provider>
675 |     </entry>
676 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torchvision/transforms/functional.py">
677 |       <provider selected="true" editor-type-id="text-editor">
678 |         <state relative-caret-position="42">
679 |           <caret line="171" column="50" selection-start-line="171" selection-start-column="36" selection-end-line="171" selection-end-column="50" />
680 |         </state>
681 |       </provider>
682 |     </entry>
683 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/imgaug/augmenters/size.py">
684 |       <provider selected="true" editor-type-id="text-editor">
685 |         <state relative-caret-position="-376">
686 |           <caret line="71" column="78" lean-forward="true" selection-start-line="71" selection-start-column="78" selection-end-line="71" selection-end-column="78" />
687 |         </state>
688 |       </provider>
689 |     </entry>
690 |     <entry file="file://$USER_HOME$/.PyCharm2018.2/system/remote_sources/-1473333495/-1135806901/torchvision/transforms/transforms.py">
691 |       <provider selected="true" editor-type-id="text-editor">
692 |         <state relative-caret-position="836">
693 |           <caret line="161" column="58" selection-start-line="161" selection-start-column="29" selection-end-line="161" selection-end-column="58" />
694 |         </state>
695 |       </provider>
696 |     </entry>
697 |     <entry file="file://$PROJECT_DIR$/demo.png">
698 |       <provider selected="true" editor-type-id="images" />
699 |     </entry>
700 |     <entry file="file://$PROJECT_DIR$/0016E5_06210_L.png" />
701 |     <entry file="file://$PROJECT_DIR$/exp.py" />
702 |     <entry file="file://$PROJECT_DIR$/eval.py">
703 |       <provider selected="true" editor-type-id="text-editor">
704 |         <state relative-caret-position="304">
705 |           <caret line="24" column="41" selection-start-line="24" selection-start-column="41" selection-end-line="24" selection-end-column="41" />
706 |           <folding>
707 |             <element signature="e#0#33#0" expanded="true" />
708 |           </folding>
709 |         </state>
710 |       </provider>
711 |     </entry>
712 |     <entry file="file://$PROJECT_DIR$/demo.py">
713 |       <provider selected="true" editor-type-id="text-editor">
714 |         <state relative-caret-position="646">
715 |           <caret line="44" column="85" selection-start-line="44" selection-start-column="85" selection-end-line="44" selection-end-column="85" />
716 |           <folding>
717 |             <element signature="e#0#10#0" expanded="true" />
718 |           </folding>
719 |         </state>
720 |       </provider>
721 |     </entry>
722 |     <entry file="file://$PROJECT_DIR$/dataset/CamVid.py">
723 |       <provider selected="true" editor-type-id="text-editor">
724 |         <state relative-caret-position="1273">
725 |           <caret line="79" column="43" selection-start-line="79" selection-start-column="43" selection-end-line="79" selection-end-column="43" />
726 |           <folding>
727 |             <element signature="e#0#12#0" expanded="true" />
728 |           </folding>
729 |         </state>
730 |       </provider>
731 |     </entry>
732 |     <entry file="file://$PROJECT_DIR$/utils.py">
733 |       <provider selected="true" editor-type-id="text-editor">
734 |         <state relative-caret-position="1615">
735 |           <caret line="90" column="24" selection-start-line="90" selection-start-column="24" selection-end-line="90" selection-end-column="24" />
736 |           <folding>
737 |             <element signature="e#0#21#0" expanded="true" />
738 |           </folding>
739 |         </state>
740 |       </provider>
741 |     </entry>
742 |     <entry file="file://$PROJECT_DIR$/model/build_contextpath.py">
743 |       <provider selected="true" editor-type-id="text-editor">
744 |         <state relative-caret-position="412">
745 |           <caret line="67" column="34" selection-start-line="67" selection-start-column="34" selection-end-line="67" selection-end-column="34" />
746 |           <folding>
747 |             <element signature="e#0#12#0" expanded="true" />
748 |           </folding>
749 |         </state>
750 |       </provider>
751 |     </entry>
752 |     <entry file="file://$PROJECT_DIR$/model/build_BiSeNet.py">
753 |       <provider selected="true" editor-type-id="text-editor">
754 |         <state relative-caret-position="469">
755 |           <caret line="164" selection-start-line="164" selection-end-line="164" />
756 |           <folding>
757 |             <element signature="e#0#12#0" expanded="true" />
758 |           </folding>
759 |         </state>
760 |       </provider>
761 |     </entry>
762 |     <entry file="file://$PROJECT_DIR$/train.py">
763 |       <provider selected="true" editor-type-id="text-editor">
764 |         <state relative-caret-position="589">
765 |           <caret line="173" column="30" selection-start-line="173" selection-start-column="30" selection-end-line="173" selection-end-column="30" />
766 |           <folding>
767 |             <element signature="e#0#15#0" expanded="true" />
768 |           </folding>
769 |         </state>
770 |       </provider>
771 |     </entry>
772 |   </component>
773 | </project>


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
  1 | # Creative Commons Attribution-NonCommercial 4.0 International
  2 | 
  3 | Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible.
  4 | 
  5 | ### Using Creative Commons Public Licenses
  6 | 
  7 | Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses.
  8 | 
  9 | * __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors).
 10 | 
 11 | * __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees).
 12 | 
 13 | ## Creative Commons Attribution-NonCommercial 4.0 International Public License
 14 | 
 15 | By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions.
 16 | 
 17 | ### Section 1 – Definitions.
 18 | 
 19 | a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image.
 20 | 
 21 | b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License.
 22 | 
 23 | c. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights.
 24 | 
 25 | d. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements.
 26 | 
 27 | e. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material.
 28 | 
 29 | f. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License.
 30 | 
 31 | g. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license.
 32 | 
 33 | h. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License.
 34 | 
 35 | i. __NonCommercial__ means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange.
 36 | 
 37 | j. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them.
 38 | 
 39 | k. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world.
 40 | 
 41 | l. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning.
 42 | 
 43 | ### Section 2 – Scope.
 44 | 
 45 | a. ___License grant.___
 46 | 
 47 |    1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to:
 48 | 
 49 |        A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and
 50 | 
 51 |        B. produce, reproduce, and Share Adapted Material for NonCommercial purposes only.
 52 | 
 53 |    2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions.
 54 | 
 55 |    3. __Term.__ The term of this Public License is specified in Section 6(a).
 56 | 
 57 |    4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material.
 58 | 
 59 |    5. __Downstream recipients.__
 60 | 
 61 |         A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License.
 62 | 
 63 |         B. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material.
 64 | 
 65 |    6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i).
 66 | 
 67 | b. ___Other rights.___
 68 | 
 69 |    1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise.
 70 | 
 71 |    2. Patent and trademark rights are not licensed under this Public License.
 72 | 
 73 |    3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties, including when the Licensed Material is used other than for NonCommercial purposes.
 74 | 
 75 | ### Section 3 – License Conditions.
 76 | 
 77 | Your exercise of the Licensed Rights is expressly made subject to the following conditions.
 78 | 
 79 | a. ___Attribution.___
 80 | 
 81 |    1. If You Share the Licensed Material (including in modified form), You must:
 82 | 
 83 |        A. retain the following if it is supplied by the Licensor with the Licensed Material:
 84 | 
 85 |          i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated);
 86 | 
 87 |          ii. a copyright notice;
 88 | 
 89 |          iii. a notice that refers to this Public License;
 90 | 
 91 |          iv. a notice that refers to the disclaimer of warranties;
 92 | 
 93 |          v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable;
 94 | 
 95 |        B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and
 96 | 
 97 |        C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License.
 98 | 
 99 |    2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information.
100 | 
101 |    3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable.
102 | 
103 |    4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License.
104 | 
105 | ### Section 4 – Sui Generis Database Rights.
106 | 
107 | Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material:
108 | 
109 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only;
110 | 
111 | b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and
112 | 
113 | c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database.
114 | 
115 | For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights.
116 | 
117 | ### Section 5 – Disclaimer of Warranties and Limitation of Liability.
118 | 
119 | a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__
120 | 
121 | b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__
122 | 
123 | c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability.
124 | 
125 | ### Section 6 – Term and Termination.
126 | 
127 | a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically.
128 | 
129 | b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates:
130 | 
131 |    1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or
132 | 
133 |    2. upon express reinstatement by the Licensor.
134 | 
135 |    For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License.
136 | 
137 | c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License.
138 | 
139 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
140 | 
141 | ### Section 7 – Other Terms and Conditions.
142 | 
143 | a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed.
144 | 
145 | b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License.
146 | 
147 | ### Section 8 – Interpretation.
148 | 
149 | a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License.
150 | 
151 | b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions.
152 | 
153 | c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor.
154 | 
155 | d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority.
156 | 
157 | > Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses.
158 | >
159 | > Creative Commons may be contacted at creativecommons.org
160 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # UnpromptedControl
 2 | 
 3 | **By sponsoring me, you're not just supporting my work - you're helping to create a more collaborative, innovative open source community 💖 [sponsor](https://github.com/sponsors/vijishmadhavan?o=sd&sc=t).**
 4 | 
 5 | [Get more updates on Twitter](https://twitter.com/Vijish68859437)
 6 | 
 7 | ControlNet is a highly regarded tool for guiding StableDiffusion models, and it has been widely acknowledged for its effectiveness. In this repository, A simple hack that allows for the restoration or removal of objects without requiring user prompts. By leveraging this approach, the workflow can be significantly streamlined, leading to enhanced process efficiency.
 8 | 
 9 | ## No-prompt
10 | 
11 | [<img src="https://colab.research.google.com/assets/colab-badge.svg" align="center">](https://colab.research.google.com/github/vijishmadhavan/UnpromptedControl/blob/master/UnpromptedControl.ipynb)
12 | 
13 | ![restore Result](examples/eg2gif.gif)
14 | ![restore Result](examples/objgif.gif)
15 | ## Image Restoration 
16 | 
17 | In this image restoration is accomplished using the controlnet-canny and stable-diffusion-2-inpainting techniques, with only "" blank input prompts. Additionally, for automatic scratch segmentation, the FT_Epoch_latest.pt model is being used. However, if the segmentation output is not satisfactory, it is possible to manually sketch and refine the mask to achieve better results. As ControlNet model is trained on pairs of images, one of which has missing parts, and it learns to predict the missing parts based on the content of the complete image.
18 | 
19 | ![restore Result](examples/eg1.jpg)
20 | 
21 | ![restore Result](examples/eg2.jpg)
22 | 
23 | ## Object Removal
24 | 
25 | Automatically removing objects from images is a challenging task that requires a combination of computer vision and deep learning techniques. This code leverages the power of OpenCV inpainting, deep learning-based image restoration, and blending techniques to achieve this task automatically, without the need for user prompts. The ControlNetModel and StableDiffusionInpaintPipeline models play a crucial role in guiding the inpainting process and restoring the image to a more natural-looking state. Overall, this code provides an efficient and effective way to remove unwanted objects from images and produce natural-looking results that are consistent with the surrounding image content. 
26 | 
27 | **"Surely, it has its limitations and might fail with certain images, especially those of faces, and may require some back and forth. To obtain good results, we need to mask not only the object but also its shadow."**
28 | 
29 | 
30 | ![restore Result](examples/obj2.jpg)
31 | ![restore Result](examples/obj1.jpg)
32 | 
33 | ## Limitation
34 | 
35 | - Limited Generalization: The algorithm currently has limitations when it comes to processing images of people's faces and bodies. It may not work as expected for these types of images, and additional work is needed to improve its performance in these areas.
36 | 
37 | - When it comes to removing an object from an image, it's important to consider the surrounding environment and any elements that may be affected by the removal process. In some cases, removing an object may require the removal of a large area surrounding the object, including its shadows.
38 | 
39 | - To obtain good results, we need to mask not only the object but also its shadow.
40 | 
41 | ## Acknowledgements
42 | 
43 | https://github.com/microsoft/Bringing-Old-Photos-Back-to-Life (Segmentation)
44 | 
45 | https://huggingface.co/thibaud/controlnet-sd21
46 | 
47 | https://github.com/lllyasviel/ControlNet
48 | 
49 | 
50 | 


--------------------------------------------------------------------------------
/__pycache__/scratch_detection.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/__pycache__/scratch_detection.cpython-311.pyc


--------------------------------------------------------------------------------
/dataset/CamVid.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import glob
  3 | import os
  4 | from torchvision import transforms
  5 | import cv2
  6 | from PIL import Image
  7 | import pandas as pd
  8 | import numpy as np
  9 | from imgaug import augmenters as iaa
 10 | import imgaug as ia
 11 | from utils import get_label_info, one_hot_it, RandomCrop, reverse_one_hot, one_hot_it_v11, one_hot_it_v11_dice
 12 | import random
 13 | 
 14 | def augmentation():
 15 |     # augment images with spatial transformation: Flip, Affine, Rotation, etc...
 16 |     # see https://github.com/aleju/imgaug for more details
 17 |     pass
 18 | 
 19 | 
 20 | def augmentation_pixel():
 21 |     # augment images with pixel intensity transformation: GaussianBlur, Multiply, etc...
 22 |     pass
 23 | 
 24 | class CamVid(torch.utils.data.Dataset):
 25 |     def __init__(self, image_path, label_path, csv_path, scale, loss='dice', mode='train'):
 26 |         super().__init__()
 27 |         self.mode = mode
 28 |         self.image_list = []
 29 |         if not isinstance(image_path, list):
 30 |             image_path = [image_path]
 31 |         for image_path_ in image_path:
 32 |             self.image_list.extend(glob.glob(os.path.join(image_path_, '*.png')))
 33 |         self.image_list.sort()
 34 |         self.label_list = []
 35 |         if not isinstance(label_path, list):
 36 |             label_path = [label_path]
 37 |         for label_path_ in label_path:
 38 |             self.label_list.extend(glob.glob(os.path.join(label_path_, '*.png')))
 39 |         self.label_list.sort()
 40 |         # self.image_name = [x.split('/')[-1].split('.')[0] for x in self.image_list]
 41 |         # self.label_list = [os.path.join(label_path, x + '_L.png') for x in self.image_list]
 42 |         self.fliplr = iaa.Fliplr(0.5)
 43 |         self.label_info = get_label_info(csv_path)
 44 |         # resize
 45 |         # self.resize_label = transforms.Resize(scale, Image.NEAREST)
 46 |         # self.resize_img = transforms.Resize(scale, Image.BILINEAR)
 47 |         # normalization
 48 |         self.to_tensor = transforms.Compose([
 49 |             transforms.ToTensor(),
 50 |             transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
 51 |             ])
 52 |         # self.crop = transforms.RandomCrop(scale, pad_if_needed=True)
 53 |         self.image_size = scale
 54 |         self.scale = [0.5, 1, 1.25, 1.5, 1.75, 2]
 55 |         self.loss = loss
 56 | 
 57 |     def __getitem__(self, index):
 58 |         # load image and crop
 59 |         seed = random.random()
 60 |         img = Image.open(self.image_list[index])
 61 |         # random crop image
 62 |         # =====================================
 63 |         # w,h = img.size
 64 |         # th, tw = self.scale
 65 |         # i = random.randint(0, h - th)
 66 |         # j = random.randint(0, w - tw)
 67 |         # img = F.crop(img, i, j, th, tw)
 68 |         # =====================================
 69 | 
 70 |         scale = random.choice(self.scale)
 71 |         scale = (int(self.image_size[0] * scale), int(self.image_size[1] * scale))
 72 | 
 73 |         # randomly resize image and random crop
 74 |         # =====================================
 75 |         if self.mode == 'train':
 76 |             img = transforms.Resize(scale, Image.BILINEAR)(img)
 77 |             img = RandomCrop(self.image_size, seed, pad_if_needed=True)(img)
 78 |         # =====================================
 79 | 
 80 |         img = np.array(img)
 81 |         # load label
 82 |         label = Image.open(self.label_list[index])
 83 | 
 84 | 
 85 |         # crop the corresponding label
 86 |         # =====================================
 87 |         # label = F.crop(label, i, j, th, tw)
 88 |         # =====================================
 89 | 
 90 |         # randomly resize label and random crop
 91 |         # =====================================
 92 |         if self.mode == 'train':
 93 |             label = transforms.Resize(scale, Image.NEAREST)(label)
 94 |             label = RandomCrop(self.image_size, seed, pad_if_needed=True)(label)
 95 |         # =====================================
 96 | 
 97 |         label = np.array(label)
 98 | 
 99 | 
100 |         # augment image and label
101 |         if self.mode == 'train':
102 |             seq_det = self.fliplr.to_deterministic()
103 |             img = seq_det.augment_image(img)
104 |             label = seq_det.augment_image(label)
105 | 
106 | 
107 |         # image -> [C, H, W]
108 |         img = Image.fromarray(img)
109 |         img = self.to_tensor(img).float()
110 | 
111 |         if self.loss == 'dice':
112 |             # label -> [num_classes, H, W]
113 |             label = one_hot_it_v11_dice(label, self.label_info).astype(np.uint8)
114 | 
115 |             label = np.transpose(label, [2, 0, 1]).astype(np.float32)
116 |             # label = label.astype(np.float32)
117 |             label = torch.from_numpy(label)
118 | 
119 |             return img, label
120 | 
121 |         elif self.loss == 'crossentropy':
122 |             label = one_hot_it_v11(label, self.label_info).astype(np.uint8)
123 |             # label = label.astype(np.float32)
124 |             label = torch.from_numpy(label).long()
125 | 
126 |             return img, label
127 | 
128 |     def __len__(self):
129 |         return len(self.image_list)
130 | 
131 | 
132 | if __name__ == '__main__':
133 |     # data = CamVid('/path/to/CamVid/train', '/path/to/CamVid/train_labels', '/path/to/CamVid/class_dict.csv', (640, 640))
134 |     data = CamVid(['/data/sqy/CamVid/train', '/data/sqy/CamVid/val'],
135 |                   ['/data/sqy/CamVid/train_labels', '/data/sqy/CamVid/val_labels'], '/data/sqy/CamVid/class_dict.csv',
136 |                   (720, 960), loss='crossentropy', mode='val')
137 |     from model.build_BiSeNet import BiSeNet
138 |     from utils import reverse_one_hot, get_label_info, colour_code_segmentation, compute_global_accuracy
139 | 
140 |     label_info = get_label_info('/data/sqy/CamVid/class_dict.csv')
141 |     for i, (img, label) in enumerate(data):
142 |         print(label.size())
143 |         print(torch.max(label))
144 | 
145 | 


--------------------------------------------------------------------------------
/dataset/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/dataset/__init__.py


--------------------------------------------------------------------------------
/dataset/__pycache__/CamVid.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/dataset/__pycache__/CamVid.cpython-36.pyc


--------------------------------------------------------------------------------
/dataset/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/dataset/__pycache__/__init__.cpython-36.pyc


--------------------------------------------------------------------------------
/demo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/demo.png


--------------------------------------------------------------------------------
/demo.py:
--------------------------------------------------------------------------------
 1 | import cv2
 2 | import argparse
 3 | from model.build_BiSeNet import BiSeNet
 4 | import os
 5 | import torch
 6 | import cv2
 7 | from imgaug import augmenters as iaa
 8 | from PIL import Image
 9 | from torchvision import transforms
10 | import numpy as np
11 | from utils import reverse_one_hot, get_label_info, colour_code_segmentation
12 | 
13 | def predict_on_image(model, args):
14 |     # pre-processing on image
15 |     image = cv2.imread(args.data, -1)
16 |     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
17 |     resize = iaa.Scale({'height': args.crop_height, 'width': args.crop_width})
18 |     resize_det = resize.to_deterministic()
19 |     image = resize_det.augment_image(image)
20 |     image = Image.fromarray(image).convert('RGB')
21 |     image = transforms.ToTensor()(image)
22 |     image = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image).unsqueeze(0)
23 |     # read csv label path
24 |     label_info = get_label_info(args.csv_path)
25 |     # predict
26 |     model.eval()
27 |     predict = model(image).squeeze()
28 |     predict = reverse_one_hot(predict)
29 |     predict = colour_code_segmentation(np.array(predict), label_info)
30 |     predict = cv2.resize(np.uint8(predict), (960, 720))
31 |     cv2.imwrite(args.save_path, cv2.cvtColor(np.uint8(predict), cv2.COLOR_RGB2BGR))
32 | 
33 | def main(params):
34 |     # basic parameters
35 |     parser = argparse.ArgumentParser()
36 |     parser.add_argument('--image', action='store_true', default=False, help='predict on image')
37 |     parser.add_argument('--video', action='store_true', default=False, help='predict on video')
38 |     parser.add_argument('--checkpoint_path', type=str, default=None, help='The path to the pretrained weights of model')
39 |     parser.add_argument('--context_path', type=str, default="resnet101", help='The context path model you are using.')
40 |     parser.add_argument('--num_classes', type=int, default=12, help='num of object classes (with void)')
41 |     parser.add_argument('--data', type=str, default=None, help='Path to image or video for prediction')
42 |     parser.add_argument('--crop_height', type=int, default=720, help='Height of cropped/resized input image to network')
43 |     parser.add_argument('--crop_width', type=int, default=960, help='Width of cropped/resized input image to network')
44 |     parser.add_argument('--cuda', type=str, default='0', help='GPU ids used for training')
45 |     parser.add_argument('--use_gpu', type=bool, default=True, help='Whether to user gpu for training')
46 |     parser.add_argument('--csv_path', type=str, default=None, required=True, help='Path to label info csv file')
47 |     parser.add_argument('--save_path', type=str, default=None, required=True, help='Path to save predict image')
48 | 
49 | 
50 |     args = parser.parse_args(params)
51 | 
52 |     # build model
53 |     os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda
54 |     model = BiSeNet(args.num_classes, args.context_path)
55 |     if torch.cuda.is_available() and args.use_gpu:
56 |         model = torch.nn.DataParallel(model).cuda()
57 | 
58 |     # load pretrained model if exists
59 |     print('load model from %s ...' % args.checkpoint_path)
60 |     model.module.load_state_dict(torch.load(args.checkpoint_path))
61 |     print('Done!')
62 | 
63 |     # predict on image
64 |     if args.image:
65 |         predict_on_image(model, args)
66 | 
67 |     # predict on video
68 |     if args.video:
69 |         pass
70 | 
71 | if __name__ == '__main__':
72 |     params = [
73 |         '--image',
74 |         '--data', 'exp.png',
75 |         '--checkpoint_path', '/path/to/ckpt',
76 |         '--cuda', '0',
77 |         '--csv_path', '/data/sqy/CamVid/class_dict.csv',
78 |         '--save_path', 'demo.png',
79 |         '--context_path', 'resnet18'
80 |     ]
81 |     main(params)


--------------------------------------------------------------------------------
/detection_models/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/__init__.py


--------------------------------------------------------------------------------
/detection_models/__pycache__/__init__.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/__pycache__/__init__.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_models/__pycache__/antialiasing.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/__pycache__/antialiasing.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_models/__pycache__/networks.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/__pycache__/networks.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_models/antialiasing.py:
--------------------------------------------------------------------------------
 1 | # Copyright (c) Microsoft Corporation.
 2 | # Licensed under the MIT License.
 3 | 
 4 | import torch
 5 | import torch.nn.parallel
 6 | import numpy as np
 7 | import torch.nn as nn
 8 | import torch.nn.functional as F
 9 | 
10 | 
11 | class Downsample(nn.Module):
12 |     # https://github.com/adobe/antialiased-cnns
13 | 
14 |     def __init__(self, pad_type="reflect", filt_size=3, stride=2, channels=None, pad_off=0):
15 |         super(Downsample, self).__init__()
16 |         self.filt_size = filt_size
17 |         self.pad_off = pad_off
18 |         self.pad_sizes = [
19 |             int(1.0 * (filt_size - 1) / 2),
20 |             int(np.ceil(1.0 * (filt_size - 1) / 2)),
21 |             int(1.0 * (filt_size - 1) / 2),
22 |             int(np.ceil(1.0 * (filt_size - 1) / 2)),
23 |         ]
24 |         self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes]
25 |         self.stride = stride
26 |         self.off = int((self.stride - 1) / 2.0)
27 |         self.channels = channels
28 | 
29 |         # print('Filter size [%i]'%filt_size)
30 |         if self.filt_size == 1:
31 |             a = np.array([1.0,])
32 |         elif self.filt_size == 2:
33 |             a = np.array([1.0, 1.0])
34 |         elif self.filt_size == 3:
35 |             a = np.array([1.0, 2.0, 1.0])
36 |         elif self.filt_size == 4:
37 |             a = np.array([1.0, 3.0, 3.0, 1.0])
38 |         elif self.filt_size == 5:
39 |             a = np.array([1.0, 4.0, 6.0, 4.0, 1.0])
40 |         elif self.filt_size == 6:
41 |             a = np.array([1.0, 5.0, 10.0, 10.0, 5.0, 1.0])
42 |         elif self.filt_size == 7:
43 |             a = np.array([1.0, 6.0, 15.0, 20.0, 15.0, 6.0, 1.0])
44 | 
45 |         filt = torch.Tensor(a[:, None] * a[None, :])
46 |         filt = filt / torch.sum(filt)
47 |         self.register_buffer("filt", filt[None, None, :, :].repeat((self.channels, 1, 1, 1)))
48 | 
49 |         self.pad = get_pad_layer(pad_type)(self.pad_sizes)
50 | 
51 |     def forward(self, inp):
52 |         if self.filt_size == 1:
53 |             if self.pad_off == 0:
54 |                 return inp[:, :, :: self.stride, :: self.stride]
55 |             else:
56 |                 return self.pad(inp)[:, :, :: self.stride, :: self.stride]
57 |         else:
58 |             return F.conv2d(self.pad(inp), self.filt, stride=self.stride, groups=inp.shape[1])
59 | 
60 | 
61 | def get_pad_layer(pad_type):
62 |     if pad_type in ["refl", "reflect"]:
63 |         PadLayer = nn.ReflectionPad2d
64 |     elif pad_type in ["repl", "replicate"]:
65 |         PadLayer = nn.ReplicationPad2d
66 |     elif pad_type == "zero":
67 |         PadLayer = nn.ZeroPad2d
68 |     else:
69 |         print("Pad type [%s] not recognized" % pad_type)
70 |     return PadLayer
71 | 


--------------------------------------------------------------------------------
/detection_models/networks.py:
--------------------------------------------------------------------------------
  1 | # Copyright (c) Microsoft Corporation.
  2 | # Licensed under the MIT License.
  3 | 
  4 | import torch
  5 | import torch.nn as nn
  6 | import torch.nn.functional as F
  7 | from detection_models.sync_batchnorm import DataParallelWithCallback
  8 | from detection_models.antialiasing import Downsample
  9 | 
 10 | 
 11 | class UNet(nn.Module):
 12 |     def __init__(
 13 |         self,
 14 |         in_channels=3,
 15 |         out_channels=3,
 16 |         depth=5,
 17 |         conv_num=2,
 18 |         wf=6,
 19 |         padding=True,
 20 |         batch_norm=True,
 21 |         up_mode="upsample",
 22 |         with_tanh=False,
 23 |         sync_bn=True,
 24 |         antialiasing=True,
 25 |     ):
 26 |         """
 27 | 		Implementation of
 28 | 		U-Net: Convolutional Networks for Biomedical Image Segmentation
 29 | 		(Ronneberger et al., 2015)
 30 | 		https://arxiv.org/abs/1505.04597
 31 | 		Using the default arguments will yield the exact version used
 32 | 		in the original paper
 33 | 		Args:
 34 | 			in_channels (int): number of input channels
 35 | 			out_channels (int): number of output channels
 36 | 			depth (int): depth of the network
 37 | 			wf (int): number of filters in the first layer is 2**wf
 38 | 			padding (bool): if True, apply padding such that the input shape
 39 | 							is the same as the output.
 40 | 							This may introduce artifacts
 41 | 			batch_norm (bool): Use BatchNorm after layers with an
 42 | 							   activation function
 43 | 			up_mode (str): one of 'upconv' or 'upsample'.
 44 | 						   'upconv' will use transposed convolutions for
 45 | 						   learned upsampling.
 46 | 						   'upsample' will use bilinear upsampling.
 47 | 		"""
 48 |         super().__init__()
 49 |         assert up_mode in ("upconv", "upsample")
 50 |         self.padding = padding
 51 |         self.depth = depth - 1
 52 |         prev_channels = in_channels
 53 | 
 54 |         self.first = nn.Sequential(
 55 |             *[nn.ReflectionPad2d(3), nn.Conv2d(in_channels, 2 ** wf, kernel_size=7), nn.LeakyReLU(0.2, True)]
 56 |         )
 57 |         prev_channels = 2 ** wf
 58 | 
 59 |         self.down_path = nn.ModuleList()
 60 |         self.down_sample = nn.ModuleList()
 61 |         for i in range(depth):
 62 |             if antialiasing and depth > 0:
 63 |                 self.down_sample.append(
 64 |                     nn.Sequential(
 65 |                         *[
 66 |                             nn.ReflectionPad2d(1),
 67 |                             nn.Conv2d(prev_channels, prev_channels, kernel_size=3, stride=1, padding=0),
 68 |                             nn.BatchNorm2d(prev_channels),
 69 |                             nn.LeakyReLU(0.2, True),
 70 |                             Downsample(channels=prev_channels, stride=2),
 71 |                         ]
 72 |                     )
 73 |                 )
 74 |             else:
 75 |                 self.down_sample.append(
 76 |                     nn.Sequential(
 77 |                         *[
 78 |                             nn.ReflectionPad2d(1),
 79 |                             nn.Conv2d(prev_channels, prev_channels, kernel_size=4, stride=2, padding=0),
 80 |                             nn.BatchNorm2d(prev_channels),
 81 |                             nn.LeakyReLU(0.2, True),
 82 |                         ]
 83 |                     )
 84 |                 )
 85 |             self.down_path.append(
 86 |                 UNetConvBlock(conv_num, prev_channels, 2 ** (wf + i + 1), padding, batch_norm)
 87 |             )
 88 |             prev_channels = 2 ** (wf + i + 1)
 89 | 
 90 |         self.up_path = nn.ModuleList()
 91 |         for i in reversed(range(depth)):
 92 |             self.up_path.append(
 93 |                 UNetUpBlock(conv_num, prev_channels, 2 ** (wf + i), up_mode, padding, batch_norm)
 94 |             )
 95 |             prev_channels = 2 ** (wf + i)
 96 | 
 97 |         if with_tanh:
 98 |             self.last = nn.Sequential(
 99 |                 *[nn.ReflectionPad2d(1), nn.Conv2d(prev_channels, out_channels, kernel_size=3), nn.Tanh()]
100 |             )
101 |         else:
102 |             self.last = nn.Sequential(
103 |                 *[nn.ReflectionPad2d(1), nn.Conv2d(prev_channels, out_channels, kernel_size=3)]
104 |             )
105 | 
106 |         if sync_bn:
107 |             self = DataParallelWithCallback(self)
108 | 
109 |     def forward(self, x):
110 |         x = self.first(x)
111 | 
112 |         blocks = []
113 |         for i, down_block in enumerate(self.down_path):
114 |             blocks.append(x)
115 |             x = self.down_sample[i](x)
116 |             x = down_block(x)
117 | 
118 |         for i, up in enumerate(self.up_path):
119 |             x = up(x, blocks[-i - 1])
120 | 
121 |         return self.last(x)
122 | 
123 | 
124 | class UNetConvBlock(nn.Module):
125 |     def __init__(self, conv_num, in_size, out_size, padding, batch_norm):
126 |         super(UNetConvBlock, self).__init__()
127 |         block = []
128 | 
129 |         for _ in range(conv_num):
130 |             block.append(nn.ReflectionPad2d(padding=int(padding)))
131 |             block.append(nn.Conv2d(in_size, out_size, kernel_size=3, padding=0))
132 |             if batch_norm:
133 |                 block.append(nn.BatchNorm2d(out_size))
134 |             block.append(nn.LeakyReLU(0.2, True))
135 |             in_size = out_size
136 | 
137 |         self.block = nn.Sequential(*block)
138 | 
139 |     def forward(self, x):
140 |         out = self.block(x)
141 |         return out
142 | 
143 | 
144 | class UNetUpBlock(nn.Module):
145 |     def __init__(self, conv_num, in_size, out_size, up_mode, padding, batch_norm):
146 |         super(UNetUpBlock, self).__init__()
147 |         if up_mode == "upconv":
148 |             self.up = nn.ConvTranspose2d(in_size, out_size, kernel_size=2, stride=2)
149 |         elif up_mode == "upsample":
150 |             self.up = nn.Sequential(
151 |                 nn.Upsample(mode="bilinear", scale_factor=2, align_corners=False),
152 |                 nn.ReflectionPad2d(1),
153 |                 nn.Conv2d(in_size, out_size, kernel_size=3, padding=0),
154 |             )
155 | 
156 |         self.conv_block = UNetConvBlock(conv_num, in_size, out_size, padding, batch_norm)
157 | 
158 |     def center_crop(self, layer, target_size):
159 |         _, _, layer_height, layer_width = layer.size()
160 |         diff_y = (layer_height - target_size[0]) // 2
161 |         diff_x = (layer_width - target_size[1]) // 2
162 |         return layer[:, :, diff_y : (diff_y + target_size[0]), diff_x : (diff_x + target_size[1])]
163 | 
164 |     def forward(self, x, bridge):
165 |         up = self.up(x)
166 |         crop1 = self.center_crop(bridge, up.shape[2:])
167 |         out = torch.cat([up, crop1], 1)
168 |         out = self.conv_block(out)
169 | 
170 |         return out
171 | 
172 | 
173 | class UnetGenerator(nn.Module):
174 |     """Create a Unet-based generator"""
175 | 
176 |     def __init__(self, input_nc, output_nc, num_downs, ngf=64, norm_type="BN", use_dropout=False):
177 |         """Construct a Unet generator
178 | 		Parameters:
179 | 			input_nc (int)  -- the number of channels in input images
180 | 			output_nc (int) -- the number of channels in output images
181 | 			num_downs (int) -- the number of downsamplings in UNet. For example, # if |num_downs| == 7,
182 | 								image of size 128x128 will become of size 1x1 # at the bottleneck
183 | 			ngf (int)       -- the number of filters in the last conv layer
184 | 			norm_layer      -- normalization layer
185 | 		We construct the U-Net from the innermost layer to the outermost layer.
186 | 		It is a recursive process.
187 | 		"""
188 |         super().__init__()
189 |         if norm_type == "BN":
190 |             norm_layer = nn.BatchNorm2d
191 |         elif norm_type == "IN":
192 |             norm_layer = nn.InstanceNorm2d
193 |         else:
194 |             raise NameError("Unknown norm layer")
195 | 
196 |         # construct unet structure
197 |         unet_block = UnetSkipConnectionBlock(
198 |             ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True
199 |         )  # add the innermost layer
200 |         for i in range(num_downs - 5):  # add intermediate layers with ngf * 8 filters
201 |             unet_block = UnetSkipConnectionBlock(
202 |                 ngf * 8,
203 |                 ngf * 8,
204 |                 input_nc=None,
205 |                 submodule=unet_block,
206 |                 norm_layer=norm_layer,
207 |                 use_dropout=use_dropout,
208 |             )
209 |         # gradually reduce the number of filters from ngf * 8 to ngf
210 |         unet_block = UnetSkipConnectionBlock(
211 |             ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer
212 |         )
213 |         unet_block = UnetSkipConnectionBlock(
214 |             ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer
215 |         )
216 |         unet_block = UnetSkipConnectionBlock(
217 |             ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer
218 |         )
219 |         self.model = UnetSkipConnectionBlock(
220 |             output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer
221 |         )  # add the outermost layer
222 | 
223 |     def forward(self, input):
224 |         return self.model(input)
225 | 
226 | 
227 | class UnetSkipConnectionBlock(nn.Module):
228 |     """Defines the Unet submodule with skip connection.
229 | 
230 | 		-------------------identity----------------------
231 | 		|-- downsampling -- |submodule| -- upsampling --|
232 | 	"""
233 | 
234 |     def __init__(
235 |         self,
236 |         outer_nc,
237 |         inner_nc,
238 |         input_nc=None,
239 |         submodule=None,
240 |         outermost=False,
241 |         innermost=False,
242 |         norm_layer=nn.BatchNorm2d,
243 |         use_dropout=False,
244 |     ):
245 |         """Construct a Unet submodule with skip connections.
246 | 		Parameters:
247 | 			outer_nc (int) -- the number of filters in the outer conv layer
248 | 			inner_nc (int) -- the number of filters in the inner conv layer
249 | 			input_nc (int) -- the number of channels in input images/features
250 | 			submodule (UnetSkipConnectionBlock) -- previously defined submodules
251 | 			outermost (bool)    -- if this module is the outermost module
252 | 			innermost (bool)    -- if this module is the innermost module
253 | 			norm_layer          -- normalization layer
254 | 			user_dropout (bool) -- if use dropout layers.
255 | 		"""
256 |         super().__init__()
257 |         self.outermost = outermost
258 |         use_bias = norm_layer == nn.InstanceNorm2d
259 |         if input_nc is None:
260 |             input_nc = outer_nc
261 |         downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4, stride=2, padding=1, bias=use_bias)
262 |         downrelu = nn.LeakyReLU(0.2, True)
263 |         downnorm = norm_layer(inner_nc)
264 |         uprelu = nn.LeakyReLU(0.2, True)
265 |         upnorm = norm_layer(outer_nc)
266 | 
267 |         if outermost:
268 |             upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc, kernel_size=4, stride=2, padding=1)
269 |             down = [downconv]
270 |             up = [uprelu, upconv, nn.Tanh()]
271 |             model = down + [submodule] + up
272 |         elif innermost:
273 |             upconv = nn.ConvTranspose2d(inner_nc, outer_nc, kernel_size=4, stride=2, padding=1, bias=use_bias)
274 |             down = [downrelu, downconv]
275 |             up = [uprelu, upconv, upnorm]
276 |             model = down + up
277 |         else:
278 |             upconv = nn.ConvTranspose2d(
279 |                 inner_nc * 2, outer_nc, kernel_size=4, stride=2, padding=1, bias=use_bias
280 |             )
281 |             down = [downrelu, downconv, downnorm]
282 |             up = [uprelu, upconv, upnorm]
283 | 
284 |             if use_dropout:
285 |                 model = down + [submodule] + up + [nn.Dropout(0.5)]
286 |             else:
287 |                 model = down + [submodule] + up
288 | 
289 |         self.model = nn.Sequential(*model)
290 | 
291 |     def forward(self, x):
292 |         if self.outermost:
293 |             return self.model(x)
294 |         else:  # add skip connections
295 |             return torch.cat([x, self.model(x)], 1)
296 | 
297 | 
298 | # ============================================
299 | # Network testing
300 | # ============================================
301 | if __name__ == "__main__":
302 |     from torchsummary import summary
303 | 
304 |     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
305 | 
306 |     model = UNet_two_decoders(
307 |         in_channels=3,
308 |         out_channels1=3,
309 |         out_channels2=1,
310 |         depth=4,
311 |         conv_num=1,
312 |         wf=6,
313 |         padding=True,
314 |         batch_norm=True,
315 |         up_mode="upsample",
316 |         with_tanh=False,
317 |     )
318 |     model.to(device)
319 | 
320 |     model_pix2pix = UnetGenerator(3, 3, 5, ngf=64, norm_type="BN", use_dropout=False)
321 |     model_pix2pix.to(device)
322 | 
323 |     print("customized unet:")
324 |     summary(model, (3, 256, 256))
325 | 
326 |     print("cyclegan unet:")
327 |     summary(model_pix2pix, (3, 256, 256))
328 | 
329 |     x = torch.zeros(1, 3, 256, 256).requires_grad_(True).cuda()
330 |     g = make_dot(model(x))
331 |     g.render("models/Digraph.gv", view=False)
332 | 
333 | 


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/__init__.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | # File   : __init__.py
 3 | # Author : Jiayuan Mao
 4 | # Email  : maojiayuan@gmail.com
 5 | # Date   : 27/01/2018
 6 | #
 7 | # This file is part of Synchronized-BatchNorm-PyTorch.
 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
 9 | # Distributed under MIT License.
10 | 
11 | from .batchnorm import set_sbn_eps_mode
12 | from .batchnorm import SynchronizedBatchNorm1d, SynchronizedBatchNorm2d, SynchronizedBatchNorm3d
13 | from .batchnorm import patch_sync_batchnorm, convert_model
14 | from .replicate import DataParallelWithCallback, patch_replication_callback
15 | 


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/__pycache__/__init__.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/sync_batchnorm/__pycache__/__init__.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/__pycache__/batchnorm.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/sync_batchnorm/__pycache__/batchnorm.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/__pycache__/comm.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/sync_batchnorm/__pycache__/comm.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/__pycache__/replicate.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_models/sync_batchnorm/__pycache__/replicate.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/batchnorm.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | # File   : batchnorm.py
  3 | # Author : Jiayuan Mao
  4 | # Email  : maojiayuan@gmail.com
  5 | # Date   : 27/01/2018
  6 | #
  7 | # This file is part of Synchronized-BatchNorm-PyTorch.
  8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
  9 | # Distributed under MIT License.
 10 | 
 11 | import collections
 12 | import contextlib
 13 | 
 14 | import torch
 15 | import torch.nn.functional as F
 16 | 
 17 | from torch.nn.modules.batchnorm import _BatchNorm
 18 | 
 19 | try:
 20 |     from torch.nn.parallel._functions import ReduceAddCoalesced, Broadcast
 21 | except ImportError:
 22 |     ReduceAddCoalesced = Broadcast = None
 23 | 
 24 | try:
 25 |     from jactorch.parallel.comm import SyncMaster
 26 |     from jactorch.parallel.data_parallel import JacDataParallel as DataParallelWithCallback
 27 | except ImportError:
 28 |     from .comm import SyncMaster
 29 |     from .replicate import DataParallelWithCallback
 30 | 
 31 | __all__ = [
 32 |     'set_sbn_eps_mode',
 33 |     'SynchronizedBatchNorm1d', 'SynchronizedBatchNorm2d', 'SynchronizedBatchNorm3d',
 34 |     'patch_sync_batchnorm', 'convert_model'
 35 | ]
 36 | 
 37 | 
 38 | SBN_EPS_MODE = 'clamp'
 39 | 
 40 | 
 41 | def set_sbn_eps_mode(mode):
 42 |     global SBN_EPS_MODE
 43 |     assert mode in ('clamp', 'plus')
 44 |     SBN_EPS_MODE = mode
 45 | 
 46 | 
 47 | def _sum_ft(tensor):
 48 |     """sum over the first and last dimention"""
 49 |     return tensor.sum(dim=0).sum(dim=-1)
 50 | 
 51 | 
 52 | def _unsqueeze_ft(tensor):
 53 |     """add new dimensions at the front and the tail"""
 54 |     return tensor.unsqueeze(0).unsqueeze(-1)
 55 | 
 56 | 
 57 | _ChildMessage = collections.namedtuple('_ChildMessage', ['sum', 'ssum', 'sum_size'])
 58 | _MasterMessage = collections.namedtuple('_MasterMessage', ['sum', 'inv_std'])
 59 | 
 60 | 
 61 | class _SynchronizedBatchNorm(_BatchNorm):
 62 |     def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, track_running_stats=True):
 63 |         assert ReduceAddCoalesced is not None, 'Can not use Synchronized Batch Normalization without CUDA support.'
 64 | 
 65 |         super(_SynchronizedBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine,
 66 |                                                      track_running_stats=track_running_stats)
 67 | 
 68 |         if not self.track_running_stats:
 69 |             import warnings
 70 |             warnings.warn('track_running_stats=False is not supported by the SynchronizedBatchNorm.')
 71 | 
 72 |         self._sync_master = SyncMaster(self._data_parallel_master)
 73 | 
 74 |         self._is_parallel = False
 75 |         self._parallel_id = None
 76 |         self._slave_pipe = None
 77 | 
 78 |     def forward(self, input):
 79 |         # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation.
 80 |         if not (self._is_parallel and self.training):
 81 |             return F.batch_norm(
 82 |                 input, self.running_mean, self.running_var, self.weight, self.bias,
 83 |                 self.training, self.momentum, self.eps)
 84 | 
 85 |         # Resize the input to (B, C, -1).
 86 |         input_shape = input.size()
 87 |         assert input.size(1) == self.num_features, 'Channel size mismatch: got {}, expect {}.'.format(input.size(1), self.num_features)
 88 |         input = input.view(input.size(0), self.num_features, -1)
 89 | 
 90 |         # Compute the sum and square-sum.
 91 |         sum_size = input.size(0) * input.size(2)
 92 |         input_sum = _sum_ft(input)
 93 |         input_ssum = _sum_ft(input ** 2)
 94 | 
 95 |         # Reduce-and-broadcast the statistics.
 96 |         if self._parallel_id == 0:
 97 |             mean, inv_std = self._sync_master.run_master(_ChildMessage(input_sum, input_ssum, sum_size))
 98 |         else:
 99 |             mean, inv_std = self._slave_pipe.run_slave(_ChildMessage(input_sum, input_ssum, sum_size))
100 | 
101 |         # Compute the output.
102 |         if self.affine:
103 |             # MJY:: Fuse the multiplication for speed.
104 |             output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std * self.weight) + _unsqueeze_ft(self.bias)
105 |         else:
106 |             output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std)
107 | 
108 |         # Reshape it.
109 |         return output.view(input_shape)
110 | 
111 |     def __data_parallel_replicate__(self, ctx, copy_id):
112 |         self._is_parallel = True
113 |         self._parallel_id = copy_id
114 | 
115 |         # parallel_id == 0 means master device.
116 |         if self._parallel_id == 0:
117 |             ctx.sync_master = self._sync_master
118 |         else:
119 |             self._slave_pipe = ctx.sync_master.register_slave(copy_id)
120 | 
121 |     def _data_parallel_master(self, intermediates):
122 |         """Reduce the sum and square-sum, compute the statistics, and broadcast it."""
123 | 
124 |         # Always using same "device order" makes the ReduceAdd operation faster.
125 |         # Thanks to:: Tete Xiao (http://tetexiao.com/)
126 |         intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device())
127 | 
128 |         to_reduce = [i[1][:2] for i in intermediates]
129 |         to_reduce = [j for i in to_reduce for j in i]  # flatten
130 |         target_gpus = [i[1].sum.get_device() for i in intermediates]
131 | 
132 |         sum_size = sum([i[1].sum_size for i in intermediates])
133 |         sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce)
134 |         mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size)
135 | 
136 |         broadcasted = Broadcast.apply(target_gpus, mean, inv_std)
137 | 
138 |         outputs = []
139 |         for i, rec in enumerate(intermediates):
140 |             outputs.append((rec[0], _MasterMessage(*broadcasted[i*2:i*2+2])))
141 | 
142 |         return outputs
143 | 
144 |     def _compute_mean_std(self, sum_, ssum, size):
145 |         """Compute the mean and standard-deviation with sum and square-sum. This method
146 |         also maintains the moving average on the master device."""
147 |         assert size > 1, 'BatchNorm computes unbiased standard-deviation, which requires size > 1.'
148 |         mean = sum_ / size
149 |         sumvar = ssum - sum_ * mean
150 |         unbias_var = sumvar / (size - 1)
151 |         bias_var = sumvar / size
152 | 
153 |         if hasattr(torch, 'no_grad'):
154 |             with torch.no_grad():
155 |                 self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data
156 |                 self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data
157 |         else:
158 |             self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data
159 |             self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data
160 | 
161 |         if SBN_EPS_MODE == 'clamp':
162 |             return mean, bias_var.clamp(self.eps) ** -0.5
163 |         elif SBN_EPS_MODE == 'plus':
164 |             return mean, (bias_var + self.eps) ** -0.5
165 |         else:
166 |             raise ValueError('Unknown EPS mode: {}.'.format(SBN_EPS_MODE))
167 | 
168 | 
169 | class SynchronizedBatchNorm1d(_SynchronizedBatchNorm):
170 |     r"""Applies Synchronized Batch Normalization over a 2d or 3d input that is seen as a
171 |     mini-batch.
172 | 
173 |     .. math::
174 | 
175 |         y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta
176 | 
177 |     This module differs from the built-in PyTorch BatchNorm1d as the mean and
178 |     standard-deviation are reduced across all devices during training.
179 | 
180 |     For example, when one uses `nn.DataParallel` to wrap the network during
181 |     training, PyTorch's implementation normalize the tensor on each device using
182 |     the statistics only on that device, which accelerated the computation and
183 |     is also easy to implement, but the statistics might be inaccurate.
184 |     Instead, in this synchronized version, the statistics will be computed
185 |     over all training samples distributed on multiple devices.
186 | 
187 |     Note that, for one-GPU or CPU-only case, this module behaves exactly same
188 |     as the built-in PyTorch implementation.
189 | 
190 |     The mean and standard-deviation are calculated per-dimension over
191 |     the mini-batches and gamma and beta are learnable parameter vectors
192 |     of size C (where C is the input size).
193 | 
194 |     During training, this layer keeps a running estimate of its computed mean
195 |     and variance. The running sum is kept with a default momentum of 0.1.
196 | 
197 |     During evaluation, this running mean/variance is used for normalization.
198 | 
199 |     Because the BatchNorm is done over the `C` dimension, computing statistics
200 |     on `(N, L)` slices, it's common terminology to call this Temporal BatchNorm
201 | 
202 |     Args:
203 |         num_features: num_features from an expected input of size
204 |             `batch_size x num_features [x width]`
205 |         eps: a value added to the denominator for numerical stability.
206 |             Default: 1e-5
207 |         momentum: the value used for the running_mean and running_var
208 |             computation. Default: 0.1
209 |         affine: a boolean value that when set to ``True``, gives the layer learnable
210 |             affine parameters. Default: ``True``
211 | 
212 |     Shape::
213 |         - Input: :math:`(N, C)` or :math:`(N, C, L)`
214 |         - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input)
215 | 
216 |     Examples:
217 |         >>> # With Learnable Parameters
218 |         >>> m = SynchronizedBatchNorm1d(100)
219 |         >>> # Without Learnable Parameters
220 |         >>> m = SynchronizedBatchNorm1d(100, affine=False)
221 |         >>> input = torch.autograd.Variable(torch.randn(20, 100))
222 |         >>> output = m(input)
223 |     """
224 | 
225 |     def _check_input_dim(self, input):
226 |         if input.dim() != 2 and input.dim() != 3:
227 |             raise ValueError('expected 2D or 3D input (got {}D input)'
228 |                              .format(input.dim()))
229 | 
230 | 
231 | class SynchronizedBatchNorm2d(_SynchronizedBatchNorm):
232 |     r"""Applies Batch Normalization over a 4d input that is seen as a mini-batch
233 |     of 3d inputs
234 | 
235 |     .. math::
236 | 
237 |         y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta
238 | 
239 |     This module differs from the built-in PyTorch BatchNorm2d as the mean and
240 |     standard-deviation are reduced across all devices during training.
241 | 
242 |     For example, when one uses `nn.DataParallel` to wrap the network during
243 |     training, PyTorch's implementation normalize the tensor on each device using
244 |     the statistics only on that device, which accelerated the computation and
245 |     is also easy to implement, but the statistics might be inaccurate.
246 |     Instead, in this synchronized version, the statistics will be computed
247 |     over all training samples distributed on multiple devices.
248 | 
249 |     Note that, for one-GPU or CPU-only case, this module behaves exactly same
250 |     as the built-in PyTorch implementation.
251 | 
252 |     The mean and standard-deviation are calculated per-dimension over
253 |     the mini-batches and gamma and beta are learnable parameter vectors
254 |     of size C (where C is the input size).
255 | 
256 |     During training, this layer keeps a running estimate of its computed mean
257 |     and variance. The running sum is kept with a default momentum of 0.1.
258 | 
259 |     During evaluation, this running mean/variance is used for normalization.
260 | 
261 |     Because the BatchNorm is done over the `C` dimension, computing statistics
262 |     on `(N, H, W)` slices, it's common terminology to call this Spatial BatchNorm
263 | 
264 |     Args:
265 |         num_features: num_features from an expected input of
266 |             size batch_size x num_features x height x width
267 |         eps: a value added to the denominator for numerical stability.
268 |             Default: 1e-5
269 |         momentum: the value used for the running_mean and running_var
270 |             computation. Default: 0.1
271 |         affine: a boolean value that when set to ``True``, gives the layer learnable
272 |             affine parameters. Default: ``True``
273 | 
274 |     Shape::
275 |         - Input: :math:`(N, C, H, W)`
276 |         - Output: :math:`(N, C, H, W)` (same shape as input)
277 | 
278 |     Examples:
279 |         >>> # With Learnable Parameters
280 |         >>> m = SynchronizedBatchNorm2d(100)
281 |         >>> # Without Learnable Parameters
282 |         >>> m = SynchronizedBatchNorm2d(100, affine=False)
283 |         >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45))
284 |         >>> output = m(input)
285 |     """
286 | 
287 |     def _check_input_dim(self, input):
288 |         if input.dim() != 4:
289 |             raise ValueError('expected 4D input (got {}D input)'
290 |                              .format(input.dim()))
291 | 
292 | 
293 | class SynchronizedBatchNorm3d(_SynchronizedBatchNorm):
294 |     r"""Applies Batch Normalization over a 5d input that is seen as a mini-batch
295 |     of 4d inputs
296 | 
297 |     .. math::
298 | 
299 |         y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta
300 | 
301 |     This module differs from the built-in PyTorch BatchNorm3d as the mean and
302 |     standard-deviation are reduced across all devices during training.
303 | 
304 |     For example, when one uses `nn.DataParallel` to wrap the network during
305 |     training, PyTorch's implementation normalize the tensor on each device using
306 |     the statistics only on that device, which accelerated the computation and
307 |     is also easy to implement, but the statistics might be inaccurate.
308 |     Instead, in this synchronized version, the statistics will be computed
309 |     over all training samples distributed on multiple devices.
310 | 
311 |     Note that, for one-GPU or CPU-only case, this module behaves exactly same
312 |     as the built-in PyTorch implementation.
313 | 
314 |     The mean and standard-deviation are calculated per-dimension over
315 |     the mini-batches and gamma and beta are learnable parameter vectors
316 |     of size C (where C is the input size).
317 | 
318 |     During training, this layer keeps a running estimate of its computed mean
319 |     and variance. The running sum is kept with a default momentum of 0.1.
320 | 
321 |     During evaluation, this running mean/variance is used for normalization.
322 | 
323 |     Because the BatchNorm is done over the `C` dimension, computing statistics
324 |     on `(N, D, H, W)` slices, it's common terminology to call this Volumetric BatchNorm
325 |     or Spatio-temporal BatchNorm
326 | 
327 |     Args:
328 |         num_features: num_features from an expected input of
329 |             size batch_size x num_features x depth x height x width
330 |         eps: a value added to the denominator for numerical stability.
331 |             Default: 1e-5
332 |         momentum: the value used for the running_mean and running_var
333 |             computation. Default: 0.1
334 |         affine: a boolean value that when set to ``True``, gives the layer learnable
335 |             affine parameters. Default: ``True``
336 | 
337 |     Shape::
338 |         - Input: :math:`(N, C, D, H, W)`
339 |         - Output: :math:`(N, C, D, H, W)` (same shape as input)
340 | 
341 |     Examples:
342 |         >>> # With Learnable Parameters
343 |         >>> m = SynchronizedBatchNorm3d(100)
344 |         >>> # Without Learnable Parameters
345 |         >>> m = SynchronizedBatchNorm3d(100, affine=False)
346 |         >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45, 10))
347 |         >>> output = m(input)
348 |     """
349 | 
350 |     def _check_input_dim(self, input):
351 |         if input.dim() != 5:
352 |             raise ValueError('expected 5D input (got {}D input)'
353 |                              .format(input.dim()))
354 | 
355 | 
356 | @contextlib.contextmanager
357 | def patch_sync_batchnorm():
358 |     import torch.nn as nn
359 | 
360 |     backup = nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d
361 | 
362 |     nn.BatchNorm1d = SynchronizedBatchNorm1d
363 |     nn.BatchNorm2d = SynchronizedBatchNorm2d
364 |     nn.BatchNorm3d = SynchronizedBatchNorm3d
365 | 
366 |     yield
367 | 
368 |     nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d = backup
369 | 
370 | 
371 | def convert_model(module):
372 |     """Traverse the input module and its child recursively
373 |        and replace all instance of torch.nn.modules.batchnorm.BatchNorm*N*d
374 |        to SynchronizedBatchNorm*N*d
375 | 
376 |     Args:
377 |         module: the input module needs to be convert to SyncBN model
378 | 
379 |     Examples:
380 |         >>> import torch.nn as nn
381 |         >>> import torchvision
382 |         >>> # m is a standard pytorch model
383 |         >>> m = torchvision.models.resnet18(True)
384 |         >>> m = nn.DataParallel(m)
385 |         >>> # after convert, m is using SyncBN
386 |         >>> m = convert_model(m)
387 |     """
388 |     if isinstance(module, torch.nn.DataParallel):
389 |         mod = module.module
390 |         mod = convert_model(mod)
391 |         mod = DataParallelWithCallback(mod, device_ids=module.device_ids)
392 |         return mod
393 | 
394 |     mod = module
395 |     for pth_module, sync_module in zip([torch.nn.modules.batchnorm.BatchNorm1d,
396 |                                         torch.nn.modules.batchnorm.BatchNorm2d,
397 |                                         torch.nn.modules.batchnorm.BatchNorm3d],
398 |                                        [SynchronizedBatchNorm1d,
399 |                                         SynchronizedBatchNorm2d,
400 |                                         SynchronizedBatchNorm3d]):
401 |         if isinstance(module, pth_module):
402 |             mod = sync_module(module.num_features, module.eps, module.momentum, module.affine)
403 |             mod.running_mean = module.running_mean
404 |             mod.running_var = module.running_var
405 |             if module.affine:
406 |                 mod.weight.data = module.weight.data.clone().detach()
407 |                 mod.bias.data = module.bias.data.clone().detach()
408 | 
409 |     for name, child in module.named_children():
410 |         mod.add_module(name, convert_model(child))
411 | 
412 |     return mod
413 | 


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/batchnorm_reimpl.py:
--------------------------------------------------------------------------------
 1 | #! /usr/bin/env python3
 2 | # -*- coding: utf-8 -*-
 3 | # File   : batchnorm_reimpl.py
 4 | # Author : acgtyrant
 5 | # Date   : 11/01/2018
 6 | #
 7 | # This file is part of Synchronized-BatchNorm-PyTorch.
 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
 9 | # Distributed under MIT License.
10 | 
11 | import torch
12 | import torch.nn as nn
13 | import torch.nn.init as init
14 | 
15 | __all__ = ['BatchNorm2dReimpl']
16 | 
17 | 
18 | class BatchNorm2dReimpl(nn.Module):
19 |     """
20 |     A re-implementation of batch normalization, used for testing the numerical
21 |     stability.
22 | 
23 |     Author: acgtyrant
24 |     See also:
25 |     https://github.com/vacancy/Synchronized-BatchNorm-PyTorch/issues/14
26 |     """
27 |     def __init__(self, num_features, eps=1e-5, momentum=0.1):
28 |         super().__init__()
29 | 
30 |         self.num_features = num_features
31 |         self.eps = eps
32 |         self.momentum = momentum
33 |         self.weight = nn.Parameter(torch.empty(num_features))
34 |         self.bias = nn.Parameter(torch.empty(num_features))
35 |         self.register_buffer('running_mean', torch.zeros(num_features))
36 |         self.register_buffer('running_var', torch.ones(num_features))
37 |         self.reset_parameters()
38 | 
39 |     def reset_running_stats(self):
40 |         self.running_mean.zero_()
41 |         self.running_var.fill_(1)
42 | 
43 |     def reset_parameters(self):
44 |         self.reset_running_stats()
45 |         init.uniform_(self.weight)
46 |         init.zeros_(self.bias)
47 | 
48 |     def forward(self, input_):
49 |         batchsize, channels, height, width = input_.size()
50 |         numel = batchsize * height * width
51 |         input_ = input_.permute(1, 0, 2, 3).contiguous().view(channels, numel)
52 |         sum_ = input_.sum(1)
53 |         sum_of_square = input_.pow(2).sum(1)
54 |         mean = sum_ / numel
55 |         sumvar = sum_of_square - sum_ * mean
56 | 
57 |         self.running_mean = (
58 |                 (1 - self.momentum) * self.running_mean
59 |                 + self.momentum * mean.detach()
60 |         )
61 |         unbias_var = sumvar / (numel - 1)
62 |         self.running_var = (
63 |                 (1 - self.momentum) * self.running_var
64 |                 + self.momentum * unbias_var.detach()
65 |         )
66 | 
67 |         bias_var = sumvar / numel
68 |         inv_std = 1 / (bias_var + self.eps).pow(0.5)
69 |         output = (
70 |                 (input_ - mean.unsqueeze(1)) * inv_std.unsqueeze(1) *
71 |                 self.weight.unsqueeze(1) + self.bias.unsqueeze(1))
72 | 
73 |         return output.view(channels, batchsize, height, width).permute(1, 0, 2, 3).contiguous()
74 | 
75 | 


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/comm.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | # File   : comm.py
  3 | # Author : Jiayuan Mao
  4 | # Email  : maojiayuan@gmail.com
  5 | # Date   : 27/01/2018
  6 | # 
  7 | # This file is part of Synchronized-BatchNorm-PyTorch.
  8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
  9 | # Distributed under MIT License.
 10 | 
 11 | import queue
 12 | import collections
 13 | import threading
 14 | 
 15 | __all__ = ['FutureResult', 'SlavePipe', 'SyncMaster']
 16 | 
 17 | 
 18 | class FutureResult(object):
 19 |     """A thread-safe future implementation. Used only as one-to-one pipe."""
 20 | 
 21 |     def __init__(self):
 22 |         self._result = None
 23 |         self._lock = threading.Lock()
 24 |         self._cond = threading.Condition(self._lock)
 25 | 
 26 |     def put(self, result):
 27 |         with self._lock:
 28 |             assert self._result is None, 'Previous result has\'t been fetched.'
 29 |             self._result = result
 30 |             self._cond.notify()
 31 | 
 32 |     def get(self):
 33 |         with self._lock:
 34 |             if self._result is None:
 35 |                 self._cond.wait()
 36 | 
 37 |             res = self._result
 38 |             self._result = None
 39 |             return res
 40 | 
 41 | 
 42 | _MasterRegistry = collections.namedtuple('MasterRegistry', ['result'])
 43 | _SlavePipeBase = collections.namedtuple('_SlavePipeBase', ['identifier', 'queue', 'result'])
 44 | 
 45 | 
 46 | class SlavePipe(_SlavePipeBase):
 47 |     """Pipe for master-slave communication."""
 48 | 
 49 |     def run_slave(self, msg):
 50 |         self.queue.put((self.identifier, msg))
 51 |         ret = self.result.get()
 52 |         self.queue.put(True)
 53 |         return ret
 54 | 
 55 | 
 56 | class SyncMaster(object):
 57 |     """An abstract `SyncMaster` object.
 58 | 
 59 |     - During the replication, as the data parallel will trigger an callback of each module, all slave devices should
 60 |     call `register(id)` and obtain an `SlavePipe` to communicate with the master.
 61 |     - During the forward pass, master device invokes `run_master`, all messages from slave devices will be collected,
 62 |     and passed to a registered callback.
 63 |     - After receiving the messages, the master device should gather the information and determine to message passed
 64 |     back to each slave devices.
 65 |     """
 66 | 
 67 |     def __init__(self, master_callback):
 68 |         """
 69 | 
 70 |         Args:
 71 |             master_callback: a callback to be invoked after having collected messages from slave devices.
 72 |         """
 73 |         self._master_callback = master_callback
 74 |         self._queue = queue.Queue()
 75 |         self._registry = collections.OrderedDict()
 76 |         self._activated = False
 77 | 
 78 |     def __getstate__(self):
 79 |         return {'master_callback': self._master_callback}
 80 | 
 81 |     def __setstate__(self, state):
 82 |         self.__init__(state['master_callback'])
 83 | 
 84 |     def register_slave(self, identifier):
 85 |         """
 86 |         Register an slave device.
 87 | 
 88 |         Args:
 89 |             identifier: an identifier, usually is the device id.
 90 | 
 91 |         Returns: a `SlavePipe` object which can be used to communicate with the master device.
 92 | 
 93 |         """
 94 |         if self._activated:
 95 |             assert self._queue.empty(), 'Queue is not clean before next initialization.'
 96 |             self._activated = False
 97 |             self._registry.clear()
 98 |         future = FutureResult()
 99 |         self._registry[identifier] = _MasterRegistry(future)
100 |         return SlavePipe(identifier, self._queue, future)
101 | 
102 |     def run_master(self, master_msg):
103 |         """
104 |         Main entry for the master device in each forward pass.
105 |         The messages were first collected from each devices (including the master device), and then
106 |         an callback will be invoked to compute the message to be sent back to each devices
107 |         (including the master device).
108 | 
109 |         Args:
110 |             master_msg: the message that the master want to send to itself. This will be placed as the first
111 |             message when calling `master_callback`. For detailed usage, see `_SynchronizedBatchNorm` for an example.
112 | 
113 |         Returns: the message to be sent back to the master device.
114 | 
115 |         """
116 |         self._activated = True
117 | 
118 |         intermediates = [(0, master_msg)]
119 |         for i in range(self.nr_slaves):
120 |             intermediates.append(self._queue.get())
121 | 
122 |         results = self._master_callback(intermediates)
123 |         assert results[0][0] == 0, 'The first result should belongs to the master.'
124 | 
125 |         for i, res in results:
126 |             if i == 0:
127 |                 continue
128 |             self._registry[i].result.put(res)
129 | 
130 |         for i in range(self.nr_slaves):
131 |             assert self._queue.get() is True
132 | 
133 |         return results[0][1]
134 | 
135 |     @property
136 |     def nr_slaves(self):
137 |         return len(self._registry)
138 | 


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/replicate.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | # File   : replicate.py
 3 | # Author : Jiayuan Mao
 4 | # Email  : maojiayuan@gmail.com
 5 | # Date   : 27/01/2018
 6 | # 
 7 | # This file is part of Synchronized-BatchNorm-PyTorch.
 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
 9 | # Distributed under MIT License.
10 | 
11 | import functools
12 | 
13 | from torch.nn.parallel.data_parallel import DataParallel
14 | 
15 | __all__ = [
16 |     'CallbackContext',
17 |     'execute_replication_callbacks',
18 |     'DataParallelWithCallback',
19 |     'patch_replication_callback'
20 | ]
21 | 
22 | 
23 | class CallbackContext(object):
24 |     pass
25 | 
26 | 
27 | def execute_replication_callbacks(modules):
28 |     """
29 |     Execute an replication callback `__data_parallel_replicate__` on each module created by original replication.
30 | 
31 |     The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)`
32 | 
33 |     Note that, as all modules are isomorphism, we assign each sub-module with a context
34 |     (shared among multiple copies of this module on different devices).
35 |     Through this context, different copies can share some information.
36 | 
37 |     We guarantee that the callback on the master copy (the first copy) will be called ahead of calling the callback
38 |     of any slave copies.
39 |     """
40 |     master_copy = modules[0]
41 |     nr_modules = len(list(master_copy.modules()))
42 |     ctxs = [CallbackContext() for _ in range(nr_modules)]
43 | 
44 |     for i, module in enumerate(modules):
45 |         for j, m in enumerate(module.modules()):
46 |             if hasattr(m, '__data_parallel_replicate__'):
47 |                 m.__data_parallel_replicate__(ctxs[j], i)
48 | 
49 | 
50 | class DataParallelWithCallback(DataParallel):
51 |     """
52 |     Data Parallel with a replication callback.
53 | 
54 |     An replication callback `__data_parallel_replicate__` of each module will be invoked after being created by
55 |     original `replicate` function.
56 |     The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)`
57 | 
58 |     Examples:
59 |         > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False)
60 |         > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1])
61 |         # sync_bn.__data_parallel_replicate__ will be invoked.
62 |     """
63 | 
64 |     def replicate(self, module, device_ids):
65 |         modules = super(DataParallelWithCallback, self).replicate(module, device_ids)
66 |         execute_replication_callbacks(modules)
67 |         return modules
68 | 
69 | 
70 | def patch_replication_callback(data_parallel):
71 |     """
72 |     Monkey-patch an existing `DataParallel` object. Add the replication callback.
73 |     Useful when you have customized `DataParallel` implementation.
74 | 
75 |     Examples:
76 |         > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False)
77 |         > sync_bn = DataParallel(sync_bn, device_ids=[0, 1])
78 |         > patch_replication_callback(sync_bn)
79 |         # this is equivalent to
80 |         > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False)
81 |         > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1])
82 |     """
83 | 
84 |     assert isinstance(data_parallel, DataParallel)
85 | 
86 |     old_replicate = data_parallel.replicate
87 | 
88 |     @functools.wraps(old_replicate)
89 |     def new_replicate(module, device_ids):
90 |         modules = old_replicate(module, device_ids)
91 |         execute_replication_callbacks(modules)
92 |         return modules
93 | 
94 |     data_parallel.replicate = new_replicate
95 | 


--------------------------------------------------------------------------------
/detection_models/sync_batchnorm/unittest.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | # File   : unittest.py
 3 | # Author : Jiayuan Mao
 4 | # Email  : maojiayuan@gmail.com
 5 | # Date   : 27/01/2018
 6 | #
 7 | # This file is part of Synchronized-BatchNorm-PyTorch.
 8 | # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
 9 | # Distributed under MIT License.
10 | 
11 | import unittest
12 | import torch
13 | 
14 | 
15 | class TorchTestCase(unittest.TestCase):
16 |     def assertTensorClose(self, x, y):
17 |         adiff = float((x - y).abs().max())
18 |         if (y == 0).all():
19 |             rdiff = 'NaN'
20 |         else:
21 |             rdiff = float((adiff / y).abs().max())
22 | 
23 |         message = (
24 |             'Tensor close check failed\n'
25 |             'adiff={}\n'
26 |             'rdiff={}\n'
27 |         ).format(adiff, rdiff)
28 |         self.assertTrue(torch.allclose(x, y, atol=1e-5, rtol=1e-3), message)
29 | 
30 | 


--------------------------------------------------------------------------------
/detection_util/__pycache__/util.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/detection_util/__pycache__/util.cpython-311.pyc


--------------------------------------------------------------------------------
/detection_util/util.py:
--------------------------------------------------------------------------------
  1 | # Copyright (c) Microsoft Corporation.
  2 | # Licensed under the MIT License.
  3 | 
  4 | import os
  5 | import sys
  6 | import time
  7 | import shutil
  8 | import platform
  9 | import numpy as np
 10 | from datetime import datetime
 11 | 
 12 | import torch
 13 | import torchvision as tv
 14 | import torch.backends.cudnn as cudnn
 15 | 
 16 | # from torch.utils.tensorboard import SummaryWriter
 17 | 
 18 | import yaml
 19 | import matplotlib.pyplot as plt
 20 | from easydict import EasyDict as edict
 21 | import torchvision.utils as vutils
 22 | 
 23 | 
 24 | ##### option parsing ######
 25 | def print_options(config_dict):
 26 |     print("------------ Options -------------")
 27 |     for k, v in sorted(config_dict.items()):
 28 |         print("%s: %s" % (str(k), str(v)))
 29 |     print("-------------- End ----------------")
 30 | 
 31 | 
 32 | def save_options(config_dict):
 33 |     from time import gmtime, strftime
 34 | 
 35 |     file_dir = os.path.join(config_dict["checkpoint_dir"], config_dict["name"])
 36 |     mkdir_if_not(file_dir)
 37 |     file_name = os.path.join(file_dir, "opt.txt")
 38 |     with open(file_name, "wt") as opt_file:
 39 |         opt_file.write(os.path.basename(sys.argv[0]) + " " + strftime("%Y-%m-%d %H:%M:%S", gmtime()) + "\n")
 40 |         opt_file.write("------------ Options -------------\n")
 41 |         for k, v in sorted(config_dict.items()):
 42 |             opt_file.write("%s: %s\n" % (str(k), str(v)))
 43 |         opt_file.write("-------------- End ----------------\n")
 44 | 
 45 | 
 46 | def config_parse(config_file, options, save=True):
 47 |     with open(config_file, "r") as stream:
 48 |         config_dict = yaml.safe_load(stream)
 49 |         config = edict(config_dict)
 50 | 
 51 |     for option_key, option_value in vars(options).items():
 52 |         config_dict[option_key] = option_value
 53 |         config[option_key] = option_value
 54 | 
 55 |     if config.debug_mode:
 56 |         config_dict["num_workers"] = 0
 57 |         config.num_workers = 0
 58 |         config.batch_size = 2
 59 |         if isinstance(config.gpu_ids, str):
 60 |             config.gpu_ids = [int(x) for x in config.gpu_ids.split(",")][0]
 61 | 
 62 |     print_options(config_dict)
 63 |     if save:
 64 |         save_options(config_dict)
 65 | 
 66 |     return config
 67 | 
 68 | 
 69 | ###### utility ######
 70 | def to_np(x):
 71 |     return x.cpu().numpy()
 72 | 
 73 | 
 74 | def prepare_device(use_gpu, gpu_ids):
 75 |     if use_gpu:
 76 |         cudnn.benchmark = True
 77 |         os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
 78 |         if isinstance(gpu_ids, str):
 79 |             gpu_ids = [int(x) for x in gpu_ids.split(",")]
 80 |             torch.cuda.set_device(gpu_ids[0])
 81 |             device = torch.device("cuda:" + str(gpu_ids[0]))
 82 |         else:
 83 |             torch.cuda.set_device(gpu_ids)
 84 |             device = torch.device("cuda:" + str(gpu_ids))
 85 |         print("running on GPU {}".format(gpu_ids))
 86 |     else:
 87 |         device = torch.device("cpu")
 88 |         print("running on CPU")
 89 | 
 90 |     return device
 91 | 
 92 | 
 93 | ###### file system ######
 94 | def get_dir_size(start_path="."):
 95 |     total_size = 0
 96 |     for dirpath, dirnames, filenames in os.walk(start_path):
 97 |         for f in filenames:
 98 |             fp = os.path.join(dirpath, f)
 99 |             total_size += os.path.getsize(fp)
100 |     return total_size
101 | 
102 | 
103 | def mkdir_if_not(dir_path):
104 |     if not os.path.exists(dir_path):
105 |         os.makedirs(dir_path)
106 | 
107 | 
108 | ##### System related ######
109 | class Timer:
110 |     def __init__(self, msg):
111 |         self.msg = msg
112 |         self.start_time = None
113 | 
114 |     def __enter__(self):
115 |         self.start_time = time.time()
116 | 
117 |     def __exit__(self, exc_type, exc_value, exc_tb):
118 |         elapse = time.time() - self.start_time
119 |         print(self.msg % elapse)
120 | 
121 | 
122 | ###### interactive ######
123 | def get_size(start_path="."):
124 |     total_size = 0
125 |     for dirpath, dirnames, filenames in os.walk(start_path):
126 |         for f in filenames:
127 |             fp = os.path.join(dirpath, f)
128 |             total_size += os.path.getsize(fp)
129 |     return total_size
130 | 
131 | 
132 | def clean_tensorboard(directory):
133 |     tensorboard_list = os.listdir(directory)
134 |     SIZE_THRESH = 100000
135 |     for tensorboard in tensorboard_list:
136 |         tensorboard = os.path.join(directory, tensorboard)
137 |         if get_size(tensorboard) < SIZE_THRESH:
138 |             print("deleting the empty tensorboard: ", tensorboard)
139 |             #
140 |             if os.path.isdir(tensorboard):
141 |                 shutil.rmtree(tensorboard)
142 |             else:
143 |                 os.remove(tensorboard)
144 | 
145 | 
146 | def prepare_tensorboard(config, experiment_name=datetime.now().strftime("%Y-%m-%d %H-%M-%S")):
147 |     tensorboard_directory = os.path.join(config.checkpoint_dir, config.name, "tensorboard_logs")
148 |     mkdir_if_not(tensorboard_directory)
149 |     clean_tensorboard(tensorboard_directory)
150 |     tb_writer = SummaryWriter(os.path.join(tensorboard_directory, experiment_name), flush_secs=10)
151 | 
152 |     # try:
153 |     #     shutil.copy('outputs/opt.txt', tensorboard_directory)
154 |     # except:
155 |     #     print('cannot find file opt.txt')
156 |     return tb_writer
157 | 
158 | 
159 | def tb_loss_logger(tb_writer, iter_index, loss_logger):
160 |     for tag, value in loss_logger.items():
161 |         tb_writer.add_scalar(tag, scalar_value=value.item(), global_step=iter_index)
162 | 
163 | 
164 | def tb_image_logger(tb_writer, iter_index, images_info, config):
165 |     ### Save and write the output into the tensorboard
166 |     tb_logger_path = os.path.join(config.output_dir, config.name, config.train_mode)
167 |     mkdir_if_not(tb_logger_path)
168 |     for tag, image in images_info.items():
169 |         if tag == "test_image_prediction" or tag == "image_prediction":
170 |             continue
171 |         image = tv.utils.make_grid(image.cpu())
172 |         image = torch.clamp(image, 0, 1)
173 |         tb_writer.add_image(tag, img_tensor=image, global_step=iter_index)
174 |         tv.transforms.functional.to_pil_image(image).save(
175 |             os.path.join(tb_logger_path, "{:06d}_{}.jpg".format(iter_index, tag))
176 |         )
177 | 
178 | 
179 | def tb_image_logger_test(epoch, iter, images_info, config):
180 | 
181 |     url = os.path.join(config.output_dir, config.name, config.train_mode, "val_" + str(epoch))
182 |     if not os.path.exists(url):
183 |         os.makedirs(url)
184 |     scratch_img = images_info["test_scratch_image"].data.cpu()
185 |     if config.norm_input:
186 |         scratch_img = (scratch_img + 1.0) / 2.0
187 |     scratch_img = torch.clamp(scratch_img, 0, 1)
188 |     gt_mask = images_info["test_mask_image"].data.cpu()
189 |     predict_mask = images_info["test_scratch_prediction"].data.cpu()
190 | 
191 |     predict_hard_mask = (predict_mask.data.cpu() >= 0.5).float()
192 | 
193 |     imgs = torch.cat((scratch_img, predict_hard_mask, gt_mask), 0)
194 |     img_grid = vutils.save_image(
195 |         imgs, os.path.join(url, str(iter) + ".jpg"), nrow=len(scratch_img), padding=0, normalize=True
196 |     )
197 | 
198 | 
199 | def imshow(input_image, title=None, to_numpy=False):
200 |     inp = input_image
201 |     if to_numpy or type(input_image) is torch.Tensor:
202 |         inp = input_image.numpy()
203 | 
204 |     fig = plt.figure()
205 |     if inp.ndim == 2:
206 |         fig = plt.imshow(inp, cmap="gray", clim=[0, 255])
207 |     else:
208 |         fig = plt.imshow(np.transpose(inp, [1, 2, 0]).astype(np.uint8))
209 |     plt.axis("off")
210 |     fig.axes.get_xaxis().set_visible(False)
211 |     fig.axes.get_yaxis().set_visible(False)
212 |     plt.title(title)
213 | 
214 | 
215 | ###### vgg preprocessing ######
216 | def vgg_preprocess(tensor):
217 |     # input is RGB tensor which ranges in [0,1]
218 |     # output is BGR tensor which ranges in [0,255]
219 |     tensor_bgr = torch.cat((tensor[:, 2:3, :, :], tensor[:, 1:2, :, :], tensor[:, 0:1, :, :]), dim=1)
220 |     # tensor_bgr = tensor[:, [2, 1, 0], ...]
221 |     tensor_bgr_ml = tensor_bgr - torch.Tensor([0.40760392, 0.45795686, 0.48501961]).type_as(tensor_bgr).view(
222 |         1, 3, 1, 1
223 |     )
224 |     tensor_rst = tensor_bgr_ml * 255
225 |     return tensor_rst
226 | 
227 | 
228 | def torch_vgg_preprocess(tensor):
229 |     # pytorch version normalization
230 |     # note that both input and output are RGB tensors;
231 |     # input and output ranges in [0,1]
232 |     # normalize the tensor with mean and variance
233 |     tensor_mc = tensor - torch.Tensor([0.485, 0.456, 0.406]).type_as(tensor).view(1, 3, 1, 1)
234 |     tensor_mc_norm = tensor_mc / torch.Tensor([0.229, 0.224, 0.225]).type_as(tensor_mc).view(1, 3, 1, 1)
235 |     return tensor_mc_norm
236 | 
237 | 
238 | def network_gradient(net, gradient_on=True):
239 |     if gradient_on:
240 |         for param in net.parameters():
241 |             param.requires_grad = True
242 |     else:
243 |         for param in net.parameters():
244 |             param.requires_grad = False
245 |     return net
246 | 


--------------------------------------------------------------------------------
/eval.py:
--------------------------------------------------------------------------------
  1 | from dataset.CamVid import CamVid
  2 | import torch
  3 | import argparse
  4 | import os
  5 | from torch.utils.data import DataLoader
  6 | from model.build_BiSeNet import BiSeNet
  7 | import numpy as np
  8 | from utils import reverse_one_hot, compute_global_accuracy, fast_hist, per_class_iu, cal_miou
  9 | import tqdm
 10 | 
 11 | 
 12 | def eval(model,dataloader, args, csv_path):
 13 |     print('start test!')
 14 |     with torch.no_grad():
 15 |         model.eval()
 16 |         precision_record = []
 17 |         tq = tqdm.tqdm(total=len(dataloader) * args.batch_size)
 18 |         tq.set_description('test')
 19 |         hist = np.zeros((args.num_classes, args.num_classes))
 20 |         for i, (data, label) in enumerate(dataloader):
 21 |             tq.update(args.batch_size)
 22 |             if torch.cuda.is_available() and args.use_gpu:
 23 |                 data = data.cuda()
 24 |                 label = label.cuda()
 25 |             predict = model(data).squeeze()
 26 |             predict = reverse_one_hot(predict)
 27 |             predict = np.array(predict)
 28 |             # predict = colour_code_segmentation(np.array(predict), label_info)
 29 | 
 30 |             label = label.squeeze()
 31 |             if args.loss == 'dice':
 32 |                 label = reverse_one_hot(label)
 33 |             label = np.array(label)
 34 |             # label = colour_code_segmentation(np.array(label), label_info)
 35 | 
 36 |             precision = compute_global_accuracy(predict, label)
 37 |             hist += fast_hist(label.flatten(), predict.flatten(), args.num_classes)
 38 |             precision_record.append(precision)
 39 |         precision = np.mean(precision_record)
 40 |         miou_list = per_class_iu(hist)[:-1]
 41 |         miou_dict, miou = cal_miou(miou_list, csv_path)
 42 |         print('IoU for each class:')
 43 |         for key in miou_dict:
 44 |             print('{}:{},'.format(key, miou_dict[key]))
 45 |         tq.close()
 46 |         print('precision for test: %.3f' % precision)
 47 |         print('mIoU for validation: %.3f' % miou)
 48 |         return precision
 49 | 
 50 | def main(params):
 51 |     # basic parameters
 52 |     parser = argparse.ArgumentParser()
 53 |     parser.add_argument('--checkpoint_path', type=str, default=None, required=True, help='The path to the pretrained weights of model')
 54 |     parser.add_argument('--crop_height', type=int, default=720, help='Height of cropped/resized input image to network')
 55 |     parser.add_argument('--crop_width', type=int, default=960, help='Width of cropped/resized input image to network')
 56 |     parser.add_argument('--data', type=str, default='/path/to/data', help='Path of training data')
 57 |     parser.add_argument('--batch_size', type=int, default=1, help='Number of images in each batch')
 58 |     parser.add_argument('--context_path', type=str, default="resnet101", help='The context path model you are using.')
 59 |     parser.add_argument('--cuda', type=str, default='0', help='GPU ids used for training')
 60 |     parser.add_argument('--use_gpu', type=bool, default=True, help='Whether to user gpu for training')
 61 |     parser.add_argument('--num_classes', type=int, default=32, help='num of object classes (with void)')
 62 |     parser.add_argument('--loss', type=str, default='dice', help='loss function, dice or crossentropy')
 63 |     args = parser.parse_args(params)
 64 | 
 65 |     # create dataset and dataloader
 66 |     test_path = os.path.join(args.data, 'test')
 67 |     # test_path = os.path.join(args.data, 'train')
 68 |     test_label_path = os.path.join(args.data, 'test_labels')
 69 |     # test_label_path = os.path.join(args.data, 'train_labels')
 70 |     csv_path = os.path.join(args.data, 'class_dict.csv')
 71 |     dataset = CamVid(test_path, test_label_path, csv_path, scale=(args.crop_height, args.crop_width), mode='test')
 72 |     dataloader = DataLoader(
 73 |         dataset,
 74 |         batch_size=1,
 75 |         shuffle=True,
 76 |         num_workers=4,
 77 |     )
 78 | 
 79 |     # build model
 80 |     os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda
 81 |     model = BiSeNet(args.num_classes, args.context_path)
 82 |     if torch.cuda.is_available() and args.use_gpu:
 83 |         model = torch.nn.DataParallel(model).cuda()
 84 | 
 85 |     # load pretrained model if exists
 86 |     print('load model from %s ...' % args.checkpoint_path)
 87 |     model.module.load_state_dict(torch.load(args.checkpoint_path))
 88 |     print('Done!')
 89 | 
 90 |     # get label info
 91 |     # label_info = get_label_info(csv_path)
 92 |     # test
 93 |     eval(model, dataloader, args, csv_path)
 94 | 
 95 | 
 96 | if __name__ == '__main__':
 97 |     params = [
 98 |         '--checkpoint_path', 'path/to/ckpt',
 99 |         '--data', '/path/to/CamVid',
100 |         '--cuda', '0',
101 |         '--context_path', 'resnet18',
102 |         '--num_classes', '12'
103 |     ]
104 |     main(params)


--------------------------------------------------------------------------------
/examples/eg1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/examples/eg1.jpg


--------------------------------------------------------------------------------
/examples/eg2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/examples/eg2.jpg


--------------------------------------------------------------------------------
/examples/eg2gif.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/examples/eg2gif.gif


--------------------------------------------------------------------------------
/examples/obj1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/examples/obj1.jpg


--------------------------------------------------------------------------------
/examples/obj2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/examples/obj2.jpg


--------------------------------------------------------------------------------
/examples/objgif.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/examples/objgif.gif


--------------------------------------------------------------------------------
/input_images/input_image.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/input_images/input_image.png


--------------------------------------------------------------------------------
/loss.py:
--------------------------------------------------------------------------------
 1 | import torch.nn as nn
 2 | import torch
 3 | import torch.nn.functional as F
 4 | 
 5 | def flatten(tensor):
 6 |     """Flattens a given tensor such that the channel axis is first.
 7 |     The shapes are transformed as follows:
 8 |        (N, C, D, H, W) -> (C, N * D * H * W)
 9 |     """
10 |     C = tensor.size(1)
11 |     # new axis order
12 |     axis_order = (1, 0) + tuple(range(2, tensor.dim()))
13 |     # Transpose: (N, C, D, H, W) -> (C, N, D, H, W)
14 |     transposed = tensor.permute(axis_order)
15 |     # Flatten: (C, N, D, H, W) -> (C, N * D * H * W)
16 |     return transposed.contiguous().view(C, -1)
17 | 
18 | 
19 | class DiceLoss(nn.Module):
20 |     def __init__(self):
21 |         super().__init__()
22 |         self.epsilon = 1e-5
23 | 
24 |     def forward(self, output, target):
25 |         assert output.size() == target.size(), "'input' and 'target' must have the same shape"
26 |         output = F.softmax(output, dim=1)
27 |         output = flatten(output)
28 |         target = flatten(target)
29 |         # intersect = (output * target).sum(-1).sum() + self.epsilon
30 |         # denominator = ((output + target).sum(-1)).sum() + self.epsilon
31 | 
32 |         intersect = (output * target).sum(-1)
33 |         denominator = (output + target).sum(-1)
34 |         dice = intersect / denominator
35 |         dice = torch.mean(dice)
36 |         return 1 - dice
37 |         # return 1 - 2. * intersect / denominator
38 | 


--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | # -*- encoding: utf-8 -*-
  3 | 
  4 | 
  5 | import torch
  6 | import torch.nn as nn
  7 | import torch.nn.functional as F
  8 | import torchvision
  9 | 
 10 | from resnet import Resnet18
 11 | # from modules.bn import InPlaceABNSync as BatchNorm2d
 12 | 
 13 | 
 14 | class ConvBNReLU(nn.Module):
 15 |     def __init__(self, in_chan, out_chan, ks=3, stride=1, padding=1, *args, **kwargs):
 16 |         super(ConvBNReLU, self).__init__()
 17 |         self.conv = nn.Conv2d(in_chan,
 18 |                 out_chan,
 19 |                 kernel_size = ks,
 20 |                 stride = stride,
 21 |                 padding = padding,
 22 |                 bias = False)
 23 |         self.bn = nn.BatchNorm2d(out_chan)
 24 |         self.init_weight()
 25 | 
 26 |     def forward(self, x):
 27 |         x = self.conv(x)
 28 |         x = F.relu(self.bn(x))
 29 |         return x
 30 | 
 31 |     def init_weight(self):
 32 |         for ly in self.children():
 33 |             if isinstance(ly, nn.Conv2d):
 34 |                 nn.init.kaiming_normal_(ly.weight, a=1)
 35 |                 if not ly.bias is None: nn.init.constant_(ly.bias, 0)
 36 | 
 37 | class BiSeNetOutput(nn.Module):
 38 |     def __init__(self, in_chan, mid_chan, n_classes, *args, **kwargs):
 39 |         super(BiSeNetOutput, self).__init__()
 40 |         self.conv = ConvBNReLU(in_chan, mid_chan, ks=3, stride=1, padding=1)
 41 |         self.conv_out = nn.Conv2d(mid_chan, n_classes, kernel_size=1, bias=False)
 42 |         self.init_weight()
 43 | 
 44 |     def forward(self, x):
 45 |         x = self.conv(x)
 46 |         x = self.conv_out(x)
 47 |         return x
 48 | 
 49 |     def init_weight(self):
 50 |         for ly in self.children():
 51 |             if isinstance(ly, nn.Conv2d):
 52 |                 nn.init.kaiming_normal_(ly.weight, a=1)
 53 |                 if not ly.bias is None: nn.init.constant_(ly.bias, 0)
 54 | 
 55 |     def get_params(self):
 56 |         wd_params, nowd_params = [], []
 57 |         for name, module in self.named_modules():
 58 |             if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d):
 59 |                 wd_params.append(module.weight)
 60 |                 if not module.bias is None:
 61 |                     nowd_params.append(module.bias)
 62 |             elif isinstance(module, nn.BatchNorm2d):
 63 |                 nowd_params += list(module.parameters())
 64 |         return wd_params, nowd_params
 65 | 
 66 | 
 67 | class AttentionRefinementModule(nn.Module):
 68 |     def __init__(self, in_chan, out_chan, *args, **kwargs):
 69 |         super(AttentionRefinementModule, self).__init__()
 70 |         self.conv = ConvBNReLU(in_chan, out_chan, ks=3, stride=1, padding=1)
 71 |         self.conv_atten = nn.Conv2d(out_chan, out_chan, kernel_size= 1, bias=False)
 72 |         self.bn_atten = nn.BatchNorm2d(out_chan)
 73 |         self.sigmoid_atten = nn.Sigmoid()
 74 |         self.init_weight()
 75 | 
 76 |     def forward(self, x):
 77 |         feat = self.conv(x)
 78 |         atten = F.avg_pool2d(feat, feat.size()[2:])
 79 |         atten = self.conv_atten(atten)
 80 |         atten = self.bn_atten(atten)
 81 |         atten = self.sigmoid_atten(atten)
 82 |         out = torch.mul(feat, atten)
 83 |         return out
 84 | 
 85 |     def init_weight(self):
 86 |         for ly in self.children():
 87 |             if isinstance(ly, nn.Conv2d):
 88 |                 nn.init.kaiming_normal_(ly.weight, a=1)
 89 |                 if not ly.bias is None: nn.init.constant_(ly.bias, 0)
 90 | 
 91 | 
 92 | class ContextPath(nn.Module):
 93 |     def __init__(self, *args, **kwargs):
 94 |         super(ContextPath, self).__init__()
 95 |         self.resnet = Resnet18()
 96 |         self.arm16 = AttentionRefinementModule(256, 128)
 97 |         self.arm32 = AttentionRefinementModule(512, 128)
 98 |         self.conv_head32 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1)
 99 |         self.conv_head16 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1)
100 |         self.conv_avg = ConvBNReLU(512, 128, ks=1, stride=1, padding=0)
101 | 
102 |         self.init_weight()
103 | 
104 |     def forward(self, x):
105 |         H0, W0 = x.size()[2:]
106 |         feat8, feat16, feat32 = self.resnet(x)
107 |         H8, W8 = feat8.size()[2:]
108 |         H16, W16 = feat16.size()[2:]
109 |         H32, W32 = feat32.size()[2:]
110 | 
111 |         avg = F.avg_pool2d(feat32, feat32.size()[2:])
112 |         avg = self.conv_avg(avg)
113 |         avg_up = F.interpolate(avg, (H32, W32), mode='nearest')
114 | 
115 |         feat32_arm = self.arm32(feat32)
116 |         feat32_sum = feat32_arm + avg_up
117 |         feat32_up = F.interpolate(feat32_sum, (H16, W16), mode='nearest')
118 |         feat32_up = self.conv_head32(feat32_up)
119 | 
120 |         feat16_arm = self.arm16(feat16)
121 |         feat16_sum = feat16_arm + feat32_up
122 |         feat16_up = F.interpolate(feat16_sum, (H8, W8), mode='nearest')
123 |         feat16_up = self.conv_head16(feat16_up)
124 | 
125 |         return feat8, feat16_up, feat32_up  # x8, x8, x16
126 | 
127 |     def init_weight(self):
128 |         for ly in self.children():
129 |             if isinstance(ly, nn.Conv2d):
130 |                 nn.init.kaiming_normal_(ly.weight, a=1)
131 |                 if not ly.bias is None: nn.init.constant_(ly.bias, 0)
132 | 
133 |     def get_params(self):
134 |         wd_params, nowd_params = [], []
135 |         for name, module in self.named_modules():
136 |             if isinstance(module, (nn.Linear, nn.Conv2d)):
137 |                 wd_params.append(module.weight)
138 |                 if not module.bias is None:
139 |                     nowd_params.append(module.bias)
140 |             elif isinstance(module, nn.BatchNorm2d):
141 |                 nowd_params += list(module.parameters())
142 |         return wd_params, nowd_params
143 | 
144 | 
145 | ### This is not used, since I replace this with the resnet feature with the same size
146 | class SpatialPath(nn.Module):
147 |     def __init__(self, *args, **kwargs):
148 |         super(SpatialPath, self).__init__()
149 |         self.conv1 = ConvBNReLU(3, 64, ks=7, stride=2, padding=3)
150 |         self.conv2 = ConvBNReLU(64, 64, ks=3, stride=2, padding=1)
151 |         self.conv3 = ConvBNReLU(64, 64, ks=3, stride=2, padding=1)
152 |         self.conv_out = ConvBNReLU(64, 128, ks=1, stride=1, padding=0)
153 |         self.init_weight()
154 | 
155 |     def forward(self, x):
156 |         feat = self.conv1(x)
157 |         feat = self.conv2(feat)
158 |         feat = self.conv3(feat)
159 |         feat = self.conv_out(feat)
160 |         return feat
161 | 
162 |     def init_weight(self):
163 |         for ly in self.children():
164 |             if isinstance(ly, nn.Conv2d):
165 |                 nn.init.kaiming_normal_(ly.weight, a=1)
166 |                 if not ly.bias is None: nn.init.constant_(ly.bias, 0)
167 | 
168 |     def get_params(self):
169 |         wd_params, nowd_params = [], []
170 |         for name, module in self.named_modules():
171 |             if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d):
172 |                 wd_params.append(module.weight)
173 |                 if not module.bias is None:
174 |                     nowd_params.append(module.bias)
175 |             elif isinstance(module, nn.BatchNorm2d):
176 |                 nowd_params += list(module.parameters())
177 |         return wd_params, nowd_params
178 | 
179 | 
180 | class FeatureFusionModule(nn.Module):
181 |     def __init__(self, in_chan, out_chan, *args, **kwargs):
182 |         super(FeatureFusionModule, self).__init__()
183 |         self.convblk = ConvBNReLU(in_chan, out_chan, ks=1, stride=1, padding=0)
184 |         self.conv1 = nn.Conv2d(out_chan,
185 |                 out_chan//4,
186 |                 kernel_size = 1,
187 |                 stride = 1,
188 |                 padding = 0,
189 |                 bias = False)
190 |         self.conv2 = nn.Conv2d(out_chan//4,
191 |                 out_chan,
192 |                 kernel_size = 1,
193 |                 stride = 1,
194 |                 padding = 0,
195 |                 bias = False)
196 |         self.relu = nn.ReLU(inplace=True)
197 |         self.sigmoid = nn.Sigmoid()
198 |         self.init_weight()
199 | 
200 |     def forward(self, fsp, fcp):
201 |         fcat = torch.cat([fsp, fcp], dim=1)
202 |         feat = self.convblk(fcat)
203 |         atten = F.avg_pool2d(feat, feat.size()[2:])
204 |         atten = self.conv1(atten)
205 |         atten = self.relu(atten)
206 |         atten = self.conv2(atten)
207 |         atten = self.sigmoid(atten)
208 |         feat_atten = torch.mul(feat, atten)
209 |         feat_out = feat_atten + feat
210 |         return feat_out
211 | 
212 |     def init_weight(self):
213 |         for ly in self.children():
214 |             if isinstance(ly, nn.Conv2d):
215 |                 nn.init.kaiming_normal_(ly.weight, a=1)
216 |                 if not ly.bias is None: nn.init.constant_(ly.bias, 0)
217 | 
218 |     def get_params(self):
219 |         wd_params, nowd_params = [], []
220 |         for name, module in self.named_modules():
221 |             if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d):
222 |                 wd_params.append(module.weight)
223 |                 if not module.bias is None:
224 |                     nowd_params.append(module.bias)
225 |             elif isinstance(module, nn.BatchNorm2d):
226 |                 nowd_params += list(module.parameters())
227 |         return wd_params, nowd_params
228 | 
229 | 
230 | class BiSeNet(nn.Module):
231 |     def __init__(self, n_classes, *args, **kwargs):
232 |         super(BiSeNet, self).__init__()
233 |         self.cp = ContextPath()
234 |         ## here self.sp is deleted
235 |         self.ffm = FeatureFusionModule(256, 256)
236 |         self.conv_out = BiSeNetOutput(256, 256, n_classes)
237 |         self.conv_out16 = BiSeNetOutput(128, 64, n_classes)
238 |         self.conv_out32 = BiSeNetOutput(128, 64, n_classes)
239 |         self.init_weight()
240 | 
241 |     def forward(self, x):
242 |         H, W = x.size()[2:]
243 |         feat_res8, feat_cp8, feat_cp16 = self.cp(x)  # here return res3b1 feature
244 |         feat_sp = feat_res8  # use res3b1 feature to replace spatial path feature
245 |         feat_fuse = self.ffm(feat_sp, feat_cp8)
246 | 
247 |         feat_out = self.conv_out(feat_fuse)
248 |         feat_out16 = self.conv_out16(feat_cp8)
249 |         feat_out32 = self.conv_out32(feat_cp16)
250 | 
251 |         feat_out = F.interpolate(feat_out, (H, W), mode='bilinear', align_corners=True)
252 |         feat_out16 = F.interpolate(feat_out16, (H, W), mode='bilinear', align_corners=True)
253 |         feat_out32 = F.interpolate(feat_out32, (H, W), mode='bilinear', align_corners=True)
254 |         return feat_out, feat_out16, feat_out32
255 | 
256 |     def init_weight(self):
257 |         for ly in self.children():
258 |             if isinstance(ly, nn.Conv2d):
259 |                 nn.init.kaiming_normal_(ly.weight, a=1)
260 |                 if not ly.bias is None: nn.init.constant_(ly.bias, 0)
261 | 
262 |     def get_params(self):
263 |         wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params = [], [], [], []
264 |         for name, child in self.named_children():
265 |             child_wd_params, child_nowd_params = child.get_params()
266 |             if isinstance(child, FeatureFusionModule) or isinstance(child, BiSeNetOutput):
267 |                 lr_mul_wd_params += child_wd_params
268 |                 lr_mul_nowd_params += child_nowd_params
269 |             else:
270 |                 wd_params += child_wd_params
271 |                 nowd_params += child_nowd_params
272 |         return wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params
273 | 
274 | 
275 | if __name__ == "__main__":
276 |     net = BiSeNet(19)
277 |     net.cuda()
278 |     net.eval()
279 |     in_ten = torch.randn(16, 3, 640, 480).cuda()
280 |     out, out16, out32 = net(in_ten)
281 |     print(out.shape)
282 | 
283 |     net.get_params()
284 | 


--------------------------------------------------------------------------------
/obrem.py:
--------------------------------------------------------------------------------
 1 | #best object removal model
 2 | 
 3 | import gradio as gr
 4 | import numpy as np
 5 | import torch
 6 | from src.pipeline_stable_diffusion_controlnet_inpaint import *
 7 | 
 8 | from diffusers import StableDiffusionInpaintPipeline, ControlNetModel, DEISMultistepScheduler
 9 | from diffusers.utils import load_image
10 | from PIL import Image
11 | import cv2
12 | 
13 | 
14 | controlnet = ControlNetModel.from_pretrained("thepowefuldeez/sd21-controlnet-canny", torch_dtype=torch.float16)
15 | 
16 | pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
17 |      "stabilityai/stable-diffusion-2-inpainting", controlnet=controlnet, torch_dtype=torch.float16
18 |  )
19 | 
20 | pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
21 | 
22 | # speed up diffusion process with faster scheduler and memory optimization
23 | # remove following line if xformers is not installed
24 | pipe.enable_xformers_memory_efficient_attention()
25 | pipe.to('cuda')
26 | 
27 | def resize_image(image, target_size):
28 |     width, height = image.size
29 |     aspect_ratio = float(width) / float(height)
30 |     if width > height:
31 |         new_width = target_size
32 |         new_height = int(target_size / aspect_ratio)
33 |     else:
34 |         new_width = int(target_size * aspect_ratio)
35 |         new_height = target_size
36 |     return image.resize((new_width, new_height), Image.BICUBIC)
37 | def predict(input_dict):
38 |     # Get the drawn input image and mask
39 |     image = input_dict["image"].convert("RGB")
40 |     input_image = input_dict["mask"].convert("RGB")
41 |     input_image = resize_image(input_image, 768)
42 |     image = resize_image(image, 768)
43 | 
44 |     # Convert images to numpy arrays
45 |     image_np = np.array(image)
46 |     input_image_np = np.array(input_image)
47 | 
48 |     # Convert input_image_np to grayscale and normalize to [0, 1] range
49 |     mask_np = cv2.cvtColor(input_image_np, cv2.COLOR_RGB2GRAY) / 255.0
50 | 
51 |     # Apply OpenCV inpainting
52 |     inpainted_image_np = cv2.inpaint(image_np, (mask_np * 255).astype(np.uint8), 3, cv2.INPAINT_TELEA)
53 | 
54 |     # Blend the original image and the inpainted image using the mask
55 |     blended_image_np = image_np * (1 - mask_np)[:, :, None] + inpainted_image_np * mask_np[:, :, None]
56 | 
57 |     # Convert the blended image back to a PIL Image
58 |     blended_image = Image.fromarray(np.uint8(blended_image_np))
59 | 
60 |     # Process the blended image
61 |     blended_image_np = np.array(blended_image)
62 |     low_threshold = 800
63 |     high_threshold = 900
64 |     canny = cv2.Canny(blended_image_np, low_threshold, high_threshold)
65 |     canny = canny[:, :, None]
66 |     canny = np.concatenate([canny, canny, canny], axis=2)
67 |     canny_image = Image.fromarray(canny)
68 |     canny_image.save("canny.png")
69 | 
70 | 
71 |     generator = torch.manual_seed(0)
72 |     output = pipe(
73 |         prompt="",
74 |         num_inference_steps=20,
75 |         generator=generator,
76 |         image=blended_image_np,
77 |         control_image=canny_image,
78 |         controlnet_conditioning_scale=0.9,
79 |         mask_image=input_image
80 |     ).images[0]
81 |     
82 |     return output
83 | image_blocks = gr.Blocks()
84 | 
85 | with image_blocks as demo:
86 |     with gr.Row():
87 |         with gr.Column():
88 |             # Allow user to draw on the input image
89 |             input_image = gr.Image(source='upload', tool='sketch', elem_id="input_image_upload", type="pil", label="Upload & Draw on Image")
90 |             # Allow user to draw the mask
91 |             #mask = gr.Image(source='upload', tool='sketch', elem_id="mask_upload", type="pil", label="Draw Mask")
92 |             #prompt = gr.Textbox(label='Your prompt (what you want to add in place of what you are removing)')
93 |             btn = gr.Button("Run")
94 |         with gr.Column():
95 |             result = gr.Image(label="Result")
96 |         btn.click(fn=predict, inputs=[input_image], outputs=result)
97 | demo.launch(share=True)
98 | 


--------------------------------------------------------------------------------
/output_masks/input/input_image.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/output_masks/input/input_image.png


--------------------------------------------------------------------------------
/output_masks/mask/input_image.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/output_masks/mask/input_image.png


--------------------------------------------------------------------------------
/resnet.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | # -*- encoding: utf-8 -*-
  3 | 
  4 | import torch
  5 | import torch.nn as nn
  6 | import torch.nn.functional as F
  7 | import torch.utils.model_zoo as modelzoo
  8 | 
  9 | # from modules.bn import InPlaceABNSync as BatchNorm2d
 10 | 
 11 | resnet18_url = 'https://download.pytorch.org/models/resnet18-5c106cde.pth'
 12 | 
 13 | 
 14 | def conv3x3(in_planes, out_planes, stride=1):
 15 |     """3x3 convolution with padding"""
 16 |     return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
 17 |                      padding=1, bias=False)
 18 | 
 19 | 
 20 | class BasicBlock(nn.Module):
 21 |     def __init__(self, in_chan, out_chan, stride=1):
 22 |         super(BasicBlock, self).__init__()
 23 |         self.conv1 = conv3x3(in_chan, out_chan, stride)
 24 |         self.bn1 = nn.BatchNorm2d(out_chan)
 25 |         self.conv2 = conv3x3(out_chan, out_chan)
 26 |         self.bn2 = nn.BatchNorm2d(out_chan)
 27 |         self.relu = nn.ReLU(inplace=True)
 28 |         self.downsample = None
 29 |         if in_chan != out_chan or stride != 1:
 30 |             self.downsample = nn.Sequential(
 31 |                 nn.Conv2d(in_chan, out_chan,
 32 |                           kernel_size=1, stride=stride, bias=False),
 33 |                 nn.BatchNorm2d(out_chan),
 34 |                 )
 35 | 
 36 |     def forward(self, x):
 37 |         residual = self.conv1(x)
 38 |         residual = F.relu(self.bn1(residual))
 39 |         residual = self.conv2(residual)
 40 |         residual = self.bn2(residual)
 41 | 
 42 |         shortcut = x
 43 |         if self.downsample is not None:
 44 |             shortcut = self.downsample(x)
 45 | 
 46 |         out = shortcut + residual
 47 |         out = self.relu(out)
 48 |         return out
 49 | 
 50 | 
 51 | def create_layer_basic(in_chan, out_chan, bnum, stride=1):
 52 |     layers = [BasicBlock(in_chan, out_chan, stride=stride)]
 53 |     for i in range(bnum-1):
 54 |         layers.append(BasicBlock(out_chan, out_chan, stride=1))
 55 |     return nn.Sequential(*layers)
 56 | 
 57 | 
 58 | class Resnet18(nn.Module):
 59 |     def __init__(self):
 60 |         super(Resnet18, self).__init__()
 61 |         self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
 62 |                                bias=False)
 63 |         self.bn1 = nn.BatchNorm2d(64)
 64 |         self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 65 |         self.layer1 = create_layer_basic(64, 64, bnum=2, stride=1)
 66 |         self.layer2 = create_layer_basic(64, 128, bnum=2, stride=2)
 67 |         self.layer3 = create_layer_basic(128, 256, bnum=2, stride=2)
 68 |         self.layer4 = create_layer_basic(256, 512, bnum=2, stride=2)
 69 |         self.init_weight()
 70 | 
 71 |     def forward(self, x):
 72 |         x = self.conv1(x)
 73 |         x = F.relu(self.bn1(x))
 74 |         x = self.maxpool(x)
 75 | 
 76 |         x = self.layer1(x)
 77 |         feat8 = self.layer2(x) # 1/8
 78 |         feat16 = self.layer3(feat8) # 1/16
 79 |         feat32 = self.layer4(feat16) # 1/32
 80 |         return feat8, feat16, feat32
 81 | 
 82 |     def init_weight(self):
 83 |         state_dict = modelzoo.load_url(resnet18_url)
 84 |         self_state_dict = self.state_dict()
 85 |         for k, v in state_dict.items():
 86 |             if 'fc' in k: continue
 87 |             self_state_dict.update({k: v})
 88 |         self.load_state_dict(self_state_dict)
 89 | 
 90 |     def get_params(self):
 91 |         wd_params, nowd_params = [], []
 92 |         for name, module in self.named_modules():
 93 |             if isinstance(module, (nn.Linear, nn.Conv2d)):
 94 |                 wd_params.append(module.weight)
 95 |                 if not module.bias is None:
 96 |                     nowd_params.append(module.bias)
 97 |             elif isinstance(module,  nn.BatchNorm2d):
 98 |                 nowd_params += list(module.parameters())
 99 |         return wd_params, nowd_params
100 | 
101 | 
102 | if __name__ == "__main__":
103 |     net = Resnet18()
104 |     x = torch.randn(16, 3, 224, 224)
105 |     out = net(x)
106 |     print(out[0].size())
107 |     print(out[1].size())
108 |     print(out[2].size())
109 |     net.get_params()
110 | 


--------------------------------------------------------------------------------
/rest.py:
--------------------------------------------------------------------------------
  1 | #best restoration model
  2 | 
  3 | import gradio as gr
  4 | import numpy as np
  5 | import torch
  6 | from src.pipeline_stable_diffusion_controlnet_inpaint import *
  7 | from scratch_detection import ScratchDetection
  8 | 
  9 | from diffusers import StableDiffusionInpaintPipeline, ControlNetModel, DEISMultistepScheduler
 10 | from diffusers.utils import load_image
 11 | from PIL import Image
 12 | import cv2
 13 | import time
 14 | import os
 15 | 
 16 | device = "cuda"
 17 | 
 18 | # load control net and stable diffusion v1-5
 19 | controlnet = ControlNetModel.from_pretrained("thepowefuldeez/sd21-controlnet-canny", torch_dtype=torch.float16)
 20 | 
 21 | pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
 22 |      "stabilityai/stable-diffusion-2-inpainting", controlnet=controlnet, torch_dtype=torch.float16
 23 |  )
 24 | 
 25 | pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)
 26 | 
 27 | # speed up diffusion process with faster scheduler and memory optimization
 28 | # remove following line if xformers is not installed
 29 | pipe.enable_xformers_memory_efficient_attention()
 30 | pipe.to('cuda')
 31 | 
 32 | def combine_masks(mask1, mask2):
 33 |     mask1_np = np.array(mask1)
 34 |     mask2_np = np.array(mask2)
 35 |     combined_mask_np = np.maximum(mask1_np, mask2_np)
 36 |     combined_mask = Image.fromarray(combined_mask_np)
 37 |     return combined_mask
 38 | 
 39 | if not os.path.exists("input_images"):
 40 |     os.makedirs("input_images")
 41 | 
 42 | def generate_scratch_mask(input_dict):
 43 |     # Save the input image to a directory
 44 |     input_image = input_dict["image"].convert("RGB")
 45 |     input_image_path = "input_images/input_image.png"
 46 |     input_image_resized = resize_image(input_image, 768)
 47 |     input_image_resized.save(input_image_path)
 48 | 
 49 |     test_path = "input_images"
 50 |     output_dir = "output_masks"
 51 |     scratch_detector = ScratchDetection(test_path, output_dir, input_size="scale_256", gpu=0)
 52 |     scratch_detector.run()
 53 |     mask_image = scratch_detector.get_mask_image("input_image.png")
 54 |     
 55 |     # Resize the mask to match the input image size
 56 |     mask_image = mask_image.resize(input_image.size, Image.BICUBIC)
 57 | 
 58 |     # Apply dilation to make the lines bigger
 59 |     kernel = np.ones((5, 5), np.uint8)
 60 |     mask_image_np = np.array(mask_image)
 61 |     mask_image_np_dilated = cv2.dilate(mask_image_np, kernel, iterations=2)
 62 |     mask_image_dilated = Image.fromarray(mask_image_np_dilated)
 63 | 
 64 |     return mask_image_dilated
 65 | 
 66 | def resize_image(image, target_size):
 67 |     width, height = image.size
 68 |     aspect_ratio = float(width) / float(height)
 69 |     if width > height:
 70 |         new_width = target_size
 71 |         new_height = int(target_size / aspect_ratio)
 72 |     else:
 73 |         new_width = int(target_size * aspect_ratio)
 74 |         new_height = target_size
 75 |     return image.resize((new_width, new_height), Image.BICUBIC)
 76 | 
 77 | with gr.Blocks() as demo:
 78 |     with gr.Row():
 79 |         input_image = gr.Image(source='upload', tool='sketch', elem_id="input_image_upload", type="pil", label="Upload & Draw on Image")
 80 |         mask_image = gr.Image(label="mask")
 81 |         output_image = gr.Image(label="output")
 82 |     with gr.Row():
 83 |         generate_mask_button = gr.Button("Generate Scratch Mask")
 84 |         submit = gr.Button("Inpaint")
 85 |     
 86 |     def inpaint(input_dict, mask):
 87 |         image = input_dict["image"].convert("RGB")
 88 |         draw_mask = input_dict["mask"].convert("RGB")
 89 | 
 90 |         image = resize_image(image, 768)
 91 |         
 92 |         mask = Image.fromarray(mask)
 93 |         mask = resize_image(mask, 768)
 94 |         draw_mask = resize_image(draw_mask, 768)
 95 | 
 96 |         image = np.array(image)
 97 |         low_threshold = 100
 98 |         high_threshold = 200
 99 |         canny = cv2.Canny(image, low_threshold, high_threshold)
100 |         canny = canny[:, :, None]
101 |         canny = np.concatenate([canny, canny, canny], axis=2)
102 |         canny_image = Image.fromarray(canny)
103 |         generator = torch.manual_seed(0)
104 | 
105 |         # Combine drawn mask and generated mask
106 |         combined_mask = combine_masks(draw_mask, mask)
107 | 
108 |         output = pipe(
109 |             prompt="",
110 |             num_inference_steps=20,
111 |             generator=generator,
112 |             image=image,
113 |             control_image=canny_image,
114 |             controlnet_conditioning_scale=0,
115 |             mask_image=combined_mask
116 |         ).images[0]
117 |         return output
118 | 
119 |     generate_mask_button.click(generate_scratch_mask, inputs=[input_image], outputs=[mask_image])
120 |     submit.click(inpaint, inputs=[input_image, mask_image], outputs=[output_image])
121 |     demo.launch(share=True)
122 | 
123 | 
124 |        
125 | 


--------------------------------------------------------------------------------
/scratch_detection.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import gc
  3 | import json
  4 | import os
  5 | import time
  6 | import warnings
  7 | 
  8 | import numpy as np
  9 | import torch
 10 | import torch.nn.functional as F
 11 | import torchvision as tv
 12 | from PIL import Image, ImageFile
 13 | 
 14 | from detection_models import networks
 15 | from detection_util.util import *
 16 | 
 17 | warnings.filterwarnings("ignore", category=UserWarning)
 18 | 
 19 | ImageFile.LOAD_TRUNCATED_IMAGES = True
 20 | 
 21 | 
 22 | def data_transforms(img, full_size, method=Image.BICUBIC):
 23 |     if full_size == "full_size":
 24 |         ow, oh = img.size
 25 |         h = int(round(oh / 16) * 16)
 26 |         w = int(round(ow / 16) * 16)
 27 |         if (h == oh) and (w == ow):
 28 |             return img
 29 |         return img.resize((w, h), method)
 30 | 
 31 |     elif full_size == "scale_256":
 32 |         ow, oh = img.size
 33 |         pw, ph = ow, oh
 34 |         if ow < oh:
 35 |             ow = 256
 36 |             oh = ph / pw * 256
 37 |         else:
 38 |             oh = 256
 39 |             ow = pw / ph * 256
 40 | 
 41 |         h = int(round(oh / 16) * 16)
 42 |         w = int(round(ow / 16) * 16)
 43 |         if (h == ph) and (w == pw):
 44 |             return img
 45 |         return img.resize((w, h), method)
 46 | 
 47 | 
 48 | def scale_tensor(img_tensor, default_scale=256):
 49 |     _, _, w, h = img_tensor.shape
 50 |     if w < h:
 51 |         ow = default_scale
 52 |         oh = h / w * default_scale
 53 |     else:
 54 |         oh = default_scale
 55 |         ow = w / h * default_scale
 56 | 
 57 |     oh = int(round(oh / 16) * 16)
 58 |     ow = int(round(ow / 16) * 16)
 59 | 
 60 |     return F.interpolate(img_tensor, [ow, oh], mode="bilinear")
 61 | 
 62 | 
 63 | def blend_mask(img, mask):
 64 | 
 65 |     np_img = np.array(img).astype("float")
 66 | 
 67 |     return Image.fromarray((np_img * (1 - mask) + mask * 255.0).astype("uint8")).convert("RGB")
 68 | 
 69 | def process_images(test_path, output_dir, input_size="scale_256", gpu=0):
 70 |     print("initializing the dataloader")
 71 | 
 72 |     # Initialize the model
 73 |     model = networks.UNet(
 74 |         in_channels=1,
 75 |         out_channels=1,
 76 |         depth=4,
 77 |         conv_num=2,
 78 |         wf=6,
 79 |         padding=True,
 80 |         batch_norm=True,
 81 |         up_mode="upsample",
 82 |         with_tanh=False,
 83 |         sync_bn=True,
 84 |         antialiasing=True,
 85 |     )
 86 | 
 87 |     ## load model
 88 |     checkpoint_path = os.path.join(os.path.dirname(__file__), "FT_Epoch_latest.pt")
 89 |     checkpoint = torch.load(checkpoint_path, map_location="cpu")
 90 |     model.load_state_dict(checkpoint["model_state"])
 91 |     print("model weights loaded")
 92 | 
 93 |     if gpu >= 0:
 94 |         model.to(gpu)
 95 |     else: 
 96 |         model.cpu()
 97 |     model.eval()
 98 | 
 99 |     ## dataloader and transformation
100 |     print("directory of testing image: " + test_path)
101 |     imagelist = os.listdir(test_path)
102 |     imagelist.sort()
103 |     total_iter = 0
104 | 
105 |     P_matrix = {}
106 |     save_url = os.path.join(output_dir)
107 |     mkdir_if_not(save_url)
108 | 
109 |     input_dir = os.path.join(save_url, "input")
110 |     output_dir = os.path.join(save_url, "mask")
111 |     mkdir_if_not(input_dir)
112 |     mkdir_if_not(output_dir)
113 | 
114 |     idx = 0
115 | 
116 |     results = []
117 |     for image_name in imagelist:
118 | 
119 |         idx += 1
120 | 
121 |         print("processing", image_name)
122 | 
123 |         scratch_file = os.path.join(test_path, image_name)
124 |         if not os.path.isfile(scratch_file):
125 |             print("Skipping non-file %s" % image_name)
126 |             continue
127 |         scratch_image = Image.open(scratch_file).convert("RGB")
128 |         w, h = scratch_image.size
129 | 
130 |         transformed_image_PIL = data_transforms(scratch_image, input_size)
131 |         scratch_image = transformed_image_PIL.convert("L")
132 |         scratch_image = tv.transforms.ToTensor()(scratch_image)
133 |         scratch_image = tv.transforms.Normalize([0.5], [0.5])(scratch_image)
134 |         scratch_image = torch.unsqueeze(scratch_image, 0)
135 |         _, _, ow, oh = scratch_image.shape
136 |         scratch_image_scale = scale_tensor(scratch_image)
137 | 
138 |         if gpu >= 0:
139 |             scratch_image_scale = scratch_image_scale.to(gpu)
140 |         else:
141 |             scratch_image_scale = scratch_image_scale.cpu()
142 |         with torch.no_grad():
143 |             P = torch.sigmoid(model(scratch_image_scale))
144 | 
145 |         P = P.data.cpu()
146 |         P = F.interpolate(P, [ow, oh], mode="nearest")
147 | 
148 |         tv.utils.save_image(
149 |             (P >= 0.4).float(),
150 |             os.path.join(
151 |                 output_dir,
152 |                 image_name[:-4] + ".png",
153 |             ),
154 |             nrow=1,
155 |             padding=0,
156 |             normalize=True,
157 |         )
158 |         transformed_image_PIL.save(os.path.join(input_dir, image_name[:-4] + ".png"))
159 |         gc.collect()
160 |         torch.cuda.empty_cache()
161 | 
162 | # Wrap the scratch detection in a class
163 | class ScratchDetection:
164 |     def __init__(self, test_path, output_dir, input_size="scale_256", gpu=0):
165 |         self.test_path = test_path
166 |         self.output_dir = output_dir
167 |         self.input_size = input_size
168 |         self.gpu = gpu
169 | 
170 |     def run(self):
171 |         process_images(self.test_path, self.output_dir, self.input_size, self.gpu)
172 | 
173 |     # Add a function to get the mask image from the output directory
174 |     def get_mask_image(self, image_name):
175 |         mask_image_path = os.path.join(self.output_dir, "mask", image_name)
176 |         return Image.open(mask_image_path)
177 | 
178 | # Keep the __main__ part, but modify it to use the new ScratchDetection class
179 | if __name__ == "__main__":
180 |     parser = argparse.ArgumentParser()
181 |     parser.add_argument("--GPU", type=int, default=0)
182 |     parser.add_argument("--test_path", type=str, default=".")
183 |     parser.add_argument("--output_dir", type=str, default=".")
184 |     parser.add_argument("--input_size", type=str, default="scale_256", help="resize_256|full_size|scale_256")
185 |     args = parser.parse_args()
186 | 
187 |     scratch_detector = ScratchDetection(args.test_path, args.output_dir, args.input_size, args.GPU)
188 |     scratch_detector.run()
189 | 


--------------------------------------------------------------------------------
/src/__pycache__/pipeline_stable_diffusion_controlnet_inpaint.cpython-311.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/src/__pycache__/pipeline_stable_diffusion_controlnet_inpaint.cpython-311.pyc


--------------------------------------------------------------------------------
/src/pipeline_stable_diffusion_controlnet_inpaint.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import PIL.Image
  3 | import numpy as np
  4 | 
  5 | from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import *
  6 | 
  7 | EXAMPLE_DOC_STRING = """
  8 |     Examples:
  9 |         ```py
 10 |         >>> # !pip install opencv-python transformers accelerate
 11 |         >>> from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, UniPCMultistepScheduler
 12 |         >>> from diffusers.utils import load_image
 13 |         >>> import numpy as np
 14 |         >>> import torch
 15 |         >>> import cv2
 16 |         >>> from PIL import Image
 17 |         >>> # download an image
 18 |         >>> image = load_image(
 19 |         ...     "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
 20 |         ... )
 21 |         >>> image = np.array(image)
 22 |         >>> mask_image = load_image(
 23 |         ...     "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
 24 |         ... )
 25 |         >>> mask_image = np.array(mask_image)
 26 |         >>> # get canny image
 27 |         >>> canny_image = cv2.Canny(image, 100, 200)
 28 |         >>> canny_image = canny_image[:, :, None]
 29 |         >>> canny_image = np.concatenate([canny_image, canny_image, canny_image], axis=2)
 30 |         >>> canny_image = Image.fromarray(canny_image)
 31 |         >>> # load control net and stable diffusion v1-5
 32 |         >>> controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
 33 |         >>> pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
 34 |         ...     "runwayml/stable-diffusion-inpainting", controlnet=controlnet, torch_dtype=torch.float16
 35 |         ... )
 36 |         >>> # speed up diffusion process with faster scheduler and memory optimization
 37 |         >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
 38 |         >>> # remove following line if xformers is not installed
 39 |         >>> pipe.enable_xformers_memory_efficient_attention()
 40 |         >>> pipe.enable_model_cpu_offload()
 41 |         >>> # generate image
 42 |         >>> generator = torch.manual_seed(0)
 43 |         >>> image = pipe(
 44 |         ...     "futuristic-looking doggo",
 45 |         ...     num_inference_steps=20,
 46 |         ...     generator=generator,
 47 |         ...     image=image,
 48 |         ...     control_image=canny_image,
 49 |         ...     mask_image=mask_image
 50 |         ... ).images[0]
 51 |         ```
 52 | """
 53 | 
 54 | 
 55 | def prepare_mask_and_masked_image(image, mask):
 56 |     """
 57 |     Prepares a pair (image, mask) to be consumed by the Stable Diffusion pipeline. This means that those inputs will be
 58 |     converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for the
 59 |     ``image`` and ``1`` for the ``mask``.
 60 |     The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be
 61 |     binarized (``mask > 0.5``) and cast to ``torch.float32`` too.
 62 |     Args:
 63 |         image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint.
 64 |             It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width``
 65 |             ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``.
 66 |         mask (_type_): The mask to apply to the image, i.e. regions to inpaint.
 67 |             It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width``
 68 |             ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``.
 69 |     Raises:
 70 |         ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask
 71 |         should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions.
 72 |         TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not
 73 |             (ot the other way around).
 74 |     Returns:
 75 |         tuple[torch.Tensor]: The pair (mask, masked_image) as ``torch.Tensor`` with 4
 76 |             dimensions: ``batch x channels x height x width``.
 77 |     """
 78 |     if isinstance(image, torch.Tensor):
 79 |         if not isinstance(mask, torch.Tensor):
 80 |             raise TypeError(f"`image` is a torch.Tensor but `mask` (type: {type(mask)} is not")
 81 | 
 82 |         # Batch single image
 83 |         if image.ndim == 3:
 84 |             assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)"
 85 |             image = image.unsqueeze(0)
 86 | 
 87 |         # Batch and add channel dim for single mask
 88 |         if mask.ndim == 2:
 89 |             mask = mask.unsqueeze(0).unsqueeze(0)
 90 | 
 91 |         # Batch single mask or add channel dim
 92 |         if mask.ndim == 3:
 93 |             # Single batched mask, no channel dim or single mask not batched but channel dim
 94 |             if mask.shape[0] == 1:
 95 |                 mask = mask.unsqueeze(0)
 96 | 
 97 |             # Batched masks no channel dim
 98 |             else:
 99 |                 mask = mask.unsqueeze(1)
100 | 
101 |         assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions"
102 |         assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions"
103 |         assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size"
104 | 
105 |         # Check image is in [-1, 1]
106 |         if image.min() < -1 or image.max() > 1:
107 |             raise ValueError("Image should be in [-1, 1] range")
108 | 
109 |         # Check mask is in [0, 1]
110 |         if mask.min() < 0 or mask.max() > 1:
111 |             raise ValueError("Mask should be in [0, 1] range")
112 | 
113 |         # Binarize mask
114 |         mask[mask < 0.5] = 0
115 |         mask[mask >= 0.5] = 1
116 | 
117 |         # Image as float32
118 |         image = image.to(dtype=torch.float32)
119 |     elif isinstance(mask, torch.Tensor):
120 |         raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not")
121 |     else:
122 |         # preprocess image
123 |         if isinstance(image, (PIL.Image.Image, np.ndarray)):
124 |             image = [image]
125 | 
126 |         if isinstance(image, list) and isinstance(image[0], PIL.Image.Image):
127 |             image = [np.array(i.convert("RGB"))[None, :] for i in image]
128 |             image = np.concatenate(image, axis=0)
129 |         elif isinstance(image, list) and isinstance(image[0], np.ndarray):
130 |             image = np.concatenate([i[None, :] for i in image], axis=0)
131 | 
132 |         image = image.transpose(0, 3, 1, 2)
133 |         image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0
134 | 
135 |         # preprocess mask
136 |         if isinstance(mask, (PIL.Image.Image, np.ndarray)):
137 |             mask = [mask]
138 | 
139 |         if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image):
140 |             mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0)
141 |             mask = mask.astype(np.float32) / 255.0
142 |         elif isinstance(mask, list) and isinstance(mask[0], np.ndarray):
143 |             mask = np.concatenate([m[None, None, :] for m in mask], axis=0)
144 | 
145 |         mask[mask < 0.5] = 0
146 |         mask[mask >= 0.5] = 1
147 |         mask = torch.from_numpy(mask)
148 | 
149 |     masked_image = image * (mask < 0.5)
150 | 
151 |     return mask, masked_image
152 | 
153 | class StableDiffusionControlNetInpaintPipeline(StableDiffusionControlNetPipeline):
154 |     r"""
155 |     Pipeline for text-guided image inpainting using Stable Diffusion with ControlNet guidance.
156 |     This model inherits from [`StableDiffusionControlNetPipeline`]. Check the superclass documentation for the generic methods the
157 |     library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
158 |     Args:
159 |         vae ([`AutoencoderKL`]):
160 |             Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
161 |         text_encoder ([`CLIPTextModel`]):
162 |             Frozen text-encoder. Stable Diffusion uses the text portion of
163 |             [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
164 |             the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
165 |         tokenizer (`CLIPTokenizer`):
166 |             Tokenizer of class
167 |             [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
168 |         unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
169 |         controlnet ([`ControlNetModel`]):
170 |             Provides additional conditioning to the unet during the denoising process
171 |         scheduler ([`SchedulerMixin`]):
172 |             A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
173 |             [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
174 |         safety_checker ([`StableDiffusionSafetyChecker`]):
175 |             Classification module that estimates whether generated images could be considered offensive or harmful.
176 |             Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.
177 |         feature_extractor ([`CLIPFeatureExtractor`]):
178 |             Model that extracts features from generated images to be used as inputs for the `safety_checker`.
179 |     """
180 |     
181 |     def prepare_mask_latents(
182 |         self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance
183 |     ):
184 |         # resize the mask to latents shape as we concatenate the mask to the latents
185 |         # we do that before converting to dtype to avoid breaking in case we're using cpu_offload
186 |         # and half precision
187 |         mask = torch.nn.functional.interpolate(
188 |             mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor)
189 |         )
190 |         mask = mask.to(device=device, dtype=dtype)
191 | 
192 |         masked_image = masked_image.to(device=device, dtype=dtype)
193 | 
194 |         # encode the mask image into latents space so we can concatenate it to the latents
195 |         if isinstance(generator, list):
196 |             masked_image_latents = [
197 |                 self.vae.encode(masked_image[i : i + 1]).latent_dist.sample(generator=generator[i])
198 |                 for i in range(batch_size)
199 |             ]
200 |             masked_image_latents = torch.cat(masked_image_latents, dim=0)
201 |         else:
202 |             masked_image_latents = self.vae.encode(masked_image).latent_dist.sample(generator=generator)
203 |         masked_image_latents = self.vae.config.scaling_factor * masked_image_latents
204 | 
205 |         # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method
206 |         if mask.shape[0] < batch_size:
207 |             if not batch_size % mask.shape[0] == 0:
208 |                 raise ValueError(
209 |                     "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
210 |                     f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
211 |                     " of masks that you pass is divisible by the total requested batch size."
212 |                 )
213 |             mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
214 |         if masked_image_latents.shape[0] < batch_size:
215 |             if not batch_size % masked_image_latents.shape[0] == 0:
216 |                 raise ValueError(
217 |                     "The passed images and the required batch size don't match. Images are supposed to be duplicated"
218 |                     f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
219 |                     " Make sure the number of images that you pass is divisible by the total requested batch size."
220 |                 )
221 |             masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1)
222 | 
223 |         mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask
224 |         masked_image_latents = (
225 |             torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents
226 |         )
227 | 
228 |         # aligning device to prevent device errors when concating it with the latent model input
229 |         masked_image_latents = masked_image_latents.to(device=device, dtype=dtype)
230 |         return mask, masked_image_latents
231 |     
232 |     @torch.no_grad()
233 |     @replace_example_docstring(EXAMPLE_DOC_STRING)
234 |     def __call__(
235 |         self,
236 |         prompt: Union[str, List[str]] = None,        
237 |         image: Union[torch.FloatTensor, PIL.Image.Image] = None,
238 |         control_image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]] = None,        
239 |         mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None,
240 |         height: Optional[int] = None,
241 |         width: Optional[int] = None,
242 |         num_inference_steps: int = 50,
243 |         guidance_scale: float = 7.5,
244 |         negative_prompt: Optional[Union[str, List[str]]] = None,
245 |         num_images_per_prompt: Optional[int] = 1,
246 |         eta: float = 0.0,
247 |         generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
248 |         latents: Optional[torch.FloatTensor] = None,
249 |         prompt_embeds: Optional[torch.FloatTensor] = None,
250 |         negative_prompt_embeds: Optional[torch.FloatTensor] = None,
251 |         output_type: Optional[str] = "pil",
252 |         return_dict: bool = True,
253 |         callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
254 |         callback_steps: int = 1,
255 |         cross_attention_kwargs: Optional[Dict[str, Any]] = None,
256 |         controlnet_conditioning_scale: float = 1.0,
257 |     ):
258 |         r"""
259 |         Function invoked when calling the pipeline for generation.
260 |         Args:
261 |             prompt (`str` or `List[str]`, *optional*):
262 |                 The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
263 |                 instead.
264 |             image (`PIL.Image.Image`):
265 |                 `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will
266 |                 be masked out with `mask_image` and repainted according to `prompt`.
267 |             control_image (`torch.FloatTensor`, `PIL.Image.Image`, `List[torch.FloatTensor]` or `List[PIL.Image.Image]`):
268 |                 The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If
269 |                 the type is specified as `Torch.FloatTensor`, it is passed to ControlNet as is. PIL.Image.Image` can
270 |                 also be accepted as an image. The control image is automatically resized to fit the output image.
271 |             mask_image (`PIL.Image.Image`):
272 |                 `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
273 |                 repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted
274 |                 to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L)
275 |                 instead of 3, so the expected shape would be `(B, H, W, 1)`.            
276 |             height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
277 |                 The height in pixels of the generated image.
278 |             width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
279 |                 The width in pixels of the generated image.
280 |             num_inference_steps (`int`, *optional*, defaults to 50):
281 |                 The number of denoising steps. More denoising steps usually lead to a higher quality image at the
282 |                 expense of slower inference.
283 |             guidance_scale (`float`, *optional*, defaults to 7.5):
284 |                 Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
285 |                 `guidance_scale` is defined as `w` of equation 2. of [Imagen
286 |                 Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
287 |                 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
288 |                 usually at the expense of lower image quality.
289 |             negative_prompt (`str` or `List[str]`, *optional*):
290 |                 The prompt or prompts not to guide the image generation. If not defined, one has to pass
291 |                 `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.
292 |                 Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).
293 |             num_images_per_prompt (`int`, *optional*, defaults to 1):
294 |                 The number of images to generate per prompt.
295 |             eta (`float`, *optional*, defaults to 0.0):
296 |                 Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
297 |                 [`schedulers.DDIMScheduler`], will be ignored for others.
298 |             generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
299 |                 One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
300 |                 to make generation deterministic.
301 |             latents (`torch.FloatTensor`, *optional*):
302 |                 Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
303 |                 generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
304 |                 tensor will ge generated by sampling using the supplied random `generator`.
305 |             prompt_embeds (`torch.FloatTensor`, *optional*):
306 |                 Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
307 |                 provided, text embeddings will be generated from `prompt` input argument.
308 |             negative_prompt_embeds (`torch.FloatTensor`, *optional*):
309 |                 Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
310 |                 weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
311 |                 argument.
312 |             output_type (`str`, *optional*, defaults to `"pil"`):
313 |                 The output format of the generate image. Choose between
314 |                 [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
315 |             return_dict (`bool`, *optional*, defaults to `True`):
316 |                 Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
317 |                 plain tuple.
318 |             callback (`Callable`, *optional*):
319 |                 A function that will be called every `callback_steps` steps during inference. The function will be
320 |                 called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
321 |             callback_steps (`int`, *optional*, defaults to 1):
322 |                 The frequency at which the `callback` function will be called. If not specified, the callback will be
323 |                 called at every step.
324 |             cross_attention_kwargs (`dict`, *optional*):
325 |                 A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under
326 |                 `self.processor` in
327 |                 [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).
328 |             controlnet_conditioning_scale (`float`, *optional*, defaults to 1.0):
329 |                 The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added
330 |                 to the residual in the original unet.
331 |         Examples:
332 |         Returns:
333 |             [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
334 |             [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
335 |             When returning a tuple, the first element is a list with the generated images, and the second element is a
336 |             list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
337 |             (nsfw) content, according to the `safety_checker`.
338 |         """
339 |         # 0. Default height and width to unet
340 |         height, width = self._default_height_width(height, width, control_image)
341 | 
342 |         # 1. Check inputs. Raise error if not correct
343 |         self.check_inputs(
344 |             prompt, control_image, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds
345 |         )
346 | 
347 |         # 2. Define call parameters
348 |         if prompt is not None and isinstance(prompt, str):
349 |             batch_size = 1
350 |         elif prompt is not None and isinstance(prompt, list):
351 |             batch_size = len(prompt)
352 |         else:
353 |             batch_size = prompt_embeds.shape[0]
354 | 
355 |         device = self._execution_device
356 |         # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
357 |         # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
358 |         # corresponds to doing no classifier free guidance.
359 |         do_classifier_free_guidance = guidance_scale > 1.0
360 | 
361 |         # 3. Encode input prompt
362 |         prompt_embeds = self._encode_prompt(
363 |             prompt,
364 |             device,
365 |             num_images_per_prompt,
366 |             do_classifier_free_guidance,
367 |             negative_prompt,
368 |             prompt_embeds=prompt_embeds,
369 |             negative_prompt_embeds=negative_prompt_embeds,
370 |         )
371 | 
372 |         # 4. Prepare image
373 |         control_image = self.prepare_image(
374 |             control_image,
375 |             width,
376 |             height,
377 |             batch_size * num_images_per_prompt,
378 |             num_images_per_prompt,
379 |             device,
380 |             self.controlnet.dtype,
381 |         )
382 | 
383 |         if do_classifier_free_guidance:
384 |             control_image = torch.cat([control_image] * 2)
385 | 
386 |         # 5. Prepare timesteps
387 |         self.scheduler.set_timesteps(num_inference_steps, device=device)
388 |         timesteps = self.scheduler.timesteps
389 | 
390 |         # 6. Prepare latent variables
391 |         num_channels_latents = self.controlnet.in_channels
392 |         latents = self.prepare_latents(
393 |             batch_size * num_images_per_prompt,
394 |             num_channels_latents,
395 |             height,
396 |             width,
397 |             prompt_embeds.dtype,
398 |             device,
399 |             generator,
400 |             latents,
401 |         )
402 |         
403 |         # EXTRA: prepare mask latents
404 |         mask, masked_image = prepare_mask_and_masked_image(image, mask_image)
405 |         mask, masked_image_latents = self.prepare_mask_latents(
406 |             mask,
407 |             masked_image,
408 |             batch_size * num_images_per_prompt,
409 |             height,
410 |             width,
411 |             prompt_embeds.dtype,
412 |             device,
413 |             generator,
414 |             do_classifier_free_guidance,
415 |         )
416 | 
417 |         # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
418 |         extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
419 | 
420 |         # 8. Denoising loop
421 |         num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
422 |         with self.progress_bar(total=num_inference_steps) as progress_bar:
423 |             for i, t in enumerate(timesteps):
424 |                 # expand the latents if we are doing classifier free guidance
425 |                 latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
426 |                 latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)                                
427 | 
428 |                 down_block_res_samples, mid_block_res_sample = self.controlnet(
429 |                     latent_model_input,
430 |                     t,
431 |                     encoder_hidden_states=prompt_embeds,
432 |                     controlnet_cond=control_image,
433 |                     return_dict=False,
434 |                 )
435 | 
436 |                 down_block_res_samples = [
437 |                     down_block_res_sample * controlnet_conditioning_scale
438 |                     for down_block_res_sample in down_block_res_samples
439 |                 ]
440 |                 mid_block_res_sample *= controlnet_conditioning_scale
441 | 
442 |                 # predict the noise residual
443 |                 latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1)
444 |                 noise_pred = self.unet(
445 |                     latent_model_input,
446 |                     t,
447 |                     encoder_hidden_states=prompt_embeds,
448 |                     cross_attention_kwargs=cross_attention_kwargs,
449 |                     down_block_additional_residuals=down_block_res_samples,
450 |                     mid_block_additional_residual=mid_block_res_sample,
451 |                 ).sample
452 | 
453 |                 # perform guidance
454 |                 if do_classifier_free_guidance:
455 |                     noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
456 |                     noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
457 | 
458 |                 # compute the previous noisy sample x_t -> x_t-1
459 |                 latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
460 | 
461 |                 # call the callback, if provided
462 |                 if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
463 |                     progress_bar.update()
464 |                     if callback is not None and i % callback_steps == 0:
465 |                         callback(i, t, latents)
466 | 
467 |         # If we do sequential model offloading, let's offload unet and controlnet
468 |         # manually for max memory savings
469 |         if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:
470 |             self.unet.to("cpu")
471 |             self.controlnet.to("cpu")
472 |             torch.cuda.empty_cache()
473 | 
474 |         if output_type == "latent":
475 |             image = latents
476 |             has_nsfw_concept = None
477 |         elif output_type == "pil":
478 |             # 8. Post-processing
479 |             image = self.decode_latents(latents)
480 | 
481 |             # 9. Run safety checker
482 |             image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
483 | 
484 |             # 10. Convert to PIL
485 |             image = self.numpy_to_pil(image)
486 |         else:
487 |             # 8. Post-processing
488 |             image = self.decode_latents(latents)
489 | 
490 |             # 9. Run safety checker
491 |             image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
492 | 
493 |         # Offload last model to CPU
494 |         if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:
495 |             self.final_offload_hook.offload()
496 | 
497 |         if not return_dict:
498 |             return (image, has_nsfw_concept)
499 | 
500 |         return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)


--------------------------------------------------------------------------------
/test.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/test.png


--------------------------------------------------------------------------------
/test_label.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/test_label.png


--------------------------------------------------------------------------------
/tfboard_loss.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/tfboard_loss.jpg


--------------------------------------------------------------------------------
/tfboard_miou.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/tfboard_miou.jpg


--------------------------------------------------------------------------------
/tfboard_precision.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/vijishmadhavan/UnpromptedControl/49c43aabecafdc0b7ed3663b3975825a6c3a4b24/tfboard_precision.jpg


--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | from torch.utils.data import Dataset
  3 | from torch.utils.data import DataLoader
  4 | from dataset.CamVid import CamVid
  5 | import os
  6 | from model.build_BiSeNet import BiSeNet
  7 | import torch
  8 | from tensorboardX import SummaryWriter
  9 | import tqdm
 10 | import numpy as np
 11 | from utils import poly_lr_scheduler
 12 | from utils import reverse_one_hot, compute_global_accuracy, fast_hist, \
 13 |     per_class_iu
 14 | from loss import DiceLoss
 15 | 
 16 | 
 17 | def val(args, model, dataloader):
 18 |     print('start val!')
 19 |     # label_info = get_label_info(csv_path)
 20 |     with torch.no_grad():
 21 |         model.eval()
 22 |         precision_record = []
 23 |         hist = np.zeros((args.num_classes, args.num_classes))
 24 |         for i, (data, label) in enumerate(dataloader):
 25 |             if torch.cuda.is_available() and args.use_gpu:
 26 |                 data = data.cuda()
 27 |                 label = label.cuda()
 28 | 
 29 |             # get RGB predict image
 30 |             predict = model(data).squeeze()
 31 |             predict = reverse_one_hot(predict)
 32 |             predict = np.array(predict)
 33 | 
 34 |             # get RGB label image
 35 |             label = label.squeeze()
 36 |             if args.loss == 'dice':
 37 |                 label = reverse_one_hot(label)
 38 |             label = np.array(label)
 39 | 
 40 |             # compute per pixel accuracy
 41 | 
 42 |             precision = compute_global_accuracy(predict, label)
 43 |             hist += fast_hist(label.flatten(), predict.flatten(), args.num_classes)
 44 | 
 45 |             # there is no need to transform the one-hot array to visual RGB array
 46 |             # predict = colour_code_segmentation(np.array(predict), label_info)
 47 |             # label = colour_code_segmentation(np.array(label), label_info)
 48 |             precision_record.append(precision)
 49 |         precision = np.mean(precision_record)
 50 |         # miou = np.mean(per_class_iu(hist))
 51 |         miou_list = per_class_iu(hist)[:-1]
 52 |         # miou_dict, miou = cal_miou(miou_list, csv_path)
 53 |         miou = np.mean(miou_list)
 54 |         print('precision per pixel for test: %.3f' % precision)
 55 |         print('mIoU for validation: %.3f' % miou)
 56 |         # miou_str = ''
 57 |         # for key in miou_dict:
 58 |         #     miou_str += '{}:{},\n'.format(key, miou_dict[key])
 59 |         # print('mIoU for each class:')
 60 |         # print(miou_str)
 61 |         return precision, miou
 62 | 
 63 | 
 64 | def train(args, model, optimizer, dataloader_train, dataloader_val):
 65 |     writer = SummaryWriter(comment=''.format(args.optimizer, args.context_path))
 66 |     if args.loss == 'dice':
 67 |         loss_func = DiceLoss()
 68 |     elif args.loss == 'crossentropy':
 69 |         loss_func = torch.nn.CrossEntropyLoss()
 70 |     max_miou = 0
 71 |     step = 0
 72 |     for epoch in range(args.num_epochs):
 73 |         lr = poly_lr_scheduler(optimizer, args.learning_rate, iter=epoch, max_iter=args.num_epochs)
 74 |         model.train()
 75 |         tq = tqdm.tqdm(total=len(dataloader_train) * args.batch_size)
 76 |         tq.set_description('epoch %d, lr %f' % (epoch, lr))
 77 |         loss_record = []
 78 |         for i, (data, label) in enumerate(dataloader_train):
 79 |             if torch.cuda.is_available() and args.use_gpu:
 80 |                 data = data.cuda()
 81 |                 label = label.cuda()
 82 |             output, output_sup1, output_sup2 = model(data)
 83 |             loss1 = loss_func(output, label)
 84 |             loss2 = loss_func(output_sup1, label)
 85 |             loss3 = loss_func(output_sup2, label)
 86 |             loss = loss1 + loss2 + loss3
 87 |             tq.update(args.batch_size)
 88 |             tq.set_postfix(loss='%.6f' % loss)
 89 |             optimizer.zero_grad()
 90 |             loss.backward()
 91 |             optimizer.step()
 92 |             step += 1
 93 |             writer.add_scalar('loss_step', loss, step)
 94 |             loss_record.append(loss.item())
 95 |         tq.close()
 96 |         loss_train_mean = np.mean(loss_record)
 97 |         writer.add_scalar('epoch/loss_epoch_train', float(loss_train_mean), epoch)
 98 |         print('loss for train : %f' % (loss_train_mean))
 99 |         if epoch % args.checkpoint_step == 0 and epoch != 0:
100 |             if not os.path.isdir(args.save_model_path):
101 |                 os.mkdir(args.save_model_path)
102 |             torch.save(model.module.state_dict(),
103 |                        os.path.join(args.save_model_path, 'latest_dice_loss.pth'))
104 | 
105 |         if epoch % args.validation_step == 0:
106 |             precision, miou = val(args, model, dataloader_val)
107 |             if miou > max_miou:
108 |                 max_miou = miou
109 |                 torch.save(model.module.state_dict(),
110 |                            os.path.join(args.save_model_path, 'best_dice_loss.pth'))
111 |             writer.add_scalar('epoch/precision_val', precision, epoch)
112 |             writer.add_scalar('epoch/miou val', miou, epoch)
113 | 
114 | 
115 | def main(params):
116 |     # basic parameters
117 |     parser = argparse.ArgumentParser()
118 |     parser.add_argument('--num_epochs', type=int, default=300, help='Number of epochs to train for')
119 |     parser.add_argument('--epoch_start_i', type=int, default=0, help='Start counting epochs from this number')
120 |     parser.add_argument('--checkpoint_step', type=int, default=1, help='How often to save checkpoints (epochs)')
121 |     parser.add_argument('--validation_step', type=int, default=1, help='How often to perform validation (epochs)')
122 |     parser.add_argument('--dataset', type=str, default="CamVid", help='Dataset you are using.')
123 |     parser.add_argument('--crop_height', type=int, default=720, help='Height of cropped/resized input image to network')
124 |     parser.add_argument('--crop_width', type=int, default=960, help='Width of cropped/resized input image to network')
125 |     parser.add_argument('--batch_size', type=int, default=1, help='Number of images in each batch')
126 |     parser.add_argument('--context_path', type=str, default="resnet101",
127 |                         help='The context path model you are using, resnet18, resnet101.')
128 |     parser.add_argument('--learning_rate', type=float, default=0.01, help='learning rate used for train')
129 |     parser.add_argument('--data', type=str, default='', help='path of training data')
130 |     parser.add_argument('--num_workers', type=int, default=4, help='num of workers')
131 |     parser.add_argument('--num_classes', type=int, default=32, help='num of object classes (with void)')
132 |     parser.add_argument('--cuda', type=str, default='0', help='GPU ids used for training')
133 |     parser.add_argument('--use_gpu', type=bool, default=True, help='whether to user gpu for training')
134 |     parser.add_argument('--pretrained_model_path', type=str, default=None, help='path to pretrained model')
135 |     parser.add_argument('--save_model_path', type=str, default=None, help='path to save model')
136 |     parser.add_argument('--optimizer', type=str, default='rmsprop', help='optimizer, support rmsprop, sgd, adam')
137 |     parser.add_argument('--loss', type=str, default='dice', help='loss function, dice or crossentropy')
138 | 
139 |     args = parser.parse_args(params)
140 | 
141 |     # create dataset and dataloader
142 |     train_path = [os.path.join(args.data, 'train'), os.path.join(args.data, 'val')]
143 |     train_label_path = [os.path.join(args.data, 'train_labels'), os.path.join(args.data, 'val_labels')]
144 |     test_path = os.path.join(args.data, 'test')
145 |     test_label_path = os.path.join(args.data, 'test_labels')
146 |     csv_path = os.path.join(args.data, 'class_dict.csv')
147 |     dataset_train = CamVid(train_path, train_label_path, csv_path, scale=(args.crop_height, args.crop_width),
148 |                            loss=args.loss, mode='train')
149 |     dataloader_train = DataLoader(
150 |         dataset_train,
151 |         batch_size=args.batch_size,
152 |         shuffle=True,
153 |         num_workers=args.num_workers,
154 |         drop_last=True
155 |     )
156 |     dataset_val = CamVid(test_path, test_label_path, csv_path, scale=(args.crop_height, args.crop_width),
157 |                          loss=args.loss, mode='test')
158 |     dataloader_val = DataLoader(
159 |         dataset_val,
160 |         # this has to be 1
161 |         batch_size=1,
162 |         shuffle=True,
163 |         num_workers=args.num_workers
164 |     )
165 | 
166 |     # build model
167 |     os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda
168 |     model = BiSeNet(args.num_classes, args.context_path)
169 |     if torch.cuda.is_available() and args.use_gpu:
170 |         model = torch.nn.DataParallel(model).cuda()
171 | 
172 |     # build optimizer
173 |     if args.optimizer == 'rmsprop':
174 |         optimizer = torch.optim.RMSprop(model.parameters(), args.learning_rate)
175 |     elif args.optimizer == 'sgd':
176 |         optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=0.9, weight_decay=1e-4)
177 |     elif args.optimizer == 'adam':
178 |         optimizer = torch.optim.Adam(model.parameters(), args.learning_rate)
179 |     else:  # rmsprop
180 |         print('not supported optimizer \n')
181 |         return None
182 | 
183 |     # load pretrained model if exists
184 |     if args.pretrained_model_path is not None:
185 |         print('load model from %s ...' % args.pretrained_model_path)
186 |         model.module.load_state_dict(torch.load(args.pretrained_model_path))
187 |         print('Done!')
188 | 
189 |     # train
190 |     train(args, model, optimizer, dataloader_train, dataloader_val)
191 | 
192 |     # val(args, model, dataloader_val, csv_path)
193 | 
194 | 
195 | if __name__ == '__main__':
196 |     params = [
197 |         '--num_epochs', '1000',
198 |         '--learning_rate', '2.5e-2',
199 |         '--data', '/path/to/CamVid',
200 |         '--num_workers', '8',
201 |         '--num_classes', '12',
202 |         '--cuda', '0',
203 |         '--batch_size', '2',  # 6 for resnet101, 12 for resnet18
204 |         '--save_model_path', './checkpoints_18_sgd',
205 |         '--context_path', 'resnet18',  # only support resnet18 and resnet101
206 |         '--optimizer', 'sgd',
207 | 
208 |     ]
209 |     main(params)
210 | 
211 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
  1 | import torch.nn as nn
  2 | import torch
  3 | from torch.nn import functional as F
  4 | from PIL import Image
  5 | import numpy as np
  6 | import pandas as pd
  7 | import random
  8 | import numbers
  9 | import torchvision
 10 | 
 11 | def poly_lr_scheduler(optimizer, init_lr, iter, lr_decay_iter=1,
 12 |                       max_iter=300, power=0.9):
 13 | 	"""Polynomial decay of learning rate
 14 | 		:param init_lr is base learning rate
 15 | 		:param iter is a current iteration
 16 | 		:param lr_decay_iter how frequently decay occurs, default is 1
 17 | 		:param max_iter is number of maximum iterations
 18 | 		:param power is a polymomial power
 19 | 
 20 | 	"""
 21 | 	# if iter % lr_decay_iter or iter > max_iter:
 22 | 	# 	return optimizer
 23 | 
 24 | 	lr = init_lr*(1 - iter/max_iter)**power
 25 | 	optimizer.param_groups[0]['lr'] = lr
 26 | 	return lr
 27 | 	# return lr
 28 | 
 29 | def get_label_info(csv_path):
 30 | 	# return label -> {label_name: [r_value, g_value, b_value, ...}
 31 | 	ann = pd.read_csv(csv_path)
 32 | 	label = {}
 33 | 	for iter, row in ann.iterrows():
 34 | 		label_name = row['name']
 35 | 		r = row['r']
 36 | 		g = row['g']
 37 | 		b = row['b']
 38 | 		class_11 = row['class_11']
 39 | 		label[label_name] = [int(r), int(g), int(b), class_11]
 40 | 	return label
 41 | 
 42 | def one_hot_it(label, label_info):
 43 | 	# return semantic_map -> [H, W]
 44 | 	semantic_map = np.zeros(label.shape[:-1])
 45 | 	for index, info in enumerate(label_info):
 46 | 		color = label_info[info]
 47 | 		# colour_map = np.full((label.shape[0], label.shape[1], label.shape[2]), colour, dtype=int)
 48 | 		equality = np.equal(label, color)
 49 | 		class_map = np.all(equality, axis=-1)
 50 | 		semantic_map[class_map] = index
 51 | 		# semantic_map.append(class_map)
 52 | 	# semantic_map = np.stack(semantic_map, axis=-1)
 53 | 	return semantic_map
 54 | 
 55 | 
 56 | def one_hot_it_v11(label, label_info):
 57 | 	# return semantic_map -> [H, W, class_num]
 58 | 	semantic_map = np.zeros(label.shape[:-1])
 59 | 	# from 0 to 11, and 11 means void
 60 | 	class_index = 0
 61 | 	for index, info in enumerate(label_info):
 62 | 		color = label_info[info][:3]
 63 | 		class_11 = label_info[info][3]
 64 | 		if class_11 == 1:
 65 | 			# colour_map = np.full((label.shape[0], label.shape[1], label.shape[2]), colour, dtype=int)
 66 | 			equality = np.equal(label, color)
 67 | 			class_map = np.all(equality, axis=-1)
 68 | 			# semantic_map[class_map] = index
 69 | 			semantic_map[class_map] = class_index
 70 | 			class_index += 1
 71 | 		else:
 72 | 			equality = np.equal(label, color)
 73 | 			class_map = np.all(equality, axis=-1)
 74 | 			semantic_map[class_map] = 11
 75 | 	return semantic_map
 76 | 
 77 | def one_hot_it_v11_dice(label, label_info):
 78 | 	# return semantic_map -> [H, W, class_num]
 79 | 	semantic_map = []
 80 | 	void = np.zeros(label.shape[:2])
 81 | 	for index, info in enumerate(label_info):
 82 | 		color = label_info[info][:3]
 83 | 		class_11 = label_info[info][3]
 84 | 		if class_11 == 1:
 85 | 			# colour_map = np.full((label.shape[0], label.shape[1], label.shape[2]), colour, dtype=int)
 86 | 			equality = np.equal(label, color)
 87 | 			class_map = np.all(equality, axis=-1)
 88 | 			# semantic_map[class_map] = index
 89 | 			semantic_map.append(class_map)
 90 | 		else:
 91 | 			equality = np.equal(label, color)
 92 | 			class_map = np.all(equality, axis=-1)
 93 | 			void[class_map] = 1
 94 | 	semantic_map.append(void)
 95 | 	semantic_map = np.stack(semantic_map, axis=-1).astype(np.float)
 96 | 	return semantic_map
 97 | 
 98 | def reverse_one_hot(image):
 99 | 	"""
100 | 	Transform a 2D array in one-hot format (depth is num_classes),
101 | 	to a 2D array with only 1 channel, where each pixel value is
102 | 	the classified class key.
103 | 
104 | 	# Arguments
105 | 		image: The one-hot format image
106 | 
107 | 	# Returns
108 | 		A 2D array with the same width and height as the input, but
109 | 		with a depth size of 1, where each pixel value is the classified
110 | 		class key.
111 | 	"""
112 | 	# w = image.shape[0]
113 | 	# h = image.shape[1]
114 | 	# x = np.zeros([w,h,1])
115 | 
116 | 	# for i in range(0, w):
117 | 	#     for j in range(0, h):
118 | 	#         index, value = max(enumerate(image[i, j, :]), key=operator.itemgetter(1))
119 | 	#         x[i, j] = index
120 | 	image = image.permute(1, 2, 0)
121 | 	x = torch.argmax(image, dim=-1)
122 | 	return x
123 | 
124 | 
125 | def colour_code_segmentation(image, label_values):
126 | 	"""
127 |     Given a 1-channel array of class keys, colour code the segmentation results.
128 | 
129 |     # Arguments
130 |         image: single channel array where each value represents the class key.
131 |         label_values
132 | 
133 |     # Returns
134 |         Colour coded image for segmentation visualization
135 |     """
136 | 
137 | 	# w = image.shape[0]
138 | 	# h = image.shape[1]
139 | 	# x = np.zeros([w,h,3])
140 | 	# colour_codes = label_values
141 | 	# for i in range(0, w):
142 | 	#     for j in range(0, h):
143 | 	#         x[i, j, :] = colour_codes[int(image[i, j])]
144 | 	label_values = [label_values[key][:3] for key in label_values if label_values[key][3] == 1]
145 | 	label_values.append([0, 0, 0])
146 | 	colour_codes = np.array(label_values)
147 | 	x = colour_codes[image.astype(int)]
148 | 
149 | 	return x
150 | 
151 | def compute_global_accuracy(pred, label):
152 | 	pred = pred.flatten()
153 | 	label = label.flatten()
154 | 	total = len(label)
155 | 	count = 0.0
156 | 	for i in range(total):
157 | 		if pred[i] == label[i]:
158 | 			count = count + 1.0
159 | 	return float(count) / float(total)
160 | 
161 | def fast_hist(a, b, n):
162 | 	'''
163 | 	a and b are predict and mask respectively
164 | 	n is the number of classes
165 | 	'''
166 | 	k = (a >= 0) & (a < n)
167 | 	return np.bincount(n * a[k].astype(int) + b[k], minlength=n ** 2).reshape(n, n)
168 | 
169 | 
170 | def per_class_iu(hist):
171 | 	epsilon = 1e-5
172 | 	return (np.diag(hist) + epsilon) / (hist.sum(1) + hist.sum(0) - np.diag(hist) + epsilon)
173 | 
174 | class RandomCrop(object):
175 | 	"""Crop the given PIL Image at a random location.
176 | 
177 | 	Args:
178 | 		size (sequence or int): Desired output size of the crop. If size is an
179 | 			int instead of sequence like (h, w), a square crop (size, size) is
180 | 			made.
181 | 		padding (int or sequence, optional): Optional padding on each border
182 | 			of the image. Default is 0, i.e no padding. If a sequence of length
183 | 			4 is provided, it is used to pad left, top, right, bottom borders
184 | 			respectively.
185 | 		pad_if_needed (boolean): It will pad the image if smaller than the
186 | 			desired size to avoid raising an exception.
187 | 	"""
188 | 
189 | 	def __init__(self, size, seed, padding=0, pad_if_needed=False):
190 | 		if isinstance(size, numbers.Number):
191 | 			self.size = (int(size), int(size))
192 | 		else:
193 | 			self.size = size
194 | 		self.padding = padding
195 | 		self.pad_if_needed = pad_if_needed
196 | 		self.seed = seed
197 | 
198 | 	@staticmethod
199 | 	def get_params(img, output_size, seed):
200 | 		"""Get parameters for ``crop`` for a random crop.
201 | 
202 | 		Args:
203 | 			img (PIL Image): Image to be cropped.
204 | 			output_size (tuple): Expected output size of the crop.
205 | 
206 | 		Returns:
207 | 			tuple: params (i, j, h, w) to be passed to ``crop`` for random crop.
208 | 		"""
209 | 		random.seed(seed)
210 | 		w, h = img.size
211 | 		th, tw = output_size
212 | 		if w == tw and h == th:
213 | 			return 0, 0, h, w
214 | 		i = random.randint(0, h - th)
215 | 		j = random.randint(0, w - tw)
216 | 		return i, j, th, tw
217 | 
218 | 	def __call__(self, img):
219 | 		"""
220 | 		Args:
221 | 			img (PIL Image): Image to be cropped.
222 | 
223 | 		Returns:
224 | 			PIL Image: Cropped image.
225 | 		"""
226 | 		if self.padding > 0:
227 | 			img = torchvision.transforms.functional.pad(img, self.padding)
228 | 
229 | 		# pad the width if needed
230 | 		if self.pad_if_needed and img.size[0] < self.size[1]:
231 | 			img = torchvision.transforms.functional.pad(img, (int((1 + self.size[1] - img.size[0]) / 2), 0))
232 | 		# pad the height if needed
233 | 		if self.pad_if_needed and img.size[1] < self.size[0]:
234 | 			img = torchvision.transforms.functional.pad(img, (0, int((1 + self.size[0] - img.size[1]) / 2)))
235 | 
236 | 		i, j, h, w = self.get_params(img, self.size, self.seed)
237 | 
238 | 		return torchvision.transforms.functional.crop(img, i, j, h, w)
239 | 
240 | 	def __repr__(self):
241 | 		return self.__class__.__name__ + '(size={0}, padding={1})'.format(self.size, self.padding)
242 | 
243 | def cal_miou(miou_list, csv_path):
244 | 	# return label -> {label_name: [r_value, g_value, b_value, ...}
245 | 	ann = pd.read_csv(csv_path)
246 | 	miou_dict = {}
247 | 	cnt = 0
248 | 	for iter, row in ann.iterrows():
249 | 		label_name = row['name']
250 | 		class_11 = int(row['class_11'])
251 | 		if class_11 == 1:
252 | 			miou_dict[label_name] = miou_list[cnt]
253 | 			cnt += 1
254 | 	return miou_dict, np.mean(miou_list)
255 | 
256 | class OHEM_CrossEntroy_Loss(nn.Module):
257 | 	def __init__(self, threshold, keep_num):
258 | 		super(OHEM_CrossEntroy_Loss, self).__init__()
259 | 		self.threshold = threshold
260 | 		self.keep_num = keep_num
261 | 		self.loss_function = nn.CrossEntropyLoss(reduction='none')
262 | 
263 | 	def forward(self, output, target):
264 | 		loss = self.loss_function(output, target).view(-1)
265 | 		loss, loss_index = torch.sort(loss, descending=True)
266 | 		threshold_in_keep_num = loss[self.keep_num]
267 | 		if threshold_in_keep_num > self.threshold:
268 | 			loss = loss[loss>self.threshold]
269 | 		else:
270 | 			loss = loss[:self.keep_num]
271 | 		return torch.mean(loss)
272 | 
273 | def group_weight(weight_group, module, norm_layer, lr):
274 | 	group_decay = []
275 | 	group_no_decay = []
276 | 	for m in module.modules():
277 | 		if isinstance(m, nn.Linear):
278 | 			group_decay.append(m.weight)
279 | 			if m.bias is not None:
280 | 				group_no_decay.append(m.bias)
281 | 		elif isinstance(m, (nn.Conv2d, nn.Conv3d)):
282 | 			group_decay.append(m.weight)
283 | 			if m.bias is not None:
284 | 				group_no_decay.append(m.bias)
285 | 		elif isinstance(m, norm_layer) or isinstance(m, nn.GroupNorm):
286 | 			if m.weight is not None:
287 | 				group_no_decay.append(m.weight)
288 | 			if m.bias is not None:
289 | 				group_no_decay.append(m.bias)
290 | 
291 | 	assert len(list(module.parameters())) == len(group_decay) + len(
292 | 		group_no_decay)
293 | 	weight_group.append(dict(params=group_decay, lr=lr))
294 | 	weight_group.append(dict(params=group_no_decay, weight_decay=.0, lr=lr))
295 | 	return weight_group
296 | 


--------------------------------------------------------------------------------