├── .clang-format ├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.rst ├── demo ├── demo.py └── kitten.off ├── img └── figRepo.jpg ├── model └── xgboost.file ├── pretrained_model └── experiments │ └── base_model │ ├── best_weights │ ├── after-epoch-73.data-00000-of-00001 │ ├── after-epoch-73.index │ ├── checkpoint │ └── learner.json │ ├── last_weights │ ├── after-epoch-100.data-00000-of-00001 │ ├── after-epoch-100.index │ ├── after-epoch-96.data-00000-of-00001 │ ├── after-epoch-96.index │ ├── after-epoch-97.data-00000-of-00001 │ ├── after-epoch-97.index │ ├── after-epoch-98.data-00000-of-00001 │ ├── after-epoch-98.index │ ├── after-epoch-99.data-00000-of-00001 │ ├── after-epoch-99.index │ └── checkpoint │ ├── metrics_eval_best_weights.json │ ├── metrics_eval_last_weights.json │ ├── metrics_test_best_weights.json │ ├── params.json │ ├── train_summaries │ └── events.out.tfevents.1590751103.WILLIAMCWU-NB0 │ └── vali_summaries │ └── events.out.tfevents.1590751104.WILLIAMCWU-NB0 ├── pymdp ├── __init__.py ├── beam_guided.py ├── learning_based.py ├── ranker │ ├── rnn_ranker.py │ ├── uRanker.py │ ├── urank │ │ ├── evaluate.py │ │ ├── evaluate_point.py │ │ ├── experiments │ │ │ └── base_model │ │ │ │ ├── best_weights │ │ │ │ ├── after-epoch-73.data-00000-of-00001 │ │ │ │ ├── after-epoch-73.index │ │ │ │ ├── checkpoint │ │ │ │ └── learner.json │ │ │ │ ├── last_weights │ │ │ │ ├── after-epoch-100.data-00000-of-00001 │ │ │ │ ├── after-epoch-100.index │ │ │ │ ├── after-epoch-96.data-00000-of-00001 │ │ │ │ ├── after-epoch-96.index │ │ │ │ ├── after-epoch-97.data-00000-of-00001 │ │ │ │ ├── after-epoch-97.index │ │ │ │ ├── after-epoch-98.data-00000-of-00001 │ │ │ │ ├── after-epoch-98.index │ │ │ │ ├── after-epoch-99.data-00000-of-00001 │ │ │ │ ├── after-epoch-99.index │ │ │ │ └── checkpoint │ │ │ │ ├── metrics_eval_best_weights.json │ │ │ │ ├── metrics_eval_last_weights.json │ │ │ │ ├── metrics_test_best_weights.json │ │ │ │ ├── params.json │ │ │ │ ├── train_summaries │ │ │ │ └── events.out.tfevents.1590751103.WILLIAMCWU-NB0 │ │ │ │ └── vali_summaries │ │ │ │ └── events.out.tfevents.1590751104.WILLIAMCWU-NB0 │ │ ├── feature_norm_for_lambdarank.py │ │ ├── label_output │ │ ├── lambda_cv_correct.py │ │ ├── lambdarank_setting │ │ │ ├── lambda_cv.py │ │ │ ├── mslr-eval-score-mslr.pl │ │ │ ├── mslr-eval-ttest-mslr.pl │ │ │ ├── process_ndcg_results.py │ │ │ ├── template_predict.conf │ │ │ └── template_train.conf │ │ ├── main.py │ │ ├── model │ │ │ ├── evaluation.py │ │ │ ├── modeling.py │ │ │ ├── reader.py │ │ │ ├── training.py │ │ │ └── utils.py │ │ ├── msltr2libsvm.py │ │ ├── prediction_output │ │ ├── prepare_data.py │ │ ├── process_ndcg_results.py │ │ ├── process_results.py │ │ ├── run.bat │ │ ├── run.sh │ │ └── util │ │ │ ├── loss_fns.py │ │ │ ├── masks.py │ │ │ ├── math_fns.py │ │ │ ├── sample.py │ │ │ ├── scores.py │ │ │ └── search_metrics.py │ └── xgboost_ranker.py ├── trajectory.py └── utility.py ├── requirements.txt ├── setup.py └── src ├── CustomisedPolyhedron.h ├── CustomisedSlicerDepracted.h ├── Exception.h ├── FillHole.cpp ├── FillHole.h ├── FillHoleCDT.cpp ├── FillHoleCDT.h ├── GeometryTools.cpp ├── GeometryTools.h ├── MeshCutEval.cpp ├── MeshCutEval.h ├── MeshSupEval.cpp ├── PlaneCut.cpp ├── PlaneCut.h ├── RoboFDM.cpp ├── RoboFDM.h └── RoboFDMUtils.cpp /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: LLVM 4 | AccessModifierOffset: -2 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlinesLeft: false 9 | AlignOperands: true 10 | AlignTrailingComments: true 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: All 15 | AllowShortIfStatementsOnASingleLine: false 16 | AllowShortLoopsOnASingleLine: false 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: false 20 | AlwaysBreakTemplateDeclarations: false 21 | BinPackArguments: true 22 | BinPackParameters: true 23 | BraceWrapping: 24 | AfterClass: false 25 | AfterControlStatement: false 26 | AfterEnum: false 27 | AfterFunction: false 28 | AfterNamespace: false 29 | AfterObjCDeclaration: false 30 | AfterStruct: false 31 | AfterUnion: false 32 | BeforeCatch: false 33 | BeforeElse: false 34 | IndentBraces: false 35 | BreakBeforeBinaryOperators: None 36 | BreakBeforeBraces: Attach 37 | BreakBeforeTernaryOperators: true 38 | BreakConstructorInitializersBeforeComma: false 39 | ColumnLimit: 120 40 | CommentPragmas: '^ IWYU pragma:' 41 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 42 | ConstructorInitializerIndentWidth: 4 43 | ContinuationIndentWidth: 4 44 | Cpp11BracedListStyle: true 45 | DerivePointerAlignment: false 46 | DisableFormat: false 47 | ExperimentalAutoDetectBinPacking: false 48 | ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] 49 | IncludeCategories: 50 | - Regex: '^"(llvm|llvm-c|clang|clang-c)/' 51 | Priority: 2 52 | - Regex: '^(<|"(gtest|isl|json)/)' 53 | Priority: 3 54 | - Regex: '.*' 55 | Priority: 1 56 | IndentCaseLabels: false 57 | IndentWidth: 2 58 | IndentWrappedFunctionNames: false 59 | KeepEmptyLinesAtTheStartOfBlocks: true 60 | MacroBlockBegin: '' 61 | MacroBlockEnd: '' 62 | MaxEmptyLinesToKeep: 1 63 | NamespaceIndentation: None 64 | ObjCBlockIndentWidth: 2 65 | ObjCSpaceAfterProperty: false 66 | ObjCSpaceBeforeProtocolList: true 67 | PenaltyBreakBeforeFirstCallParameter: 19 68 | PenaltyBreakComment: 300 69 | PenaltyBreakFirstLessLess: 120 70 | PenaltyBreakString: 1000 71 | PenaltyExcessCharacter: 1000000 72 | PenaltyReturnTypeOnItsOwnLine: 60 73 | PointerAlignment: Left 74 | ReflowComments: true 75 | SortIncludes: true 76 | SpaceAfterCStyleCast: false 77 | SpaceBeforeAssignmentOperators: true 78 | SpaceBeforeParens: ControlStatements 79 | SpaceInEmptyParentheses: false 80 | SpacesBeforeTrailingComments: 1 81 | SpacesInAngles: false 82 | SpacesInContainerLiterals: true 83 | SpacesInCStyleCastParentheses: false 84 | SpacesInParentheses: false 85 | SpacesInSquareBrackets: false 86 | Standard: Cpp11 87 | TabWidth: 8 88 | UseTab: Never 89 | ... 90 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/visualstudio 3 | 4 | ### VisualStudio ### 5 | ## Ignore Visual Studio temporary files, build results, and 6 | ## files generated by popular Visual Studio add-ons. 7 | ## 8 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 9 | 10 | build 11 | *.DS_Store 12 | results 13 | data 14 | 15 | # User-specific files 16 | *.suo 17 | *.user 18 | *.userosscache 19 | *.sln.docstates 20 | *.dll 21 | 22 | *.npy 23 | *.iph 24 | 25 | # User-specific files (MonoDevelop/Xamarin Studio) 26 | *.userprefs 27 | 28 | # Build results 29 | [Dd]ebug/ 30 | [Dd]ebugPublic/ 31 | [Rr]elease/ 32 | [Rr]eleases/ 33 | x64/ 34 | x86/ 35 | bld/ 36 | [Bb]in/ 37 | [Oo]bj/ 38 | [Ll]og/ 39 | 40 | # Visual Studio 2015 cache/options directory 41 | .vs/ 42 | # Uncomment if you have tasks that create the project's static files in wwwroot 43 | #wwwroot/ 44 | 45 | # MSTest test Results 46 | [Tt]est[Rr]esult*/ 47 | [Bb]uild[Ll]og.* 48 | 49 | # NUNIT 50 | *.VisualState.xml 51 | TestResult.xml 52 | 53 | # Build Results of an ATL Project 54 | [Dd]ebugPS/ 55 | [Rr]eleasePS/ 56 | dlldata.c 57 | 58 | # .NET Core 59 | project.lock.json 60 | project.fragment.lock.json 61 | artifacts/ 62 | **/Properties/launchSettings.json 63 | 64 | *_i.c 65 | *_p.c 66 | *_i.h 67 | *.ilk 68 | *.meta 69 | *.obj 70 | *.pch 71 | *.pdb 72 | *.pgc 73 | *.pgd 74 | *.rsp 75 | *.sbr 76 | *.tlb 77 | *.tli 78 | *.tlh 79 | *.tmp 80 | *.tmp_proj 81 | *.log 82 | *.vspscc 83 | *.vssscc 84 | .builds 85 | *.pidb 86 | *.svclog 87 | *.scc 88 | 89 | # Chutzpah Test files 90 | _Chutzpah* 91 | 92 | # Visual C++ cache files 93 | ipch/ 94 | *.aps 95 | *.ncb 96 | *.opendb 97 | *.opensdf 98 | *.sdf 99 | *.cachefile 100 | *.VC.db 101 | *.VC.VC.opendb 102 | 103 | # Visual Studio profiler 104 | *.psess 105 | *.vsp 106 | *.vspx 107 | *.sap 108 | 109 | # TFS 2012 Local Workspace 110 | $tf/ 111 | 112 | # Guidance Automation Toolkit 113 | *.gpState 114 | 115 | # ReSharper is a .NET coding add-in 116 | _ReSharper*/ 117 | *.[Rr]e[Ss]harper 118 | *.DotSettings.user 119 | 120 | # JustCode is a .NET coding add-in 121 | .JustCode 122 | 123 | # TeamCity is a build add-in 124 | _TeamCity* 125 | 126 | # DotCover is a Code Coverage Tool 127 | *.dotCover 128 | 129 | # Visual Studio code coverage results 130 | *.coverage 131 | *.coveragexml 132 | 133 | # NCrunch 134 | _NCrunch_* 135 | .*crunch*.local.xml 136 | nCrunchTemp_* 137 | 138 | # MightyMoose 139 | *.mm.* 140 | AutoTest.Net/ 141 | 142 | # Web workbench (sass) 143 | .sass-cache/ 144 | 145 | # Installshield output folder 146 | [Ee]xpress/ 147 | 148 | # DocProject is a documentation generator add-in 149 | DocProject/buildhelp/ 150 | DocProject/Help/*.HxT 151 | DocProject/Help/*.HxC 152 | DocProject/Help/*.hhc 153 | DocProject/Help/*.hhk 154 | DocProject/Help/*.hhp 155 | DocProject/Help/Html2 156 | DocProject/Help/html 157 | 158 | # Click-Once directory 159 | publish/ 160 | 161 | # Publish Web Output 162 | *.[Pp]ublish.xml 163 | *.azurePubxml 164 | # TODO: Uncomment the next line to ignore your web deploy settings. 165 | # By default, sensitive information, such as encrypted password 166 | # should be stored in the .pubxml.user file. 167 | #*.pubxml 168 | *.pubxml.user 169 | *.publishproj 170 | 171 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 172 | # checkin your Azure Web App publish settings, but sensitive information contained 173 | # in these scripts will be unencrypted 174 | PublishScripts/ 175 | 176 | # NuGet Packages 177 | *.nupkg 178 | # The packages folder can be ignored because of Package Restore 179 | **/packages/* 180 | # except build/, which is used as an MSBuild target. 181 | !**/packages/build/ 182 | # Uncomment if necessary however generally it will be regenerated when needed 183 | #!**/packages/repositories.config 184 | # NuGet v3's project.json files produces more ignorable files 185 | *.nuget.props 186 | *.nuget.targets 187 | 188 | # Microsoft Azure Build Output 189 | csx/ 190 | *.build.csdef 191 | 192 | # Microsoft Azure Emulator 193 | ecf/ 194 | rcf/ 195 | 196 | # Windows Store app package directories and files 197 | AppPackages/ 198 | BundleArtifacts/ 199 | Package.StoreAssociation.xml 200 | _pkginfo.txt 201 | 202 | # Visual Studio cache files 203 | # files ending in .cache can be ignored 204 | *.[Cc]ache 205 | # but keep track of directories ending in .cache 206 | !*.[Cc]ache/ 207 | 208 | # Others 209 | ClientBin/ 210 | ~$* 211 | *~ 212 | *.dbmdl 213 | *.dbproj.schemaview 214 | *.jfm 215 | *.pfx 216 | *.publishsettings 217 | orleans.codegen.cs 218 | 219 | # Since there are multiple workflows, uncomment next line to ignore bower_components 220 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 221 | #bower_components/ 222 | 223 | # RIA/Silverlight projects 224 | Generated_Code/ 225 | 226 | # Backup & report files from converting an old project file 227 | # to a newer Visual Studio version. Backup files are not needed, 228 | # because we have git ;-) 229 | _UpgradeReport_Files/ 230 | Backup*/ 231 | UpgradeLog*.XML 232 | UpgradeLog*.htm 233 | 234 | # SQL Server files 235 | *.mdf 236 | *.ldf 237 | *.ndf 238 | 239 | # Business Intelligence projects 240 | *.rdl.data 241 | *.bim.layout 242 | *.bim_*.settings 243 | 244 | # Microsoft Fakes 245 | FakesAssemblies/ 246 | 247 | # GhostDoc plugin setting file 248 | *.GhostDoc.xml 249 | 250 | # Node.js Tools for Visual Studio 251 | .ntvs_analysis.dat 252 | node_modules/ 253 | 254 | # Typescript v1 declaration files 255 | typings/ 256 | 257 | # Visual Studio 6 build log 258 | *.plg 259 | 260 | # Visual Studio 6 workspace options file 261 | *.opt 262 | 263 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 264 | *.vbw 265 | 266 | # Visual Studio LightSwitch build output 267 | **/*.HTMLClient/GeneratedArtifacts 268 | **/*.DesktopClient/GeneratedArtifacts 269 | **/*.DesktopClient/ModelManifest.xml 270 | **/*.Server/GeneratedArtifacts 271 | **/*.Server/ModelManifest.xml 272 | _Pvt_Extensions 273 | 274 | # Paket dependency manager 275 | .paket/paket.exe 276 | paket-files/ 277 | 278 | # FAKE - F# Make 279 | .fake/ 280 | 281 | # JetBrains Rider 282 | .idea/ 283 | *.sln.iml 284 | 285 | # CodeRush 286 | .cr/ 287 | 288 | # Python Tools for Visual Studio (PTVS) 289 | __pycache__/ 290 | *.pyc 291 | 292 | # Cake - Uncomment if you are using it 293 | # tools/** 294 | # !tools/packages.config 295 | 296 | # Telerik's JustMock configuration file 297 | *.jmconfig 298 | 299 | # BizTalk build output 300 | *.btp.cs 301 | *.btm.cs 302 | *.odx.cs 303 | *.xsd.cs 304 | 305 | ### VisualStudio Patch ### 306 | # By default, sensitive information, such as encrypted password 307 | # should be stored in the .pubxml.user file. 308 | 309 | 310 | # End of https://www.gitignore.io/api/visualstudio 311 | 312 | ### C++ template 313 | # Compiled Object files 314 | *.slo 315 | *.lo 316 | *.o 317 | *.obj 318 | 319 | # Precompiled Headers 320 | *.gch 321 | *.pch 322 | 323 | # Compiled Dynamic libraries 324 | *.so 325 | *.dylib 326 | *.dll 327 | 328 | # Fortran module files 329 | *.mod 330 | 331 | # Compiled Static libraries 332 | *.lai 333 | *.la 334 | *.a 335 | *.lib 336 | 337 | # Executables 338 | *.exe 339 | *.out 340 | *.app 341 | 342 | ### JetBrains template 343 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 344 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 345 | 346 | # User-specific stuff: 347 | .idea/workspace.xml 348 | .idea/tasks.xml 349 | .idea/dictionaries 350 | .idea/vcs.xml 351 | .idea/jsLibraryMappings.xml 352 | 353 | # Sensitive or high-churn files: 354 | .idea/dataSources.ids 355 | .idea/dataSources.xml 356 | .idea/dataSources.local.xml 357 | .idea/sqlDataSources.xml 358 | .idea/dynamic.xml 359 | .idea/uiDesigner.xml 360 | 361 | # Gradle: 362 | .idea/gradle.xml 363 | .idea/libraries 364 | 365 | # Mongo Explorer plugin: 366 | .idea/mongoSettings.xml 367 | 368 | ## File-based project format: 369 | *.iws 370 | 371 | ## Plugin-specific files: 372 | 373 | # IntelliJ 374 | /out/ 375 | 376 | # mpeltonen/sbt-idea plugin 377 | .idea_modules/ 378 | 379 | # JIRA plugin 380 | atlassian-ide-plugin.xml 381 | 382 | # Crashlytics plugin (for Android Studio and IntelliJ) 383 | com_crashlytics_export_strings.xml 384 | crashlytics.properties 385 | crashlytics-build.properties 386 | fabric.properties 387 | 388 | cmake-build-debug 389 | cmake-build-release 390 | 391 | *logs 392 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | cmake_minimum_required(VERSION 3.16) 3 | 4 | project(RoboFDM_py) 5 | 6 | set(CMAKE_VERBOSE_MAKEFILE ON) 7 | 8 | IF (WIN32) 9 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MD") 10 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd") 11 | set(MSVC_RUNTIME "dynamic") 12 | ENDIF () 13 | 14 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) 15 | set(CGAL_DO_NOT_WARN_ABOUT_CMAKE_BUILD_TYPE TRUE) 16 | 17 | find_package(CGAL CONFIG REQUIRED) 18 | # Don't let CGAL override flags 19 | set(CGAL_DONT_OVERRIDE_CMAKE_FLAGS 20 | TRUE 21 | CACHE BOOL "Force CGAL to maintain CMAKE flags") 22 | set(CGAL_DO_NOT_WARN_ABOUT_CMAKE_BUILD_TYPE TRUE) 23 | 24 | include(FetchContent) 25 | 26 | FetchContent_Declare( 27 | pybind11_sources 28 | GIT_REPOSITORY https://github.com/pybind/pybind11.git 29 | GIT_TAG v2.2 30 | ) 31 | 32 | FetchContent_GetProperties(pybind11_sources) 33 | 34 | if (NOT pybind11_sources_POPULATED) 35 | FetchContent_Populate(pybind11_sources) 36 | 37 | add_subdirectory( 38 | ${pybind11_sources_SOURCE_DIR} 39 | ${pybind11_sources_BINARY_DIR} 40 | ) 41 | endif () 42 | 43 | find_package(Eigen3 CONFIG REQUIRED) 44 | 45 | # include helper file 46 | include(${CGAL_USE_FILE}) 47 | 48 | # Boost and its components 49 | find_package(Boost REQUIRED) 50 | 51 | find_package(OpenMP REQUIRED) 52 | 53 | if (OPENMP_FOUND) 54 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 55 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 56 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") 57 | endif () 58 | 59 | 60 | # create python module 61 | 62 | add_library(RoboFDM 63 | MODULE 64 | src/GeometryTools.cpp 65 | src/GeometryTools.h 66 | src/FillHole.cpp 67 | src/FillHole.h 68 | src/FillHoleCDT.cpp 69 | src/FillHoleCDT.h 70 | src/PlaneCut.cpp 71 | src/PlaneCut.h 72 | src/MeshCutEval.cpp 73 | src/MeshCutEval.h 74 | src/MeshSupEval.cpp 75 | src/RoboFDM.cpp 76 | src/RoboFDM.h 77 | src/RoboFDMUtils.cpp 78 | ) 79 | 80 | 81 | if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") 82 | if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL "7.0" OR 83 | CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "7.0") 84 | target_link_libraries(RoboFDM PUBLIC 85 | pybind11::module 86 | Eigen3::Eigen PRIVATE ${OpenMP_libomp_LIBRARY}) 87 | endif() 88 | else() 89 | target_link_libraries(RoboFDM 90 | PUBLIC 91 | pybind11::module 92 | Eigen3::Eigen 93 | ) 94 | endif() 95 | 96 | set_target_properties(RoboFDM 97 | PROPERTIES 98 | PREFIX "${PYTHON_MODULE_PREFIX}" 99 | SUFFIX "${PYTHON_MODULE_EXTENSION}" 100 | ) 101 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Chenming Wu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =========================================================== 2 | PyMDP: A decomposition tool for multi-direction 3D printing 3 | =========================================================== 4 | 5 | .. image:: /img/figRepo.jpg 6 | 7 | Multi-directional 3D printing by robotic arms or multi-axis systems is a new way of manufacturing. As a strong complementary of layer-wise additive manufacturing, multi-directional printing has the capability of decreasing or eliminating the need for support structures. 8 | 9 | ------ 10 | Notice 11 | ------ 12 | This library is **no longer actively maintained**. If you come across any complications during the compilation process, we suggest exploring the option of using a previous version of VCPKG from the year 2020. This approach has proven to be effective for numerous users who encountered similar issues and reached out to us via email. 13 | 14 | ---------- 15 | Dependency 16 | ---------- 17 | 18 | `Eigen `_ `CGAL `_ `PyBind11 `_ 19 | 20 | 21 | ------- 22 | Install 23 | ------- 24 | 25 | We use CMake (>=3.16) and vcpkg to facilate the compilation process. You can download and install CMake from their official website, and install vcpkg by 26 | 27 | .. code-block:: bash 28 | 29 | git clone https://github.com/Microsoft/vcpkg.git 30 | cd vcpkg 31 | ./bootstrap-vcpkg.sh 32 | ./vcpkg integrate install 33 | 34 | Next, you will need to install CGAL dependency: 35 | 36 | .. code-block:: bash 37 | 38 | vcpkg install eigen3 39 | vcpkg install cgal 40 | 41 | 42 | Note: If you are using a Windows system, please be aware that vcpkg will install the 32-bit package as the default option. If you encounter this situation, you may need to utilize the following command. 43 | 44 | .. code-block:: bash 45 | 46 | vcpkg install eigen3:x64-windows 47 | vcpkg install cgal:x64-windows 48 | 49 | Then you can easily install the library by using the following command. 50 | 51 | .. code-block:: bash 52 | 53 | pip install . --install-option="--vcpkg=YOUR_VCPKG_FOLDER" 54 | 55 | Please change "YOUR_VCPKG_FOLDER" to the folder where VCPKG is installed. 56 | 57 | ------- 58 | Demo 59 | ------- 60 | 61 | .. code-block:: python 62 | 63 | from pymdp import BGS 64 | 65 | if __name__ == "__main__": 66 | proc = BGS(filename='kitten.off') 67 | proc.set_beam_width(10) 68 | proc.set_output_folder('kitten') 69 | proc.start_search() 70 | 71 | 72 | We have recently introduced a learning-based approach to enhance the original search algorithm, utilizing learning-to-rank techniques. The source codes for this method can be found in the "learning_based.py" file, which is available for access. 73 | 74 | 75 | 76 | ------- 77 | Credits 78 | ------- 79 | We kindly request that any scientific publications utilizing PyMDP cite our work, as we greatly appreciate your support. 80 | 81 | .. code-block:: bibtex 82 | 83 | @inproceedings{wu2017robofdm, 84 | title={RoboFDM: A robotic system for support-free fabrication using FDM}, 85 | author={Wu, Chenming and Dai, Chengkai and Fang, Guoxin and Liu, Yong-Jin and Wang, Charlie CL}, 86 | booktitle={2017 IEEE International Conference on Robotics and Automation (ICRA)}, 87 | pages={1175--1180}, 88 | year={2017}, 89 | organization={IEEE} 90 | } 91 | 92 | .. code-block:: bibtex 93 | 94 | @article{wu2019general, 95 | title={General Support-Effective Decomposition for Multi-Directional 3-D Printing}, 96 | author={Wu, Chenming and Dai, Chengkai and Fang, Guoxin and Liu, Yong-Jin and Wang, Charlie CL}, 97 | journal={IEEE Transactions on Automation Science and Engineering}, 98 | year={2019}, 99 | publisher={IEEE} 100 | } 101 | 102 | .. code-block:: bibtex 103 | 104 | @article{wu2020learning, 105 | title={Learning to accelerate decomposition for multi-directional 3D printing}, 106 | author={Wu, Chenming and Liu, Yong-Jin and Wang, Charlie CL}, 107 | journal={IEEE Robotics and Automation Letters}, 108 | volume={5}, 109 | number={4}, 110 | pages={5897--5904}, 111 | year={2020}, 112 | publisher={IEEE} 113 | } 114 | 115 | 116 | In our learning-to-accelerate work, we use `urank `_ impelementation provided by Xiaofeng Zhu. Please consider cite their work if you also found it helpful. 117 | 118 | -------------------------------------------------------------------------------- /demo/demo.py: -------------------------------------------------------------------------------- 1 | from pymdp.beam_guided import BGS 2 | 3 | if __name__ == "__main__": 4 | proc = BGS(filename='kitten.off', export=True) 5 | proc.set_beam_width(10) 6 | proc.set_output_folder('kitten') 7 | proc.start_search() -------------------------------------------------------------------------------- /img/figRepo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/img/figRepo.jpg -------------------------------------------------------------------------------- /model/xgboost.file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/model/xgboost.file -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/best_weights/after-epoch-73.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/best_weights/after-epoch-73.data-00000-of-00001 -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/best_weights/after-epoch-73.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/best_weights/after-epoch-73.index -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/best_weights/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "after-epoch-73" 2 | all_model_checkpoint_paths: "after-epoch-73" 3 | -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/best_weights/learner.json: -------------------------------------------------------------------------------- 1 | { 2 | "stopped_at_learner": 0.0 3 | } -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-100.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-100.data-00000-of-00001 -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-100.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-100.index -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-96.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-96.data-00000-of-00001 -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-96.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-96.index -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-97.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-97.data-00000-of-00001 -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-97.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-97.index -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-98.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-98.data-00000-of-00001 -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-98.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-98.index -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-99.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-99.data-00000-of-00001 -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/after-epoch-99.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/last_weights/after-epoch-99.index -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/last_weights/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "after-epoch-100" 2 | all_model_checkpoint_paths: "after-epoch-96" 3 | all_model_checkpoint_paths: "after-epoch-97" 4 | all_model_checkpoint_paths: "after-epoch-98" 5 | all_model_checkpoint_paths: "after-epoch-99" 6 | all_model_checkpoint_paths: "after-epoch-100" 7 | -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/metrics_eval_best_weights.json: -------------------------------------------------------------------------------- 1 | { 2 | "ndcg_1": 0.42354467511177063, 3 | "ndcg_3": 0.4819876253604889, 4 | "ndcg_5": 0.5298611521720886, 5 | "ndcg_10": 0.5841916799545288 6 | } -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/metrics_eval_last_weights.json: -------------------------------------------------------------------------------- 1 | { 2 | "ndcg_1": 0.41776859760284424, 3 | "ndcg_3": 0.47838959097862244, 4 | "ndcg_5": 0.5276974439620972, 5 | "ndcg_10": 0.5813385844230652 6 | } -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/metrics_test_best_weights.json: -------------------------------------------------------------------------------- 1 | { 2 | "ndcg_1": 0.42255860567092896, 3 | "ndcg_2": 0.455078125, 4 | "ndcg_3": 0.4832306504249573, 5 | "ndcg_4": 0.5097373127937317, 6 | "ndcg_5": 0.5315667986869812, 7 | "err_1": 0.40935495495796204, 8 | "err_2": 0.47491729259490967, 9 | "err_3": 0.5008284449577332, 10 | "err_4": 0.5139602422714233, 11 | "err_5": 0.5219849348068237 12 | } -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/params.json: -------------------------------------------------------------------------------- 1 | { 2 | "learning_rate": 1e-4, 3 | "batch_size": 1, 4 | "num_epochs": 1000, 5 | "buffer_size": 1, 6 | "save_summary_steps": 100, 7 | "early_stoping_epochs": 200, 8 | "gradient_clip_value": 5, 9 | "mlp_sizes": [100, 100], 10 | "residual_mlp_sizes": [100, 50], 11 | "pooling": "MP", 12 | "rnn": "C1", 13 | "num_learners": 4, 14 | "top_ks": [1,2,3,4,5], 15 | "save_predictions": true, 16 | "use_residual": false, 17 | 18 | "training_keep_prob": 1.0, 19 | "top_k": 10, 20 | "pre_training": 0, 21 | "use_regularization": false, 22 | "dropout_rate": 0.3, 23 | "decay_size": 126, 24 | "decay_rate": 0.9, 25 | "mask": "diag_mask", 26 | "exploration": 0.7 27 | } 28 | -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/train_summaries/events.out.tfevents.1590751103.WILLIAMCWU-NB0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/train_summaries/events.out.tfevents.1590751103.WILLIAMCWU-NB0 -------------------------------------------------------------------------------- /pretrained_model/experiments/base_model/vali_summaries/events.out.tfevents.1590751104.WILLIAMCWU-NB0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pretrained_model/experiments/base_model/vali_summaries/events.out.tfevents.1590751104.WILLIAMCWU-NB0 -------------------------------------------------------------------------------- /pymdp/__init__.py: -------------------------------------------------------------------------------- 1 | from pymdp.beam_guided import BGS 2 | -------------------------------------------------------------------------------- /pymdp/beam_guided.py: -------------------------------------------------------------------------------- 1 | import RoboFDM 2 | import numpy as np 3 | import copy 4 | import os 5 | import sys 6 | from .utility import run_cut_process, write_mesh 7 | from .trajectory import TrajStation, Trajectory 8 | 9 | from sys import platform 10 | if platform == "linux" or platform == "linux2": 11 | pass 12 | elif platform == "darwin": 13 | pass 14 | elif platform == "win32": 15 | import ctypes 16 | SEM_NOGPFAULTERRORBOX = 0x8007 17 | ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX) 18 | 19 | 20 | class BGS: 21 | def __init__(self, filename, ranknet=None, export=False): 22 | self.b_objs = [] 23 | self.b_polys = [] 24 | self.b_poly_sequence = [] 25 | self.b_r_sequence = [] 26 | self.b_rew = [] 27 | self.b_residual = [] 28 | self.b_envs = [] 29 | self.b_round = 0 30 | self.b_best_so_far = 0 31 | self.b_trajs = TrajStation() 32 | self.filename = filename 33 | self.env = RoboFDM.init() 34 | self.env.reset(filename) 35 | self.params = np.array([0.05, 0.55, 0.0, 0.00, 0.25]) 36 | self.threshold = 0.02 37 | self.n_features = self.env.n_features() 38 | self.output_folder = None 39 | self.b_width = None 40 | self.export_polys = [] 41 | self.export = export 42 | 43 | @staticmethod 44 | def search_type(): 45 | return "Normal" 46 | 47 | def set_beam_width(self, w): 48 | self.b_width = w 49 | 50 | def set_output_folder(self, f): 51 | self.output_folder = f 52 | 53 | def is_diverse(self, a, b): 54 | dist = abs(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]) 55 | if dist > 0.95: 56 | if abs(a[3] - b[3]) < 2.0 * dist: 57 | return False 58 | return True 59 | 60 | def r_distance(self, a, b): 61 | return np.linalg.norm(a-b) 62 | 63 | def query_poly_idx(self, all_range, p): 64 | sum_range = 0 65 | for i in range(len(all_range)): 66 | sum_range += all_range[i] 67 | if p < sum_range: 68 | return i 69 | return 0 70 | 71 | def feedforward_search(self): 72 | all_r = None 73 | all_range = [] 74 | self.b_best_so_far = np.max(self.b_rew) 75 | print('best so far ', self.b_best_so_far) 76 | print("polys = ", len(self.b_polys)) 77 | print('residual = ', self.b_residual) 78 | for i in range(len(self.b_polys)): 79 | self.env.set_poly(self.b_polys[i]) 80 | r = self.env.render() 81 | r = np.insert(r, 5, 0, axis=1) 82 | r[:, 5] = r[:, 1] 83 | #r[:, 1] -= r[:, 4] 84 | dr = r[:, 1] 85 | r[:, 1] += self.b_rew[i] 86 | r[:, 4] += self.b_residual[i] 87 | all_range.append(len(dr)) 88 | if all_r is None: 89 | all_r = r 90 | else: 91 | all_r = np.concatenate((all_r, r), axis=0) 92 | 93 | # filter out candidates that not satisfy volume constraint 94 | violated_vol = (all_r[:, 0] < 0.1) 95 | all_r[violated_vol, 0:6] = [0, 0, 0, 0, 0, 0] 96 | 97 | cur_sel = [] 98 | cur_polys = [] 99 | cur_r = [] 100 | epsilon = 0.0001 101 | area_sorted = np.argsort(all_r[:, 1])[::-1] 102 | 103 | poly_sequence = self.b_poly_sequence.copy() 104 | r_sequence = self.b_r_sequence.copy() 105 | self.b_poly_sequence.clear() 106 | self.b_r_sequence.clear() 107 | self.b_rew.clear() 108 | self.b_residual.clear() 109 | 110 | # self.b_trajs.display() 111 | self.b_trajs.move_to_next_level() 112 | has_impr = False 113 | 114 | '''Construct trajectory features here''' 115 | traj_feats = [] 116 | cur_traj_node = [] 117 | cur_export_polys = [] 118 | epsilon_best = 0 119 | while len(cur_sel) < self.b_width: 120 | for i in range(len(all_r)): 121 | cur_idx = area_sorted[i] 122 | 123 | if len(cur_sel) >= self.b_width: 124 | break 125 | if all_r[cur_idx, 5] < 1e-4: 126 | break 127 | if all_r[cur_idx, 4] > epsilon: 128 | continue 129 | if all_r[cur_idx, 1] < epsilon_best: 130 | break 131 | 132 | poly_idx = self.query_poly_idx(all_range, cur_idx) 133 | 134 | # diversity 135 | cur_plane = all_r[cur_idx, 6::] 136 | flag_satisfied = True 137 | for tmp_r in cur_r: 138 | r, pid = tmp_r 139 | if pid != poly_idx: # don't filter if they come from different poly idx 140 | continue 141 | if self.is_diverse(r[6::], cur_plane) is False: 142 | flag_satisfied = False 143 | break 144 | 145 | if not flag_satisfied: 146 | #print('reject: ', all_r[cur_idx, :]) 147 | continue 148 | 149 | #print('epsilon = ', all_r[cur_idx, -1], epsilon) 150 | 151 | # print('current feature = ', cur_idx, all_r[cur_idx, 0:5]) 152 | 153 | ret_poly = run_cut_process( 154 | self.b_polys[poly_idx], cur_plane, self.export) 155 | if ret_poly == None: 156 | print('plane cut failed.') 157 | continue 158 | 159 | if type(ret_poly) == tuple: 160 | cur_export_polys.append(ret_poly) 161 | ret_poly = ret_poly[0] 162 | 163 | cur_sel.append(i) 164 | new_reward = all_r[cur_idx, 1] 165 | self.b_rew.append(new_reward) 166 | self.b_residual.append(all_r[cur_idx, 4]) 167 | if new_reward > self.b_best_so_far: 168 | has_impr = True 169 | #print(all_r[cur_idx, :]) 170 | cur_poly_sequence = poly_sequence[poly_idx] 171 | cur_poly_sequence.append(ret_poly) 172 | self.b_poly_sequence.append(cur_poly_sequence) 173 | cur_r_sequence = r_sequence[poly_idx].copy() 174 | cur_r_sequence.append(all_r[cur_idx, 1]) 175 | self.b_r_sequence.append(cur_r_sequence) 176 | cur_polys.append(ret_poly) 177 | #cur_r.append(np.concatenate((all_r[cur_idx,:], cur_plane), axis=0)) 178 | cur_r.append((all_r[cur_idx, :], poly_idx)) 179 | '''Output mesh''' 180 | #write_mesh(ret_poly, str(self.b_round) + '-' + str(poly_idx) + '-' + str(len(cur_polys)-1) +'.OFF') 181 | 182 | # create/maintain a trajectory 183 | traj_feats.append(all_r[cur_idx]) 184 | cur_traj_node.append((poly_idx, len(traj_feats)-1, new_reward)) 185 | #self.b_trajs.add_node(poly_idx, len(traj_feats)-1, new_reward) 186 | 187 | #write_mesh(ret_poly, str(self.b_round) + '-' + str(poly_idx) + '-' + str(itr)+'.OFF') 188 | # compute diversity function 189 | # average(L2, pos/weight) < threshold 190 | epsilon = 5.0 * epsilon 191 | if len(self.b_rew) != 0: 192 | epsilon_best = np.max(np.array(self.b_rew)) 193 | 194 | if epsilon > 1e3: 195 | break 196 | 197 | self.export_polys.append(cur_export_polys) 198 | print('has_import ', has_impr) 199 | # while loop here 200 | if has_impr == False: 201 | return False 202 | 203 | for tn in cur_traj_node: 204 | (pid, pl, pr) = tn 205 | self.b_trajs.add_node(pid, pl, pr) 206 | 207 | # print(cur_r) 208 | feat_valid = len(traj_feats) 209 | 210 | self.b_trajs.add_feature(traj_feats, feat_valid) 211 | 212 | self.b_polys = cur_polys 213 | # print(self.b_rew) 214 | return True 215 | 216 | def start_search(self): 217 | self.b_envs.append(self.env) 218 | self.b_polys.append(self.env.get_poly()) 219 | self.b_poly_sequence.append(self.b_polys) 220 | self.b_r_sequence.append([0.0]) 221 | self.b_rew.append(0.0) 222 | self.b_residual.append(0.0) 223 | print(self.filename) 224 | while True: 225 | pos_rew = self.feedforward_search() 226 | self.b_round += 1 227 | if pos_rew == False: 228 | break 229 | 230 | print(os.path.basename(self.filename)[:-4]) 231 | self.b_trajs.prepare_data(os.path.join( 232 | self.output_folder, os.path.basename(self.filename)[:-4])) 233 | 234 | self.b_trajs.prepare_data_edge(os.path.join( 235 | self.output_folder, os.path.basename(self.filename)[:-4])) 236 | 237 | if self.export: 238 | self.b_trajs.export_best_segmentation(os.path.join( 239 | self.output_folder, os.path.basename(self.filename)[:-4]), self.export_polys) -------------------------------------------------------------------------------- /pymdp/learning_based.py: -------------------------------------------------------------------------------- 1 | import RoboFDM 2 | import numpy as np 3 | import copy 4 | import os 5 | import sys 6 | from .utility import run_cut_process, write_mesh 7 | from .trajectory import TrajStation, Trajectory 8 | 9 | from sys import platform 10 | if platform == "linux" or platform == "linux2": 11 | pass 12 | elif platform == "darwin": 13 | pass 14 | elif platform == "win32": 15 | import ctypes 16 | SEM_NOGPFAULTERRORBOX = 0x8007 17 | ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX) 18 | 19 | class BGS: 20 | def __init__(self, filename, ranknet, export=False): 21 | self.export = export 22 | self.b_objs = [] 23 | self.b_polys = [] 24 | self.b_poly_sequence = [] 25 | self.b_r_sequence = [] 26 | self.b_rew = [] 27 | self.b_residual = [] 28 | self.b_envs = [] 29 | self.b_round = 0 30 | self.b_best_so_far = 0 31 | self.b_trajs = TrajStation() 32 | self.filename = filename 33 | self.env = RoboFDM.init() 34 | self.env.reset(filename) 35 | self.params = np.array([0.05, 0.55, 0.0, 0.00, 0.25]) 36 | self.threshold = 0.02 37 | self.n_features = self.env.n_features() 38 | self.output_folder = None 39 | self.b_width = None 40 | self.rank_net = ranknet 41 | self.export_polys = [] 42 | self.k = 2 43 | 44 | @staticmethod 45 | def search_type(): 46 | return "Learning" 47 | 48 | def set_beam_width(self, w): 49 | self.b_width = w 50 | 51 | def set_k(self, k): 52 | self.k = k 53 | 54 | def set_output_folder(self, f): 55 | self.output_folder = f 56 | 57 | def is_diverse(self, a, b): 58 | dist = abs(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]) 59 | if dist > 0.95: 60 | if abs(a[3] - b[3]) < 2.0 * dist: 61 | return False 62 | return True 63 | 64 | def r_distance(self, a, b): 65 | return np.linalg.norm(a-b) 66 | 67 | def query_poly_idx(self, all_range, p): 68 | sum_range = 0 69 | for i in range(len(all_range)): 70 | sum_range += all_range[i] 71 | if p < sum_range: 72 | return i 73 | return 0 74 | 75 | def select_from_features(self, features, k=5): 76 | sort_idx = self.rank_net.rank_features(features) 77 | res = [sort_idx[i] for i in range(k)] 78 | return res 79 | 80 | def feedforward_search(self): 81 | all_r = None 82 | all_range = [] 83 | self.b_best_so_far = np.max(self.b_rew) 84 | print('best so far ', self.b_best_so_far) 85 | print("polys = ", len(self.b_polys)) 86 | print(self.b_residual) 87 | for i in range(len(self.b_polys)): 88 | self.env.set_poly(self.b_polys[i]) 89 | r = self.env.render() 90 | r = np.insert(r, 5, 0, axis=1) 91 | r[:, 5] = r[:, 1] 92 | #r[:, 1] -= r[:, 4] 93 | dr = r[:, 1] 94 | r[:, 1] += self.b_rew[i] 95 | # r[:,4] += self.b_residual[i] 96 | all_range.append(len(dr)) 97 | if all_r is None: 98 | all_r = r 99 | else: 100 | all_r = np.concatenate((all_r, r), axis=0) 101 | 102 | # filter out candidates that not satisfy volume constraint 103 | violated_vol = (all_r[:, 0] < 0.1) 104 | all_r[violated_vol, 0:6] = [0, 0, 0, 0, 0, 0] 105 | cur_sel = [] 106 | cur_polys = [] 107 | cur_r = [] 108 | epsilon = 0.0001 109 | area_sorted = np.argsort(all_r[:, 1])[::-1] 110 | 111 | poly_sequence = self.b_poly_sequence.copy() 112 | r_sequence = self.b_r_sequence.copy() 113 | self.b_poly_sequence.clear() 114 | self.b_r_sequence.clear() 115 | self.b_rew.clear() 116 | self.b_residual.clear() 117 | 118 | # self.b_trajs.display() 119 | self.b_trajs.move_to_next_level() 120 | has_impr = False 121 | 122 | '''Construct trajectory features here''' 123 | traj_feats = [] 124 | cur_traj_node = [] 125 | epsilon_best = 0 126 | epsilon_best_arr = [] 127 | while len(cur_sel) < self.b_width: 128 | for i in range(len(all_r)): 129 | cur_idx = area_sorted[i] 130 | if len(cur_sel) >= self.b_width: 131 | break 132 | if all_r[cur_idx, 5] < 1e-4: 133 | break 134 | if all_r[cur_idx, 4] > epsilon: 135 | continue 136 | if all_r[cur_idx, 1] < epsilon_best: 137 | break 138 | 139 | poly_idx = self.query_poly_idx(all_range, cur_idx) 140 | 141 | # diversity 142 | cur_plane = all_r[cur_idx, 6::] 143 | flag_satisfied = True 144 | for tmp_r in cur_r: 145 | r, pid = tmp_r 146 | if pid != poly_idx: # don't filter if they come from different poly idx 147 | continue 148 | if self.is_diverse(r[6::], cur_plane) is False: 149 | flag_satisfied = False 150 | break 151 | # tmp_dist = self.r_distance(r[5::], cur_plane) 152 | # if tmp_dist < self.threshold: 153 | # flag_satisfied = False 154 | # break 155 | 156 | if not flag_satisfied: 157 | #print('reject: ', all_r[cur_idx, :]) 158 | continue 159 | 160 | #print('epsilon = ', all_r[cur_idx, -1], epsilon) 161 | 162 | #print('current feature = ', cur_idx, all_r[cur_idx, 0:5]) 163 | 164 | cur_sel.append((cur_idx, poly_idx)) 165 | cur_r.append((all_r[cur_idx, :], poly_idx)) 166 | epsilon_best_arr.append(all_r[cur_idx, 1]) 167 | 168 | #write_mesh(ret_poly, str(self.b_round) + '-' + str(poly_idx) + '-' + str(itr)+'.OFF') 169 | # compute diversity function 170 | # average(L2, pos/weight) < threshold 171 | epsilon = 5.0 * epsilon 172 | if len(epsilon_best_arr) != 0: 173 | epsilon_best = np.max(np.array(epsilon_best_arr)) 174 | 175 | if epsilon > 1e3: 176 | break 177 | 178 | select_k = min(len(cur_sel), self.k) 179 | print(select_k) 180 | if len(cur_sel) > 1: 181 | prev_feats = self.b_trajs.get_feats_previous() 182 | # rank and select here 183 | if prev_feats == None: 184 | temp_features = [np.concatenate((np.array([0, 0, 0, 0, 0, 0]), all_r[i, 0:6]), axis=0) for (i, _) in cur_sel] 185 | else: 186 | prev_feats = np.array(prev_feats) 187 | temp_features = [np.concatenate((prev_feats[p, 0:6], all_r[i, 0:6]), axis=0) for (i, p) in cur_sel] 188 | 189 | sel_idx = self.select_from_features(temp_features, select_k) 190 | temp_features = np.array(temp_features) 191 | # max_ele = np.argmax(temp_features[:, 1]) 192 | 193 | if 0 not in sel_idx: 194 | sel_idx.append(0) 195 | cur_sel = [cur_sel[i] for i in sel_idx] 196 | 197 | cur_export_polys = [] 198 | for sel in cur_sel: 199 | cur_idx, poly_idx = sel 200 | cur_plane = all_r[cur_idx, 6::] 201 | ret_poly = run_cut_process( 202 | self.b_polys[poly_idx], cur_plane, self.export) 203 | if ret_poly == None: 204 | print('plane cut failed.') 205 | continue 206 | else: 207 | print('cut successed') 208 | 209 | if type(ret_poly) == tuple: 210 | cur_export_polys.append(copy.copy(ret_poly)) 211 | ret_poly = ret_poly[0] 212 | 213 | new_reward = all_r[cur_idx, 1] 214 | self.b_rew.append(new_reward) 215 | self.b_residual.append(all_r[cur_idx, 4]) 216 | 217 | if new_reward > self.b_best_so_far: 218 | has_impr = True 219 | 220 | cur_poly_sequence = poly_sequence[poly_idx] 221 | cur_poly_sequence.append(ret_poly) 222 | self.b_poly_sequence.append(cur_poly_sequence) 223 | cur_r_sequence = r_sequence[poly_idx].copy() 224 | cur_r_sequence.append(all_r[cur_idx, 1]) 225 | self.b_r_sequence.append(cur_r_sequence) 226 | cur_polys.append(ret_poly) 227 | #cur_r.append(np.concatenate((all_r[cur_idx,:], cur_plane), axis=0)) 228 | cur_r.append((all_r[cur_idx, :], poly_idx)) 229 | '''Output mesh''' 230 | #write_mesh(ret_poly, str(self.b_round) + '-' + str(poly_idx) + '-' + str(len(cur_polys)-1) +'.OFF') 231 | 232 | # create/maintain a trajectory 233 | traj_feats.append(all_r[cur_idx, :]) 234 | cur_traj_node.append((poly_idx, len(traj_feats)-1, new_reward)) 235 | #self.b_trajs.add_node(poly_idx, len(traj_feats)-1, new_reward) 236 | 237 | self.export_polys.append(cur_export_polys) 238 | 239 | # while loop here 240 | if has_impr == False: 241 | return False 242 | 243 | for tn in cur_traj_node: 244 | (pid, pl, pr) = tn 245 | self.b_trajs.add_node(pid, pl, pr) 246 | 247 | # print(cur_r) 248 | feat_valid = len(traj_feats) 249 | 250 | self.b_trajs.add_feature(traj_feats, feat_valid) 251 | 252 | self.b_polys = cur_polys 253 | # print(self.b_rew) 254 | return True 255 | 256 | def start_search(self): 257 | self.b_envs.append(self.env) 258 | self.b_polys.append(self.env.get_poly()) 259 | self.b_poly_sequence.append(self.b_polys) 260 | self.b_r_sequence.append([0.0]) 261 | self.b_rew.append(0.0) 262 | self.b_residual.append(0.0) 263 | # print(self.filename) 264 | while True: 265 | pos_rew = self.feedforward_search() 266 | self.b_round += 1 267 | if pos_rew == False: 268 | break 269 | 270 | # print(os.path.basename(self.filename)[:-4]) 271 | self.b_trajs.prepare_data(os.path.join( 272 | self.output_folder, os.path.basename(self.filename)[:-4])) 273 | 274 | if self.export: 275 | self.b_trajs.export_best_segmentation(os.path.join( 276 | self.output_folder, os.path.basename(self.filename)[:-4]), self.export_polys) 277 | -------------------------------------------------------------------------------- /pymdp/ranker/rnn_ranker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sklearn.datasets 4 | import torch 5 | import numpy as np 6 | import torch.nn.functional as F 7 | from sklearn.metrics import accuracy_score 8 | import random 9 | 10 | #use_cuda = torch.cuda.is_available() 11 | use_cuda = False 12 | device = torch.device("cuda:0" if use_cuda else "cpu") 13 | 14 | np.random.seed(0) 15 | X, y = sklearn.datasets.make_moons(200,noise=0.2) 16 | 17 | 18 | import matplotlib.pyplot as plt 19 | 20 | plt.scatter(X[:,0],X[:,1],s=40,c=y,cmap=plt.cm.binary) 21 | 22 | import torch.nn as nn 23 | import torch.nn.functional as F 24 | 25 | from torch.utils.data import DataLoader, Dataset, TensorDataset 26 | 27 | #our class must extend nn.Module 28 | class ClsNet(nn.Module): 29 | def __init__(self, input_size, hidden_size): 30 | super(ClsNet,self).__init__() 31 | #Our network consists of 3 layers. 1 input, 1 hidden and 1 output layer 32 | self.hidden_size = hidden_size 33 | 34 | #This applies Linear transformation to input data. 35 | self.fc1 = nn.Linear(input_size+hidden_size, 24) 36 | 37 | #This applies linear transformation to produce output data 38 | self.fc2 = nn.Linear(24, 6) 39 | 40 | self.fc3 = nn.Linear(6, 1) 41 | 42 | self.i2h = nn.Linear(input_size + hidden_size, hidden_size) 43 | 44 | def forward(self, x, hidden): 45 | hidden = hidden.repeat((x.size()[0], 1)) 46 | combined = torch.cat((x, hidden), 1) 47 | # Output hidden layer 48 | hidden = self.i2h(combined) 49 | #Output of the first layer 50 | x = self.fc1(combined) 51 | #Activation function is Relu. Feel free to experiment with this 52 | x = torch.relu(x) 53 | #This produces output 54 | x = self.fc2(x) 55 | x = torch.relu(x) 56 | output = self.fc3(x) 57 | hidden = torch.mean(hidden, dim=0) 58 | return output, hidden 59 | 60 | def init_hidden(self): 61 | return torch.zeros(1, self.hidden_size).to(device) 62 | 63 | #todo: finish it 64 | def predict(self, x, hidden): 65 | #Apply softmax to output 66 | output, hidden = self.forward(x, hidden) 67 | pred = torch.sigmoid(output).detach() > 0.5 68 | return pred, hidden 69 | 70 | def save_model(model, file): 71 | torch.save(model.state_dict(), file) 72 | 73 | def load_model(model, file, default_device='cpu'): 74 | device = torch.device(default_device) 75 | model.load_state_dict(torch.load(file, map_location=device)) 76 | model.eval() 77 | return model 78 | 79 | 80 | def adjust_learning_rate(optimizer, epoch, init_lr): 81 | """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" 82 | lr = init_lr * (0.4 ** (epoch // 1000)) 83 | for param_group in optimizer.param_groups: 84 | param_group['lr'] = lr 85 | 86 | 87 | def train(model, feats, ys, optimizer): 88 | global device 89 | hidden = model.init_hidden() 90 | criterion = nn.BCEWithLogitsLoss() 91 | for i in range(len(feats)): 92 | tx = feats[i] 93 | ty = ys[i] 94 | if tx.size()[0] == 0: 95 | return 96 | output, hidden = model.forward(tx, hidden) 97 | loss = criterion(output, ty) 98 | optimizer.zero_grad() 99 | loss.backward(retain_graph=True) 100 | optimizer.step() 101 | 102 | def predict(model, feats, ys, acc, acc_all): 103 | global device 104 | hidden = model.init_hidden() 105 | for i in range(len(feats)): 106 | tx = feats[i] 107 | ty = ys[i] 108 | if tx.size()[0] == 0: 109 | return 110 | _y, hidden = model.predict(tx, hidden) 111 | acc.append(accuracy_score(ty, _y)) 112 | acc_all.append(_y.size()[0]) 113 | 114 | class RNN_Ranker(): 115 | def __init__(self, timestamp, load=True): 116 | self.model = ClsNet(12, 3) 117 | load_model(self.model, timestamp+'.pth') 118 | self.factor = 1.0 119 | self.hidden = self.model.init_hidden() 120 | 121 | def set_factor(self, factor): 122 | self.factor = factor 123 | self.hidden = self.model.init_hidden() 124 | 125 | def rank_features(self, features): 126 | _features = np.copy(features) 127 | for f in _features: 128 | f[1] *= self.factor 129 | f[4] *= self.factor 130 | f[5] *= self.factor 131 | # return np.array([0, 1, 2, 3, 4]) 132 | 133 | test_x = [] 134 | for i in range(len(_features)): 135 | for j in range(len(_features)): 136 | if i == j: 137 | continue 138 | test_x.append(np.concatenate( 139 | (_features[i], _features[j]), axis=0)) 140 | 141 | test_x = np.array(test_x) 142 | print(test_x.shape) 143 | test_x = torch.from_numpy(test_x).type(torch.FloatTensor).to(device) 144 | y, self.hidden = self.model.predict(test_x, self.hidden) 145 | y = y.detach().cpu().numpy().reshape(len(_features), len(_features)-1) 146 | y = np.sum(y, axis=1) 147 | # print(y) 148 | return np.argsort(y)[::-1] 149 | -------------------------------------------------------------------------------- /pymdp/ranker/uRanker.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .urank.evaluate_point import EvalPoint 3 | 4 | class uRanker(): 5 | def __init__(self, load=True): 6 | self.model = EvalPoint() 7 | self.factor = 1.0 8 | 9 | def set_factor(self, factor): 10 | self.factor = factor 11 | 12 | def rank_features(self, features): 13 | _features = np.copy(features) 14 | for f in _features: 15 | f[1] *= self.factor 16 | f[4] *= self.factor 17 | f[5] *= self.factor 18 | f[7] *= self.factor 19 | f[10] *= self.factor 20 | f[11] *= self.factor 21 | 22 | test_x = np.array(_features) 23 | test_x = test_x[:, 6::] 24 | print(test_x.shape) 25 | y = self.model.evaluate(test_x) 26 | return y -------------------------------------------------------------------------------- /pymdp/ranker/urank/evaluate.py: -------------------------------------------------------------------------------- 1 | """Evaluate the model""" 2 | 3 | import argparse 4 | import logging 5 | import os 6 | 7 | import numpy as np 8 | import tensorflow as tf 9 | 10 | from model.utils import Params 11 | from model.utils import set_logger, load_best_ndcgs 12 | from model.evaluation import evaluate 13 | from model.reader import input_fn 14 | from model.reader import load_dataset_from_tfrecords 15 | from model.modeling import model_fn 16 | 17 | 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('--model_dir', default='experiments/base_model', 20 | help="Directory containing params.json") 21 | parser.add_argument('--residual_model_dir', default='experiments/residual_model', 22 | help="Directory containing params.json") 23 | # loss functions 24 | # grank, urrank, ranknet, listnet, listmle, lambdarank, mdprank 25 | parser.add_argument('--loss_fn', default='grank', 26 | help="model loss function") 27 | # tf data folder for 28 | # OHSUMED, MQ2007, MSLR-WEB10K, MSLR-WEB30K 29 | parser.add_argument('--data_dir', default='../data/OHSUMED/5', 30 | help="Directory containing the dataset") 31 | # OHSUMED, MQ2007, MSLR-WEB10K, MSLR-WEB30K 32 | parser.add_argument('--tfrecords_filename', default='OHSUMED.tfrecords', 33 | help="Directory containing the dataset") 34 | parser.add_argument('--restore_from', default='best_weights', 35 | help="Subdirectory of the best weights") 36 | 37 | if __name__ == '__main__': 38 | # Set the random seed for the whole graph 39 | tf.set_random_seed(230) 40 | # Load the parameters 41 | args = parser.parse_args() 42 | json_path = os.path.join(args.model_dir, 'params.json') 43 | assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path) 44 | params = Params(json_path) 45 | if params.mlp_sizes is None or len(params.mlp_sizes) == 0: 46 | logging.error('mlp_sizes are not set correctly, at least one MLP layer is required') 47 | params.dict['loss_fn'] = args.loss_fn 48 | if params.num_learners > 1: 49 | params.dict['use_residual'] = True 50 | # Load the parameters from the dataset, that gives the size etc. into params 51 | json_path = os.path.join(args.data_dir, 'dataset_params.json') 52 | assert os.path.isfile(json_path), "No json file found at {}, run build.py".format(json_path) 53 | params.update(json_path) 54 | # Set the logger 55 | set_logger(os.path.join(args.model_dir, 'evaluate.log')) 56 | # # Get paths for tfrecords 57 | path_eval_tfrecords = os.path.join(args.data_dir, 'test_' + args.tfrecords_filename) 58 | # Create the input data pipeline 59 | logging.info("Creating the dataset...") 60 | eval_dataset = load_dataset_from_tfrecords(path_eval_tfrecords) 61 | # Create iterator over the test set 62 | eval_inputs = input_fn('test', eval_dataset, params) 63 | logging.info("- done.") 64 | # Define the model 65 | logging.info("Creating the model...") 66 | weak_learner_id = load_best_ndcgs(os.path.join(args.model_dir, args.restore_from, 'learner.json'))[0] 67 | eval_model_spec = model_fn('test', eval_inputs, params, reuse=False, \ 68 | weak_learner_id=int(weak_learner_id)) 69 | # node_names = [n.name for n in tf.get_default_graph().as_graph_def().node] 70 | # print(node_names) 71 | logging.info("- done.") 72 | logging.info("Starting evaluation") 73 | logging.info("Optimized using {} learners".format(weak_learner_id)) 74 | evaluate(eval_model_spec, args.model_dir, params, args.restore_from) 75 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/evaluate_point.py: -------------------------------------------------------------------------------- 1 | """Evaluate the model""" 2 | 3 | import argparse 4 | import logging 5 | import os 6 | import warnings 7 | os.environ['KMP_WARNINGS'] = '0' 8 | os.environ['CUDA_VISIBLE_DEVICES']="-1" 9 | 10 | from prepare_data import normalize_min_max_feature_array, normalize_mean_max_feature_array 11 | 12 | warnings.filterwarnings('ignore', category=FutureWarning) 13 | import numpy as np 14 | import tensorflow as tf 15 | 16 | tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) 17 | tf.get_logger().setLevel(logging.ERROR) 18 | 19 | from model.utils import Params 20 | from model.utils import set_logger, load_best_ndcgs 21 | from model.reader import input_fn 22 | from model.reader import load_dataset_from_tfrecords 23 | from model.modeling import model_fn 24 | 25 | import logging 26 | import os 27 | 28 | from model.utils import save_dict_to_json, save_predictions_to_file 29 | from model.utils import get_expaned_metrics 30 | 31 | class EvaluatePointConfig: 32 | def __init__(self): 33 | self.model_dir = "E:/RAL2020/ranker/src/experiments/base_model" 34 | self.residual_model_dir = 'E:/RAL2020/ranker/src/experiments/residual_model' 35 | self.loss_fn = "urank" 36 | self.data_dir = "E:/RAL2020/ranker/data/RAL-6/1" 37 | self.tfrecords_filename = 'RAL.tfrecords' 38 | self.restore_from = 'best_weights' 39 | 40 | """Tensorflow utility functions for evaluation""" 41 | 42 | 43 | def evaluate_sess(sess, model_spec, num_steps, features, labels, writer=None, params=None): 44 | """Train the model on `num_steps` batches. 45 | 46 | Args: 47 | sess: (tf.Session) current session 48 | model_spec: (dict) contains the graph operations or nodes needed for training 49 | num_steps: (int) train for this number of batches 50 | writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything 51 | params: (Params) hyperparameters 52 | """ 53 | update_metrics = model_spec['update_metrics'] 54 | eval_metrics = model_spec['metrics'] 55 | global_step = tf.train.get_global_step() 56 | # Load the evaluation dataset into the pipeline and initialize the metrics init op 57 | # sess.run([model_spec['iterator_init_op'], model_spec['metrics_init_op']]) 58 | 59 | # sess.run(model_spec['iterator_init_op']) 60 | sess.run(model_spec['metrics_init_op']) 61 | 62 | if params.save_predictions: 63 | # save the predictions and lable_qid to files 64 | prediction_list = [] 65 | label_list = [] 66 | # compute metrics over the dataset 67 | for temp_query_id in range(int(1)): 68 | # prediction_per_query, label_per_query, height = sess.run([predictions, labels, model_spec["height"]]) 69 | # logging.info("- height per query: \n" + str(height)) 70 | prediction_per_query, _ = sess.run([model_spec["predictions"], 71 | update_metrics], feed_dict={ 72 | "features:0": features, 73 | "labels:0": labels, 74 | "height:0": features.shape[0], 75 | "width:0": features.shape[1], 76 | "unique_rating:0": len(set(labels)), 77 | "label_gains:0": [2**v-1 for v in labels]}) 78 | return prediction_per_query 79 | save_predictions_to_file(prediction_list, "./prediction_output") 80 | # tensorflow mess up test input orders 81 | save_predictions_to_file(label_list, "./label_output") 82 | else: 83 | # only update metrics 84 | for temp_query_id in range(int(num_steps)): 85 | sess.run(update_metrics) 86 | # Get the values of the metrics 87 | metrics_values = {k: v[0] for k, v in eval_metrics.items()} 88 | metrics_val = sess.run(metrics_values) 89 | expanded_metrics_val = get_expaned_metrics(metrics_val, params.top_ks) 90 | metrics_string = " ; ".join("{}: {:05.3f}".format(k, v) for k, v in expanded_metrics_val.items()) 91 | logging.info("- Eval metrics: " + metrics_string) 92 | # Add summaries manually to writer at global_step_val 93 | if writer is not None: 94 | global_step_val = sess.run(global_step) 95 | for tag, val in expanded_metrics_val.items(): 96 | summ = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=val)]) 97 | writer.add_summary(summ, global_step_val) 98 | return expanded_metrics_val 99 | 100 | 101 | class EvalPoint: 102 | def __init__(self): 103 | # Load the parameters 104 | args = EvaluatePointConfig() 105 | json_path = os.path.join(args.model_dir, 'params.json') 106 | assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path) 107 | params = Params(json_path) 108 | if params.mlp_sizes is None or len(params.mlp_sizes) == 0: 109 | logging.error('mlp_sizes are not set correctly, at least one MLP layer is required') 110 | params.dict['loss_fn'] = args.loss_fn 111 | 112 | # Load the parameters from the dataset, that gives the size etc. into params 113 | json_path = os.path.join(args.data_dir, 'dataset_params.json') 114 | assert os.path.isfile(json_path), "No json file found at {}, run build.py".format(json_path) 115 | params.update(json_path) 116 | # Set the logger 117 | set_logger(os.path.join(args.model_dir, 'evaluate.log')) 118 | # # Get paths for tfrecords 119 | path_eval_tfrecords = os.path.join(args.data_dir, 'test_' + args.tfrecords_filename) 120 | # Create the input data pipeline 121 | logging.info("Creating the dataset...") 122 | eval_dataset = load_dataset_from_tfrecords(path_eval_tfrecords) 123 | # Create iterator over the test set 124 | # eval_inputs = input_fn('test', eval_dataset, params) 125 | eval_inputs = online_input_fn() 126 | logging.info("- done.") 127 | # print(type(eval_inputs)) 128 | 129 | # Define the model 130 | logging.info("Creating the model...") 131 | weak_learner_id = load_best_ndcgs(os.path.join(args.model_dir, args.restore_from, 'learner.json'))[0] 132 | self.model_spec = model_fn('test', eval_inputs, params, reuse=False, weak_learner_id=int(weak_learner_id)) 133 | # node_names = [n.name for n in tf.get_default_graph().as_graph_def().node] 134 | # print(node_names) 135 | logging.info("- done.") 136 | logging.info("Starting evaluation") 137 | logging.info("Optimized using {} learners".format(weak_learner_id)) 138 | self.saver = tf.train.Saver() 139 | self.sess = tf.Session() 140 | self.params = params 141 | self.sess.run(self.model_spec['variable_init_op']) 142 | save_path = os.path.join(args.model_dir, args.restore_from) 143 | if os.path.isdir(save_path): 144 | save_path = tf.train.latest_checkpoint(save_path) 145 | self.saver.restore(self.sess, save_path) 146 | 147 | def evaluate(self, features): 148 | num_steps = 1 149 | features = normalize_min_max_feature_array(features) 150 | n_features = features.shape[0] 151 | labels = [0 for i in range(n_features)] 152 | predicted_scores = evaluate_sess(self.sess, self.model_spec, num_steps, features, labels, params=self.params) 153 | predicted_scores = np.squeeze(predicted_scores) 154 | return np.argsort(predicted_scores)[::-1] 155 | 156 | def online_input_fn(): 157 | features = tf.placeholder(tf.float32, name="features") 158 | labels = tf.placeholder(tf.float32, name="labels") 159 | height = tf.placeholder(tf.int32, name="height") 160 | width = tf.placeholder(tf.int32, name="width") 161 | unique_rating = tf.placeholder(tf.int32, name="unique_rating") 162 | label_gains = tf.placeholder(tf.float32, name="label_gains") 163 | inputs = { 164 | 'features': features, 165 | 'labels': labels, 166 | 'height': height, 167 | 'width': width, 168 | 'unique_rating': unique_rating, 169 | 'label_gains': label_gains, 170 | } 171 | return inputs 172 | 173 | 174 | if __name__ == '__main__': 175 | # Set the random seed for the whole graph 176 | tf.set_random_seed(230) 177 | evaluator = EvalPoint() 178 | features = np.array([[0, 0, 0, 0, 0, 0, 0.238741, 0.270131, 0.346482, 0.025711, 0.000000, 0.270131], 179 | [0, 0, 0, 0, 0, 0, 0.242391, 0.298806, 0.327038, 0.098476, 0.000000, 0.298806]]) 180 | 181 | res = evaluator.evaluate(features) 182 | print("evaluated result = ", res) 183 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/best_weights/after-epoch-73.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/best_weights/after-epoch-73.data-00000-of-00001 -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/best_weights/after-epoch-73.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/best_weights/after-epoch-73.index -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/best_weights/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "after-epoch-73" 2 | all_model_checkpoint_paths: "after-epoch-73" 3 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/best_weights/learner.json: -------------------------------------------------------------------------------- 1 | { 2 | "stopped_at_learner": 0.0 3 | } -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-100.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-100.data-00000-of-00001 -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-100.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-100.index -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-96.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-96.data-00000-of-00001 -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-96.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-96.index -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-97.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-97.data-00000-of-00001 -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-97.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-97.index -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-98.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-98.data-00000-of-00001 -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-98.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-98.index -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-99.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-99.data-00000-of-00001 -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-99.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/last_weights/after-epoch-99.index -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/last_weights/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "after-epoch-100" 2 | all_model_checkpoint_paths: "after-epoch-96" 3 | all_model_checkpoint_paths: "after-epoch-97" 4 | all_model_checkpoint_paths: "after-epoch-98" 5 | all_model_checkpoint_paths: "after-epoch-99" 6 | all_model_checkpoint_paths: "after-epoch-100" 7 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/metrics_eval_best_weights.json: -------------------------------------------------------------------------------- 1 | { 2 | "ndcg_1": 0.42354467511177063, 3 | "ndcg_3": 0.4819876253604889, 4 | "ndcg_5": 0.5298611521720886, 5 | "ndcg_10": 0.5841916799545288 6 | } -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/metrics_eval_last_weights.json: -------------------------------------------------------------------------------- 1 | { 2 | "ndcg_1": 0.41776859760284424, 3 | "ndcg_3": 0.47838959097862244, 4 | "ndcg_5": 0.5276974439620972, 5 | "ndcg_10": 0.5813385844230652 6 | } -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/metrics_test_best_weights.json: -------------------------------------------------------------------------------- 1 | { 2 | "ndcg_1": 0.42255860567092896, 3 | "ndcg_2": 0.455078125, 4 | "ndcg_3": 0.4832306504249573, 5 | "ndcg_4": 0.5097373127937317, 6 | "ndcg_5": 0.5315667986869812, 7 | "err_1": 0.40935495495796204, 8 | "err_2": 0.47491729259490967, 9 | "err_3": 0.5008284449577332, 10 | "err_4": 0.5139602422714233, 11 | "err_5": 0.5219849348068237 12 | } -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/params.json: -------------------------------------------------------------------------------- 1 | { 2 | "learning_rate": 1e-4, 3 | "batch_size": 1, 4 | "num_epochs": 1000, 5 | "buffer_size": 1, 6 | "save_summary_steps": 100, 7 | "early_stoping_epochs": 200, 8 | "gradient_clip_value": 5, 9 | "mlp_sizes": [100, 100], 10 | "residual_mlp_sizes": [100, 50], 11 | "pooling": "MP", 12 | "rnn": "C1", 13 | "num_learners": 4, 14 | "top_ks": [1,2,3,4,5], 15 | "save_predictions": true, 16 | "use_residual": false, 17 | 18 | "training_keep_prob": 1.0, 19 | "top_k": 10, 20 | "pre_training": 0, 21 | "use_regularization": false, 22 | "dropout_rate": 0.3, 23 | "decay_size": 126, 24 | "decay_rate": 0.9, 25 | "mask": "diag_mask", 26 | "exploration": 0.7 27 | } 28 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/train_summaries/events.out.tfevents.1590751103.WILLIAMCWU-NB0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/train_summaries/events.out.tfevents.1590751103.WILLIAMCWU-NB0 -------------------------------------------------------------------------------- /pymdp/ranker/urank/experiments/base_model/vali_summaries/events.out.tfevents.1590751104.WILLIAMCWU-NB0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/pymdp/ranker/urank/experiments/base_model/vali_summaries/events.out.tfevents.1590751104.WILLIAMCWU-NB0 -------------------------------------------------------------------------------- /pymdp/ranker/urank/feature_norm_for_lambdarank.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Generate serialized TF records 3 | usage: python prepare_data.py 4 | ''' 5 | 6 | import os 7 | import argparse 8 | import json 9 | import numpy as np 10 | import tensorflow as tf 11 | import argparse 12 | import logging 13 | from model.utils import save_dict_to_json 14 | 15 | 16 | # change RAW_RANK_DATA and TF_RANK_DATA accordingly 17 | # for example the full path for '../../learning_to_rank_data_sets_OHSUMED' 18 | RAW_RANK_DATA = os.environ.get('RAW_RANK_DATA') 19 | MODIFIED_RANK_DATA = os.environ.get('RAW_RANK_DATA') + '_modified' 20 | 21 | def get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type): 22 | OHSUMED_data_folder = os.path.join('OHSUMED', 'Feature-min', 'Fold{}'.format(fold_str)) 23 | # OHSUMED 24 | # print('file_type', file_type) 25 | full_file_name = os.path.join(RAW_RANK_DATA, OHSUMED_data_folder, file_type) 26 | if file_type == 'train': 27 | full_file_name += 'ing' 28 | if file_type == 'vali': 29 | full_file_name += 'dation' 30 | full_file_name += 'set' 31 | data_path = full_file_name + '.txt' 32 | return data_path 33 | 34 | def get_data_path(tfrecords_folder, fold_str, file_type): 35 | data_path = '' 36 | # OHSUMED 37 | if tfrecords_folder == 'OHSUMED': 38 | data_path = get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type) 39 | else: 40 | # MQ2007_data 41 | MS_data_folder = os.path.join(tfrecords_folder, 'Fold{}'.format(fold_str)) 42 | data_path = os.path.join(RAW_RANK_DATA, MS_data_folder, file_type + ".txt") 43 | return data_path 44 | 45 | def normalize_mean_max_feature_array(array): 46 | mean = array.mean(axis = 0) 47 | abs_max = abs(array.max(axis = 0)) 48 | epilson = 1e-8 49 | abs_max = abs_max + epilson 50 | normalized_array = (array - mean) / abs_max 51 | return normalized_array 52 | 53 | def normalize_min_max_feature_array(array): 54 | mini = array.min(axis = 0) 55 | maxi = array.max(axis = 0) 56 | epilson = 1e-8 57 | value_range = maxi - mini + epilson 58 | normalized_array = (array - mini) / value_range 59 | return normalized_array 60 | 61 | def _bytes_feature(value): 62 | value = value if type(value) == list else [value] 63 | return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) 64 | 65 | def _int64_feature(value): 66 | value = value if type(value) == list else [value] 67 | return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) 68 | 69 | def _float_feature(value): 70 | value = value if type(value) == list else [value] 71 | return tf.train.Feature(float_list=tf.train.FloatList(value=value)) 72 | 73 | def convert(tfrecords_folder, file_type, fold): 74 | group_features = {} 75 | group_labels = {} 76 | fold = str(fold) 77 | data_path = get_data_path(tfrecords_folder, fold, file_type) 78 | print('data_path', data_path) 79 | 80 | # if file_type == 'vali': 81 | # file_type = 'eval' 82 | # tfrecords_filename = tfrecords_folder + '.tfrecords' 83 | # complete_file_name = os.path.join(TF_RANK_DATA, tfrecords_folder, fold, \ 84 | # file_type + "_" + tfrecords_filename) 85 | # writer = tf.python_io.TFRecordWriter(complete_file_name) 86 | full_file_name = file_type 87 | if file_type == 'train': 88 | full_file_name += 'ing' 89 | if file_type == 'vali': 90 | full_file_name += 'dation' 91 | full_file_name += 'set' 92 | complete_file_name = os.path.join(MODIFIED_RANK_DATA, tfrecords_folder, fold, \ 93 | full_file_name + ".txt") 94 | if tfrecords_folder == 'OHSUMED': 95 | complete_file_name = os.path.join(MODIFIED_RANK_DATA, tfrecords_folder, 'Feature-min', fold, \ 96 | full_file_name + ".txt") 97 | 98 | writer = open(complete_file_name, 'w') 99 | max_height = 0 100 | with open(data_path, "r") as f: 101 | for line in f: 102 | if not line: 103 | break 104 | if "#" in line: 105 | line = line[:line.index("#")] 106 | splits = line.strip().split(" ") 107 | label = float(splits[0]) 108 | group = int(splits[1].split(":")[1]) 109 | features = [float(split.split(":")[1]) for split in splits[2:]] 110 | 111 | if group in group_features: 112 | new_feature_list = group_features[group] 113 | new_feature_list.append(features) 114 | group_features[group] = new_feature_list 115 | 116 | new_label_list = group_labels[group] 117 | new_label_list.append(label) 118 | group_labels[group] = new_label_list 119 | else: 120 | feature_list = [] 121 | feature_list.append(features) 122 | group_features[group] = feature_list 123 | 124 | label_list = [] 125 | label_list.append(label) 126 | group_labels[group] = label_list 127 | 128 | query_ids = list(group_features.keys()) 129 | query_ids.sort() 130 | # print('fold', fold, ', len', len(query_ids), ', file_type', file_type, ', query_ids', query_ids) 131 | num_queries = 0 132 | feature_dim = 0 133 | doc_count = 0 134 | 135 | for group in group_features: 136 | label_list = group_labels[group] 137 | label_array = np.asarray(label_list, dtype=np.float32) 138 | # remove line 136-138 to keep the original data 139 | # # remove all 0 label entries 140 | 141 | # if label_array.sum() < 1: 142 | # print('All 0 label entries: ', str(group), str(label_array.sum())) 143 | # continue 144 | # printing out queries that only had 0 labels 145 | # if label_array.sum() < 1: 146 | # print('All 0 label entries: ', str(group), str(label_array.sum())) 147 | if label_array.sum() == np.amax(label_array) * label_array.size: 148 | # print('All same label entricleares: {}, max/min rating: {}, number of docs: {}'.format(group, \ 149 | # np.amax(label_array), label_array.size)) 150 | continue 151 | # if file_type == 'test' and label_array.sum() == np.amax(label_array) * label_array.size: 152 | # print('All same label entries in test: {}, max/min rating: {}, number of docs: {}'.format(group, \ 153 | # np.amax(label_array), label_array.size)) 154 | 155 | # if file_type != 'test' and label_array.sum() == np.amax(label_array) * label_array.size: 156 | # # keep the test data unchanged 157 | # # but save some steps in training and validation/eval 158 | # # print('All same label entries: {}, max/min rating: {}, number of docs: {}'.format(group, \ 159 | # # np.amax(label_array), label_array.size)) 160 | # continue 161 | feature_array = np.asarray(group_features[group], dtype=np.float32) 162 | normalized_feature_array = normalize_min_max_feature_array(feature_array) 163 | # feature_raw = normalized_feature_array.tostring() 164 | # print('feature_raw', normalized_feature_array) 165 | for i in range(0, len(normalized_feature_array)): 166 | qid = group 167 | label = label_list[i] 168 | r = list(normalized_feature_array[i]) 169 | feature_string = [' {}:{}'.format(i, r[i-1]) for i in range(1, len(r) + 1)] 170 | feature_string = ' '.join(feature_string) 171 | feature_string = '{} qid:{}{}'.format(label, qid, feature_string) 172 | # print(feature_string) 173 | writer.write(feature_string) 174 | writer.flush() 175 | 176 | # the number of documents of a query 177 | height = normalized_feature_array.shape[0] 178 | if height > max_height: 179 | max_height = height 180 | # feature dim (same for all queries) 181 | width = normalized_feature_array.shape[1] 182 | label_list = group_labels[group] 183 | unique_rating = len(set(label_list)) 184 | label_gain_list = [2**v-1 for v in label_list] 185 | doc_count += height 186 | num_queries += 1 187 | # example = tf.train.Example(features=tf.train.Features(feature={ 188 | # 'height': _int64_feature(height), 189 | # 'width': _int64_feature(width), 190 | # 'feature_raw': _bytes_feature(feature_raw), 191 | # 'label_gain': _float_feature(label_gain_list), 192 | # 'unique_rating': _int64_feature(unique_rating), 193 | # 'label': _float_feature(label_list)})) 194 | # writer.write(example.SerializeToString()) 195 | 196 | # writer.close() 197 | writer.close() 198 | # print('max_height in {} : {}'.format(tfrecords_folder, max_height)) 199 | # query_ids = list(group_features.keys()) 200 | feature_list_0 = group_features[query_ids[0]] 201 | feature_dim = len(feature_list_0[0]) 202 | # return len(query_ids), feature_dim, doc_count 203 | return num_queries, feature_dim, doc_count 204 | 205 | def main(): 206 | tfrecords_folders = ['MSLR-WEB10K', 'MSLR-WEB30K']# 'OHSUMED', 'MQ2007', 'MSLR-WEB10K', 'MSLR-WEB30K' 207 | folds = 5 208 | for tfrecords_folder in tfrecords_folders: 209 | for fold in range(1, folds + 1): 210 | write2folder = os.path.join(MODIFIED_RANK_DATA, tfrecords_folder, str(fold)) 211 | if tfrecords_folder == 'OHSUMED': 212 | write2folder = os.path.join(MODIFIED_RANK_DATA, tfrecords_folder, 'Feature-min', str(fold)) 213 | if not os.path.exists(write2folder): 214 | os.makedirs(write2folder) 215 | # use eval in the write part of tfrecords for now 216 | eval_size, eval_feature_dim, eval_doc_count = convert(tfrecords_folder, 'vali', fold) 217 | test_size, test_feature_dim, test_doc_count = convert(tfrecords_folder, 'test', fold) 218 | train_size, train_feature_dim, train_doc_count = convert(tfrecords_folder, 'train', fold) 219 | # # Save datasets properties in json file 220 | # sizes = { 221 | # 'feature_dim': train_feature_dim, 222 | # 'train_size': train_size, 223 | # 'train_doc_count': train_doc_count, 224 | # 'eval_size': eval_size, 225 | # 'eval_doc_count': eval_doc_count, 226 | # 'test_size': test_size, 227 | # 'test_doc_count': test_doc_count 228 | # } 229 | # save_dict_to_json(sizes, os.path.join(write2folder, 'dataset_params.json')) 230 | 231 | if __name__ == "__main__": 232 | main() 233 | print("Done!") 234 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/lambda_cv_correct.py: -------------------------------------------------------------------------------- 1 | import os, sys, time 2 | 3 | RAW_RANK_DATA = os.environ.get('RAW_RANK_DATA') 4 | LIGHTGBM_DATA = os.environ.get('LIGHTGBM_DATA') 5 | # PREDICTION_RESULT_FILE = 'LightGBM_predict_result' 6 | 7 | def get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type): 8 | OHSUMED_data_folder = os.path.join('OHSUMED', 'Feature-min', 'Fold{}'.format(fold_str)) 9 | # OHSUMED 10 | # print('file_type', file_type) 11 | full_file_name = os.path.join(RAW_RANK_DATA, OHSUMED_data_folder, file_type) 12 | if file_type == 'train': 13 | full_file_name += 'ing' 14 | if file_type == 'vali': 15 | full_file_name += 'dation' 16 | full_file_name += 'set' 17 | data_path = full_file_name + '.txt' 18 | return data_path 19 | 20 | def get_data_path(tfrecords_folder, fold_str, file_type): 21 | data_path = '' 22 | # OHSUMED 23 | if tfrecords_folder == 'OHSUMED': 24 | data_path = get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type) 25 | else: 26 | # MQ2007_data 27 | MS_data_folder = os.path.join(tfrecords_folder, 'Fold{}'.format(fold_str)) 28 | data_path = os.path.join(RAW_RANK_DATA, MS_data_folder, file_type + ".txt") 29 | return data_path 30 | 31 | def run_pl(lightgbm_folder, fold, PREDICTION_RESULT_FILE): 32 | fold = str(fold) 33 | data_path = get_data_path(lightgbm_folder, fold, 'test') 34 | print('data_path', data_path) 35 | fold_result_file = '{}_{}_ndcg'.format(lightgbm_folder, fold) 36 | os.system('perl mslr-eval-score-mslr.pl {} {} {} 0'.format(data_path, \ 37 | PREDICTION_RESULT_FILE, fold_result_file)) 38 | complete_result_file = '{}_ndcg.txt'.format(lightgbm_folder) 39 | os.system('cat "{}" >> "{}"'.format(fold_result_file, complete_result_file)) 40 | os.system('echo "\n" >> "{}"'.format(complete_result_file)) 41 | # for the original ndcg pl script 42 | os.system('perl mslr-eval-score-mslr-has0.pl {} {} {} 0'.format(data_path, \ 43 | PREDICTION_RESULT_FILE, fold_result_file + '-has0s')) 44 | complete_result_file_original = '{}_ndcg-has0s.txt'.format(lightgbm_folder) 45 | os.system('cat "{}" >> "{}"'.format(fold_result_file + '-has0s', complete_result_file_original)) 46 | os.system('echo "\n" >> "{}"'.format(complete_result_file_original)) 47 | 48 | def main(): 49 | lightgbm_folders = ['MSLR-WEB30K']# 'OHSUMED', 'MQ2007', 'MSLR-WEB10K', 'MSLR-WEB30K' 50 | folds = 5 51 | for lightgbm_folder in lightgbm_folders: 52 | complete_result_file = '{}_ndcg.txt'.format(lightgbm_folder) 53 | if os.path.isfile(complete_result_file): 54 | os.system('rm {}'.format(complete_result_file)) 55 | os.system('rm {}_*_ndcg'.format(lightgbm_folder)) 56 | for fold in range(1, folds + 1): 57 | input_data_folder = os.path.join(LIGHTGBM_DATA, lightgbm_folder, str(fold)) 58 | # print(input_data_folder) 59 | os.system('cp template_train.conf train.conf') 60 | os.system('cp template_predict.conf predict.conf') 61 | data_path = os.path.join(input_data_folder, 'rank.train') 62 | valid_data_path = os.path.join(input_data_folder, 'rank.vali') 63 | test_data_path = os.path.join(input_data_folder, 'rank.test') 64 | # 'data = {}'.format(data_path) 65 | # 'valid_data = {}'.format(valid_data_path) 66 | # 'data = {}'.format(test_data_path) 67 | os.system('echo "data = {}\n" >> train.conf'.format(data_path)) 68 | os.system('echo "valid_data = {}\n" >> train.conf'.format(valid_data_path)) 69 | os.system('echo "output_model = {}_LightGBM_model\n" >> train.conf'.format(lightgbm_folder)) 70 | os.system('./lightgbm config=train.conf') 71 | 72 | os.system('echo "input_model = {}_LightGBM_model\n" >> predict.conf'.format(lightgbm_folder)) 73 | os.system('echo "data = {}\n" >> predict.conf'.format(test_data_path)) 74 | PREDICTION_RESULT_FILE = 'LightGBM_predict_result-{}-{}'.format(lightgbm_folder, fold) 75 | os.system('echo "output_result = {}\n" >> predict.conf'.format(PREDICTION_RESULT_FILE)) 76 | os.system('./lightgbm config=predict.conf') 77 | # # prediction scores in LightGBM_predict_result.txt 78 | run_pl(lightgbm_folder, fold, PREDICTION_RESULT_FILE) 79 | 80 | complete_result_file = '{}_ndcg.txt'.format(lightgbm_folder) 81 | os.system('cat "template_train.conf" >> "{}"'.format(complete_result_file)) 82 | os.system('cat "../../src/objective/rank_objective.hpp" >> "{}"'.format(complete_result_file)) 83 | 84 | if __name__ == '__main__': 85 | start_time = time.time() 86 | main() 87 | print('Done') 88 | print('-----{}----'.format((time.time() - start_time)/5/1000)) 89 | # python lambda_cv.py 2>&1 | tee lightgbm_msltr_accuracy.log 90 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/lambdarank_setting/lambda_cv.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | RAW_RANK_DATA = os.environ.get('RAW_RANK_DATA') 4 | LIGHTGBM_DATA = os.environ.get('LIGHTGBM_DATA') 5 | PREDICTION_RESULT_FILE = 'LightGBM_predict_result' 6 | 7 | def get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type): 8 | OHSUMED_data_folder = os.path.join('OHSUMED', 'Feature-min', 'Fold{}'.format(fold_str)) 9 | # OHSUMED 10 | # print('file_type', file_type) 11 | full_file_name = os.path.join(RAW_RANK_DATA, OHSUMED_data_folder, file_type) 12 | if file_type == 'train': 13 | full_file_name += 'ing' 14 | if file_type == 'vali': 15 | full_file_name += 'dation' 16 | full_file_name += 'set' 17 | data_path = full_file_name + '.txt' 18 | return data_path 19 | 20 | def get_data_path(tfrecords_folder, fold_str, file_type): 21 | data_path = '' 22 | # OHSUMED 23 | if tfrecords_folder == 'OHSUMED': 24 | data_path = get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type) 25 | else: 26 | # MQ2007_data 27 | MS_data_folder = os.path.join(tfrecords_folder, 'Fold{}'.format(fold_str)) 28 | data_path = os.path.join(RAW_RANK_DATA, MS_data_folder, file_type + ".txt") 29 | return data_path 30 | 31 | def run_pl(lightgbm_folder, fold): 32 | fold = str(fold) 33 | data_path = get_data_path(lightgbm_folder, fold, 'test') 34 | print('data_path', data_path) 35 | fold_result_file = '{}_{}_ndcg'.format(lightgbm_folder, fold) 36 | os.system('perl mslr-eval-score-mslr.pl {} {} {} 0'.format(data_path, \ 37 | PREDICTION_RESULT_FILE, fold_result_file)) 38 | complete_result_file = '{}_ndcg.txt'.format(lightgbm_folder) 39 | os.system('cat "{}" >> "{}"'.format(fold_result_file, complete_result_file)) 40 | os.system('echo "\n" >> "{}"'.format(complete_result_file)) 41 | 42 | def main(): 43 | lightgbm_folders = ['MSLR-WEB30K']# 'OHSUMED', 'MQ2007', 'MSLR-WEB10K', 'MSLR-WEB30K' 44 | folds = 5 45 | for lightgbm_folder in lightgbm_folders: 46 | complete_result_file = '{}_ndcg.txt'.format(lightgbm_folder) 47 | if os.path.isfile(complete_result_file): 48 | os.system('rm {}'.format(complete_result_file)) 49 | for fold in range(1, folds + 1): 50 | input_data_folder = os.path.join(LIGHTGBM_DATA, lightgbm_folder, str(fold)) 51 | # print(input_data_folder) 52 | os.system('cp template_train.conf train.conf') 53 | os.system('cp template_predict.conf predict.conf') 54 | data_path = os.path.join(input_data_folder, 'rank.train') 55 | valid_data_path = os.path.join(input_data_folder, 'rank.vali') 56 | test_data_path = os.path.join(input_data_folder, 'rank.test') 57 | # 'data = {}'.format(data_path) 58 | # 'valid_data = {}'.format(valid_data_path) 59 | # 'data = {}'.format(test_data_path) 60 | os.system('echo "data = {}\n" >> train.conf'.format(data_path)) 61 | os.system('echo "valid_data = {}\n" >> train.conf'.format(valid_data_path)) 62 | os.system('echo "output_model = LightGBM_model\n" >> train.conf'.format(valid_data_path)) 63 | os.system('./lightgbm config=train.conf') 64 | 65 | os.system('echo "input_model = LightGBM_model\n" >> predict.conf') 66 | os.system('echo "data = {}\n" >> predict.conf'.format(test_data_path)) 67 | os.system('echo "output_result = LightGBM_predict_result\n" >> predict.conf') 68 | os.system('./lightgbm config=predict.conf') 69 | # prediction scores in LightGBM_predict_result.txt 70 | run_pl(lightgbm_folder, fold) 71 | 72 | 73 | if __name__ == '__main__': 74 | main() 75 | print('Done') 76 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/lambdarank_setting/mslr-eval-ttest-mslr.pl: -------------------------------------------------------------------------------- 1 | #! 2 | # author: Jun Xu 3 | # 4 | use strict; 5 | use Statistics::DependantTTest; 6 | use Statistics::Distributions; 7 | 8 | my ($fnInputA, $fnInputB, $fnOutput) = @ARGV; 9 | my $argc = $#ARGV+1; 10 | if($argc != 3) 11 | { 12 | print "Invalid command line.\n"; 13 | print "Usage: $0 argv[1] argv[2] argv[3]\n"; 14 | print "argv[1]: evaluation file A\n"; 15 | print "argv[2]: evaluation file B\n"; 16 | print "argv[3]: result (output) file\n"; 17 | exit -1; 18 | } 19 | 20 | 21 | my %hsQueryPerf; 22 | my %hsTTestResult; 23 | 24 | ReadInputFile($fnInputA, "A"); 25 | ReadInputFile($fnInputB, "B"); 26 | 27 | open(FOUT, ">$fnOutput") or die "$!: Cannot create $fnOutput\n"; 28 | print FOUT "t-test results for A: $fnInputA and B: $fnInputB\n\n\n"; 29 | print FOUT "Measure\tNumber of queries\tMean A\tMean B\tt-value\tp-value\n"; 30 | my @measures = sort {MeasureStringComp($a) <=> MeasureStringComp($b)} keys(%hsQueryPerf); 31 | 32 | for(my $i = 0; $i < @measures; $i ++) 33 | { 34 | my $curMeasure = $measures[$i]; 35 | my @A_values; 36 | my @B_values; 37 | my $meanA; 38 | my $meanB; 39 | foreach my $qid (keys(%{$hsQueryPerf{$curMeasure}})) 40 | { 41 | if (exists($hsQueryPerf{$curMeasure}{$qid}{"A"}) 42 | && exists($hsQueryPerf{$curMeasure}{$qid}{"B"}) ) 43 | { 44 | push @A_values, $hsQueryPerf{$curMeasure}{$qid}{"A"}; 45 | $meanA += $hsQueryPerf{$curMeasure}{$qid}{"A"}; 46 | push @B_values, $hsQueryPerf{$curMeasure}{$qid}{"B"}; 47 | $meanB += $hsQueryPerf{$curMeasure}{$qid}{"B"}; 48 | } 49 | else 50 | { 51 | die "Empty value for $curMeasure, qid = $qid\n"; 52 | } 53 | } 54 | my $numQuery = @A_values; 55 | $meanA /= $numQuery; 56 | $meanB /= $numQuery; 57 | 58 | my $t_test = new Statistics::DependantTTest; 59 | $t_test->load_data('A',@A_values); 60 | $t_test->load_data('B',@B_values); 61 | my ($t_value, $deg_freedom) = $t_test->perform_t_test('A','B'); 62 | my ($p_value) = Statistics::Distributions::tprob($deg_freedom, $t_value); 63 | 64 | print FOUT "$curMeasure\t$numQuery\t$meanA\t$meanB\t$t_value\t$p_value\n"; 65 | } 66 | close(FOUT); 67 | 68 | #subs 69 | sub ReadInputFile 70 | { 71 | my ($fnInput, $flag) = @_; 72 | open(FIN, $fnInput) or die "$!: Cannot open file $fnInput.\n"; 73 | 74 | my @headline; 75 | while(defined(my $ln = )) 76 | { 77 | chomp($ln); 78 | next if (length($ln) < 2); 79 | if ($ln =~ m/^qid\t/i) 80 | { 81 | @headline = split(/\t/, $ln); 82 | } 83 | elsif ($ln =~ m/^average/i) 84 | { 85 | next; 86 | } 87 | else 88 | { 89 | my @vals = split(/\t/, $ln); 90 | for(my $idx = 1; $idx < @vals; $idx ++) 91 | { 92 | #vals[0]: $qid, vals[1...N]: $values 93 | #headline[0]: qid, headline[1...N]: measures 94 | $hsQueryPerf{$headline[$idx]}{$vals[0]}{$flag} = $vals[$idx]; 95 | } 96 | } 97 | } 98 | close(FIN); 99 | } 100 | 101 | sub MeasureStringComp 102 | { 103 | my $strMeasure = $_[0]; 104 | my $prec = 0; 105 | my $map = 100; 106 | my $ndcg = 500; 107 | my $meanNDCG = 600; 108 | my $other = 1000; 109 | if ($strMeasure =~ m/^MAP/i) 110 | { 111 | return $map; 112 | } 113 | elsif($strMeasure =~ m/^MeanNDCG/i) 114 | { 115 | return $meanNDCG; 116 | } 117 | elsif ($strMeasure =~ m/^NDCG\@(\d+)$/i) 118 | { 119 | my $iPos = $1; 120 | die "Error: Invalide measure string $strMeasure\n" if ($iPos > 100); 121 | return $ndcg + $iPos; 122 | } 123 | elsif ($strMeasure =~ m/^P\@(\d+)$/i) 124 | { 125 | my $iPos = $1; 126 | die "Error: Invalide measure string $strMeasure\n" if ($iPos > 100); 127 | return $prec + $iPos; 128 | } 129 | else 130 | { 131 | #die "Error: Invalide measure string $strMeasure\n"; 132 | return $other; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/lambdarank_setting/process_ndcg_results.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import pandas as pd 3 | 4 | def getPaths(): 5 | txt_files = glob.glob('*.txt') 6 | return txt_files 7 | # for file_path in txt_files: 8 | # print(file_path) 9 | 10 | # e.g., file_path = 'MQ2007_glrank.txt' 11 | def get_data_model(file_path): 12 | fields = file_path.split('.txt')[0] 13 | dataset_model = fields.split('_') 14 | return dataset_model[0], dataset_model[1] 15 | 16 | 17 | def get_ndcgs(file_path): 18 | with open(file_path, 'r') as f: 19 | ndcg_1s = [] 20 | ndcg_3s = [] 21 | ndcg_5s = [] 22 | ndcg_10s = [] 23 | 24 | lines = f.readlines() 25 | for line in lines: 26 | if line.startswith('- Eval metrics:'): 27 | # print(line) 28 | fields = line.split(' ') 29 | one_fold = [] 30 | ndcg_1 = float(fields[4]) 31 | ndcg_3 = float(fields[7]) 32 | ndcg_5 = float(fields[10]) 33 | ndcg_10 = float(fields[13]) 34 | 35 | ndcg_1s.append(ndcg_1) 36 | ndcg_3s.append(ndcg_3) 37 | ndcg_5s.append(ndcg_5) 38 | ndcg_10s.append(ndcg_10) 39 | if (len(ndcg_1s) != 5 or len(ndcg_3s) != 5 or len(ndcg_5s) != 5 or len(ndcg_10s) != 5): 40 | print('ERROR! in ', file_path) 41 | a_ndcg_1 = sum(ndcg_1s) / float(len(ndcg_1s)) 42 | a_ndcg_3 = sum(ndcg_3s) / float(len(ndcg_3s)) 43 | a_ndcg_5 = sum(ndcg_5s) / float(len(ndcg_5s)) 44 | a_ndcg_10 = sum(ndcg_10s) / float(len(ndcg_10s)) 45 | # print('a_ndcg_1 : ', a_ndcg_1) 46 | # print('a_ndcg_3 : ', a_ndcg_3) 47 | # print('a_ndcg_5 : ', a_ndcg_5) 48 | # print('a_ndcg_10 : ', a_ndcg_10) 49 | return [a_ndcg_1, a_ndcg_3, a_ndcg_5, a_ndcg_10] 50 | 51 | txt_files = getPaths() 52 | txt_files.sort() 53 | for file_path in txt_files: 54 | dataset, model = get_data_model(file_path) 55 | ncdgs = get_ndcgs(file_path) 56 | # df = pd.DataFrame({"PassengerId": testset['PassengerId'],"Survived": result}) 57 | # df.to_csv('reult.csv',header=True, index=False) 58 | print(dataset, model, ncdgs) -------------------------------------------------------------------------------- /pymdp/ranker/urank/lambdarank_setting/template_predict.conf: -------------------------------------------------------------------------------- 1 | 2 | task = predict 3 | 4 | # data = MQ2007/2/rank.test 5 | 6 | #input_model= LightGBM_model.txt 7 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/lambdarank_setting/template_train.conf: -------------------------------------------------------------------------------- 1 | # task type, support train and predict 2 | task = train 3 | 4 | # boosting type, support gbdt for now, alias: boosting, boost 5 | boosting_type = gbdt 6 | 7 | # application type, support following application 8 | # regression , regression task 9 | # binary , binary classification task 10 | # lambdarank , lambdarank task 11 | # alias: application, app 12 | objective = lambdarank 13 | 14 | # eval metrics, support multi metric, delimite by ',' , support following metrics 15 | # l1 16 | # l2 , default metric for regression 17 | # ndcg , default metric for lambdarank 18 | # auc 19 | # binary_logloss , default metric for binary 20 | # binary_error 21 | metric = ndcg 22 | 23 | # evaluation position for ndcg metric, alias : ndcg_at 24 | ndcg_eval_at = 10 25 | 26 | # frequence for metric output 27 | metric_freq = 1 28 | 29 | # true if need output metric for training data, alias: tranining_metric, train_metric 30 | is_training_metric = true 31 | 32 | # number of bins for feature bucket, 255 is a recommend setting, it can save memories, and also has good accuracy. 33 | max_bin = 10000 34 | 35 | # training data 36 | # if exsting weight file, should name to "rank.train.weight" 37 | # if exsting query file, should name to "rank.train.query" 38 | # alias: train_data, train 39 | # data = MQ2007/2/rank.train 40 | 41 | # validation data, support multi validation data, separated by ',' 42 | # if exsting weight file, should name to "rank.test.weight" 43 | # if exsting query file, should name to "rank.test.query" 44 | # alias: valid, test, test_data, 45 | # valid_data = MQ2007/2/rank.vali 46 | 47 | # number of trees(iterations), alias: num_tree, num_iteration, num_iterations, num_round, num_rounds 48 | num_trees = 1000 49 | 50 | # shrinkage rate , alias: shrinkage_rate 51 | learning_rate = 0.05 52 | 53 | # number of leaves for one tree, alias: num_leaf 54 | num_leaves = 64 55 | 56 | # type of tree learner, support following types: 57 | # serial , single machine version 58 | # feature , use feature parallel to train 59 | # data , use data parallel to train 60 | # voting , use voting based parallel to train 61 | # alias: tree 62 | tree_learner = serial 63 | 64 | # number of threads for multi-threading. One thread will use one CPU, defalut is setted to #cpu. 65 | # num_threads = 8 66 | 67 | # feature sub-sample, will random select 80% feature to train on each iteration 68 | # alias: sub_feature 69 | feature_fraction = 1.0 70 | 71 | # Support bagging (data sub-sample), will perform bagging every 5 iterations 72 | bagging_freq = 1 73 | 74 | # Bagging farction, will random select 80% data on bagging 75 | # alias: sub_row 76 | bagging_fraction = 0.9 77 | 78 | # minimal number data for one leaf, use this to deal with over-fit 79 | # alias : min_data_per_leaf, min_data 80 | min_data_in_leaf = 5 81 | 82 | # minimal sum hessians for one leaf, use this to deal with over-fit 83 | min_sum_hessian_in_leaf = 5.0 84 | 85 | # save memory and faster speed for sparse feature, alias: is_sparse 86 | is_enable_sparse = true 87 | 88 | # when data is bigger than memory size, set this to true. otherwise set false will have faster speed 89 | # alias: two_round_loading, two_round 90 | use_two_round_loading = false 91 | 92 | # true if need to save data to binary file and application will auto load data from binary file next time 93 | # alias: is_save_binary, save_binary 94 | is_save_binary_file = false 95 | 96 | # output model file 97 | #output_model = LightGBM_model.txt 98 | 99 | # support continuous train from trained gbdt model 100 | # input_model= trained_model.txt 101 | 102 | # output prediction file for predict task 103 | # output_result= prediction.txt 104 | 105 | # support continuous train from initial score file 106 | # input_init_score= init_score.txt 107 | 108 | 109 | # number of machines in parallel training, alias: num_machine 110 | num_machines = 1 111 | 112 | # local listening port in parallel training, alias: local_port 113 | local_listen_port = 12400 114 | 115 | # machines list file for parallel training, alias: mlist 116 | machine_list_file = mlist.txt 117 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/main.py: -------------------------------------------------------------------------------- 1 | # rm *.txt & ./bash.sh 2 | # experiments/base_model/params.json 3 | # /Users/xiaofengzhu/Documents/GitHub/uRank_urRank/uRank_urRank/src_temp 4 | # tensorboard --logdir 5 | import argparse 6 | import logging 7 | import os 8 | import time 9 | import tensorflow as tf 10 | from model.utils import Params 11 | from model.utils import set_logger 12 | from model.training import train_and_evaluate 13 | from model.reader import load_dataset_from_tfrecords 14 | from model.reader import input_fn 15 | from model.modeling import model_fn 16 | from model.evaluation import evaluate 17 | 18 | 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument('--model_dir', default='experiments/base_model', 21 | help="Directory containing params.json") 22 | # loss functions 23 | # grank, urrank, ranknet, listnet, listmle, lambdarank, mdprank 24 | parser.add_argument('--loss_fn', default='grank', help="model loss function") 25 | # tf data folder for 26 | # OHSUMED, MQ2007, MSLR-WEB10K, MSLR-WEB30K 27 | parser.add_argument('--data_dir', default='../data/OHSUMED/5', 28 | help="Directory containing the dataset") 29 | # OHSUMED, MQ2007, MSLR-WEB10K, MSLR-WEB30K 30 | parser.add_argument('--tfrecords_filename', default='OHSUMED.tfrecords', 31 | help="Dataset-filename for the tfrecords") 32 | # usage: python main.py --restore_dir experiments/base_model/best_weights 33 | parser.add_argument('--restore_dir', default=None, # experimens/base_model/best_weights 34 | help="Optional, directory containing weights to reload") 35 | 36 | if __name__ == '__main__': 37 | tf.reset_default_graph() 38 | # Set the random seed for the whole graph for reproductible experiments 39 | tf.set_random_seed(230) 40 | # Load the parameters from the experiment params.json file in model_dir 41 | args = parser.parse_args() 42 | json_path = os.path.join(args.model_dir, 'params.json') 43 | assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path) 44 | params = Params(json_path) 45 | # print('params.mlp_sizes\n', params.mlp_sizes) 46 | # print('params.top_ks\n', params.top_ks) 47 | if params.mlp_sizes is None or len(params.mlp_sizes) == 0: 48 | logging.error('mlp_sizes are not set correctly, at least one MLP layer is required') 49 | params.dict['loss_fn'] = args.loss_fn 50 | 51 | # # Load the parameters from the dataset, that gives the size etc. into params 52 | json_path = os.path.join(args.data_dir, 'dataset_params.json') 53 | assert os.path.isfile(json_path), "No json file found at {}, please run prepare_data.py".format(json_path) 54 | params.update(json_path) 55 | # Set the logger 56 | set_logger(os.path.join(args.model_dir, 'train.log')) 57 | path_train_tfrecords = os.path.join(args.data_dir, 'train_' + args.tfrecords_filename) 58 | path_eval_tfrecords = os.path.join(args.data_dir, 'vali_' + args.tfrecords_filename) 59 | # Create the input data pipeline 60 | logging.info("Creating the datasets...") 61 | train_dataset = load_dataset_from_tfrecords(path_train_tfrecords) 62 | eval_dataset = load_dataset_from_tfrecords(path_eval_tfrecords) 63 | # Specify other parameters for the dataset and the model 64 | # Create the two iterators over the two datasets 65 | train_inputs = input_fn('train', train_dataset, params) 66 | eval_inputs = input_fn('vali', eval_dataset, params) 67 | logging.info("- done.") 68 | # Define the models (2 different set of nodes that share weights for train and validation) 69 | logging.info("Creating the model...") 70 | train_model_spec = model_fn('train', train_inputs, params) 71 | eval_model_spec = model_fn('vali', eval_inputs, params, reuse=True) 72 | logging.info("- done.") 73 | # Train the model 74 | # log time 75 | start_time = time.time() 76 | logging.info("Starting training for at most {} epoch(s) for the initial learner".format(params.num_epochs)) 77 | global_epoch = train_and_evaluate(train_model_spec, eval_model_spec, args.model_dir, params, \ 78 | learner_id=0, restore_from=args.restore_dir) 79 | logging.info("global_epoch: {} epoch(s) at learner 0".format(global_epoch)) 80 | print("--- %s seconds ---" % (time.time() - start_time)) 81 | # start gradient boosting 82 | last_global_epoch = global_epoch 83 | if (params.num_learners > 1): 84 | ######################################################### 85 | for learner_id in range(1, params.num_learners): 86 | ######################################################### 87 | logging.info("Creating residual learner ".format(learner_id)) 88 | params.dict['use_residual'] = True 89 | ###END TO DO 90 | residual_train_model_spec = model_fn('train', train_inputs, params, reuse=tf.AUTO_REUSE, \ 91 | weak_learner_id=learner_id) 92 | residual_eval_model_spec = model_fn('vali', eval_inputs, params, reuse=True, \ 93 | weak_learner_id=learner_id) 94 | logging.info("- done.") 95 | args.restore_dir = 'best_weights' 96 | global_epoch = train_and_evaluate(residual_train_model_spec, residual_eval_model_spec, 97 | args.model_dir, params, learner_id=learner_id, restore_from=args.restore_dir, \ 98 | global_epoch=global_epoch) 99 | logging.info("global_epoch: {} epoch(s) at learner {}".format(global_epoch, learner_id)) 100 | if global_epoch - last_global_epoch == params.early_stoping_epochs: 101 | logging.info("boosting has stopped early at learner {}".format(learner_id)) 102 | break 103 | last_global_epoch = global_epoch 104 | print("--- %s seconds using boosting ---" % (time.time() - start_time)) 105 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/model/evaluation.py: -------------------------------------------------------------------------------- 1 | """Tensorflow utility functions for evaluation""" 2 | 3 | import logging 4 | import os 5 | 6 | from tqdm import trange 7 | import tensorflow as tf 8 | 9 | from model.utils import save_dict_to_json, save_predictions_to_file 10 | from model.utils import get_expaned_metrics 11 | 12 | 13 | def evaluate_sess(sess, model_spec, num_steps, writer=None, params=None): 14 | """Train the model on `num_steps` batches. 15 | 16 | Args: 17 | sess: (tf.Session) current session 18 | model_spec: (dict) contains the graph operations or nodes needed for training 19 | num_steps: (int) train for this number of batches 20 | writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything 21 | params: (Params) hyperparameters 22 | """ 23 | update_metrics = model_spec['update_metrics'] 24 | eval_metrics = model_spec['metrics'] 25 | global_step = tf.train.get_global_step() 26 | # Load the evaluation dataset into the pipeline and initialize the metrics init op 27 | # sess.run([model_spec['iterator_init_op'], model_spec['metrics_init_op']]) 28 | sess.run(model_spec['iterator_init_op']) 29 | sess.run(model_spec['metrics_init_op']) 30 | if params.save_predictions: 31 | # save the predictions and lable_qid to files 32 | prediction_list = [] 33 | label_list = [] 34 | # compute metrics over the dataset 35 | for temp_query_id in range(int(num_steps)): 36 | # prediction_per_query, label_per_query, height = sess.run([predictions, labels, model_spec["height"]]) 37 | # logging.info("- height per query: \n" + str(height)) 38 | prediction_per_query, label_per_query, label_gains, _ = sess.run([model_spec["predictions"], \ 39 | model_spec["labels"], model_spec["label_gains"], update_metrics]) 40 | prediction_list.extend([v[0] for v in prediction_per_query.tolist()]) 41 | # prediction_string = "\n".join(str(v[0]) for v in prediction_per_query.tolist()) 42 | # logging.info("- prediction_per_query: \n" + str(prediction_string)) 43 | label_per_query_list = label_per_query.tolist() 44 | label_gains_list = label_gains.tolist() 45 | # label_per_query_list_string = "\n".join(str(v[0]) for v in label_per_query_list) 46 | # logging.warning("- label_per_query_list_string: \n" + label_per_query_list_string) 47 | # label_gains_list_string = "\n".join(str(v[0]) for v in label_gains_list) 48 | # logging.info("- label_gains_list: \n" + label_gains_list_string) 49 | label_list.extend(['{} qid:{} 1:{}'.format(int(label_per_query_list[i][0]), \ 50 | temp_query_id, \ 51 | label_gains_list[i][0]) \ 52 | for i in range(0, len(label_per_query_list))]) 53 | save_predictions_to_file(prediction_list, "./prediction_output") 54 | # tensorflow mess up test input orders 55 | save_predictions_to_file(label_list, "./label_output") 56 | else: 57 | # only update metrics 58 | for temp_query_id in range(int(num_steps)): 59 | sess.run(update_metrics) 60 | # Get the values of the metrics 61 | metrics_values = {k: v[0] for k, v in eval_metrics.items()} 62 | metrics_val = sess.run(metrics_values) 63 | expanded_metrics_val = get_expaned_metrics(metrics_val, params.top_ks) 64 | metrics_string = " ; ".join("{}: {:05.3f}".format(k, v) for k, v in expanded_metrics_val.items()) 65 | logging.info("- Eval metrics: " + metrics_string) 66 | # Add summaries manually to writer at global_step_val 67 | if writer is not None: 68 | global_step_val = sess.run(global_step) 69 | for tag, val in expanded_metrics_val.items(): 70 | summ = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=val)]) 71 | writer.add_summary(summ, global_step_val) 72 | return expanded_metrics_val 73 | 74 | 75 | def evaluate(model_spec, model_dir, params, restore_from): 76 | """Evaluate the model 77 | 78 | Args: 79 | model_spec: (dict) contains the graph operations or nodes needed for evaluation 80 | model_dir: (string) directory containing config, weights and log 81 | params: (Params) contains hyperparameters of the model. 82 | Must define: num_epochs, train_size, batch_size, eval_size, save_summary_steps 83 | restore_from: (string) directory or file containing weights to restore the graph 84 | """ 85 | # Initialize tf.Saver 86 | saver = tf.train.Saver() 87 | # tf.reset_default_graph() 88 | with tf.Session() as sess: 89 | 90 | # Initialize the lookup table 91 | sess.run(model_spec['variable_init_op']) 92 | 93 | # Reload weights from the weights subdirectory 94 | save_path = os.path.join(model_dir, restore_from) 95 | if os.path.isdir(save_path): 96 | save_path = tf.train.latest_checkpoint(save_path) 97 | saver.restore(sess, save_path) 98 | 99 | # Evaluate 100 | num_steps = (params.test_size + params.batch_size - 1) // params.batch_size 101 | metrics = evaluate_sess(sess, model_spec, num_steps, params=params) 102 | metrics_name = '_'.join(restore_from.split('/')) 103 | save_path = os.path.join(model_dir, "metrics_test_{}.json".format(metrics_name)) 104 | save_dict_to_json(metrics, save_path) -------------------------------------------------------------------------------- /pymdp/ranker/urank/model/reader.py: -------------------------------------------------------------------------------- 1 | # python model/reader.py 2 | import os 3 | import argparse 4 | import numpy as np 5 | import tensorflow as tf 6 | import argparse 7 | import logging 8 | 9 | from model.utils import Params 10 | # from utils import Params 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument('--model_dir', default='experiments/base_model', 14 | help="Directory containing params.json") 15 | 16 | 17 | def _parse_function(serialized_example_proto): 18 | keys_to_features = { 19 | 'height': tf.FixedLenFeature([], tf.int64), 20 | 'width': tf.FixedLenFeature([], tf.int64), 21 | 'feature_raw': tf.FixedLenFeature([], tf.string), 22 | 'label_gain': tf.VarLenFeature(tf.float32), 23 | 'unique_rating': tf.FixedLenFeature([], tf.int64), 24 | 'label': tf.VarLenFeature(tf.float32) 25 | } 26 | inputs = tf.parse_single_example( 27 | serialized_example_proto, 28 | # Defaults are not specified since both keys are required. 29 | features=keys_to_features) 30 | height = tf.cast(inputs['height'], tf.int32) 31 | # height = tf.reshape(height, []) 32 | width = tf.cast(inputs['width'], tf.int32) 33 | # width = tf.reshape(width, []) 34 | feature = tf.decode_raw(inputs['feature_raw'], tf.float32) 35 | # feature = tf.reshape(feature, [height, width]) 36 | label_sparse = tf.cast(inputs['label'], tf.float32) 37 | # label = tf.sparse.to_dense(label_sparse, validate_indices=False) 38 | label = tf.sparse_tensor_to_dense(label_sparse, validate_indices=False) 39 | label_gain_sparse = tf.cast(inputs['label_gain'], tf.float32) 40 | # label_gain = tf.sparse.to_dense(label_gain_sparse, validate_indices=False) 41 | label_gain = tf.sparse_tensor_to_dense(label_gain_sparse, validate_indices=False) 42 | unique_rating = tf.cast(inputs['unique_rating'], tf.int32) 43 | # no need to add padding later 44 | return feature, label, height, width, label_gain, unique_rating 45 | 46 | 47 | 48 | def load_dataset_from_tfrecords(path_tfrecords_filename): 49 | # tfrecords_filename 50 | # file_type + "_" + tfrecords_filename 51 | dataset = tf.data.TFRecordDataset(path_tfrecords_filename) 52 | # Parse the record into tensors. 53 | dataset = dataset.map(_parse_function) 54 | return dataset 55 | 56 | 57 | def input_fn(mode, dataset, params): 58 | # Shuffle the dataset 59 | is_training = (mode == 'train') 60 | buffer_size = params.buffer_size if is_training else 1 61 | if mode != 'test': 62 | dataset = dataset.shuffle(buffer_size=buffer_size) 63 | # batch_size = 1 64 | # Generate batches 65 | dataset = dataset.batch(params.batch_size) 66 | # Repeat the input ## num_epochs times 67 | dataset = dataset.repeat() 68 | # prefetch a batch 69 | dataset = dataset.prefetch(1) 70 | # # Create a one-shot iterator 71 | # iterator = dataset.make_one_shot_iterator() 72 | iterator = dataset.make_initializable_iterator() 73 | # Get batch X and Y 74 | features, labels, height, width, label_gains, unique_rating = iterator.get_next() 75 | width = tf.squeeze(width) 76 | height = tf.squeeze(height) 77 | features = tf.reshape(features, [height, width]) 78 | labels = tf.reshape(labels, [-1, 1]) 79 | label_gains = tf.reshape(label_gains, [-1, 1]) 80 | unique_rating = tf.squeeze(unique_rating) 81 | 82 | iterator_init_op = iterator.initializer 83 | inputs = { 84 | 'features': features, 85 | 'labels': labels, 86 | 'height': height, 87 | 'width': width, 88 | 'unique_rating': unique_rating, 89 | 'label_gains': label_gains, 90 | # 'use_predicted_order':False, 91 | 'iterator_init_op': iterator_init_op 92 | } 93 | return inputs 94 | 95 | def _shuffle_docs(labels, features, height, width): 96 | n_data = tf.shape(labels)[0] 97 | indices = tf.range(n_data) 98 | labels_features = tf.concat([labels, features], 1) 99 | shuffled = tf.random_shuffle(labels_features) 100 | column_rows = tf.transpose(shuffled) 101 | new_labels = tf.gather(column_rows, [0]) 102 | new_features = tf.gather(column_rows, tf.range(1, width + 1)) 103 | # transpose back 104 | new_labels = tf.transpose(new_labels) # , [-1, 1] 105 | new_features = tf.transpose(new_features) # , [height, width] 106 | return new_labels, new_features 107 | 108 | if __name__ == "__main__": 109 | tf.set_random_seed(230) 110 | dataset = load_dataset_from_tfrecords("../data/OHSUMED/0/test_OHSUMED.tfrecords") 111 | args = parser.parse_args() 112 | json_path = os.path.join(args.model_dir, 'params.json') 113 | assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path) 114 | params = Params(json_path) 115 | mode = 'eval' 116 | # iterator_init_op = iterator.initializer 117 | inputs = input_fn(mode, dataset, params) 118 | iterator_init_op = inputs['iterator_init_op'] 119 | features, labels, label_gains, unique_rating = inputs['features'], inputs['labels'], inputs['label_gains'], inputs['unique_rating'] 120 | logging.info("- done loading dataset.") 121 | with tf.Session() as sess: 122 | init_op = tf.global_variables_initializer() 123 | sess.run(init_op) 124 | sess.run(iterator_init_op) 125 | for i in range(3): 126 | print(sess.run([label_gains, labels, unique_rating])) 127 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/model/training.py: -------------------------------------------------------------------------------- 1 | """Tensorflow utility functions for training""" 2 | # tensorboard --logdir=experiments/base_model/ 3 | # tensorboard --logdir=experiments/base_model/train_summaries 4 | # tensorboard --logdir=experiments/base_model/eval_summaries 5 | 6 | import logging 7 | import os 8 | 9 | from tqdm import trange 10 | import tensorflow as tf 11 | 12 | from model.utils import save_dict_to_json, load_best_ndcgs, get_expaned_metrics 13 | from model.evaluation import evaluate_sess 14 | 15 | 16 | def train_sess(sess, model_spec, num_steps, writer, params): 17 | """Train the model on `num_steps` batches 18 | 19 | Args: 20 | sess: (tf.Session) current session 21 | model_spec: (dict) contains the graph operations or nodes needed for training 22 | num_steps: (int) train for this number of batches 23 | writer: (tf.summary.FileWriter) writer for summaries 24 | params: (Params) hyperparameters 25 | """ 26 | # Get relevant graph operations or nodes needed for training 27 | 28 | loss = model_spec['loss'] 29 | train_op = model_spec['train_op'] 30 | update_metrics = model_spec['update_metrics'] 31 | metrics = model_spec['metrics'] 32 | summary_op = model_spec['summary_op'] 33 | global_step = tf.train.get_global_step() 34 | 35 | # Load the training dataset into the pipeline and initialize the metrics local variables 36 | # sess.run([model_spec['iterator_init_op'], model_spec['metrics_init_op']]) 37 | sess.run(model_spec['iterator_init_op']) 38 | sess.run(model_spec['metrics_init_op']) 39 | # Use tqdm for progress bar 40 | t = trange(int(num_steps)) 41 | for i in t: 42 | # Evaluate summaries for tensorboard only once in a while 43 | if i == params.save_summary_steps - 1: 44 | # if i % params.save_summary_steps == 0: 45 | # Perform a mini-batch update 46 | _, _, loss_val, summ, global_step_val = sess.run([train_op, update_metrics, loss, 47 | summary_op, global_step]) 48 | # Write summaries for tensorboard 49 | writer.add_summary(summ, global_step_val) 50 | else: 51 | _, _, loss_val = sess.run([train_op, update_metrics, loss]) 52 | # Log the loss in the tqdm progress bar 53 | t.set_postfix(loss='{:05.3f}'.format(loss_val)) 54 | metrics_values = {k: v[0] for k, v in metrics.items()} 55 | metrics_val = sess.run(metrics_values) 56 | expanded_metrics_val = get_expaned_metrics(metrics_val, params.top_ks) 57 | metrics_string = " ; ".join("{}: {:05.3f}".format(k, v) for k, v in expanded_metrics_val.items()) 58 | logging.info("- Train metrics: " + metrics_string) 59 | 60 | 61 | # NDCG@10 62 | # def isSavingWeights(eval_metrics, best_eval_metrics): 63 | # if eval_metrics[len(eval_metrics) - 1] >= best_eval_metrics[len(eval_metrics) - 1]: 64 | # return True 65 | # return False 66 | 67 | # NDCG@1, 3, 5, 10 68 | def isSavingWeights(eval_metrics, best_eval_metrics): 69 | for i in range(len(eval_metrics)): 70 | if eval_metrics[i] > best_eval_metrics[i]: 71 | return True 72 | elif eval_metrics[i] < best_eval_metrics[i]: 73 | return False 74 | else: 75 | continue 76 | return False 77 | 78 | def train_and_evaluate(train_model_spec, eval_model_spec, 79 | model_dir, params, learner_id=0, restore_from=None, global_epoch=1): 80 | """Train the model and evaluate every epoch. 81 | 82 | Args: 83 | train_model_spec: (dict) contains the graph operations or nodes needed for training 84 | eval_model_spec: (dict) contains the graph operations or nodes needed for evaluation 85 | model_dir: (string) directory containing config, weights and log 86 | params: (Params) contains hyperparameters of the model. 87 | Must define: num_epochs, train_size, batch_size, eval_size, save_summary_steps 88 | restore_from: (string) directory or file containing weights to restore the graph 89 | """ 90 | # Initialize tf.Saver instances to save weights during training 91 | last_saver = tf.train.Saver() # will keep last 5 epochs 92 | best_saver = tf.train.Saver(max_to_keep=1) # only keep 1 best checkpoint (best on eval) 93 | begin_at_epoch = 0 94 | with tf.Session() as sess: 95 | # Initialize model variables 96 | sess.run(train_model_spec['variable_init_op']) 97 | # For tensorboard (takes care of writing summaries to files) 98 | train_writer = tf.summary.FileWriter(os.path.join(model_dir, 'train_summaries'), sess.graph) 99 | eval_writer = tf.summary.FileWriter(os.path.join(model_dir, 'vali_summaries'), sess.graph) 100 | best_json_path = os.path.join(model_dir, "metrics_eval_best_weights.json") 101 | 102 | best_eval_metric = 0.0 # ndcg_1 103 | # best_loss_metric = float('inf') 104 | second_eval_metric = 0.0 # ndcg_3 105 | third_eval_metric = 0.0 # ndcg_5 106 | forth_eval_metric = 0.0 # ndcg_10 107 | # global_epoch = 0 108 | # Reload weights from directory if specified 109 | # restor from the previous learner 110 | if restore_from is not None: 111 | save_path = os.path.join(model_dir, restore_from) 112 | if os.path.isdir(save_path): 113 | save_path = tf.train.latest_checkpoint(save_path) 114 | # begin_at_epoch = int(restore_from.split('-')[-1]) 115 | logging.info("Restoring parameters from {}".format(save_path)) 116 | # last_saver = tf.train.import_meta_graph(save_path+".meta") 117 | pretrained_include = ['model/mlp'] 118 | if params.loss_fn=='urrank': 119 | pretrained_include = ['model/ur'] 120 | for i in range(1, learner_id): 121 | pretrained_include.append('residual_mlp_{}'.format(learner_id)) 122 | 123 | pretrained_vars = tf.contrib.framework.get_variables_to_restore(include=pretrained_include) 124 | pretrained_saver = tf.train.Saver(pretrained_vars, name="pretrained_saver") 125 | pretrained_saver.restore(sess, save_path) 126 | [best_eval_metric, second_eval_metric, third_eval_metric, forth_eval_metric] = \ 127 | load_best_ndcgs(best_json_path) 128 | # print('[best_eval_metric, second_eval_metric, third_eval_metric, forth_eval_metric]', \ 129 | # [best_eval_metric, second_eval_metric, third_eval_metric, forth_eval_metric]) 130 | # for each learner 131 | early_stopping_count = 0 132 | for epoch in range(begin_at_epoch, begin_at_epoch + params.num_epochs): 133 | if early_stopping_count == int(params.early_stoping_epochs): 134 | logging.info("Early stopping at learner {}, epoch {}/{}".format(learner_id, epoch + 1, \ 135 | begin_at_epoch + params.num_epochs)) 136 | break 137 | # Run one epoch 138 | logging.info("Learner {}, Epoch {}/{}".format(learner_id, epoch + 1, \ 139 | begin_at_epoch + params.num_epochs)) 140 | # Compute number of batches in one epoch (one full pass over the training set) 141 | num_steps = (params.train_size + params.batch_size - 1) // params.batch_size 142 | train_sess(sess, train_model_spec, num_steps, train_writer, params) 143 | # Save weights 144 | last_save_path = os.path.join(model_dir, 'last_weights', 'after-epoch') 145 | # global_epoch = int(params.num_learners) * int(params.num_epochs) + epoch + 1 146 | last_saver.save(sess, last_save_path, global_step=global_epoch) 147 | # Evaluate for one epoch on validation set 148 | num_steps = (params.eval_size + params.batch_size - 1) // params.batch_size 149 | metrics = evaluate_sess(sess, eval_model_spec, num_steps, eval_writer, params) 150 | # If best_eval, best_save_path 151 | # eval_metric = metrics['dcg'] 152 | # loss_metric = metrics['loss'] 153 | eval_metric = round(metrics['ndcg_1'], 6) 154 | eval_metric_2 = round(metrics['ndcg_3'], 6) 155 | eval_metric_3 = round(metrics['ndcg_5'], 6) 156 | eval_metric_4 = round(metrics['ndcg_10'], 6) 157 | # eval_metric = metrics['ndcg_1'] 158 | # eval_metric_2 = metrics['ndcg_3'] 159 | # eval_metric_3 = metrics['ndcg_5'] 160 | # eval_metric_4 = metrics['ndcg_10'] 161 | eval_metrics = [eval_metric, eval_metric_2, eval_metric_3, eval_metric_4] 162 | best_eval_metrics = [best_eval_metric, second_eval_metric, third_eval_metric, \ 163 | forth_eval_metric] 164 | if isSavingWeights(eval_metrics, best_eval_metrics): 165 | # rest early_stopping_count 166 | early_stopping_count = 0 167 | # Store new best ndcg_1 168 | # this worsk better than eval_metric > best_eval_metric 169 | # and isSavingWeights 170 | best_eval_metric = eval_metric 171 | # loss_metric = best_loss_metric 172 | second_eval_metric = eval_metric_2 173 | third_eval_metric = eval_metric_3 174 | forth_eval_metric = eval_metric_4 175 | # Save weights 176 | best_save_path = os.path.join(model_dir, 'best_weights', 'after-epoch') 177 | # global_epoch = int(params.num_learners) * int(params.num_epochs) + epoch + 1 178 | best_save_path = best_saver.save(sess, best_save_path, global_step=global_epoch) 179 | logging.info("- Found new best metric score, saving in {}".format(best_save_path)) 180 | # Save best eval metrics in a json file in the model directory 181 | save_dict_to_json(metrics, best_json_path) 182 | save_dict_to_json({'stopped_at_learner': learner_id}, \ 183 | os.path.join(model_dir, 'best_weights', 'learner.json')) 184 | else: 185 | early_stopping_count = early_stopping_count + 1 186 | # Save latest eval metrics in a json file in the model directory 187 | last_json_path = os.path.join(model_dir, "metrics_eval_last_weights.json") 188 | save_dict_to_json(metrics, last_json_path) 189 | global_epoch += 1 190 | return global_epoch 191 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/model/utils.py: -------------------------------------------------------------------------------- 1 | """General utility functions""" 2 | 3 | import json 4 | import logging 5 | 6 | 7 | class Params(): 8 | """Class that loads hyperparameters from a json file. 9 | 10 | Example: 11 | ``` 12 | params = Params(json_path) 13 | print(params.learning_rate) 14 | params.learning_rate = 0.5 # change the value of learning_rate in params 15 | ``` 16 | """ 17 | 18 | def __init__(self, json_path): 19 | self.update(json_path) 20 | 21 | def save(self, json_path): 22 | """Saves parameters to json file""" 23 | with open(json_path, 'w') as f: 24 | json.dump(self.__dict__, f, indent=4) 25 | 26 | def update(self, json_path): 27 | """Loads parameters from json file""" 28 | with open(json_path) as f: 29 | params = json.load(f) 30 | self.__dict__.update(params) 31 | 32 | @property 33 | def dict(self): 34 | """Gives dict-like access to Params instance by `params.dict['learning_rate']`""" 35 | return self.__dict__ 36 | 37 | 38 | def set_logger(log_path): 39 | """Sets the logger to log info in terminal and file `log_path`. 40 | 41 | In general, it is useful to have a logger so that every output to the terminal is saved 42 | in a permanent file. Here we save it to `model_dir/train.log`. 43 | 44 | Example: 45 | ``` 46 | logging.info("Starting training...") 47 | ``` 48 | 49 | Args: 50 | log_path: (string) where to log 51 | """ 52 | logger = logging.getLogger() 53 | logger.setLevel(logging.INFO) 54 | 55 | if not logger.handlers: 56 | # Logging to a file 57 | file_handler = logging.FileHandler(log_path) 58 | file_handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s: %(message)s')) 59 | logger.addHandler(file_handler) 60 | 61 | # Logging to console 62 | stream_handler = logging.StreamHandler() 63 | stream_handler.setFormatter(logging.Formatter('%(message)s')) 64 | logger.addHandler(stream_handler) 65 | 66 | 67 | def save_dict_to_json(d, json_path): 68 | """Saves dict of floats in json file 69 | 70 | Args: 71 | d: (dict) of float-castable values (np.float, int, float, etc.) 72 | json_path: (string) path to json file 73 | """ 74 | with open(json_path, 'w') as f: 75 | # We need to convert the values to float for json (it doesn't accept np.array, np.float, ) 76 | d = {k: float(v) for k, v in d.items()} 77 | json.dump(d, f, indent=4) 78 | 79 | def save_predictions_to_file(l, prediction_output_path): 80 | """Saves list of floats in txt file 81 | 82 | Args: 83 | l: (list) of float-castable values (np.float, int, float, etc.) 84 | prediction_output_path: (string) path to txt file 85 | """ 86 | with open(prediction_output_path, 'w') as f: 87 | for v in l: 88 | f.write(str(v) + '\n') 89 | f.flush() 90 | 91 | def load_best_ndcgs(json_path): 92 | """Saves dict of floats in json file 93 | 94 | Args: 95 | d: (dict) of float-castable values (np.float, int, float, etc.) 96 | json_path: (string) path to json file 97 | """ 98 | with open(json_path, 'r') as f: 99 | # We need to convert the values to float for json (it doesn't accept np.array, np.float, ) 100 | data = json.load(f) 101 | return list(data.values()) 102 | 103 | def get_expaned_metrics(metrics_val, top_ks): 104 | expanded_metrics_val = {} 105 | for tag, val in metrics_val.items(): 106 | if tag == 'ndcg': 107 | for i in range(len(val)): 108 | expanded_metrics_val['{}_{}'.format(tag, top_ks[i])] = val[i][0] 109 | else: 110 | expanded_metrics_val[tag] = val 111 | return expanded_metrics_val -------------------------------------------------------------------------------- /pymdp/ranker/urank/msltr2libsvm.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | RAW_RANK_DATA = os.environ.get('RAW_RANK_DATA') 4 | LIGHTGBM_DATA = os.environ.get('LIGHTGBM_DATA') 5 | 6 | def get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type): 7 | OHSUMED_data_folder = os.path.join('OHSUMED', 'Feature-min', 'Fold{}'.format(fold_str)) 8 | # OHSUMED 9 | # print('file_type', file_type) 10 | full_file_name = os.path.join(RAW_RANK_DATA, OHSUMED_data_folder, file_type) 11 | if file_type == 'train': 12 | full_file_name += 'ing' 13 | if file_type == 'vali': 14 | full_file_name += 'dation' 15 | full_file_name += 'set' 16 | data_path = full_file_name + '.txt' 17 | return data_path 18 | 19 | def get_data_path(tfrecords_folder, fold_str, file_type): 20 | data_path = '' 21 | # OHSUMED 22 | if tfrecords_folder == 'OHSUMED': 23 | data_path = get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type) 24 | else: 25 | # MQ2007_data 26 | MS_data_folder = os.path.join(tfrecords_folder, 'Fold{}'.format(fold_str)) 27 | data_path = os.path.join(RAW_RANK_DATA, MS_data_folder, file_type + ".txt") 28 | return data_path 29 | 30 | def convert(write2folder, lightgbm_folder, file_type, \ 31 | fold, out_data_filename, out_query_filename): 32 | group_features = {} 33 | group_labels = {} 34 | fold = str(fold) 35 | data_path = get_data_path(lightgbm_folder, fold, file_type) 36 | print('data_path', data_path) 37 | 38 | raw_rank_data_input = open(data_path,"r") 39 | output_feature = open(os.path.join(write2folder, out_data_filename), "w") 40 | output_query = open(os.path.join(write2folder, out_query_filename), "w") 41 | cur_cnt = 0 42 | cur_doc_cnt = 0 43 | last_qid = -1 44 | while True: 45 | line = raw_rank_data_input.readline() 46 | if not line: 47 | break 48 | line = line.split(' #')[0] 49 | tokens = line.split(' ') 50 | tokens[-1] = tokens[-1].strip() 51 | label = tokens[0] 52 | qid = int(tokens[1].split(':')[1]) 53 | if qid != last_qid: 54 | if cur_doc_cnt > 0: 55 | output_query.write(str(cur_doc_cnt) + '\n') 56 | cur_cnt += 1 57 | cur_doc_cnt = 0 58 | last_qid = qid 59 | cur_doc_cnt += 1 60 | output_feature.write(label+' ') 61 | output_feature.write(' '.join(tokens[2:]) + '\n') 62 | output_query.write(str(cur_doc_cnt) + '\n') 63 | raw_rank_data_input.close() 64 | output_query.close() 65 | output_feature.close() 66 | 67 | 68 | def main(): 69 | lightgbm_folders = ['OHSUMED', 'MQ2007']# 'OHSUMED', 'MQ2007', 'MSLR-WEB10K', 'MSLR-WEB30K' 70 | folds = 5 71 | for lightgbm_folder in lightgbm_folders: 72 | for fold in range(1, folds + 1): 73 | write2folder = os.path.join(LIGHTGBM_DATA, lightgbm_folder, str(fold)) 74 | print(write2folder) 75 | if not os.path.exists(write2folder): 76 | os.makedirs(write2folder) 77 | convert(write2folder, lightgbm_folder, "train", fold, "rank.train", "rank.train.query") 78 | convert(write2folder, lightgbm_folder, "test", fold, "rank.test", "rank.test.query") 79 | convert(write2folder, lightgbm_folder, "vali", fold, "rank.vali", "rank.vali.query") 80 | 81 | if __name__ == '__main__': 82 | main() 83 | print('Done') -------------------------------------------------------------------------------- /pymdp/ranker/urank/prepare_data.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Generate serialized TF records 3 | usage: python prepare_data.py 4 | ''' 5 | 6 | import os 7 | import argparse 8 | import json 9 | import numpy as np 10 | import tensorflow as tf 11 | import argparse 12 | import logging 13 | from model.utils import save_dict_to_json 14 | 15 | 16 | # change RAW_RANK_DATA and TF_RANK_DATA accordingly 17 | # for example the full path for '../../learning_to_rank_data_sets_OHSUMED' 18 | RAW_RANK_DATA = os.environ.get('RAW_RANK_DATA') 19 | TF_RANK_DATA = os.environ.get('TF_RANK_DATA') 20 | 21 | def get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type): 22 | OHSUMED_data_folder = os.path.join('OHSUMED', 'Feature-min', 'Fold{}'.format(fold_str)) 23 | # OHSUMED 24 | # print('file_type', file_type) 25 | full_file_name = os.path.join(RAW_RANK_DATA, OHSUMED_data_folder, file_type) 26 | if file_type == 'train': 27 | full_file_name += 'ing' 28 | if file_type == 'vali': 29 | full_file_name += 'dation' 30 | full_file_name += 'set' 31 | data_path = full_file_name + '.txt' 32 | return data_path 33 | 34 | def get_data_path(tfrecords_folder, fold_str, file_type): 35 | data_path = '' 36 | # OHSUMED 37 | if tfrecords_folder == 'OHSUMED': 38 | data_path = get_OHSUMED_data_path(tfrecords_folder, fold_str, file_type) 39 | else: 40 | # MQ2007_data 41 | MS_data_folder = os.path.join(tfrecords_folder, 'Fold{}'.format(fold_str)) 42 | data_path = os.path.join(RAW_RANK_DATA, MS_data_folder, file_type + ".txt") 43 | return data_path 44 | 45 | def normalize_mean_max_feature_array(array): 46 | mean = array.mean(axis = 0) 47 | abs_max = abs(array.max(axis = 0)) 48 | epilson = 1e-8 49 | abs_max = abs_max + epilson 50 | normalized_array = (array - mean) / abs_max 51 | return normalized_array 52 | # this one is better than normalize_mean_max_feature_array 53 | def normalize_min_max_feature_array(array): 54 | mini = array.min(axis = 0) 55 | maxi = array.max(axis = 0) 56 | epilson = 1e-8 57 | value_range = maxi - mini + epilson 58 | normalized_array = (array - mini) / value_range 59 | return normalized_array 60 | 61 | def _bytes_feature(value): 62 | value = value if type(value) == list else [value] 63 | return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) 64 | 65 | def _int64_feature(value): 66 | value = value if type(value) == list else [value] 67 | return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) 68 | 69 | def _float_feature(value): 70 | value = value if type(value) == list else [value] 71 | return tf.train.Feature(float_list=tf.train.FloatList(value=value)) 72 | 73 | def convert(tfrecords_folder, file_type, fold): 74 | group_features = {} 75 | group_labels = {} 76 | fold = str(fold) 77 | data_path = get_data_path(tfrecords_folder, fold, file_type) 78 | print('data_path', data_path) 79 | 80 | # if file_type == 'vali': 81 | # file_type = 'eval' 82 | tfrecords_filename = tfrecords_folder + '.tfrecords' 83 | complete_file_name = os.path.join(TF_RANK_DATA, tfrecords_folder, fold, \ 84 | file_type + "_" + tfrecords_filename) 85 | writer = tf.python_io.TFRecordWriter(complete_file_name) 86 | max_height = 0 87 | with open(data_path, "r") as f: 88 | for line in f: 89 | if not line: 90 | break 91 | if "#" in line: 92 | line = line[:line.index("#")] 93 | splits = line.strip().split(" ") 94 | label = float(splits[0]) 95 | group = int(splits[1].split(":")[1]) 96 | features = [float(split.split(":")[1]) for split in splits[2:]] 97 | 98 | if group in group_features: 99 | new_feature_list = group_features[group] 100 | new_feature_list.append(features) 101 | group_features[group] = new_feature_list 102 | 103 | new_label_list = group_labels[group] 104 | new_label_list.append(label) 105 | group_labels[group] = new_label_list 106 | else: 107 | feature_list = [] 108 | feature_list.append(features) 109 | group_features[group] = feature_list 110 | 111 | label_list = [] 112 | label_list.append(label) 113 | group_labels[group] = label_list 114 | 115 | query_ids = list(group_features.keys()) 116 | query_ids.sort() 117 | # print('fold', fold, ', len', len(query_ids), ', file_type', file_type, ', query_ids', query_ids) 118 | num_queries = 0 119 | feature_dim = 0 120 | doc_count = 0 121 | 122 | for group in group_features: 123 | label_list = group_labels[group] 124 | label_array = np.asarray(label_list, dtype=np.float32) 125 | # remove line 136-138 to keep the original data 126 | # # remove all 0 label entries 127 | 128 | if label_array.sum() < 1: 129 | # print('All 0 label entries: ', str(group), str(label_array.sum())) 130 | continue 131 | # printing out queries that only had 0 labels 132 | # if label_array.sum() < 1: 133 | # print('All 0 label entries: ', str(group), str(label_array.sum())) 134 | # if label_array.sum() == np.amax(label_array) * label_array.size: 135 | # print('All same label entries: {}, max/min rating: {}, number of docs: {}'.format(group, \ 136 | # np.amax(label_array), label_array.size)) 137 | # continue 138 | # if file_type == 'test' and label_array.sum() == np.amax(label_array) * label_array.size: 139 | # print('All same label entries in test: {}, max/min rating: {}, number of docs: {}'.format(group, \ 140 | # np.amax(label_array), label_array.size)) 141 | if file_type != 'test' and label_array.sum() == np.amax(label_array) * label_array.size: 142 | # keep the test data unchanged 143 | # but save some steps in training and validation/eval 144 | # print('All same label entries: {}, max/min rating: {}, number of docs: {}'.format(group, \ 145 | # np.amax(label_array), label_array.size)) 146 | continue 147 | feature_array = np.asarray(group_features[group], dtype=np.float32) 148 | normalized_feature_array = normalize_min_max_feature_array(feature_array) 149 | feature_raw = normalized_feature_array.tostring() 150 | # the number of documents of a query 151 | height = normalized_feature_array.shape[0] 152 | if height > max_height: 153 | max_height = height 154 | # feature dim (same for all queries) 155 | width = normalized_feature_array.shape[1] 156 | label_list = group_labels[group] 157 | unique_rating = len(set(label_list)) 158 | label_gain_list = [2**v-1 for v in label_list] 159 | doc_count += height 160 | num_queries += 1 161 | example = tf.train.Example(features=tf.train.Features(feature={ 162 | 'height': _int64_feature(height), 163 | 'width': _int64_feature(width), 164 | 'feature_raw': _bytes_feature(feature_raw), 165 | 'label_gain': _float_feature(label_gain_list), 166 | 'unique_rating': _int64_feature(unique_rating), 167 | 'label': _float_feature(label_list)})) 168 | writer.write(example.SerializeToString()) 169 | 170 | writer.close() 171 | print('max_height in {} : {}'.format(tfrecords_folder, max_height)) 172 | # query_ids = list(group_features.keys()) 173 | feature_list_0 = group_features[query_ids[0]] 174 | feature_dim = len(feature_list_0[0]) 175 | # return len(query_ids), feature_dim, doc_count 176 | return num_queries, feature_dim, doc_count 177 | 178 | def main(): 179 | tfrecords_folders = ['RAL-I']#, 'MSLR-WEB10K', 'MSLR-WEB30K']# 'OHSUMED', 'MQ2007', 'MSLR-WEB10K', 'MSLR-WEB30K' 180 | folds = 1 181 | for tfrecords_folder in tfrecords_folders: 182 | for fold in range(1, folds + 1): 183 | write2folder = os.path.join(TF_RANK_DATA, tfrecords_folder, str(fold)) 184 | if not os.path.exists(write2folder): 185 | os.makedirs(write2folder) 186 | # use eval in the write part of tfrecords for now 187 | eval_size, eval_feature_dim, eval_doc_count = convert(tfrecords_folder, 'vali', fold) 188 | test_size, test_feature_dim, test_doc_count = convert(tfrecords_folder, 'test', fold) 189 | train_size, train_feature_dim, train_doc_count = convert(tfrecords_folder, 'train', fold) 190 | # Save datasets properties in json file 191 | sizes = { 192 | 'feature_dim': train_feature_dim, 193 | 'train_size': train_size, 194 | 'train_doc_count': train_doc_count, 195 | 'eval_size': eval_size, 196 | 'eval_doc_count': eval_doc_count, 197 | 'test_size': test_size, 198 | 'test_doc_count': test_doc_count 199 | } 200 | save_dict_to_json(sizes, os.path.join(write2folder, 'dataset_params.json')) 201 | 202 | if __name__ == "__main__": 203 | main() 204 | print("Done!") 205 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/process_ndcg_results.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import pandas as pd 3 | 4 | def getPaths(): 5 | txt_files = glob.glob('*.txt') 6 | return txt_files 7 | # for file_path in txt_files: 8 | # print(file_path) 9 | 10 | # e.g., file_path = 'MQ2007_glrank.txt' 11 | def get_data_model(file_path): 12 | fields = file_path.split('.txt')[0] 13 | dataset_model = fields.split('_') 14 | return dataset_model[0], dataset_model[1] 15 | 16 | 17 | def get_ndcgs(file_path): 18 | with open(file_path, 'r') as f: 19 | ndcg_1s = [] 20 | ndcg_3s = [] 21 | ndcg_5s = [] 22 | ndcg_10s = [] 23 | 24 | lines = f.readlines() 25 | for line in lines: 26 | if line.startswith('- Eval metrics:'): 27 | # print(line) 28 | fields = line.split(' ') 29 | one_fold = [] 30 | ndcg_1 = float(fields[4]) 31 | ndcg_3 = float(fields[7]) 32 | ndcg_5 = float(fields[10]) 33 | ndcg_10 = float(fields[13]) 34 | 35 | ndcg_1s.append(ndcg_1) 36 | ndcg_3s.append(ndcg_3) 37 | ndcg_5s.append(ndcg_5) 38 | ndcg_10s.append(ndcg_10) 39 | if (len(ndcg_1s) != 5 or len(ndcg_3s) != 5 or len(ndcg_5s) != 5 or len(ndcg_10s) != 5): 40 | print('ERROR! in ', file_path) 41 | a_ndcg_1 = sum(ndcg_1s) / float(len(ndcg_1s)) 42 | a_ndcg_3 = sum(ndcg_3s) / float(len(ndcg_3s)) 43 | a_ndcg_5 = sum(ndcg_5s) / float(len(ndcg_5s)) 44 | a_ndcg_10 = sum(ndcg_10s) / float(len(ndcg_10s)) 45 | # print('a_ndcg_1 : ', a_ndcg_1) 46 | # print('a_ndcg_3 : ', a_ndcg_3) 47 | # print('a_ndcg_5 : ', a_ndcg_5) 48 | # print('a_ndcg_10 : ', a_ndcg_10) 49 | return [a_ndcg_1, a_ndcg_3, a_ndcg_5, a_ndcg_10] 50 | 51 | txt_files = getPaths() 52 | txt_files.sort() 53 | for file_path in txt_files: 54 | dataset, model = get_data_model(file_path) 55 | ncdgs = get_ndcgs(file_path) 56 | # df = pd.DataFrame({"PassengerId": testset['PassengerId'],"Survived": result}) 57 | # df.to_csv('reult.csv',header=True, index=False) 58 | print(dataset, model, ncdgs) -------------------------------------------------------------------------------- /pymdp/ranker/urank/process_results.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import pandas as pd 3 | 4 | def getPaths(): 5 | txt_files = glob.glob('*.txt') 6 | return txt_files 7 | # for file_path in txt_files: 8 | # print(file_path) 9 | 10 | # e.g., file_path = 'MQ2007_glrank.txt' 11 | def get_data_model(file_path): 12 | fields = file_path.split('.txt')[0] 13 | dataset_model = fields.split('_') 14 | return dataset_model[0], dataset_model[1] 15 | 16 | 17 | def get_ndcgs_errs(file_path): 18 | # print(file_path) 19 | with open(file_path, 'r') as f: 20 | ndcg_1s = [] 21 | ndcg_3s = [] 22 | ndcg_5s = [] 23 | ndcg_10s = [] 24 | 25 | err_1s = [] 26 | err_3s = [] 27 | err_5s = [] 28 | err_10s = [] 29 | 30 | lines = f.readlines() 31 | for line in lines: 32 | if line.startswith('- Eval metrics:'): 33 | # print(line) 34 | fields = line.split(' ') 35 | one_fold = [] 36 | ndcg_1 = float(fields[4]) 37 | ndcg_3 = float(fields[7]) 38 | ndcg_5 = float(fields[10]) 39 | ndcg_10 = float(fields[13]) 40 | 41 | err_1 = float(fields[16]) 42 | err_3 = float(fields[19]) 43 | err_5 = float(fields[22]) 44 | err_10 = float(fields[25]) 45 | 46 | ndcg_1s.append(ndcg_1) 47 | ndcg_3s.append(ndcg_3) 48 | ndcg_5s.append(ndcg_5) 49 | ndcg_10s.append(ndcg_10) 50 | 51 | err_1s.append(err_1) 52 | err_3s.append(err_3) 53 | err_5s.append(err_5) 54 | err_10s.append(err_10) 55 | 56 | if (len(ndcg_1s) != 5 or len(ndcg_3s) != 5 or len(ndcg_5s) != 5 or len(ndcg_10s) != 5 \ 57 | or len(err_1s) != 5 or len(err_3s) != 5 or len(err_5s) != 5 or len(err_10s) != 5): 58 | print('ERROR! in ', file_path) 59 | 60 | a_ndcg_1 = float("{0:.3f}".format(sum(ndcg_1s) / float(len(ndcg_1s)))) 61 | a_ndcg_3 = float("{0:.3f}".format(sum(ndcg_3s) / float(len(ndcg_3s)))) 62 | a_ndcg_5 = float("{0:.3f}".format(sum(ndcg_5s) / float(len(ndcg_5s)))) 63 | a_ndcg_10 = float("{0:.3f}".format(sum(ndcg_10s) / float(len(ndcg_10s)))) 64 | # print('a_ndcg_1 : ', a_ndcg_1) 65 | # print('a_ndcg_3 : ', a_ndcg_3) 66 | # print('a_ndcg_5 : ', a_ndcg_5) 67 | # print('a_ndcg_10 : ', a_ndcg_10) 68 | 69 | a_err_1 = float("{0:.3f}".format(sum(err_1s) / float(len(err_1s)))) 70 | a_err_3 = float("{0:.3f}".format(sum(err_3s) / float(len(err_3s)))) 71 | a_err_5 = float("{0:.3f}".format(sum(err_5s) / float(len(err_5s)))) 72 | a_err_10 = float("{0:.3f}".format(sum(err_10s) / float(len(err_10s)))) 73 | 74 | return [a_ndcg_1, a_err_1, a_ndcg_3, a_err_3, a_ndcg_5, a_err_5, a_ndcg_10, a_err_10] 75 | 76 | txt_files = getPaths() 77 | txt_files.sort() 78 | for file_path in txt_files: 79 | dataset, model = get_data_model(file_path) 80 | results = get_ndcgs_errs(file_path) 81 | print(dataset, model, ' '.join('& * ' + str(v) for v in results)) -------------------------------------------------------------------------------- /pymdp/ranker/urank/run.bat: -------------------------------------------------------------------------------- 1 | cd E:\RAL2020\ 2 | python "generate_dataset - 0529.py" 3 | cd E:\RAL2020\ranker\src 4 | python prepare_data.py 5 | python evaluate.py --loss_fn grank --data_dir ../data/RAL-I/1 --tfrecords_filename RAL-I.tfrecords -------------------------------------------------------------------------------- /pymdp/ranker/urank/run.sh: -------------------------------------------------------------------------------- 1 | #./run_OHSUMED_uRank.sh 2 | # ./run_OHSUMED_gRank.sh 3 | #./run_MQ2007_uRank.sh 4 | # ./run_MQ2007_gRank.sh 5 | # ./run_OHSUMED_ranknet.sh 6 | # ./run_MQ2007_ranknet.sh 7 | 8 | ./run_MSLR-WEB10K_gRank.sh 9 | ./run_MSLR-WEB30K_gRank.sh -------------------------------------------------------------------------------- /pymdp/ranker/urank/util/masks.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def diag_mask(pairwise_label_scores): 5 | masks = tf.ones(tf.shape(pairwise_label_scores)) 6 | n_data = tf.shape(pairwise_label_scores)[0] 7 | # line 8 -9 == line 10 8 | not_consider = tf.diag(tf.ones([n_data])) 9 | masks = tf.subtract(masks, not_consider) 10 | # masks = tf.matrix_band_part(masks, 0, -1) 11 | masks = tf.cast(masks, dtype=tf.float32) 12 | pair_count = tf.reduce_sum(masks) 13 | return masks, pair_count 14 | 15 | def full_mask(pairwise_label_scores): 16 | masks = tf.ones(tf.shape(pairwise_label_scores)) 17 | # line 19 == line 20 18 | not_consider = tf.less_equal(pairwise_label_scores, 0.5) 19 | # not_consider = tf.equal(pairwise_label_scores, 0.5) 20 | not_consider = tf.cast(not_consider, tf.float32) 21 | masks = tf.subtract(masks, not_consider) 22 | masks = tf.cast(masks, dtype=tf.float32) 23 | pair_count = tf.reduce_sum(masks) 24 | return masks, pair_count 25 | 26 | def pruned_mask(pairwise_label_scores): 27 | # 0 1 2 28 | # 0 0, -1, -2 29 | # 1 1, 0, -1 30 | # 2 2, 1, 0 31 | # ONLY KEEP THE POSITIVE DIFFS 32 | # 0 1 2 33 | # 0 0, 0, 0 34 | # 1 1, 0, 0 35 | # 2 2, 1, 0 36 | masks = tf.greater(pairwise_label_scores, 0.0) 37 | masks = tf.cast(masks, dtype=tf.float32) 38 | pair_count = tf.reduce_sum(masks) 39 | return masks, pair_count 40 | 41 | def equal_mask(pairwise_label_scores): 42 | masks = tf.equal(pairwise_label_scores, 0) 43 | masks = tf.cast(masks, dtype=tf.float32) 44 | # take the uppder triangle (leave out diag) 45 | masks = tf.matrix_band_part(masks, 0, -1) 46 | pair_count = tf.reduce_sum(masks) 47 | return masks, pair_count 48 | 49 | def list_mask(raw_pairwise_label_scores): 50 | masks = tf.ones(tf.shape(raw_pairwise_label_scores)) 51 | not_consider = tf.less_equal(raw_pairwise_label_scores, 0.0) 52 | not_consider = tf.cast(not_consider, tf.float32) 53 | masks = tf.subtract(masks, not_consider) 54 | masks = tf.cast(masks, dtype=tf.float32) 55 | return masks 56 | 57 | def list_negative_mask(raw_pairwise_label_scores): 58 | masks = tf.ones(tf.shape(raw_pairwise_label_scores)) 59 | not_consider = tf.greater_equal(raw_pairwise_label_scores, 0.0) 60 | not_consider = tf.cast(not_consider, tf.float32) 61 | masks = tf.subtract(masks, not_consider) 62 | masks = -tf.cast(masks, dtype=tf.float32) 63 | return masks 64 | -------------------------------------------------------------------------------- /pymdp/ranker/urank/util/scores.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | def get_pairwise_scores(predicted_scores): 4 | pairwise_predicted_scores = predicted_scores - tf.transpose(predicted_scores) 5 | return pairwise_predicted_scores 6 | 7 | 8 | def get_pairwise_label_scores(labels): 9 | pairwise_label_scores = labels - tf.transpose(labels) 10 | differences_ij = tf.maximum(tf.minimum(1.0, pairwise_label_scores), -1.0) 11 | pairwise_label_scores = (1.0 / 2.0) * (1.0 + differences_ij) 12 | return pairwise_label_scores 13 | 14 | 15 | def get_softmax_pairwise_scores(predicted_scores): 16 | exp_predicted_scores = 2 ** predicted_scores 17 | exp_predicted_scores = tf.divide(exp_predicted_scores, tf.reduce_sum(exp_predicted_scores)) 18 | pairwise_predicted_scores = exp_predicted_scores - tf.transpose(exp_predicted_scores) 19 | return pairwise_predicted_scores 20 | -------------------------------------------------------------------------------- /pymdp/ranker/xgboost_ranker.py: -------------------------------------------------------------------------------- 1 | from xgboost import XGBClassifier 2 | from sklearn.model_selection import train_test_split 3 | from sklearn.metrics import accuracy_score 4 | import numpy as np 5 | 6 | class XGBoost_Ranker(): 7 | def __init__(self, timestamp, load=True): 8 | self.model = XGBClassifier() 9 | self.model.load_model(timestamp+'.file') 10 | self.factor = 1.0 11 | 12 | def set_factor(self, factor): 13 | self.factor = factor 14 | 15 | def rank_features(self, features): 16 | _features = np.copy(features) 17 | for f in _features: 18 | f[1] *= self.factor 19 | f[4] *= self.factor 20 | f[5] *= self.factor 21 | # return np.array([0, 1, 2, 3, 4]) 22 | 23 | test_x = [] 24 | for i in range(len(_features)): 25 | for j in range(len(_features)): 26 | if i == j: 27 | continue 28 | test_x.append(np.concatenate( 29 | (_features[i], _features[j]), axis=0)) 30 | 31 | test_x = np.array(test_x) 32 | print(test_x.shape) 33 | y = self.model.predict(test_x).reshape(len(_features), len(_features)-1) 34 | y = np.sum(y, axis=1) 35 | # print(y) 36 | return np.argsort(y)[::-1] -------------------------------------------------------------------------------- /pymdp/trajectory.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | import os 4 | from xml.etree.ElementTree import ElementTree 5 | from xml.etree.ElementTree import Element, SubElement 6 | 7 | def export_config_xml(file, planes): 8 | root_node = Element('root') 9 | tree = ElementTree(root_node) 10 | # n_planes = len(planes) - 1 11 | 12 | for i in range(len(planes)): 13 | part_dict = { "File":"result-"+str(i)+".off", "Space":"1"} 14 | part_node = SubElement(root_node, 'Part', part_dict) 15 | plane_node = SubElement(part_node, 'Planes') 16 | plane = planes[i] 17 | plane_node.text = '%f %f %f %f\n' % (plane[0], plane[1], plane[2], plane[3]) 18 | 19 | tree.write(file) 20 | 21 | class Trajectory: 22 | def __init__(self): 23 | self.nodes = [] 24 | self.value = 0.0 25 | self.active = True 26 | 27 | def is_active(self): 28 | return self.active 29 | 30 | def update_node(self, node, value): 31 | self.nodes.append(node) 32 | self.value = value 33 | 34 | def display(self): 35 | for node in self.nodes: 36 | print(str(node), end = ' ') 37 | print('Score = ', self.value) 38 | 39 | 40 | class TrajStation: 41 | def __init__(self): 42 | self.trajs = [[Trajectory()]] 43 | self.features = [] 44 | self.feat_valid = [] 45 | self.level = 0 46 | 47 | def get_trajs_by_level(self, level): 48 | if len(self.trajs) == level: 49 | self.trajs.append([]) 50 | return self.trajs[level] 51 | 52 | def get_trajs_current(self): 53 | return self.trajs[self.level] 54 | 55 | def get_trajs_previous(self): 56 | return self.trajs[self.level -1] 57 | 58 | def get_feats_previous(self): 59 | print("level = ", self.level) 60 | if self.level == 1: 61 | return None 62 | else: 63 | return self.features[-1] 64 | 65 | def add_node(self, _from, _to, _val): 66 | trajs = self.get_trajs_previous() 67 | new_traj = copy.deepcopy(trajs[_from]) 68 | new_traj.update_node(_to, _val) 69 | self.trajs[-1].append(new_traj) 70 | 71 | def move_to_next_level(self): 72 | self.level += 1 73 | if len(self.trajs) <= self.level: 74 | self.trajs.append([]) 75 | 76 | def move_to_previous_level(self): 77 | self.level -=1 78 | 79 | def add_feature(self, feat, feat_valid): 80 | self.features.append(feat) 81 | self.feat_valid.append(feat_valid) 82 | 83 | def display(self): 84 | trajs = self.get_trajs_current() 85 | for i in range(len(trajs)): 86 | print('Traj ', i, ' : ') 87 | trajs[i].display() 88 | 89 | def prepare_data_edge(self, folder): 90 | all_trajs = [x for sublist in self.trajs for x in sublist] 91 | all_trajs.sort(key=lambda x : x.value, reverse=True) 92 | adjacent_matrices = [[] for i in range(len(self.features))] 93 | adjacent_dicts = [{} for i in range(len(self.features))] 94 | 95 | for traj in all_trajs: 96 | for i in range(len(traj.nodes)): 97 | prev_id = traj.nodes[i-1] if i > 0 else -1 98 | cur_id = traj.nodes[i] 99 | is_in = (prev_id, cur_id) in adjacent_dicts[i] 100 | if is_in == False: 101 | prev_feat = self.features[i-1][traj.nodes[i-1]][0:6] if i > 0 else np.array([0, 0, 0, 0, 0, 0]) 102 | cur_feat = self.features[i][traj.nodes[i]][0:6] 103 | cat_feat = np.concatenate((prev_feat, cur_feat), axis=0) 104 | adjacent_matrices[i].append(cat_feat) 105 | adjacent_dicts[i][(prev_id, cur_id)] = 1 106 | 107 | for i in range(len(self.features)): 108 | np.save(folder +'/adjm-'+str(i)+'.npy', adjacent_matrices[i]) 109 | 110 | def prepare_data(self, folder): 111 | # first, sort all trajectories 112 | all_trajs = [x for sublist in self.trajs for x in sublist] 113 | #all_trajs = np.concatenate(self.trajs) #self.data_all.sort(key=lambda x: float(x[5]), reverse=False) 114 | all_trajs.sort(key=lambda x : x.value, reverse=False) 115 | adjacent_matrices = [] 116 | print('length of features = ', len(self.features)) 117 | for i in range(len(self.features)): 118 | adjacent_matrices.append(np.zeros((self.feat_valid[i], self.feat_valid[i]), dtype=bool)) 119 | 120 | for traj in all_trajs: 121 | # print("score = ", str(traj.value)) 122 | for i in range(len(traj.nodes)): 123 | k = traj.nodes[i] 124 | m = adjacent_matrices[i] 125 | m[k, :] = True 126 | m[:, k] = False 127 | # print(i, k) 128 | 129 | if len(adjacent_matrices) == 0: 130 | return 131 | 132 | if os.path.exists(folder) is False: 133 | os.makedirs(folder) 134 | 135 | for i in range(len(adjacent_matrices)): 136 | rows, cols = np.where(adjacent_matrices[i] == True) 137 | ind = np.array([rows, cols]) 138 | np.save(folder +'/adj-'+str(i)+'.py', ind) 139 | 140 | for i in range(len(self.features)): 141 | np.save(folder+'/feat-' + str(i) + '.npy', self.features[i]) 142 | 143 | def export_best_segmentation(self, folder, export_polys): 144 | all_trajs = [x for sublist in self.trajs for x in sublist] 145 | best_traj = max(all_trajs, key=lambda x : x.value) 146 | print('val: ', best_traj.value) 147 | planes = [] 148 | 149 | if os.path.exists(folder) is False: 150 | os.makedirs(folder) 151 | 152 | for i in range(len(best_traj.nodes)): 153 | k = best_traj.nodes[i] 154 | # print("node: ", k) 155 | # print(self.features[i][k][0:10]) 156 | planes.append(self.features[i][k][6:10]) 157 | if i == len(best_traj.nodes) - 1: 158 | planes.append(np.array([0, 1, 0, 0])) 159 | with open(os.path.join(folder, 'result-'+str(i+1)+'.off'), 'w') as f: 160 | f.write(export_polys[i][k][0]) 161 | 162 | with open(os.path.join(folder, 'result-'+str(i)+'.off'), 'w') as f: 163 | f.write(export_polys[i][k][1]) 164 | 165 | export_config_xml(os.path.join(folder, "result.xml"), planes) -------------------------------------------------------------------------------- /pymdp/utility.py: -------------------------------------------------------------------------------- 1 | import RoboFDM 2 | from multiprocessing import Process, Manager,TimeoutError 3 | 4 | def apply_cut(poly, plane, return_dict): 5 | try: 6 | ra = RoboFDM.init() 7 | ra.reset("bunny.off") 8 | ra.set_poly(poly) 9 | #print('--> Actual plane: ', plane) 10 | #print('--> Actual evaluation: ', ra.step(plane)) 11 | ra.plane_cut(plane) 12 | poly=ra.get_poly() 13 | return_dict[0] = poly 14 | except Exception: 15 | pass 16 | 17 | def apply_cut_both(poly, plane, return_dict): 18 | try: 19 | ra = RoboFDM.init() 20 | ra.reset("bunny.off") 21 | ra.set_poly(poly) 22 | #print('--> Actual plane: ', plane) 23 | #print('--> Actual evaluation: ', ra.step(plane)) 24 | ra.plane_cut_both(plane) 25 | poly=ra.get_poly() 26 | poly_pos = ra.get_positive_poly() 27 | return_dict[0] = poly 28 | return_dict[1] = poly_pos 29 | except Exception: 30 | pass 31 | 32 | def run_cut_process(poly, plane, export=False): 33 | manager = Manager() 34 | return_dict = manager.dict() 35 | if export == False: 36 | t = Process(target=apply_cut,args=(poly, plane, return_dict)) 37 | else: 38 | t = Process(target=apply_cut_both, args=(poly, plane, return_dict)) 39 | 40 | t.start() 41 | t.join(timeout=5.0) 42 | ret = return_dict.values() 43 | t.terminate() 44 | if len(ret) == 0: 45 | return None 46 | else: 47 | if export: 48 | return (ret[0], ret[1]) 49 | else: 50 | return ret[0] 51 | 52 | def write_mesh(mesh_str, filename): 53 | with open(filename, "w") as f: 54 | f.write(mesh_str) 55 | 56 | def sample_poly(poly, outfile): 57 | ra = RoboFDM.init() 58 | result = ra.sample_mesh(poly, outfile) 59 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.17.4 2 | xgboost>=1.0.2 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import platform 5 | import subprocess 6 | 7 | from setuptools import setup, Extension, find_packages 8 | from setuptools.command.build_ext import build_ext 9 | from setuptools.command.install import install 10 | from distutils.version import LooseVersion 11 | 12 | VCPKG = None 13 | 14 | try: 15 | # for pip >= 10 16 | from pip._internal.req import parse_requirements 17 | except ImportError: 18 | # for pip <= 9.0.3 19 | from pip.req import parse_requirements 20 | 21 | 22 | 23 | class InstallCommand(install): 24 | user_options = install.user_options + [ 25 | ('vcpkg=', None, ""), # a 'flag' option 26 | #('someval=', None, None) # an option that takes a value 27 | ] 28 | 29 | def initialize_options(self): 30 | install.initialize_options(self) 31 | self.vcpkg = None 32 | #self.someval = None 33 | 34 | def finalize_options(self): 35 | install.finalize_options(self) 36 | 37 | def run(self): 38 | global VCPKG 39 | VCPKG = self.vcpkg # will be 1 or None 40 | install.run(self) 41 | 42 | def load_requirements(fname): 43 | reqs = parse_requirements(fname, session="test") 44 | return [str(ir.req) for ir in reqs] 45 | 46 | 47 | class CMakeExtension(Extension): 48 | def __init__(self, name, sourcedir=''): 49 | Extension.__init__(self, name, sources=[]) 50 | self.sourcedir = os.path.abspath(sourcedir) 51 | 52 | 53 | class CMakeBuild(build_ext): 54 | def run(self): 55 | try: 56 | out = subprocess.check_output(['cmake', '--version']) 57 | except OSError: 58 | raise RuntimeError("CMake must be installed to build the following extensions: " + 59 | ", ".join(e.name for e in self.extensions)) 60 | 61 | cmake_version = LooseVersion( 62 | re.search(r'version\s*([\d.]+)', out.decode()).group(1)) 63 | if cmake_version < LooseVersion('3.16.0'): 64 | raise RuntimeError("CMake >= 3.16.0 is required") 65 | 66 | for ext in self.extensions: 67 | self.build_extension(ext) 68 | 69 | def build_extension(self, ext): 70 | global VCPKG 71 | extdir = os.path.abspath(os.path.dirname( 72 | self.get_ext_fullpath(ext.name))) 73 | cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, 74 | '-DPYTHON_EXECUTABLE=' + sys.executable] 75 | 76 | build_type = os.environ.get("BUILD_TYPE", "Release") 77 | build_args = ['--config', build_type] 78 | 79 | # Pile all .so in one place and use $ORIGIN as RPATH 80 | cmake_args += ["-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE"] 81 | cmake_args += ["-DCMAKE_INSTALL_RPATH={}".format("$ORIGIN")] 82 | 83 | if VCPKG != None: 84 | vcpkg_cmake = os.path.join( 85 | str(VCPKG), "scripts", "buildsystems", "vcpkg.cmake") 86 | cmake_args += ["-DCMAKE_TOOLCHAIN_FILE="+vcpkg_cmake] 87 | 88 | if platform.system() == "Windows": 89 | cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format( 90 | build_type.upper(), extdir)] 91 | if sys.maxsize > 2**32: 92 | cmake_args += ['-A', 'x64'] 93 | build_args += ['--', '/m'] 94 | else: 95 | cmake_args += ['-DCMAKE_BUILD_TYPE=' + build_type] 96 | build_args += ['--', '-j4'] 97 | 98 | env = os.environ.copy() 99 | env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), 100 | self.distribution.get_version()) 101 | if not os.path.exists(self.build_temp): 102 | os.makedirs(self.build_temp) 103 | subprocess.check_call(['cmake', ext.sourcedir] + 104 | cmake_args, cwd=self.build_temp, env=env) 105 | subprocess.check_call(['cmake', 106 | '--build', '.', 107 | '--target', ext.name 108 | ] + build_args, 109 | cwd=self.build_temp) 110 | 111 | 112 | setup( 113 | name='pymdp', 114 | version=0.1, 115 | author='Chenming Wu', 116 | author_email='wcm1994@gmail.com', 117 | description='A python package for multi-directional printing decomposition', 118 | long_description=open("README.rst").read(), 119 | ext_modules=[CMakeExtension('RoboFDM')], 120 | packages=find_packages(), 121 | cmdclass=dict(install=InstallCommand, build_ext=CMakeBuild), 122 | url="https://github.com/chenming-wu/pymdp", 123 | zip_safe=False, 124 | install_requires=load_requirements("requirements.txt"), 125 | ) 126 | -------------------------------------------------------------------------------- /src/CustomisedPolyhedron.h: -------------------------------------------------------------------------------- 1 | #ifndef CUSTOMISED_POLYHEDRON_H 2 | #define CUSTOMISED_POLYHEDRON_H 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | //#define USEDEBUG 21 | #ifdef USEDEBUG 22 | #define Debug(x) std::cout << x 23 | #else 24 | #define Debug(x) 25 | #endif 26 | 27 | #define NDebug(x) std::cerr << x 28 | 29 | // Microsoft Visual Studio 2019 doesn't have M_PI anymore 30 | #ifndef M_PI 31 | #define M_PI 3.14159265358979323846 32 | #endif 33 | 34 | template 35 | class Polyhedron_demo_vertex : 36 | public CGAL::HalfedgeDS_vertex_base 37 | { 38 | public: 39 | typedef std::set Set_of_indices; 40 | Vector_ vertexNormal; 41 | private: 42 | typedef CGAL::HalfedgeDS_vertex_base Pdv_base; 43 | 44 | Set_of_indices indices; 45 | std::size_t mID; 46 | std::size_t time_stamp_; 47 | 48 | public: 49 | int nb_of_feature_edges; 50 | 51 | bool is_corner() const 52 | { 53 | return nb_of_feature_edges > 2; 54 | } 55 | 56 | bool is_feature_vertex() const 57 | { 58 | return nb_of_feature_edges != 0; 59 | } 60 | 61 | void add_incident_patch(const Patch_id i) 62 | { 63 | indices.insert(i); 64 | } 65 | 66 | /// For the determinism of Compact_container iterators 67 | ///@{ 68 | typedef CGAL::Tag_true Has_timestamp; 69 | 70 | std::size_t time_stamp() const 71 | { 72 | return time_stamp_; 73 | } 74 | 75 | void set_time_stamp(const std::size_t& ts) 76 | { 77 | time_stamp_ = ts; 78 | } 79 | 80 | ///}@ 81 | 82 | const Set_of_indices& 83 | incident_patches_ids_set() const 84 | { 85 | return indices; 86 | } 87 | 88 | std::size_t& id() { return mID; } 89 | std::size_t id() const { return mID; } 90 | 91 | Polyhedron_demo_vertex() : Pdv_base(), mID(-1), nb_of_feature_edges(0) 92 | { 93 | } 94 | 95 | Polyhedron_demo_vertex(const Point& p) : Pdv_base(p), mID(-1), nb_of_feature_edges(0) 96 | { 97 | } 98 | }; 99 | 100 | 101 | template 102 | class Polyhedron_demo_halfedge : 103 | public CGAL::HalfedgeDS_halfedge_base 104 | { 105 | private: 106 | bool feature_edge; 107 | std::size_t time_stamp_; 108 | std::size_t mask_; 109 | 110 | public: 111 | 112 | Polyhedron_demo_halfedge() 113 | : feature_edge(false), mask_(0) 114 | { 115 | }; 116 | 117 | bool is_feature_edge() const 118 | { 119 | return feature_edge; 120 | } 121 | 122 | void set_feature_edge(const bool b) 123 | { 124 | feature_edge = b; 125 | this->opposite()->feature_edge = b; 126 | } 127 | 128 | std::size_t& mask() { return mask_; } 129 | std::size_t mask() const { return mask_; } 130 | 131 | void set_mask(std::size_t m) { mask_ = m; } 132 | 133 | /// For the determinism of Compact_container iterators 134 | ///@{ 135 | typedef CGAL::Tag_true Has_timestamp; 136 | 137 | std::size_t time_stamp() const 138 | { 139 | return time_stamp_; 140 | } 141 | 142 | void set_time_stamp(const std::size_t& ts) 143 | { 144 | time_stamp_ = ts; 145 | } 146 | 147 | ///@} 148 | }; 149 | 150 | // Defined facet base 151 | // auxID is used in customised_mesh_slicer, to identify the id of cross-section's faces 152 | template 153 | class Polyhedron_demo_face : 154 | public CGAL::HalfedgeDS_face_base 155 | { 156 | public: 157 | Patch_id_ patch_id_; 158 | 159 | int auxID; 160 | std::size_t time_stamp_; 161 | bool isVisited; 162 | Vector_ facetNormal; 163 | Vector_ facetCentroid; 164 | 165 | bool isSupported; 166 | double area_; 167 | //std::vector vecIDs; 168 | public: 169 | typedef Patch_id_ Patch_id; 170 | 171 | Polyhedron_demo_face() 172 | : patch_id_(1), isVisited(false), isSupported(true), auxID(-1), 173 | area_(0) 174 | { 175 | //vecIDs.reserve(1); 176 | } 177 | 178 | int patch_id() const 179 | { 180 | return patch_id_; 181 | } 182 | 183 | void set_patch_id(const int i) 184 | { 185 | patch_id_ = i; 186 | } 187 | 188 | void set_face_area(const double a) 189 | { 190 | area_ = a; 191 | } 192 | 193 | const double& area() { return area_; } 194 | const double area() const { return area_; } 195 | 196 | bool& visited() { return isVisited; } 197 | bool visited() const { return isVisited; } 198 | 199 | bool& supported() { return isSupported; } 200 | bool supported() const { return isSupported; } 201 | 202 | /// For the determinism of Compact_container iterators 203 | ///@{ 204 | typedef CGAL::Tag_true Has_timestamp; 205 | 206 | std::size_t time_stamp() const 207 | { 208 | return time_stamp_; 209 | } 210 | 211 | void set_time_stamp(const std::size_t& ts) 212 | { 213 | time_stamp_ = ts; 214 | } 215 | 216 | ///@} 217 | }; 218 | 219 | template 220 | class Polyhedron_demo_items : public CGAL::Polyhedron_items_3 221 | { 222 | public: 223 | // wrap vertex 224 | template 225 | struct Vertex_wrapper 226 | { 227 | typedef typename Traits::Point_3 Point; 228 | typedef Polyhedron_demo_vertex Vertex; 233 | }; 234 | 235 | // wrap face 236 | template 237 | struct Face_wrapper 238 | { 239 | typedef Polyhedron_demo_face Face; 244 | }; 245 | 246 | // wrap halfedge 247 | template 248 | struct Halfedge_wrapper 249 | { 250 | typedef Polyhedron_demo_halfedge Halfedge; 254 | }; 255 | }; 256 | 257 | typedef CGAL::Exact_predicates_inexact_constructions_kernel K; 258 | typedef CGAL::Tetrahedron_3 Tetra; 259 | typedef CGAL::Polyhedron_3> Polyhedron; 260 | typedef Polyhedron::Vertex_handle Vertex_handle; 261 | typedef Polyhedron::Facet_handle Facet_handle; 262 | typedef Polyhedron::Halfedge_handle Halfedge_handle; 263 | typedef Polyhedron::Edge_iterator Edge_iterator; 264 | typedef Polyhedron::Facet_iterator Facet_iterator; 265 | typedef Polyhedron::Halfedge_const_iterator Halfedge_iterator; 266 | typedef Polyhedron::Facet::Halfedge_around_facet_const_circulator HF_circulator; 267 | 268 | // constant typedefs 269 | typedef Polyhedron::Edge_const_iterator Edge_const_iterator; 270 | typedef Polyhedron::Halfedge_const_handle Halfedge_const_handle; 271 | typedef Polyhedron::Halfedge_const_iterator Halfedge_const_iterator; 272 | typedef Polyhedron::Vertex_const_handle Vertex_const_handle; 273 | typedef Polyhedron::Vertex_const_iterator Vertex_const_iterator; 274 | typedef Polyhedron::Facet_const_handle Facet_const_handle; 275 | typedef Polyhedron::Facet_const_iterator Facet_const_iterator; 276 | 277 | typedef K::Point_3 Point3; 278 | typedef K::Plane_3 Plane; 279 | typedef K::Vector_3 Vector3; 280 | typedef K::Line_3 Line3; 281 | typedef K::Triangle_3 Triangle3; 282 | typedef K::Segment_3 Segment3; 283 | typedef K::Ray_3 Ray3; 284 | typedef K::Point_2 Point2; 285 | typedef K::Vector_2 Vector2; 286 | typedef CGAL::Polygon_2 Polygon2; 287 | typedef CGAL::AABB_face_graph_triangle_primitive FGTP; 288 | typedef CGAL::AABB_traits AABB_traits_FGTP; 289 | typedef CGAL::AABB_tree AABB_tree_FGTP; 290 | typedef AABB_tree_FGTP::Primitive_id Primitive_id; 291 | typedef boost::optional::Type> Ray_intersection; 292 | typedef CGAL::Polygon_mesh_slicer Slicer; 293 | typedef std::vector> Polylines; 294 | typedef CGAL::Bbox_3 Bbox; 295 | typedef CGAL::Min_sphere_annulus_d_traits_3 Traits; 296 | typedef CGAL::Min_sphere_d MinSphere; 297 | 298 | typedef struct bsp { 299 | bsp() 300 | { 301 | initialize(); 302 | } 303 | 304 | void initialize() 305 | { 306 | first = 0.0; 307 | second = 0.0; 308 | third = 0.0; 309 | fourth = 0.0; 310 | fifth = 0.0; 311 | collided = false; 312 | support_free = true; 313 | done = false; 314 | } 315 | double first; 316 | double second; 317 | double third; 318 | double fourth; 319 | double fifth; 320 | bool collided; 321 | bool support_free; 322 | bool done; 323 | } BSPNode; 324 | 325 | typedef boost::tuple BSP_Result; 326 | 327 | #endif //ICRA17_BJUT_CUSTOMISED_POLYHEDRON_H -------------------------------------------------------------------------------- /src/Exception.h: -------------------------------------------------------------------------------- 1 | /* This file is part of PyMesh. Copyright (c) 2015 by Qingnan Zhou */ 2 | #pragma once 3 | #include 4 | #include 5 | 6 | namespace PyMesh { 7 | 8 | class PyMeshException : public std::exception { 9 | public: 10 | PyMeshException(const std::string& description) : 11 | exception(), m_description(description) {} 12 | virtual ~PyMeshException() throw() {} 13 | 14 | public: 15 | virtual const char* what() const throw() { 16 | return m_description.c_str(); 17 | } 18 | 19 | private: 20 | std::string m_description; 21 | }; 22 | 23 | class IOError : public PyMeshException { 24 | public: 25 | IOError(const std::string& description) : 26 | PyMeshException(description) {} 27 | virtual ~IOError() throw() {} 28 | }; 29 | 30 | class RuntimeError : public PyMeshException { 31 | public: 32 | RuntimeError(const std::string& description) : 33 | PyMeshException(description) {} 34 | virtual ~RuntimeError() throw() {} 35 | }; 36 | 37 | class NotImplementedError : public PyMeshException { 38 | public: 39 | NotImplementedError(const std::string& description) : 40 | PyMeshException(description) {} 41 | virtual ~NotImplementedError() throw() {} 42 | }; 43 | } -------------------------------------------------------------------------------- /src/FillHole.cpp: -------------------------------------------------------------------------------- 1 | #include "FillHole.h" 2 | 3 | #include 4 | 5 | void FillHoleCGAL::fill_hole(Polyhedron& poly, Vector3& nr, const double density) 6 | { 7 | //#define FILL_AND_REFINE 8 | double alpha = 0.8; 9 | bool use_DT = true; 10 | unsigned int nb_holes = 0; 11 | BOOST_FOREACH(Halfedge_handle h, halfedges(poly)) 12 | { 13 | if (h->is_border()) 14 | { 15 | std::vector patch_facets; 16 | #ifdef FILL_AND_REFINE 17 | CGAL::Polygon_mesh_processing::triangulate_refine_and_fair_hole(poly, 18 | h, std::back_inserter(patch_facets), 19 | CGAL::Emptyset_iterator(), 20 | CGAL::Polygon_mesh_processing::parameters:: 21 | density_control_factor(alpha). 22 | use_delaunay_triangulation(use_DT)); 23 | #else 24 | CGAL::Polygon_mesh_processing::triangulate_hole(poly, 25 | h, std::back_inserter(patch_facets), 26 | CGAL::Polygon_mesh_processing::parameters::use_delaunay_triangulation(use_DT)); 27 | #endif 28 | ++nb_holes; 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /src/FillHole.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "CustomisedPolyhedron.h" 4 | 5 | class FillHole 6 | { 7 | public: 8 | FillHole() = default; 9 | ~FillHole() = default; 10 | 11 | virtual void fill_hole(Polyhedron& poly, Vector3& nr, const double density = 0.4) = 0; 12 | }; 13 | 14 | class FillHoleCGAL : public FillHole 15 | { 16 | public: 17 | FillHoleCGAL() = default; 18 | ~FillHoleCGAL() = default; 19 | 20 | void fill_hole(Polyhedron& poly, Vector3& nr, const double density = 0.4); 21 | }; 22 | -------------------------------------------------------------------------------- /src/FillHoleCDT.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenming-wu/pymdp/8275851bc7e7f0c34e7f19a706bd212fb4cd25e7/src/FillHoleCDT.cpp -------------------------------------------------------------------------------- /src/FillHoleCDT.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "FillHole.h" 3 | 4 | class FillHoleCDT : public FillHole { 5 | public: 6 | FillHoleCDT() = default; 7 | ~FillHoleCDT() = default; 8 | 9 | void fill_hole(Polyhedron& poly, Vector3& nr, const double density = 0.4); 10 | }; -------------------------------------------------------------------------------- /src/GeometryTools.h: -------------------------------------------------------------------------------- 1 | #ifndef GEOMETRY_TOOL_HEADER 2 | #define GEOMETRY_TOOL_HEADER 3 | 4 | #include "CustomisedPolyhedron.h" 5 | 6 | class Geotools { 7 | public: 8 | Geotools(); 9 | 10 | // Point conversions 11 | static Point3 point_to_3d(const Point2& p, Plane& pl); 12 | 13 | static Point2 point_to_2d(const Point3& p, Plane& pl); 14 | 15 | // Convex hull 16 | static void construct_CH(const std::vector& pin, std::vector& pout); 17 | 18 | // Position determinations 19 | static bool positive(const Vector3& p, const double c, const Point3& a); 20 | 21 | static bool negative(const Vector3& p, const double c, const Point3& a); 22 | 23 | static bool negative(const Plane& pl, const Point3& a); 24 | 25 | static bool positive(const Plane& pl, const Point3& a); 26 | 27 | static bool positive(const Plane& pl, const Facet_handle fh); 28 | 29 | static bool has_negative_vertex(Facet_iterator& t, const Plane& pl); 30 | 31 | static bool has_positive_vertex(Facet_iterator& t, const Plane& pl); 32 | 33 | static bool has_positive_vertex(Facet_const_iterator& t, const Plane& pl); 34 | 35 | static Plane plane_equation(Facet_iterator& f); 36 | 37 | static Point3 points_centroid(Point3& a, Point3& b, Point3& c, Point3& d); 38 | 39 | static double face_point_vol(const Facet_iterator &f, const Point3 &p); 40 | 41 | static double face_point_vol(const Point3 &p1, const Point3 &p2, const Point3 &p3, const Point3 &p); 42 | 43 | static double point_to_plane_dist(Point3 &p, Plane &pl); 44 | 45 | static Point3 *get_int_point(Point3 &p1, Point3 &p2, Point3 &p3, Point3 &p4); 46 | 47 | static double get_min_y(const Facet_handle &fh); 48 | 49 | static double get_min_y(const Facet_const_handle &fh); 50 | 51 | static double get_min_z(const Facet_handle &fh); 52 | 53 | static double get_max_y(const Facet_handle &fh); 54 | 55 | static Vector3 get_facet_centroid(const Facet_handle &fh); 56 | 57 | static Vector3 get_facet_nomal(const Facet_handle &fh, const Polyhedron &poly); 58 | 59 | static double get_facet_area(const Facet_handle &fh); 60 | 61 | static double triangle_area(Point3 &a, Point3 &b, Point3 &c); 62 | 63 | static Eigen::Matrix3d make_rotation(const Eigen::Vector3d& a, const Eigen::Vector3d& b); 64 | 65 | static Eigen::Quaterniond euler2quaternion(const Eigen::Vector3d& euler); 66 | 67 | static Eigen::Matrix3d quaternion2mat(const Eigen::Quaterniond& q); 68 | 69 | static Eigen::Vector3d mat2euler(const Eigen::Matrix3d& m); 70 | 71 | static Eigen::Quaterniond mat2quaternion(const Eigen::Matrix3d& m); 72 | 73 | static Eigen::Matrix3d euler2mat(const Eigen::Vector3d& euler); 74 | 75 | static Eigen::Vector3d quaternion2euler(const Eigen::Quaterniond& q); 76 | 77 | static Point3 project_to_3d(const Point2& p, const Plane& pl); 78 | 79 | static Point2 project_to_2d(const Point3& p, const Plane& pl); 80 | }; 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /src/MeshCutEval.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | //#include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "CustomisedPolyhedron.h" 10 | #include "GeometryTools.h" 11 | 12 | constexpr double alpha_max_ = 0.70711; 13 | constexpr double FRAGILE_EPS = 0.9; 14 | constexpr double CONNECT_EPS = 1; 15 | 16 | class MeshCutEval { 17 | public: 18 | typedef std::unordered_map MapEdgePoint; 19 | typedef boost::tuple TupleRisky; 20 | 21 | // Functions 22 | MeshCutEval(); 23 | 24 | ~MeshCutEval(); 25 | 26 | static bool initialization(const Polyhedron &poly); 27 | 28 | static BSPNode 29 | apply_plane_cut(const Polyhedron &poly_, const Plane &pl, const Bbox &box, const std::vector &plt); 30 | 31 | // Evaluate and cut 32 | static bool cut_with_plane(const Polyhedron &poly_, const Plane &plane, MapEdgePoint &mapEdgePoint, 33 | std::set &intFaceSet); 34 | 35 | static std::vector 36 | euler_update_facets(const Polyhedron &poly_, const Plane &pl, const std::set &intFaceSet); 37 | 38 | static BSPNode exact_cutting_volume_and_area(const Polyhedron &poly_, const Plane &pl, 39 | const MapEdgePoint &map_edge_point, 40 | std::vector &vec_bsps); 41 | 42 | static BSPNode exact_cutting_volume_and_area(const Polyhedron &poly_, const Plane &pl, 43 | const MapEdgePoint &map_edge_point, 44 | std::vector &vec_bsps, std::vector &inRev); 45 | 46 | 47 | static bool is_supported(Vector3 &f_normal, const Vector3 &dir); 48 | 49 | static std::vector initialize_supports(Polyhedron &poly_); 50 | 51 | static std::vector get_platform_cross(double radius, double platformZ); 52 | 53 | static Plane convert_xyzab_to_plane(double x, double y, double z, double alpha, double beta, Bbox &box_); 54 | 55 | static Plane convert_abg_to_plane(double alpha, double beta, double gamma, const MinSphere& sphere); 56 | 57 | static void evaluation_direction(const Polyhedron& poly, const Vector3& current_dir, const Bbox& box, const std::vector& plt, std::vector& vec_tu); 58 | 59 | static std::pair find_suitable_platform(const Polyhedron& poly); 60 | 61 | static void get_inrev_risky_area(const Polyhedron& poly, const Vector3& current_dir, const Bbox& box, const std::vector& plt, std::vector& inRev); 62 | 63 | private: 64 | static double compute_min_d(const Vector3& dir, const Bbox& box_); 65 | 66 | static double compute_max_d(const Vector3& dir, const Bbox& box_); 67 | }; 68 | -------------------------------------------------------------------------------- /src/MeshSupEval.cpp: -------------------------------------------------------------------------------- 1 | #include "MeshCutEval.h" 2 | 3 | BSPNode MeshCutEval::exact_cutting_volume_and_area(const Polyhedron& poly_, const Plane& pl, 4 | const MapEdgePoint& map_edge_point, 5 | std::vector& vec_bsps, std::vector& inRev) 6 | { 7 | BSPNode VA; 8 | VA.initialize(); 9 | 10 | const Point3& pop = pl.point(); 11 | Vector3 cut(pl.a(), pl.b(), pl.c()); 12 | double C = pl.d(); 13 | cut = cut / CGAL::sqrt(cut.squared_length()); // Compute unit vector 14 | C = C / CGAL::sqrt(cut.squared_length()); 15 | 16 | std::vector cpnts3; 17 | std::vector> cpnts(vec_bsps.size()); 18 | 19 | // Evaluate area 20 | const auto rotMat = Geotools::make_rotation(Eigen::Vector3d(cut.x(), cut.y(), cut.z()), 21 | Eigen::Vector3d(0, 1, 0)); 22 | auto sumArea = 0.0; 23 | int nIntersect = 0; 24 | 25 | for (const auto fit : faces(poly_)) { 26 | Halfedge_const_handle he = fit->halfedge(); 27 | std::pair p[3]; 28 | p[0].first = he->prev(); 29 | p[1].first = he; 30 | p[2].first = he->next(); 31 | p[0].second = Geotools::negative(cut, C, p[0].first->vertex()->point()); 32 | p[1].second = Geotools::negative(cut, C, p[1].first->vertex()->point()); 33 | p[2].second = Geotools::negative(cut, C, p[2].first->vertex()->point()); 34 | 35 | double triArea = fit->area(); 36 | double triVol = 0; 37 | 38 | // from small to great 39 | std::sort(std::begin(p), std::end(p), 40 | [](const std::pair& a, const std::pair& b) { 41 | return a.second < b.second; 42 | }); 43 | 44 | int sum = 0; 45 | for (auto it = std::begin(p); it != std::end(p); ++it) { 46 | sum += it->second; 47 | } 48 | 49 | if (sum == 0) { 50 | triVol = Geotools::face_point_vol(fit, pop); 51 | } 52 | else if (sum == 1) // two points are at positive side 53 | { 54 | const Halfedge_const_handle e1 = p[2].first, e2 = e1->next(); 55 | const auto& e1_in_map = map_edge_point.find(e1); 56 | const auto& e2_in_map = map_edge_point.find(e2); 57 | if (e1_in_map != map_edge_point.end() && e2_in_map != map_edge_point.end()) { 58 | const double sArea = CGAL::sqrt(CGAL::squared_area(e1_in_map->second, 59 | p[2].first->vertex()->point(), 60 | e2_in_map->second)); 61 | triVol = (Geotools::face_point_vol(fit, pop) - Geotools::face_point_vol( 62 | e1_in_map->second, p[2].first->vertex()->point(), e2_in_map->second, pop)); 63 | triArea = triArea - sArea; 64 | } 65 | else { 66 | std::cout << "sum == 1 error" << std::endl; 67 | const Point3& p0 = p[0].first->vertex()->point(); 68 | const Point3& p1 = p[1].first->vertex()->point(); 69 | const double dis0 = CGAL::abs(cut * Vector3(p0.x(), p0.y(), p0.z()) + C); 70 | const double dis1 = CGAL::abs(cut * Vector3(p1.x(), p1.y(), p1.z()) + C); 71 | int nOp = 0; 72 | if (dis0 < 1e-10) ++nOp; 73 | if (dis1 < 1e-10) ++nOp; 74 | if (2 == nOp) triVol = Geotools::face_point_vol(fit, pop); 75 | else triArea = 0; 76 | } 77 | } 78 | else if (sum == 2) // one point is at positive side 79 | { 80 | const Halfedge_const_handle e1 = p[0].first, e2 = e1->next(); 81 | const auto& e1_in_map = map_edge_point.find(e1); 82 | const auto& e2_in_map = map_edge_point.find(e2); 83 | if (map_edge_point.find(e1) != map_edge_point.end() && map_edge_point.find(e2) != map_edge_point.end()) { 84 | triArea = CGAL::sqrt(CGAL::squared_area(e1_in_map->second, 85 | p[0].first->vertex()->point(), 86 | e2_in_map->second)); 87 | triVol = Geotools::face_point_vol(e1_in_map->second, 88 | p[0].first->vertex()->point(), 89 | e2_in_map->second, 90 | pop); 91 | } 92 | else { 93 | std::cout << "sum == 2 error" << std::endl; 94 | const Point3& p1 = p[1].first->vertex()->point(); 95 | const Point3& p2 = p[2].first->vertex()->point(); 96 | const double dis1 = CGAL::abs(cut * Vector3(p1.x(), p1.y(), p1.z()) + C); 97 | const double dis2 = CGAL::abs(cut * Vector3(p2.x(), p2.y(), p2.z()) + C); 98 | int nOp = 0; 99 | if (dis1 < 1e-10) ++nOp; 100 | if (dis2 < 1e-10) ++nOp; 101 | if (2 == nOp) triVol = Geotools::face_point_vol(fit, pop); 102 | else triArea = 0; 103 | } 104 | } 105 | 106 | // Computation 107 | if (sum != 3) { 108 | const bool isSupported = is_supported(fit->facetNormal, cut); 109 | 110 | if (!isSupported) VA.support_free = false; 111 | 112 | // insert rotated centroid points of faces to cpnts(2D) and cpnts3(3D) 113 | auto& tmp_p = fit->facetCentroid; 114 | Eigen::Vector3d tmp_pe(tmp_p.x(), tmp_p.y(), tmp_p.z()); 115 | auto rot_p = rotMat * tmp_pe; 116 | //cpnts[fit->id() - 2].emplace_back(rot_p.x(), rot_p.z()); 117 | cpnts3.emplace_back(rot_p.x(), rot_p.y(), rot_p.z()); 118 | //std::cout << rot_p.transpose() << std::endl; 119 | 120 | // update volume 121 | VA.first += triVol; 122 | //vec_bsps[fit->id() - 2].first += triVol; 123 | 124 | // update area 125 | if (fit->supported() && !isSupported) // was safe before but now is risky 126 | { 127 | VA.second += triArea; 128 | } 129 | 130 | if (!fit->supported() && isSupported) // was risky before but now is safe 131 | { 132 | //vec_bsps[fit->id() - 2].second -= projArea; 133 | VA.second -= triArea; 134 | 135 | inRev[fit->patch_id()] = true; 136 | } 137 | 138 | if (!fit->supported() && !isSupported) // was risky before and now is also risky 139 | { 140 | //vec_bsps[fit->id() - 2].second -= (fit->projArea() - projArea); 141 | //VA.second -= (fit->projArea() - projArea); 142 | //VA.second -= 0; 143 | // Absolute decrease 144 | //VA.fourth += (fit->projArea() - projArea); 145 | } 146 | 147 | if (!isSupported) VA.fifth += triArea; 148 | 149 | sumArea += triArea; 150 | } 151 | 152 | ++nIntersect; 153 | } 154 | 155 | // compute the volume of bounding box 156 | if (nIntersect <= 3) { 157 | VA.first = 0.; 158 | VA.second = 0.; 159 | } 160 | 161 | //VA.third /= boxVol_; 162 | VA.first = std::abs(VA.first); 163 | //VA.fifth /= sumArea; 164 | //VA.fifth = 1.0 - VA.fifth; 165 | return VA; 166 | } 167 | 168 | void MeshCutEval::get_inrev_risky_area(const Polyhedron& poly, const Vector3& current_dir, const Bbox& box, const std::vector& plt, std::vector& inRev) 169 | { 170 | std::vector vec_tu; 171 | 172 | Plane min_plane; 173 | const double a = current_dir.x(), b = current_dir.y(), c = current_dir.z(); 174 | double dmin = compute_min_d(current_dir, box); 175 | double dmax = compute_max_d(current_dir, box); 176 | double cdmax = -DBL_MAX; 177 | 178 | for (auto &p : plt) { 179 | Vector3 vp(p.x(), p.y(), p.z()); 180 | const double tmp = vp * current_dir; 181 | if (tmp > cdmax) cdmax = tmp; 182 | } 183 | const int n_int = static_cast(dmax - dmin + 1); 184 | 185 | // Find possible fragile points 186 | std::vector vecPosPnts, vecNegPnts; 187 | for (auto vit = poly.vertices_begin(); vit != poly.vertices_end(); ++vit) { 188 | const double faceDot = vit->vertexNormal * Vector3(a, b, c); 189 | if (faceDot > 0.92) vecPosPnts.emplace_back(vit); 190 | else if (faceDot < -0.92) vecNegPnts.emplace_back(vit); 191 | } 192 | 193 | Debug("dmin = " << dmin << " dmax = " << dmax << std::endl); 194 | Debug("cdmax " << cdmax << std::endl); 195 | Debug("interval = " << n_int << std::endl); 196 | 197 | for (auto j = 0; j < n_int; ++j) { 198 | double d = dmin + static_cast(j); 199 | if (d < cdmax) continue; 200 | Plane plane(a, b, c, -d); 201 | 202 | bool isFragile = false; 203 | double minFragile = DBL_MAX; 204 | for (auto v : vecPosPnts) 205 | { 206 | auto& pnt = v->point(); 207 | auto dis = pnt.x() * a + pnt.y() * b + pnt.z() * c - d; 208 | if (dis > 0 && dis < 1) 209 | { 210 | isFragile = true; 211 | break; 212 | } 213 | if (dis > 0 && dis < minFragile) minFragile = dis; 214 | } 215 | 216 | if (isFragile) continue; 217 | 218 | for (auto v : vecNegPnts) { 219 | auto &pnt = v->point(); 220 | auto dis = pnt.x() * a + pnt.y() * b + pnt.z() * c - d; 221 | if (dis < 0 && dis > -1) { 222 | isFragile = true; 223 | break; 224 | } 225 | if (dis < 0 && -dis < minFragile) minFragile = -dis; 226 | } 227 | 228 | if (isFragile) continue; 229 | 230 | std::unordered_map mapEdgePoint; 231 | std::set intersectedFaces; 232 | if (!cut_with_plane(poly, plane, mapEdgePoint, intersectedFaces)) continue; 233 | 234 | // Label intersected faces (euler update) 235 | Plane plane_neg(plane.a(), plane.b(), plane.c(), plane.d()); 236 | auto nbc = euler_update_facets(poly, plane_neg, intersectedFaces); 237 | 238 | // if it has multiple connected component after cutting, then check if they are connected with 239 | // the physical platform, this is done by approximating its minimal y values 240 | if (nbc.size() > 1) { 241 | auto maxMinValueY = *(std::max_element(nbc.begin(), nbc.end())); 242 | if (maxMinValueY > plt[0].y() + CONNECT_EPS) continue; 243 | } 244 | std::vector vecBsps; 245 | auto nRetRes = exact_cutting_volume_and_area(poly, plane, mapEdgePoint, vecBsps, inRev); 246 | 247 | BSP_Result tu; 248 | boost::get<0>(tu) = plane; 249 | boost::get<1>(tu) = nRetRes.first; 250 | boost::get<2>(tu) = -nRetRes.second; 251 | boost::get<3>(tu) = d - cdmax; 252 | boost::get<4>(tu) = minFragile; 253 | boost::get<5>(tu) = nRetRes.fifth; 254 | boost::get<6>(tu) = nRetRes.support_free; // Support-Free 255 | vec_tu.push_back(tu); 256 | } 257 | } -------------------------------------------------------------------------------- /src/PlaneCut.cpp: -------------------------------------------------------------------------------- 1 | #include "PlaneCut.h" 2 | #include 3 | 4 | constexpr double eps = 1e-8; 5 | constexpr double eps10 = 1e-7; 6 | 7 | inline bool negative(Vector3& p, double C, Point3& a) 8 | { 9 | Vector3 probe(a.x(), a.y(), a.z()); 10 | return probe * p + C > eps; 11 | } 12 | 13 | inline bool positive(Vector3& p, double C, Point3& a) 14 | { 15 | Vector3 probe(a.x(), a.y(), a.z()); 16 | return probe * p + C < -eps; 17 | } 18 | 19 | bool PlaneCutter::cut(Polyhedron& poly_left, Polyhedron& poly_right, const Plane& pl) { 20 | 21 | int IntrCnt = 0; 22 | std::vector Edges; 23 | std::vector ModifiedEdges; //Side to be modified 24 | Edges.reserve(poly_left.size_of_halfedges() / 2); 25 | 26 | // clear degenerate edges 27 | for (const auto& e : edges(poly_left)) 28 | { 29 | if (CGAL::Polygon_mesh_processing::is_degenerate_edge(e, poly_left)) 30 | { 31 | poly_left.join_vertex(e.halfedge()); 32 | } 33 | } 34 | 35 | for (Polyhedron::Edge_iterator it = poly_left.edges_begin(); 36 | it != poly_left.edges_end(); ++it) 37 | { 38 | Edges.push_back(it); 39 | } 40 | 41 | Vector3 cut(pl.a(), pl.b(), pl.c()); 42 | double C = pl.d(); 43 | cut = cut / CGAL::sqrt(cut.squared_length()); 44 | C = C / CGAL::sqrt(cut.squared_length()); 45 | 46 | double d0, d1; 47 | for (std::vector::iterator it = Edges.begin(); 48 | it != Edges.end(); ++it) 49 | { 50 | Halfedge_handle h = *it; 51 | Vector3 p1(h->prev()->vertex()->point().x(), h->prev()->vertex()->point().y(), h->prev()->vertex()->point().z()); 52 | Vector3 p2(h->vertex()->point().x(), h->vertex()->point().y(), h->vertex()->point().z()); 53 | d0 = cut * p1 + C; 54 | d1 = cut * p2 + C; 55 | Vector3 Q; 56 | if ((d0 >= 0 && d1 < 0) || (d0 < 0 && d1 >= 0)) 57 | { 58 | Q = p1 + ((d0 / (d0 - d1)) * (p2 - p1)); 59 | Point3 newPnt(Q.x(), Q.y(), Q.z()); 60 | if (std::abs(d0) < eps10) 61 | { 62 | h->prev()->vertex()->point() = newPnt; 63 | } 64 | else if (std::abs(d1) < eps10) 65 | { 66 | h->vertex()->point() = newPnt; 67 | } 68 | else 69 | { 70 | IntrCnt++; 71 | Halfedge_handle t = poly_left.split_edge(h); 72 | t->vertex()->point() = newPnt; 73 | ModifiedEdges.push_back(t); 74 | } 75 | } 76 | } 77 | //std::cout << "Modified edges = " << ModifiedEdges.size() << std::endl; 78 | 79 | for (std::vector::iterator it = ModifiedEdges.begin(); 80 | it != ModifiedEdges.end(); it++) 81 | { 82 | Halfedge_handle h = *it; 83 | Halfedge_handle g = h->opposite()->prev(); 84 | Facet_handle f_h = h->facet(); 85 | Facet_handle g_h = g->facet(); 86 | Halfedge_handle tmp_he; 87 | if (f_h != nullptr && !f_h->is_triangle()) 88 | { 89 | if (f_h->is_quad()) 90 | { 91 | tmp_he = poly_left.split_facet(h, h->next()->next()); 92 | } 93 | else 94 | { 95 | tmp_he = poly_left.split_facet(h, h->next()->next()); 96 | poly_left.split_facet(h, h->next()->next()); 97 | } 98 | } 99 | 100 | if (g_h != nullptr && !g_h->is_triangle()) 101 | { 102 | if (g_h->is_quad()) 103 | { 104 | tmp_he = poly_left.split_facet(g, g->next()->next()); 105 | } 106 | else 107 | { 108 | tmp_he = poly_left.split_facet(g, g->next()->next()); 109 | poly_left.split_facet(g, g->next()->next()); 110 | } 111 | } 112 | } 113 | 114 | poly_right = poly_left; 115 | 116 | for (Facet_iterator it = poly_left.facets_begin(), nd = poly_left.facets_end(); 117 | it != nd;) 118 | { 119 | Facet_iterator itNext = it; 120 | ++itNext; 121 | Halfedge_handle h = it->halfedge(); 122 | if (h == NULL) 123 | { 124 | it = itNext; 125 | continue; 126 | } 127 | Halfedge_handle e = h; 128 | do 129 | { 130 | if (negative(cut, C, h->vertex()->point())) 131 | { 132 | poly_left.erase_facet(e); 133 | break; 134 | } 135 | h = h->next(); 136 | } while (h != e); 137 | it = itNext; 138 | } 139 | 140 | for (Facet_iterator it = poly_right.facets_begin(), nd = poly_right.facets_end(); 141 | it != nd;) 142 | { 143 | Facet_iterator itNext = it; 144 | ++itNext; 145 | Halfedge_handle h = it->halfedge(); 146 | if (h == NULL) 147 | { 148 | it = itNext; 149 | continue; 150 | } 151 | Halfedge_handle e = h; 152 | do 153 | { 154 | if (positive(cut, C, h->vertex()->point())) 155 | { 156 | poly_right.erase_facet(e); 157 | break; 158 | } 159 | h = h->next(); 160 | } while (h != e); 161 | it = itNext; 162 | } 163 | 164 | return true; 165 | } 166 | 167 | std::pair PlaneCutter::cut(const Polyhedron& poly, const Plane& pl) { 168 | Polyhedron o1; 169 | Polyhedron o2; 170 | o1 = poly; 171 | cut(o1, o2, pl); 172 | return std::make_pair(o1, o2); 173 | } -------------------------------------------------------------------------------- /src/PlaneCut.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "CustomisedPolyhedron.h" 4 | #include "FillHoleCDT.h" 5 | 6 | class PlaneCutter 7 | { 8 | public: 9 | PlaneCutter() = default; 10 | ~PlaneCutter() = default; 11 | 12 | // Approach 1: this would modify original polyhedron 13 | bool cut(Polyhedron& poly_left, Polyhedron& poly_right, const Plane& pl); 14 | 15 | template 16 | bool cut_and_fill(Polyhedron& poly_left, Polyhedron& poly_right, const Plane& pl) { 17 | bool res = cut(poly_left, poly_right, pl); 18 | if (!res) return res; 19 | Vector3 planeDir(pl.a(), pl.b(), pl.c()); 20 | FH fh; 21 | fh.fill_hole(poly_left, planeDir); 22 | //fh.fill_hole(poly_right, -planeDir); 23 | return res; 24 | } 25 | 26 | template 27 | bool cut_and_fill_both(Polyhedron& poly_left, Polyhedron& poly_right, const Plane& pl) { 28 | bool res = cut(poly_left, poly_right, pl); 29 | if (!res) return res; 30 | Vector3 planeDir(pl.a(), pl.b(), pl.c()); 31 | FH fh; 32 | fh.fill_hole(poly_left, planeDir); 33 | fh.fill_hole(poly_right, -planeDir); 34 | return res; 35 | } 36 | 37 | // Approach 2: this won't affect original polyhedron 38 | std::pair cut(const Polyhedron& poly, const Plane& pl); 39 | 40 | template 41 | std::pair cut_and_fill(const Polyhedron& poly, const Plane& pl) { 42 | Polyhedron o1 = poly, o2; 43 | cut(o1, o2, pl); 44 | Vector3 planeDir(pl.a(), pl.b(), pl.c()); 45 | FH fh; 46 | fh.fill_hole(o1, -planeDir); 47 | fh.fill_hole(o2, planeDir); 48 | return std::make_pair(o1, o2); 49 | } 50 | 51 | }; 52 | -------------------------------------------------------------------------------- /src/RoboFDM.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "FillHoleCDT.h" 8 | #include "PlaneCut.h" 9 | #include "RoboFDM.h" 10 | 11 | #include 12 | 13 | std::default_random_engine generator(static_cast(time(nullptr))); 14 | std::uniform_real_distribution distribution(0.01, 0.99); 15 | 16 | RoboFDM::RoboFDM() : newLoaded_(true), first_init_(false) { load_candidate_dirs(candDirs_); } 17 | 18 | RoboFDM::~RoboFDM() {} 19 | 20 | BSPNode RoboFDM::apply_action(double alpha, double beta, double gamma, Plane& pl) { 21 | // TODO: Apply this action to FineDecomposition 22 | const auto plt = MeshCutEval::get_platform_cross(rPlatform.first, rPlatform.second); 23 | pl = MeshCutEval::convert_abg_to_plane(alpha, beta, gamma, bsphere); 24 | auto res = MeshCutEval::apply_plane_cut(poly, pl, bbox_, plt); 25 | return res; 26 | } 27 | 28 | BSPNode RoboFDM::apply_action(double alpha, double beta, double gamma) { 29 | // TODO: Apply this action to FineDecomposition 30 | const auto plt = MeshCutEval::get_platform_cross(rPlatform.first, rPlatform.second); 31 | auto pl = MeshCutEval::convert_abg_to_plane(alpha, beta, gamma, bsphere); 32 | auto res = MeshCutEval::apply_plane_cut(poly, pl, bbox_, plt); 33 | return res; 34 | } 35 | 36 | py::tuple RoboFDM::step(py::array_t& input) { 37 | py::tuple data(5); 38 | py::buffer_info buf = input.request(); 39 | auto* ptr = (double*)buf.ptr; 40 | Plane pl; 41 | 42 | const auto plt = MeshCutEval::get_platform_cross(rPlatform.first, rPlatform.second); 43 | if (input.size() == 3) 44 | pl = MeshCutEval::convert_abg_to_plane(ptr[0], ptr[1], ptr[2], bsphere); 45 | else 46 | pl = Plane(ptr[0], ptr[1], ptr[2], ptr[3]); 47 | 48 | auto res = MeshCutEval::apply_plane_cut(poly, pl, bbox_, plt); 49 | 50 | data[0] = res.first; 51 | data[1] = res.second; 52 | data[2] = res.third; 53 | data[3] = res.fourth; 54 | data[4] = res.fifth; 55 | return data; 56 | } 57 | 58 | bool RoboFDM::plane_cut(py::array_t& input) { 59 | auto* data = input.data(); 60 | Plane pl; 61 | if (input.size() == 3) { 62 | pl = MeshCutEval::convert_abg_to_plane(data[0], data[1], data[2], bsphere); 63 | } else { 64 | pl = Plane(data[0], data[1], data[2], data[3]); 65 | } 66 | 67 | PlaneCutter pc; 68 | Polyhedron _; 69 | pc.cut_and_fill(poly, _, pl); 70 | return true; 71 | } 72 | 73 | bool RoboFDM::plane_cut_both(py::array_t& input) { 74 | auto* data = input.data(); 75 | Plane pl; 76 | if (input.size() == 3) { 77 | pl = MeshCutEval::convert_abg_to_plane(data[0], data[1], data[2], bsphere); 78 | } else { 79 | pl = Plane(data[0], data[1], data[2], data[3]); 80 | } 81 | 82 | poly_pos.clear(); 83 | PlaneCutter pc; 84 | pc.cut_and_fill_both(poly, poly_pos, pl); 85 | return true; 86 | } 87 | 88 | py::array_t RoboFDM::planes() { 89 | const auto dim = 3; 90 | 91 | // auto result = py::array_t({Na, Nb, Nc, dim}); 92 | auto result = py::array_t({N3, dim}); 93 | py::buffer_info buf_result = result.request(); 94 | auto* arrEvalRes = (double*)buf_result.ptr; 95 | 96 | #pragma omp parallel for 97 | for (auto i = 0; i < N3; i++) { 98 | const auto pos_dir = i % N2; 99 | const auto pos_off = static_cast(i / N2); 100 | 101 | const auto gamma = pos_off / static_cast(Nc); 102 | const auto alpha = static_cast(pos_dir / Nb) / static_cast(Na); 103 | const auto beta = static_cast(pos_dir % Nb) / static_cast(Nb); 104 | 105 | arrEvalRes[i * dim + 0] = alpha; 106 | arrEvalRes[i * dim + 1] = beta; 107 | arrEvalRes[i * dim + 2] = gamma; 108 | } 109 | return result; 110 | } 111 | 112 | double RoboFDM::get_far_risky_area() { 113 | double sumArea = 0.; 114 | for (auto& f : faces(poly)) { 115 | if (f->facetCentroid.y() > rPlatform.second + 1.4 && !f->supported()) { 116 | sumArea += f->area(); 117 | } 118 | } 119 | 120 | return sumArea; 121 | } 122 | 123 | double RoboFDM::get_inrev_risky_area() { 124 | omp_set_num_threads(nThread); // Set Thread Number 125 | const auto plt = MeshCutEval::get_platform_cross(rPlatform.first, rPlatform.second); 126 | 127 | std::vector> evalResults; 128 | evalResults.resize(nThread); 129 | 130 | std::vector isInrev(poly.size_of_facets(), false); 131 | 132 | int cnt = 0; 133 | for (auto f : faces(poly)) { 134 | f->set_patch_id(cnt++); 135 | } 136 | 137 | #pragma omp parallel for schedule(dynamic) 138 | for (auto i = 0; i < candDirs_.size(); ++i) { 139 | int tid = omp_get_thread_num(); 140 | const auto& curDir = candDirs_[i]; 141 | MeshCutEval::get_inrev_risky_area(poly, curDir, bbox_, plt, isInrev); 142 | } 143 | 144 | double sumInrevArea = 0.; 145 | for (auto f : faces(poly)) { 146 | if (f->supported()) 147 | continue; 148 | 149 | if (isInrev[f->patch_id()]) 150 | sumInrevArea += f->area(); 151 | } 152 | 153 | return sumInrevArea; 154 | } 155 | 156 | py::array_t RoboFDM::reset(const std::string& meshfile) { 157 | poly.clear(); 158 | std::ifstream f(meshfile); 159 | f >> poly; 160 | f.close(); 161 | if (poly.is_valid()) { 162 | if (!poly.is_closed()) { 163 | unsigned int nb_holes = 0; 164 | for (Halfedge_handle h : halfedges(poly)) { 165 | if (h->is_border()) { 166 | std::vector patch_facets; 167 | CGAL::Polygon_mesh_processing::triangulate_hole( 168 | poly, h, std::back_inserter(patch_facets), 169 | CGAL::Polygon_mesh_processing::parameters::use_delaunay_triangulation(true)); 170 | 171 | ++nb_holes; 172 | } 173 | } 174 | } 175 | } 176 | 177 | rPlatform = MeshCutEval::find_suitable_platform(poly); 178 | 179 | initResults = MeshCutEval::initialize_supports(poly); 180 | bbox_ = bbox_3(poly.points_begin(), poly.points_end()); 181 | bsphere = MinSphere(poly.points_begin(), poly.points_end()); 182 | 183 | initVolume = initResults[0]; 184 | initArea = initResults[1]; 185 | initRiskyArea = initResults[2]; 186 | 187 | // std::cout << initResults[0] << " " << initResults[1] << " " << initResults[2] << std::endl; 188 | 189 | return py::array_t({1}); 190 | 191 | if (!first_init_) { 192 | init_features_ = render(); 193 | first_init_ = true; 194 | } 195 | return init_features_; 196 | } 197 | 198 | py::array_t RoboFDM::render() { 199 | omp_set_num_threads(nThread); // Set Thread Number 200 | // auto result = py::array_t({Na, Nb, Nc, dim}); 201 | 202 | const auto plt = MeshCutEval::get_platform_cross(rPlatform.first, rPlatform.second); 203 | 204 | std::vector> evalResults; 205 | evalResults.resize(nThread); 206 | 207 | #pragma omp parallel for schedule(dynamic) 208 | for (auto i = 0; i < candDirs_.size(); ++i) { 209 | int tid = omp_get_thread_num(); 210 | const auto& curDir = candDirs_[i]; 211 | MeshCutEval::evaluation_direction(poly, curDir, bbox_, plt, evalResults[tid]); 212 | } 213 | 214 | int numEvals = 0; 215 | #pragma omp parallel for schedule(dynamic) 216 | for (auto i = 0; i < evalResults.size(); ++i) { 217 | #pragma omp critical 218 | numEvals += evalResults[i].size(); 219 | } 220 | 221 | const auto dim = 9; 222 | auto result = py::array_t({numEvals, dim}); 223 | py::buffer_info buf_result = result.request(); 224 | auto* arrEvalRes = (double*)buf_result.ptr; 225 | 226 | numEvals = 0; 227 | for (auto i = 0; i < evalResults.size(); ++i) { 228 | #pragma omp parallel for schedule(dynamic) 229 | for (auto j = 0; j < evalResults[i].size(); ++j) { 230 | const auto curIdx = numEvals + j; 231 | const auto& tmpEval = evalResults[i][j]; 232 | arrEvalRes[dim * curIdx + 0] = tmpEval.get<1>() / initVolume; 233 | arrEvalRes[dim * curIdx + 1] = tmpEval.get<2>() / initRiskyArea; 234 | arrEvalRes[dim * curIdx + 2] = 235 | std::max(0.0, std::min(tmpEval.get<3>() / std::sqrt(bsphere.squared_radius()), 1.0)); 236 | arrEvalRes[dim * curIdx + 3] = 237 | std::max(0.0, std::min(tmpEval.get<4>() / std::sqrt(bsphere.squared_radius()), 1.0)); 238 | arrEvalRes[dim * curIdx + 4] = tmpEval.get<5>() / initRiskyArea; 239 | const auto& pl = tmpEval.get<0>(); 240 | arrEvalRes[dim * curIdx + 5] = pl.a(); 241 | arrEvalRes[dim * curIdx + 6] = pl.b(); 242 | arrEvalRes[dim * curIdx + 7] = pl.c(); 243 | arrEvalRes[dim * curIdx + 8] = pl.d(); 244 | } 245 | numEvals += evalResults[i].size(); 246 | } 247 | return result; 248 | } 249 | 250 | std::string RoboFDM::get_poly() { 251 | std::stringstream ss; 252 | ss << std::setprecision(8) << poly; 253 | return ss.str(); 254 | } 255 | 256 | std::string RoboFDM::get_positive_poly() { 257 | std::stringstream ss; 258 | ss << poly_pos; 259 | return ss.str(); 260 | } 261 | 262 | bool RoboFDM::set_poly(const std::string& str) { 263 | std::stringstream ss(str); 264 | poly.clear(); 265 | ss >> poly; 266 | if (poly.is_valid()) { 267 | if (!poly.is_closed()) { 268 | unsigned int nb_holes = 0; 269 | for (Halfedge_handle h : halfedges(poly)) { 270 | if (h->is_border()) { 271 | std::vector patch_facets; 272 | CGAL::Polygon_mesh_processing::triangulate_hole( 273 | poly, h, std::back_inserter(patch_facets), 274 | CGAL::Polygon_mesh_processing::parameters::use_delaunay_triangulation(true)); 275 | 276 | ++nb_holes; 277 | } 278 | } 279 | } 280 | 281 | auto initResults = MeshCutEval::initialize_supports(poly); 282 | //std::cout << initResults[0] << " " << initResults[1] << " " << initResults[2] << std::endl; 283 | 284 | return true; 285 | } else { 286 | return false; 287 | } 288 | } 289 | -------------------------------------------------------------------------------- /src/RoboFDM.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include "MeshCutEval.h" 7 | 8 | namespace py = pybind11; 9 | using namespace pybind11::literals; 10 | 11 | 12 | 13 | class RoboFDM { 14 | public: 15 | RoboFDM(); 16 | 17 | ~RoboFDM(); 18 | 19 | void load_tet_mesh(const std::string &file); 20 | 21 | void load_tri_mesh(const std::string &file); 22 | 23 | bool mesh_to_polyhedron(const Eigen::MatrixXd &V, const Eigen::MatrixXi &F, Polyhedron &poly); 24 | 25 | py::array_t reset(const std::string& meshfile); 26 | 27 | BSPNode apply_old_action(double x, double y, double z, double alpha, double beta); 28 | 29 | BSPNode apply_action(double alpha, double beta, double gamma, Plane& pl); 30 | 31 | BSPNode apply_action(double alpha, double beta, double gamma); 32 | 33 | py::tuple step(py::array_t& input); 34 | 35 | py::array_t render(); 36 | 37 | std::string get_poly(); 38 | 39 | std::string get_positive_poly(); 40 | 41 | bool set_poly(const std::string& str); 42 | 43 | bool plane_cut(py::array_t& input); 44 | 45 | bool plane_cut_both(py::array_t& input); 46 | 47 | py::array_t planes(); 48 | 49 | int n_features(); 50 | 51 | double get_far_risky_area(); 52 | 53 | double get_inrev_risky_area(); 54 | 55 | double get_risky_area() const { return initRiskyArea; }; 56 | 57 | double get_volume() const { return initVolume; }; 58 | 59 | double get_area() const { return initArea; }; 60 | 61 | private: 62 | static void load_candidate_dirs(std::vector& candidate_dirs_); 63 | 64 | public: 65 | Polyhedron poly, poly_pos; 66 | std::vector initResults; 67 | 68 | MinSphere bsphere; 69 | 70 | double initVolume; 71 | double initArea; 72 | double initRiskyArea; 73 | 74 | private: 75 | Eigen::MatrixXd V_; 76 | Eigen::MatrixXi F_; 77 | bool first_init_; 78 | py::array_t init_features_; 79 | std::vector candDirs_; 80 | Bbox bbox_; 81 | const int Na = 64, Nb = 16, Nc = 64; 82 | const int N2 = Na * Nb; 83 | const int N3 = N2 * Nc; 84 | const int nThread = omp_get_num_procs(); 85 | std::pair rPlatform = std::make_pair(20, 0); 86 | bool newLoaded_; 87 | }; 88 | --------------------------------------------------------------------------------