├── .dockerignore ├── .flake8 ├── .github └── workflows │ └── ci-unittests.yml ├── .gitignore ├── .gitlab-ci.yml ├── .mypy.ini ├── .pytest.coveragerc ├── .readthedocs.yaml ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── TPL ├── CMakeLists.txt ├── README ├── flann-1.8.4 │ ├── CMakeLists.txt │ ├── COPYING │ ├── ChangeLog │ ├── README.md │ ├── bin │ │ ├── download_checkmd5.py │ │ ├── indent.sh │ │ ├── make_release.sh │ │ ├── run_test.py │ │ └── uncrustify.cfg │ ├── cmake │ │ ├── CMakeLists.txt │ │ ├── FindFlann.cmake │ │ ├── UseLATEX.cmake │ │ ├── flann.pc.in │ │ ├── flann_utils.cmake │ │ └── uninstall_target.cmake.in │ ├── doc │ │ ├── CMakeLists.txt │ │ ├── images │ │ │ └── cmake-gui.png │ │ ├── manual.tex │ │ └── references.bib │ ├── examples │ │ ├── CMakeLists.txt │ │ ├── README │ │ ├── flann_example.c │ │ ├── flann_example.cpp │ │ └── flann_example_mpi.cpp │ ├── src │ │ ├── CMakeLists.txt │ │ ├── cpp │ │ │ ├── CMakeLists.txt │ │ │ └── flann │ │ │ │ ├── algorithms │ │ │ │ ├── all_indices.h │ │ │ │ ├── autotuned_index.h │ │ │ │ ├── center_chooser.h │ │ │ │ ├── composite_index.h │ │ │ │ ├── dist.h │ │ │ │ ├── hierarchical_clustering_index.h │ │ │ │ ├── kdtree_cuda_3d_index.cu │ │ │ │ ├── kdtree_cuda_3d_index.h │ │ │ │ ├── kdtree_cuda_builder.h │ │ │ │ ├── kdtree_index.h │ │ │ │ ├── kdtree_single_index.h │ │ │ │ ├── kmeans_index.h │ │ │ │ ├── linear_index.h │ │ │ │ ├── lsh_index.h │ │ │ │ └── nn_index.h │ │ │ │ ├── config.h │ │ │ │ ├── config.h.in │ │ │ │ ├── defines.h │ │ │ │ ├── flann.cpp │ │ │ │ ├── flann.h │ │ │ │ ├── flann.hpp │ │ │ │ ├── flann_cpp.cpp │ │ │ │ ├── general.h │ │ │ │ ├── io │ │ │ │ └── hdf5.h │ │ │ │ ├── mpi │ │ │ │ ├── client.h │ │ │ │ ├── flann_mpi_client.cpp │ │ │ │ ├── flann_mpi_server.cpp │ │ │ │ ├── index.h │ │ │ │ ├── matrix.h │ │ │ │ ├── queries.h │ │ │ │ └── server.h │ │ │ │ ├── nn │ │ │ │ ├── ground_truth.h │ │ │ │ ├── index_testing.h │ │ │ │ └── simplex_downhill.h │ │ │ │ └── util │ │ │ │ ├── allocator.h │ │ │ │ ├── any.h │ │ │ │ ├── cuda │ │ │ │ ├── heap.h │ │ │ │ └── result_set.h │ │ │ │ ├── cutil_math.h │ │ │ │ ├── dynamic_bitset.h │ │ │ │ ├── heap.h │ │ │ │ ├── logger.h │ │ │ │ ├── lsh_table.h │ │ │ │ ├── matrix.h │ │ │ │ ├── object_factory.h │ │ │ │ ├── params.h │ │ │ │ ├── random.h │ │ │ │ ├── result_set.h │ │ │ │ ├── sampling.h │ │ │ │ ├── saving.h │ │ │ │ ├── serialization.h │ │ │ │ └── timer.h │ │ ├── matlab │ │ │ ├── CMakeLists.txt │ │ │ ├── flann_build_index.m │ │ │ ├── flann_free_index.m │ │ │ ├── flann_load_index.m │ │ │ ├── flann_save_index.m │ │ │ ├── flann_search.m │ │ │ ├── flann_set_distance_type.m │ │ │ ├── nearest_neighbors.cpp │ │ │ └── test_flann.m │ │ └── python │ │ │ ├── CMakeLists.txt │ │ │ ├── pyflann │ │ │ ├── __init__.py │ │ │ ├── exceptions.py │ │ │ ├── flann_ctypes.py │ │ │ └── index.py │ │ │ └── setup.py.tpl │ └── test │ │ ├── CMakeLists.txt │ │ ├── flann_autotuned_test.cpp │ │ ├── flann_cuda_test.cu │ │ ├── flann_hierarchical_test.cpp │ │ ├── flann_kdtree_single_test.cpp │ │ ├── flann_kdtree_test.cpp │ │ ├── flann_kmeans_test.cpp │ │ ├── flann_linear_test.cpp │ │ ├── flann_lsh_test.cpp │ │ ├── flann_multithreaded_test.cpp │ │ ├── flann_tests.h │ │ ├── memusage_clustering.py │ │ ├── memusage_nn.py │ │ ├── test_clustering.py │ │ ├── test_index_save.py │ │ ├── test_nn.py │ │ ├── test_nn_autotune.py │ │ └── test_nn_index.py └── libsvm-3.1-custom │ ├── COPYRIGHT │ ├── FAQ.html │ ├── Makefile │ ├── Makefile.original │ ├── Makefile.win │ ├── README │ ├── README-Kitware.md │ ├── binary-predict.c │ ├── binary-train.c │ ├── convert.c │ ├── cross-validation │ ├── Makefile.txt │ ├── eval.cpp │ └── eval.h │ ├── data │ ├── leu │ ├── leu.bz2 │ ├── leu.combined │ ├── leu.t │ ├── leu.t.bz2 │ ├── range1 │ ├── svmguide1 │ ├── svmguide1.model │ ├── svmguide1.range │ ├── svmguide1.scale │ ├── svmguide1.scale.model │ ├── svmguide1.scale.out │ ├── svmguide1.scale.png │ ├── svmguide1.t │ ├── svmguide1.t.predict │ ├── svmguide1.t.scale │ ├── svmguide2 │ ├── svmguide4 │ ├── svmguide4.scale │ ├── svmguide4.scale.range │ ├── svmguide4.scale.scale │ ├── svmguide4.t │ └── svmguide4.t.scale │ ├── eval.cpp │ ├── eval.h │ ├── heart_scale │ ├── hik │ ├── Makefile │ ├── class1_histintsc.cfg │ ├── kernel_libSVMList │ ├── kernel_libSVMList.cpp │ ├── kernel_libSVMList.o │ ├── kernelizerList.cpp │ ├── kernelizerList.h │ ├── kernelizerList.o │ ├── libSVMListData.cpp │ ├── libSVMListData.h │ ├── libSVMListData.o │ └── svmNode.h │ ├── java │ ├── Makefile │ ├── libsvm.jar │ ├── libsvm │ │ ├── svm.java │ │ ├── svm.m4 │ │ ├── svm_model.java │ │ ├── svm_node.java │ │ ├── svm_parameter.java │ │ ├── svm_print_interface.java │ │ └── svm_problem.java │ ├── svm_predict.java │ ├── svm_scale.java │ ├── svm_toy.java │ ├── svm_train.java │ └── test_applet.html │ ├── matlab │ ├── Makefile │ ├── README │ ├── libsvmread.c │ ├── libsvmwrite.c │ ├── make.m │ ├── svm_model_matlab.c │ ├── svm_model_matlab.h │ ├── svmpredict.c │ └── svmtrain.c │ ├── python │ ├── Makefile │ ├── README │ ├── plotroc.py │ ├── setup.py │ ├── svm │ │ └── __init__.py │ └── svmutil │ │ └── __init__.py │ ├── svm-predict.c │ ├── svm-prob.c │ ├── svm-scale.c │ ├── svm-toy │ ├── gtk │ │ ├── Makefile │ │ ├── callbacks.cpp │ │ ├── callbacks.h │ │ ├── interface.c │ │ ├── interface.h │ │ ├── main.c │ │ └── svm-toy.glade │ ├── qt │ │ ├── Makefile │ │ └── svm-toy.cpp │ └── windows │ │ └── svm-toy.cpp │ ├── svm-train.c │ ├── svm.cpp │ ├── svm.cpp.original │ ├── svm.def │ ├── svm.h │ ├── svm.h.original │ └── tools │ ├── README │ ├── checkdata.py │ ├── easy.py │ ├── easy.py.original │ ├── grid.py │ ├── grid_shawk.py │ ├── linear-easy.py │ ├── linear_grid.py.dated │ ├── plotroc.py │ ├── subset.py │ ├── train_fusion_classifier.py │ └── xvalidation.py ├── data └── caffenet │ ├── cat.jpg │ ├── cat_gray.jpg │ ├── file_list.txt.in │ ├── fish-bike.jpg │ └── imagenet_val.prototxt ├── devops ├── ansible │ └── roles │ │ ├── kitware.caffe │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── caffe.yml │ │ │ ├── dep_install.yml │ │ │ ├── dep_python.yml │ │ │ └── main.yml │ │ └── kitware.smqtk │ │ ├── README.md │ │ ├── defaults │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── tasks │ │ ├── daemon.yml │ │ ├── main.yml │ │ ├── model_install.yml │ │ └── smqtk_build.yml │ │ └── templates │ │ ├── isd.conf.json.j2 │ │ └── upstart.smqtk_iqr_app.conf.j2 └── docker │ ├── .env │ ├── docker-compose.build.yml │ ├── smqtk_classifier_service │ ├── Dockerfile │ ├── README.rst │ ├── default_server.cpu.json │ ├── default_server.gpu.json │ └── entrypoint.sh │ ├── smqtk_iqr_playground │ ├── Dockerfile │ ├── README.md │ ├── TODO.txt │ ├── default_confs │ │ ├── cpu │ │ │ ├── compute_many_descriptors.json │ │ │ └── runApp.IqrService.json │ │ ├── entrypoint.conf │ │ ├── generate_image_transform.tiles.json │ │ ├── gpu │ │ │ ├── compute_many_descriptors.json │ │ │ └── runApp.IqrService.json │ │ ├── nn_index_tool.json │ │ ├── psql_table_init.sql │ │ └── runApp.IqrSearchDispatcher.json │ ├── descr_comp_test.cpu.py │ ├── descr_comp_test.gpu.py │ ├── entrypoint.sh │ ├── matplotlibrc │ ├── run_container.cpu.sh │ └── run_container.gpu.sh │ └── tpl │ ├── caffe1 │ ├── Dockerfile │ ├── README.md │ └── setup.py │ └── faiss │ ├── README.md │ ├── gpu │ └── Dockerfile │ └── patch │ ├── 1.5.1 │ └── python │ │ └── Makefile │ ├── 1.6.0 │ └── python │ │ └── Makefile │ └── 1.6.2 │ └── python │ └── Makefile ├── docs ├── Makefile ├── algorithminterfaces.rst ├── algorithmmodels.rst ├── algorithms.rst ├── architecture.rst ├── architecture │ └── plugins_configuration.rst ├── conf.py ├── dataabstraction.rst ├── examples │ ├── caffe_build_index.rst │ ├── nnss_incremental_update │ │ ├── 1.download_leeds_butterfly.sh │ │ ├── 2.ingest_files_1.txt │ │ ├── 2a.config.compute_many_descriptors.json │ │ ├── 2a.run.sh │ │ ├── 2b.config.train_itq.json │ │ ├── 2b.run.sh │ │ ├── 2c.config.compute_hash_codes.json │ │ ├── 2c.extract_ingest_uuids.sh │ │ ├── 2c.run.sh │ │ ├── 2c.uuids_for_processing.txt │ │ ├── 2d.config.nnss_app.json │ │ ├── 3.config.compute_hash_codes.json │ │ ├── 3.ingest_files_2.txt │ │ ├── 3.run.sh │ │ ├── 4.config.compute_hash_codes.json │ │ ├── 4.ingest_files_3.txt │ │ ├── 4.run.sh │ │ ├── incremental_example.rst │ │ └── test_in_index.sh │ ├── overview.rst │ └── simple_feature_computation.rst ├── index.rst ├── installation.rst ├── make.bat ├── quickstarts.rst ├── quickstarts │ └── create_external_plugin.rst ├── readthedocs-reqs.txt ├── release_notes.rst ├── release_notes │ ├── pending_release.rst │ ├── v0.10.0.rst │ ├── v0.11.0.rst │ ├── v0.12.0.rst │ ├── v0.13.0.rst │ ├── v0.14.0.rst │ ├── v0.2.0.rst │ ├── v0.2.1.rst │ ├── v0.2.2.rst │ ├── v0.3.0.rst │ ├── v0.4.0.rst │ ├── v0.5.0.rst │ ├── v0.6.0.rst │ ├── v0.6.1.rst │ ├── v0.6.2.rst │ ├── v0.7.0.rst │ ├── v0.8.0.rst │ ├── v0.8.1.rst │ └── v0.9.0.rst ├── release_process.rst ├── releasing.rst ├── sphinx_server.py ├── utilities.rst ├── webservices.rst └── webservices │ ├── descriptorservice.rst │ ├── figures │ ├── IQRWithSMQTK.png │ ├── iqr-butterflies-link.png │ ├── iqrinitialize.png │ ├── iqrinitialresults.png │ ├── iqrlogin-entry.png │ ├── iqrlogin.png │ └── iqrrefine.png │ └── iqrdemonstration.rst ├── etc └── smqtk │ ├── postgres │ ├── classification_element │ │ └── example_table_init.sql │ ├── descriptor_element │ │ └── example_table_init.sql │ ├── descriptor_set │ │ └── example_table_init.sql │ └── example_pgbouncer.ini │ └── proxymanager.config ├── girder ├── external_smqtk_iqr │ ├── README.rst │ ├── plugin.yml │ └── web_client │ │ ├── js │ │ └── exiqr_folderView.js │ │ ├── stylesheets │ │ └── exiqr_folderView.styl │ │ └── templates │ │ └── exiqr_folderView.jade ├── smqtk_girder │ ├── plugin.cmake │ ├── plugin.json │ ├── plugin_tests │ │ ├── fixtures │ │ │ ├── girder │ │ │ │ └── images │ │ │ │ │ ├── .smqtk │ │ │ │ │ └── .gitignore │ │ │ │ │ ├── 001_0001.jpg │ │ │ │ │ ├── 001_0002.jpg │ │ │ │ │ ├── 001_0003.jpg │ │ │ │ │ ├── 001_0004.jpg │ │ │ │ │ ├── 001_0005.jpg │ │ │ │ │ ├── 002_0001.jpg │ │ │ │ │ ├── 002_0002.jpg │ │ │ │ │ ├── 002_0003.jpg │ │ │ │ │ ├── 002_0004.jpg │ │ │ │ │ ├── 002_0005.jpg │ │ │ │ │ ├── 003_0001.jpg │ │ │ │ │ ├── 003_0002.jpg │ │ │ │ │ ├── 003_0003.jpg │ │ │ │ │ └── 003_0004.jpg │ │ │ └── images.yml │ │ ├── gallerySpec.js │ │ └── testUtils.js │ ├── server │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── iqr.py │ │ ├── nearest_neighbors.py │ │ └── utils.py │ └── web_client │ │ ├── collections │ │ ├── IqrResultsCollection.js │ │ ├── IqrSessionCollection.js │ │ └── ResultsCollection.js │ │ ├── main.js │ │ ├── models │ │ └── IqrSessionModel.js │ │ ├── routes.js │ │ ├── stylesheets │ │ ├── galleryItemView.styl │ │ └── iqrItemView.styl │ │ ├── templates │ │ ├── configView.pug │ │ ├── galleryDetailWidget.pug │ │ ├── galleryItemView.pug │ │ ├── galleryView.pug │ │ ├── hierarchyWidgetMenu.pug │ │ └── iqrItemView.pug │ │ └── views │ │ ├── ConfigView.js │ │ ├── GalleryDetailWidget.js │ │ ├── GalleryItemView.js │ │ ├── GalleryView.js │ │ ├── HierarchyWidget.js │ │ ├── IqrItemView.js │ │ └── IqrView.js └── smqtk_worker │ ├── .gitignore │ ├── setup.py │ └── smqtk_worker │ ├── __init__.py │ ├── tasks.py │ └── utils.py ├── pytest.ini ├── python └── smqtk │ ├── __init__.py │ ├── algorithms │ ├── __init__.py │ ├── _interface.py │ ├── classifier │ │ ├── __init__.py │ │ ├── _classifier_collection.py │ │ ├── _defaults.py │ │ ├── _interface_classifier.py │ │ ├── _interface_supervised.py │ │ ├── _plugins.py │ │ ├── index_label.py │ │ ├── libsvm.py │ │ └── sklearn_logistic_regression.py │ ├── descriptor_generator │ │ ├── __init__.py │ │ ├── _interface.py │ │ ├── _plugins.py │ │ ├── caffe_descriptor.py │ │ ├── colordescriptor │ │ │ ├── INSTALL.md │ │ │ ├── __init__.py │ │ │ ├── colordescriptor.py │ │ │ ├── encode_FLANN.py │ │ │ └── utils.py │ │ └── kwcnndescriptor │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── autoencoder_model_def.py │ │ │ ├── kwcnndescriptor.py │ │ │ └── kwcnnmodel.npy │ ├── image_io │ │ ├── __init__.py │ │ ├── _interface.py │ │ ├── _plugins.py │ │ ├── gdal_io.py │ │ └── pil_io.py │ ├── nn_index │ │ ├── __init__.py │ │ ├── _interface_nn_index.py │ │ ├── _plugins.py │ │ ├── faiss.py │ │ ├── flann.py │ │ ├── hash_index │ │ │ ├── __init__.py │ │ │ ├── _interface_hash_index.py │ │ │ ├── _plugins.py │ │ │ ├── linear.py │ │ │ └── sklearn_balltree.py │ │ ├── lsh │ │ │ ├── __init__.py │ │ │ └── functors │ │ │ │ ├── __init__.py │ │ │ │ ├── _plugins.py │ │ │ │ ├── itq.py │ │ │ │ └── simple_rp.py │ │ └── mrpt.py │ ├── object_detection │ │ ├── __init__.py │ │ ├── _defaults.py │ │ └── _interface.py │ ├── rank_relevancy │ │ ├── __init__.py │ │ ├── _interface.py │ │ ├── _plugins.py │ │ ├── margin_sampling.py │ │ └── wrap_classifier.py │ └── relevancy_index │ │ ├── __init__.py │ │ ├── _plugins.py │ │ ├── classifier_wrapper.py │ │ └── libsvm_hik.py │ ├── bin │ ├── __init__.py │ ├── check_images.py │ ├── classifier_kfold_validation.py │ ├── classifier_model_validation.py │ ├── classifyFiles.py │ ├── computeDescriptor.py │ ├── compute_classifications.py │ ├── compute_hash_codes.py │ ├── compute_many_descriptors.py │ ├── createFileIngest.py │ ├── createGirderIngest.py │ ├── descriptors_to_svmtrainfile.py │ ├── generate_image_transform.py │ ├── iqrTrainClassifier.py │ ├── iqr_app_model_generation.py │ ├── make_balltree.py │ ├── make_train_test_sets.py │ ├── minibatch_kmeans_clusters.py │ ├── nearest_neighbors.py │ ├── nn_index_tool.py │ ├── proxyManagerServer.py │ ├── removeOldFiles.py │ ├── runApplication.py │ ├── summarizePlugins.py │ └── train_itq.py │ ├── compute_functions.py │ ├── exceptions │ └── __init__.py │ ├── iqr │ ├── __init__.py │ ├── iqr_controller.py │ └── iqr_session.py │ ├── representation │ ├── __init__.py │ ├── _interface.py │ ├── bbox.py │ ├── classification_element │ │ ├── __init__.py │ │ ├── _plugins.py │ │ ├── file.py │ │ ├── memory.py │ │ └── postgres.py │ ├── classification_element_factory.py │ ├── data_element │ │ ├── __init__.py │ │ ├── _plugins.py │ │ ├── file_element.py │ │ ├── girder.py │ │ ├── hbase_element.py │ │ ├── matrix.py │ │ ├── memory_element.py │ │ ├── psql.py │ │ └── url_element.py │ ├── data_set │ │ ├── __init__.py │ │ ├── _plugins.py │ │ ├── file_set.py │ │ ├── kvstore_backed.py │ │ ├── memory_set.py │ │ └── psql.py │ ├── descriptor_element │ │ ├── __init__.py │ │ ├── _io.py │ │ ├── _plugins.py │ │ ├── cached_element_wrapper.py │ │ ├── local_elements.py │ │ ├── postgres.py │ │ └── solr_element.py │ ├── descriptor_element_factory.py │ ├── descriptor_set │ │ ├── __init__.py │ │ ├── _plugins.py │ │ ├── memory.py │ │ ├── postgres.py │ │ └── solr_index.py │ ├── detection_element │ │ ├── __init__.py │ │ ├── _interface.py │ │ ├── _plugins.py │ │ └── memory.py │ ├── detection_element_factory.py │ └── key_value │ │ ├── __init__.py │ │ ├── _plugins.py │ │ ├── memory.py │ │ └── postgres.py │ ├── utils │ ├── __init__.py │ ├── base_object.py │ ├── bits.py │ ├── cli.py │ ├── combinatorics.py │ ├── configuration.py │ ├── content_type_validator.py │ ├── database_info.py │ ├── dict.py │ ├── distance_kernel.py │ ├── factors.py │ ├── feature_memory.py │ ├── file.py │ ├── girder.py │ ├── image.py │ ├── iter_validation.py │ ├── metrics.py │ ├── mimetype.py │ ├── mongo_sessions.py │ ├── parallel.py │ ├── plugin.py │ ├── postgres.py │ ├── preview_cache.py │ ├── probability.py │ ├── proxy_manager.py │ ├── read_write_lock.py │ ├── signal_handler.py │ ├── simple_timer.py │ ├── string.py │ ├── timed_cache.py │ ├── url.py │ ├── video.py │ └── web.py │ └── web │ ├── __init__.py │ ├── _plugins.py │ ├── classifier_service │ ├── README.rst │ ├── __init__.py │ ├── api.rst │ ├── classifier_server.py │ └── example │ │ ├── .gitignore │ │ ├── README.md │ │ ├── classifier_test.ipynb │ │ ├── dummy_classifier.pkl.b64 │ │ ├── dummy_classifier.py │ │ ├── dummy_descriptor_generator.py │ │ ├── fish-bike.jpg.b64 │ │ ├── test_classifier_responses.txt │ │ └── test_classifier_server_curl.sh │ ├── descriptor_service │ ├── __init__.py │ └── example_config.json │ ├── iqr_service │ ├── __init__.py │ ├── api.rst │ ├── iqr_server.py │ └── use_case_example.rst │ ├── nearestneighbor_service │ ├── __init__.py │ ├── example_config.json │ └── service.py │ └── search_app │ ├── README.rst │ ├── __init__.py │ ├── modules │ ├── __init__.py │ ├── file_upload │ │ ├── FileUploadMod.py │ │ ├── __init__.py │ │ └── static │ │ │ ├── css │ │ │ └── smqtk_upload.css │ │ │ └── js │ │ │ ├── flow.js │ │ │ ├── flow.min.js │ │ │ └── smqtk_upload.js │ ├── iqr │ │ ├── __init__.py │ │ ├── iqr_search.py │ │ ├── static │ │ │ ├── css │ │ │ │ └── smqtk.iqr.css │ │ │ ├── img │ │ │ │ ├── carbon-rejected.png │ │ │ │ ├── carbon-rejected_on.png │ │ │ │ ├── carbon-verified.png │ │ │ │ ├── carbon-verified_on.png │ │ │ │ ├── explicit_marker_off.png │ │ │ │ ├── explicit_marker_on.png │ │ │ │ ├── explicit_overlay.png │ │ │ │ └── loading.gif │ │ │ └── js │ │ │ │ ├── FileSaver.LICENSE.md │ │ │ │ ├── FileSaver.js │ │ │ │ ├── FileSaver.min.js │ │ │ │ ├── smqtk.data_view.js │ │ │ │ ├── smqtk.iqr_refine_view.js │ │ │ │ ├── smqtk.iqr_status_view.js │ │ │ │ ├── smqtk.iqr_view.js │ │ │ │ └── smqtk.progress_bars.js │ │ └── templates │ │ │ ├── base.html │ │ │ └── iqr_search_index.html │ ├── login │ │ ├── LoginMod.py │ │ ├── __init__.py │ │ ├── templates │ │ │ └── login.html │ │ └── users.json │ └── static_host.py │ ├── sample_configs │ ├── config.IqrRestService.json │ ├── config.IqrSearchApp.json │ ├── config.classifyFiles.json │ └── config.iqrTrainClassifier.json │ ├── static │ ├── css │ │ ├── bootstrap-theme.css │ │ ├── bootstrap-theme.min.css │ │ ├── bootstrap.css │ │ ├── bootstrap.min.css │ │ ├── clip.view.css │ │ ├── images │ │ │ ├── ui-bg_diagonals-thick_18_b81900_40x40.png │ │ │ ├── ui-bg_diagonals-thick_20_666666_40x40.png │ │ │ ├── ui-bg_flat_10_000000_40x100.png │ │ │ ├── ui-bg_glass_100_f6f6f6_1x400.png │ │ │ ├── ui-bg_glass_100_fdf5ce_1x400.png │ │ │ ├── ui-bg_glass_65_ffffff_1x400.png │ │ │ ├── ui-bg_gloss-wave_35_f6a828_500x100.png │ │ │ ├── ui-bg_highlight-soft_100_eeeeee_1x100.png │ │ │ ├── ui-bg_highlight-soft_75_ffe45c_1x100.png │ │ │ ├── ui-icons_222222_256x240.png │ │ │ ├── ui-icons_228ef1_256x240.png │ │ │ ├── ui-icons_ef8c08_256x240.png │ │ │ ├── ui-icons_ffd27a_256x240.png │ │ │ └── ui-icons_ffffff_256x240.png │ │ ├── jquery-ui.css │ │ ├── jquery-ui.min.css │ │ ├── jquery-ui.structure.css │ │ ├── jquery-ui.structure.min.css │ │ ├── jquery-ui.theme.css │ │ └── jquery-ui.theme.min.css │ ├── img │ │ ├── Kitwarelogo-gradation.png │ │ ├── cancel.png │ │ ├── done.png │ │ ├── pause.png │ │ └── resume.png │ └── js │ │ ├── bootbox.js │ │ ├── bootbox.min.js │ │ ├── bootstrap.js │ │ ├── bootstrap.min.js │ │ ├── jquery-1.11.1.js │ │ ├── jquery-1.11.1.min.js │ │ ├── jquery-ui.js │ │ ├── jquery-ui.min.js │ │ ├── jquery.timer.js │ │ ├── setZeroTimeout.js │ │ ├── smqtk.alerts.js │ │ └── smqtk.vars.js │ └── templates │ ├── base.html │ └── index.html ├── requirements.txt ├── requirements ├── ci.txt ├── docs.txt ├── optional.txt ├── runtime.txt └── test.txt ├── scripts ├── build_iqr_models.sh ├── memex │ ├── README.md │ ├── hackathon_2016_04 │ │ ├── ad-images.split-train-test.py │ │ ├── save_ad_images.py │ │ ├── save_phone_images.py │ │ └── score_phone_numbers.py │ ├── hackathon_2016_07 │ │ └── cp1 │ │ │ ├── ad_image_classification │ │ │ ├── 01.train_test_segmentation.ipynb │ │ │ ├── 02.train_tiered_classifiers.ipynb │ │ │ ├── 03.image_classifier_based_scoring.ipynb │ │ │ └── cnn_finetuning │ │ │ │ ├── 01.format_train_test_files.ipynb │ │ │ │ ├── alexnet │ │ │ │ ├── LICENSE │ │ │ │ ├── README.txt │ │ │ │ ├── deploy.prototxt │ │ │ │ ├── solver.prototxt │ │ │ │ └── train_val.prototxt │ │ │ │ ├── labels.txt │ │ │ │ └── score_eval_data.py │ │ │ ├── data_retreival │ │ │ ├── notebooks │ │ │ │ └── CP1 Data Curation.ipynb │ │ │ └── scripts │ │ │ │ ├── download_url_as_sha.py │ │ │ │ ├── es_config.example.json │ │ │ │ └── get_es_child_documents.py │ │ │ └── tensorflow_retraining │ │ │ ├── README.md │ │ │ ├── aggregate.py │ │ │ ├── evaluate.py │ │ │ ├── retrain.py │ │ │ └── split.py │ ├── live_update_pipeline │ │ ├── get_cdr_images.py │ │ ├── list_ido_solr_images.py │ │ ├── run.update.sh │ │ ├── update_process.md │ │ └── watch_cdr_get.sh │ └── machine_run_scripts │ │ ├── dyndns │ │ └── process_roxyimages.csift.py │ │ └── hbase_gpu_machine │ │ ├── cnn_config.prototxt.tmpl │ │ ├── compute.py │ │ └── transfer_file_descriptors_to_psql.py └── sample_build_configs │ ├── compute_hash_codes.json │ ├── compute_many_descriptors.json │ ├── generate_image_transform.tiles.json │ ├── make_balltree.json │ └── train_itq.json ├── setup.py ├── setup_env.build.sh.in ├── setup_env.install.sh.in ├── src ├── CMakeLists.txt ├── cnn_feature_extractor │ ├── CMakeLists.txt │ ├── README.txt │ └── cnn_feature_extractor.cxx └── frame_extractor │ ├── CMakeLists.txt │ ├── README.txt │ └── frame_extractor.cxx └── tests ├── __init__.py ├── algorithms ├── __init__.py ├── classifier │ ├── __init__.py │ ├── test_ClassifierAbstract.py │ ├── test_ClassifierCollection.py │ ├── test_IndexLabelClassifier.py │ ├── test_SupervisedClassifierAbstract.py │ ├── test_libsvm.py │ └── test_sklearn_logistic_regression.py ├── descriptor_generator │ ├── __init__.py │ ├── test_DG_abstract.py │ ├── test_caffe.py │ └── test_colordescriptor.py ├── image_io │ ├── __init__.py │ ├── test_gdal_io.py │ ├── test_interface_ImageReader.py │ └── test_pil_io.py ├── nn_index │ ├── __init__.py │ ├── hash_index │ │ ├── __init__.py │ │ ├── test_abstract.py │ │ ├── test_balltree.py │ │ └── test_linear.py │ ├── lsh_hash_functors │ │ ├── __init__.py │ │ ├── test_abstract.py │ │ └── test_itq.py │ ├── test_NNI_FAISS.py │ ├── test_NNI_FLANN.py │ ├── test_NNI_MRPT.py │ ├── test_NNI_abstract.py │ └── test_NNI_lsh.py ├── object_detection │ ├── __init__.py │ ├── test_interface_ImageMatrixObjectDetector.py │ └── test_interface_ObjectDetection.py ├── rank_relevancy │ ├── __init__.py │ ├── test_RR_abstract.py │ ├── test_margin_sampling.py │ └── test_wrap_sc.py └── relevancy_index │ ├── __init__.py │ ├── test_RI_abstract.py │ ├── test_sc_wrapper.py │ └── test_svm_hik.py ├── data ├── Hopper.alexnet_fc7_output.npy ├── Lenna.alexnet_fc7_output.npy ├── Lenna.png ├── caffe.dummpy_network.prototxt ├── caffe.dummy_mean.npy ├── caffe.empty_model.caffemodel ├── grace_hopper.100x100+100+100.png ├── grace_hopper.png ├── test_data_file_set_tree │ ├── 0 │ │ ├── 0 │ │ │ ├── UUID_000.dataElement │ │ │ ├── UUID_001.dataElement │ │ │ └── UUID_003.dataelemenT │ │ └── 1 │ │ │ └── UUID_012.dataElement │ ├── 1 │ │ ├── 3 │ │ │ └── something_else │ │ └── 8 │ │ │ └── UUID_180.dataElement │ ├── 3 │ │ └── 1 │ │ │ └── UUID_317.dataElement │ ├── 4 │ │ └── 3 │ │ │ └── 2 │ │ │ └── 1 │ │ │ └── UUID_43210.dataElement │ ├── UUID_0.dataElement │ ├── UUID_1.dataElement │ ├── UUID_2.dataElement │ └── nonmatching_filename ├── test_file.dat ├── test_labels.txt └── text_file ├── iqr ├── __init__.py └── test_iqr_session.py ├── representation ├── ClassificationElement │ ├── __init__.py │ ├── test_CE_abstract.py │ ├── test_CE_file.py │ └── test_CE_memory.py ├── DataElement │ ├── __init__.py │ ├── test_DataElement_abstract.py │ ├── test_DataFileElement.py │ ├── test_DataMemoryElement.py │ ├── test_DataUrlElement.py │ ├── test_GirderDataElement.py │ ├── test_HBaseDataElement.py │ ├── test_MatrixDataElement.py │ └── test_from_uri.py ├── DataSet │ ├── __init__.py │ ├── test_DataSet_abstract.py │ ├── test_FileSet.py │ ├── test_MemorySet.py │ └── test_kvstore_backed.py ├── DescriptorElement │ ├── __init__.py │ ├── test_DescriptorElement_abstract.py │ ├── test_DescriptorFileElement.py │ ├── test_DescriptorMemoryElement.py │ └── test_DescriptorSolrElement.py ├── DescriptorSet │ ├── __init__.py │ ├── test_DS_abstract.py │ └── test_DS_memory.py ├── DetectionElement │ ├── __init__.py │ ├── test_interface.py │ └── test_memory.py ├── KeyValueStore │ ├── __init__.py │ ├── test_KeyValueStoreAbstract.py │ ├── test_MemoryKeyValueStore.py │ └── test_PostgresKeyValueStore.py ├── __init__.py ├── test_DescriptorElementFactory.py ├── test_DetectionElementFactory.py └── test_bounding_box.py ├── test_cf_compute_descriptors.py ├── test_cf_mbkmeans_clustering.py ├── utils ├── __init__.py ├── file_utils │ ├── __init__.py │ ├── test_file_mimetype_filemagic.py │ ├── test_file_mimetype_tika.py │ ├── test_safe_create_dir.py │ └── test_safe_file_write.py ├── test_adjust_proba.py ├── test_bit_utils.py ├── test_configuration.py ├── test_factors.py ├── test_image_utils.py ├── test_merge_dict.py ├── test_metrics.py ├── test_parallel.py ├── test_plugin.py ├── test_plugin_dir │ ├── __init__.py │ ├── module_of_more_stuff.py │ ├── module_of_stuff.py │ └── module_with_exception.py ├── test_postgres.py ├── test_rwlock.py ├── test_string_utils.py ├── test_url_tools.py └── test_web.py └── web ├── __init__.py ├── classifier_service ├── __init__.py ├── dummy_classifier.py ├── dummy_descriptor_generator.py └── test_classifier_server.py └── iqr_service ├── __init__.py ├── stubs.py └── test_iqr_service.py /.dockerignore: -------------------------------------------------------------------------------- 1 | /.git 2 | 3 | ### JetBrains template 4 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 5 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 6 | 7 | # User-specific stuff: 8 | .idea/**/workspace.xml 9 | .idea/**/tasks.xml 10 | .idea/dictionaries 11 | 12 | # Sensitive or high-churn files: 13 | .idea/**/dataSources/ 14 | .idea/**/dataSources.ids 15 | .idea/**/dataSources.xml 16 | .idea/**/dataSources.local.xml 17 | .idea/**/sqlDataSources.xml 18 | .idea/**/dynamic.xml 19 | .idea/**/uiDesigner.xml 20 | 21 | # Gradle: 22 | .idea/**/gradle.xml 23 | .idea/**/libraries 24 | 25 | # CMake 26 | cmake-build-debug/ 27 | cmake-build-release/ 28 | 29 | # Mongo Explorer plugin: 30 | .idea/**/mongoSettings.xml 31 | 32 | ## File-based project format: 33 | *.iws 34 | 35 | ## Plugin-specific files: 36 | 37 | # IntelliJ 38 | out/ 39 | 40 | # mpeltonen/sbt-idea plugin 41 | .idea_modules/ 42 | 43 | # JIRA plugin 44 | atlassian-ide-plugin.xml 45 | 46 | # Cursive Clojure plugin 47 | .idea/replstate.xml 48 | 49 | # Crashlytics plugin (for Android Studio and IntelliJ) 50 | com_crashlytics_export_strings.xml 51 | crashlytics.properties 52 | crashlytics-build.properties 53 | fabric.properties 54 | ### CMake template 55 | CMakeCache.txt 56 | CMakeFiles 57 | CMakeScripts 58 | Testing 59 | Makefile 60 | cmake_install.cmake 61 | install_manifest.txt 62 | compile_commands.json 63 | CTestTestfile.cmake 64 | 65 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = 3 | *.ipynb_checkpoints, 4 | __pycache__, 5 | .eggs, 6 | docs, 7 | scripts, 8 | TPL 9 | 10 | # While a max length of 79 is encouraged, sometimes going over makes more sense 11 | # for overall clarity. 12 | max-line-length = 120 13 | -------------------------------------------------------------------------------- /.pytest.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = tests/* 3 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | formats: 3 | - pdf 4 | python: 5 | version: 3.6 6 | install: 7 | - requirements: docs/readthedocs-reqs.txt 8 | - method: pip 9 | path: . 10 | sphinx: 11 | builder: html 12 | configuration: docs/conf.py 13 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2014-2019 Kitware, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither name of Kitware, Inc. nor the names of any contributors may be used 15 | to endorse or promote products derived from this software without specific 16 | prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTING.md 2 | include LICENSE.txt 3 | include README.md 4 | 5 | # Grab everything in ./etc/ 6 | recursive-include etc * 7 | -------------------------------------------------------------------------------- /TPL/README: -------------------------------------------------------------------------------- 1 | The current libSVM package provided here is based on version 3.1, however we 2 | have made modifications in order to extend functionality. 3 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/COPYING: -------------------------------------------------------------------------------- 1 | 2 | The BSD License 3 | 4 | Copyright (c) 2008-2011 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | Copyright (c) 2008-2011 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 11 | * Neither the name of the "University of British Columbia" nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 15 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/ChangeLog: -------------------------------------------------------------------------------- 1 | Version 1.6.11 2 | * bug fixes 3 | 4 | Version 1.6.10 5 | * fixed a radiusSearch bug introduced in 1.6.9 6 | 7 | Version 1.6.9 8 | * bug fixes 9 | * fixed radius search bug on MSVC compiler 10 | * fixed windows linking problems 11 | 12 | Version 1.6.8 13 | * bug fixes, low dimensional search speedup 14 | 15 | Version 1.6.7 16 | * bug fixes 17 | 18 | Version 1.6.6 19 | * misc bug fixes 20 | 21 | Version 1.6.5 22 | * fix compilation problem on some C++ compilers 23 | * fixes in the python bindings 24 | 25 | Version 1.6.4 26 | * small bug fix 27 | 28 | Version 1.6.3 29 | * radius search speedup 30 | 31 | Version 1.6.2 32 | * slight API changes to the C++ bindings, now the main index type is templated 33 | * on the distance functor which makes it easier to use custom distances 34 | * new kd-tree implementation optimized for low dimensionality data 35 | * experimental MPI support for cluster computing 36 | 37 | Version 1.5 38 | * new C++ templated API 39 | * saving/loading of indices to disk 40 | * threadsafe search 41 | * new distance types (thanks to Radu Bogdan Rusu and Romain Thibaux for the patch) 42 | * (api change) autotuned is no longer selected by passing a precision >0, it's used when the algorithm type is set to autotuned 43 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/bin/indent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=`dirname $0` 4 | uncrustify --no-backup -c ${DIR}/uncrustify.cfg $1 5 | 6 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/bin/make_release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | VERSION=`grep "set(FLANN_VERSION" CMakeLists.txt | sed 's/[^0-9]*\([0-9]*\.[0-9]*\.[0-9]*\)[^0-9]*/\1/'` 3 | 4 | echo "Creating flann-$VERSION-src.zip" 5 | 6 | git archive --prefix=flann-$VERSION-src/ -o flann-$VERSION-src.zip $VERSION-src 7 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/bin/run_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | 4 | import sys 5 | import os.path as op 6 | import unittest 7 | 8 | if __name__ == "__main__": 9 | if len(sys.argv)==1: 10 | print "Usage: %s file"%sys.argv[0] 11 | sys.exit(1) 12 | 13 | python_path = op.abspath(op.join( op.dirname(__file__),"..","src","python")) 14 | sys.path.append(python_path) 15 | 16 | test_file = sys.argv[1] 17 | sys.argv = sys.argv[1:] 18 | execfile(test_file) 19 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/cmake/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(PKG_DESC "Fast Library for Approximate Nearest Neighbors") 2 | set(pkg_conf_file ${CMAKE_CURRENT_BINARY_DIR}/flann.pc) 3 | configure_file(flann.pc.in ${pkg_conf_file} @ONLY) 4 | install(FILES ${pkg_conf_file} 5 | DESTINATION ${FLANN_LIB_INSTALL_DIR}/pkgconfig/ COMPONENT pkgconfig) 6 | 7 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/cmake/FindFlann.cmake: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Find Flann 3 | # 4 | # This sets the following variables: 5 | # FLANN_FOUND - True if FLANN was found. 6 | # FLANN_INCLUDE_DIRS - Directories containing the FLANN include files. 7 | # FLANN_LIBRARIES - Libraries needed to use FLANN. 8 | # FLANN_DEFINITIONS - Compiler flags for FLANN. 9 | 10 | find_package(PkgConfig) 11 | pkg_check_modules(PC_FLANN flann) 12 | set(FLANN_DEFINITIONS ${PC_FLANN_CFLAGS_OTHER}) 13 | 14 | find_path(FLANN_INCLUDE_DIR flann/flann.hpp 15 | HINTS ${PC_FLANN_INCLUDEDIR} ${PC_FLANN_INCLUDE_DIRS}) 16 | 17 | find_library(FLANN_LIBRARY flann 18 | HINTS ${PC_FLANN_LIBDIR} ${PC_FLANN_LIBRARY_DIRS}) 19 | 20 | set(FLANN_INCLUDE_DIRS ${FLANN_INCLUDE_DIR}) 21 | set(FLANN_LIBRARIES ${FLANN_LIBRARY}) 22 | 23 | include(FindPackageHandleStandardArgs) 24 | find_package_handle_standard_args(Flann DEFAULT_MSG 25 | FLANN_LIBRARY FLANN_INCLUDE_DIR) 26 | 27 | mark_as_advanced(FLANN_LIBRARY FLANN_INCLUDE_DIR) 28 | 29 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/cmake/flann.pc.in: -------------------------------------------------------------------------------- 1 | # This file was generated by CMake for @PROJECT_NAME@ 2 | prefix=@CMAKE_INSTALL_PREFIX@ 3 | exec_prefix=${prefix} 4 | libdir=${prefix}/@FLANN_LIB_INSTALL_DIR@ 5 | includedir=${prefix}/include 6 | 7 | Name: @PROJECT_NAME@ 8 | Description: @PKG_DESC@ 9 | Version: @FLANN_VERSION@ 10 | Requires: @PKG_EXTERNAL_DEPS@ 11 | Libs: -L${libdir} -lflann_cpp 12 | Cflags: -I${includedir} 13 | 14 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/cmake/uninstall_target.cmake.in: -------------------------------------------------------------------------------- 1 | if(NOT EXISTS "@PROJECT_BINARY_DIR@/install_manifest.txt") 2 | message(FATAL_ERROR "Cannot find install manifest: \"@PROJECT_BINARY_DIR@/install_manifest.txt\"") 3 | endif(NOT EXISTS "@PROJECT_BINARY_DIR@/install_manifest.txt") 4 | 5 | file(READ "@PROJECT_BINARY_DIR@/install_manifest.txt" files) 6 | string(REGEX REPLACE "\n" ";" files "${files}") 7 | foreach(file ${files}) 8 | message(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") 9 | if(EXISTS "$ENV{DESTDIR}${file}") 10 | exec_program("@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" 11 | OUTPUT_VARIABLE rm_out RETURN_VALUE rm_retval) 12 | if(NOT "${rm_retval}" STREQUAL 0) 13 | message(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"") 14 | endif(NOT "${rm_retval}" STREQUAL 0) 15 | else(EXISTS "$ENV{DESTDIR}${file}") 16 | message(STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.") 17 | endif(EXISTS "$ENV{DESTDIR}${file}") 18 | endforeach(file) 19 | 20 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/doc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(LATEX) 2 | 3 | if (EXISTS ${PDFLATEX_COMPILER} AND EXISTS ${BIBTEX_COMPILER}) 4 | include(${PROJECT_SOURCE_DIR}/cmake/UseLATEX.cmake) 5 | 6 | add_latex_document(manual.tex BIBFILES references.bib IMAGE_DIRS images DEFAULT_PDF) 7 | 8 | add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/manual.pdf 9 | COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/manual.pdf ${CMAKE_CURRENT_SOURCE_DIR}/manual.pdf 10 | DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/manual.pdf 11 | ) 12 | add_custom_target(doc DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/manual.pdf) 13 | endif() 14 | 15 | install( 16 | FILES manual.pdf 17 | DESTINATION share/doc/flann 18 | OPTIONAL 19 | ) 20 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/doc/images/cmake-gui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/flann-1.8.4/doc/images/cmake-gui.png -------------------------------------------------------------------------------- /TPL/flann-1.8.4/examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_custom_target(examples ALL) 2 | 3 | 4 | if (BUILD_C_BINDINGS) 5 | add_executable(flann_example_c flann_example.c) 6 | target_link_libraries(flann_example_c flann) 7 | set_target_properties(flann_example_c PROPERTIES COMPILE_FLAGS -std=c99) 8 | 9 | add_dependencies(examples flann_example_c) 10 | install (TARGETS flann_example_c DESTINATION bin ) 11 | endif() 12 | 13 | if (HDF5_FOUND) 14 | include_directories(${HDF5_INCLUDE_DIR}) 15 | 16 | add_executable(flann_example_cpp flann_example.cpp) 17 | target_link_libraries(flann_example_cpp ${HDF5_LIBRARIES}) 18 | if (HDF5_IS_PARALLEL) 19 | target_link_libraries(flann_example_cpp ${MPI_LIBRARIES}) 20 | endif() 21 | 22 | add_dependencies(examples flann_example_cpp) 23 | install (TARGETS flann_example_cpp DESTINATION bin) 24 | 25 | 26 | if (USE_MPI AND HDF5_IS_PARALLEL) 27 | add_executable(flann_example_mpi flann_example_mpi.cpp) 28 | target_link_libraries(flann_example_mpi flann_cpp ${HDF5_LIBRARIES} ${MPI_LIBRARIES} ${Boost_LIBRARIES}) 29 | 30 | add_dependencies(examples flann_example_mpi) 31 | install (TARGETS flann_example_mpi DESTINATION bin) 32 | endif() 33 | else() 34 | message("hdf5 library not found, not compiling flann_example.cpp") 35 | endif() 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/examples/README: -------------------------------------------------------------------------------- 1 | 2 | These examples use some datasets that are not included in the source distribution. You can download the datasets from here: 3 | 4 | http://people.cs.ubc.ca/~mariusm/uploads/FLANN/datasets/dataset.hdf5 5 | http://people.cs.ubc.ca/~mariusm/uploads/FLANN/datasets/dataset.dat 6 | http://people.cs.ubc.ca/~mariusm/uploads/FLANN/datasets/testset.dat 7 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/examples/flann_example.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | using namespace flann; 8 | 9 | int main(int argc, char** argv) 10 | { 11 | int nn = 3; 12 | 13 | Matrix dataset; 14 | Matrix query; 15 | load_from_file(dataset, "dataset.hdf5","dataset"); 16 | load_from_file(query, "dataset.hdf5","query"); 17 | 18 | Matrix indices(new int[query.rows*nn], query.rows, nn); 19 | Matrix dists(new float[query.rows*nn], query.rows, nn); 20 | 21 | // construct an randomized kd-tree index using 4 kd-trees 22 | Index > index(dataset, flann::KDTreeIndexParams(4)); 23 | index.buildIndex(); 24 | 25 | // do a knn search, using 128 checks 26 | index.knnSearch(query, indices, dists, nn, flann::SearchParams(128)); 27 | 28 | flann::save_to_file(indices,"result.hdf5","result"); 29 | 30 | delete[] dataset.ptr(); 31 | delete[] query.ptr(); 32 | delete[] indices.ptr(); 33 | delete[] dists.ptr(); 34 | 35 | return 0; 36 | } 37 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_subdirectory( cpp ) 3 | 4 | if (BUILD_MATLAB_BINDINGS) 5 | add_subdirectory( matlab ) 6 | endif() 7 | 8 | if (BUILD_PYTHON_BINDINGS) 9 | add_subdirectory( python ) 10 | endif() 11 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/src/cpp/flann/mpi/flann_mpi_server.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, char* argv[]) 7 | { 8 | boost::mpi::environment env(argc, argv); 9 | 10 | try { 11 | if (argc != 4) { 12 | std::cout << "Usage: " << argv[0] << " \n"; 13 | return 1; 14 | } 15 | flann::mpi::Server > server(argv[1], argv[2], std::atoi(argv[3]), 16 | flann::KDTreeIndexParams(4)); 17 | 18 | server.run(); 19 | } 20 | catch (std::exception& e) { 21 | std::cerr << "Exception: " << e.what() << "\n"; 22 | } 23 | 24 | return 0; 25 | } 26 | 27 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/src/python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | configure_file( setup.py.tpl setup.py ) 2 | 3 | install( DIRECTORY pyflann DESTINATION share/flann/python ) 4 | install( FILES ${CMAKE_CURRENT_BINARY_DIR}/setup.py DESTINATION share/flann/python ) 5 | 6 | 7 | # python instalation 8 | if (PYTHON_EXECUTABLE) 9 | install(CODE "execute_process( 10 | COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/setup.py install 11 | WORKING_DIRECTORY \"${CMAKE_CURRENT_SOURCE_DIR}\")") 12 | endif() 13 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/src/python/pyflann/__init__.py: -------------------------------------------------------------------------------- 1 | #Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 2 | #Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 3 | # 4 | #THE BSD LICENSE 5 | # 6 | #Redistribution and use in source and binary forms, with or without 7 | #modification, are permitted provided that the following conditions 8 | #are met: 9 | # 10 | #1. Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | #2. Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | #THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 | #IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 | #OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 | #IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 | #INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 | #NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | #DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | #THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | #(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 | #THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | #import sys 28 | #import os 29 | #sys.path.insert(0, os.path.split(__file__)[0]) # make python3 happy 30 | 31 | from pyflann.index import * 32 | -------------------------------------------------------------------------------- /TPL/flann-1.8.4/src/python/setup.py.tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | from distutils.core import setup 4 | from os.path import exists, abspath, dirname, join 5 | import os 6 | import sys 7 | 8 | 9 | def find_path(): 10 | lib_paths = [ os.path.abspath('@LIBRARY_OUTPUT_PATH@'), abspath(join(dirname(dirname(sys.argv[0])), '../../../lib')) ] 11 | possible_libs = ['libflann.so', 'flann.dll', 'libflann.dll', 'libflann.dylib'] 12 | 13 | for path in lib_paths: 14 | for lib in possible_libs: 15 | if exists(join(path,lib)): 16 | return path 17 | 18 | setup(name='flann', 19 | version='@FLANN_VERSION@', 20 | description='Fast Library for Approximate Nearest Neighbors', 21 | author='Marius Muja', 22 | author_email='mariusm@cs.ubc.ca', 23 | license='BSD', 24 | url='http://www.cs.ubc.ca/~mariusm/flann/', 25 | packages=['pyflann', 'pyflann.lib'], 26 | package_dir={'pyflann.lib': find_path() }, 27 | package_data={'pyflann.lib': ['libflann.so', 'flann.dll', 'libflann.dll', 'libflann.dylib']}, 28 | ) 29 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/COPYRIGHT: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) 2000-2011 Chih-Chung Chang and Chih-Jen Lin 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | 16 | 3. Neither name of copyright holders nor the names of its contributors 17 | may be used to endorse or promote products derived from this software 18 | without specific prior written permission. 19 | 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 25 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 28 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 29 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 30 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/Makefile: -------------------------------------------------------------------------------- 1 | CXX ?= g++ 2 | CFLAGS = -Wall -Wconversion -O3 -fPIC 3 | SHVER = 2 4 | 5 | all: svm-train svm-predict svm-scale svm-prob binary-train binary-predict 6 | 7 | lib: svm.o 8 | $(CXX) -shared -dynamiclib svm.o -o libsvm.so.$(SHVER) 9 | 10 | svm-predict: svm-predict.c svm.o 11 | $(CXX) $(CFLAGS) svm-predict.c svm.o -o svm-predict -lm 12 | svm-train: svm-train.c svm.o 13 | $(CXX) $(CFLAGS) svm-train.c svm.o -o svm-train -lm 14 | svm-scale: svm-scale.c 15 | $(CXX) $(CFLAGS) svm-scale.c -o svm-scale 16 | svm-prob: svm-prob.c 17 | $(CXX) $(CFLAGS) svm-prob.c -o svm-prob 18 | svm.o: svm.cpp svm.h 19 | $(CXX) $(CFLAGS) -c svm.cpp 20 | 21 | eval.o: eval.cpp eval.h svm.h 22 | $(CXX) $(CFLAGS) -c eval.cpp -o eval.o 23 | 24 | binary-predict: binary-predict.c eval.o svm.o 25 | $(CXX) $(CFLAGS) binary-predict.c eval.o svm.o -o binary-predict -lm 26 | binary-train: binary-train.c eval.o svm.o 27 | $(CXX) $(CFLAGS) binary-train.c eval.o svm.o -o binary-train -lm 28 | 29 | 30 | clean: 31 | rm -f *~ svm.o eval.o svm-train svm-predict svm-scale svm-prob binary-train binary-predict libsvm.so.$(SHVER) 32 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/Makefile.original: -------------------------------------------------------------------------------- 1 | CXX ?= g++ 2 | CFLAGS = -Wall -Wconversion -O3 -fPIC 3 | SHVER = 2 4 | 5 | all: svm-train svm-predict svm-scale 6 | 7 | lib: svm.o 8 | $(CXX) -shared -dynamiclib svm.o -o libsvm.so.$(SHVER) 9 | 10 | svm-predict: svm-predict.c svm.o 11 | $(CXX) $(CFLAGS) svm-predict.c svm.o -o svm-predict -lm 12 | svm-train: svm-train.c svm.o 13 | $(CXX) $(CFLAGS) svm-train.c svm.o -o svm-train -lm 14 | svm-scale: svm-scale.c 15 | $(CXX) $(CFLAGS) svm-scale.c -o svm-scale 16 | svm.o: svm.cpp svm.h 17 | $(CXX) $(CFLAGS) -c svm.cpp 18 | clean: 19 | rm -f *~ svm.o svm-train svm-predict svm-scale libsvm.so.$(SHVER) 20 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/Makefile.win: -------------------------------------------------------------------------------- 1 | #You must ensure nmake.exe, cl.exe, link.exe are in system path. 2 | #VCVARS32.bat 3 | #Under dosbox prompt 4 | #nmake -f Makefile.win 5 | 6 | ########################################## 7 | CXX = cl.exe 8 | CFLAGS = -nologo -O2 -EHsc -I. -D __WIN32__ -D _CRT_SECURE_NO_DEPRECATE 9 | TARGET = windows 10 | 11 | all: $(TARGET)\svm-train.exe $(TARGET)\svm-predict.exe $(TARGET)\svm-scale.exe $(TARGET)\svm-toy.exe lib 12 | 13 | $(TARGET)\svm-predict.exe: svm.h svm-predict.c svm.obj 14 | $(CXX) $(CFLAGS) svm-predict.c svm.obj -Fe$(TARGET)\svm-predict.exe 15 | 16 | $(TARGET)\svm-train.exe: svm.h svm-train.c svm.obj 17 | $(CXX) $(CFLAGS) svm-train.c svm.obj -Fe$(TARGET)\svm-train.exe 18 | 19 | $(TARGET)\svm-scale.exe: svm.h svm-scale.c 20 | $(CXX) $(CFLAGS) svm-scale.c -Fe$(TARGET)\svm-scale.exe 21 | 22 | $(TARGET)\svm-toy.exe: svm.h svm.obj svm-toy\windows\svm-toy.cpp 23 | $(CXX) $(CFLAGS) svm-toy\windows\svm-toy.cpp svm.obj user32.lib gdi32.lib comdlg32.lib -Fe$(TARGET)\svm-toy.exe 24 | 25 | svm.obj: svm.cpp svm.h 26 | $(CXX) $(CFLAGS) -c svm.cpp 27 | 28 | lib: svm.cpp svm.h svm.def 29 | $(CXX) $(CFLAGS) -LD svm.cpp -Fe$(TARGET)\libsvm -link -DEF:svm.def 30 | 31 | clean: 32 | -erase /Q *.obj $(TARGET)\. 33 | 34 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/README-Kitware.md: -------------------------------------------------------------------------------- 1 | This is a custom libSVM repository as provided by Harry Sun. -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/convert.c: -------------------------------------------------------------------------------- 1 | 2 | /* convert cvs data to libsvm/svm-light format */ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | char buf[10000000]; 9 | float feature[100000]; 10 | 11 | int main(int argc, char **argv) 12 | { 13 | FILE *fp; 14 | 15 | if(argc!=2) { fprintf(stderr,"Usage %s filename\n",argv[0]); } 16 | if((fp=fopen(argv[1],"r"))==NULL) 17 | { 18 | fprintf(stderr,"Can't open input file %s\n",argv[1]); 19 | } 20 | 21 | while(fscanf(fp,"%[^\n]\n",buf)==1) 22 | { 23 | int i=0,j; 24 | char *p=strtok(buf,","); 25 | 26 | feature[i++]=atof(p); 27 | 28 | while((p=strtok(NULL,","))) 29 | feature[i++]=atof(p); 30 | 31 | // --i; 32 | /* 33 | if ((int) feature[i]==1) 34 | printf("-1 "); 35 | else 36 | printf("+1 "); 37 | */ 38 | // printf("%f ", feature[1]); 39 | printf("%d ", (int) feature[0]); 40 | for(j=1;j 5 | #include "svm.h" 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | /* cross validation function */ 12 | double binary_class_cross_validation(const struct svm_problem *prob, const struct svm_parameter *param, int nr_fold); 13 | 14 | /* predict function */ 15 | void binary_class_predict(FILE *input, FILE *output); 16 | 17 | extern struct svm_model* model; 18 | void exit_input_error(int line_num); 19 | 20 | #ifdef __cplusplus 21 | } 22 | #endif 23 | 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/data/leu.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/data/leu.bz2 -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/data/leu.t.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/data/leu.t.bz2 -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/data/range1: -------------------------------------------------------------------------------- 1 | x 2 | -1 1 3 | 1 0 297.05 4 | 2 -4.555206 581.0731 5 | 3 -0.7524385 0.7170606 6 | 4 8.157474000000001 180 7 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/data/svmguide1.range: -------------------------------------------------------------------------------- 1 | x 2 | -1 1 3 | 1 0 297.05 4 | 2 -4.555206 581.0731 5 | 3 -0.7524385 0.7170606 6 | 4 8.157474000000001 180 7 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/data/svmguide1.scale.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/data/svmguide1.scale.png -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/data/svmguide4.scale.range: -------------------------------------------------------------------------------- 1 | x 2 | -1 1 3 | 1 0 1 4 | 2 0 1 5 | 3 0 1 6 | 4 0 1 7 | 5 0 1 8 | 6 0 1 9 | 7 0 1 10 | 8 0 1 11 | 9 0 1 12 | 10 0 1 13 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/eval.h: -------------------------------------------------------------------------------- 1 | #ifndef _EVAL_H 2 | #define _EVAL_H 3 | 4 | #include 5 | #include "svm.h" 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | /* cross validation function */ 12 | double binary_class_cross_validation(const struct svm_problem *prob, const struct svm_parameter *param, int nr_fold); 13 | 14 | /* predict function */ 15 | void binary_class_predict(FILE *input, FILE *output); 16 | 17 | extern struct svm_model* model; 18 | void exit_input_error(int line_num); 19 | 20 | #ifdef __cplusplus 21 | } 22 | #endif 23 | 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/Makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | CFLAGS=-c -O3 -Wall 3 | LDFLAGS=-O3 -Wall 4 | SOURCES=kernel_libSVMList.cpp libSVMListData.cpp kernelizerList.cpp 5 | OBJS=$(SOURCES:.cpp=.o) 6 | EXECUTABLE=kernel_libSVMList 7 | 8 | all: $(SOURCES) $(EXECUTABLE) 9 | 10 | $(EXECUTABLE): $(OBJS) 11 | $(CC) $(LDFLAGS) $(OBJS) -o $@ 12 | .cpp.o: 13 | $(CC) $(CFLAGS) $< -o $@ 14 | 15 | 16 | clean: 17 | rm -rf core *.o 18 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/class1_histintsc.cfg: -------------------------------------------------------------------------------- 1 | DATA_TRAIN ../dataset/div1/div1_train.libsvm_0 2 | DATA_TEST ../dataset/div1/div1_test.libsvm_0 3 | KERNELTYPE HIST_INTSC 4 | ALPHA NULL 5 | BETA NULL 6 | RHO NULL 7 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/kernel_libSVMList: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/hik/kernel_libSVMList -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/kernel_libSVMList.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/hik/kernel_libSVMList.o -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/kernelizerList.h: -------------------------------------------------------------------------------- 1 | #ifndef _KERNELIZERLIST_H 2 | #define _KERNELIZERLIST_H 3 | 4 | #include "libSVMListData.h" 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | 10 | class kernelizerList 11 | { 12 | public: 13 | kernelizerList(libSVMListData &trainData, libSVMListData &testData); 14 | ~kernelizerList(); 15 | int mTrainNum; 16 | int mTestNum; 17 | int mMaxFtrDim; 18 | int mKernelDim; 19 | void setParam(float alpha, float beta, float rho); 20 | void saveKernelMatrix_libSVMFmt(string trainFName, string testFName); 21 | void computeKernelMatrix(string kernelType); 22 | void scale(); 23 | private: 24 | list *mlTrainData; 25 | list *mlTestData; 26 | float **mTrainKernel; 27 | float **mTestKernel; 28 | int *mTrainLabel; 29 | int *mTestLabel; 30 | float mAlpha, mBeta, mRho; 31 | void memsetKernelMatrix(); 32 | void computeHistIntsc(); 33 | void computeLinear(); 34 | }; 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/kernelizerList.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/hik/kernelizerList.o -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/libSVMListData.h: -------------------------------------------------------------------------------- 1 | #ifndef _LIBSVMLISTDATA_H 2 | #define _LIBSVMLISTDATA_H 3 | 4 | #include 5 | #include 6 | #include "svmNode.h" 7 | 8 | using namespace std; 9 | 10 | class libSVMListData 11 | { 12 | public: 13 | libSVMListData(string dataFName); 14 | ~libSVMListData(); 15 | int mNumSample; 16 | int mMaxFtrDim; 17 | void saveData(string dataFName); 18 | list* getDataPointer() {return mDataList;}; 19 | int* getLabelPointer() {return mLabel;}; 20 | private: 21 | list *mDataList; 22 | int *mLabel; 23 | int countSample(string dataFName); 24 | void loadData(string dataFName); 25 | }; 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/libSVMListData.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/hik/libSVMListData.o -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/hik/svmNode.h: -------------------------------------------------------------------------------- 1 | #ifndef _SVMNODE_H 2 | #define _SVMNODE_H 3 | 4 | struct svmNode 5 | { 6 | int nodeNum; 7 | float value; 8 | }; 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/Makefile: -------------------------------------------------------------------------------- 1 | .SUFFIXES: .class .java 2 | FILES = libsvm/svm.class libsvm/svm_model.class libsvm/svm_node.class \ 3 | libsvm/svm_parameter.class libsvm/svm_problem.class \ 4 | libsvm/svm_print_interface.class \ 5 | svm_train.class svm_predict.class svm_toy.class svm_scale.class 6 | 7 | #JAVAC = jikes 8 | JAVAC_FLAGS = -target 1.5 -source 1.5 9 | JAVAC = javac 10 | # JAVAC_FLAGS = 11 | 12 | all: $(FILES) 13 | jar cvf libsvm.jar *.class libsvm/*.class 14 | 15 | .java.class: 16 | $(JAVAC) $(JAVAC_FLAGS) $< 17 | 18 | libsvm/svm.java: libsvm/svm.m4 19 | m4 libsvm/svm.m4 > libsvm/svm.java 20 | 21 | clean: 22 | rm -f libsvm/*.class *.class *.jar libsvm/*~ *~ libsvm/svm.java 23 | 24 | dist: clean all 25 | rm *.class libsvm/*.class 26 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/libsvm.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/TPL/libsvm-3.1-custom/java/libsvm.jar -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/libsvm/svm_model.java: -------------------------------------------------------------------------------- 1 | // 2 | // svm_model 3 | // 4 | package libsvm; 5 | public class svm_model implements java.io.Serializable 6 | { 7 | public svm_parameter param; // parameter 8 | public int nr_class; // number of classes, = 2 in regression/one class svm 9 | public int l; // total #SV 10 | public svm_node[][] SV; // SVs (SV[l]) 11 | public double[][] sv_coef; // coefficients for SVs in decision functions (sv_coef[k-1][l]) 12 | public double[] rho; // constants in decision functions (rho[k*(k-1)/2]) 13 | public double[] probA; // pariwise probability information 14 | public double[] probB; 15 | 16 | // for classification only 17 | 18 | public int[] label; // label of each class (label[k]) 19 | public int[] nSV; // number of SVs for each class (nSV[k]) 20 | // nSV[0] + nSV[1] + ... + nSV[k-1] = l 21 | }; 22 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/libsvm/svm_node.java: -------------------------------------------------------------------------------- 1 | package libsvm; 2 | public class svm_node implements java.io.Serializable 3 | { 4 | public int index; 5 | public double value; 6 | } 7 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/libsvm/svm_parameter.java: -------------------------------------------------------------------------------- 1 | package libsvm; 2 | public class svm_parameter implements Cloneable,java.io.Serializable 3 | { 4 | /* svm_type */ 5 | public static final int C_SVC = 0; 6 | public static final int NU_SVC = 1; 7 | public static final int ONE_CLASS = 2; 8 | public static final int EPSILON_SVR = 3; 9 | public static final int NU_SVR = 4; 10 | 11 | /* kernel_type */ 12 | public static final int LINEAR = 0; 13 | public static final int POLY = 1; 14 | public static final int RBF = 2; 15 | public static final int SIGMOID = 3; 16 | public static final int PRECOMPUTED = 4; 17 | 18 | public int svm_type; 19 | public int kernel_type; 20 | public int degree; // for poly 21 | public double gamma; // for poly/rbf/sigmoid 22 | public double coef0; // for poly/sigmoid 23 | 24 | // these are for training only 25 | public double cache_size; // in MB 26 | public double eps; // stopping criteria 27 | public double C; // for C_SVC, EPSILON_SVR and NU_SVR 28 | public int nr_weight; // for C_SVC 29 | public int[] weight_label; // for C_SVC 30 | public double[] weight; // for C_SVC 31 | public double nu; // for NU_SVC, ONE_CLASS, and NU_SVR 32 | public double p; // for EPSILON_SVR 33 | public int shrinking; // use the shrinking heuristics 34 | public int probability; // do probability estimates 35 | 36 | public Object clone() 37 | { 38 | try 39 | { 40 | return super.clone(); 41 | } catch (CloneNotSupportedException e) 42 | { 43 | return null; 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/libsvm/svm_print_interface.java: -------------------------------------------------------------------------------- 1 | package libsvm; 2 | public interface svm_print_interface 3 | { 4 | public void print(String s); 5 | } 6 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/libsvm/svm_problem.java: -------------------------------------------------------------------------------- 1 | package libsvm; 2 | public class svm_problem implements java.io.Serializable 3 | { 4 | public int l; 5 | public double[] y; 6 | public svm_node[][] x; 7 | } 8 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/java/test_applet.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/matlab/make.m: -------------------------------------------------------------------------------- 1 | % This make.m is used under Windows 2 | 3 | % add -largeArrayDims on 64-bit machines 4 | 5 | mex -O -largeArrayDims -I..\ -c ..\svm.cpp 6 | mex -O -largeArrayDims -I..\ -c svm_model_matlab.c 7 | mex -O -largeArrayDims -I..\ svmtrain.c svm.obj svm_model_matlab.obj 8 | mex -O -largeArrayDims -I..\ svmpredict.c svm.obj svm_model_matlab.obj 9 | mex -O -largeArrayDims libsvmread.c 10 | mex -O -largeArrayDims libsvmwrite.c 11 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/matlab/svm_model_matlab.h: -------------------------------------------------------------------------------- 1 | const char *model_to_matlab_structure(mxArray *plhs[], int num_of_feature, struct svm_model *model); 2 | struct svm_model *matlab_matrix_to_model(const mxArray *matlab_struct, const char **error_message); 3 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/python/Makefile: -------------------------------------------------------------------------------- 1 | all = lib 2 | 3 | lib: 4 | cd ..; make lib; cd - 5 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/python/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup( 4 | name='libsvm', 5 | version="3.1-custom", 6 | packages=setuptools.find_packages(), 7 | ) 8 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/svm-toy/gtk/Makefile: -------------------------------------------------------------------------------- 1 | CC? = gcc 2 | CXX? = g++ 3 | CFLAGS = -Wall -O3 -g `pkg-config --cflags gtk+-2.0` 4 | LIBS = `pkg-config --libs gtk+-2.0` 5 | 6 | svm-toy: main.o interface.o callbacks.o ../../svm.o 7 | $(CXX) $(CFLAGS) main.o interface.o callbacks.o ../../svm.o -o svm-toy $(LIBS) 8 | 9 | main.o: main.c 10 | $(CC) $(CFLAGS) -c main.c 11 | 12 | interface.o: interface.c interface.h 13 | $(CC) $(CFLAGS) -c interface.c 14 | 15 | callbacks.o: callbacks.cpp callbacks.h 16 | $(CXX) $(CFLAGS) -c callbacks.cpp 17 | 18 | ../../svm.o: 19 | cd ../..; make svm.o 20 | 21 | clean: 22 | rm -f *~ callbacks.o svm-toy main.o interface.o callbacks.o ../../svm.o 23 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/svm-toy/gtk/interface.h: -------------------------------------------------------------------------------- 1 | /* 2 | * DO NOT EDIT THIS FILE - it is generated by Glade. 3 | */ 4 | 5 | #ifdef __cplusplus 6 | extern "C" { 7 | #endif 8 | 9 | GtkWidget* create_window (void); 10 | GtkWidget* create_fileselection (void); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/svm-toy/gtk/main.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Initial main.c file generated by Glade. Edit as required. 3 | * Glade will not overwrite this file. 4 | */ 5 | 6 | #include 7 | #include "interface.h" 8 | void svm_toy_initialize(); 9 | 10 | int main (int argc, char *argv[]) 11 | { 12 | GtkWidget *window; 13 | 14 | gtk_set_locale (); 15 | gtk_init (&argc, &argv); 16 | 17 | window = create_window (); 18 | gtk_widget_show (window); 19 | 20 | svm_toy_initialize(); 21 | gtk_main (); 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/svm-toy/qt/Makefile: -------------------------------------------------------------------------------- 1 | CXX? = g++ 2 | CFLAGS = -Wall -O3 -I$(INCLUDE) -I$(INCLUDE)/QtGui -lQtGui 3 | INCLUDE = /usr/include/qt4 4 | MOC = /usr/bin/moc-qt4 5 | 6 | svm-toy: svm-toy.cpp svm-toy.moc ../../svm.o 7 | $(CXX) $(CFLAGS) svm-toy.cpp ../../svm.o -o svm-toy 8 | 9 | svm-toy.moc: svm-toy.cpp 10 | $(MOC) svm-toy.cpp -o svm-toy.moc 11 | 12 | ../../svm.o: 13 | cd ../..; make svm.o 14 | 15 | clean: 16 | rm -f *~ svm-toy svm-toy.moc ../../svm.o 17 | 18 | -------------------------------------------------------------------------------- /TPL/libsvm-3.1-custom/svm.def: -------------------------------------------------------------------------------- 1 | LIBRARY libsvm 2 | EXPORTS 3 | svm_train @1 4 | svm_cross_validation @2 5 | svm_save_model @3 6 | svm_load_model @4 7 | svm_get_svm_type @5 8 | svm_get_nr_class @6 9 | svm_get_labels @7 10 | svm_get_svr_probability @8 11 | svm_predict_values @9 12 | svm_predict @10 13 | svm_predict_probability @11 14 | svm_free_model_content @12 15 | svm_free_and_destroy_model @13 16 | svm_destroy_param @14 17 | svm_check_parameter @15 18 | svm_check_probability_model @16 19 | svm_set_print_string_function @17 20 | -------------------------------------------------------------------------------- /data/caffenet/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/data/caffenet/cat.jpg -------------------------------------------------------------------------------- /data/caffenet/cat_gray.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/data/caffenet/cat_gray.jpg -------------------------------------------------------------------------------- /data/caffenet/file_list.txt.in: -------------------------------------------------------------------------------- 1 | ${SMQTK_SOURCE_DIR}/data/caffenet/fish-bike.jpg 0 2 | ${SMQTK_SOURCE_DIR}/data/caffenet/cat_gray.jpg 0 3 | ${SMQTK_SOURCE_DIR}/data/caffenet/cat.jpg 0 4 | -------------------------------------------------------------------------------- /data/caffenet/fish-bike.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/data/caffenet/fish-bike.jpg -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.caffe/README.md: -------------------------------------------------------------------------------- 1 | kitware.caffe 2 | ============= 3 | 4 | Ansible role to install [Caffe](https://github.com/BVLC/caffe). 5 | 6 | Role Variables 7 | -------------- 8 | 9 | The following variables may be overridden: 10 | 11 | * `caffe_path` 12 | * Path to edownload and build Caffe in. 13 | * `caffe_repo` 14 | * URL to the story to clone sm. 15 | * `caffe_version` 16 | * Git commit/branch/tag for fetching Caffe. 17 | * `caffe_install_prefix` 18 | * Path to the root to install Caffe into. 19 | * This must be somewhere root permission is not required. 20 | * `caffe_virtual_env` 21 | * Install python bindings into this virtual env root. 22 | * This must be somewhere root permission is not required. 23 | 24 | TODO: Data/model download options? Leave that to upload in girder? 25 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.caffe/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | caffe_path: "{{ ansible_user_dir }}/caffe" 3 | caffe_repo: "https://github.com/BVLC/caffe.git" 4 | caffe_version: "master" 5 | caffe_install_prefix: "{{ caffe_path }}/install" 6 | # No virtual env value by default 7 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.caffe/tasks/dep_install.yml: -------------------------------------------------------------------------------- 1 | # Install system/python dependencies 2 | --- 3 | # I don't think these are actually required? 4 | # Possibly include with variables? 5 | #- libopencv-dev 6 | #- liblmdb-dev 7 | #- libleveldb-dev 8 | - name: Install system dependencies 9 | apt: 10 | name: "{{item}}" 11 | state: present 12 | with_items: 13 | - git 14 | - cmake 15 | - libatlas-base-dev 16 | - libatlas-dev 17 | - libboost1.55-all-dev 18 | - libprotobuf-dev 19 | - protobuf-compiler 20 | - libgoogle-glog-dev 21 | - libgflags-dev 22 | - libhdf5-dev 23 | - libsnappy-dev 24 | become: yes 25 | become_user: root 26 | 27 | - include: dep_python.yml 28 | when: caffe_virtualenv is not defined 29 | become: yes 30 | become_user: root 31 | 32 | - include: dep_python.yml 33 | when: caffe_virtualenv is defined 34 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.caffe/tasks/dep_python.yml: -------------------------------------------------------------------------------- 1 | # Install python dependencies 2 | --- 3 | - name: Install python deps 4 | pip: 5 | name: "{{ item }}" 6 | state: present 7 | virtualenv: "{{ caffe_virtualenv|default(omit) }}" 8 | with_items: 9 | # Minimal deps required to start processing Caffe networks 10 | - numpy 11 | - scikit-image 12 | - protobuf 13 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.caffe/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install system and python dependencies 3 | - include: dep_install.yml 4 | 5 | # Checkout/Build/Install Caffe 6 | - include: caffe.yml 7 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/README.md: -------------------------------------------------------------------------------- 1 | kitware.smqtk 2 | ============= 3 | 4 | Ansible role to install [SMQTK](https://github.com/Kitware/SMQTK). 5 | 6 | 7 | Role Variables 8 | -------------- 9 | 10 | The following variables may be overridden: 11 | 12 | * `smqtk_path`: Path to download and build SMQTK in. 13 | * `smqtk_repo`: URL to the repository to clone from. 14 | * `smqtk_version`: Git commit/branch/tag for fetching smqtk. 15 | * `smqtk_virtualenv`: Path to the python virtual environment to install SMQTK into. 16 | * `smqtk_iqr_secret_key`: String to use as server Flask secret key. 17 | This should be changed from the default. 18 | * `smqtk_iqr_host`: Host address to listen to. 19 | * `smqtk_iqr_port`: Port to use 20 | 21 | 22 | Dependencies 23 | ------------ 24 | 25 | This role depends on the following roles: 26 | 27 | * `Stouts.monogodb` 28 | * `ANXS.postgresql` 29 | * `kitware.caffe` 30 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Default variables for SMQTK 3 | 4 | smqtk_path: "{{ ansible_user_dir }}/kitware/smqtk" 5 | smqtk_repo: "https://github.com/Kitware/SMQTK.git" 6 | smqtk_version: "master" 7 | smqtk_app_config_path: "{{ smqtk_path }}/python/smqtk/web/search_app/config.ISD_ansible.json" 8 | smqtk_app_secret_key: "MySuperUltraSecret" 9 | smqtk_iqr_host: "0.0.0.0" 10 | smqtk_iqr_port: 5000 11 | # `smqtk_virtualenv` is optional 12 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: smqtk_iqr_app restart 3 | service: 4 | name: smqtk_iqr_app 5 | state: restarted 6 | become: yes 7 | become_user: root 8 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/tasks/daemon.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install config file 3 | template: 4 | src: "isd.conf.json.j2" 5 | dest: "{{ smqtk_app_config_path }}" 6 | backup: yes 7 | 8 | - name: Install service (upstart) 9 | template: 10 | src: "upstart.smqtk_iqr_app.conf.j2" 11 | dest: "/etc/init/smqtk_iqr_app.conf" 12 | become: yes 13 | become_user: root 14 | 15 | - name: (Re)Start service 16 | service: 17 | name: smqtk_iqr_app 18 | state: restarted 19 | become: yes 20 | become_user: root 21 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install system requirements 3 | apt: 4 | name: "{{ item }}" 5 | state: present 6 | update_cache: True 7 | cache_valid_time: 3600 8 | with_items: 9 | - git 10 | - cmake 11 | - g++ 12 | - make 13 | - python-dev 14 | - python-pip 15 | become: yes 16 | become_user: root 17 | 18 | - name: Ensure base directory exists 19 | file: 20 | path: "{{ smqtk_path }}" 21 | state: directory 22 | group: "{{ ansible_user_id }}" 23 | owner: "{{ ansible_user_id }}" 24 | mode: 0755 25 | become: yes 26 | become_user: root # because the path specified could be anywhere on the system 27 | 28 | - name: Fetch SMQTK source tree 29 | git: 30 | repo: "{{ smqtk_repo }}" 31 | dest: "{{ smqtk_path }}" 32 | accept_hostkey: yes 33 | version: "{{ smqtk_version }}" 34 | 35 | # Different configure/build/install path depending on virtualenv use 36 | - include: smqtk_build.yml 37 | when: smqtk_virtualenv is not defined 38 | become: yes 39 | become_user: root 40 | vars: 41 | install_prefix: "/usr/local" 42 | python_executable: "/usr/bin/python" 43 | 44 | - include: smqtk_build.yml 45 | when: smqtk_virtualenv is defined 46 | vars: 47 | install_prefix: "{{ smqtk_virtualenv }}" 48 | python_executable: "{{ smqtk_virtualenv }}/bin/python" 49 | 50 | # Make/Add server service via upstart script (see girder's) 51 | - include: daemon.yml 52 | 53 | # Install SMQTK model files to known directory 54 | - include: model_install.yml 55 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/tasks/smqtk_build.yml: -------------------------------------------------------------------------------- 1 | # Build and install SMQTK components from checkout 2 | # Assumes the following varables/facts defined: 3 | # - install_prefix 4 | # - Root path to install SMQTK into 5 | # - python_executable 6 | # - Path to the python executable to use for the given install_prefix 7 | --- 8 | 9 | - name: Ensure build directory 10 | file: 11 | state: directory 12 | path: "{{ smqtk_path }}/build" 13 | mode: 0755 14 | 15 | - name: CMake configure SMQTK 16 | command: > 17 | cmake 18 | -DCMAKE_INSTALL_PREFIX:PATH={{install_prefix}} 19 | -DPYTHON_EXECUTABLE:STRING={{python_executable}} 20 | {{smqtk_path}} 21 | args: 22 | chdir: "{{ smqtk_path }}/build" 23 | creates: "{{ smqtk_path }}/build/CMakeCache.txt" 24 | 25 | - name: CMake build SMQTK 26 | command: "make -j4" 27 | args: 28 | chdir: "{{ smqtk_path }}/build" 29 | creates: "{{ smqtk_path }}/build/TPL/install/lib" 30 | 31 | # This will correctly install into a venv if the python executable is set 32 | # appropriately. 33 | - name: CMake install SMQTK 34 | command: "make install" 35 | args: 36 | chdir: "{{ smqtk_path }}/build" 37 | creates: "{{ install_prefix }}/setup_smqtk.sh" 38 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/templates/isd.conf.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "flask_app": { 3 | "BASIC_AUTH_PASSWORD": "demo", 4 | "BASIC_AUTH_USERNAME": "demo", 5 | "SECRET_KEY": "{{ smqtk_app_secret_key }}" 6 | }, 7 | "iqr_tabs": { 8 | }, 9 | "mongo": { 10 | "database": "smqtk", 11 | "server": "127.0.0.1:{{ mongodb_conf_port }}" 12 | }, 13 | "server": { 14 | "host": "{{ smqtk_iqr_host }}", 15 | "port": {{ smqtk_iqr_port }} 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /devops/ansible/roles/kitware.smqtk/templates/upstart.smqtk_iqr_app.conf.j2: -------------------------------------------------------------------------------- 1 | description "SMQTK IQR webapp server" 2 | 3 | start on startup 4 | stop on shutdown 5 | 6 | # Automatically Respawn: 7 | respawn 8 | respawn limit 20 5 9 | 10 | setuid "{{ ansible_user_id }}" 11 | setgid "{{ ansible_user_id }}" 12 | 13 | script 14 | cd "{{ smqtk_path }}" 15 | {% if smqtk_virtualenv is defined %} 16 | . "{{ smqtk_virtualenv }}/bin/activate" && runApplication -a IqrSearchDispatcher -vtc {{ smqtk_app_config_path }} 2>&1 17 | {% else %} 18 | exec runApplication -a IqrSearchDispatcher -vtc {{ smqtk_app_config_path }} 2>&1 19 | {% endif %} 20 | end script 21 | -------------------------------------------------------------------------------- /devops/docker/.env: -------------------------------------------------------------------------------- 1 | SMQTK_REGISTRY=kitware/smqtk 2 | 3 | # Dependency Image/Source versions to be used 4 | CUDA_DEVEL_IMAGE_TAG=9.2-cudnn7-devel-ubuntu18.04 5 | CUDA_RUNTIME_IMAGE_TAG=9.2-cudnn7-runtime-ubuntu18.04 6 | 7 | FAISS_VERSION=1.6.2 8 | 9 | # OUR Images 10 | SMQTK_TAG=latest 11 | -------------------------------------------------------------------------------- /devops/docker/smqtk_classifier_service/README.rst: -------------------------------------------------------------------------------- 1 | Base Classifier Service Docker Image 2 | ==================================== 3 | This docker image provides a base for building turn-key classifier service 4 | images. 5 | 6 | This base image provides a SMQTK installation as well as an entrypoint to run 7 | the classifier service with a fixed configuration file path. 8 | 9 | Port 5002 is exposed and the configured server should use this port in any child 10 | images. Make sure to configure the host to ``0.0.0.0`` for the service to be 11 | accessible from outside the container. 12 | 13 | 14 | Server Configuration 15 | -------------------- 16 | Child images extending this should add their specific configuration and 17 | model files in the ``/configuration`` directory inside the image. This directory 18 | must contain the file ``server.json`` that will be used by the entrypoint 19 | script as the configuration file for the classification server. 20 | 21 | A default example of this configuration file can be output by running:: 22 | 23 | runApplication -l SmqtkClassifierService -g /path/to/output/file.json 24 | 25 | A default configuration file is pre-installed that loads no classifier models 26 | but can still be used with IQR session state files at runtime. 27 | -------------------------------------------------------------------------------- /devops/docker/smqtk_classifier_service/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Docker entrypoint script for the classifier service server. 4 | # 5 | set -e 6 | 7 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 8 | 9 | SERVICE_CONFIG_PATH="/configuration/server.json" 10 | 11 | exec runApplication -vt \ 12 | -a SmqtkClassifierService \ 13 | -c "${SERVICE_CONFIG_PATH}" 14 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/TODO.txt: -------------------------------------------------------------------------------- 1 | [ ] Resolve why Ctrl-C'ing the container does not correctly pass into shutdown 2 | clean-up. 3 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/default_confs/entrypoint.conf: -------------------------------------------------------------------------------- 1 | # Paths should be relative and will be interpreted relative to the working 2 | # directory of the entrypoint script. 3 | 4 | 5 | ### Directory definitions. Relative to ``WORKING_DIR``, which is `${HOME}/data` 6 | 7 | # Path to the directory that should contain the data files 8 | IMAGE_DIR="images" 9 | # Path to the directory containing configuration files 10 | CONFIG_DIR="configs" 11 | # Path to the directory containing PostgreSQL 9.4 database files 12 | PSQL_DATA_DIR="db.psql" 13 | # Path to the directory containing MongoDB data files 14 | MONGO_DATA_DIR="db.mongo" 15 | # Path to the directory containing SMQTK computed model files 16 | MODEL_DIR="models" 17 | # Path to the directory that log files will be written to 18 | LOG_DIR="logs" 19 | 20 | 21 | ### Files to be found in ${CONFIG_DIR} for various utilities 22 | 23 | PSQL_TABLE_INIT="psql_table_init.sql" 24 | SMQTK_GEN_IMG_TILES="generate_image_transform.tiles.json" 25 | SMQTK_CMD_CONFIG="compute_many_descriptors.json" 26 | SMQTK_NN_INDEX_TOOL_CONFIG="nn_index_tool.json" 27 | SMQTK_GUI_IQR_CONFIG="runApp.IqrSearchDispatcher.json" 28 | SMQTK_REST_IQR_CONFIG="runApp.IqrService.json" 29 | 30 | 31 | ### PSQL Connection params 32 | 33 | PSQL_HOST="/dev/shm" 34 | PSQL_NAME="postgres" 35 | PSQL_USER="smqtk" 36 | 37 | 38 | ### Build parameters 39 | 40 | # Batch size for compute_many_descriptors utility 41 | DESCRIPTOR_BATCH_SIZE=20 42 | # CSV file generated by ``compute_many_descriptors`` listing files processed. 43 | # Will be written to models directory if computed. Otherwise not read by this 44 | # application. 45 | DESCRIPTOR_PROCESSED_CSV="cmd_processed.csv" 46 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/default_confs/generate_image_transform.tiles.json: -------------------------------------------------------------------------------- 1 | { 2 | "brightness_levels": null, 3 | "contrast_levels": null, 4 | "crop": { 5 | "center_levels": null, 6 | "quadrant_pyramid_levels": null, 7 | "tile_shape": [128, 128], 8 | "tile_stride": [64, 64] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/default_confs/psql_table_init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS descriptors_resnet50_pool5 ( 2 | type_str TEXT NOT NULL, 3 | uid TEXT NOT NULL, 4 | vector BYTEA NOT NULL, 5 | 6 | PRIMARY KEY (uid, type_str) 7 | ); 8 | CREATE TABLE IF NOT EXISTS descriptor_set_resnet50_pool5 ( 9 | uid TEXT NOT NULL, 10 | element BYTEA NOT NULL, 11 | 12 | PRIMARY KEY (uid) 13 | ); 14 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/default_confs/runApp.IqrSearchDispatcher.json: -------------------------------------------------------------------------------- 1 | { 2 | "flask_app": { 3 | "BASIC_AUTH_PASSWORD": "demo", 4 | "BASIC_AUTH_USERNAME": "demo", 5 | "SECRET_KEY": "MySuperUltraSecret" 6 | }, 7 | "iqr_tabs": { 8 | "Data-set Iqr": { 9 | "data_set": { 10 | "smqtk.representation.data_set.memory_set.DataMemorySet": { 11 | "cache_element": { 12 | "smqtk.representation.data_element.file_element.DataFileElement": { 13 | "explicit_mimetype": null, 14 | "filepath": "models/image_elements.dms_cache", 15 | "readonly": true 16 | }, 17 | "type": "smqtk.representation.data_element.file_element.DataFileElement" 18 | }, 19 | "pickle_protocol": -1 20 | }, 21 | "type": "smqtk.representation.data_set.memory_set.DataMemorySet" 22 | }, 23 | "iqr_service_url": "localhost:5001", 24 | "working_directory": "data/iqr_app_work" 25 | } 26 | }, 27 | "mongo": { 28 | "database": "smqtk", 29 | "server": "127.0.0.1:27017" 30 | }, 31 | "server": { 32 | "host": "0.0.0.0", 33 | "port": 5000 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/descr_comp_test.cpu.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test computing a descriptor on something. 3 | """ 4 | from smqtk.algorithms.descriptor_generator.caffe_descriptor \ 5 | import CaffeDescriptorGenerator 6 | from smqtk.representation.data_element.file_element import DataFileElement 7 | from smqtk.representation.data_element.url_element import DataUrlElement 8 | 9 | 10 | grace_hopper_img_url = "https://upload.wikimedia.org/wikipedia/commons/5/55/Grace_Hopper.jpg" 11 | e = DataUrlElement(grace_hopper_img_url) 12 | 13 | gen = CaffeDescriptorGenerator( 14 | DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet-50-deploy.prototxt"), 15 | DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet-50-model.caffemodel"), 16 | DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet_mean.binaryproto"), 17 | return_layer="pool5", 18 | use_gpu=False, load_truncated_images=True 19 | ) 20 | 21 | # Uses default DescriptorMemoryElement factory. 22 | d = gen.generate_one_element(e) 23 | 24 | assert d.vector() is not None 25 | print(d.vector()) 26 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/descr_comp_test.gpu.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test computing a descriptor on something. 3 | """ 4 | from smqtk.algorithms.descriptor_generator.caffe_descriptor \ 5 | import CaffeDescriptorGenerator 6 | from smqtk.representation.data_element.file_element import DataFileElement 7 | from smqtk.representation.data_element.url_element import DataUrlElement 8 | 9 | 10 | grace_hopper_img_url = "https://upload.wikimedia.org/wikipedia/commons/5/55/Grace_Hopper.jpg" 11 | e = DataUrlElement(grace_hopper_img_url) 12 | 13 | gen = CaffeDescriptorGenerator( 14 | DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet-50-deploy.prototxt"), 15 | DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet-50-model.caffemodel"), 16 | DataFileElement("/home/smqtk/caffe/msra_resnet/ResNet_mean.binaryproto"), 17 | return_layer="pool5", 18 | use_gpu=True, load_truncated_images=True 19 | ) 20 | 21 | # Uses default DescriptorMemoryElement factory. 22 | d = gen.generate_one_element(e) 23 | 24 | assert d.vector() is not None 25 | print(d.vector()) 26 | -------------------------------------------------------------------------------- /devops/docker/smqtk_iqr_playground/matplotlibrc: -------------------------------------------------------------------------------- 1 | backend : Agg 2 | -------------------------------------------------------------------------------- /devops/docker/tpl/caffe1/README.md: -------------------------------------------------------------------------------- 1 | Docker image to build the BAIR/BVLC Caffe 1.0 libraries and python package 2 | 3 | This image is intended to be used with a "COPY --from=..." statement in docker 4 | images that would like to use the built Caffe libraries or statically-built 5 | python "module". 6 | 7 | Runtime depedencies required: 8 | docker (GPU-only): Base image descending from the development or runtime 9 | `nvidia/cuda` image of the same CUDA Version this image was built 10 | against (see CUDA_DEVEL_IMAGE_TAG arg used). 11 | apt: 12 | libboost-python1.65.1=1.65.1+dfsg-0ubuntu5 13 | libboost-system1.65.1=1.65.1+dfsg-0ubuntu5 14 | libboost-thread1.65.1=1.65.1+dfsg-0ubuntu5 15 | libgoogle-glog0v5=0.3.5-1 16 | libgflags2.2=2.2.1-1 17 | libhdf5-100=1.10.0-patch1+docs-4 18 | libprotobuf10=3.0.0-9.1ubuntu1 19 | libopenblas-base=0.2.20+ds-4 20 | python3=3.6.7-1~18.04 21 | python3-pip=9.0.1-2.3~ubuntu1.18.04.1 22 | pip: 23 | numpy==1.18.4 24 | scikit-image==0.16.2 25 | protobuf==3.11.3 26 | 27 | COPY locations: 28 | A local installation tree can be found in `/opt/caffe/install/`. 29 | This includes the `/opt/caffe/install/python/caffe` python package 30 | directory. 31 | A wheel is additionally generated and is located at 32 | `/caffe-1.0-py3-none-any.whl`. 33 | 34 | NOTE: The Ubuntu package manager "caffe" installs are not being utilized 35 | because they attempt to install a lot of dependencies. The local use-case of 36 | this image is to use the python bindings that can be built. 37 | -------------------------------------------------------------------------------- /devops/docker/tpl/caffe1/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import setuptools 3 | 4 | 5 | setuptools.setup( 6 | name="caffe", 7 | version="1.0", 8 | description="Install setup for Caffe 1.0 that didn't come with caffe.", 9 | packages=setuptools.find_packages(), 10 | package_data={ 11 | 'caffe': ['_caffe.so', 'imagenet/ilsvrc_2012_mean.npy'], 12 | }, 13 | install_requires=["numpy", "scikit-image", "protobuf"], 14 | ) 15 | -------------------------------------------------------------------------------- /devops/docker/tpl/faiss/README.md: -------------------------------------------------------------------------------- 1 | See the top of ``./gpu/Dockerfile`` for result image usage 2 | description/instructions. 3 | -------------------------------------------------------------------------------- /devops/docker/tpl/faiss/patch/1.5.1/python/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015-present, Facebook, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD+Patents license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | -include ../makefile.inc 8 | 9 | ifneq ($(strip $(NVCC)),) 10 | SWIGFLAGS = -DGPU_WRAPPER 11 | endif 12 | 13 | all: build 14 | 15 | # Also silently generates swigfaiss.py. 16 | swigfaiss.cpp: swigfaiss.swig ../libfaiss.a 17 | $(SWIG) -python -c++ -Doverride= -I../ $(SWIGFLAGS) -o $@ $< 18 | 19 | swigfaiss.o: swigfaiss.cpp 20 | $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) $(PYTHONCFLAGS) \ 21 | -I../ -c $< -o $@ 22 | 23 | # Extension is .so even on OSX. 24 | _swigfaiss.so: swigfaiss.o ../libfaiss.a 25 | $(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS) 26 | 27 | build: _swigfaiss.so faiss.py 28 | $(PYTHON) setup.py build 29 | 30 | install: build 31 | $(PYTHON) setup.py install --prefix=$(prefix) 32 | 33 | clean: 34 | rm -f swigfaiss.{cpp,o,py} 35 | rm -f _swigfaiss.so 36 | rm -rf build/ 37 | 38 | .PHONY: all build clean install 39 | -------------------------------------------------------------------------------- /devops/docker/tpl/faiss/patch/1.6.0/python/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | -include ../makefile.inc 7 | 8 | ifneq ($(strip $(NVCC)),) 9 | SWIGFLAGS += -DGPU_WRAPPER 10 | endif 11 | 12 | all: build 13 | 14 | # Also silently generates swigfaiss.py. 15 | swigfaiss.cpp: swigfaiss.swig ../libfaiss.a 16 | $(SWIG) -python -c++ -Doverride= -I../ $(SWIGFLAGS) -o $@ $< 17 | 18 | swigfaiss_avx2.cpp: swigfaiss.swig ../libfaiss.a 19 | $(SWIG) -python -c++ -Doverride= -module swigfaiss_avx2 -I../ $(SWIGFLAGS) -o $@ $< 20 | 21 | %.o: %.cpp 22 | $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) $(PYTHONCFLAGS) \ 23 | -I../ -c $< -o $@ 24 | 25 | # Extension is .so even on OSX. 26 | _%.so: %.o ../libfaiss.a 27 | $(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS) 28 | 29 | build: _swigfaiss.so faiss.py 30 | $(PYTHON) setup.py build 31 | 32 | install: build 33 | $(PYTHON) setup.py install --prefix=$(prefix) 34 | 35 | clean: 36 | rm -f swigfaiss*.cpp swigfaiss*.o swigfaiss*.py _swigfaiss*.so 37 | rm -rf build/ 38 | 39 | .PHONY: all build clean install 40 | -------------------------------------------------------------------------------- /devops/docker/tpl/faiss/patch/1.6.2/python/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | -include ../makefile.inc 7 | 8 | ifneq ($(strip $(NVCC)),) 9 | SWIGFLAGS += -DGPU_WRAPPER 10 | endif 11 | 12 | all: build 13 | 14 | # Also silently generates swigfaiss.py. 15 | swigfaiss.cpp: swigfaiss.swig ../libfaiss.a 16 | $(SWIG) -python -c++ -Doverride= -I../ $(SWIGFLAGS) -o $@ $< 17 | 18 | swigfaiss_avx2.cpp: swigfaiss.swig ../libfaiss.a 19 | $(SWIG) -python -c++ -Doverride= -module swigfaiss_avx2 -I../ $(SWIGFLAGS) -o $@ $< 20 | 21 | %.o: %.cpp 22 | $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) $(PYTHONCFLAGS) \ 23 | -I../ -c $< -o $@ 24 | 25 | # Extension is .so even on OSX. 26 | _%.so: %.o ../libfaiss.a 27 | $(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS) 28 | 29 | build: _swigfaiss.so faiss.py 30 | $(PYTHON) setup.py build 31 | 32 | install: build 33 | $(PYTHON) setup.py install --prefix=$(prefix) 34 | 35 | clean: 36 | rm -f swigfaiss*.cpp swigfaiss*.o swigfaiss*.py _swigfaiss*.so 37 | rm -rf build/ 38 | 39 | .PHONY: all build clean install 40 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/algorithms.rst: -------------------------------------------------------------------------------- 1 | Algorithms 2 | ========== 3 | 4 | .. toctree:: 5 | :maxdepth: 3 6 | 7 | algorithminterfaces 8 | algorithmmodels 9 | -------------------------------------------------------------------------------- /docs/architecture.rst: -------------------------------------------------------------------------------- 1 | SMQTK Architecture Overview 2 | =========================== 3 | 4 | SMQTK provides at its lowest level semantics for plugins and configuration. 5 | These are provided by some utility functions and two mixin classes: 6 | :py:class:`smqtk.utils.plugin.Pluggable` and 7 | :py:class:`smqtk.utils.configuration.Configurable`. 8 | These are explained further in the "Plugins and Configuration" section. 9 | 10 | Subsequent to these two mixin classes, SMQTK provides two main categories of 11 | interfaces: algorithms and data representations. 12 | This organization of philosophy roughly aligns with the concept of data 13 | oriented design. 14 | Algorithms are usually interfaces defining a behavioral or transformative 15 | action(s), abstracting away how that behavior or transformation is achieved. 16 | Data representation interfaces define the encapsulation of some data structure, 17 | abstracting away where that data is stored.. 18 | 19 | Building upon algorithm and data representation interfaces, there is a 20 | sub-module providing some general web services: :mod:`smqtk.web`. 21 | Of likely interest is headless IQR web-service 22 | (:py:mod:`smqtk.web.iqr_service`). 23 | There is also a demonstration IQR web application with a simple UI as well as 24 | other headless web services (:py:mod:`smqtk.web.search_app`). 25 | 26 | 27 | .. toctree:: 28 | :maxdepth: 2 29 | 30 | architecture/plugins_configuration 31 | dataabstraction 32 | algorithms 33 | webservices 34 | utilities 35 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/1.download_leeds_butterfly.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd "${SCRIPT_DIR}" 5 | 6 | wget http://www.comp.leeds.ac.uk/scs6jwks/dataset/leedsbutterfly/files/leedsbutterfly_dataset_v1.0.zip 7 | unzip leedsbutterfly_dataset_v1.0.zip 8 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/2a.run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd "${SCRIPT_DIR}" 5 | 6 | ../../../bin/scripts/compute_many_descriptors.py \ 7 | -v \ 8 | -c 2a.config.compute_many_descriptors.json \ 9 | -f 2.ingest_files_1.txt \ 10 | --completed-files 2a.completed_files.csv 11 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/2b.config.train_itq.json: -------------------------------------------------------------------------------- 1 | { 2 | "descriptor_set": { 3 | "smqtk.representation.descriptor_set.postgres.PostgresDescriptorSet": { 4 | "db_host": "/dev/shm", 5 | "db_name": "postgres", 6 | "db_pass": null, 7 | "db_port": null, 8 | "db_user": null, 9 | "element_col": "element", 10 | "multiquery_batch_size": 1000, 11 | "pickle_protocol": -1, 12 | "read_only": false, 13 | "table_name": "descriptor_set", 14 | "uuid_col": "uid" 15 | }, 16 | "type": "smqtk.representation.descriptor_set.postgres.PostgresDescriptorSet" 17 | }, 18 | "itq_config": { 19 | "bit_length": 256, 20 | "itq_iterations": 50, 21 | "mean_vec_filepath": "2b.itq.256bit.mean_vec.npy", 22 | "random_seed": 0, 23 | "rotation_filepath": "2b.itq.256bit.rotation.npy" 24 | }, 25 | "parallel": { 26 | "index_load_cores": 4, 27 | "use_multiprocessing": true 28 | }, 29 | "uuids_list_filepath": null 30 | } 31 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/2b.run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd "${SCRIPT_DIR}" 5 | 6 | ../../../bin/scripts/train_itq.py -v -c 2b.config.train_itq.json 7 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/2c.extract_ingest_uuids.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd "${SCRIPT_DIR}" 5 | 6 | cat 2a.completed_files.csv | cut -d',' -f2 >2c.uuids_for_processing.txt 7 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/2c.run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd "${SCRIPT_DIR}" 5 | 6 | ../../../bin/scripts/compute_hash_codes.py -v -c 2c.config.compute_hash_codes.json 7 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/3.run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd "${SCRIPT_DIR}" 5 | 6 | # Compute descriptors for new files, outputing a file that matches input 7 | # files to thair SHA1 checksum values (their UUIDs) 8 | ../../../bin/scripts/compute_many_descriptors.py \ 9 | -d \ 10 | -c 2a.config.compute_many_descriptors.json \ 11 | -f 3.ingest_files_2.txt \ 12 | --completed-files 3.completed_files.csv 13 | 14 | # Extract UUIDs of files/descriptors just generated 15 | cat 3.completed_files.csv | cut -d, -f2 > 3.uuids_for_processing.txt 16 | 17 | # Compute hash codes for descriptors just generated, updating the target 18 | # hash2uuids model file. 19 | ../../../bin/scripts/compute_hash_codes.py -v -c 3.config.compute_hash_codes.json 20 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/4.run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd "${SCRIPT_DIR}" 5 | 6 | # Compute descriptors for new files, outputing a file that matches input 7 | # files to thair SHA1 checksum values (their UUIDs) 8 | ../../../bin/scripts/compute_many_descriptors.py \ 9 | -d \ 10 | -c 2a.config.compute_many_descriptors.json \ 11 | -f 4.ingest_files_3.txt \ 12 | --completed-files 4.completed_files.csv 13 | 14 | # Extract UUIDs of files/descriptors just generated 15 | cat 4.completed_files.csv | cut -d, -f2 > 4.uuids_for_processing.txt 16 | 17 | # Compute hash codes for descriptors just generated, updating the target 18 | # hash2uuids model file. 19 | ../../../bin/scripts/compute_hash_codes.py -v -c 4.config.compute_hash_codes.json 20 | -------------------------------------------------------------------------------- /docs/examples/nnss_incremental_update/test_in_index.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | function usage() { 5 | echo Usage: $0 FILEPATH INDEX_SIZE 6 | } 7 | 8 | filepath="$(readlink -f $1)" 9 | n="$2" 10 | sha1=$(sha1sum $filepath 2>/dev/null | cut -d' ' -f1) 11 | 12 | if [ -z "${sha1}" ] 13 | then 14 | echo "ERROR: Not a valid file path: ${filepath}" 15 | usage 16 | exit 1 17 | fi 18 | if [[ ! "$n" =~ ^[0-9]+$ ]] 19 | then 20 | echo "ERROR: Not given a positive integer for the second parameter" 21 | usage 22 | exit 1 23 | fi 24 | if [ ! "$n" -gt 0 ] 25 | then 26 | echo "ERROR: Index size must be >0 (given '$n')" 27 | usage 28 | exit 1 29 | fi 30 | 31 | # Usingi server address given default configuration. 32 | if [ -n "$(curl http://127.0.0.1:5000/nn/n=${n}/file://${filepath} 2>/dev/null | grep "$sha1")" ] 33 | then 34 | echo "File in index" 35 | exit 0 36 | else 37 | echo "File NOT in index" 38 | exit 1 39 | fi 40 | -------------------------------------------------------------------------------- /docs/examples/overview.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | simple_feature_computation 8 | caffe_build_index 9 | nnss_incremental_update/incremental_example 10 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. SMQTK documentation master file, created by 2 | sphinx-quickstart on Thu Oct 22 16:40:18 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Social Media Query ToolKit 7 | ========================== 8 | 9 | `GitHub `_ 10 | 11 | Python toolkit for pluggable algorithms and data structures for multimedia-based machine learning. 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | 16 | installation 17 | quickstarts 18 | architecture 19 | examples/overview 20 | releasing 21 | 22 | Indices and tables 23 | ================== 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/quickstarts.rst: -------------------------------------------------------------------------------- 1 | Quick-starts 2 | ------------ 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | quickstarts/create_external_plugin 8 | -------------------------------------------------------------------------------- /docs/readthedocs-reqs.txt: -------------------------------------------------------------------------------- 1 | # For some reason RTD really wants to install an old version of sphinx *just* 2 | # for smqtk, but not newer projects... Not what what I'm missing... 3 | sphinx ~= 3.2 4 | sphinx-rtd-theme ~= 0.5 5 | 6 | # SMQTK specific documentation dependencies 7 | sphinx-argparse==0.2.5 8 | sphinx-prompt==1.3.0 9 | -------------------------------------------------------------------------------- /docs/release_notes.rst: -------------------------------------------------------------------------------- 1 | Release Notes 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | release_notes/v0.2.0 8 | release_notes/v0.2.1 9 | release_notes/v0.2.2 10 | release_notes/v0.3.0 11 | release_notes/v0.4.0 12 | release_notes/v0.5.0 13 | release_notes/v0.6.0 14 | release_notes/v0.6.1 15 | release_notes/v0.6.2 16 | release_notes/v0.7.0 17 | release_notes/v0.8.0 18 | release_notes/v0.8.1 19 | release_notes/v0.9.0 20 | release_notes/v0.10.0 21 | release_notes/v0.11.0 22 | release_notes/v0.12.0 23 | release_notes/v0.13.0 24 | release_notes/v0.14.0 25 | -------------------------------------------------------------------------------- /docs/release_notes/v0.12.0.rst: -------------------------------------------------------------------------------- 1 | SMQTK v0.12.0 Release Notes 2 | =========================== 3 | This minor release includes minor fixes and known dependency version updates. 4 | 5 | Fixes 6 | ----- 7 | Docker 8 | 9 | * Fix issue with IQR playground image where matplotlib was attempting to use 10 | the TkAgg backend by default by adding a ``matplotlibrc`` file to specify the 11 | use of the ``Agg`` backend. 12 | 13 | Misc 14 | 15 | * Update requirements versions for: Flask, Flask-Cors, Pillow 16 | 17 | * Update Travis-CI configuration to assume less default values. 18 | 19 | Web 20 | 21 | * IQR Service 22 | 23 | * Broaden base64 parsing error catch. Specific message of the error changed 24 | with python 3.7. 25 | -------------------------------------------------------------------------------- /docs/release_notes/v0.2.1.rst: -------------------------------------------------------------------------------- 1 | SMQTK v0.2.1 Release Notes 2 | ========================== 3 | This is a minor release with a necessary bug fix for installing SMQTK. This 4 | release also has a minor documentation update regarding Caffe AlexNet default 5 | model files and how/where to get them. 6 | 7 | 8 | Updates / New Features since v0.2 9 | --------------------------------- 10 | 11 | Documentation 12 | 13 | * Added segment on acquiring necessary Caffe model files for use with the 14 | current caffe wrapper implementation. 15 | 16 | Fixes since v0.2 17 | ---------------- 18 | 19 | Build 20 | 21 | * Fix an issue where the CMake was trying to install directories no longer 22 | in the source tree due to earlier removal. 23 | -------------------------------------------------------------------------------- /docs/release_notes/v0.6.1.rst: -------------------------------------------------------------------------------- 1 | SMQTK v0.6.1 Release Notes 2 | ========================== 3 | This is a patch release with bug fixs for the Docker wrapping of RESTful 4 | services introduced in v0.6.0. 5 | 6 | Fixes since v0.6.0 7 | ------------------ 8 | 9 | Docker 10 | 11 | * Fixed issue where `smqtk_services.run_images.sh` wasn't properly pulling containers 12 | from Dockerhub. 13 | 14 | * Fixed typo in default configuration files installed into the container. 15 | 16 | * Fixed IQR service function layout to be more explicit in errors caught and 17 | raised which maintaining thread safety. 18 | -------------------------------------------------------------------------------- /docs/release_notes/v0.6.2.rst: -------------------------------------------------------------------------------- 1 | SMQTK v0.6.1 Release Notes 2 | ========================== 3 | This is a patch release with a bug fix for Caffe descriptor generation 4 | introduced in v0.6.0. 5 | 6 | Fixes since v0.6.0 7 | ------------------ 8 | 9 | Descriptor Generation 10 | 11 | * Fixed bug in Caffe wrapper image array loading where loaded arrays were 12 | not in the correctly associated with data identifiers. 13 | -------------------------------------------------------------------------------- /docs/release_notes/v0.8.1.rst: -------------------------------------------------------------------------------- 1 | SMQTK v0.8.1 Release Notes 2 | =========================== 3 | This patch release addresses a bug with PostgreSQL implementations incorrectly 4 | calling a helper class. 5 | 6 | 7 | Fixes since v0.8.0 8 | ------------------ 9 | 10 | Descriptor Index Plugins 11 | 12 | * Fix bug in PostgreSQL plugin where the helper class was not being called 13 | appropriately. 14 | 15 | Utilities 16 | 17 | * Fix bug in PostgreSQL connection helper where the connection object was 18 | being called upon when it may not have been initialized. 19 | -------------------------------------------------------------------------------- /docs/release_notes/v0.9.0.rst: -------------------------------------------------------------------------------- 1 | SMQTK v0.9.0 Release Notes 2 | ========================== 3 | 4 | This minor release represents an update to supporting python 3 versions 5 | as well as adding connection pooling support to the PostgreSQL helper 6 | class. 7 | 8 | 9 | Updates / New Features since v0.8.1 10 | ----------------------------------- 11 | 12 | General 13 | 14 | - Added support for Python 3. 15 | - Made some optimizations to the Postgres database access. 16 | 17 | Travis CI 18 | 19 | - Removed use of Miniconda installation since it wasn't being utilized in 20 | special way. 21 | 22 | Fixes since v0.8.1 23 | ------------------ 24 | 25 | Tests 26 | 27 | - Fixed ambiguous ordering check in libsvm-hik implementation of 28 | RelevancyIndex algorithm. 29 | -------------------------------------------------------------------------------- /docs/releasing.rst: -------------------------------------------------------------------------------- 1 | Release Process and Notes 2 | ========================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | release_process 8 | release_notes 9 | -------------------------------------------------------------------------------- /docs/sphinx_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | This module is designed to used with _livereload to 5 | make it a little easier to write Sphinx documentation. 6 | Simply run the command:: 7 | python sphinx_server.py 8 | 9 | and browse to http://localhost:5500 10 | 11 | livereload_: https://pypi.python.org/pypi/livereload 12 | """ 13 | import os 14 | 15 | from livereload import Server, shell 16 | 17 | rebuild_cmd = shell('make html', cwd='.') 18 | 19 | watch_dirs = [ 20 | '.', 21 | 'architecture', 22 | 'examples', 'examples/nnss_incremental_update', 23 | 'quickstarts', 24 | 'release_notes', 25 | 'webservices', 26 | ] 27 | 28 | watch_globs = [ 29 | '*.rst', '*.ipynb' 30 | ] 31 | 32 | server = Server() 33 | server.watch('conf.py', rebuild_cmd) 34 | # Cover above configured watch dirs and globs matrix. 35 | for d in watch_dirs: 36 | for g in watch_globs: 37 | server.watch(os.path.join(d, g), rebuild_cmd) 38 | # Watch source python files. 39 | for dirpath, dirnames, filenames in os.walk('../python/smqtk'): 40 | server.watch(os.path.join(dirpath, '*.py'), rebuild_cmd) 41 | # Optionally change to host="0.0.0.0" to make available outside localhost. 42 | server.serve(root='_build/html') 43 | -------------------------------------------------------------------------------- /docs/webservices.rst: -------------------------------------------------------------------------------- 1 | .. _webapplabel: 2 | 3 | Web Service and Demonstration Applications 4 | ========================================== 5 | 6 | Included in SMQTK are a few web-based service and demonstration applications, providing a view into the functionality provided by SMQTK algorithms and utilities. 7 | 8 | 9 | .. _run_application: 10 | 11 | runApplication 12 | -------------- 13 | 14 | This script can be used to run any conforming (derived from `SmqtkWebApp`) SMQTK web based application. 15 | Web services should be runnable via the ``bin/runApplication.py`` script. 16 | 17 | .. argparse:: 18 | :ref: smqtk.bin.runApplication.cli_parser 19 | :prog: runApplication 20 | 21 | 22 | SmqtkWebApp 23 | ----------- 24 | This is the base class for all web applications and services in SMQTK. 25 | 26 | .. autoclass:: smqtk.web.SmqtkWebApp 27 | :members: 28 | 29 | 30 | Sample Web Applications 31 | ----------------------- 32 | 33 | .. toctree:: 34 | :maxdepth: 3 35 | 36 | webservices/descriptorservice 37 | webservices/iqrdemonstration 38 | 39 | -------------------------------------------------------------------------------- /docs/webservices/descriptorservice.rst: -------------------------------------------------------------------------------- 1 | Descriptor Similarity Service 2 | ============================= 3 | 4 | * Provides a web-accessible API for computing content descriptor vectors for available descriptor generator labels. 5 | * Descriptor generators that are available to the service are based on the a configuration file provided to the server. 6 | 7 | .. autoclass:: smqtk.web.descriptor_service.DescriptorServiceServer 8 | :members: 9 | 10 | -------------------------------------------------------------------------------- /docs/webservices/figures/IQRWithSMQTK.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/docs/webservices/figures/IQRWithSMQTK.png -------------------------------------------------------------------------------- /docs/webservices/figures/iqr-butterflies-link.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/docs/webservices/figures/iqr-butterflies-link.png -------------------------------------------------------------------------------- /docs/webservices/figures/iqrinitialize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/docs/webservices/figures/iqrinitialize.png -------------------------------------------------------------------------------- /docs/webservices/figures/iqrinitialresults.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/docs/webservices/figures/iqrinitialresults.png -------------------------------------------------------------------------------- /docs/webservices/figures/iqrlogin-entry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/docs/webservices/figures/iqrlogin-entry.png -------------------------------------------------------------------------------- /docs/webservices/figures/iqrlogin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/docs/webservices/figures/iqrlogin.png -------------------------------------------------------------------------------- /docs/webservices/figures/iqrrefine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/docs/webservices/figures/iqrrefine.png -------------------------------------------------------------------------------- /etc/smqtk/postgres/classification_element/example_table_init.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS classifications; 2 | CREATE TABLE IF NOT EXISTS classifications ( 3 | type_name TEXT NOT NULL, 4 | uid TEXT NOT NULL, 5 | classification BYTEA NOT NULL, 6 | 7 | PRIMARY KEY (uid, type_name) 8 | ); 9 | -------------------------------------------------------------------------------- /etc/smqtk/postgres/descriptor_element/example_table_init.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS descriptors; 2 | CREATE TABLE IF NOT EXISTS descriptors ( 3 | type_str TEXT NOT NULL, 4 | uid TEXT NOT NULL, 5 | vector BYTEA NOT NULL, 6 | 7 | PRIMARY KEY (uid, type_str) 8 | ); 9 | -------------------------------------------------------------------------------- /etc/smqtk/postgres/descriptor_set/example_table_init.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS descriptor_set; 2 | CREATE TABLE IF NOT EXISTS descriptor_set ( 3 | uid TEXT NOT NULL, 4 | element BYTEA NOT NULL, 5 | 6 | PRIMARY KEY (uid) 7 | ); 8 | -------------------------------------------------------------------------------- /etc/smqtk/postgres/example_pgbouncer.ini: -------------------------------------------------------------------------------- 1 | [databases] 2 | * = host=/dev/shm port=5432 user=postgres 3 | 4 | [pgbouncer] 5 | listen_addr = localhost 6 | listen_port = 6432 7 | unix_socket_dir = /dev/shm 8 | 9 | auth_type = any 10 | 11 | logfile = pgbouncer.log 12 | pidfile = pgbouncer.pid 13 | 14 | pool_mode=session 15 | 16 | max_client_conn=10000 17 | default_pool_size=500 18 | reserve_pool_size=200 19 | 20 | ; See PgBouncer documentation for additional configuration properties 21 | -------------------------------------------------------------------------------- /etc/smqtk/proxymanager.config: -------------------------------------------------------------------------------- 1 | [server] 2 | port = 30000 3 | authkey = test 4 | 5 | -------------------------------------------------------------------------------- /girder/external_smqtk_iqr/README.rst: -------------------------------------------------------------------------------- 1 | SMQTK External IQR Plugin 2 | ######################### 3 | This plugin adds a small widget to folders in girder if certain metadata fields 4 | are present for that folder's model. If they exist, then a bar with a link 5 | is generated at the top of the view that initializes the IQR configuration 6 | with the remote server and opens a new tab/window to that service. 7 | 8 | We expect the fields: 9 | 10 | smqtk_iqr_root 11 | smqtk_iqr 12 | 13 | ``smqtk_iqr_root`` should specify the URL to the root of a running 14 | ``IqrSearchDispatcher`` instance (see :command:`runApplication`). 15 | 16 | ``smqtk_iqr`` should be a valid configuration for a tab in 17 | ``IqrSearchDispatcher`` (see the ``__default__`` section in a generated config 18 | for an example template). 19 | -------------------------------------------------------------------------------- /girder/external_smqtk_iqr/plugin.yml: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: SQMTK External IQR 4 | description: Link to external SMQTK IQR service from folder of data items, 5 | passing folder metadata specified configuration options for the session. 6 | version: "0.1" 7 | -------------------------------------------------------------------------------- /girder/external_smqtk_iqr/web_client/stylesheets/exiqr_folderView.styl: -------------------------------------------------------------------------------- 1 | .g-exiqrFolderView-header 2 | background-color #f0f0f0 3 | color #555 4 | font-size 16px 5 | font-weight bold 6 | margin-top 10px 7 | margin-bottom 10px 8 | padding 5px 6px 9 | -------------------------------------------------------------------------------- /girder/external_smqtk_iqr/web_client/templates/exiqr_folderView.jade: -------------------------------------------------------------------------------- 1 | .g-exiqrFolderView-header 2 | i.icon-filter 3 | a.g-exiqr-link Launch Image Query 4 | button.g-exiqr-md-toggle-btn.pull-right Toggle Metadata 5 | -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin.cmake: -------------------------------------------------------------------------------- 1 | get_filename_component(PLUGIN ${CMAKE_CURRENT_LIST_DIR} NAME) 2 | 3 | add_web_client_test( 4 | tasks "${PROJECT_SOURCE_DIR}/plugins/${PLUGIN}/plugin_tests/gallerySpec.js" PLUGIN ${PLUGIN} 5 | SETUP_DATABASE "${CMAKE_CURRENT_LIST_DIR}/plugin_tests/fixtures/images.yml") 6 | 7 | add_python_test(utils PLUGIN ${PLUGIN}) 8 | -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SMQTK Girder", 3 | "description": "Provides support for Nearest neighbors and IQR services within Girder", 4 | "version": "0.0.1", 5 | "dependencies": ["worker", "jobs"] 6 | } 7 | -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/.smqtk/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/.smqtk/.gitignore -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0001.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0002.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0003.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0004.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0005.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/001_0005.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0001.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0002.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0003.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0004.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0005.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/002_0005.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0001.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0002.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0003.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/girder/smqtk_girder/plugin_tests/fixtures/girder/images/003_0004.jpg -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/fixtures/images.yml: -------------------------------------------------------------------------------- 1 | users: 2 | - login: 'girder' 3 | password: 'girder' 4 | firstName: 'girder' 5 | lastName: 'girder' 6 | email: 'girder@girder.girder' 7 | admin: true 8 | import: 'girder' 9 | -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/gallerySpec.js: -------------------------------------------------------------------------------- 1 | girderTest.addCoveredScripts([ 2 | '/plugins/smqtk_girder/plugin_tests/testUtils.js', 3 | '/clients/web/static/built/plugins/smqtk_girder/plugin.min.js' 4 | ]); 5 | 6 | girderTest.importStylesheet('/static/built/plugins/smqtk_girder/plugin.min.css'); 7 | 8 | girderTest.startApp(); 9 | 10 | $(function () { 11 | describe('Test viewing a folder as a gallery', function () { 12 | it('Tries to view an image', function () { 13 | smqtkGirderTest.viewAsGallery()(); 14 | 15 | runs(function () { 16 | $('img.im-image-thumbnail:first').click(); 17 | 18 | waitsFor(function () { 19 | return $('.modal a.btn:contains(Close):visible').length === 1; 20 | }, 'the dialog close button to appear'); 21 | 22 | runs(function () { 23 | $('.modal a.btn:contains(Close)').click(); 24 | }); 25 | girderTest.waitForLoad(); 26 | }); 27 | }); 28 | }); 29 | 30 | // Test sort order for distance in NNS 31 | }); 32 | -------------------------------------------------------------------------------- /girder/smqtk_girder/plugin_tests/testUtils.js: -------------------------------------------------------------------------------- 1 | window.smqtkGirderTest = {}; 2 | 3 | smqtkGirderTest.viewAsGallery = function () { 4 | return function () { 5 | girderTest.login('girder', 'girder', 'girder', 'girder')(); 6 | 7 | runs(function () { 8 | expect($('#g-user-action-menu.open').length).toBe(0); 9 | $('.g-user-text>a:first').click(); 10 | }); 11 | girderTest.waitForLoad(); 12 | 13 | runs(function () { 14 | expect($('#g-user-action-menu.open').length).toBe(1); 15 | $('a.g-my-folders').click(); 16 | }); 17 | girderTest.waitForLoad(); 18 | 19 | runs(function () { 20 | $('a.g-folder-list-link:last').click(); 21 | }); 22 | girderTest.waitForLoad(); 23 | 24 | runs(function () { 25 | $('.g-folder-actions-button').click(); 26 | }); 27 | 28 | waitsFor(function () { 29 | return $('.smqtk-view-as-gallery:visible').length === 1; 30 | }, 'the view as gallery action to appear'); 31 | 32 | 33 | runs(function () { 34 | $('.smqtk-view-as-gallery').click(); 35 | }); 36 | 37 | waitsFor(function () { 38 | return $('#gallery-images').length === 1 && 39 | $('.im-thumbnail').length === 14; 40 | }, 'items to appear as thumbnails'); 41 | }; 42 | }; 43 | -------------------------------------------------------------------------------- /girder/smqtk_girder/server/constants.py: -------------------------------------------------------------------------------- 1 | class PluginSettings(object): 2 | DB_HOST = 'smqtk_girder.db_host' 3 | DB_NAME = 'smqtk_girder.db_name' 4 | DB_USER = 'smqtk_girder.db_user' 5 | DB_PASS = 'smqtk_girder.db_pass' 6 | DB_DESCRIPTORS_TABLE = 'smqtk_girder.db_descriptors_table' 7 | IMAGE_BATCH_SIZE = 'smqtk_girder.image_batch_size' 8 | CAFFE_NETWORK_MODEL = 'smqtk_girder.caffe_network_model' 9 | CAFFE_NETWORK_PROTOTXT = 'smqtk_girder.caffe_network_prototxt' 10 | CAFFE_IMAGE_MEAN = 'smqtk_girder.caffe_image_mean' 11 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/collections/IqrResultsCollection.js: -------------------------------------------------------------------------------- 1 | import ItemCollection from 'girder/collections/ItemCollection'; 2 | import ItemModel from 'girder/models/ItemModel'; 3 | 4 | var IqrResultsCollection = ItemCollection.extend({ 5 | resourceName: 'item', 6 | model: ItemModel, 7 | altUrl: 'smqtk_iqr/results', 8 | pageLimit: 100, 9 | comparator: function (item) { 10 | return -item.get('meta').smqtk_iqr_confidence; 11 | } 12 | }); 13 | 14 | export default IqrResultsCollection; 15 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/collections/IqrSessionCollection.js: -------------------------------------------------------------------------------- 1 | import ItemCollection from 'girder/collections/ItemCollection'; 2 | import IqrSessionModel from '../models/IqrSessionModel'; 3 | 4 | var IqrSessionCollection = ItemCollection.extend({ 5 | model: IqrSessionModel, 6 | altUrl: IqrSessionModel.altUrl 7 | }); 8 | 9 | export default IqrSessionCollection; 10 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/collections/ResultsCollection.js: -------------------------------------------------------------------------------- 1 | import ItemCollection from 'girder/collections/ItemCollection'; 2 | import ItemModel from 'girder/models/ItemModel'; 3 | 4 | var ResultsCollection = ItemCollection.extend({ 5 | resourceName: 'item', 6 | model: ItemModel, 7 | altUrl: 'smqtk_nearest_neighbors/nn', 8 | pageLimit: 100, 9 | comparator: 'smqtk_distance' 10 | }); 11 | 12 | export default ResultsCollection; 13 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/main.js: -------------------------------------------------------------------------------- 1 | import './routes'; 2 | 3 | import './views/HierarchyWidget'; 4 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/routes.js: -------------------------------------------------------------------------------- 1 | import router from 'girder/router'; 2 | import GalleryView from './views/GalleryView'; 3 | import ConfigView from './views/ConfigView'; 4 | import events from 'girder/events'; 5 | import { exposePluginConfig } from 'girder/utilities/PluginUtils'; 6 | 7 | exposePluginConfig('smqtk_girder', 'plugins/smqtk_girder/config'); 8 | 9 | router.route('plugins/smqtk_girder/config', 'smqtkGirderConfig', function () { 10 | events.trigger('g:navigateTo', ConfigView); 11 | }); 12 | 13 | router.route('gallery/:indexId/:id', 'gallery', function (indexId, id, params) { 14 | GalleryView.fetchAndInit(indexId, id, params); 15 | }); 16 | 17 | router.route('gallery/nearest-neighbors/:indexId/:id', 'gallery-nearest-neighbors', function (indexId, id, params) { 18 | GalleryView.fetchAndInitNns(indexId, id, params); 19 | }); 20 | 21 | // this :id is the IQR Session ID 22 | router.route('gallery/iqr/:indexId/:id', 'gallery-iqr', function (indexId, id, params) { 23 | GalleryView.fetchAndInitIqr(indexId, id, params); 24 | }); 25 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/stylesheets/galleryItemView.styl: -------------------------------------------------------------------------------- 1 | .im-image-area 2 | overflow-x hidden 3 | 4 | .im-details 5 | display inline-block 6 | padding 6px 12px 7 | margin-bottom 0 8 | vertical-align middle 9 | cursor pointer 10 | 11 | .im-actions 12 | padding-top 30px 13 | 14 | img.im-image-thumbnail 15 | height 100px !important 16 | 17 | img.im-image-detail 18 | max-height 100% 19 | max-width 100% 20 | display block 21 | margin 0 auto 22 | 23 | .im-caption 24 | height 70px 25 | overflow hidden 26 | 27 | .smqtk-nn-search 28 | margin-top 10px 29 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/stylesheets/iqrItemView.styl: -------------------------------------------------------------------------------- 1 | .smqtk-iqr-annotation 2 | margin-left 40px 3 | height 60px 4 | 5 | .smqtk-iqr-annotation button 6 | opacity .5 7 | 8 | .smqtk-iqr-annotation button.selected 9 | opacity 1 10 | 11 | .smqtk-iqr-annotation button.smqtk-iqr-positive 12 | background-color #95cd7e 13 | 14 | .smqtk-iqr-annotation button.smqtk-iqr-negative 15 | background-color #cc7f7f 16 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/templates/galleryDetailWidget.pug: -------------------------------------------------------------------------------- 1 | .modal-dialog 2 | .modal-content 3 | .modal-header 4 | button.close(data-dismiss="modal", aria-hidden="true", type="button") × 5 | h4.modal-title= item.get('name') 6 | .modal-body 7 | .image-container 8 | .scroll-controls 9 | if canScroll && hasPrev 10 | i.icon-left-open.prev-image(title='Previous image') 11 | if canScroll && hasNext 12 | i.icon-right-open.next-image(title='Next image') 13 | 14 | a.im-link(href=item.downloadUrl({'contentDisposition': 'inline'}), target="_blank") 15 | img.im-image-detail(src=item.downloadUrl({'contentDisposition': 'inline'})) 16 | 17 | a.btn.btn-default.smqtk-nn-search Find nearest neighbors 18 | 19 | .modal-footer 20 | a.btn.btn-small.btn-default(data-dismiss="modal") Close 21 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/templates/galleryItemView.pug: -------------------------------------------------------------------------------- 1 | .im-image-area.col-xs-6.col-md-3 2 | .im-details(im-id=item.get('id')) 3 | .im-thumbnail 4 | img.im-image-thumbnail(src=item.downloadUrl({'contentDisposition': 'inline'})) 5 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/templates/galleryView.pug: -------------------------------------------------------------------------------- 1 | .g-user-header 2 | .btn-group.pull-right 3 | button.g-user-actions-button.btn.btn-default.dropdown-toggle( 4 | data-toggle="dropdown", title="Gallery actions") 5 | i.icon-picture 6 | | Actions 7 | i.icon-down-dir 8 | ul.g-item-actions-menu.dropdown-menu.pull-right(role="menu") 9 | if iqrEnabled 10 | li(role="presentation") 11 | a.smqtk-iqr-refine(role="menuitem") 12 | i.icon-edit 13 | | Refine results 14 | li(role="presentation") 15 | a.smqtk-iqr-save-adjudications(role="menuitem") 16 | i.icon-edit 17 | | Save adjudications 18 | li(role="presentation") 19 | a.smqtk-iqr-quit(role="menuitem") 20 | i.icon-edit 21 | | Quit IQR session 22 | else 23 | li(role="presentation") 24 | a.smqtk-start-iqr(role="menuitem") 25 | i.icon-edit 26 | | Start IQR session 27 | li.divider(role="presentation") 28 | li(role="presentation") 29 | a.smqtk-manage-iqr-sessions(role="menuitem") 30 | i.icon-ok-squared 31 | | Manage IQR sessions 32 | .g-body-subtitle(style='height:45px;') 33 | | Folder Name 34 | 35 | #gallery-images 36 | a.show-more-items Show more items 37 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/templates/hierarchyWidgetMenu.pug: -------------------------------------------------------------------------------- 1 | li(role="presentation") 2 | a.smqtk-process-images(role="menuitem") 3 | i.icon-doc 4 | | Process Images 5 | li(role="presentation") 6 | a.smqtk-view-as-gallery(role="menuitem") 7 | i.icon-doc 8 | | View as Gallery 9 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/templates/iqrItemView.pug: -------------------------------------------------------------------------------- 1 | .im-image-area.col-xs-6.col-md-3 2 | .im-details(im-id=item.get('id')) 3 | .im-thumbnail 4 | img.im-image-thumbnail(src=item.downloadUrl({'contentDisposition': 'inline'})) 5 | 6 | .smqtk-iqr-annotation 7 | .btn-group 8 | button.btn.btn-default.smqtk-iqr-annotate.smqtk-iqr-positive(class=(positive ? 'selected' : '')) 9 | | Pos 10 | button.btn.btn-default.smqtk-iqr-annotate.smqtk-iqr-negative(class=(negative ? 'selected' : '')) 11 | | Neg 12 | -------------------------------------------------------------------------------- /girder/smqtk_girder/web_client/views/GalleryItemView.js: -------------------------------------------------------------------------------- 1 | import $ from 'jquery'; 2 | import _ from 'underscore'; 3 | 4 | import FolderCollection from 'girder/collections/FolderCollection'; 5 | import LoadingAnimation from 'girder/views/widgets/LoadingAnimation'; 6 | import View from 'girder/views/View'; 7 | 8 | import GalleryDetailWidget from './GalleryDetailWidget'; 9 | import GalleryItemViewTemplate from '../templates/galleryItemView.pug'; 10 | 11 | import '../stylesheets/galleryItemView.styl'; 12 | 13 | var GalleryItemView = View.extend({ 14 | events: { 15 | 'click .im-details': function (event) { 16 | this.galleryDetailWidget = new GalleryDetailWidget({ 17 | el: $('#g-dialog-container'), 18 | item: this.item, 19 | parentView: this 20 | }); 21 | this.galleryDetailWidget.render(); 22 | } 23 | }, 24 | 25 | initialize: function (settings) { 26 | this.item = settings.item; 27 | }, 28 | 29 | render: function () { 30 | this.$el.html(GalleryItemViewTemplate({ 31 | item: this.item 32 | })); 33 | 34 | return this; 35 | } 36 | }); 37 | 38 | export default GalleryItemView; 39 | -------------------------------------------------------------------------------- /girder/smqtk_worker/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore for development installs 2 | smqtk_worker.egg-info/ 3 | -------------------------------------------------------------------------------- /girder/smqtk_worker/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup( 4 | name='smqtk_worker', 5 | version='0.0.1', 6 | description='A Girder Worker plugin for distributed processing of images using SMQTK.', 7 | author='Kitware, Inc.', 8 | author_email='kitware@kitware.com', 9 | license='Apache 2.0', 10 | packages=['smqtk_worker'], 11 | entry_points={ 12 | 'girder_worker_plugins': [ 13 | 'smqtk_worker = smqtk_worker:GirderWorkerPlugin' 14 | ] 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /girder/smqtk_worker/smqtk_worker/__init__.py: -------------------------------------------------------------------------------- 1 | from girder_worker import GirderWorkerPluginABC 2 | 3 | 4 | class GirderWorkerPlugin(GirderWorkerPluginABC): 5 | def __init__(self, app, *args, **kwargs): 6 | self.app = app 7 | 8 | def task_imports(self): 9 | return ['smqtk_worker.tasks'] 10 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | # -l :: Show local in trace-backs. 3 | # -v :: Increased verbosity. 4 | # --tb :: Trace-back print mode. 5 | addopts = -lv --doctest-modules --tb=long 6 | testpaths = 7 | tests 8 | python/smqtk 9 | norecursedirs = 10 | tests/data 11 | tests/utils/test_plugin_dir 12 | doctest_optionflags = 13 | NORMALIZE_WHITESPACE 14 | IGNORE_EXCEPTION_DETAIL 15 | -------------------------------------------------------------------------------- /python/smqtk/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.14.0" 2 | 3 | from . import ( # noqa: F401 4 | algorithms, 5 | representation, 6 | iqr 7 | ) 8 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface import SmqtkAlgorithm # noqa: F401 2 | 3 | # Import module abstracts and plugin getter functions 4 | from .classifier import Classifier, SupervisedClassifier # noqa: F401 5 | from .descriptor_generator import DescriptorGenerator # noqa: F401 6 | from .image_io import ImageReader # noqa: F401 7 | from .object_detection import ObjectDetector, ImageMatrixObjectDetector # noqa: F401 8 | from .nn_index import NearestNeighborsIndex # noqa: F401 9 | from .nn_index.hash_index import HashIndex # noqa: F401 10 | from .nn_index.lsh.functors import LshFunctor # noqa: F401 11 | from .rank_relevancy import RankRelevancy, RankRelevancyWithFeedback # noqa: F401 12 | from .relevancy_index import RelevancyIndex # noqa: F401 13 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/_interface.py: -------------------------------------------------------------------------------- 1 | from smqtk.utils import SmqtkObject 2 | from smqtk.utils.configuration import Configurable 3 | from smqtk.utils.plugin import Pluggable 4 | 5 | 6 | # noinspection PyAbstractClass 7 | class SmqtkAlgorithm (SmqtkObject, Configurable, Pluggable): 8 | """ 9 | Parent class for all algorithm interfaces. 10 | """ 11 | 12 | __slots__ = () 13 | 14 | @property 15 | def name(self): 16 | """ 17 | :return: The name of this class type. 18 | :rtype: str 19 | """ 20 | return self.__class__.__name__ 21 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/classifier/__init__.py: -------------------------------------------------------------------------------- 1 | from ._classifier_collection import ClassifierCollection # noqa: F401 2 | from ._interface_classifier import Classifier # noqa: F401 3 | from ._interface_supervised import SupervisedClassifier # noqa: F401 4 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/classifier/_defaults.py: -------------------------------------------------------------------------------- 1 | from smqtk.representation import ClassificationElementFactory 2 | from smqtk.representation.classification_element.memory import \ 3 | MemoryClassificationElement 4 | 5 | 6 | # Default classifier element factory for interfaces. 7 | DFLT_CLASSIFIER_FACTORY = ClassificationElementFactory( 8 | MemoryClassificationElement, {} 9 | ) 10 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/classifier/_plugins.py: -------------------------------------------------------------------------------- 1 | from .index_label import IndexLabelClassifier # noqa: F401 2 | from .libsvm import LibSvmClassifier # noqa: F401 3 | from .sklearn_logistic_regression import SkLearnLogisticRegression # noqa: F401 4 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/descriptor_generator/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface import DFLT_DESCRIPTOR_FACTORY, DescriptorGenerator # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/descriptor_generator/_plugins.py: -------------------------------------------------------------------------------- 1 | from .caffe_descriptor import CaffeDescriptorGenerator # noqa: F401 2 | from .colordescriptor.colordescriptor import ( # noqa: F401 3 | ColorDescriptor_Image_rgbhistogram, 4 | ColorDescriptor_Video_rgbhistogram, 5 | ColorDescriptor_Image_opponenthistogram, 6 | ColorDescriptor_Video_opponenthistogram, 7 | ColorDescriptor_Image_huehistogram, 8 | ColorDescriptor_Video_huehistogram, 9 | ColorDescriptor_Image_nrghistogram, 10 | ColorDescriptor_Video_nrghistogram, 11 | ColorDescriptor_Image_transformedcolorhistogram, 12 | ColorDescriptor_Video_transformedcolorhistogram, 13 | ColorDescriptor_Image_colormoments, 14 | ColorDescriptor_Video_colormoments, 15 | ColorDescriptor_Image_colormomentinvariants, 16 | ColorDescriptor_Video_colormomentinvariants, 17 | ColorDescriptor_Image_sift, 18 | ColorDescriptor_Video_sift, 19 | ColorDescriptor_Image_huesift, 20 | ColorDescriptor_Video_huesift, 21 | ColorDescriptor_Image_hsvsift, 22 | ColorDescriptor_Video_hsvsift, 23 | ColorDescriptor_Image_opponentsift, 24 | ColorDescriptor_Video_opponentsift, 25 | ColorDescriptor_Image_rgsift, 26 | ColorDescriptor_Video_rgsift, 27 | ColorDescriptor_Image_csift, 28 | ColorDescriptor_Video_csift, 29 | ColorDescriptor_Image_rgbsift, 30 | ColorDescriptor_Video_rgbsift, 31 | ) 32 | from .kwcnndescriptor import KWCNNDescriptorGenerator # noqa: F401 33 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/descriptor_generator/colordescriptor/INSTALL.md: -------------------------------------------------------------------------------- 1 | # ColorDescriptor Installation 2 | The ColorDescriptor content description implementation hinges on the 3 | ColorDescriptor package that can be found at: 4 | 5 | http://koen.me/research/colordescriptors/ 6 | 7 | Provided that this is being used for non-commercial purposes, or a license has 8 | been acquired, this site provides a method of downloading a ZIP archive of 9 | the necessary binaries (for Windows, Max OSX and Linux systems) and required 10 | python IO module. 11 | 12 | ## Setup 13 | 1. Add the unzipped directory to the PYTHONPATH to allow access to the 14 | DescriptorsIO.py module. 15 | 2. Add the system specific sub-directory to the PATH depending on your platform. 16 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/descriptor_generator/colordescriptor/__init__.py: -------------------------------------------------------------------------------- 1 | from .colordescriptor import cd_type_list 2 | SMQTK_PLUGIN_CLASS = cd_type_list 3 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/descriptor_generator/kwcnndescriptor/LICENSE: -------------------------------------------------------------------------------- 1 | All files recursively contained in this directory follow the following license: 2 | 3 | Copyright 2016 Kitware, Inc. 4 | All rights reserved. 5 | 6 | Government Unlimited Rights 7 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/descriptor_generator/kwcnndescriptor/__init__.py: -------------------------------------------------------------------------------- 1 | from .kwcnndescriptor import KWCNNDescriptorGenerator 2 | 3 | 4 | __author__ = 'jason.parham@kitware.com,paul.tunison@kitware.com' 5 | 6 | 7 | DESCRIPTOR_GENERATOR_CLASS = KWCNNDescriptorGenerator 8 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/descriptor_generator/kwcnndescriptor/kwcnnmodel.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/algorithms/descriptor_generator/kwcnndescriptor/kwcnnmodel.npy -------------------------------------------------------------------------------- /python/smqtk/algorithms/image_io/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface import ImageReader # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/image_io/_plugins.py: -------------------------------------------------------------------------------- 1 | from .gdal_io import GdalImageReader # noqa: F401 2 | from .pil_io import PilImageReader # noqa: F401 3 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/nn_index/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface_nn_index import NearestNeighborsIndex # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/nn_index/_plugins.py: -------------------------------------------------------------------------------- 1 | from .faiss import FaissNearestNeighborsIndex # noqa: F401 2 | from .flann import FlannNearestNeighborsIndex # noqa: F401 3 | from .lsh import LSHNearestNeighborIndex # noqa: F401 4 | from .mrpt import MRPTNearestNeighborsIndex # noqa: F401 5 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/nn_index/hash_index/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface_hash_index import HashIndex # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/nn_index/hash_index/_plugins.py: -------------------------------------------------------------------------------- 1 | from .linear import LinearHashIndex # noqa: F401 2 | from .sklearn_balltree import SkLearnBallTreeHashIndex # noqa: F401 3 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/nn_index/lsh/functors/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Interface and plugin getter for LSH algorithm hash generation functors. 3 | """ 4 | import abc 5 | 6 | from smqtk.algorithms import SmqtkAlgorithm 7 | 8 | 9 | class LshFunctor (SmqtkAlgorithm): 10 | """ 11 | Locality-sensitive hashing functor interface. 12 | 13 | The aim of such a function is to be able to generate hash codes 14 | (bit-vectors) such that similar items map to the same or similar hashes 15 | with a high probability. In other words, it aims to maximize hash collision 16 | for similar items. 17 | 18 | **Building Models** 19 | 20 | Some hash functions want to build a model based on some training set of 21 | descriptors. Due to the non-standard nature of algorithm training and model 22 | building, please refer to the specific implementation for further 23 | information on whether model training is needed and how it is accomplished. 24 | 25 | """ 26 | 27 | def __call__(self, descriptor): 28 | return self.get_hash(descriptor) 29 | 30 | @abc.abstractmethod 31 | def get_hash(self, descriptor): 32 | """ 33 | Get the locality-sensitive hash code for the input descriptor. 34 | 35 | :param descriptor: Descriptor vector we should generate the hash of. 36 | :type descriptor: numpy.ndarray[float] 37 | 38 | :return: Generated bit-vector as a numpy array of booleans. 39 | :rtype: numpy.ndarray[bool] 40 | 41 | """ 42 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/nn_index/lsh/functors/_plugins.py: -------------------------------------------------------------------------------- 1 | from .itq import ItqFunctor # noqa: F401 2 | from .simple_rp import SimpleRPFunctor # noqa: F401 3 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/object_detection/__init__.py: -------------------------------------------------------------------------------- 1 | # noinspection PyUnresolvedReferences 2 | # - Convenience pass-through from "private" sub-module. 3 | from ._defaults import DFLT_CLASSIFIER_FACTORY, DFLT_DETECTION_FACTORY # noqa: F401 4 | from ._interface import ObjectDetector, ImageMatrixObjectDetector # noqa: F401 5 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/object_detection/_defaults.py: -------------------------------------------------------------------------------- 1 | """ 2 | Default values and instances for the ObjectDetection interface. 3 | """ 4 | # - Using the same default factory for ObjectDetector as the Classifier 5 | # interface. 6 | # - Providing classifier default here for convenience. 7 | # noinspection PyProtectedMember 8 | from smqtk.algorithms.classifier._defaults import DFLT_CLASSIFIER_FACTORY # noqa: F401, lgtm[py/unused-import] 9 | from smqtk.representation import DetectionElementFactory 10 | from smqtk.representation.detection_element.memory \ 11 | import MemoryDetectionElement 12 | 13 | DFLT_DETECTION_FACTORY = DetectionElementFactory( 14 | MemoryDetectionElement, {} 15 | ) 16 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/rank_relevancy/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface import RankRelevancy, RankRelevancyWithFeedback # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/rank_relevancy/_plugins.py: -------------------------------------------------------------------------------- 1 | from .margin_sampling import \ 2 | RankRelevancyWithMarginSampledFeedback # noqa: F401 3 | from .wrap_classifier import \ 4 | RankRelevancyWithSupervisedClassifier # noqa: F401 5 | -------------------------------------------------------------------------------- /python/smqtk/algorithms/relevancy_index/_plugins.py: -------------------------------------------------------------------------------- 1 | from .classifier_wrapper import SupervisedClassifierRelevancyIndex # noqa: F401 2 | from .libsvm_hik import LibSvmHikRelevancyIndex # noqa: F401 3 | -------------------------------------------------------------------------------- /python/smqtk/bin/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /python/smqtk/bin/createFileIngest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Add a set of local system files to a data set via explicit paths or shell-style 3 | glob strings. 4 | """ 5 | 6 | import glob 7 | import logging 8 | import os.path as osp 9 | 10 | from smqtk.representation import DataSet 11 | from smqtk.representation.data_element.file_element import DataFileElement 12 | from smqtk.utils import cli 13 | from smqtk.utils.configuration import ( 14 | from_config_dict, 15 | make_default_config, 16 | ) 17 | 18 | 19 | def default_config(): 20 | return { 21 | "data_set": make_default_config(DataSet.get_impls()) 22 | } 23 | 24 | 25 | def cli_parser(): 26 | parser = cli.basic_cli_parser(__doc__) 27 | parser.add_argument("input_files", metavar='GLOB', nargs='*') 28 | return parser 29 | 30 | 31 | def main(): 32 | parser = cli_parser() 33 | args = parser.parse_args() 34 | config = cli.utility_main_helper(default_config, args) 35 | log = logging.getLogger(__name__) 36 | 37 | log.debug("Script arguments:\n%s" % args) 38 | 39 | def iter_input_elements(): 40 | for f in args.input_files: 41 | f = osp.expanduser(f) 42 | if osp.isfile(f): 43 | yield DataFileElement(f) 44 | else: 45 | log.debug("Expanding glob: %s" % f) 46 | for g in glob.glob(f): 47 | yield DataFileElement(g) 48 | 49 | log.info("Adding elements to data set") 50 | ds = from_config_dict(config['data_set'], DataSet.get_impls()) 51 | ds.add_data(*iter_input_elements()) 52 | 53 | 54 | if __name__ == '__main__': 55 | main() 56 | -------------------------------------------------------------------------------- /python/smqtk/bin/proxyManagerServer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Server for hosting proxy manager which hosts proxy object instances. 3 | 4 | This takes a simple configuration file that looks like the following: 5 | 6 | | [server] 7 | | port = 8 | | authkey = 9 | 10 | """ 11 | # Copyright 2013-2016 by Kitware, Inc. All Rights Reserved. Please refer to 12 | # LICENSE.txt for licensing information, or contact General Counsel, 13 | # Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065. 14 | 15 | from smqtk.utils import cli 16 | from smqtk.utils.proxy_manager import ProxyManager 17 | 18 | 19 | def default_config(): 20 | return { 21 | "port": 5000, 22 | "authkey": "CHANGE_ME", 23 | } 24 | 25 | 26 | def cli_parser(): 27 | return cli.basic_cli_parser(__doc__) 28 | 29 | 30 | def main(): 31 | parser = cli_parser() 32 | args = parser.parse_args() 33 | 34 | # Default config options for this util are technically valid for running, 35 | # its just a bad authkey. 36 | config = cli.utility_main_helper(default_config, args, 37 | default_config_valid=True) 38 | 39 | port = int(config['port']) 40 | authkey = bytes(config['authkey']) 41 | 42 | mgr = ProxyManager(('', port), authkey) 43 | mgr.get_server().serve_forever() 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /python/smqtk/exceptions/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | class NoClassificationError (Exception): 3 | """ 4 | When a ClassificationElement has no mapping yet set, but an operation 5 | required it. 6 | """ 7 | 8 | 9 | class NoDetectionError (Exception): 10 | """ 11 | When a DetectionElement has no stored data (paired bounding box and classification 12 | """ 13 | 14 | 15 | class ReadOnlyError (Exception): 16 | """ 17 | For when an attempt at modifying an immutable container is made. 18 | """ 19 | 20 | 21 | class NoUriResolutionError (Exception): 22 | """ 23 | Standard exception thrown by base DataElement from_uri method when a 24 | subclass does not implement URI resolution. 25 | """ 26 | 27 | 28 | class InvalidUriError (Exception): 29 | """ 30 | An invalid URI was provided. 31 | """ 32 | 33 | def __init__(self, uri_value, reason): 34 | super(InvalidUriError, self).__init__(uri_value, reason) 35 | self.uri = uri_value 36 | self.reason = reason 37 | 38 | 39 | class MissingLabelError(Exception): 40 | """ 41 | Raised by ClassifierCollection.classify when requested classifier labels 42 | are missing from collection. 43 | """ 44 | def __init__(self, labels): 45 | """ 46 | :param labels: The labels missing from the collection 47 | :type labels: set[str] 48 | """ 49 | super(MissingLabelError, self).__init__(labels) 50 | self.labels = labels 51 | -------------------------------------------------------------------------------- /python/smqtk/iqr/__init__.py: -------------------------------------------------------------------------------- 1 | from .iqr_session import IqrSession 2 | from .iqr_controller import IqrController 3 | 4 | __all__ = [ 5 | 'IqrController', 6 | 'IqrSession', 7 | ] 8 | -------------------------------------------------------------------------------- /python/smqtk/representation/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface import SmqtkRepresentation # noqa: F401 2 | 3 | from .bbox import AxisAlignedBoundingBox # noqa: F401 4 | 5 | from .classification_element import ClassificationElement # noqa: F401 6 | from .data_element import DataElement # noqa: F401 7 | from .data_set import DataSet # noqa: F401 8 | from .descriptor_element import DescriptorElement # noqa: F401 9 | from .descriptor_set import DescriptorSet # noqa: F401 10 | from .detection_element import DetectionElement # noqa: F401 11 | from .key_value import KeyValueStore # noqa: F401 12 | 13 | from .classification_element_factory import ClassificationElementFactory # noqa: F401 14 | from .descriptor_element_factory import DescriptorElementFactory # noqa: F401 15 | from .detection_element_factory import DetectionElementFactory # noqa: F401 16 | -------------------------------------------------------------------------------- /python/smqtk/representation/_interface.py: -------------------------------------------------------------------------------- 1 | from smqtk.utils import SmqtkObject 2 | from smqtk.utils.configuration import Configurable 3 | 4 | 5 | # noinspection PyAbstractClass 6 | class SmqtkRepresentation (SmqtkObject, Configurable): 7 | """ 8 | Interface for data representation interfaces and implementations. 9 | 10 | Data should be serializable, so this interface adds abstract methods for 11 | serializing and de-serializing SMQTK data representation instances. 12 | 13 | """ 14 | 15 | __slots__ = () 16 | 17 | # TODO(paul.tunison): Add serialization abstract method signatures here. 18 | # - Could start with just requiring implementing sub-classes to 19 | # ``__getstate__`` and ``__setstate__`` methods required for pickle 20 | # interface. 21 | -------------------------------------------------------------------------------- /python/smqtk/representation/classification_element/_plugins.py: -------------------------------------------------------------------------------- 1 | from .file import FileClassificationElement # noqa: F401 2 | from .memory import MemoryClassificationElement # noqa: F401 3 | from .postgres import PostgresClassificationElement # noqa: F401 4 | -------------------------------------------------------------------------------- /python/smqtk/representation/data_element/_plugins.py: -------------------------------------------------------------------------------- 1 | from .file_element import DataFileElement # noqa: F401 2 | from .girder import GirderDataElement # noqa: F401 3 | from .hbase_element import HBaseDataElement # noqa: F401 4 | from .matrix import MatrixDataElement # noqa: F401 5 | from .memory_element import DataMemoryElement # noqa: F401 6 | from .psql import PostgresDataElement # noqa: F401 7 | from .url_element import DataUrlElement # noqa: F401 8 | -------------------------------------------------------------------------------- /python/smqtk/representation/data_set/_plugins.py: -------------------------------------------------------------------------------- 1 | from .file_set import DataFileSet # noqa: F401 2 | from .kvstore_backed import KVSDataSet # noqa: F401 3 | from .memory_set import DataMemorySet # noqa: F401 4 | from .psql import PostgresNativeDataSet # noqa: F401 5 | -------------------------------------------------------------------------------- /python/smqtk/representation/descriptor_element/_plugins.py: -------------------------------------------------------------------------------- 1 | from .local_elements import DescriptorMemoryElement, DescriptorFileElement # noqa: F401 2 | from .postgres import PostgresDescriptorElement # noqa: F401 3 | from .solr_element import SolrDescriptorElement # noqa: F401 4 | -------------------------------------------------------------------------------- /python/smqtk/representation/descriptor_set/_plugins.py: -------------------------------------------------------------------------------- 1 | from .memory import MemoryDescriptorSet # noqa: F401 2 | from .postgres import PostgresDescriptorSet # noqa: F401 3 | from .solr_index import SolrDescriptorSet # noqa: F401 4 | -------------------------------------------------------------------------------- /python/smqtk/representation/detection_element/__init__.py: -------------------------------------------------------------------------------- 1 | from ._interface import DetectionElement # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/representation/detection_element/_plugins.py: -------------------------------------------------------------------------------- 1 | from .memory import MemoryDetectionElement # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/representation/key_value/_plugins.py: -------------------------------------------------------------------------------- 1 | from .memory import MemoryKeyValueStore # noqa: F401 2 | from .postgres import PostgresKeyValueStore # noqa: F401 3 | -------------------------------------------------------------------------------- /python/smqtk/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | LICENCE 3 | ------- 4 | Copyright 2013-2016 by Kitware, Inc. All Rights Reserved. Please refer to 5 | KITWARE_LICENSE.TXT for licensing information, or contact General Counsel, 6 | Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065. 7 | 8 | """ 9 | ### 10 | # In specific ordering for dependency resolution 11 | # 12 | from .base_object import SmqtkObject # noqa: F401 13 | from .cli import initialize_logging # noqa: F401 14 | from .content_type_validator import ContentTypeValidator # noqa: F401 15 | from .database_info import DatabaseInfo # noqa: F401 16 | from .iter_validation import check_empty_iterable # noqa: F401 17 | from .read_write_lock import ReaderUpdateException, DummyRWLock, ReadWriteLock # noqa: F401 18 | from .signal_handler import SignalHandler # noqa: F401 19 | from .simple_timer import SimpleTimer # noqa: F401 20 | -------------------------------------------------------------------------------- /python/smqtk/utils/base_object.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | class SmqtkObject (object): 5 | """ 6 | Highest level object interface for classes defined in SMQTK. 7 | 8 | Currently defines logging methods. 9 | 10 | """ 11 | 12 | __slots__ = () 13 | 14 | @classmethod 15 | def get_logger(cls): 16 | """ 17 | :return: logging object for this class 18 | :rtype: logging.Logger 19 | """ 20 | return logging.getLogger('.'.join((cls.__module__, cls.__name__))) 21 | 22 | @property 23 | def _log(self): 24 | """ 25 | :return: logging object for this class as a property 26 | :rtype: logging.Logger 27 | """ 28 | return self.get_logger() 29 | -------------------------------------------------------------------------------- /python/smqtk/utils/combinatorics.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import math 3 | import operator as op 4 | 5 | from six.moves import range 6 | 7 | 8 | def ncr(n, r): 9 | """ 10 | N-choose-r method, returning the number of combinations possible in integer 11 | form. 12 | 13 | From dheerosaur: 14 | http://stackoverflow.com/questions/4941753/is-there-a-math-ncr-function-in-python 15 | 16 | :param n: Selection pool size. 17 | :type n: int 18 | 19 | :param r: permutation selection size. 20 | :type r: int 21 | 22 | :return: Number of n-choose-r permutations for the given n and r. 23 | :rtype: int 24 | 25 | """ 26 | r = min(r, n - r) 27 | if r == 0: 28 | return 1 29 | numer = functools.reduce(op.mul, range(n, n - r, -1), 1) 30 | # denom = functools.reduce(op.mul, range(1, r+1), 1) 31 | denom = math.factorial(r) 32 | return numer // denom 33 | -------------------------------------------------------------------------------- /python/smqtk/utils/dict.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions pertaining to python dictionaries. 3 | """ 4 | 5 | import copy 6 | 7 | 8 | def merge_dict(a, b, deep_copy=False): 9 | """ 10 | Merge dictionary b into dictionary a. 11 | 12 | This is different than normal dictionary update in that we don't bash 13 | nested dictionaries, instead recursively updating them. 14 | 15 | For congruent keys, values are are overwritten, while new keys in ``b`` are 16 | simply added to ``a``. 17 | 18 | Values are assigned (not copied) by default. Setting ``deep_copy`` causes 19 | values from ``b`` to be deep-copied into ``a``. 20 | 21 | :param a: The "base" dictionary that is updated in place. 22 | :type a: dict 23 | 24 | :param b: The dictionary to merge into ``a`` recursively. 25 | :type b: dict 26 | 27 | :param deep_copy: Optionally deep-copy values from ``b`` when assigning into 28 | ``a``. 29 | :type deep_copy: bool 30 | 31 | :return: ``a`` dictionary after merger (not a copy). 32 | :rtype: dict 33 | 34 | """ 35 | for k in b: 36 | if k in a and isinstance(a[k], dict) and isinstance(b[k], dict): 37 | merge_dict(a[k], b[k], deep_copy) 38 | elif deep_copy: 39 | a[k] = copy.deepcopy(b[k]) 40 | else: 41 | a[k] = b[k] 42 | return a 43 | -------------------------------------------------------------------------------- /python/smqtk/utils/iter_validation.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | 4 | def check_empty_iterable(iterable, callback, exception_inst): 5 | """ 6 | Check that the given iterable is not empty, then call the given callback 7 | function with the reconstructed iterable when it is not empty. 8 | 9 | :param iterable: Iterable to check. 10 | :type iterable: collections.abc.Iterable 11 | 12 | :param callback: Function to call with the reconstructed, not-empty 13 | iterable. 14 | :type callback: (collections.abc.Iterable) -> None 15 | 16 | :param exception_inst: The exception to throw if the iterable is empty 17 | :type exception_inst: Exception 18 | 19 | """ 20 | i = iter(iterable) 21 | try: 22 | first = next(i) 23 | except StopIteration: 24 | raise exception_inst 25 | callback(itertools.chain([first], i)) 26 | -------------------------------------------------------------------------------- /python/smqtk/utils/mimetype.py: -------------------------------------------------------------------------------- 1 | import mimetypes as pymimetypes 2 | import threading 3 | 4 | 5 | MT_CACHE_LOCK = threading.RLock() 6 | MT_CACHE = None 7 | 8 | 9 | def get_mimetypes(): 10 | """ 11 | Get the singleton SMQTK-adjusted mimetypes instance. 12 | 13 | This is desired due to the presence of some odd mimetype and extension 14 | conversion, e.g. image/jpeg -> .jfif. 15 | 16 | :return: SMQTK singleton MimeTypes instance. 17 | :rtype: mimetypes.MimeTypes 18 | 19 | """ 20 | global MT_CACHE 21 | with MT_CACHE_LOCK: 22 | if MT_CACHE is None: 23 | MT_CACHE = pymimetypes.MimeTypes() 24 | # Remove weird extensions from map for getting extensions-by-type 25 | for jpg_ext in ['.jfif', '.jpe']: 26 | if jpg_ext in MT_CACHE.types_map_inv[1]['image/jpeg']: 27 | # Incorrect stub for `.types_map_inv`. Actually contains 28 | # lists at the leaves. 29 | # noinspection PyUnresolvedReferences 30 | MT_CACHE.types_map_inv[1]['image/jpeg'].remove(jpg_ext) # type: ignore 31 | return MT_CACHE 32 | -------------------------------------------------------------------------------- /python/smqtk/utils/simple_timer.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from smqtk.utils import SmqtkObject 4 | 5 | 6 | class SimpleTimer (SmqtkObject): 7 | """ 8 | Little class to wrap the timing of things. To be use with the ``with`` 9 | statement. 10 | """ 11 | 12 | def __init__(self, msg, log_func=None, *args): 13 | """ 14 | Additional arguments are passed to the logging method 15 | :param msg: 16 | :param log_func: 17 | :param args: 18 | :return: 19 | """ 20 | self._log_func = log_func 21 | self._msg = msg 22 | self._msg_args = args 23 | self._s = 0.0 24 | 25 | def __enter__(self): 26 | if self._log_func: 27 | self._log_func(self._msg, *self._msg_args) 28 | else: 29 | self._log.info(self._msg % self._msg_args) 30 | self._s = time.time() 31 | 32 | def __exit__(self, *_): 33 | if self._log_func: 34 | self._log_func("%s -> %f s", self._msg % self._msg_args, 35 | time.time() - self._s) 36 | else: 37 | self._log.info("%s -> %f s" % (self._msg % self._msg_args, 38 | time.time() - self._s)) 39 | -------------------------------------------------------------------------------- /python/smqtk/web/_plugins.py: -------------------------------------------------------------------------------- 1 | from .classifier_service import SmqtkClassifierService # noqa: F401 2 | from .descriptor_service import DescriptorServiceServer # noqa: F401 3 | from .iqr_service import IqrService # noqa: F401 4 | from .nearestneighbor_service import NearestNeighborServiceServer # noqa: F401 5 | from .search_app import IqrSearchDispatcher # noqa: F401 6 | -------------------------------------------------------------------------------- /python/smqtk/web/classifier_service/__init__.py: -------------------------------------------------------------------------------- 1 | from .classifier_server import SmqtkClassifierService # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/web/classifier_service/example/.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | -------------------------------------------------------------------------------- /python/smqtk/web/classifier_service/example/README.md: -------------------------------------------------------------------------------- 1 | This directory contains a Jupyter notebook for testing the classifier service, 2 | along with some associated files: 3 | 4 | - classifier_test.ipynb: You can run this notebook to test out the server. 5 | - dummy_classifier.py, dummy_descriptor_generator.py: Files with dummy 6 | implementations of necessary classes. 7 | - dummy_classifier.pkl.b64: a base64-encoded pickle file of an instance of 8 | DummyClassifier, for testing upload 9 | - fish-bike.jpg.b64: a base64-encoded JPG file for use as input to classify 10 | - test_classifier_server.sh: A script that runs several curl commands, to test 11 | the output of the server. Run this after starting the server in the notebook. 12 | - test_classifier_responses.txt: A text file containing the output of a run of 13 | `test_classifier_server.sh` 14 | 15 | Simpy run all cells in the notebook, and the server will be up and running 16 | (assuming your system allows connections to `localhost`). Once the server is 17 | running, it can take requests from `curl` or other scripts like 18 | `test_classifier_server.sh`. -------------------------------------------------------------------------------- /python/smqtk/web/classifier_service/example/dummy_classifier.pkl.b64: -------------------------------------------------------------------------------- 1 | Y2NvcHlfcmVnCl9yZWNvbnN0cnVjdG9yCnAxCihjZHVtbXlfY2xhc3NpZmllcgpEdW1teUNsYXNzaWZpZXIKcDIKY19fYnVpbHRpbl9fCm9iamVjdApwMwpOdFJwNAou -------------------------------------------------------------------------------- /python/smqtk/web/iqr_service/__init__.py: -------------------------------------------------------------------------------- 1 | from .iqr_server import IqrService # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/web/nearestneighbor_service/__init__.py: -------------------------------------------------------------------------------- 1 | from .service import NearestNeighborServiceServer 2 | 3 | SMQTK_PLUGIN_CLASS = NearestNeighborServiceServer 4 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/README.rst: -------------------------------------------------------------------------------- 1 | IQR Demo Search Application 2 | =========================== 3 | Requires two servers: this server for the client/server interface and the IQR 4 | RESTful service server to provide IQR functionality. 5 | 6 | Until tested otherwise, both servers should be run on the same machine with 7 | different ports in order to avoid CORS issues. 8 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/__init__.py -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/file_upload/__init__.py: -------------------------------------------------------------------------------- 1 | from .FileUploadMod import FileUploadMod # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/file_upload/static/css/smqtk_upload.css: -------------------------------------------------------------------------------- 1 | /* Flow.js related elements */ 2 | /*noinspection CssUnusedSymbol*/ 3 | .flow-drop { 4 | padding:15px; 5 | font-size:13px; 6 | text-align:center; 7 | color:#666; 8 | font-weight:bold; 9 | background-color:#eee; 10 | border: 2px dashed #aaa; 11 | border-radius:10px; 12 | margin-top:40px; 13 | z-index:9999; 14 | } 15 | 16 | /*noinspection CssUnusedSymbol*/ 17 | .flow-dragover { 18 | padding:30px; 19 | color:#555; 20 | background-color:#ddd; 21 | border:1px solid #999; 22 | } 23 | 24 | /*noinspection CssUnusedSymbol*/ 25 | .flow-browse { 26 | text-decoration: underline; 27 | } 28 | 29 | /*noinspection CssUnusedSymbol*/ 30 | .flow-error { 31 | font-size:14px; 32 | font-style:italic; 33 | } 34 | 35 | /*noinspection CssUnusedSymbol*/ 36 | .flow-progress { 37 | /*margin:30px 0 30px 0;*/ 38 | width:100%; 39 | } 40 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/__init__.py: -------------------------------------------------------------------------------- 1 | from .iqr_search import IqrSearch # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/css/smqtk.iqr.css: -------------------------------------------------------------------------------- 1 | /* IQR Result display elements */ 2 | .iqr-result { 3 | display: inline-block; 4 | text-align: center; 5 | margin: 5px; 6 | padding: 5px; 7 | border: 2px solid lightblue; 8 | -webkit-border-radius: 5px; 9 | border-radius: 5px; 10 | background-color:rgba(220, 220, 250, 0.1); 11 | vertical-align: top; 12 | } 13 | .iqr-result-img-container { 14 | height: 192px; 15 | width: 192px; 16 | } 17 | .result-positive { 18 | border-color: lightgreen; 19 | background-color:rgba(154, 230, 154, 0.3); 20 | } 21 | .result-negative { 22 | border-color: red; 23 | background-color:rgba(230, 154, 154, 0.3); 24 | } 25 | .adjudication-box { 26 | background: #373737; 27 | opacity: 0.8; 28 | margin-bottom: 2px; 29 | text-align: left; 30 | } 31 | 32 | /* IQR View display elements */ 33 | .iqr-view-results-container { 34 | display: inline-block; 35 | text-align: center; 36 | vertical-align: top; 37 | } 38 | .explicit { 39 | filter: blur(10px); 40 | } 41 | .explicit:hover { 42 | filter: blur(0); 43 | } 44 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/carbon-rejected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/carbon-rejected.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/carbon-rejected_on.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/carbon-rejected_on.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/carbon-verified.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/carbon-verified.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/carbon-verified_on.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/carbon-verified_on.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/explicit_marker_off.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/explicit_marker_off.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/explicit_marker_on.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/explicit_marker_on.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/explicit_overlay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/explicit_overlay.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/img/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/modules/iqr/static/img/loading.gif -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/static/js/FileSaver.LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright © 2015 [Eli Grey][1]. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | [1]: http://eligrey.com 10 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/iqr/templates/base.html: -------------------------------------------------------------------------------- 1 | ../../../templates/base.html -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/login/__init__.py: -------------------------------------------------------------------------------- 1 | from .LoginMod import LoginMod # noqa: F401 2 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/login/templates/login.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% set active_page = "login" %} 3 | 4 | {% block body_content %} 5 |
6 |
7 |

Sign In

8 |
9 | 10 |
12 |
13 | 14 | 17 |
18 |
19 | 20 | 22 |
23 |
24 | 25 | 26 | 28 |
29 | {# csrf_token_gen() function defined in base dispatcher app -#} 30 | 31 |
32 |
33 | {% endblock body_content %} 34 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/login/users.json: -------------------------------------------------------------------------------- 1 | { 2 | "demo": { 3 | "fullname": "Demo User", 4 | "passwd": "demo", 5 | "roles": [ 6 | "guest" 7 | ] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/modules/static_host.py: -------------------------------------------------------------------------------- 1 | import flask 2 | 3 | 4 | __author__ = 'paul.tunison@kitware.com' 5 | 6 | 7 | class StaticDirectoryHost (flask.Blueprint): 8 | """ 9 | Module that will host a given directory to the given URL prefix (relative to 10 | the parent module's prefix). 11 | 12 | Instances of this class will have nothing set to their static URL path, as a 13 | blank string is used. Please reference the URL prefix value. 14 | 15 | """ 16 | 17 | def __init__(self, name, static_dir, url_prefix): 18 | # make sure URL prefix starts with a slash 19 | if not url_prefix.startswith('/'): 20 | url_prefix = '/' + url_prefix 21 | 22 | super(StaticDirectoryHost, self).__init__(name, __name__, 23 | static_folder=static_dir, 24 | static_url_path="", 25 | url_prefix=url_prefix) 26 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/sample_configs/config.IqrSearchApp.json: -------------------------------------------------------------------------------- 1 | { 2 | "flask_app": { 3 | "BASIC_AUTH_PASSWORD": "demo", 4 | "BASIC_AUTH_USERNAME": "demo", 5 | "SECRET_KEY": "MySuperUltraSecret" 6 | }, 7 | "server": { 8 | "host": "127.0.0.1", 9 | "port": 5000 10 | }, 11 | "mongo": { 12 | "database": "smqtk", 13 | "server": "127.0.0.1:27017" 14 | }, 15 | "iqr_tabs": { 16 | "LEEDS Butterflies": { 17 | "working_directory": "workdir", 18 | "data_set": { 19 | "smqtk.representation.data_set.memory_set.DataMemorySet": { 20 | "cache_element": { 21 | "smqtk.representation.data_element.file_element.DataFileElement": { 22 | "explicit_mimetype": null, 23 | "filepath": "workdir/butterflies_alexnet_fc7/data.memorySet.cache", 24 | "readonly": false 25 | }, 26 | "type": "smqtk.representation.data_element.file_element.DataFileElement" 27 | }, 28 | "pickle_protocol": -1 29 | }, 30 | "type": "smqtk.representation.data_set.memory_set.DataMemorySet" 31 | }, 32 | "iqr_service_url": "http://localhost:5001" 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/sample_configs/config.iqrTrainClassifier.json: -------------------------------------------------------------------------------- 1 | { 2 | "classifier": { 3 | "smqtk.algorithms.classifier.libsvm.LibSvmClassifier": { 4 | "normalize": 2, 5 | "svm_label_map_uri": "workdir/iqr_classifier/label_map", 6 | "svm_model_uri": "workdir/iqr_classifier/model", 7 | "train_params": { 8 | "-b": 1, 9 | "-c": 2, 10 | "-s": 0, 11 | "-t": 0 12 | } 13 | }, 14 | "type": "smqtk.algorithms.classifier.libsvm.LibSvmClassifier" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/clip.view.css: -------------------------------------------------------------------------------- 1 | .clip-results { 2 | text-align: center; 3 | /*font-size: 0;*/ 4 | } 5 | 6 | .clip-view-image { 7 | width: 200px; 8 | height:150px; 9 | position : relative; 10 | background-position:center; 11 | -webkit-border-radius: 5px; 12 | border-radius: 5px; 13 | cursor: pointer; 14 | } 15 | 16 | .clip-view { 17 | display:inline-block; 18 | text-align: left; 19 | position:relative; 20 | margin:7px; 21 | padding : 5px; 22 | border-style: solid; 23 | border-color: lightblue; 24 | border-width : 2px; 25 | -webkit-border-radius: 5px; 26 | border-radius: 5px; 27 | background-color:rgba(220, 220, 250, 0.1); 28 | } 29 | 30 | 31 | .adjudication-yes{ 32 | border-color: limegreen; 33 | background-color:rgba(154, 230, 154, 0.3); 34 | } 35 | 36 | .adjudication-no{ 37 | border-color: red; 38 | background-color:rgba(230, 154, 154, 0.3); 39 | } 40 | 41 | .adjudication-star{ 42 | border-color: #fbcb09; 43 | position: relative; 44 | background-color:rgba(220, 220, 144, 0.5); 45 | } 46 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_diagonals-thick_20_666666_40x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_diagonals-thick_20_666666_40x40.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_flat_10_000000_40x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_flat_10_000000_40x100.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_glass_100_f6f6f6_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_glass_100_f6f6f6_1x400.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_glass_100_fdf5ce_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_glass_100_fdf5ce_1x400.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_glass_65_ffffff_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_glass_65_ffffff_1x400.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-icons_222222_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-icons_222222_256x240.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-icons_228ef1_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-icons_228ef1_256x240.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-icons_ef8c08_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-icons_ef8c08_256x240.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-icons_ffd27a_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-icons_ffd27a_256x240.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/css/images/ui-icons_ffffff_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/css/images/ui-icons_ffffff_256x240.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/img/Kitwarelogo-gradation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/img/Kitwarelogo-gradation.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/img/cancel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/img/cancel.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/img/done.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/img/done.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/img/pause.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/img/pause.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/img/resume.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/python/smqtk/web/search_app/static/img/resume.png -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/js/setZeroTimeout.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Reference: http://dbaron.org/log/20100309-faster-timeouts 3 | */ 4 | // Only add setZeroTimeout to the window object, and hide everything 5 | // else in a closure. 6 | (function() { 7 | var timeouts = []; 8 | var messageName = "zero-timeout-message"; 9 | 10 | // Like setTimeout, but only takes a function argument. There's 11 | // no time argument (always zero) and no arguments (you have to 12 | // use a closure). 13 | function setZeroTimeout(fn) { 14 | timeouts.push(fn); 15 | window.postMessage(messageName, "*"); 16 | } 17 | 18 | function handleMessage(event) { 19 | if (event.source == window && event.data == messageName) { 20 | event.stopPropagation(); 21 | if (timeouts.length > 0) { 22 | var fn = timeouts.shift(); 23 | fn(); 24 | } 25 | } 26 | } 27 | 28 | window.addEventListener("message", handleMessage, true); 29 | 30 | // Add the one thing we want added to the window object. 31 | window.setZeroTimeout = setZeroTimeout; 32 | })(); -------------------------------------------------------------------------------- /python/smqtk/web/search_app/static/js/smqtk.vars.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Common global variables used in SMQTK javascript. 3 | */ 4 | 5 | // Header to set the CSRF header to in AJAX requests. 6 | CSRF_HEADER_KEY = "X-Csrf-Token"; 7 | CSRF_FORM_KEY = "_csrf_token"; 8 | -------------------------------------------------------------------------------- /python/smqtk/web/search_app/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% set active_page = "home" %} 3 | 4 | {% block body_content %} 5 |
6 |

Welcome to SMQTK

7 |

Current IQR instances:

8 | 9 |
    10 | {%- if instance_keys|length > 0 %} 11 | {%- for prefix in instance_keys %} 12 |
  • {{ prefix|escape }}
    • 13 | {%- endfor %} 14 | {%- else %} 15 |
  • None
    • 16 | {%- endif %} 17 |
18 |
19 | {% endblock %} 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements/runtime.txt 2 | -r requirements/test.txt 3 | -r requirements/docs.txt 4 | -r requirements/ci.txt 5 | -------------------------------------------------------------------------------- /requirements/ci.txt: -------------------------------------------------------------------------------- 1 | # Additional requirements for CI utility. 2 | 3 | # This is an implicit value, here for clarity 4 | --index-url https://pypi.python.org/simple/ 5 | 6 | flake8==3.8.3 7 | flake8-mutable==1.2.0 # for triggering on mutable default values. 8 | mypy==0.790 9 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | # Sphinx (pinned here for CI/stability, unpinned for read-the-docs) 2 | Sphinx==3.2.1 3 | sphinx-rtd-theme==0.5.0 4 | 5 | # SMQTK specific documentation dependencies 6 | sphinx-argparse==0.2.5 7 | sphinx-prompt==1.3.0 8 | 9 | # For local server tool. 10 | livereload==2.6.3 11 | -------------------------------------------------------------------------------- /requirements/optional.txt: -------------------------------------------------------------------------------- 1 | # This is an implicit value, here for clarity 2 | --index-url https://pypi.python.org/simple/ 3 | 4 | # # Elastic search interaction 5 | # certifi==2016.8.8 6 | # 7 | # # For use with 1.x versions of source elasticsearch instance 8 | # elasticsearch==1.9.0 9 | # elasticsearch_dsl==0.0.11 10 | # # For use with 2.x versions of source elasticsearch instance 11 | # elasticsearch==2.4.0 12 | # elasticsearch_dsl==2.1.0 13 | 14 | # # PostgreSQL 15 | # psycopg2-binary==2.7.6 16 | 17 | # # Caffe 18 | # protobuf==3.0.0 19 | # scikit-image==0.12.3 20 | 21 | # # Solr plugin dependency 22 | # solrpy==0.9.7 23 | -------------------------------------------------------------------------------- /requirements/runtime.txt: -------------------------------------------------------------------------------- 1 | # Core SMQTK requirements 2 | 3 | # This is an implicit value, here for clarity 4 | --index-url https://pypi.python.org/simple/ 5 | 6 | # Core deps 7 | Click==7.1.2 8 | Flask==1.1.2 9 | Flask-BasicAuth==0.2.0 10 | Flask-login==0.5.0 11 | Flask-Cors==3.0.9 12 | imageio==2.8.0 13 | jinja2==2.11.3 14 | jsmin==3.0.1 15 | matplotlib==3.2.1 16 | numpy==1.18.4 17 | Pillow==8.4.0 18 | pymongo==3.10.1 19 | requests==2.23.0 20 | scikit-learn==0.23.1 21 | scipy==1.4.1 22 | 23 | # Transitive deps that need downgrading relative to latest versions 24 | itsdangerous==2.0.1 25 | MarkupSafe==2.0.1 26 | werkzeug==2.0.3 27 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | # This is an implicit value, here for clarity 2 | --index-url https://pypi.python.org/simple/ 3 | 4 | coverage==6.2 5 | pytest==6.2.5 6 | pytest-cov==3.0.0 7 | 8 | -------------------------------------------------------------------------------- /scripts/memex/README.md: -------------------------------------------------------------------------------- 1 | Here lies scripts intended specifically for the DARPA MEMEX program, and may 2 | not be generally applicable. 3 | -------------------------------------------------------------------------------- /scripts/memex/hackathon_2016_07/cp1/ad_image_classification/cnn_finetuning/alexnet/README.txt: -------------------------------------------------------------------------------- 1 | Original configuration files for AlexNet are from the Caffe repository: 2 | 3 | ./models/bvlc_alexnet/ 4 | 5 | Checkout hash: 42cd785e4b5ed824a9b2a02a19aa534042b64325 6 | 7 | Please refer to the ``LICENSE`` file in this directory, which pertains to 8 | files in this directory. 9 | -------------------------------------------------------------------------------- /scripts/memex/hackathon_2016_07/cp1/ad_image_classification/cnn_finetuning/alexnet/solver.prototxt: -------------------------------------------------------------------------------- 1 | net: "train_val.prototxt" 2 | test_iter: 40 3 | test_interval: 1000 4 | 5 | type: "Adam" 6 | # Taking advice from: 7 | # https://github.com/BVLC/caffe/blob/master/examples/mnist/lenet_solver_adam.prototxt 8 | base_lr: 0.0001 9 | momentum: 0.9 10 | momentum2: 0.999 11 | lr_policy: "fixed" 12 | #gamma: 0.1 13 | #weight_decay: 0.0005 14 | 15 | max_iter: 200000 16 | 17 | # Shows training loss every this many iterations 18 | display: 100 19 | # Also shows moving window average of this many iterations 20 | average_loss: 100 21 | 22 | snapshot: 5000 23 | snapshot_prefix: "caffe_alexnet_ht.adam.train" 24 | 25 | solver_mode: GPU 26 | -------------------------------------------------------------------------------- /scripts/memex/hackathon_2016_07/cp1/ad_image_classification/cnn_finetuning/labels.txt: -------------------------------------------------------------------------------- 1 | negative 2 | positive 3 | -------------------------------------------------------------------------------- /scripts/memex/hackathon_2016_07/cp1/data_retreival/scripts/download_url_as_sha.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | import requests 4 | import sys 5 | 6 | # Downloads an imageurl to a location based on its sha1 hash 7 | # Then print out the mapping of the URL to the sha 8 | 9 | NUM_RETRIES = 5 10 | 11 | if __name__ == '__main__': 12 | url = sys.argv[1] 13 | 14 | DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 15 | '../data') 16 | 17 | i = 0 18 | while True: 19 | try: 20 | r = requests.get(url, stream=True) 21 | except Exception as ex: 22 | print("Encountered exception: {}".format(str(ex))) 23 | sys.exit(3) 24 | 25 | if (r.status_code >= 400 and r.status_code < 500): 26 | sys.exit(1) 27 | elif i == NUM_RETRIES: 28 | sys.exit(2) 29 | elif not r.ok: 30 | i += 1 31 | continue 32 | else: 33 | sha1 = hashlib.sha1(r.content).hexdigest() 34 | 35 | try: 36 | os.makedirs(os.path.join(DATA_DIR, 'CP1_imageset', sha1[0:3])) 37 | except OSError: 38 | pass 39 | 40 | with open(os.path.join(DATA_DIR, 'CP1_imageset', sha1[0:3], sha1), 'wb') as outfile: 41 | outfile.write(r.content) 42 | 43 | print('%s %s' % (url, sha1)) 44 | 45 | sys.exit(0) 46 | -------------------------------------------------------------------------------- /scripts/memex/hackathon_2016_07/cp1/data_retreival/scripts/es_config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://my-elasticsearch-server.org:1234", 3 | "user": "myesuser", 4 | "password": "hunter2" 5 | } 6 | -------------------------------------------------------------------------------- /scripts/memex/hackathon_2016_07/cp1/tensorflow_retraining/aggregate.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | 4 | score_file = 'run16_sha_scores.csv' 5 | matches_file = 'cp1eval/CP1_clusters_ads_images.csv' 6 | 7 | matches = list(csv.reader(open(matches_file)))[1:] 8 | scores = list(csv.reader(open(score_file)))[1:] 9 | 10 | cluster_map = {} 11 | 12 | for match in matches: 13 | images = cluster_map.get(match[0], []) 14 | images.append(match[2]) 15 | cluster_map[match[0]] = images 16 | 17 | score_map = {} 18 | 19 | for score in scores: 20 | score_map[score[0].split('/')[1]] = float(score[2]) 21 | 22 | cluster_scores = [] 23 | 24 | for cluster, images in cluster_map.iteritems(): 25 | cur_scores = [] 26 | for image in images: 27 | if image in score_map: 28 | cur_scores.append(score_map[image]) 29 | if len(cur_scores) > 0: 30 | # agg_score = sum(cur_scores) / len(cur_scores) 31 | agg_score = max(cur_scores) 32 | cluster_scores.append({'cluster_id': cluster, 'score': agg_score}) 33 | 34 | for cluster_score in cluster_scores: 35 | print json.dumps(cluster_score) 36 | -------------------------------------------------------------------------------- /scripts/memex/hackathon_2016_07/cp1/tensorflow_retraining/split.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import pickle 3 | import os 4 | import itertools 5 | 6 | parts = { 7 | 'test': 'test', 8 | 'train1': 'train', 9 | 'train2': 'train', 10 | 'train3': 'validate' 11 | } 12 | 13 | suffix = 'shas.pickle' 14 | 15 | data_dir = 'cp1data' 16 | image_dir = 'subset' 17 | 18 | size = 5000 19 | 20 | for group in ['pos', 'neg']: 21 | for prefix, part in parts.iteritems(): 22 | shas = pickle.load(open('_'.join([prefix, group, suffix]))) 23 | # for sha in itertools.islice(shas, size): 24 | for sha in shas: 25 | sha_bin = sha[:3] 26 | src = os.path.join(data_dir, group, sha_bin, sha) 27 | dst = os.path.join(image_dir, group, part + '_' + sha + '.jpg') 28 | shutil.copyfile(src, dst) 29 | -------------------------------------------------------------------------------- /scripts/memex/live_update_pipeline/update_process.md: -------------------------------------------------------------------------------- 1 | # Data and Product Update Process 2 | This is an informational listing of the process for syncing data and computing products for the new data. 3 | 4 | ## Sync Image files 5 | Currently using ``rsync`` to get new image files from imagecat.dyndns.org via the ``bin/memex/image_dump.rsync_from_imagecat_dyndns.sh`` script. 6 | This results in a log and a test file listing newly added files. 7 | 8 | ## Computing Descriptors/Hash Codes 9 | Follow the incremental update example at: 10 | 11 | http://smqtk.readthedocs.org/en/latest/examples/nnss_incremental_update/incremental_example.html 12 | 13 | These steps are encapsulated in the ``run.compute_many_descriptors.sh`` and ``run.compute_hash_codes.sh`` scripts. 14 | 15 | ## Full-stack 16 | Now, the full update process is encapsulated in the ``run.update.sh`` script. 17 | 18 | ### 3-way gun classifier 19 | Model files: 20 | 21 | - `classifier.svm.weapons_3_class_alexnet.labels` 22 | - `classifier.svm.weapons_3_class_alexnet.model` 23 | 24 | Use the ``run.compute_classifications.sh`` script, which wraps a call to the ``compute_classifications.py`` script but with more tightly defined I/O paths. 25 | See the configuration file ``config.jpl_weapons_v3.compute_classifications.json``. 26 | This will classify descriptors based on a UUIDs list, which is created when executing ``run.compute_hash_codes.sh``. 27 | Classifications are stored in the postgres database, as well as output to a CSV file for transport elsewhere (``image_dump..classifications.data.csv``). 28 | -------------------------------------------------------------------------------- /scripts/memex/live_update_pipeline/watch_cdr_get.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | output_dir="test_output" 4 | output_csv="test_output.csv" 5 | log_file="test_output.log" 6 | interval=1 # seconds 7 | 8 | watch -n ${interval} " 9 | # This first one will be expensive for large archives 10 | #echo \"All Files Downloaded : \$(find \"${output_dir}/\" -type f | wc -l)\"; 11 | echo \"CDR records scanned : \$(grep \"report_progress\" \"${log_file}\" | tail -1 | cut -d' ' -f20)\"; 12 | echo \"Download stored-data failures : \$(grep \"stored-data URL\" \"${log_file}\" | wc -l)\"; 13 | echo \"Download fallback failures : \$(grep \"original URL\" \"${log_file}\" | wc -l)\"; 14 | echo \"DL Failures caused by exceptions: \$(grep \"\(Exception\|Error\|FAILED\)\" \"${log_file}\" | wc -l)\" 15 | echo \"Files Recorded : \$(cat \"${output_csv}\" | wc -l)\"; 16 | echo \"Files downloaded : \$(grep \"Saving to file\" \"${log_file}\" | wc -l)\"; 17 | echo \"Scan Restarts : \$(grep \"Restarting query\" \"${log_file}\" | wc -l)\" 18 | " 19 | -------------------------------------------------------------------------------- /scripts/sample_build_configs/generate_image_transform.tiles.json: -------------------------------------------------------------------------------- 1 | { 2 | "brightness_levels": null, 3 | "contrast_levels": null, 4 | "crop": { 5 | "center_levels": null, 6 | "quadrant_pyramid_levels": null, 7 | "tile_shape": [128, 128] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /scripts/sample_build_configs/make_balltree.json: -------------------------------------------------------------------------------- 1 | { 2 | "hash2uuid_kv_store": { 3 | "MemoryKeyValueStore": { 4 | "cache_element": { 5 | "DataFileElement": { 6 | "explicit_mimetype": null, 7 | "filepath": "models/lsh_hash2uuids_kvstore.pickle", 8 | "readonly": true 9 | }, 10 | "type": "DataFileElement" 11 | } 12 | }, 13 | "type": "MemoryKeyValueStore" 14 | }, 15 | "itq_bit_length": 256, 16 | "sklearn_balltree": { 17 | "cache_element": { 18 | "DataFileElement": { 19 | "explicit_mimetype": null, 20 | "filepath": "models/hash_index_btree", 21 | "readonly": false 22 | }, 23 | "type": "DataFileElement" 24 | }, 25 | "leaf_size": 1000, 26 | "random_seed": 0 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /scripts/sample_build_configs/train_itq.json: -------------------------------------------------------------------------------- 1 | { 2 | "descriptor_set": { 3 | "PostgresDescriptorSet": { 4 | "create_table": true, 5 | "db_host": "localhost", 6 | "db_name": "postgres", 7 | "db_pass": null, 8 | "db_port": 5432, 9 | "db_user": "postgres", 10 | "element_col": "element", 11 | "multiquery_batch_size": 1000, 12 | "pickle_protocol": -1, 13 | "read_only": false, 14 | "table_name": "descriptor_set_alexnet_fc7", 15 | "uuid_col": "uid" 16 | }, 17 | "type": "PostgresDescriptorSet" 18 | }, 19 | "itq_config": { 20 | "bit_length": 256, 21 | "itq_iterations": 100, 22 | "mean_vec_cache": { 23 | "DataFileElement": { 24 | "explicit_mimetype": null, 25 | "filepath": "models/itq.model.b256_i100_r0_n2.mean_vec.npy", 26 | "readonly": false 27 | }, 28 | "type": "DataFileElement" 29 | }, 30 | "normalize": 2, 31 | "random_seed": 0, 32 | "rotation_cache": { 33 | "DataFileElement": { 34 | "explicit_mimetype": null, 35 | "filepath": "models/itq.model.b256_i100_r0_n2.rotation.npy", 36 | "readonly": false 37 | }, 38 | "type": "DataFileElement" 39 | } 40 | }, 41 | "uuids_list_filepath": null 42 | } 43 | -------------------------------------------------------------------------------- /setup_env.build.sh.in: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # SMQTK system setup script (build version) 4 | # 5 | SMQTK_SOURCE="@SMQTK_SOURCE_DIR@" 6 | SMQTK_BINARY="@SMQTK_BINARY_DIR@" 7 | SMQTK_TPL="@TPL_LOCAL_INSTALL@" 8 | SMQTK_TPL_PYTHON="@TPL_PYTHON_SP@" 9 | 10 | export PATH="${SMQTK_SOURCE}/bin:${SMQTK_BINARY}/src/frame_extractor/:${SMQTK_BINARY}/src/cnn_feature_extractor/:${PATH}" 11 | 12 | # Only add third party libraries if the directory exists, they might 13 | # not if users disabled building SVM/Flann during install. 14 | if [ -d "${SMQTK_TPL}" ]; then 15 | export PATH="${SMQTK_TPL}/bin:${PATH}" 16 | fi 17 | 18 | export PYTHONPATH="${SMQTK_SOURCE}/python:$PYTHONPATH" 19 | 20 | if [ -d "${SMQTK_TPL_PYTHON}" ]; then 21 | export PYTHONPATH="${SMQTK_TPL_PYTHON}:$PYTHONPATH" 22 | fi 23 | -------------------------------------------------------------------------------- /setup_env.install.sh.in: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Setup the run environment (install environment) 4 | # 5 | 6 | # Assuming bash environment 7 | SMQTK_INSTALL="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 8 | export PATH="${SMQTK_INSTALL}/bin:${PATH}" 9 | export PYTHONPATH="${SMQTK_INSTALL}/@python_site_packages@:${PYTHONPATH}" 10 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | option(SMQTK_BUILD_FRAME_EXTRACTOR 2 | "Enable building of the frame_extractor tool. This requires VXL." 3 | OFF 4 | ) 5 | if( SMQTK_BUILD_FRAME_EXTRACTOR ) 6 | add_subdirectory( frame_extractor ) 7 | endif() 8 | 9 | option(SMQTK_BUILD_CNN_FEATURE_EXTRACTOR 10 | "Enable building of the deep learning feature extraction. This requires Caffe." 11 | OFF 12 | ) 13 | if( SMQTK_BUILD_CNN_FEATURE_EXTRACTOR ) 14 | add_subdirectory( cnn_feature_extractor ) 15 | endif() 16 | -------------------------------------------------------------------------------- /src/cnn_feature_extractor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(smqtk_cnn_feature_extractor) 2 | cmake_minimum_required(VERSION 2.8) 3 | 4 | find_package(OpenCV REQUIRED) 5 | find_package(Caffe REQUIRED) 6 | add_definitions( ${Caffe_DEFINITIONS} ) 7 | include_directories(SYSTEM ${Caffe_INCLUDE_DIRS}) 8 | 9 | set(cnn_feature_extractor_srcs 10 | cnn_feature_extractor.cxx 11 | ) 12 | 13 | add_executable(cnn_feature_extractor 14 | ${cnn_feature_extractor_srcs}) 15 | target_link_libraries(cnn_feature_extractor 16 | ${Caffe_LIBRARIES}) 17 | 18 | install( 19 | TARGETS cnn_feature_extractor 20 | DESTINATION bin 21 | COMPONENT tools 22 | ) 23 | 24 | configure_file( "${SMQTK_SOURCE_DIR}/data/caffenet/file_list.txt.in" 25 | "${SMQTK_BINARY_DIR}/data/caffenet/file_list.txt" ) 26 | -------------------------------------------------------------------------------- /src/cnn_feature_extractor/README.txt: -------------------------------------------------------------------------------- 1 | Using Caffe (http://caffe.berkeleyvision.org/), cnn_feature_extractor extracts a feature vector from each input image. It is assumed Caffe has been built with CMake, and the header files and library (libcaffe.so) are available through CMake-generated CaffeConfig.cmake. 2 | 3 | To run the program, first download two files from Caffe and copy them to 4 | ${SMQTK_BINARY_DIR}/data/caffenet. 5 | 1) bvlc_reference_caffenet.caffemodel 6 | run `scripts/download_model_binary.py models/bvlc_reference_caffenet` from Caffe root directory 7 | 2) imagenet_mean.binaryproto 8 | wget http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz 9 | 10 | Then customize imagenet_val.prototxt. Make sure layer.transform_param.mean_file points to imagenet_mean.binaryprot, and layer.image_data_param.soruce points to file_list.txt. Change layer.image_data_param.batch_size to 50 (or 1) and the second to last parameter on the following command line to 10 (or 3). 11 | 12 | Finally run the following command from the shell 13 | cd ${SMQTK_BINARY_DIR} 14 | 15 | ./src/cnn_feature_extractor/cnn_feature_extractor data/caffenet/bvlc_reference_caffenet.caffemodel data/caffenet/imagenet_val.prototxt fc7 data/caffenet/cnn 3 csv 16 | 17 | ./src/cnn_feature_extractor/cnn_feature_extractor data/caffenet/bvlc_reference_caffenet.caffemodel data/caffenet/imagenet_val.prototxt fc7 data/caffenet/cnn 3 svm 18 | 19 | ./src/cnn_feature_extractor/cnn_feature_extractor data/caffenet/bvlc_reference_caffenet.caffemodel data/caffenet/imagenet_val.prototxt fc7 data/caffenet/cnn 3 stdout >& /dev/null 20 | 21 | If you get a segfault, double check the settings in imagenet_val.prototxt. 22 | -------------------------------------------------------------------------------- /src/frame_extractor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(smqtk_frame_extractor) 2 | cmake_minimum_required(VERSION 2.8) 3 | 4 | find_package(Boost 1.50 REQUIRED) #COMPONENTS 5 | add_definitions(-DBOOST_ALL_NO_LIB) 6 | include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) 7 | 8 | find_package(VXL REQUIRED) 9 | include("${VXL_CMAKE_DIR}/UseVXL.cmake") 10 | 11 | set(frame_extractor_srcs 12 | frame_extractor.cxx) 13 | 14 | add_executable(frame_extractor 15 | ${frame_extractor_srcs}) 16 | target_link_libraries(frame_extractor 17 | vidl vul) 18 | 19 | set_property( 20 | TARGET frame_extractor 21 | PROPERTY 22 | COMPILE_FLAGS "${flags}") 23 | 24 | install( 25 | TARGETS frame_extractor 26 | DESTINATION bin 27 | COMPONENT tools 28 | ) 29 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") 4 | -------------------------------------------------------------------------------- /tests/algorithms/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/algorithms/classifier/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/algorithms/classifier/__init__.py -------------------------------------------------------------------------------- /tests/algorithms/descriptor_generator/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/algorithms/image_io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/algorithms/image_io/__init__.py -------------------------------------------------------------------------------- /tests/algorithms/nn_index/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/algorithms/nn_index/__init__.py -------------------------------------------------------------------------------- /tests/algorithms/nn_index/hash_index/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/algorithms/nn_index/hash_index/__init__.py -------------------------------------------------------------------------------- /tests/algorithms/nn_index/lsh_hash_functors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/algorithms/nn_index/lsh_hash_functors/__init__.py -------------------------------------------------------------------------------- /tests/algorithms/nn_index/lsh_hash_functors/test_abstract.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | import unittest 3 | 4 | import unittest.mock as mock 5 | 6 | from smqtk.algorithms.nn_index.lsh.functors import LshFunctor 7 | 8 | 9 | class DummyLshFunctor (LshFunctor): 10 | 11 | @classmethod 12 | def is_usable(cls): 13 | return True 14 | 15 | def get_config(self): 16 | pass 17 | 18 | def get_hash(self, descriptor): 19 | pass 20 | 21 | 22 | class testLshFunctorAbstract (unittest.TestCase): 23 | 24 | def test_call(self): 25 | # calling an instance should get us to the get_hash method. 26 | f = DummyLshFunctor() 27 | f.get_hash = mock.MagicMock() 28 | 29 | expected_descriptor = 'pretend descriptor element' 30 | f(expected_descriptor) 31 | f.get_hash.assert_called_once_with(expected_descriptor) 32 | -------------------------------------------------------------------------------- /tests/algorithms/object_detection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/algorithms/object_detection/__init__.py -------------------------------------------------------------------------------- /tests/algorithms/rank_relevancy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/algorithms/rank_relevancy/__init__.py -------------------------------------------------------------------------------- /tests/algorithms/relevancy_index/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/algorithms/relevancy_index/test_RI_abstract.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | import unittest 3 | 4 | import unittest.mock as mock 5 | 6 | from smqtk.algorithms.relevancy_index import RelevancyIndex 7 | 8 | 9 | class DummyRI (RelevancyIndex): 10 | 11 | @classmethod 12 | def is_usable(cls): 13 | return True 14 | 15 | def rank(self, pos, neg): 16 | pass 17 | 18 | def get_config(self): 19 | pass 20 | 21 | def count(self): 22 | return 0 23 | 24 | def build_index(self, descriptors): 25 | pass 26 | 27 | 28 | class TestSimilarityIndexAbstract (unittest.TestCase): 29 | 30 | def test_count(self): 31 | index = DummyRI() 32 | self.assertEqual(index.count(), 0) 33 | self.assertEqual(index.count(), len(index)) 34 | 35 | # Pretend that there were things in there. Len should pass it though 36 | index.count = mock.Mock() 37 | index.count.return_value = 5 38 | self.assertEqual(len(index), 5) 39 | -------------------------------------------------------------------------------- /tests/data/Hopper.alexnet_fc7_output.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/Hopper.alexnet_fc7_output.npy -------------------------------------------------------------------------------- /tests/data/Lenna.alexnet_fc7_output.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/Lenna.alexnet_fc7_output.npy -------------------------------------------------------------------------------- /tests/data/Lenna.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/Lenna.png -------------------------------------------------------------------------------- /tests/data/caffe.dummpy_network.prototxt: -------------------------------------------------------------------------------- 1 | name: "AlexNet" 2 | layer { 3 | name: "data" 4 | type: "Input" 5 | top: "data" 6 | input_param { shape: { dim: 10 dim: 3 dim: 227 dim: 227 } } 7 | } 8 | layer { 9 | name: "conv" 10 | type: "Convolution" 11 | bottom: "data" 12 | top: "conv" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | convolution_param { 22 | num_output: 96 23 | kernel_size: 11 24 | stride: 4 25 | } 26 | } 27 | layer { 28 | name: "norm" 29 | type: "LRN" 30 | bottom: "conv" 31 | top: "norm" 32 | lrn_param { 33 | local_size: 5 34 | alpha: 0.0001 35 | beta: 0.75 36 | } 37 | } 38 | layer { 39 | name: "pool" 40 | type: "Pooling" 41 | bottom: "norm" 42 | top: "pool" 43 | pooling_param { 44 | pool: MAX 45 | kernel_size: 3 46 | stride: 2 47 | } 48 | } 49 | layer { 50 | name: "fc" 51 | type: "InnerProduct" 52 | bottom: "pool" 53 | top: "fc" 54 | param { 55 | lr_mult: 1 56 | decay_mult: 1 57 | } 58 | param { 59 | lr_mult: 2 60 | decay_mult: 0 61 | } 62 | inner_product_param { 63 | num_output: 4096 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tests/data/caffe.dummy_mean.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/caffe.dummy_mean.npy -------------------------------------------------------------------------------- /tests/data/caffe.empty_model.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/caffe.empty_model.caffemodel -------------------------------------------------------------------------------- /tests/data/grace_hopper.100x100+100+100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/grace_hopper.100x100+100+100.png -------------------------------------------------------------------------------- /tests/data/grace_hopper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/grace_hopper.png -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/0/0/UUID_000.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/0/0/UUID_000.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/0/0/UUID_001.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/0/0/UUID_001.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/0/0/UUID_003.dataelemenT: -------------------------------------------------------------------------------- 1 | Name case difference from expected. 2 | -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/0/1/UUID_012.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/0/1/UUID_012.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/1/3/something_else: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/1/3/something_else -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/1/8/UUID_180.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/1/8/UUID_180.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/3/1/UUID_317.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/3/1/UUID_317.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/4/3/2/1/UUID_43210.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/4/3/2/1/UUID_43210.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/UUID_0.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/UUID_0.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/UUID_1.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/UUID_1.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/UUID_2.dataElement: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/UUID_2.dataElement -------------------------------------------------------------------------------- /tests/data/test_data_file_set_tree/nonmatching_filename: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_data_file_set_tree/nonmatching_filename -------------------------------------------------------------------------------- /tests/data/test_file.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/data/test_file.dat -------------------------------------------------------------------------------- /tests/data/test_labels.txt: -------------------------------------------------------------------------------- 1 | label_1 2 | label_2 3 | negative 4 | label_3 5 | Kitware 6 | label_4 7 | -------------------------------------------------------------------------------- /tests/data/text_file: -------------------------------------------------------------------------------- 1 | Some text content. 2 | -------------------------------------------------------------------------------- /tests/iqr/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/iqr/__init__.py -------------------------------------------------------------------------------- /tests/representation/ClassificationElement/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/representation/DataElement/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/representation/DataElement/__init__.py -------------------------------------------------------------------------------- /tests/representation/DataSet/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/representation/DescriptorElement/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/representation/DescriptorElement/test_DescriptorSolrElement.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import unittest.mock as mock 4 | import pytest 5 | 6 | from smqtk.representation.descriptor_element.solr_element import \ 7 | SolrDescriptorElement 8 | from smqtk.utils.configuration import configuration_test_helper 9 | 10 | 11 | @pytest.mark.skipif(not SolrDescriptorElement.is_usable(), 12 | reason='SolrDescriptorElement reports as not usable.') 13 | class TestDescriptorSolrElement (unittest.TestCase): 14 | 15 | TEST_URL = 'http://localhost:8983/solr' 16 | 17 | @mock.patch("solr.Solr") 18 | def test_configuration(self, _mock_Solr): 19 | inst = SolrDescriptorElement( 20 | 'test', 'a', 21 | solr_conn_addr=self.TEST_URL, 22 | type_field='type_s', uuid_field='uuid_s', vector_field='vector_fs', 23 | timestamp_field='timestamp_f', timeout=101, 24 | persistent_connection=True, commit_on_set=False, 25 | ) 26 | for i in configuration_test_helper(inst, {'type_str', 'uuid'}, 27 | ('test', 'abcd')): # type: SolrDescriptorElement 28 | assert i.solr_conn_addr == self.TEST_URL 29 | assert i.type_field == 'type_s' 30 | assert i.uuid_field == 'uuid_s' 31 | assert i.vector_field == 'vector_fs' 32 | assert i.timestamp_field == 'timestamp_f' 33 | assert i.solr_timeout == 101 34 | assert i.solr_persistent_connection is True 35 | assert i.solr_commit_on_set is False 36 | -------------------------------------------------------------------------------- /tests/representation/DescriptorSet/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/representation/DetectionElement/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/representation/DetectionElement/__init__.py -------------------------------------------------------------------------------- /tests/representation/KeyValueStore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/representation/KeyValueStore/__init__.py -------------------------------------------------------------------------------- /tests/representation/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "paul.tunison@kitware.com" 2 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/utils/__init__.py -------------------------------------------------------------------------------- /tests/utils/file_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/utils/file_utils/__init__.py -------------------------------------------------------------------------------- /tests/utils/file_utils/test_file_mimetype_tika.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | from unittest import TestCase 3 | 4 | from smqtk.utils.file import file_mimetype_tika 5 | 6 | from tests import TEST_DATA_DIR 7 | 8 | try: 9 | from tika import detector as tika_detector # type: ignore 10 | except ImportError: 11 | tika_detector = None 12 | 13 | 14 | if tika_detector is not None: 15 | 16 | class TestFile_mimetype_tika(TestCase): 17 | 18 | def test_file_doesnt_exist(self): 19 | try: 20 | file_mimetype_tika('/this/path/probably/doesnt/exist.txt') 21 | except IOError as ex: 22 | self.assertEqual(ex.errno, 2, 23 | "Expected directory IO error #2. " 24 | "Got %d" % ex.errno) 25 | 26 | def test_directory_provided(self): 27 | try: 28 | file_mimetype_tika(TEST_DATA_DIR) 29 | except IOError as ex: 30 | self.assertEqual(ex.errno, 21, 31 | "Expected directory IO error #21. " 32 | "Got %d" % ex.errno) 33 | 34 | def test_get_mimetype_lenna(self): 35 | m = file_mimetype_tika( 36 | os.path.join(TEST_DATA_DIR, 'Lenna.png') 37 | ) 38 | self.assertEqual(m, 'image/png') 39 | 40 | def test_get_mimetype_no_extension(self): 41 | m = file_mimetype_tika( 42 | os.path.join(TEST_DATA_DIR, 'text_file') 43 | ) 44 | self.assertEqual(m, 'text/plain') 45 | -------------------------------------------------------------------------------- /tests/utils/test_plugin_dir/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/utils/test_plugin_dir/__init__.py -------------------------------------------------------------------------------- /tests/utils/test_plugin_dir/module_of_more_stuff.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | # intentional duplicate w.r.t. "stuff" module. 3 | from pathlib import Path # noqa: F401 4 | import tests.utils.test_plugin_dir.module_of_stuff as module_of_stuff 5 | 6 | 7 | class NewCustomType(argparse.ArgumentParser): 8 | ... 9 | 10 | 11 | class AnotherDerived(module_of_stuff.ClassDefinition): 12 | def cool_thing(self) -> str: 13 | return "something different" 14 | 15 | 16 | nct_type = type(NewCustomType) 17 | -------------------------------------------------------------------------------- /tests/utils/test_plugin_dir/module_of_stuff.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test module defining various things for testing type value extraction. 3 | """ 4 | import abc 5 | import os # noqa: F401 6 | from pathlib import Path # noqa: F401 7 | 8 | a = 1 # an instance 9 | b = str # exporting an alias to the type! 10 | 11 | 12 | class ClassDefinition(metaclass=abc.ABCMeta): 13 | @abc.abstractmethod 14 | def cool_thing(self) -> str: ... 15 | 16 | 17 | class Derived(ClassDefinition): 18 | 19 | def cool_thing(self) -> str: 20 | return "it's cool" 21 | 22 | 23 | class StillAbstract(ClassDefinition): 24 | ... 25 | 26 | 27 | class_instance = Derived() 28 | -------------------------------------------------------------------------------- /tests/utils/test_plugin_dir/module_with_exception.py: -------------------------------------------------------------------------------- 1 | from .module_of_stuff import ClassDefinition 2 | 3 | 4 | class WizzyBang(ClassDefinition): 5 | ... 6 | 7 | 8 | raise RuntimeError("Expected error on import") 9 | -------------------------------------------------------------------------------- /tests/utils/test_postgres.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import pytest 4 | 5 | from smqtk.utils.postgres import psycopg2, PsqlConnectionHelper 6 | 7 | 8 | @pytest.mark.skipif(psycopg2 is None, 9 | reason="Psycopg2 module is not importable, postgres " 10 | "utilities are not available.") 11 | class TestPsqlConnectionHelper (unittest.TestCase): 12 | 13 | def setUp(self): 14 | self.conn_helper = PsqlConnectionHelper() 15 | 16 | def test_batch_execute_on_empty_iterable(self): 17 | # noinspection PyUnusedLocal 18 | def exec_hook(cur, batch): 19 | raise Exception('This line shouldn\'t be reached with an empty ' 20 | 'iterable.') 21 | 22 | list(self.conn_helper.batch_execute(iter(()), exec_hook, 1)) 23 | -------------------------------------------------------------------------------- /tests/utils/test_string_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | import unittest 3 | 4 | from smqtk.utils.string import random_characters 5 | 6 | 7 | class TestRandomCharacters (unittest.TestCase): 8 | 9 | def test_zero_n(self): 10 | self.assertEqual(random_characters(0), '') 11 | 12 | def test_no_char_set(self): 13 | self.assertRaisesRegex( 14 | ValueError, 15 | "Empty char_set given", 16 | random_characters, 5, () 17 | ) 18 | 19 | def test_negative_n(self): 20 | self.assertRaisesRegex( 21 | ValueError, 22 | "n must be a positive integer", 23 | random_characters, -1234 24 | ) 25 | 26 | def test_floating_point_n(self): 27 | # Testing that this should cast down to the integer 4. 28 | # noinspection PyTypeChecker 29 | s = random_characters(4.7) 30 | self.assertEqual(len(s), 4) 31 | -------------------------------------------------------------------------------- /tests/utils/test_web.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from smqtk.utils.web import ServiceProxy 3 | 4 | 5 | class TestServiceProxy (unittest.TestCase): 6 | """ Tests for the ServiceProxy helper class """ 7 | 8 | def test_init_no_scheme_fill_in(self): 9 | """ Test that when no http/https scheme is present at the beginning 10 | of the URL provided at construction time that a "http://" scheme is 11 | prepended. 12 | """ 13 | test_url = "foo/bar" 14 | expected = "http://foo/bar" 15 | assert ServiceProxy(test_url).url == expected 16 | 17 | test_url = "://foo/bar" 18 | expected = "http://://foo/bar" 19 | assert ServiceProxy(test_url).url == expected 20 | 21 | def test_init_url_hash_scheme(self): 22 | """ Test that if a scheme is present at the head of the URL that it is 23 | not changed. 24 | """ 25 | test_url = 'http://this.site/bar' 26 | assert ServiceProxy(test_url).url == test_url 27 | 28 | test_url = 'http://http://this.site/bar' 29 | assert ServiceProxy(test_url).url == test_url 30 | 31 | test_url = 'https://this.site/bar' 32 | assert ServiceProxy(test_url).url == test_url 33 | -------------------------------------------------------------------------------- /tests/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/web/__init__.py -------------------------------------------------------------------------------- /tests/web/classifier_service/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/web/classifier_service/__init__.py -------------------------------------------------------------------------------- /tests/web/iqr_service/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kitware/SMQTK/ef4282dd9b648ccf6cb25fd2b4e7c58463439640/tests/web/iqr_service/__init__.py --------------------------------------------------------------------------------