├── .gitignore ├── FacebookAI.png ├── README.md ├── css ├── bootstrap.css ├── bootstrap.min.css ├── mdb.css └── mdb.min.css ├── demo ├── 2017266_map.jpg ├── 2017266_raw.jpg ├── 6862937532_map.jpg ├── 6862937532_raw.jpg ├── NoisyLady.png ├── Picture1.jpg ├── Picture2.png ├── Picture3.jpg ├── i07_08_4.bmp ├── i07_08_5.bmp ├── magma.png ├── motion0022_map.jpg ├── motion0022_raw.jpg ├── motion0072_map.jpg ├── motion0072_raw.jpg ├── motion0089_map.jpg ├── motion0089_raw.jpg ├── motion0114.jpg ├── motion0160_map.jpg ├── motion0160_raw.jpg ├── mp4 │ ├── 12255205254.mp4 │ ├── 6293871635.mp4 │ ├── 8069880437.mp4 │ └── 8440795977.mp4 ├── out_of_focus0083_map.jpg ├── out_of_focus0083_raw.jpg ├── out_of_focus0161_map.jpg ├── out_of_focus0161_raw.jpg ├── out_of_focus0385_map.jpg ├── out_of_focus0385_raw.jpg ├── out_of_focus0470_map.jpg ├── out_of_focus0470_raw.jpg └── out_of_focus0689.jpg ├── fastiqa ├── __init__.py ├── _loss.py ├── all.py ├── basics.py ├── browser.py ├── bunch.py ├── bunches │ ├── __init__.py │ ├── _feat.py │ ├── feat2mos.py │ ├── iqa │ │ ├── __init__.py │ │ ├── im2mos.py │ │ ├── im_bbox2mos.py │ │ ├── im_feat2mos.py │ │ ├── im_roi2mos.py │ │ └── test_images.py │ ├── v1 │ │ └── im2mos.py │ └── vqa │ │ ├── __init__.py │ │ ├── _clip.py │ │ ├── single_vid2mos.py │ │ ├── test_feats.py │ │ ├── test_videos.py │ │ ├── vid2mos.py │ │ ├── vid_sp2mos.py │ │ └── video_utils.py ├── common.py ├── dev.py ├── inference_model.py ├── iqa.py ├── iqa_exp.py ├── learn.py ├── log.py ├── loss.py ├── models │ ├── _3d.py │ ├── __inceptiontime_layers.py │ ├── __init__.py │ ├── _body_head.py │ ├── _inceptiontime.py │ ├── _roi_pool.py │ ├── inception_head.py │ ├── mobilenet │ │ ├── __init__.py │ │ ├── _v2.py │ │ └── v2.py │ ├── nima.py │ ├── resnet3d │ │ ├── __init__.py │ │ ├── densenet.py │ │ ├── pre_act_resnet.py │ │ ├── resnet.py │ │ ├── resnet2p1d.py │ │ ├── resnext.py │ │ ├── utils.py │ │ └── wide_resnet.py │ ├── resnet_3d.py │ ├── tfm_pth.py │ ├── total_params.py │ └── vsfa.py ├── qmap.py ├── utils │ └── cached_property.py └── vqa.py ├── favicon.ico ├── file_upload.htm ├── filepond ├── filepond.min.css ├── filepond.min.js └── filepond_example.htm ├── font └── roboto │ ├── Roboto-Bold.eot │ ├── Roboto-Bold.ttf │ ├── Roboto-Bold.woff │ ├── Roboto-Bold.woff2 │ ├── Roboto-Light.eot │ ├── Roboto-Light.ttf │ ├── Roboto-Light.woff │ ├── Roboto-Light.woff2 │ ├── Roboto-Medium.eot │ ├── Roboto-Medium.ttf │ ├── Roboto-Medium.woff │ ├── Roboto-Medium.woff2 │ ├── Roboto-Regular.eot │ ├── Roboto-Regular.ttf │ ├── Roboto-Regular.woff │ ├── Roboto-Regular.woff2 │ ├── Roboto-Thin.eot │ ├── Roboto-Thin.ttf │ ├── Roboto-Thin.woff │ └── Roboto-Thin.woff2 ├── foot.htm ├── head.htm ├── heatmap.htm ├── hist.htm ├── images ├── A021.gif ├── A037.gif ├── Noisy Lady.png ├── Picture1.jpg ├── Picture2.png ├── Picture3.jpg ├── demo.jpg ├── ex1.gif ├── ex2.gif └── pvq_db.png ├── imgs └── CVPR_Logo_Horz2_web.jpg ├── index.html ├── js ├── bootstrap.js ├── bootstrap.min.js ├── include_once.js ├── jquery-3.3.1.min.js ├── mdb.js ├── mdb.min.js ├── plotly-latest.min.js └── popper.min.js ├── json ├── AVA.json ├── CLIVE.json ├── KoNViD.json ├── KonIQ.json ├── LIVE_FB_IQA.json ├── LIVE_VQC.json └── LSVQ.json ├── license-free.txt ├── main.py ├── paq2piq ├── abstract.htm ├── applications.htm ├── download_zone.htm └── online_demo.htm ├── pvq.ini ├── pvq ├── abstract.htm └── download_zone.htm ├── styles ├── column.css ├── heatmap.css ├── main.css ├── reset-this.css ├── social_bar.css └── switch.css ├── template.htm └── test_PVQ_on_new_datasets.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | *.m~ 2 | *.pyc 3 | *.torch 4 | *.pth 5 | *.t7 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 104 | __pypackages__/ 105 | 106 | # Celery stuff 107 | celerybeat-schedule 108 | celerybeat.pid 109 | 110 | # SageMath parsed files 111 | *.sage.py 112 | 113 | # Environments 114 | .env 115 | .venv 116 | env/ 117 | venv/ 118 | ENV/ 119 | env.bak/ 120 | venv.bak/ 121 | 122 | # Spyder project settings 123 | .spyderproject 124 | .spyproject 125 | 126 | # Rope project settings 127 | .ropeproject 128 | 129 | # mkdocs documentation 130 | /site 131 | 132 | # mypy 133 | .mypy_cache/ 134 | .dmypy.json 135 | dmypy.json 136 | 137 | # Pyre type checker 138 | .pyre/ 139 | 140 | # pytype static type analyzer 141 | .pytype/ 142 | 143 | # Cython debug symbols 144 | cython_debug/ 145 | -------------------------------------------------------------------------------- /FacebookAI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/FacebookAI.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PatchVQ 2 | Patch-VQ: ‘Patching Up’ the Video Quality Problem 3 | 4 | ***Please email yingzhenqiang at gmail dot com for any questions. Thank you!*** 5 | 6 | ## Demo 7 | 8 | Please follow [this](https://colab.research.google.com/drive/1612Y7tiAgiutmt6rXHCjjKgNdJXJ5Hvo) to test the Patch VQ model pretrained on the LSVQ database. 9 | Please follow [this](test_PVQ_on_new_datasets.ipynb) to test our Patch VQ model on your database. 10 | 11 | # Note 12 | 13 | * Due to the [breaking changes](https://github.com/fastai/fastai/blob/master/CHANGELOG.md#2018) in `fastai 2.0.18 --> 2.1`, the code is incompatible with the latest fastai! Please make sure the following versions are installed: 14 | ``` 15 | fastai 2.0.18 16 | fastcore 1.2.5 17 | torch 1.6.0 18 | torchvision 0.7.0 19 | ``` 20 | 21 | ​ a new version is still work-in-progress. See [here](https://github.com/baidut/fastiqa). 22 | 23 | * To reproduce the results in the paper, make sure to use the pretrained weights provided here: 24 | * [RoIPoolModel-fit.10.bs.120.pth](https://github.com/baidut/PatchVQ/releases/download/v0.1/RoIPoolModel-fit.10.bs.120.pth) for PaQ2PiQ (don't use weights from other sources) 25 | * [fastai-r3d18_K_200ep.pth](https://github.com/baidut/PatchVQ/releases/download/v0.1/fastai-r3d18_K_200ep.pth) for resnet 3d (don't use pytorch builtin `r3d_18`) 26 | 27 | **Common Issues:** 28 | 29 | * `NameError: name '_C' is not defined` --> `pip3 install Cython` 30 | * `RuntimeError: Could not infer dtype of PILImage` --> downgrade: pip install pillow==8.2 31 | * `AttributeError: module 'PIL.Image' has no attribute 'Resampling'` --> check your fastai and fastcore version 32 | * `ImportError: cannot import name 'default_generator' from 'torch._C' (unknown location)` --> `pip install torch==1.6.0 torchvision==0.7.0` 33 | 34 | ## Download LSVQ database 35 | 36 | **Description** 37 | 38 | No-reference (NR) perceptual video quality assessment (VQA) is a complex, unsolved, and important problem to social and streaming media applications. Efficient and accurate video quality predictors are needed to monitor and guide the processing of billions of shared, often imperfect, user-generated content (UGC). Unfortunately, current NR models are limited in their prediction capabilities on real-world, "in-the-wild" UGC video data. To advance progress on this problem, we created the largest (by far) subjective video quality dataset, containing 39, 000 real-world distorted videos and 117, 000 space-time localized video patches ("v-patches"), and 5.5M human perceptual quality annotations. Using this, we created two unique NR-VQA models: (a) a local-to-global region-based NR VQA architecture (called PVQ) that learns to predict global video quality and achieves state-of-the-art performance on 3 UGC datasets, and (b) a first-of-a-kind space-time video quality mapping engine (called PVQ Mapper) that helps localize and visualize perceptual distortions in space and time. We will make the new database and prediction models available immediately following the review process. 39 | 40 | **Investigators** 41 | 42 | * Zhenqiang Ying () -- Graduate Student, Dept. of ECE, UT Austin 43 | * Maniratnam Mandal () -- Graduate Student, Dept. of ECE, UT Austin 44 | * Deepti Ghadiyaram (), Facebook Inc. 45 | * Alan Bovik ([bovik@ece.utexas.edu](mailto:bovik@ece.utexas.edu)) -- Professor, Dept. of ECE, UT Austin 46 | 47 | **Download** 48 | 49 | **We are making the LSVQ Database available to the research community free of charge. If you use this database in your research, we kindly ask that you reference our papers listed below:** 50 | 51 | > - Z. Ying, M. Mandal, D. Ghadiyaram and A.C. Bovik, "Patch-VQ: ‘Patching Up’ the Video Quality Problem," arXiv 2020.[[paper\]](https://arxiv.org/pdf/2011.13544.pdf) 52 | > - Z. Ying, M. Mandal, D. Ghadiyaram and A.C. Bovik, "LIVE Large-Scale Social Video Quality (LSVQ) Database", Online:https://github.com/baidut/PatchVQ, 2020. 53 | 54 | 55 | **Please fill [THIS FORM ](https://forms.gle/cdxArkHUuuqRCX4Z9) to download our database.** 56 | 57 | # Update 2025: We found some issues: automatic form reply was broken and some videos might not be accessible from their original sites (IA/YFCC) anymore. Sorry for the inconvience! You can directly download all videos (IA, YFCC) from the [Box](https://utexas.box.com/s/3x10cuh5m2r85gcjmatgagkpf2ekgqwo) instead of running the script that downloads from the original sites. 58 | Please use this password to access the Box: `LiveLsvq@2021` and if any of the zip files is password-protected, please use: `LIVE2020` to unzip. Please feel free to contact me via yingzhenqiang-at-gmail-dot-com for downloading issues. 59 | 60 | 1. follow '[download_from_internetarchive.ipynb](https://colab.research.google.com/drive/16C4cEe-DRxwMUnMS-PQzjzGPs2HfIX6a)' to download Internet archive videos 61 | 2. download YFCC videos from [Box](https://utexas.box.com/s/3x10cuh5m2r85gcjmatgagkpf2ekgqwo) (The password will be sent to your email after you submit the request form.) 62 | 3. download label files (coordinates and scores). 63 | * [labels_test_1080p.csv](https://github.com/baidut/PatchVQ/releases/download/v0.1/labels_test_1080p.csv) 1.05 MB 64 | * [labels_train_test.csv](https://github.com/baidut/PatchVQ/releases/download/v0.1/labels_train_test.csv) 10.8 MB (`is_test` column denotes if a video is in the train set or the test set ) 65 | 4. [optional] follow [this](https://drive.google.com/file/d/16B1UvoIE2iCzSt7moXRwfBGpB_nRy5qE/view?usp=sharing) crop patches from videos 66 | 67 | 68 | **Copyright Notice** 69 | 70 | -----------COPYRIGHT NOTICE STARTS WITH THIS LINE------------ 71 | Copyright (c) 2020 The University of Texas at Austin 72 | All rights reserved. 73 | 74 | Permission is hereby granted, without written agreement and without license or royalty fees, to use, copy, modify, and distribute this database (the images, the results and the source files) and its documentation for any purpose, provided that the copyright notice in its entirety appear in all copies of this database, and the original source of this database, Laboratory for Image and Video Engineering (LIVE, [http://live.ece.utexas.edu ](http://live.ece.utexas.edu/)) at the University of Texas at Austin (UT Austin, [http://www.utexas.edu ](http://www.utexas.edu/)), is acknowledged in any publication that reports research using this database. 75 | 76 | The following papers are to be cited in the bibliography whenever the database is used as: 77 | 78 | > - Z. Ying, M. Mandal, D. Ghadiyaram and A.C. Bovik, "Patch-VQ: ‘Patching Up’ the Video Quality Problem," arXiv 2020.[[paper\]](https://arxiv.org/pdf/2011.13544.pdf) 79 | > - Z. Ying, M. Mandal, D. Ghadiyaram and A.C. Bovik, "LIVE Large-Scale Social Video Quality (LSVQ) Database", Online:https://github.com/baidut/PatchVQ, 2020. 80 | 81 | IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT AUSTIN BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF THIS DATABASE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF TEXAS AT AUSTIN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 82 | 83 | THE UNIVERSITY OF TEXAS AT AUSTIN SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE DATABASE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF TEXAS AT AUSTIN HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 84 | 85 | -----------COPYRIGHT NOTICE ENDS WITH THIS LINE------------ 86 | -------------------------------------------------------------------------------- /demo/2017266_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/2017266_map.jpg -------------------------------------------------------------------------------- /demo/2017266_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/2017266_raw.jpg -------------------------------------------------------------------------------- /demo/6862937532_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/6862937532_map.jpg -------------------------------------------------------------------------------- /demo/6862937532_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/6862937532_raw.jpg -------------------------------------------------------------------------------- /demo/NoisyLady.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/NoisyLady.png -------------------------------------------------------------------------------- /demo/Picture1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/Picture1.jpg -------------------------------------------------------------------------------- /demo/Picture2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/Picture2.png -------------------------------------------------------------------------------- /demo/Picture3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/Picture3.jpg -------------------------------------------------------------------------------- /demo/i07_08_4.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/i07_08_4.bmp -------------------------------------------------------------------------------- /demo/i07_08_5.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/i07_08_5.bmp -------------------------------------------------------------------------------- /demo/magma.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/magma.png -------------------------------------------------------------------------------- /demo/motion0022_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0022_map.jpg -------------------------------------------------------------------------------- /demo/motion0022_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0022_raw.jpg -------------------------------------------------------------------------------- /demo/motion0072_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0072_map.jpg -------------------------------------------------------------------------------- /demo/motion0072_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0072_raw.jpg -------------------------------------------------------------------------------- /demo/motion0089_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0089_map.jpg -------------------------------------------------------------------------------- /demo/motion0089_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0089_raw.jpg -------------------------------------------------------------------------------- /demo/motion0114.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0114.jpg -------------------------------------------------------------------------------- /demo/motion0160_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0160_map.jpg -------------------------------------------------------------------------------- /demo/motion0160_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/motion0160_raw.jpg -------------------------------------------------------------------------------- /demo/mp4/12255205254.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/mp4/12255205254.mp4 -------------------------------------------------------------------------------- /demo/mp4/6293871635.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/mp4/6293871635.mp4 -------------------------------------------------------------------------------- /demo/mp4/8069880437.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/mp4/8069880437.mp4 -------------------------------------------------------------------------------- /demo/mp4/8440795977.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/mp4/8440795977.mp4 -------------------------------------------------------------------------------- /demo/out_of_focus0083_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0083_map.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0083_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0083_raw.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0161_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0161_map.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0161_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0161_raw.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0385_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0385_map.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0385_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0385_raw.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0470_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0470_map.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0470_raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0470_raw.jpg -------------------------------------------------------------------------------- /demo/out_of_focus0689.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/demo/out_of_focus0689.jpg -------------------------------------------------------------------------------- /fastiqa/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | logging.basicConfig() 3 | -------------------------------------------------------------------------------- /fastiqa/_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class EDMLoss(nn.Module): 5 | def __init__(self): 6 | super(EDMLoss, self).__init__() 7 | 8 | def forward(self, p_target: torch.Tensor, p_estimate: torch.Tensor): 9 | assert p_target.shape == p_estimate.shape 10 | # cdf for values [1, 2, ..., 10] 11 | cdf_target = torch.cumsum(p_target, dim=-1) 12 | # cdf for values [1, 2, ..., 10] 13 | cdf_estimate = torch.cumsum(p_estimate, dim=-1) 14 | cdf_diff = cdf_estimate - cdf_target 15 | emd = torch.sqrt(torch.mean(torch.pow(torch.abs(cdf_diff), 2), -1)) 16 | return emd.mean() 17 | # samplewise_emd = torch.sqrt(torch.mean(torch.pow(torch.abs(cdf_diff), 2))) 18 | # return samplewise_emd.mean() 19 | 20 | # np.abs(np.cumsum(y_true, axis=-1) - np.cumsum(y_pred, axis=-1)) 21 | # np.sqrt(np.mean(np.square( ) ) ) 22 | -------------------------------------------------------------------------------- /fastiqa/all.py: -------------------------------------------------------------------------------- 1 | from .iqa import * 2 | from .vqa import * 3 | -------------------------------------------------------------------------------- /fastiqa/basics.py: -------------------------------------------------------------------------------- 1 | from fastai.vision.all import * 2 | import json # save settings 3 | import logging 4 | from .loss import * 5 | 6 | # class IqaData(): 7 | # db = None 8 | # 9 | # def __init__(self, db, filter=None, **kwargs): 10 | # self.label = db() if isinstance(db, type) else db 11 | # self.name = f'{self.db.name}_{self.__class__.__name__}' 12 | # self.__dict__.update(kwargs) 13 | # 14 | # def __getattr__(self, k: str): 15 | # return getattr(self.db, k) 16 | 17 | 18 | class IqaModel(Module): 19 | __name__ = None 20 | n_out = 1 21 | 22 | def __init__(self, **kwargs): 23 | self.__dict__.update(kwargs) 24 | super().__init__() 25 | if self.__name__ is None: self.__name__ = self.__class__.__name__.split('__')[0] 26 | 27 | def bunch(self, dls): 28 | # assert not isinstance(dls, (tuple, list)), "do dls.bunch() first" 29 | # logging.info(f'bunching ... {self.__name__}@{dls.__name__}') 30 | # 31 | # if isinstance(dls.label_col, (list, tuple)): 32 | # if len(dls.label_col) != self.n_out: 33 | # dls.label_col = dls.label_col[:self.n_out] 34 | # print(f'Changed dls.label_col to ({dls.label_col}) to fit model.n_out ({dls.__name__})') 35 | 36 | return dls 37 | 38 | def serialize(obj): 39 | """JSON serializer for objects not serializable by default json code""" 40 | # if isinstance(obj, (datetime, date)): 41 | # return obj.isoformat() 42 | # https://stackoverflow.com/questions/19628421/how-to-check-if-str-is-implemented-by-an-object 43 | if type(obj).__str__ is not object.__str__: # pathlib 44 | return str(obj) 45 | if obj.__class__.__name__ == 'function': 46 | return obj.__name__ 47 | if hasattr(obj, 'to_json'): 48 | return obj.to_json() 49 | d = {k:(obj.__dict__[k]) for k in obj.__dict__ if not k.startswith('_')} 50 | d['__class__.__name__'] = obj.__class__.__name__ 51 | return d 52 | # return {k:(obj.k) for k in dir(obj) if not k.startswith('_')} 53 | # raise TypeError ("Type %s not serializable" % type(obj)) 54 | 55 | def to_json(self, file=None): 56 | if file is None: 57 | return json.dumps(self, default=serialize) 58 | with open(file, 'w', encoding='utf-8') as f: 59 | json.dump(self, f, default=serialize, ensure_ascii=False, indent=4) 60 | 61 | # https://stackoverflow.com/questions/279561/what-is-the-python-equivalent-of-static-variables-inside-a-function 62 | def static_vars(**kwargs): 63 | def decorate(func): 64 | for k in kwargs: 65 | setattr(func, k, kwargs[k]) 66 | return func 67 | return decorate 68 | -------------------------------------------------------------------------------- /fastiqa/bunches/__init__.py: -------------------------------------------------------------------------------- 1 | from ..bunch import IqaDataBunch 2 | from fastai.vision.all import * 3 | -------------------------------------------------------------------------------- /fastiqa/bunches/feat2mos.py: -------------------------------------------------------------------------------- 1 | """ 2 | load features for image/video quality assessment databases 3 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 4 | %load_ext autoreload 5 | %autoreload 2 6 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 7 | # Export configurations 8 | from fastiqa.bunches.feat2mos import * 9 | feats = Feature2dBlock('paq2piq2x2', clip_num=8), Feature3dBlock() 10 | dls = Feat2MOS.from_json('json/LIVE_VQC.json', bs=16, feats = feats) 11 | dls.to_json() 12 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 13 | from fastiqa.bunches.feat2mos import * 14 | dls = Feat2MOS.from_json('json/KoNViD.json', bs=3) 15 | dls.show_batch() 16 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 17 | from fastiqa.bunches.feat2mos import * 18 | feats = FeatureBlock('effnet3'), 19 | dls = Feat2MOS.from_json('json/LIVE_FB_IQA.json', bs=3, feats=feats) 20 | dls.show_batch() 21 | # %% 22 | dls.vars 23 | dls.c 24 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 25 | %load_ext autoreload 26 | %autoreload 2 27 | from fastiqa.bunches.feat2mos import * 28 | dls = Feat2MOS.from_json('json/LIVE_FB_VQA_1k.json', bs=3) 29 | dls.show_batch() 30 | #%% 31 | # TODO: feature visualization? 32 | 33 | # 34 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 35 | """ 36 | 37 | from . import * 38 | from ._feat import * 39 | 40 | class Feat2MOS(IqaDataBunch): 41 | bs = 8 42 | feat_folder = 'features' 43 | feats = [] # (None,) # Feature2dBlock('paq2piq2x2'), Feature3dBlock('r3d18-2x2') 44 | suff = None 45 | # @property 46 | # def roi_index(self): 47 | # return self.feats[0].roi_index 48 | # 49 | # @roi_index.setter 50 | # def roi_index(self, index): 51 | # for feat in self.feats: 52 | # feat.roi_index = val 53 | # #assert len(self.label_col) == len(index) 54 | # if len(self.label_col) != len(index): 55 | # self._label_col = self.label_col[index] 56 | # # self._roi_col = np.array(self.roi_col)[[0,2], :].flatten().tolist() 57 | # # roi_col not used 58 | 59 | def get_block(self): 60 | return DataBlock( 61 | blocks = self.feats + (RegressionBlock, ), 62 | getters = [ColReader(self.fn_col, pref=self.path/self.feat_folder/f.name) for f in self.feats] + [ColReader(self.label_col)], 63 | splitter = self.get_splitter(), 64 | n_inp=len(self.feats), 65 | ) 66 | 67 | def get_df(self): 68 | df = super().get_df() 69 | # remove file extension if exists 70 | if self.suff is '': 71 | df[self.fn_col] = df[self.fn_col].str.rsplit('.',1).str[0] 72 | return df 73 | 74 | @property 75 | def vars(self): 76 | b = self._data.one_batch() 77 | return sum([b[x].shape[-1] for x in range(len(self.feats))]) 78 | 79 | 80 | def check_file_exists(self): 81 | df = self.get_df() 82 | files = [] 83 | dir = self.path/self.feat_folder 84 | for f in df[self.fn_col]: 85 | #if not (path/'jpg'/f).exists(): # file all exists 86 | for feat in self.feats: 87 | name = feat.name 88 | if not (dir/f'{name}/{f}.npy').exists(): 89 | files.append(dir/f'{name}'/f) 90 | #print(path/f'features/{feature}'/f) 91 | return files 92 | 93 | def show_batch(self, *args, **kwargs): 94 | b = self._data.one_batch() 95 | print(f'batch size = {self.bs}') 96 | for x in b[:-1]: 97 | print(list(x.shape)) 98 | print('--->', list(b[-1].shape)) 99 | 100 | def bunch(self, x, **kwargs): 101 | # format as the same database class using the new configurations (json) 102 | a = super().bunch(x, **kwargs) 103 | a.feats = self.feats 104 | return a 105 | 106 | 107 | 108 | 109 | # 110 | -------------------------------------------------------------------------------- /fastiqa/bunches/iqa/__init__.py: -------------------------------------------------------------------------------- 1 | from fastai.vision.all import * 2 | 3 | from ...bunch import * 4 | -------------------------------------------------------------------------------- /fastiqa/bunches/iqa/im2mos.py: -------------------------------------------------------------------------------- 1 | __all__ = ['Im2MOS'] 2 | 3 | """ 4 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 5 | from fastiqa.bunches.iqa.im2mos import * 6 | dls = Im2MOS.from_json('json/CLIVE.json', bs=3) 7 | # dls = Im2MOS.from_json('json/LIVE_FB_IQA.json', bs=3) 8 | dls.show_batch() 9 | dls.path 10 | 11 | # %% 12 | from fastiqa.bunches.iqa.im2mos import * 13 | dls = Im2MOS() << 'json/CLIVE.json' 14 | dls.show_batch() 15 | # %% 16 | dls.path 17 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 18 | """ 19 | 20 | from . import * 21 | 22 | """ 23 | dls = CLIVE(bs=2).get_data() 24 | dls.show_batch() 25 | """ 26 | 27 | # https://docs.fast.ai/vision.data#ImageDataLoaders.from_df 28 | class Im2MOS(IqaDataBunch): 29 | def get_block(self): 30 | getters = [ 31 | ColReader(self.fn_col, pref=self.path/self.folder, suff=self.fn_suffix), 32 | ColReader(self.label_col), 33 | ] 34 | return DataBlock(blocks=(ImageBlock, RegressionBlock), 35 | getters=getters, 36 | item_tfms = self.item_tfms, 37 | splitter = self.get_splitter()) 38 | -------------------------------------------------------------------------------- /fastiqa/bunches/iqa/im_bbox2mos.py: -------------------------------------------------------------------------------- 1 | # instead inputing 4 numbers, provide bounding box 2 | 3 | getters = [lambda o: path/'train'/o, lambda o: img2bbox[o][0], lambda o: img2bbox[o][1]] 4 | 5 | 6 | __all__ = ['ImBBox2MOS'] 7 | 8 | """ 9 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 10 | from fastiqa.bunches.iqa.im_bbox2mos import * 11 | from fastai.vision.all import * 12 | #dls = ImBBox2MOS.from_json('json/LIVE_FB_IQA.json', bs=3) 13 | #dls 14 | 15 | item_tfms = [Resize(128, method='pad'),] 16 | # batch_tfms = [Rotate(), Flip(), Dihedral(), Normalize.from_stats(*imagenet_stats)] 17 | dls = ImBBox2MOS(item_tfms=item_tfms, bs=3) << 'json/LIVE_FB_IQA.json' 18 | dls.show_batch() 19 | 20 | # %% 21 | from fastiqa.bunches.iqa.im_bbox2mos import * 22 | from fastai.vision.all import * 23 | # before padding coordinates 24 | # CropPad(800, pad_mode=PadMode.Zeros) 25 | # item_tfms = [CropPad(800, pad_mode=PadMode.Zeros),] 26 | # some images are broken 27 | item_tfms = [Resize(128, method='pad'),] 28 | # batch_tfms = [Rotate(), Flip(), Dihedral(), Normalize.from_stats(*imagenet_stats)] 29 | dls = ImBBox2MOS(item_tfms=item_tfms, bs=3) << 'json/AVA.json' 30 | dls.show_batch() 31 | 32 | # TypeError: clip_remove_empty() takes 2 positional arguments but 3 were given 33 | 34 | # %% 35 | 36 | from fastiqa.bunches.iqa.im_bbox2mos import * 37 | from fastai.vision.all import * 38 | from fastiqa.iqa import * 39 | 40 | item_tfms = [Resize(256, method='pad', pad_mode=PadMode.Zeros), RandomCrop(224),] 41 | dls = ImBBox2MOS(item_tfms=item_tfms, bs=3) << TestSet(AVA) 42 | dls.show_batch() 43 | # %% 44 | 45 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 46 | """ 47 | 48 | from . import * 49 | 50 | def my_bb_pad(samples, pad_idx=0): 51 | "Function that collect `samples` of labelled bboxes and adds padding with `pad_idx`." 52 | # don't remove empty 53 | # samples = [(s[0], *clip_remove_empty(*s[1:])) for s in samples] 54 | # samples = [(s[0], *clip_remove_empty(s[1], s[2]), s[2:]) for s in samples] 55 | max_len = max([len(s[2]) for s in samples]) 56 | def _f(img,bbox,lbl,scores): 57 | bbox = torch.cat([bbox,bbox.new_zeros(max_len-bbox.shape[0], 4)]) 58 | lbl = torch.cat([lbl, lbl .new_zeros(max_len-lbl .shape[0])+pad_idx]) 59 | return img,bbox,lbl,scores 60 | return [_f(*s) for s in samples] 61 | 62 | MyBBoxBlock = TransformBlock(type_tfms=TensorBBox.create, item_tfms=PointScaler, dls_kwargs = {'before_batch': my_bb_pad}) 63 | 64 | # reshape to four coordinates 65 | class BBoxColReader(ColReader): 66 | # _do_one is dealing with one cell located at (r,c) 67 | # return [super()._do_one(r, c)] 68 | def __call__(self, o, **kwargs): 69 | b = super().__call__(o, **kwargs) 70 | # a batch of coordinates, 71 | res = [b[i:i+4] for i in range(0, len(b), 4)] 72 | # print(res) 73 | return res 74 | 75 | class ImBBox2MOS(IqaDataBunch): 76 | roi_col = ["left", "top", "right", "bottom"] 77 | # wont save 78 | # pad the images 79 | def get_df(self): 80 | # prefix = ['top', 'left', 'bottom', 'right', 'height', 'width'] 81 | df = super().get_df() 82 | # wont use the padded coordinates 83 | if 'height' in df.columns and ('width' in df.columns): 84 | print('add bbox coordinates (no padding applied)') 85 | df['top'] = 0 86 | df['left'] = 0 87 | df['bottom'] = df['height'] 88 | df['right'] = df['width'] 89 | return df 90 | 91 | def get_block(self): 92 | if len(self.roi_col) == 4*4: 93 | bbox_lbl = ['image', 'patch1', 'patch2', 'patch3'] 94 | elif len(self.roi_col) == 4: 95 | bbox_lbl = ['image'] 96 | else: 97 | raise NotImplementedError 98 | print() 99 | getters = [ 100 | ColReader(self.fn_col, pref=self.path/self.folder, suff=self.fn_suffix), 101 | BBoxColReader(self.roi_col), 102 | lambda o: bbox_lbl, # only one label 103 | ColReader(self.label_col), 104 | ] 105 | return DataBlock(blocks=(ImageBlock, MyBBoxBlock, BBoxLblBlock, RegressionBlock), 106 | item_tfms = self.item_tfms, 107 | getters=getters, n_inp=3, splitter = self.get_splitter()) 108 | -------------------------------------------------------------------------------- /fastiqa/bunches/iqa/im_feat2mos.py: -------------------------------------------------------------------------------- 1 | """will be removed soon. use feat2mos.py 2 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | from fastiqa.bunches.iqa.im_feat2mos import * 4 | dls = ImFeat2MOS.from_json('json/LIVE_FB_IQA.json', bs=3) 5 | dls.show_batch() 6 | 7 | 8 | FeatureBlock 9 | # 10 | # try the freeze version (but bn not freezed. we could save the finetuned bn version ) 11 | 12 | Step1: convert one npy file to seperate files 13 | 14 | unpack_results 15 | 16 | 17 | # %% 18 | 19 | # %% 20 | files[0] 21 | # when finetuning the PaQ2PiQ branch, we made a new sota model 22 | # patch quality is not important, better patch quality map should not be the direction, focus on global quality 23 | 24 | # %% 25 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 26 | """ 27 | 28 | from . import * 29 | 30 | class NpyReader(): 31 | """ load features from a numpy file""" 32 | def __init__(self, file, roi_index=None): 33 | if not Path(file).exists(): 34 | raise FileNotFoundError(file) 35 | with open(file, 'rb') as f: 36 | features = np.load(f) 37 | 38 | if roi_index is not None: 39 | if features.ndim == 2: 40 | features = features.reshape([4, -1, features.shape[-1]]) 41 | features = features[roi_index, :] 42 | else: 43 | features = features[:, roi_index, :] 44 | self._data = features 45 | 46 | def __call__(self, row): 47 | print(row.index) 48 | return L(self._data[row.index, :]) 49 | 50 | 51 | def TensorBlock(): 52 | return TransformBlock(type_tfms=torch.tensor) 53 | 54 | # after loading, no index available, must use a seperate feature file for each image.... 55 | # unless we have a name_index mapping (search vs load and combine) 56 | # seperate feature file make more sense 57 | 58 | class ImFeat2MOS(IqaDataBunch): 59 | feat_folder = 'features' 60 | feats = 'effnet3', 61 | roi_index = None 62 | 63 | def get_block(self): 64 | return DataBlock( 65 | blocks = [TensorBlock for _ in self.feats] + [RegressionBlock], 66 | getters = [NpyReader(self.path/self.feat_folder/f'{feat}.npy', self.roi_index) for feat in self.feats] + [ColReader(self.label_col)], 67 | splitter = self.get_splitter(), 68 | n_inp=len(self.feats), 69 | ) 70 | 71 | @property 72 | def vars(self): 73 | b = self._data.one_batch() 74 | return sum([b[x].shape[-1] for x in range(len(b)-1)]) 75 | 76 | def check_file_exists(self): 77 | df = self.get_df() 78 | files = [] 79 | dir = self.path/self.feat_folder 80 | for f in df[self.fn_col]: 81 | #if not (path/'jpg'/f).exists(): # file all exists 82 | for feat in self.feats: 83 | name = feat.name 84 | if not (dir/f'{name}/{f}.npy').exists(): 85 | files.append(dir/f'{name}'/f) 86 | #print(path/f'features/{feature}'/f) 87 | return files 88 | 89 | def show_batch(self, *args, **kwargs): 90 | b = self._data.one_batch() 91 | print(f'batch size = {self.bs}') 92 | for x in b[:-1]: 93 | print(list(x.shape)) 94 | print('--->', list(b[-1].shape)) 95 | 96 | def bunch(self, x, **kwargs): 97 | # format as the same database class using the new configurations (json) 98 | a = super().bunch(x, **kwargs) 99 | a.feats = self.feats 100 | return a 101 | -------------------------------------------------------------------------------- /fastiqa/bunches/iqa/im_roi2mos.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ImRoI2MOS'] 2 | 3 | """ 4 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 5 | from fastiqa.bunches.iqa.im_roi2mos import * 6 | dls = ImRoI2MOS.from_json('json/LIVE_FB_IQA.json', bs=3) 7 | dls.show_batch() 8 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 9 | """ 10 | 11 | from . import * 12 | 13 | def show_roi_batch(dls, b=None, max_n=4, figsize=(15, 5), **kwargs): 14 | # rb --> rois in a batch 15 | max_n = min(max_n, dls.bs) 16 | if b is None: b = dls.one_batch() 17 | xb, rb, yb = b 18 | fig, axes = plt.subplots(ncols=max_n, nrows=1, figsize=figsize, **kwargs) # , figsize=(12,6), dpi=120 19 | for i in range(max_n): 20 | xs, rs, ys = xb[i], rb[i], yb[i] 21 | # axes[i].imshow(x.permute(1,2,0).cpu().numpy()) 22 | # axes[i].set_title(f'{ys.item():.02f}') 23 | rois = rs.cpu().numpy().reshape(-1,4).tolist() 24 | labels = ['image', 'p1', 'p2', 'p3'] 25 | tbbox = LabeledBBox(TensorBBox(rois), labels) 26 | timg = TensorImage(xs.cpu())*255 27 | tpil = PILImage.create(timg) 28 | ctx = tpil.show(ax=axes[i]) 29 | tbbox.show(ctx=ctx) 30 | axes[i].set_title(','.join("%.2f" % x for x in ys.tolist()) if ys.dim() > 0 else "%.2f" % ys) 31 | axes[i].axis('off') 32 | 33 | class ImRoI2MOS(IqaDataBunch): 34 | roi_col = ["left_image", "top_image", "right_image", "bottom_image"] 35 | pad = None 36 | # if loss_func is None: 37 | # loss_func = getattr(dls.train_ds, 'loss_func', None) 38 | 39 | # pad the images 40 | def get_df(self): 41 | # prefix = ['top', 'left', 'bottom', 'right', 'height', 'width'] 42 | df = super().get_df() 43 | """prepare roi label""" 44 | if any([x not in df.columns for x in self.roi_col]): 45 | if 'height' not in df.columns: 46 | df['height'] = self.height 47 | if 'width' not in df.columns: 48 | df['width'] = self.width 49 | 50 | print('add coordinate information to csv file') 51 | # unpack position 52 | # for p in ['p1', 'p2', 'p3']: 53 | # df[p] = df[p].apply(literal_eval) 54 | # columns = [x+'_'+p for x in ['left', 'right', 'top', 'bottom', 'start', 'end']] 55 | # df[columns] = pd.DataFrame(df[p].to_list(), columns=columns) 56 | df['top_image'] = 0 57 | df['left_image'] = 0 58 | df['bottom_image'] = df['height'] 59 | df['right_image'] = df['width'] 60 | df['height_image'] = df['height'] 61 | df['width_image'] = df['width'] 62 | 63 | # shift if needed 64 | if self.pad: 65 | top_shift = (self.pad - df['height']) // 2 66 | left_shift = (self.pad - df['width']) // 2 67 | 68 | for s in self.label_suffixes: # by deafult, "" 69 | df['top' + s] += top_shift 70 | df['left' + s] += left_shift 71 | df['bottom' + s] = df['top' + s] + df['height' + s] 72 | df['right' + s] = df['left' + s] + df['width' + s] 73 | 74 | # df.to_csv(self.path/self.csv_labels, index=False) 75 | return df 76 | 77 | def get_block(self): 78 | getters = [ 79 | ColReader(self.fn_col, pref=self.path/self.folder, suff=self.fn_suffix), 80 | ColReader(self.roi_col), 81 | ColReader(self.label_col), 82 | ] 83 | return DataBlock(blocks=(ImageBlock, RegressionBlock, RegressionBlock), 84 | item_tfms = self.item_tfms, 85 | getters=getters, n_inp=2, splitter = self.get_splitter()) 86 | 87 | ImRoI2MOS.show_batch = show_roi_batch 88 | -------------------------------------------------------------------------------- /fastiqa/bunches/iqa/test_images.py: -------------------------------------------------------------------------------- 1 | """fastai1 code, not yet converted""" 2 | 3 | from ..label import * 4 | from tqdm import tqdm 5 | from fastai.vision import open_image 6 | import os 7 | from PIL import Image as PIL_Image 8 | 9 | """ 10 | # %% 11 | from fastiqa.all import * 12 | learn = RoIPoolLearner.from_cls(FLIVE, RoIPoolModel) 13 | learn.path = Path('.') 14 | learn.load('RoIPoolModel-fit(10,bs=120)') 15 | learn.export('trained_model.pkl') 16 | 17 | from fastiqa.all import *; # Im2MOS(TestImages(path='/var/www/yourapplication')) 18 | data.df 19 | # %% 20 | """ 21 | 22 | class TestImages(IqaLabel): # Rois0123Label 23 | path = '.' 24 | img_tfm_size = None 25 | valid_pct = 1 26 | batch_size = 1 27 | csv_labels = 'scores.csv' 28 | 29 | @classmethod 30 | def from_learner(cls, learn, path=None, dir_qmap=None, sz=None, **kwargs): 31 | def proc_file(f): 32 | im = open_image(os.path.join(path, f)) 33 | if dir_qmap is not None: 34 | qmap = learn.predict_quality_map(im, [32, 32]) 35 | 36 | name = os.path.basename(f).split('.')[0] 37 | 38 | qmap.plot() 39 | qmap.savefig(os.path.join(dir_qmap, name + '.jpg')) 40 | score = qmap.global_score 41 | 42 | if sz is not None: 43 | height, width = qmap.img.size 44 | new_width = sz # 500 45 | new_height = new_width * height // width 46 | qmap.pil_image.resize((new_width, new_height), PIL_Image.ANTIALIAS).save(os.path.join(dir_qmap, name + '_raw.jpg')) 47 | qmap.blend(mos_range=(None, None)).resize((new_width, new_height), PIL_Image.ANTIALIAS).save(os.path.join(dir_qmap, name + '_map.jpg')) 48 | else: 49 | score = learn.predict(im)[0].obj[0] 50 | del im 51 | del qmap 52 | return score 53 | 54 | 55 | if dir_qmap is not None: 56 | os.makedirs(dir_qmap, exist_ok=True) 57 | 58 | valid_images = (".jpg",".jpeg",".png",".bmp",".tif") 59 | files = os.listdir(path if path is not None else cls.path) 60 | files = [f for f in files if f.lower().endswith(valid_images)] 61 | scores = [proc_file(f) for f in tqdm(files)] 62 | df = pd.DataFrame({'mos': scores, 'name': files}) 63 | df.to_csv('scores.csv', index=False) 64 | 65 | return cls(path=path) 66 | pass 67 | -------------------------------------------------------------------------------- /fastiqa/bunches/v1/im2mos.py: -------------------------------------------------------------------------------- 1 | from . import IqaDataBunch 2 | from fastai.vision import MSELossFlat 3 | 4 | """ 5 | IM2MOS 6 | Input: data_cls = ImageList 7 | Output: label_cls = FloatList 8 | 9 | # %% 10 | %matplotlib inline 11 | from fastiqa.bunches import * 12 | # %% 13 | Im2MOS(CLIVE) 14 | Im2MOS(FLIVE640) 15 | Patch1toMOS(FLIVE_2k, batch_size=1) 16 | 17 | # %% for debugging 18 | from fastiqa.bunches import *; Im2MOS(CLIVE).get_data() 19 | 20 | 21 | # 22 | # data = IM 23 | """ 24 | 25 | # dls # dataloaders 26 | class IqaDataBunch(IqaData): 27 | batch_size = 64 # 16 28 | # num_workers = 16 29 | label_cls = FloatList 30 | data_cls = ImageList 31 | device = torch.device('cuda') 32 | 33 | name = None 34 | 35 | def __getattr__(self, k: str): 36 | # print(f'__getattr__: {k}') 37 | try: 38 | return getattr(self.label, k) 39 | except AttributeError: 40 | return getattr(self.data, k) 41 | 42 | @cached_property 43 | def data(self): 44 | print(f'loading data...{self.name}') 45 | data = self.get_data() 46 | print(f'DONE loading data {self.name}') 47 | return data 48 | 49 | def get_data(self): 50 | return NotImplementedError 51 | 52 | def reset(self, **kwargs): 53 | self.label._df = None 54 | self._data = None 55 | self.__dict__.update(kwargs) 56 | return self 57 | 58 | def _repr_html_(self): 59 | self.data.show_batch(rows=2, ds_type=DatasetType.Valid) 60 | print(self.data.__str__()) 61 | return 62 | 63 | def show(self, fn): 64 | # TODO not found? 65 | # locate the image 66 | ds = self.df[self.df[self.fn_col] == fn] 67 | try: idx = int(ds.index[0]) 68 | except IndexError: print('Not found!'); return 69 | 70 | # put selected sample on the top of the dataframe 71 | self.df.loc[0, :] = self.df.iloc[idx] 72 | self.df.loc[0, 'is_valid'] = True 73 | 74 | # avoid loading the whole database 75 | self.df = self.df.iloc[:self.batch_size] 76 | 77 | # backup the data 78 | data = self._data 79 | 80 | # reload the database 81 | self._data = None 82 | self.show_batch(ds_type=DatasetType.Single) 83 | self.reset(_data = data) 84 | return ds.T # otherwise won't show all columns 85 | 86 | 87 | class Im2MOS(IqaDataBunch): 88 | _data = None 89 | label_col_idx = 0 # only one output 90 | loss_func = MSELossFlat() 91 | 92 | def get_data(self): 93 | data = self.get_list() 94 | data = self.get_split(data) 95 | data = self.get_label(data) 96 | data = self.get_augment(data) 97 | data = self.get_bunch(data) 98 | return data 99 | 100 | def get_list(self): 101 | return self.data_cls.from_df(self.df, self.path, suffix=self.fn_suffix, cols=self.fn_col, 102 | folder=self.folder) 103 | 104 | def get_split(self, data): 105 | if self.valid_pct is None: 106 | data = data.split_from_df(col='is_valid') 107 | else: 108 | if self.valid_pct == 1: # TODO ignore_empty=True 109 | print('all samples are in the validation set!') 110 | data = data.split_by_valid_func(lambda x: True) # All valid 111 | data.train = data.valid # train set cannot be empty 112 | elif self.valid_pct == 0: 113 | print('all samples are in the training set!') 114 | data = data.split_none() 115 | else: # 0 < self.valid_pct < 1 116 | print('We suggest using a fixed split with a fixed random seed') 117 | data = data.split_by_rand_pct(valid_pct=self.valid_pct, seed=2019) 118 | return data 119 | 120 | def get_label(self, data): 121 | return data.label_from_df(cols=self.label_cols[self.label_col_idx], label_cls=self.label_cls) 122 | 123 | def get_augment(self, data): 124 | return data if self.img_tfm_size is None else data.transform(size=self.img_tfm_size) 125 | 126 | def get_bunch(self, data, **kwargs): 127 | return data.databunch(bs=self.batch_size, val_bs=self.val_bs, **kwargs) 128 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from fastai.vision.all import * 3 | from ast import literal_eval # unpack postion data 4 | from functools import partial 5 | 6 | from ...bunch import * 7 | from ._clip import * 8 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/_clip.py: -------------------------------------------------------------------------------- 1 | __all__ = ['Clip', 'ClipBlock', 'MultiClip', 'MultiClipBlock', 'show_sequence_batch', 'takespread', 'create_sequence_batch'] 2 | 3 | from fastai.vision.all import * 4 | from math import ceil 5 | 6 | # https://stackoverflow.com/questions/9873626/choose-m-evenly-spaced-elements-from-a-sequence-of-length-n 7 | # [x for x in takespread([3,4,5], 5, 1)] # [[3], [4], [4], [5], [5]] 8 | # list(x for x in takespread(list(range(192)),3,2)) 9 | def takespread(sequence, num, clip_size=1): 10 | sequence = list(sequence) 11 | length = float(len(sequence)-clip_size+1) 12 | for i in range(num): 13 | start = int(ceil(i * (length-1) / num)) 14 | yield sequence[start:start+clip_size] 15 | 16 | # show_sequence_batch, create_batch 17 | 18 | """fastuple will put the same kind of items together, should use something else 19 | otherwise frame1 all together, then followed by frame2 in a batch 20 | """ 21 | 22 | # https://note.nkmk.me/en/python-pillow-concat-images/ 23 | def get_concat_h(frames): 24 | im = frames[0] 25 | column = len(frames) 26 | dst = Image.new('RGB', (im.width * column, im.height)) 27 | for x in range(1, column): 28 | dst.paste(frames[x], (x * im.width, 0)) 29 | return dst 30 | 31 | def get_concat_v(frames): 32 | im = frames[0] 33 | row = len(frames) 34 | dst = Image.new('RGB', (im.width, im.height * row)) 35 | for y in range(1, row): 36 | dst.paste(frames[y], (0, y * im.width)) 37 | return dst 38 | 39 | 40 | def int2float(o:TensorImage): 41 | return o.float().div_(255.) 42 | 43 | 44 | class Clip(fastuple): 45 | @classmethod 46 | # first_frame 47 | def create(cls, fn_last_frame, clip_size=None): # one file name: id 48 | a = fn_last_frame.rsplit('_', 1) 49 | base = a[0] 50 | end_frame = int(a[1].rsplit('.', 1)[0]) # .jpg 51 | 52 | fns = [f'{base}_{n+1:05d}'+'.jpg' for n in range(end_frame-clip_size, end_frame)] 53 | return cls(tuple(PILImage.create(f) for f in fns)) 54 | 55 | def show(self, **kwargs): 56 | return self[0].show(**kwargs) # show first frame 57 | 58 | 59 | @property 60 | def size(self): 61 | return self[0].size 62 | 63 | # ClipBlock renamed to MultiClipBlock 64 | def ClipBlock(clip_size=8): 65 | f = partial(Clip.create, clip_size=clip_size) 66 | return TransformBlock(type_tfms=f, batch_tfms=int2float) 67 | 68 | ################################################################################ 69 | 70 | # MultiClipFromLastFrame 71 | class MultiClip(fastuple): 72 | @classmethod 73 | def create(cls, fn_last_frame, clip_num=None, clip_size=None): # one file name: id 74 | a = fn_last_frame.rsplit('_', 1) 75 | base = a[0] 76 | n_frames = int(a[1]) 77 | 78 | if clip_size is not None and clip_num is None: # get all clips without interval or overlap 79 | # if last clip has less than clip_size frames? 80 | # drop last clip 81 | # sample per second? how to reflect frame rate? --- low frame rate 82 | raise NotImplementedError 83 | fns = [f'{base}_{n+1:05d}'+'.jpg' for n in range(n_frames)] 84 | else: 85 | fns = [] 86 | for idx in takespread(range(n_frames), clip_num, clip_size): 87 | fns += [f'{base}_{n+1:05d}'+'.jpg' for n in idx] 88 | return cls(tuple(PILImage.create(f) for f in fns)) 89 | 90 | @property 91 | def size(self): 92 | return self[0].size 93 | 94 | # ClipBlock renamed to MultiClipBlock 95 | def MultiClipBlock(clip_num=8, clip_size=8): 96 | f = partial(MultiClip.create, clip_num=clip_num, clip_size=clip_size) 97 | return TransformBlock(type_tfms=f, batch_tfms=int2float) 98 | 99 | def create_sequence_batch(data): 100 | xs, ys = [], [] 101 | for d in data: 102 | xs.append(d[0]) 103 | ys.append(d[1]) 104 | 105 | try: 106 | xs_ = torch.cat([TensorImage(torch.cat([im[None] for im in x], dim=0))[None] for x in xs], dim=0) 107 | ys_ = torch.cat([y[None] for y in ys], dim=0) 108 | except: 109 | print('Error@create_sequence_batch') 110 | for idx, x in enumerate(xs): 111 | print(idx, x.size()) # torch.Size([3, 500, 500]) 112 | xs_ = torch.cat([TensorImage(torch.cat([im[None] for im in x], dim=0))[None] for x in xs], dim=0) 113 | print('xs_ is ok') 114 | print(xs_) 115 | raise 116 | 117 | return TensorImage(xs_), TensorCategory(ys_) 118 | 119 | 120 | def show_sequence_batch(dls, max_n=4): 121 | xb, yb = dls.one_batch() 122 | max_n = min(dls.bs, max_n) 123 | ncols = len(xb[0]) 124 | fig, axes = plt.subplots(ncols=ncols, nrows=max_n, figsize=(12,6), dpi=120) 125 | for i in range(max_n): 126 | xs, ys = xb[i], yb[i] 127 | for j, x in enumerate(xs): 128 | axes[i,j].imshow(x.permute(1,2,0).cpu().numpy()) 129 | axes[i,j].set_title(f'{ys.item():.02f}') 130 | axes[i,j].axis('off') 131 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/single_vid2mos.py: -------------------------------------------------------------------------------- 1 | __all__ = ['SingleVideo2MOS'] 2 | 3 | """ --> MOS 4 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 5 | %load_ext autoreload 6 | %autoreload 2 7 | # %% 8 | from fastiqa.bunches.vqa.single_vid2mos import * 9 | dls = SingleVideo2MOS.from_json('json/LIVE_FB_VQA.json', bs=3, clip_num=3, clip_size=2) 10 | dls.set_vid_id('ia-batch8/North_Reading_School_Committee_Part_1_-_April_23_2012') 11 | dls.show_batch() 12 | dls.fn_col 13 | # %% 14 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 15 | from fastiqa.bunches.vqa.single_vid2mos import * 16 | dls = SingleVideo2MOS.from_json('json/KoNViD.json', bs=3, clip_num=3, clip_size=2) 17 | # dls.path 18 | dls.set_vid_id('3240926995') # singing woman # 2999049224 19 | dls.show_batch() 20 | # %% 21 | from fastiqa.bunches.vqa.single_vid2mos import * 22 | file = 'json/LIVE_FB_VQA_30k.json' # LSVQ_Test 23 | bs = 2 24 | shuffle = False 25 | dls = SingleVideo2MOS.from_json( 26 | file, # LIVE_FB_VQA_1k 27 | use_nan_label=True, clip_num=None, clip_size=16, 28 | bs=bs, 29 | shuffle=shuffle) 30 | dls.set_vid_id('ia-batch8/North_Reading_School_Committee_Part_1_-_April_23_2012') 31 | dls.show_batch() 32 | 33 | # %% 34 | dls._path 35 | type(dls.path) 36 | dls.path 37 | # %% 38 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 39 | """ 40 | 41 | """view one video as a collection of frames/clips""" 42 | """ #73 single video as an image database """ 43 | 44 | """ 45 | # %% 46 | from fastiqa.all import * 47 | dls = SingleVideo2MOS.from_json('json/KoNViD.json', bs=20, valid_bs=23) 48 | dls.set_vid_id('2999049224') 49 | self= dls.get_block() 50 | self.summary(dls.df) 51 | # %% 52 | either use a single video database, or use learn.predict_from_video() 53 | form the x for the model... 54 | # %% 55 | """ 56 | 57 | from . import * 58 | from .vid2mos import * 59 | # the same corrdincate 60 | # append df? 61 | # build a df? no need 62 | # generate image file list from df 63 | # 64 | # 65 | # def _AllAsTrainTestSplitter(idx): 66 | # "Split `items` so that `val_idx` are in the validation set and the others in the training set" 67 | # def _inner(o): 68 | # return L(idx, use_list=True), L(idx, use_list=True) 69 | # return _inner 70 | 71 | class SingleVideo2MOS(Vid2MOS): 72 | """TODO: skip frame """ 73 | """may include roi 74 | clip_size is 1 for extracting 2d features 75 | clip_size > 1 for extracting 3d features 76 | 77 | """ 78 | _vid_prop = None 79 | _df = None # frames and labels 80 | _df_vid_list = None # konvid vid list 81 | bs = 64 82 | vid_id = None 83 | use_nan_label = False 84 | clip_size = 1 85 | clip_num = 16 # 16x16 = 256 # 16x8 = 128 # cover all video 86 | # clip_num == bs, then 1 batch finish all feature extraction 87 | item_tfms = None # make sure no preprocessing is applied 88 | roi_col = None 89 | 90 | # @property 91 | # def __name__(self): 92 | # return self.__class__.__name__ + '-' + self.vid_id 93 | 94 | def set_vid_id(self, x): 95 | # buggy: self.reset(vid_id=x, __name__=str(x), _df = None) 96 | self.vid_id = x 97 | self._df = None 98 | self.__name__ = str(x) 99 | return self 100 | 101 | # clip num is unset? -- just one clip 102 | def get_input_blocks(self): 103 | """input is one frame or one clip""" 104 | return ImageBlock if self.clip_size==1\ 105 | else partial(ClipBlock, clip_size=self.clip_size), 106 | #else partial(ClipBlock, clip_size=self.clip_size), 107 | 108 | def get_input_getters(self): 109 | # 'fn' 110 | # not self.fn_col, 111 | # Note, here we use a generated df for each video 112 | return ColReader('fn', pref=self.path/self.folder/f'{self.vid_id}/'), 113 | 114 | @property 115 | def video_list(self): 116 | if self._df_vid_list is None: 117 | self._df_vid_list = super().get_df().set_index(self.fn_col) 118 | return self._df_vid_list 119 | 120 | def get_df(self): 121 | if self._df is None: 122 | assert self.clip_size >= 1 123 | path = self.path # Path('/media/zq/DB/db/KoNViD/') 124 | assert self.vid_id is not None 125 | if pd.api.types.is_integer_dtype(self.video_list.index.dtype): 126 | self.vid_id = int(self.vid_id) 127 | row = self.video_list.loc[self.vid_id] # KoNViD int id 128 | self._vid_prop = row 129 | # logging.debug(f'video id: {self.vid_id} ') 130 | # logging.debug(f'row: {row} ') 131 | # logging.debug(f'num_frame: {row[self.frame_num_col]} ') 132 | num_frame = int(row[self.frame_num_col]) 133 | # print(type(row[self.frame_num_col])) 134 | # print((row[self.frame_num_col])) 300. 135 | 136 | if self.clip_num is None: 137 | # image case (self.clip_size is 1) : 138 | # files = [f'image_{x+1:05d}.jpg' for x in range(num_frame) ] 139 | # last_frame_index = (self.clip_size, row[self.frame_num_col], self.clip_num).astype(int).tolist() 140 | 141 | # for 2d case, we need .jpg 142 | # for 3d case, we don't need .jpg 143 | files = [f'image_{x+self.clip_size:05d}.jpg' for x in range(0, num_frame-self.clip_size+1, self.clip_size)] # drop last clip 144 | else: 145 | # fixed clip_num cases ( we used before) 146 | # index start with 1, not 0 147 | last_frame_index = np.linspace(self.clip_size, row[self.frame_num_col], self.clip_num).astype(int).tolist() 148 | files = [f'image_{x:05d}.jpg' for x in last_frame_index ] 149 | 150 | # use a different 3d backbone? smaller but faster? 151 | # change to use pytorch built in r3d 18 152 | self._df = pd.DataFrame(files, columns=['fn']) 153 | return self._df 154 | 155 | def get_block(self): 156 | self.get_df() # cache vid prop 157 | row = self._vid_prop 158 | InputBlocks = self.get_input_blocks() 159 | InputGetters = self.get_input_getters() 160 | 161 | if self.roi_col is not None: 162 | InputBlocks += RegressionBlock, 163 | InputGetters += lambda x: [row[x] for x in self.roi_col], 164 | 165 | # add roi information to this dataframe, the same roi for all models 166 | # do it at model side? not a good idea 167 | OutputBlocks = RegressionBlock, 168 | OutputGetters = lambda x: np.nan if self.use_nan_label else row[self.label_col].tolist() , 169 | 170 | num_frame = int(row[self.frame_num_col]) 171 | self.split_mode = 'all_as_train_test' 172 | return DataBlock( 173 | blocks = InputBlocks + OutputBlocks, 174 | getters = InputGetters + OutputGetters, 175 | n_inp = len(InputBlocks), 176 | splitter = self.get_splitter(), 177 | #_AllAsTrainTestSplitter(range(num_frame if self.clip_size==1 else self.clip_num)) #RandomSplitter(valid_pct=1), # all as valid 178 | ) 179 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/test_feats.py: -------------------------------------------------------------------------------- 1 | __all__ = ['TestFeatures'] 2 | 3 | """No need to use this, simply forward the features! 4 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 5 | from fastiqa.bunches.vqa.test_feats import * 6 | from fastiqa.bunches.vqa._feat import * 7 | dls = TestFeatures.from_json('json/TestVideos.json', bs=2, feats=Feature2dBlock('paq2piq2x2', clip_num=16)) 8 | b = dls.set_vid_id('VID_20201024_113442') 9 | b = dls.one_batch() 10 | b[0].shape 11 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 12 | """ 13 | 14 | from . import * 15 | from .test_videos import * 16 | from tqdm import tqdm 17 | 18 | class TestFeatures(TestVideos): 19 | def get_input_blocks(self): 20 | return self.feats 21 | 22 | def get_input_getters(self): 23 | return [ColReader(self.fn_col, pref=self.path/self.feat_folder/f.name) for f in self.feats] 24 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/test_videos.py: -------------------------------------------------------------------------------- 1 | from . import * 2 | from .single_vid2mos import * 3 | from tqdm import tqdm 4 | from .video_utils import video_process 5 | # from ....util_scripts.generate_video_jpgs import video_process 6 | 7 | class TestVideos(SingleVideo2MOS): 8 | 9 | def get_df(self): 10 | # always generate a new one? - no, unless it doesn't exist 11 | csv_file = self.path/self.csv_labels 12 | if not csv_file.exists(): 13 | print('generating jpg files:') 14 | files = list((self.path/'mp4').glob('*.mp4')) 15 | print(f'Found {len(files)} files.') 16 | files = [f for f in files if not (self.path/self.folder/(f.stem)).exists()] 17 | print(f'Generating {len(files)} files.') 18 | for f in tqdm(files): 19 | print(f'Extracting frames: {f}') 20 | video_process(video_file_path=f, 21 | dst_root_path=self.path/self.folder, 22 | ext='.mp4', fps=-1, size=-1) 23 | # print(f'Skip cuz exists: {jpg_folder}') 24 | 25 | print('prepare label csv files') 26 | df = pd.DataFrame({self.fn_col: [f.stem for f in files], self.label_col: -1}) # np.nan 27 | df.to_csv(csv_file, index=False) 28 | return super().get_df() 29 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/vid2mos.py: -------------------------------------------------------------------------------- 1 | __all__ = ['Vid2MOS', 'clip2image'] 2 | 3 | """ 4 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 5 | from fastiqa.bunches.vqa.vid2mos import * 6 | # dls = Vid2MOS.from_json('json/LIVE_VQC.json', bs=3, clip_num=3, clip_size=2) 7 | dls = Vid2MOS.from_json('json/KoNViD.json', bs=3, clip_num=3, clip_size=2) 8 | # dls = Vid2MOS.from_json('json/LIVE_FB_VQA_pad500.json', item_tfms=CropPad(500), bs=3, clip_num=3, clip_size=2) 9 | dls.show_batch() 10 | dls.bs 11 | '_data' in dls.__dict__ 12 | del dls.__dict__['_data'] 13 | dls.bs = 2 14 | dir(dls) 15 | dls._data.bs 16 | dls = dls.reset(bs=2) 17 | dls.show_batch() 18 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 19 | """ 20 | 21 | from . import * 22 | 23 | """ 24 | Note here, a video means a collection of jpg files 25 | """ 26 | 27 | def clip2image(t, vertical=False): 28 | # [n, 3, H, W] --> [3, n, H, W] --> [3, n*H, W] 29 | if vertical:# vertical concat 30 | return t.transpose(0, 1).reshape(3, -1, t.size(-1)) 31 | else: 32 | # [n, 3, H, W] --> [3, H, n, W] --> [3, H, n*W] 33 | return t.transpose(0, 1).transpose(1, 2).reshape(3, t.size(-2), -1) 34 | 35 | class Vid2MOS(IqaDataBunch): 36 | clip_size = 1 37 | clip_num = 8 38 | bs = 8 39 | fn_last_frame_col = "fn_last_frame" 40 | folder = "jpg" 41 | frame_num_col = 'frame_number' 42 | 43 | def get_df(self): 44 | df = super().get_df() 45 | # add_fn_last_frame col if not exists 46 | if self.frame_num_col not in df.columns: 47 | print('add frame_num_col') 48 | frame_numbers = [] 49 | for folder in df[self.fn_col].tolist(): 50 | n_max = 0 51 | # .split('.')[0] 52 | for file in (self.path/self.folder/folder).glob('*.jpg'): 53 | n = int(str(file)[:-4].split('_')[-1]) 54 | if n > n_max: 55 | n_max = n; 56 | frame_numbers.append(n_max) 57 | df[self.frame_num_col] = frame_numbers 58 | df.to_csv(self.path/self.csv_labels, index=False) 59 | if self.fn_last_frame_col not in df.columns: 60 | print('add fn_last_frame_col') 61 | df[self.fn_last_frame_col] = df[self.fn_col] + '/image_' + df[self.frame_num_col].astype(str).str.zfill(5) 62 | df.to_csv(self.path/self.csv_labels, index=False) 63 | return df 64 | 65 | def get_block(self): 66 | df = self.get_df() 67 | VideoBlock = partial(MultiClipBlock, clip_size=self.clip_size, clip_num=self.clip_num) 68 | return DataBlock( 69 | blocks = (VideoBlock, RegressionBlock), 70 | getters = [ 71 | ColReader(self.fn_last_frame_col, pref=self.path/(self.folder + '/') ), 72 | ColReader(self.label_col), # mos_vid 73 | ], 74 | item_tfms = self.item_tfms, 75 | splitter = self.get_splitter(), 76 | ) 77 | 78 | def show_batch(self, b=None, max_n=4, figsize=(30, 6), vertical=False, **kwargs): 79 | # rb --> rois in a batch 80 | dls = self 81 | max_n = min(max_n, dls.bs) 82 | xb, yb = dls.one_batch() if b is None else b 83 | if vertical: 84 | fig, axes = plt.subplots(ncols=max_n, nrows=1, figsize=figsize, **kwargs) # , figsize=(12,6), dpi=120 85 | else: 86 | fig, axes = plt.subplots(nrows=max_n, ncols=1, figsize=figsize, **kwargs) # , figsize=(12,6), dpi=120 87 | if max_n == 1: axes = [axes] # only one item 88 | for i in range(max_n): 89 | xs, ys = xb[i], yb[i] 90 | # axes[i].imshow(x.permute(1,2,0).cpu().numpy()) 91 | # axes[i].set_title(f'{ys.item():.02f}') 92 | timg = TensorImage(clip2image(xs).cpu())*255 93 | tpil = PILImage.create(timg) 94 | ctx = tpil.show(ax=axes[i]) 95 | 96 | # ys 0 dim tensor 97 | lbl = str(ys.tolist()) # "%.2f" % ys if ys.dim() == 0 else 98 | axes[i].set_title(lbl) 99 | axes[i].axis('off') 100 | 101 | def create_batch(self, data): 102 | # cannot call self in this function 103 | return create_sequence_batch(data) 104 | # if self.clip_size > 1 else None 105 | 106 | # Vid2MOS.show_batch = show_sequence_batch # old display method 107 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/vid_sp2mos.py: -------------------------------------------------------------------------------- 1 | __all__ = ['VidSP2MOS', 'SP2MOS'] 2 | 3 | """Video + Spatial patch --> MOS 4 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 5 | from fastiqa.bunches.vqa.vid_sp2mos import * 6 | from fastai.vision.all import CropPad 7 | dls = VidSP2MOS.from_json('json/LIVE_FB_VQA_pad500.json', bs=2, clip_num=3, clip_size=2, item_tfms=CropPad(500)) 8 | dls.show_batch() 9 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 10 | from fastiqa.bunches.vqa.vid_sp2mos import * 11 | from fastai.vision.all import CropPad 12 | dls = VidSP2MOS.from_json('json/LIVE_FB_VQA.json', bs=2, clip_num=3, clip_size=2, item_tfms=CropPad(500)) 13 | dls.show_batch() 14 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 15 | from fastiqa.bunches.vqa.vid_sp2mos import * 16 | # dls = VidSP2MOS.from_json('json/LIVE_FB_VQA_1k.json', bs=1, clip_num=3, clip_size=2, item_tfms=None) 17 | dls = VidSP2MOS.from_json('json/LIVE_FB_VQA_30k.json', bs=1, clip_num=3, clip_size=2, item_tfms=None) 18 | dls.show_batch() 19 | 20 | # %% 21 | dls.show_batch() 22 | df = dls.get_df() 23 | df.columns 24 | # %% 25 | self = dls 26 | df = dls.get_df() 27 | [x in df.columns for x in self.vid_p1_label_col] 28 | all([x in df.columns for x in self.vid_p1_rois_col]) 29 | dls.get_df().columns 30 | # %% 31 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 32 | """ 33 | 34 | 35 | # DataBlock, RegressionBlock, ColReader, RandomSplitter, TensorImage, TensorCategory, TensorBBox, LabeledBBox, PILImage 36 | # import pandas as pd 37 | # import torch 38 | # import matplotlib.pyplot as plt 39 | 40 | 41 | from . import * 42 | from .vid2mos import * 43 | 44 | 45 | 46 | def create_roi_batch(data): 47 | xs, rois, ys = [], [], [] 48 | for d in data: 49 | xs.append(d[0]) 50 | rois.append(d[1]) 51 | ys.append(d[2]) 52 | 53 | try: 54 | xs_ = torch.cat([TensorImage(torch.cat([im[None] for im in x], dim=0))[None] for x in xs], dim=0) 55 | ys_ = torch.cat([y[None] for y in ys], dim=0) 56 | rois_ = torch.cat([roi[None] for roi in rois], dim=0) 57 | except: 58 | print('Error@create_batch') 59 | for idx, x in enumerate(xs): 60 | print(idx, x.size()) # torch.Size([3, 500, 500]) 61 | xs_ = torch.cat([TensorImage(torch.cat([im[None] for im in x], dim=0))[None] for x in xs], dim=0) 62 | print('xs_ is ok') 63 | print(xs_) 64 | raise 65 | 66 | return TensorImage(xs_), TensorCategory(rois_), TensorCategory(ys_) 67 | 68 | 69 | # https://docs.fast.ai/data.core#TfmdDL.show_batch 70 | def show_vid_roi_batch(dls, b=None, max_n=4, figsize=(30, 6), vertical=False, **kwargs): 71 | # rb --> rois in a batch 72 | max_n = min(max_n, dls.bs) 73 | xb, rb, yb = dls.one_batch() if b is None else b 74 | if vertical: 75 | fig, axes = plt.subplots(ncols=max_n, nrows=1, figsize=figsize, **kwargs) # , figsize=(12,6), dpi=120 76 | else: 77 | fig, axes = plt.subplots(nrows=max_n, ncols=1, figsize=figsize, **kwargs) # , figsize=(12,6), dpi=120 78 | if max_n == 1: axes = [axes] # only one item 79 | for i in range(max_n): 80 | xs, rs, ys = xb[i], rb[i], yb[i] 81 | # axes[i].imshow(x.permute(1,2,0).cpu().numpy()) 82 | # axes[i].set_title(f'{ys.item():.02f}') 83 | rois = rs.cpu().numpy().reshape(-1,4).tolist() 84 | tbbox = LabeledBBox(TensorBBox(rois), ['video', 'p1', 'p2', 'p3']) 85 | timg = TensorImage(clip2image(xs).cpu())*255 86 | 87 | tpil = PILImage.create(timg) 88 | ctx = tpil.show(ax=axes[i]) 89 | tbbox.show(ctx=ctx) 90 | 91 | # ys 0 dim tensor 92 | lbl = ','.join("%.2f" % x for x in ys.tolist()) if ys.dim() > 0 else "%.2f" % ys 93 | axes[i].set_title(lbl) 94 | axes[i].axis('off') 95 | 96 | class VidSP2MOS(Vid2MOS): 97 | pad = None # shift_based on padding 98 | # if loss_func is None: 99 | # loss_func = getattr(dls.train_ds, 'loss_func', None) 100 | def get_df(self): 101 | # prefix = ['top', 'left', 'bottom', 'right', 'height', 'width'] 102 | df = super().get_df() 103 | """prepare roi label""" 104 | if any([x not in df.columns for x in self.roi_col]): 105 | # unpack position 106 | for p in ['p1', 'p2', 'p3']: 107 | df[p] = df[p].apply(literal_eval) 108 | columns = [x+'_'+p for x in ['left', 'right', 'top', 'bottom', 'start', 'end']] 109 | df[columns] = pd.DataFrame(df[p].to_list(), columns=columns) 110 | df['top_vid'] = 0 111 | df['left_vid'] = 0 112 | df['bottom_vid'] = df['height'] # df['height_vid'] 113 | df['right_vid'] = df['width'] # df['width_vid'] 114 | df['start_vid'] = 0 115 | df['end_vid'] = df[self.frame_num_col] 116 | 117 | # shift if needed 118 | if self.pad: 119 | top_shift = (self.pad - df['height_vid']) // 2 120 | left_shift = (self.pad - df['width_vid']) // 2 121 | 122 | for s in ['_vid', '_p1', '_p2', '_p3']: 123 | df['top' + s] += top_shift 124 | df['left' + s] += left_shift 125 | df['bottom' + s] = df['top' + s] + df['height' + s] 126 | df['right' + s] = df['left' + s] + df['width' + s] 127 | 128 | df.to_csv(self.path/self.csv_labels, index=False) 129 | return df 130 | 131 | def get_block(self): 132 | VideoBlock = partial(MultiClipBlock, clip_size=self.clip_size, clip_num=self.clip_num) 133 | return DataBlock( 134 | blocks = (VideoBlock, RegressionBlock, RegressionBlock), 135 | getters = [ 136 | ColReader(self.fn_last_frame_col, pref=self.path/self.folder), 137 | ColReader(self.roi_col), 138 | ColReader(self.label_col), 139 | ], 140 | n_inp = 2, 141 | splitter = self.get_splitter(), 142 | item_tfms = self.item_tfms, # RandomCrop(500), 143 | ) 144 | 145 | def create_batch(self, data): 146 | return create_roi_batch(data) 147 | 148 | def show_crops(self, fn): 149 | """ only for validate patch coordinates""" 150 | df = self.get_df() 151 | row = df[df[self.fn_col] == fn].iloc[0] 152 | # db = 'ia' 153 | # if type(row['identifier']) == str else 'yfcc' 154 | # folder = self.path/self.folder/f"{db}-batch{row['batch']}/{row['identifier']}" 155 | folder = self.path/self.folder/fn 156 | p1 = row['left_p1'], row['top_p1'], row['right_p1'], row['bottom_p1'] 157 | tbbox = LabeledBBox(TensorBBox([p1]), ['p1']) 158 | 159 | tpil = PILImage.create(folder/f"image_{1+row['start_p1']:05d}.jpg") 160 | ctx = tpil.show() 161 | tbbox.show(ctx=ctx) 162 | 163 | tpil = PILImage.create(folder/f"image_{1+row['start_p2']:05d}.jpg") 164 | tpil.show() 165 | 166 | p3 = row['left_p3'], row['top_p3'], row['right_p3'], row['bottom_p3'] 167 | tbbox = LabeledBBox(TensorBBox([p3]), ['p3']) 168 | 169 | tpil = PILImage.create(folder/f"image_{1+row['start_p3']:05d}.jpg") 170 | ctx = tpil.show() 171 | tbbox.show(ctx=ctx) 172 | print(row) 173 | print(folder) 174 | 175 | 176 | class SP2MOS(VidSP2MOS): 177 | """Spatial patch to mos""" 178 | 179 | # if loss_func is None: 180 | # loss_func = getattr(dls.train_ds, 'loss_func', None) 181 | 182 | def get_block(self): 183 | VideoBlock = partial(MultiClipBlock, clip_size=self.clip_size, clip_num=self.clip_num) 184 | return DataBlock( 185 | blocks = (VideoBlock, RegressionBlock, RegressionBlock), 186 | getters = [ 187 | ColReader(self.fn_last_frame_col, pref=self.path/self.folder), 188 | ColReader(self.p1_rois_col), 189 | ColReader(self.p1_label_col), 190 | ], 191 | n_inp = 2, 192 | splitter = self.get_splitter(), 193 | item_tfms = self.item_tfms, # RandomCrop(500), 194 | ) 195 | 196 | 197 | VidSP2MOS.show_batch = show_vid_roi_batch 198 | -------------------------------------------------------------------------------- /fastiqa/bunches/vqa/video_utils.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | def video_process(video_file_path, dst_root_path, ext, fps=-1, size=240): 4 | if ext != video_file_path.suffix: 5 | return 6 | 7 | ffprobe_cmd = ('ffprobe -v error -select_streams v:0 ' 8 | '-of default=noprint_wrappers=1:nokey=1 -show_entries ' 9 | 'stream=width,height,avg_frame_rate,duration').split() 10 | ffprobe_cmd.append(str(video_file_path)) 11 | 12 | p = subprocess.run(ffprobe_cmd, capture_output=True) 13 | res = p.stdout.decode('utf-8').splitlines() 14 | if len(res) < 4: 15 | return 16 | 17 | frame_rate = [float(r) for r in res[2].split('/')] 18 | frame_rate = frame_rate[0] / frame_rate[1] 19 | duration = float(res[3]) 20 | n_frames = int(frame_rate * duration) 21 | 22 | name = video_file_path.stem 23 | dst_dir_path = dst_root_path / name 24 | dst_dir_path.mkdir(exist_ok=True) 25 | n_exist_frames = len([ 26 | x for x in dst_dir_path.iterdir() 27 | if x.suffix == '.jpg' and x.name[0] != '.' 28 | ]) 29 | 30 | if n_exist_frames >= n_frames: 31 | return 32 | 33 | if size > 0: 34 | width = int(res[0]) 35 | height = int(res[1]) 36 | 37 | if width > height: 38 | vf_param = 'scale=-1:{}'.format(size) 39 | else: 40 | vf_param = 'scale={}:-1'.format(size) 41 | 42 | if fps > 0: 43 | vf_param += ',minterpolate={}'.format(fps) 44 | 45 | ffmpeg_cmd = ['ffmpeg', '-i', str(video_file_path), '-vf', vf_param] 46 | else: 47 | ffmpeg_cmd = ['ffmpeg', '-i', str(video_file_path)] 48 | ffmpeg_cmd += ['-threads', '1', '{}/image_%05d.jpg'.format(dst_dir_path)] 49 | print(ffmpeg_cmd) 50 | subprocess.run(ffmpeg_cmd) 51 | print('\n') 52 | -------------------------------------------------------------------------------- /fastiqa/common.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torchvision import transforms 4 | 5 | from PIL import Image 6 | 7 | # render_output 8 | import matplotlib.pyplot as plt 9 | 10 | IMAGE_NET_MEAN = [0.485, 0.456, 0.406] 11 | IMAGE_NET_STD = [0.229, 0.224, 0.225] 12 | 13 | 14 | class Transform: 15 | def __init__(self): 16 | # normalize = transforms.Normalize(mean=IMAGE_NET_MEAN, std=IMAGE_NET_STD) 17 | 18 | self._train_transform = transforms.Compose( 19 | [ 20 | transforms.ToTensor(), 21 | ] 22 | ) 23 | 24 | self._val_transform = transforms.Compose([transforms.ToTensor()]) 25 | 26 | @property 27 | def train_transform(self): 28 | return self._train_transform 29 | 30 | @property 31 | def val_transform(self): 32 | return self._val_transform 33 | 34 | 35 | 36 | def set_up_seed(seed=42): 37 | torch.manual_seed(seed) 38 | torch.backends.cudnn.deterministic = True 39 | torch.backends.cudnn.benchmark = False 40 | np.random.seed(seed) 41 | 42 | 43 | def format_output(global_score, local_scores=None): 44 | if local_scores is None: 45 | return {"global_score": float(global_score)} 46 | else: 47 | return {"global_score": float(global_score), "local_scores": local_scores} 48 | 49 | 50 | def blend_output(input_image, output, vmin=0, vmax=100, alpha=0.8, resample=Image.BILINEAR): 51 | def stretch(image, minimum, maximum): 52 | if maximum is None: 53 | maximum = image.max() 54 | if minimum is None: 55 | minimum = image.min() 56 | image = (image - minimum) / (maximum - minimum) 57 | image[image < 0] = 0 58 | image[image > 1] = 1 59 | return image 60 | 61 | cm = plt.get_cmap('magma') 62 | # min-max normalize the image, you can skip this step 63 | qmap_matrix = output['local_scores'] 64 | qmap_matrix = 100*stretch(np.array(qmap_matrix), vmin, vmax) 65 | qmap_matrix = (np.array(qmap_matrix) * 255 / 100).astype(np.uint8) 66 | colored_map = cm(qmap_matrix) 67 | # Obtain a 4-channel image (R,G,B,A) in float [0, 1] 68 | # But we want to convert to RGB in uint8 and save it: 69 | heatmap = Image.fromarray((colored_map[:, :, :3] * 255).astype(np.uint8)) 70 | sz = input_image.size 71 | heatmap = heatmap.resize(sz, resample=resample) 72 | 73 | return Image.blend(input_image, heatmap, alpha=alpha) 74 | 75 | def render_output(input_image, output, vmin=0, vmax=100, fig=None): 76 | # QualityMap.plot 77 | fig, axes = plt.subplots(1, 3, figsize=(12, 8 * 3)) 78 | 79 | raw_img = axes[0].imshow(input_image) 80 | blend_img = axes[1].imshow(input_image, alpha=0.2) 81 | 82 | # _, H, W = input_image.shape # fastai 83 | W, H = input_image.size # PIL 84 | h, w = output['local_scores'].shape 85 | extent = (0, W // w * w, H // h * h, 0) 86 | 87 | blend_mat = axes[1].imshow(output['local_scores'], alpha=0.8, cmap='magma', 88 | extent=extent, interpolation='bilinear') 89 | mat = axes[2].imshow(output['local_scores'], cmap='gray', extent=extent, 90 | vmin=vmin, vmax=vmax) 91 | 92 | axes[0].set_title('Input image') 93 | score = axes[1].set_title(f"Predicted: {output['global_score']:.2f}") 94 | axes[2].set_title(f'Quality map {h}x{w}') 95 | return fig, [raw_img, blend_img, blend_mat, mat, score] 96 | -------------------------------------------------------------------------------- /fastiqa/dev.py: -------------------------------------------------------------------------------- 1 | from fastiqa_dev.models.rnn_head import * 2 | from fastiqa_dev.models.seq_body_head import * 3 | -------------------------------------------------------------------------------- /fastiqa/inference_model.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import torch 4 | import numpy as np 5 | # from PIL.Image import Image 6 | from PIL import Image 7 | from torchvision.datasets.folder import default_loader 8 | 9 | from .common import Transform, format_output, render_output 10 | 11 | import cv2 12 | 13 | """ 14 | ####################### 15 | # %% show quality map 16 | ####################### 17 | %matplotlib inline 18 | from paq2piq.inference_model import *; 19 | 20 | file = '/media/zq/Seagate/Git/fastiqa/images/Picture1.jpg' 21 | model = InferenceModel(RoIPoolModel(), 'paq2piq/RoIPoolModel.pth') 22 | image = Image.open(file) 23 | output = model.predict_from_pil_image(image) 24 | render_output(image, output) 25 | # %% 26 | 27 | ################################ 28 | # %% show quality map of a video 29 | ################################ 30 | 31 | # %% 32 | 33 | """ 34 | 35 | # use cuda if available 36 | use_cuda = torch.cuda.is_available() 37 | device = torch.device("cuda" if use_cuda else "cpu") 38 | 39 | 40 | class InferenceModel: 41 | blk_size = 20, 20 42 | categories = 'Bad', 'Poor', 'Fair', 'Good', 'Excellent' 43 | 44 | def __init__(self, model, path_to_model_state: Path): 45 | self.transform = Transform().val_transform 46 | model_state = torch.load(path_to_model_state, map_location=lambda storage, loc: storage) 47 | self.model = model 48 | self.model.load_state_dict(model_state["model"]) 49 | self.model = self.model.to(device) 50 | self.model.eval() 51 | 52 | def predict_from_file(self, image_path: Path, render=False): 53 | image = default_loader(image_path) 54 | return self.predict(image) 55 | 56 | def predict_from_pil_image(self, image: Image): 57 | image = image.convert("RGB") 58 | return self.predict(image) 59 | 60 | def predict_from_cv2_image(self, image): 61 | image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 62 | return self.predict(image) 63 | 64 | def predict_from_vid_file(self, vid_path: Path): 65 | im = Image.open(vid_path) 66 | index = 1 67 | for frame in ImageSequence.Iterator(im): 68 | frame.save("frame%d.png" % index) 69 | index = index + 1 70 | pass 71 | 72 | @torch.no_grad() 73 | def predict(self, image): 74 | image = self.transform(image) 75 | image = image.unsqueeze_(0) 76 | image = image.to(device) 77 | self.model.input_block_rois(self.blk_size, [image.shape[-2], image.shape[-1]], device=device) 78 | t = self.model(image).data.cpu().numpy()[0] 79 | 80 | local_scores = np.reshape(t[1:], self.blk_size) 81 | global_score = t[0] 82 | 83 | # normalize the global score 84 | x_mean, std_left, std_right = 72.59696108881171, 7.798274017370107, 4.118047289170692 85 | if global_score < x_mean: 86 | x = x_mean + x_mean*(global_score-x_mean)/(4*std_left) 87 | if x < 0: x = 0 88 | elif global_score > x_mean: 89 | x = x_mean + (100-x_mean)*(global_score-x_mean)/(4*std_right) 90 | if x > 100: x = 100 91 | else: 92 | x = x_mean 93 | category = self.categories[int(x//20)] 94 | return {"global_score": global_score, 95 | "normalized_global_score": x, 96 | "local_scores": local_scores, 97 | "category": category} 98 | -------------------------------------------------------------------------------- /fastiqa/iqa.py: -------------------------------------------------------------------------------- 1 | from fastai.vision.all import * 2 | from fastai.distributed import * 3 | 4 | from fastiqa.bunch import * # load_json 5 | 6 | from fastiqa.bunches.iqa.im2mos import * 7 | from fastiqa.bunches.iqa.im_roi2mos import * 8 | from fastiqa.bunches.iqa.im_bbox2mos import * 9 | # from fastiqa.bunches.iqa.test_images import * 10 | 11 | from fastiqa.models._body_head import * 12 | from fastiqa.models._roi_pool import * 13 | from fastiqa.models.nima import NIMA 14 | 15 | from fastiqa.learn import * 16 | from fastiqa.iqa_exp import * 17 | 18 | from torchvision.models.video.resnet import * #r3d_18, r2plus1d_18 19 | # https://pytorch.org/docs/stable/_modules/torchvision/models/video/resnet.html#r3d_18 20 | from fastiqa.models.mobilenet.v2 import mobilenet3d_v2 21 | from torchvision.models import * 22 | -------------------------------------------------------------------------------- /fastiqa/log.py: -------------------------------------------------------------------------------- 1 | # import logging 2 | # # logging.basicConfig(level=logging.WARNING) 3 | # logging.basicConfig(level=logging.DEBUG) 4 | 5 | 6 | # import sys 7 | # from loguru import logger 8 | # logger.remove() 9 | # logger.add(sys.stderr, level="DEBUG") 10 | # logger.debug('haha') 11 | # logger.info('haha') 12 | 13 | 14 | import sys 15 | from loguru import logger 16 | 17 | DEBUG = False 18 | 19 | if DEBUG: 20 | logger.remove() 21 | logger.add(sys.stderr, level="DEBUG") 22 | else: 23 | logger.remove() 24 | logger.add(sys.stderr, level="WARNING") 25 | -------------------------------------------------------------------------------- /fastiqa/loss.py: -------------------------------------------------------------------------------- 1 | from ._loss import * 2 | from fastai.vision.all import * 3 | # EDMLoss 4 | 5 | @use_kwargs_dict(reduction='mean') 6 | def EDMLossFlat(*args, axis=-1, floatify=True, **kwargs): 7 | "Same as `nn.EDMLoss`, but flattens input and target." 8 | return BaseLoss(EDMLoss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs) 9 | 10 | @use_kwargs_dict(reduction='mean') 11 | def HuberLossFlat(*args, axis=-1, floatify=True, **kwargs): 12 | "Same as `nn.L1Loss`, but flattens input and target." 13 | return BaseLoss(nn.SmoothL1Loss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs) 14 | -------------------------------------------------------------------------------- /fastiqa/models/_3d.py: -------------------------------------------------------------------------------- 1 | from fastai.vision.all import * 2 | 3 | def in_channels_3d(m): 4 | "Return the shape of the first weight layer in `m`." 5 | for l in flatten_model(m): 6 | if getattr(l, 'weight', None) is not None and l.weight.ndim==5: #4 7 | return l.weight.shape[1] 8 | raise Exception('No weight layer') 9 | 10 | def dummy_eval_3d(m, size=(64,64)): 11 | "Evaluate `m` on a dummy input of a certain `size`" 12 | ch_in = in_channels_3d(m) # weight of size [64, 3, 7, 7, 7] 13 | # 1, ch_in, 64 64 14 | # 1, ch_in, 2, 64 64 # 2 frames per video 15 | x = one_param(m).new(1, ch_in, 2, *size).requires_grad_(False).uniform_(-1.,1.) 16 | with torch.no_grad(): return m.eval()(x) 17 | 18 | # Cell 19 | def model_sizes_3d(m, size=(64,64)): 20 | "Pass a dummy input through the model `m` to get the various sizes of activations." 21 | with hook_outputs(m) as hooks: 22 | _ = dummy_eval_3d(m, size=size) 23 | return [o.stored.shape for o in hooks] 24 | 25 | def num_features_model_3d(m): 26 | "Return the number of output features for `m`." 27 | sz,ch_in = 32,in_channels_3d(m) 28 | while True: 29 | #Trying for a few sizes in case the model requires a big input size. 30 | try: 31 | return model_sizes_3d(m, (sz,sz))[-1][1] 32 | except Exception as e: 33 | sz *= 2 34 | if sz > 2048: raise e 35 | 36 | class AdaptiveConcatPool3d(Module): 37 | "Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`" 38 | def __init__(self, size=None): 39 | self.size = size or 1 40 | self.ap = nn.AdaptiveAvgPool3d(self.size) 41 | self.mp = nn.AdaptiveMaxPool3d(self.size) 42 | def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1) 43 | 44 | def create_head_3d(nf, n_out, lin_ftrs=None, ps=0.5, concat_pool=True, bn_final=False, lin_first=False, y_range=None): 45 | "Model head that takes `nf` features, runs through `lin_ftrs`, and out `n_out` classes." 46 | lin_ftrs = [nf, 512, n_out] if lin_ftrs is None else [nf] + lin_ftrs + [n_out] 47 | ps = L(ps) 48 | if len(ps) == 1: ps = [ps[0]/2] * (len(lin_ftrs)-2) + ps 49 | actns = [nn.ReLU(inplace=True)] * (len(lin_ftrs)-2) + [None] 50 | pool = AdaptiveConcatPool3d() if concat_pool else nn.AdaptiveAvgPool3d(1) 51 | layers = [pool, Flatten()] 52 | if lin_first: layers.append(nn.Dropout(ps.pop(0))) 53 | for ni,no,p,actn in zip(lin_ftrs[:-1], lin_ftrs[1:], ps, actns): 54 | layers += LinBnDrop(ni, no, bn=True, p=p, act=actn, lin_first=lin_first) 55 | if lin_first: layers.append(nn.Linear(lin_ftrs[-2], n_out)) 56 | if bn_final: layers.append(nn.BatchNorm1d(lin_ftrs[-1], momentum=0.01)) 57 | if y_range is not None: layers.append(SigmoidRange(*y_range)) 58 | return nn.Sequential(*layers) 59 | -------------------------------------------------------------------------------- /fastiqa/models/__inceptiontime_layers.py: -------------------------------------------------------------------------------- 1 | # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/100_layers.ipynb (unless otherwise specified). 2 | 3 | __all__ = ['noop', 'mish', 'Mish', 'get_act_layer', 'same_padding1d', 'Pad1d', 'Conv1dSame', 'Chomp1d', 'Conv1dCausal', 4 | 'Conv1d', 'CoordConv1D', 'LambdaPlus', 'Flatten', 'Squeeze', 'Unsqueeze', 'YRange', 'Temp'] 5 | 6 | # Cell 7 | from fastai.torch_core import Module 8 | from fastai.data.all import * 9 | 10 | # Cell 11 | def noop(x): return x 12 | 13 | # Cell 14 | # Misra, D. (2019). Mish: A Self Regularized Non-Monotonic Neural Activation Function. arXiv preprint arXiv:1908.08681. 15 | # https://arxiv.org/abs/1908.08681 16 | # GitHub: https://github.com/digantamisra98/Mish 17 | @torch.jit.script 18 | def mish(input): 19 | '''Applies the mish function element-wise: mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))''' 20 | return input * torch.tanh(F.softplus(input)) 21 | 22 | class Mish(Module): 23 | def forward(self, input): 24 | return mish(input) 25 | 26 | # Cell 27 | def get_act_layer(act_fn, act_kwargs={}): 28 | act_fn = act_fn.lower() 29 | assert act_fn in ['relu', 'leakyrelu', 'prelu', 'elu', 'mish', 'swish'], 'incorrect act_fn' 30 | if act_fn == 'relu': return nn.ReLU() 31 | elif act_fn == 'leakyrelu': return nn.LeakyReLU(**act_kwargs) 32 | elif act_fn == 'prelu': return nn.PReLU(**act_kwargs) 33 | elif act_fn == 'elu': return nn.ELU(**act_kwargs) 34 | elif act_fn == 'mish': return Mish() 35 | elif act_fn == 'swish': return Swish() 36 | 37 | # Cell 38 | def same_padding1d(seq_len, ks, stride=1, dilation=1): 39 | effective_ks = (ks - 1) * dilation + 1 40 | out_dim = (seq_len + stride - 1) // stride 41 | p = max(0, (out_dim - 1) * stride + effective_ks - seq_len) 42 | padding_before = p // 2 43 | padding_after = p - padding_before 44 | return padding_before, padding_after 45 | 46 | 47 | class Pad1d(nn.ConstantPad1d): 48 | def __init__(self, padding, value=0.): 49 | super().__init__(padding, value) 50 | 51 | 52 | class Conv1dSame(Module): 53 | "Conv1d with padding='same'" 54 | 55 | def __init__(self, c_in, c_out, ks=3, stride=1, dilation=1, **kwargs): 56 | self.ks, self.stride, self.dilation = ks, stride, dilation 57 | self.conv1d_same = nn.Conv1d(c_in, c_out, ks, stride=stride, dilation=dilation, **kwargs) 58 | self.pad = Pad1d 59 | 60 | 61 | def forward(self, x): 62 | self.padding = same_padding1d(x.shape[-1],self.ks,stride=self.stride,dilation=self.dilation) 63 | return self.conv1d_same(self.pad(self.padding)(x)) 64 | 65 | # Cell 66 | # https://github.com/locuslab/TCN/blob/master/TCN/tcn.py 67 | class Chomp1d(Module): 68 | def __init__(self, chomp_size): 69 | self.chomp_size = chomp_size 70 | 71 | def forward(self, x): 72 | return x[:, :, :-self.chomp_size].contiguous() 73 | 74 | 75 | class Conv1dCausal(Module): 76 | def __init__(self, c_in, c_out, ks, stride=1, dilation=1, **kwargs): 77 | padding = (ks - 1) * dilation 78 | self.conv = nn.Conv1d(c_in, c_out, ks, stride=stride, padding=padding, dilation=dilation, **kwargs) 79 | self.chomp = Chomp1d(math.ceil(padding / stride)) 80 | 81 | def forward(self, x): 82 | return self.chomp(self.conv(x)) 83 | 84 | # Cell 85 | def Conv1d(c_in, c_out, ks=3, stride=1, padding='same', dilation=1, bias=True, act_fn='relu', act_kwargs={}, 86 | bn_before_conv=False, bn_before_act=True, bn_after_act=False, zero_bn=False, **kwargs): 87 | '''conv1d with default padding='same', bn and act_fn (default = 'relu')''' 88 | layers = [] 89 | if bn_before_conv: layers.append(nn.BatchNorm1d(c_in)) 90 | if padding == 'same': layers.append(Conv1dSame(c_in, c_out, ks, stride=stride, dilation=dilation, bias=bias, **kwargs)) 91 | elif padding == 'causal': layers.append(Conv1dCausal(c_in, c_out, ks, stride=stride, dilation=dilation, bias=bias, **kwargs)) 92 | else: 93 | if padding == 'valid': padding = 0 94 | layers.append(nn.Conv1d(c_in, c_out, ks, stride=stride, padding=padding, dilation=dilation, bias=bias, **kwargs)) 95 | if bn_before_act: layers.append(nn.BatchNorm1d(c_out)) 96 | if act_fn: layers.append(get_act_layer(act_fn, act_kwargs)) 97 | if bn_after_act: 98 | bn = nn.BatchNorm1d(c_out) 99 | nn.init.constant_(bn.weight, 0. if zero_bn else 1.) 100 | layers.append(bn) 101 | return nn.Sequential(*layers) 102 | 103 | # Cell 104 | class CoordConv1D(Module): 105 | def forward(self, x): 106 | bs, _, seq_len = x.size() 107 | cc = torch.arange(seq_len, device=device, dtype=torch.float) / (seq_len - 1) 108 | cc = cc * 2 - 1 109 | cc = cc.repeat(bs, 1, 1) 110 | x = torch.cat([x, cc], dim=1) 111 | return x 112 | 113 | # Cell 114 | class LambdaPlus(Module): 115 | def __init__(self, func, *args, **kwargs): self.func,self.args,self.kwargs=func,args,kwargs 116 | def forward(self, x): return self.func(x, *self.args, **self.kwargs) 117 | 118 | # Cell 119 | class Flatten(Module): 120 | def forward(self, x): return x.view(x.size(0), -1) 121 | 122 | class Squeeze(Module): 123 | def __init__(self, dim=-1): 124 | self.dim = dim 125 | def forward(self, x): return x.squeeze(dim=self.dim) 126 | 127 | class Unsqueeze(Module): 128 | def __init__(self, dim=-1): 129 | self.dim = dim 130 | def forward(self, x): return x.unsqueeze(dim=self.dim) 131 | 132 | class YRange(Module): 133 | def __init__(self, y_range:tuple): 134 | self.y_range = y_range 135 | self.sigmoid = torch.sigmoid 136 | def forward(self, x): 137 | x = self.sigmoid(x) 138 | return x * (self.y_range[1] - self.y_range[0]) + self.y_range[0] 139 | 140 | class Temp(Module): 141 | def __init__(self, temp): 142 | self.temp = float(temp) 143 | self.temp = nn.Parameter(torch.Tensor(1).fill_(self.temp).to(device)) 144 | def forward(self, x): 145 | return x.div_(self.temp) 146 | -------------------------------------------------------------------------------- /fastiqa/models/__init__.py: -------------------------------------------------------------------------------- 1 | from ..basics import * 2 | -------------------------------------------------------------------------------- /fastiqa/models/_body_head.py: -------------------------------------------------------------------------------- 1 | __all__ = ['BodyHeadModel'] 2 | 3 | """ 4 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 5 | from fastiqa.models._body_head import * 6 | from fastiqa.models.resnet_3d import * 7 | from fastai.vision.all import * 8 | 9 | BodyHeadModel.compare_paramters( 10 | backbones = { 11 | 'ResNet-18' : resnet18, 12 | 'ResNet(2+1)D-18' : r2p1d18_K_200ep, 13 | 'ResNet3D-18' : r3d18_K_200ep, 14 | 'ResNet-50' : resnet50, 15 | 'ResNet(2+)1D-50' : r2p1d50_K_200ep, 16 | 'ResNet3D-50' : r3d50_KM_200ep 17 | } 18 | ) 19 | # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 20 | """ 21 | 22 | from fastai.vision.all import * 23 | from ..basics import * # IqaModel 24 | from ._3d import * 25 | # from ..utils.cached_property import cached_property 26 | 27 | class BodyHeadModel(IqaModel): 28 | backbone = None 29 | is_3d = None 30 | clip_num = 1 31 | _num_features_model = None 32 | _create_head = None 33 | cut = -1 # https://docs.fast.ai/vision.learner#Cut-a-pretrained-model 34 | n_out_per_roi = 1 35 | 36 | @staticmethod 37 | def split_on(m): 38 | return [[m.body], [m.head]] 39 | 40 | def create_body(self): 41 | try: 42 | return create_body(self.backbone) 43 | except StopIteration: 44 | logging.warning('Cut pretraiend {self.backbone.__name__} at -1') 45 | return create_body(self.backbone, cut=self.cut) 46 | 47 | def create_head(self): 48 | num_features = self._num_features_model(self.body) 49 | return self._create_head(num_features * 2 * self.clip_num, self.n_out_per_roi) 50 | # output 1 score per image/video location 51 | # to learn distributions, set it to 5 52 | 53 | def __init__(self, backbone=None, **kwargs): 54 | # remove simply fc 55 | # one could try only modify the last layer 56 | super().__init__(**kwargs) 57 | if backbone is not None: 58 | # self.__name__ += f' ({backbone.__name__})' 59 | self.backbone = backbone 60 | 61 | if self.is_3d is None: 62 | name = self.backbone.__name__ 63 | self.is_3d = '3d' in name or '2p1d' in name or '2plus1d' in name or 'mc3' in name 64 | 65 | self._num_features_model = num_features_model_3d if self.is_3d else num_features_model 66 | self._create_head = create_head_3d if self.is_3d else create_head 67 | self.body = self.create_body() 68 | self.head = self.create_head() 69 | 70 | def forward(self, x: Tensor, *args, **kwargs) -> Tensor: 71 | # multi clip: 72 | # bs x clip_num x clip_size x 3 x H x W 73 | # --> bs x clip_num x 3 x clip_size x H x W 74 | if self.is_3d and x.size()[-4] != 3 and x.size()[-3] == 3: 75 | x = x.transpose(-4,-3) 76 | 77 | batch_size = x.size(0) 78 | # video data 79 | # if x.size()[1] != 3 and x.size()[2] == 3: 80 | # x = x.transpose(1,2) 81 | base_feat = self.body(x) 82 | # print('base_feat:', base_feat.size()) # torch.Size([64, 8192, 128]) 83 | pred = self.head(base_feat) 84 | return pred.view(batch_size, -1) 85 | 86 | @staticmethod 87 | def compare_paramters(backbones): 88 | body_params = [] 89 | head_params = [] 90 | 91 | if type(backbones) is dict: 92 | labels = backbones.keys() 93 | backbones = backbones.values() 94 | elif type(backbones) is list or tuple: 95 | labels = [backbone.__name__ for backbone in backbones] 96 | else: 97 | raise TypeError('backbones must be a list, tuple or dict') 98 | 99 | for backbone in backbones: 100 | model = BodyHeadModel(backbone=backbone) 101 | body_params.append(total_params(model.body)[0]) 102 | head_params.append(total_params(model.head)[0]) 103 | 104 | width = 0.35 # the width of the bars: can also be len(x) sequence 105 | ind = np.arange(len(backbones)) 106 | 107 | p1 = plt.barh(ind, body_params, width) 108 | p2 = plt.barh(ind, head_params, width, left=body_params) # botoom for bar, left for barh 109 | 110 | plt.xlabel('#parameters') 111 | plt.title('Model parameters') 112 | plt.yticks(ind, labels) 113 | #plt.yticks(np.arange(0, 81, 10)) 114 | plt.legend((p1[0], p2[0]), ('backbone', 'head')) 115 | 116 | plt.show() 117 | 118 | 119 | 120 | # if fc: 121 | # return nn.Sequential( 122 | # nn.AdaptiveAvgPool2d(output_size=(1, 1)), 123 | # nn.Linear(in_features=num_features, out_features=self.n_out, bias=True) 124 | # ) 125 | # if fc: 126 | # return nn.Sequential( 127 | # nn.AdaptiveAvgPool3d(output_size=(1, 1, 1)), 128 | # nn.Linear(in_features=num_features, out_features=self.n_out, bias=True) 129 | # ) 130 | -------------------------------------------------------------------------------- /fastiqa/models/_inceptiontime.py: -------------------------------------------------------------------------------- 1 | # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/102_InceptionTime.ipynb (unless otherwise specified). 2 | 3 | __all__ = ['shortcut', 'Inception', 'InceptionBlock', 'InceptionTime'] 4 | 5 | # Cell 6 | from fastai.data.all import * 7 | from .__inceptiontime_layers import * 8 | 9 | # Cell 10 | # This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on: 11 | 12 | # Fawaz, H. I., Lucas, B., Forestier, G., Pelletier, C., Schmidt, D. F., Weber, J., ... & Petitjean, F. (2019). 13 | # InceptionTime: Finding AlexNet for Time Series Classification. arXiv preprint arXiv:1909.04939. 14 | # Official InceptionTime tensorflow implementation: https://github.com/hfawaz/InceptionTime 15 | 16 | 17 | def shortcut(c_in, c_out): 18 | return nn.Sequential(*[nn.Conv1d(c_in, c_out, kernel_size=1), nn.BatchNorm1d(c_out)]) 19 | 20 | 21 | class Inception(Module): 22 | def __init__(self, c_in, bottleneck=32, ks=40, nb_filters=32): 23 | self.bottleneck = nn.Conv1d(c_in, bottleneck, 1) if bottleneck and c_in > 1 else noop 24 | mts_feat = bottleneck or c_in 25 | conv_layers = [] 26 | kss = [ks // (2**i) for i in range(3)] 27 | # ensure odd kss until nn.Conv1d with padding='same' is available in pytorch 28 | kss = [ksi if ksi % 2 != 0 else ksi - 1 for ksi in kss] 29 | for i in range(len(kss)): conv_layers.append(nn.Conv1d(mts_feat, nb_filters, kernel_size=kss[i], padding=kss[i]//2)) 30 | self.conv_layers = nn.ModuleList(conv_layers) 31 | self.maxpool = nn.MaxPool1d(3, stride=1, padding=1) 32 | self.conv = nn.Conv1d(c_in, nb_filters, kernel_size=1) 33 | self.bn = nn.BatchNorm1d(nb_filters * 4) 34 | self.act = nn.ReLU() 35 | 36 | def forward(self, x): 37 | input_tensor = x 38 | x = self.bottleneck(input_tensor) 39 | for i in range(3): 40 | out_ = self.conv_layers[i](x) 41 | if i == 0: out = out_ 42 | else: out = torch.cat((out, out_), 1) 43 | mp = self.conv(self.maxpool(input_tensor)) 44 | inc_out = torch.cat((out, mp), 1) 45 | return self.act(self.bn(inc_out)) 46 | 47 | 48 | class InceptionBlock(Module): 49 | def __init__(self,c_in,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6): 50 | self.residual = residual 51 | self.depth = depth 52 | 53 | #inception & residual layers 54 | inc_mods = [] 55 | res_layers = [] 56 | res = 0 57 | for d in range(depth): 58 | inc_mods.append(Inception(c_in if d == 0 else nb_filters * 4, bottleneck=bottleneck if d > 0 else 0,ks=ks, nb_filters=nb_filters)) 59 | if self.residual and d % 3 == 2: 60 | res_layers.append(shortcut(c_in if res == 0 else nb_filters * 4, nb_filters * 4)) 61 | res += 1 62 | else: res_layer = res_layers.append(None) 63 | self.inc_mods = nn.ModuleList(inc_mods) 64 | self.res_layers = nn.ModuleList(res_layers) 65 | self.act = nn.ReLU() 66 | 67 | def forward(self, x): 68 | res = x 69 | for d, l in enumerate(range(self.depth)): 70 | x = self.inc_mods[d](x) 71 | if self.residual and d % 3 == 2: 72 | res = self.res_layers[d](res) 73 | x += res 74 | res = x 75 | x = self.act(x) 76 | return x 77 | 78 | class InceptionTime(Module): 79 | def __init__(self,c_in,c_out,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6): 80 | self.block = InceptionBlock(c_in, bottleneck=bottleneck, ks=ks, nb_filters=nb_filters, residual=residual, depth=depth) 81 | self.gap = nn.AdaptiveAvgPool1d(1) 82 | self.squeeze = Squeeze(-1) 83 | self.fc = nn.Linear(nb_filters * 4, c_out) 84 | 85 | def forward(self, x): 86 | x = self.block(x) 87 | x = self.squeeze(self.gap(x)) 88 | x = self.fc(x) 89 | return x 90 | -------------------------------------------------------------------------------- /fastiqa/models/inception_head.py: -------------------------------------------------------------------------------- 1 | 2 | from fastai.vision.all import * 3 | from ._body_head import * 4 | from ..basics import IqaModel 5 | from ._inceptiontime import * 6 | import logging 7 | 8 | class InceptionTimeModel(IqaModel): 9 | """ 10 | c_out inception_time output 11 | n_out model output 12 | """ 13 | siamese = False 14 | def __init__(self, c_in, bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6, **kwargs): 15 | super().__init__(**kwargs) 16 | c_out = 1 if self.siamese else self.n_out 17 | self.inception_time = InceptionTime(c_in=c_in,c_out=c_out, bottleneck=bottleneck,ks=ks,nb_filters=nb_filters,residual=residual,depth=depth) 18 | 19 | @classmethod 20 | def from_dls(cls, dls, n_out=None, **kwargs): 21 | if n_out is None: n_out = dls.c 22 | return cls(c_in = dls.vars, n_out=n_out, **kwargs) 23 | 24 | def forward(self, x, x2=None): # more features 25 | # if self.training == False: 26 | # self.siamese = False 27 | # self.n_out = 1 28 | 29 | if x2 is not None: 30 | x = torch.cat([x, x2], dim=-1) # 4096 features 31 | if self.siamese: 32 | x = x.view(self.n_out*x.shape[0], -1, x.shape[-1]) # *x.shape[2:] 33 | # [bs, n_out*length, features] --> [bs*n_out, length, features] 34 | # [bs*n_out, length, features] --> [bs*n_out, features, length ] 35 | y = self.inception_time(x.transpose(1, 2)) 36 | return y.view(-1, self.n_out) if self.siamese else y 37 | 38 | def input_sois(self, clip_num=16): 39 | raise NotImplementedError 40 | -------------------------------------------------------------------------------- /fastiqa/models/mobilenet/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | """ 4 | # %% 5 | 6 | 7 | 8 | 9 | 10 | # %% 11 | """ 12 | -------------------------------------------------------------------------------- /fastiqa/models/mobilenet/_v2.py: -------------------------------------------------------------------------------- 1 | '''MobilenetV2 in PyTorch. 2 | 3 | See the paper "MobileNetV2: Inverted Residuals and Linear Bottlenecks" for more details. 4 | # https://github.com/okankop/Efficient-3DCNNs/blob/master/models/mobilenetv2.py 5 | ''' 6 | import torch 7 | import math 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | from torch.autograd import Variable 11 | 12 | 13 | 14 | 15 | def conv_bn(inp, oup, stride): 16 | return nn.Sequential( 17 | nn.Conv3d(inp, oup, kernel_size=3, stride=stride, padding=(1,1,1), bias=False), 18 | nn.BatchNorm3d(oup), 19 | nn.ReLU6(inplace=True) 20 | ) 21 | 22 | 23 | def conv_1x1x1_bn(inp, oup): 24 | return nn.Sequential( 25 | nn.Conv3d(inp, oup, 1, 1, 0, bias=False), 26 | nn.BatchNorm3d(oup), 27 | nn.ReLU6(inplace=True) 28 | ) 29 | 30 | 31 | class InvertedResidual(nn.Module): 32 | def __init__(self, inp, oup, stride, expand_ratio): 33 | super(InvertedResidual, self).__init__() 34 | self.stride = stride 35 | 36 | hidden_dim = round(inp * expand_ratio) 37 | self.use_res_connect = self.stride == (1,1,1) and inp == oup 38 | 39 | if expand_ratio == 1: 40 | self.conv = nn.Sequential( 41 | # dw 42 | nn.Conv3d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), 43 | nn.BatchNorm3d(hidden_dim), 44 | nn.ReLU6(inplace=True), 45 | # pw-linear 46 | nn.Conv3d(hidden_dim, oup, 1, 1, 0, bias=False), 47 | nn.BatchNorm3d(oup), 48 | ) 49 | else: 50 | self.conv = nn.Sequential( 51 | # pw 52 | nn.Conv3d(inp, hidden_dim, 1, 1, 0, bias=False), 53 | nn.BatchNorm3d(hidden_dim), 54 | nn.ReLU6(inplace=True), 55 | # dw 56 | nn.Conv3d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), 57 | nn.BatchNorm3d(hidden_dim), 58 | nn.ReLU6(inplace=True), 59 | # pw-linear 60 | nn.Conv3d(hidden_dim, oup, 1, 1, 0, bias=False), 61 | nn.BatchNorm3d(oup), 62 | ) 63 | 64 | def forward(self, x): 65 | if self.use_res_connect: 66 | return x + self.conv(x) 67 | else: 68 | return self.conv(x) 69 | 70 | 71 | class MobileNetV2(nn.Module): 72 | def __init__(self, num_classes=1000, sample_size=224, width_mult=1.): 73 | super(MobileNetV2, self).__init__() 74 | block = InvertedResidual 75 | input_channel = 32 76 | last_channel = 1280 77 | interverted_residual_setting = [ 78 | # t, c, n, s 79 | [1, 16, 1, (1,1,1)], 80 | [6, 24, 2, (2,2,2)], 81 | [6, 32, 3, (2,2,2)], 82 | [6, 64, 4, (2,2,2)], 83 | [6, 96, 3, (1,1,1)], 84 | [6, 160, 3, (2,2,2)], 85 | [6, 320, 1, (1,1,1)], 86 | ] 87 | 88 | # building first layer 89 | assert sample_size % 16 == 0. 90 | input_channel = int(input_channel * width_mult) 91 | self.last_channel = int(last_channel * width_mult) if width_mult > 1.0 else last_channel 92 | self.features = [conv_bn(3, input_channel, (1,2,2))] 93 | # building inverted residual blocks 94 | for t, c, n, s in interverted_residual_setting: 95 | output_channel = int(c * width_mult) 96 | for i in range(n): 97 | stride = s if i == 0 else (1,1,1) 98 | self.features.append(block(input_channel, output_channel, stride, expand_ratio=t)) 99 | input_channel = output_channel 100 | # building last several layers 101 | self.features.append(conv_1x1x1_bn(input_channel, self.last_channel)) 102 | # make it nn.Sequential 103 | self.features = nn.Sequential(*self.features) 104 | 105 | # building classifier 106 | self.classifier = nn.Sequential( 107 | nn.Dropout(0.2), 108 | nn.Linear(self.last_channel, num_classes), 109 | ) 110 | 111 | self._initialize_weights() 112 | 113 | def forward(self, x): 114 | x = self.features(x) 115 | x = F.avg_pool3d(x, x.data.size()[-3:]) 116 | x = x.view(x.size(0), -1) 117 | x = self.classifier(x) 118 | return x 119 | 120 | def _initialize_weights(self): 121 | for m in self.modules(): 122 | if isinstance(m, nn.Conv3d): 123 | n = m.kernel_size[0] * m.kernel_size[1] * m.kernel_size[2] * m.out_channels 124 | m.weight.data.normal_(0, math.sqrt(2. / n)) 125 | if m.bias is not None: 126 | m.bias.data.zero_() 127 | elif isinstance(m, nn.BatchNorm3d): 128 | m.weight.data.fill_(1) 129 | m.bias.data.zero_() 130 | elif isinstance(m, nn.Linear): 131 | n = m.weight.size(1) 132 | m.weight.data.normal_(0, 0.01) 133 | m.bias.data.zero_() 134 | 135 | 136 | def get_fine_tuning_parameters(model, ft_portion): 137 | if ft_portion == "complete": 138 | return model.parameters() 139 | 140 | elif ft_portion == "last_layer": 141 | ft_module_names = [] 142 | ft_module_names.append('classifier') 143 | 144 | parameters = [] 145 | for k, v in model.named_parameters(): 146 | for ft_module in ft_module_names: 147 | if ft_module in k: 148 | parameters.append({'params': v}) 149 | break 150 | else: 151 | parameters.append({'params': v, 'lr': 0.0}) 152 | return parameters 153 | 154 | else: 155 | raise ValueError("Unsupported ft_portion: 'complete' or 'last_layer' expected") 156 | 157 | 158 | def get_model(**kwargs): 159 | """ 160 | Returns the model. 161 | """ 162 | model = MobileNetV2(**kwargs) 163 | return model 164 | 165 | 166 | if __name__ == "__main__": 167 | model = get_model(num_classes=600, sample_size=112, width_mult=1.) 168 | model = model.cuda() 169 | model = nn.DataParallel(model, device_ids=None) 170 | print(model) 171 | 172 | 173 | input_var = Variable(torch.randn(8, 3, 16, 112, 112)) 174 | output = model(input_var) 175 | print(output.shape) 176 | -------------------------------------------------------------------------------- /fastiqa/models/mobilenet/v2.py: -------------------------------------------------------------------------------- 1 | from ._v2 import get_model 2 | __all__ = ['mobilenet3d_v2'] 3 | # https://discuss.pytorch.org/t/solved-keyerror-unexpected-key-module-encoder-embedding-weight-in-state-dict/1686/3 4 | 5 | import torch 6 | from collections import OrderedDict 7 | 8 | def remove_module_in_state_dict(state_dict): 9 | new_state_dict = OrderedDict() 10 | for k, v in state_dict.items(): 11 | name = k[7:] # remove `module.` 12 | new_state_dict[name] = v 13 | return new_state_dict 14 | 15 | 16 | def mobilenet3d_v2(pretrained=True, **kwargs): 17 | path_to_model_state = '/media/zq/DB/pth/kinetics_mobilenetv2_1.0x_RGB_16_best.pth' 18 | model = get_model(num_classes=600, sample_size=112, width_mult=1.) 19 | if pretrained: 20 | model_state = torch.load(path_to_model_state, map_location=lambda storage, loc: storage) 21 | model.load_state_dict(remove_module_in_state_dict(model_state['state_dict'])) 22 | return model 23 | -------------------------------------------------------------------------------- /fastiqa/models/nima.py: -------------------------------------------------------------------------------- 1 | # https://github.com/kentsyx/Neural-IMage-Assessment/blob/master/model/model.py 2 | import torch 3 | import torch.nn as nn 4 | import torchvision.models as models 5 | 6 | import numpy as np # use gpu might be better? whatever 7 | 8 | def get_mean_score(score): # this is just for one sample, not for a batch 9 | buckets = np.arange(1, 11) 10 | mu = (buckets * score).sum() 11 | return mu 12 | 13 | 14 | def get_std_score(scores): 15 | si = np.arange(1, 11) 16 | mean = get_mean_score(scores) 17 | std = np.sqrt(np.sum(((si - mean) ** 2) * scores)) 18 | return std 19 | 20 | class NIMA(nn.Module): 21 | 22 | """Neural IMage Assessment model by Google""" 23 | def __init__(self, base_model=None, num_classes=10): 24 | if base_model is None: 25 | base_model = models.vgg16(pretrained=True) 26 | super().__init__() 27 | self.features = base_model.features 28 | self.classifier = nn.Sequential( 29 | nn.Dropout(p=0.75), 30 | nn.Linear(in_features=25088, out_features=num_classes), 31 | nn.Softmax()) 32 | 33 | def forward(self, x): 34 | out = self.features(x) 35 | out = out.view(out.size(0), -1) 36 | out = self.classifier(out) 37 | 38 | return out 39 | # if self.training: 40 | # return out 41 | # else: 42 | # mean_scores = [] 43 | # for prob in out.data.cpu().numpy(): 44 | # mean_scores.append(get_mean_score(prob)) 45 | # # std_score = get_std_score(prob) 46 | # return torch.tensor(mean_scores) 47 | -------------------------------------------------------------------------------- /fastiqa/models/resnet3d/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/fastiqa/models/resnet3d/__init__.py -------------------------------------------------------------------------------- /fastiqa/models/resnet3d/densenet.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | from collections import OrderedDict 7 | 8 | 9 | class _DenseLayer(nn.Sequential): 10 | 11 | def __init__(self, num_input_features, growth_rate, bn_size, drop_rate): 12 | super().__init__() 13 | self.add_module('norm1', nn.BatchNorm3d(num_input_features)) 14 | self.add_module('relu1', nn.ReLU(inplace=True)) 15 | self.add_module( 16 | 'conv1', 17 | nn.Conv3d(num_input_features, 18 | bn_size * growth_rate, 19 | kernel_size=1, 20 | stride=1, 21 | bias=False)) 22 | self.add_module('norm2', nn.BatchNorm3d(bn_size * growth_rate)) 23 | self.add_module('relu2', nn.ReLU(inplace=True)) 24 | self.add_module( 25 | 'conv2', 26 | nn.Conv3d(bn_size * growth_rate, 27 | growth_rate, 28 | kernel_size=3, 29 | stride=1, 30 | padding=1, 31 | bias=False)) 32 | self.drop_rate = drop_rate 33 | 34 | def forward(self, x): 35 | new_features = super().forward(x) 36 | if self.drop_rate > 0: 37 | new_features = F.dropout(new_features, 38 | p=self.drop_rate, 39 | training=self.training) 40 | return torch.cat([x, new_features], 1) 41 | 42 | 43 | class _DenseBlock(nn.Sequential): 44 | 45 | def __init__(self, num_layers, num_input_features, bn_size, growth_rate, 46 | drop_rate): 47 | super().__init__() 48 | for i in range(num_layers): 49 | layer = _DenseLayer(num_input_features + i * growth_rate, 50 | growth_rate, bn_size, drop_rate) 51 | self.add_module('denselayer{}'.format(i + 1), layer) 52 | 53 | 54 | class _Transition(nn.Sequential): 55 | 56 | def __init__(self, num_input_features, num_output_features): 57 | super().__init__() 58 | self.add_module('norm', nn.BatchNorm3d(num_input_features)) 59 | self.add_module('relu', nn.ReLU(inplace=True)) 60 | self.add_module( 61 | 'conv', 62 | nn.Conv3d(num_input_features, 63 | num_output_features, 64 | kernel_size=1, 65 | stride=1, 66 | bias=False)) 67 | self.add_module('pool', nn.AvgPool3d(kernel_size=2, stride=2)) 68 | 69 | 70 | class DenseNet(nn.Module): 71 | """Densenet-BC model class 72 | Args: 73 | growth_rate (int) - how many filters to add each layer (k in paper) 74 | block_config (list of 4 ints) - how many layers in each pooling block 75 | num_init_features (int) - the number of filters to learn in the first convolution layer 76 | bn_size (int) - multiplicative factor for number of bottle neck layers 77 | (i.e. bn_size * k features in the bottleneck layer) 78 | drop_rate (float) - dropout rate after each dense layer 79 | num_classes (int) - number of classification classes 80 | """ 81 | 82 | def __init__(self, 83 | n_input_channels=3, 84 | conv1_t_size=7, 85 | conv1_t_stride=1, 86 | no_max_pool=False, 87 | growth_rate=32, 88 | block_config=(6, 12, 24, 16), 89 | num_init_features=64, 90 | bn_size=4, 91 | drop_rate=0, 92 | num_classes=1000): 93 | 94 | super().__init__() 95 | 96 | # First convolution 97 | self.features = [('conv1', 98 | nn.Conv3d(n_input_channels, 99 | num_init_features, 100 | kernel_size=(conv1_t_size, 7, 7), 101 | stride=(conv1_t_stride, 2, 2), 102 | padding=(conv1_t_size // 2, 3, 3), 103 | bias=False)), 104 | ('norm1', nn.BatchNorm3d(num_init_features)), 105 | ('relu1', nn.ReLU(inplace=True))] 106 | if not no_max_pool: 107 | self.features.append( 108 | ('pool1', nn.MaxPool3d(kernel_size=3, stride=2, padding=1))) 109 | self.features = nn.Sequential(OrderedDict(self.features)) 110 | 111 | # Each denseblock 112 | num_features = num_init_features 113 | for i, num_layers in enumerate(block_config): 114 | block = _DenseBlock(num_layers=num_layers, 115 | num_input_features=num_features, 116 | bn_size=bn_size, 117 | growth_rate=growth_rate, 118 | drop_rate=drop_rate) 119 | self.features.add_module('denseblock{}'.format(i + 1), block) 120 | num_features = num_features + num_layers * growth_rate 121 | if i != len(block_config) - 1: 122 | trans = _Transition(num_input_features=num_features, 123 | num_output_features=num_features // 2) 124 | self.features.add_module('transition{}'.format(i + 1), trans) 125 | num_features = num_features // 2 126 | 127 | # Final batch norm 128 | self.features.add_module('norm5', nn.BatchNorm3d(num_features)) 129 | 130 | for m in self.modules(): 131 | if isinstance(m, nn.Conv3d): 132 | m.weight = nn.init.kaiming_normal(m.weight, mode='fan_out') 133 | elif isinstance(m, nn.BatchNorm3d) or isinstance(m, nn.BatchNorm2d): 134 | m.weight.data.fill_(1) 135 | m.bias.data.zero_() 136 | 137 | # Linear layer 138 | self.classifier = nn.Linear(num_features, num_classes) 139 | 140 | for m in self.modules(): 141 | if isinstance(m, nn.Conv3d): 142 | nn.init.kaiming_normal_(m.weight, 143 | mode='fan_out', 144 | nonlinearity='relu') 145 | elif isinstance(m, nn.BatchNorm3d): 146 | nn.init.constant_(m.weight, 1) 147 | nn.init.constant_(m.bias, 0) 148 | elif isinstance(m, nn.Linear): 149 | nn.init.constant_(m.bias, 0) 150 | 151 | def forward(self, x): 152 | features = self.features(x) 153 | out = F.relu(features, inplace=True) 154 | out = F.adaptive_avg_pool3d(out, 155 | output_size=(1, 1, 156 | 1)).view(features.size(0), -1) 157 | out = self.classifier(out) 158 | return out 159 | 160 | 161 | def generate_model(model_depth, **kwargs): 162 | assert model_depth in [121, 169, 201, 264] 163 | 164 | if model_depth == 121: 165 | model = DenseNet(num_init_features=64, 166 | growth_rate=32, 167 | block_config=(6, 12, 24, 16), 168 | **kwargs) 169 | elif model_depth == 169: 170 | model = DenseNet(num_init_features=64, 171 | growth_rate=32, 172 | block_config=(6, 12, 32, 32), 173 | **kwargs) 174 | elif model_depth == 201: 175 | model = DenseNet(num_init_features=64, 176 | growth_rate=32, 177 | block_config=(6, 12, 48, 32), 178 | **kwargs) 179 | elif model_depth == 264: 180 | model = DenseNet(num_init_features=64, 181 | growth_rate=32, 182 | block_config=(6, 12, 64, 48), 183 | **kwargs) 184 | 185 | return model -------------------------------------------------------------------------------- /fastiqa/models/resnet3d/pre_act_resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from .resnet import conv3x3x3, conv1x1x1, get_inplanes, ResNet 6 | 7 | 8 | class PreActivationBasicBlock(nn.Module): 9 | expansion = 1 10 | 11 | def __init__(self, inplanes, planes, stride=1, downsample=None): 12 | super().__init__() 13 | 14 | self.bn1 = nn.BatchNorm3d(inplanes) 15 | self.conv1 = conv3x3x3(inplanes, planes, stride) 16 | self.bn2 = nn.BatchNorm3d(planes) 17 | self.conv2 = conv3x3x3(planes, planes) 18 | self.relu = nn.ReLU(inplace=True) 19 | self.downsample = downsample 20 | self.stride = stride 21 | 22 | def forward(self, x): 23 | residual = x 24 | 25 | out = self.bn1(x) 26 | out = self.relu(out) 27 | out = self.conv1(out) 28 | 29 | out = self.bn2(out) 30 | out = self.relu(out) 31 | out = self.conv2(out) 32 | 33 | if self.downsample is not None: 34 | residual = self.downsample(x) 35 | 36 | out += residual 37 | 38 | return out 39 | 40 | 41 | class PreActivationBottleneck(nn.Module): 42 | expansion = 4 43 | 44 | def __init__(self, inplanes, planes, stride=1, downsample=None): 45 | super().__init__() 46 | 47 | self.bn1 = nn.BatchNorm3d(inplanes) 48 | self.conv1 = conv1x1x1(inplanes, planes) 49 | self.bn2 = nn.BatchNorm3d(planes) 50 | self.conv2 = conv3x3x3(planes, planes, stride) 51 | self.bn3 = nn.BatchNorm3d(planes) 52 | self.conv3 = conv1x1x1(planes, planes * self.expansion) 53 | self.relu = nn.ReLU(inplace=True) 54 | self.downsample = downsample 55 | self.stride = stride 56 | 57 | def forward(self, x): 58 | residual = x 59 | 60 | out = self.bn1(x) 61 | out = self.relu(out) 62 | out = self.conv1(out) 63 | 64 | out = self.bn2(out) 65 | out = self.relu(out) 66 | out = self.conv2(out) 67 | 68 | out = self.bn3(out) 69 | out = self.relu(out) 70 | out = self.conv3(out) 71 | 72 | if self.downsample is not None: 73 | residual = self.downsample(x) 74 | 75 | out += residual 76 | 77 | return out 78 | 79 | 80 | def generate_model(model_depth, **kwargs): 81 | assert model_depth in [10, 18, 34, 50, 101, 152, 200] 82 | 83 | if model_depth == 10: 84 | model = ResNet(PreActivationBasicBlock, [1, 1, 1, 1], get_inplanes(), 85 | **kwargs) 86 | elif model_depth == 18: 87 | model = ResNet(PreActivationBasicBlock, [2, 2, 2, 2], get_inplanes(), 88 | **kwargs) 89 | elif model_depth == 34: 90 | model = ResNet(PreActivationBasicBlock, [3, 4, 6, 3], get_inplanes(), 91 | **kwargs) 92 | elif model_depth == 50: 93 | model = ResNet(PreActivationBottleneck, [3, 4, 6, 3], get_inplanes(), 94 | **kwargs) 95 | elif model_depth == 101: 96 | model = ResNet(PreActivationBottleneck, [3, 4, 23, 3], get_inplanes(), 97 | **kwargs) 98 | elif model_depth == 152: 99 | model = ResNet(PreActivationBottleneck, [3, 8, 36, 3], get_inplanes(), 100 | **kwargs) 101 | elif model_depth == 200: 102 | model = ResNet(PreActivationBottleneck, [3, 24, 36, 3], get_inplanes(), 103 | **kwargs) 104 | 105 | return model 106 | -------------------------------------------------------------------------------- /fastiqa/models/resnet3d/resnext.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.autograd import Variable 5 | import math 6 | from functools import partial 7 | 8 | __all__ = ['ResNeXt', 'resnext3d50', 'resnext3d101'] 9 | 10 | 11 | def conv3x3x3(in_planes, out_planes, stride=1): 12 | # 3x3x3 convolution with padding 13 | return nn.Conv3d( 14 | in_planes, 15 | out_planes, 16 | kernel_size=3, 17 | stride=stride, 18 | padding=1, 19 | bias=False) 20 | 21 | 22 | def downsample_basic_block(x, planes, stride): 23 | out = F.avg_pool3d(x, kernel_size=1, stride=stride) 24 | zero_pads = torch.Tensor( 25 | out.size(0), planes - out.size(1), out.size(2), out.size(3), 26 | out.size(4)).zero_() 27 | if isinstance(out.data, torch.cuda.FloatTensor): 28 | zero_pads = zero_pads.cuda() 29 | 30 | out = Variable(torch.cat([out.data, zero_pads], dim=1)) 31 | 32 | return out 33 | 34 | 35 | class ResNeXtBottleneck(nn.Module): 36 | expansion = 2 37 | 38 | def __init__(self, inplanes, planes, cardinality, stride=1, 39 | downsample=None): 40 | super(ResNeXtBottleneck, self).__init__() 41 | mid_planes = cardinality * int(planes / 32) 42 | self.conv1 = nn.Conv3d(inplanes, mid_planes, kernel_size=1, bias=False) 43 | self.bn1 = nn.BatchNorm3d(mid_planes) 44 | self.conv2 = nn.Conv3d( 45 | mid_planes, 46 | mid_planes, 47 | kernel_size=3, 48 | stride=stride, 49 | padding=1, 50 | groups=cardinality, 51 | bias=False) 52 | self.bn2 = nn.BatchNorm3d(mid_planes) 53 | self.conv3 = nn.Conv3d( 54 | mid_planes, planes * self.expansion, kernel_size=1, bias=False) 55 | self.bn3 = nn.BatchNorm3d(planes * self.expansion) 56 | self.relu = nn.ReLU(inplace=True) 57 | self.downsample = downsample 58 | self.stride = stride 59 | 60 | def forward(self, x): 61 | residual = x 62 | 63 | out = self.conv1(x) 64 | out = self.bn1(out) 65 | out = self.relu(out) 66 | 67 | out = self.conv2(out) 68 | out = self.bn2(out) 69 | out = self.relu(out) 70 | 71 | out = self.conv3(out) 72 | out = self.bn3(out) 73 | 74 | if self.downsample is not None: 75 | residual = self.downsample(x) 76 | 77 | out += residual 78 | out = self.relu(out) 79 | 80 | return out 81 | 82 | 83 | class ResNeXt(nn.Module): 84 | 85 | def __init__(self, 86 | block, 87 | layers, 88 | sample_size, 89 | sample_duration, 90 | shortcut_type='B', 91 | cardinality=32, 92 | num_classes=400): 93 | self.inplanes = 64 94 | super(ResNeXt, self).__init__() 95 | self.conv1 = nn.Conv3d( 96 | 3, 97 | 64, 98 | kernel_size=7, 99 | stride=(1, 2, 2), 100 | padding=(3, 3, 3), 101 | bias=False) 102 | #self.conv1 = nn.Conv3d( 103 | # 3, 104 | # 64, 105 | # kernel_size=(3,7,7), 106 | # stride=(1, 2, 2), 107 | # padding=(1, 3, 3), 108 | # bias=False) 109 | self.bn1 = nn.BatchNorm3d(64) 110 | self.relu = nn.ReLU(inplace=True) 111 | self.maxpool = nn.MaxPool3d(kernel_size=(3, 3, 3), stride=2, padding=1) 112 | self.layer1 = self._make_layer(block, 128, layers[0], shortcut_type, 113 | cardinality) 114 | self.layer2 = self._make_layer( 115 | block, 256, layers[1], shortcut_type, cardinality, stride=2) 116 | self.layer3 = self._make_layer( 117 | block, 512, layers[2], shortcut_type, cardinality, stride=2) 118 | self.layer4 = self._make_layer( 119 | block, 1024, layers[3], shortcut_type, cardinality, stride=2) 120 | last_duration = int(math.ceil(sample_duration / 16)) 121 | #last_duration = 1 122 | last_size = int(math.ceil(sample_size / 32)) 123 | self.avgpool = nn.AvgPool3d( 124 | (last_duration, last_size, last_size), stride=1) 125 | self.fc = nn.Linear(cardinality * 32 * block.expansion, num_classes) 126 | 127 | for m in self.modules(): 128 | if isinstance(m, nn.Conv3d): 129 | m.weight = nn.init.kaiming_normal(m.weight, mode='fan_out') 130 | elif isinstance(m, nn.BatchNorm3d): 131 | m.weight.data.fill_(1) 132 | m.bias.data.zero_() 133 | 134 | def _make_layer(self, 135 | block, 136 | planes, 137 | blocks, 138 | shortcut_type, 139 | cardinality, 140 | stride=1): 141 | downsample = None 142 | if stride != 1 or self.inplanes != planes * block.expansion: 143 | if shortcut_type == 'A': 144 | downsample = partial( 145 | downsample_basic_block, 146 | planes=planes * block.expansion, 147 | stride=stride) 148 | else: 149 | downsample = nn.Sequential( 150 | nn.Conv3d( 151 | self.inplanes, 152 | planes * block.expansion, 153 | kernel_size=1, 154 | stride=stride, 155 | bias=False), nn.BatchNorm3d(planes * block.expansion)) 156 | 157 | layers = [] 158 | layers.append( 159 | block(self.inplanes, planes, cardinality, stride, downsample)) 160 | self.inplanes = planes * block.expansion 161 | for i in range(1, blocks): 162 | layers.append(block(self.inplanes, planes, cardinality)) 163 | 164 | return nn.Sequential(*layers) 165 | 166 | def forward(self, x): 167 | x = self.conv1(x) 168 | x = self.bn1(x) 169 | x = self.relu(x) 170 | x = self.maxpool(x) 171 | 172 | x = self.layer1(x) 173 | x = self.layer2(x) 174 | x = self.layer3(x) 175 | x = self.layer4(x) 176 | 177 | x = self.avgpool(x) 178 | 179 | x = x.view(x.size(0), -1) 180 | x = self.fc(x) 181 | 182 | return x 183 | 184 | 185 | def get_fine_tuning_parameters(model, ft_portion): 186 | if ft_portion == "complete": 187 | return model.parameters() 188 | 189 | elif ft_portion == "last_layer": 190 | ft_module_names = [] 191 | ft_module_names.append('fc') 192 | 193 | parameters = [] 194 | for k, v in model.named_parameters(): 195 | for ft_module in ft_module_names: 196 | if ft_module in k: 197 | parameters.append({'params': v}) 198 | break 199 | else: 200 | parameters.append({'params': v, 'lr': 0.0}) 201 | return parameters 202 | 203 | else: 204 | raise ValueError("Unsupported ft_portion: 'complete' or 'last_layer' expected") 205 | 206 | 207 | def resnext3d(layers, pretrained=False, sample_size = 112, sample_duration = 8, num_classes=600, **kwargs): 208 | return ResNeXt(ResNeXtBottleneck, layers, 209 | sample_size=sample_size, sample_duration=sample_duration, num_classes=num_classes, **kwargs) 210 | 211 | 212 | 213 | def resnext3d50(**kwargs): 214 | return resnext3d([3, 4, 6, 3], **kwargs) 215 | 216 | 217 | def resnext3d101(pretrained=False, **kwargs): 218 | model = resnext3d([3, 4, 23, 3], **kwargs) 219 | if pretrained: 220 | model.load_state_dict(torch.load('fastai-kinetics_resnext_101_RGB_16_best.pth')) 221 | # model.load_state_dict(torch.load('kinetics_resnext_101_RGB_16_best.pth')['state_dict']) 222 | #model.load_state_dict(torch.load('resnext-101-64f-kinetics.pth')['state_dict']) 223 | 224 | return model 225 | 226 | def resnext3d152(**kwargs): 227 | return resnext3d([3, 8, 36, 3], **kwargs) 228 | -------------------------------------------------------------------------------- /fastiqa/models/resnet3d/utils.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import random 3 | from functools import partialmethod 4 | 5 | import torch 6 | import numpy as np 7 | from sklearn.metrics import precision_recall_fscore_support 8 | 9 | 10 | class AverageMeter(object): 11 | """Computes and stores the average and current value""" 12 | 13 | def __init__(self): 14 | self.reset() 15 | 16 | def reset(self): 17 | self.val = 0 18 | self.avg = 0 19 | self.sum = 0 20 | self.count = 0 21 | 22 | def update(self, val, n=1): 23 | self.val = val 24 | self.sum += val * n 25 | self.count += n 26 | self.avg = self.sum / self.count 27 | 28 | 29 | class Logger(object): 30 | 31 | def __init__(self, path, header): 32 | self.log_file = path.open('w') 33 | self.logger = csv.writer(self.log_file, delimiter='\t') 34 | 35 | self.logger.writerow(header) 36 | self.header = header 37 | 38 | def __del(self): 39 | self.log_file.close() 40 | 41 | def log(self, values): 42 | write_values = [] 43 | for col in self.header: 44 | assert col in values 45 | write_values.append(values[col]) 46 | 47 | self.logger.writerow(write_values) 48 | self.log_file.flush() 49 | 50 | 51 | def calculate_accuracy(outputs, targets): 52 | with torch.no_grad(): 53 | batch_size = targets.size(0) 54 | 55 | _, pred = outputs.topk(1, 1, largest=True, sorted=True) 56 | pred = pred.t() 57 | correct = pred.eq(targets.view(1, -1)) 58 | n_correct_elems = correct.float().sum().item() 59 | 60 | return n_correct_elems / batch_size 61 | 62 | 63 | def calculate_precision_and_recall(outputs, targets, pos_label=1): 64 | with torch.no_grad(): 65 | _, pred = outputs.topk(1, 1, largest=True, sorted=True) 66 | precision, recall, _, _ = precision_recall_fscore_support( 67 | targets.view(-1, 1).cpu().numpy(), 68 | pred.cpu().numpy()) 69 | 70 | return precision[pos_label], recall[pos_label] 71 | 72 | 73 | def worker_init_fn(worker_id): 74 | torch_seed = torch.initial_seed() 75 | 76 | random.seed(torch_seed + worker_id) 77 | 78 | if torch_seed >= 2**32: 79 | torch_seed = torch_seed % 2**32 80 | np.random.seed(torch_seed + worker_id) 81 | 82 | 83 | def get_lr(optimizer): 84 | lrs = [] 85 | for param_group in optimizer.param_groups: 86 | lr = float(param_group['lr']) 87 | lrs.append(lr) 88 | 89 | return max(lrs) 90 | 91 | 92 | def partialclass(cls, *args, **kwargs): 93 | 94 | class PartialClass(cls): 95 | __init__ = partialmethod(cls.__init__, *args, **kwargs) 96 | 97 | return PartialClass -------------------------------------------------------------------------------- /fastiqa/models/resnet3d/wide_resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from . import resnet 6 | 7 | 8 | class WideBottleneck(resnet.Bottleneck): 9 | expansion = 2 10 | 11 | 12 | def generate_model(model_depth, k, **kwargs): 13 | assert model_depth in [50, 101, 152, 200] 14 | 15 | inplanes = [x * k for x in resnet.get_inplanes()] 16 | if model_depth == 50: 17 | model = resnet.ResNet(WideBottleneck, [3, 4, 6, 3], inplanes, **kwargs) 18 | elif model_depth == 101: 19 | model = resnet.ResNet(WideBottleneck, [3, 4, 23, 3], inplanes, **kwargs) 20 | elif model_depth == 152: 21 | model = resnet.ResNet(WideBottleneck, [3, 8, 36, 3], inplanes, **kwargs) 22 | elif model_depth == 200: 23 | model = resnet.ResNet(WideBottleneck, [3, 24, 36, 3], inplanes, 24 | **kwargs) 25 | 26 | return model 27 | -------------------------------------------------------------------------------- /fastiqa/models/resnet_3d.py: -------------------------------------------------------------------------------- 1 | from .resnet3d import resnet, resnet2p1d, resnext 2 | import torch 3 | # from fastiqa.models import resnet 4 | 5 | DIR = 'pth/' 6 | 7 | def r3d18_K_200ep(pretrained=False, **kwargs): 8 | model = resnet.generate_model(model_depth=18, n_classes=700, **kwargs) 9 | PATH = DIR + 'fastai-r3d18_K_200ep.pth' 10 | # PATH = '/media/zq/DB/code/fastiqa2_dev/r2p1d18_K_200ep.pth' 11 | if pretrained: 12 | model.load_state_dict(torch.load(PATH)) 13 | return model 14 | 15 | # K 700 16 | # KM 1039 17 | # KMS 1139 18 | def r3d50_KM_200ep(pretrained=False, **kwargs): 19 | model = resnet.generate_model(model_depth=50, n_classes=1039, **kwargs) 20 | PATH = DIR + 'fastai-r3d50_KM_200ep.pth' 21 | if pretrained: 22 | model.load_state_dict(torch.load(PATH)) 23 | return model 24 | 25 | 26 | def r2p1d50_K_200ep(pretrained=False, **kwargs): 27 | model = resnet2p1d.generate_model(model_depth=50, n_classes=700, **kwargs) 28 | PATH = DIR + 'fastai-r2p1d50_K_200ep.pth' 29 | if pretrained: 30 | model.load_state_dict(torch.load(PATH)) 31 | return model 32 | 33 | 34 | def r2p1d18_K_200ep(pretrained=False, **kwargs): 35 | model = resnet2p1d.generate_model(model_depth=18, n_classes=700, **kwargs) 36 | PATH = DIR + 'fastai-r2p1d18_K_200ep.pth' 37 | if pretrained: 38 | model.load_state_dict(torch.load(PATH)) 39 | return model 40 | 41 | 42 | # def resnext3d(pretrained=False, depth=50, **kwargs): 43 | # model = resnext.generate_model(model_depth=depth, n_classes=700, **kwargs) 44 | # #PATH = '/media/zq/DB/code/fastiqa2/fastai-r2p1d18_K_200ep.pth' 45 | # if pretrained: 46 | # pass 47 | # #model.load_state_dict(torch.load(PATH)) 48 | # return model 49 | # 50 | # def resnext3d18(pretrained=False, **kwargs): 51 | # return resnext3d(pretrained, depth=18) 52 | # 53 | # def resnext3d50(pretrained=False, **kwargs): 54 | # return resnext3d(pretrained, depth=50) 55 | -------------------------------------------------------------------------------- /fastiqa/models/tfm_pth.py: -------------------------------------------------------------------------------- 1 | """convert pretrained models""" 2 | 3 | def key_transformation(old_key): 4 | # print(old_key) 5 | if '.' in old_key: 6 | a, b = old_key.split('.', 1) 7 | if a == "cnn": 8 | return f"body.{b}" 9 | return old_key 10 | # body.0.weight", "cnn.1.weight 11 | 12 | rename_state_dict_keys(e['NoRoIPoolModel'].path/'models'/'bestmodel.pth', key_transformation) 13 | -------------------------------------------------------------------------------- /fastiqa/models/total_params.py: -------------------------------------------------------------------------------- 1 | from fastai.vision.all import * 2 | 3 | def abbreviate(x): 4 | abbreviations = ["", "K", "M", "B", "T", "Qd", "Qn", "Sx", "Sp", "O", "N", 5 | "De", "Ud", "DD"] 6 | thing = "1" 7 | a = 0 8 | while len(thing) < len(str(x)) - 3: 9 | thing += "000" 10 | a += 1 11 | b = int(thing) 12 | thing = round(x / b, 2) 13 | return str(thing) + " " + abbreviations[a] 14 | 15 | def total_params(self): 16 | total_params = total_trainable_params = 0 17 | info = layers_info(self) 18 | for layer, size, params, trainable in info: 19 | if size is None: continue 20 | total_params += int(params) 21 | total_trainable_params += int(params) * trainable 22 | return {'Total params': total_params, 23 | 'Total trainable params': total_trainable_params, 24 | 'Total non-trainable params': total_params - total_trainable_params, 25 | 'Summary': f'{abbreviate(total_trainable_params)} / {abbreviate(total_params)}' 26 | } 27 | 28 | Learner.total_params = total_params 29 | -------------------------------------------------------------------------------- /fastiqa/models/vsfa.py: -------------------------------------------------------------------------------- 1 | 2 | import torch.nn as nn 3 | 4 | # https://github.com/lidq92/VSFA/blob/master/VSFA.py 5 | class ANN(nn.Module): 6 | def __init__(self, input_size=4096, reduced_size=128, n_ANNlayers=1, dropout_p=0.5): 7 | super(ANN, self).__init__() 8 | self.n_ANNlayers = n_ANNlayers 9 | self.fc0 = nn.Linear(input_size, reduced_size) # 10 | self.dropout = nn.Dropout(p=dropout_p) # 11 | self.fc = nn.Linear(reduced_size, reduced_size) # 12 | 13 | def forward(self, input): 14 | input = self.fc0(input) # linear 15 | for i in range(self.n_ANNlayers-1): # nonlinear 16 | input = self.fc(self.dropout(F.relu(input))) 17 | return input 18 | 19 | 20 | def TP(q, tau=12, beta=0.5): 21 | """subjectively-inspired temporal pooling""" 22 | q = torch.unsqueeze(torch.t(q), 0) 23 | qm = -float('inf')*torch.ones((1, 1, tau-1)).to(q.device) 24 | qp = 10000.0 * torch.ones((1, 1, tau - 1)).to(q.device) # 25 | l = -F.max_pool1d(torch.cat((qm, -q), 2), tau, stride=1) 26 | m = F.avg_pool1d(torch.cat((q * torch.exp(-q), qp * torch.exp(-qp)), 2), tau, stride=1) 27 | n = F.avg_pool1d(torch.cat((torch.exp(-q), torch.exp(-qp)), 2), tau, stride=1) 28 | m = m / n 29 | return beta * m + (1 - beta) * l 30 | 31 | 32 | class VSFA(nn.Module): 33 | def __init__(self, input_size=4096, reduced_size=128, hidden_size=32): 34 | 35 | super(VSFA, self).__init__() 36 | self.hidden_size = hidden_size 37 | self.ann = ANN(input_size, reduced_size, 1) 38 | self.rnn = nn.GRU(reduced_size, hidden_size, batch_first=True) 39 | self.q = nn.Linear(hidden_size, 1) 40 | 41 | def forward(self, input, input_length): 42 | input = self.ann(input) # dimension reduction 43 | outputs, _ = self.rnn(input, self._get_initial_state(input.size(0), input.device)) 44 | q = self.q(outputs) # frame quality 45 | score = torch.zeros_like(input_length, device=q.device) # 46 | for i in range(input_length.shape[0]): # 47 | qi = q[i, :np.int(input_length[i].numpy())] 48 | qi = TP(qi) 49 | score[i] = torch.mean(qi) # video overall quality 50 | return score 51 | 52 | def _get_initial_state(self, batch_size, device): 53 | h0 = torch.zeros(1, batch_size, self.hidden_size, device=device) 54 | return h0 55 | -------------------------------------------------------------------------------- /fastiqa/qmap.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from PIL import Image 4 | from .learn import IqaLearner 5 | 6 | class QualityMap: 7 | """Note here the image is fastai image""" 8 | def __init__(self, mat, img, global_score=0): 9 | self.mat, self.img = mat, img 10 | self.global_score = global_score 11 | 12 | # fastai1 13 | # from fastai.vision import image2np 14 | # from PIL import Image as PIL_Image 15 | # @property 16 | # def pil_image(self): 17 | # return PIL_Image.fromarray((255 * image2np(self.img.px)).astype(np.uint8)) 18 | 19 | # @property 20 | def pil_image(self): 21 | # PIL.Image.frombytes 22 | # PIL.Image.new 23 | return PILImage.create((255 * (self.img.px)).astype(np.uint8)) 24 | 25 | def plot(self, vmin=0, vmax=100): 26 | fig, axes = plt.subplots(1, 3, figsize=(12, 8 * 3)) 27 | 28 | # fastai image 29 | self.img.show(axes[0], title='Input image') # title mos 30 | self.img.show(axes[1], title=f'Predicted: {self.global_score:.2f}') # title prediction 31 | 32 | _, H, W = self.img.shape 33 | h, w = self.mat.shape # self.mat.size() 34 | extent = (0, W // w * w, H // h * h, 0) 35 | 36 | axes[1].imshow(self.mat, alpha=0.8, cmap='magma', 37 | extent=extent, interpolation='bilinear') 38 | axes[2].imshow(self.mat, cmap='gray', extent=extent, 39 | vmin=vmin, vmax=vmax) 40 | axes[2].set_title(f'Quality map {h}x{w}') 41 | 42 | def _repr_html_(self): 43 | self.plot() 44 | return '' 45 | 46 | def savefig(self, filename): 47 | plt.savefig(filename, bbox_inches='tight') 48 | 49 | def blend(self, mos_range=(0, 100), alpha=0.8, resample=Image.BILINEAR): 50 | """qmap.blend().save('qmap.jpg')""" 51 | 52 | def stretch(image, minimum, maximum): 53 | if maximum is None: 54 | maximum = image.max() 55 | if minimum is None: 56 | minimum = image.min() 57 | image = (image - minimum) / (maximum - minimum) 58 | image[image < 0] = 0 59 | image[image > 1] = 1 60 | return image 61 | 62 | cm = plt.get_cmap('magma') 63 | # min-max normalize the image, you can skip this step 64 | qmap_matrix = self.mat 65 | if mos_range is not None: 66 | qmap_matrix = 100*stretch(np.array(qmap_matrix), mos_range[0], mos_range[1]) 67 | qmap_matrix = (np.array(qmap_matrix) * 255 / 100).astype(np.uint8) 68 | colored_map = cm(qmap_matrix) 69 | # Obtain a 4-channel image (R,G,B,A) in float [0, 1] 70 | # But we want to convert to RGB in uint8 and save it: 71 | heatmap = PIL_Image.fromarray((colored_map[:, :, :3] * 255).astype(np.uint8)) 72 | sz = self.img.shape[-1], self.img.shape[-2] 73 | heatmap = heatmap.resize(sz, resample=resample) 74 | 75 | return PIL_Image.blend(self.pil_image, heatmap, alpha=alpha) 76 | 77 | 78 | def predict_quality_map_without_roi_pool(self, img, blk_size=None): 79 | if blk_size is None: 80 | blk_size = [4, 4] # [32, 32] # very poor 81 | 82 | batch_x = img2patches(img.data, blk_size).cuda() 83 | batch_y = torch.zeros(1, blk_size[0] * blk_size[1]).cuda() 84 | predicts = self.pred_batch(batch=[batch_x, batch_y]) 85 | # img.shape 3 x H x W 86 | h = blk_size[0] # + int(img.shape[1] % blk_size[0] != 0) 87 | w = blk_size[1] # + int(img.shape[2] % blk_size[1] != 0) 88 | pred = predicts.reshape(h, w) 89 | return QualityMap(pred, img) 90 | 91 | def get_quality_maps(self): 92 | # TODO predict in batch (according to data.batch_size) 93 | # get_qmaps 94 | """ 95 | save all quality maps to quality maps folder 96 | ======================================= 97 | im = open_image('/media/zq/Seagate/data/TestImages/Noisy Lady.png') # 98 | learn = e['Pool1RoIModel'] 99 | learn.data = TestImages() 100 | t, a = learn.predict_quality_map(im, [[8,8]]) 101 | print(a) 102 | print(t[0].data[0]) # image score 103 | # im 104 | plt.savefig('foo.png', bbox_inches='tight') 105 | :return: 106 | """ 107 | # cannot do it in parallel (GPU) 108 | # only generate on valid set 109 | # must be data, inner_df only contain train samples and it's shuffled 110 | # must be loaded, since we need to call data.one_item 111 | 112 | # use learn.data.valid_ds with df.fn_col, no need 113 | # this will not load the database 114 | # only load the dataframe 115 | df = self.dls.df[self.dls.df.is_valid] #.reset_index() 116 | #for file in df[self.dls.fn_col] 117 | # img = open_image(self.dls.path/self.dls.folder/file) 118 | 119 | # _, ax = plt.subplots() 120 | fig, (ax1, ax2) = plt.subplots(2) 121 | names = df[self.dls.fn_col].tolist() 122 | for n in range(len(self.dls.valid_ds)): # add progress bar 123 | sample = self.dls.valid_ds[n][0] 124 | qmap = self.predict_quality_map(sample) 125 | qmap.show(ax=ax2) 126 | ax1.imshow(sample) 127 | dir = self.dls.path/'quality_maps'/self.dls.folder 128 | filename = str(dir/names[n]).split('.')[-2] + '.jpg' 129 | dir = filename.rsplit('/', 1)[0] # 'data/CLIVE/quality_maps/Images/trainingImages/t1.jpg' 130 | if not os.path.exists(dir): 131 | os.makedirs(dir) 132 | plt.savefig(filename, bbox_inches='tight') 133 | 134 | def predict_quality_map(self, sample, blk_size=None): 135 | """ 136 | :param sample: fastai.vision.image.Image open_image() 137 | :param blk_size: 138 | :return: 139 | if you simply wanna quality map matrix: 140 | sample = open_image('img.jpg') 141 | self.model.input_block_rois(blk_size, [sample.shape[-2], sample.shape[-1]]) 142 | t = self.predict(sample) 143 | """ 144 | 145 | if blk_size is None: 146 | blk_size = [[32, 32]] #[[8, 8]] 147 | 148 | if not isinstance(blk_size[0], list): 149 | blk_size = [blk_size] 150 | # input_block_rois(self, blk_size=None, img_size=[1, 1], batch_size=1, include_image=True): 151 | # [8, 8] # [5, 5] 152 | cuda = self.dls.device.type == 'cuda' 153 | self.model.input_block_rois(blk_size, [sample.shape[-2], sample.shape[-1]], cuda=cuda) # self.dls.img_raw_size 154 | # [768, 1024] [8, 8] is too big learn.data.batch_size 155 | 156 | # predict will first convert image to a batch according to learn.data 157 | # TODO backup self.dls 158 | # this is very inefficient 159 | # data = self.dls 160 | # self.dls = TestImages() 161 | # if type(sample) == fastai.vision.core.PILImage: 162 | # sample = image2tensor(sample) 163 | t = self.predict(sample) 164 | # self.dls = data 165 | 166 | a = t[0].data[1:].reshape(blk_size[0]) # TODO only allow one blk size 167 | 168 | # convert sample to PIL image 169 | # image = PIL_Image.fromarray((255 * image2np(sample.px)).astype(np.uint8)) 170 | return QualityMap(a, sample, t[0].data[0]) 171 | 172 | 173 | IqaLearner.predict_quality_map_without_roi_pool = predict_quality_map_without_roi_pool 174 | IqaLearner.predict_quality_map = predict_quality_map 175 | IqaLearner.get_quality_maps = get_quality_maps 176 | -------------------------------------------------------------------------------- /fastiqa/utils/cached_property.py: -------------------------------------------------------------------------------- 1 | try: 2 | from cached_property import cached_property 3 | except ImportError: 4 | class cached_property(object): 5 | '''Computes attribute value and caches it in the instance. 6 | Python Cookbook (Denis Otkidach) https://stackoverflow.com/users/168352/denis-otkidach 7 | This decorator allows you to create a property which can be computed once and 8 | accessed many times. Sort of like memoization. 9 | ''' 10 | 11 | def __init__(self, method, name=None): 12 | # record the unbound-method and the name 13 | self.method = method 14 | self.name = name or method.__name__ 15 | self.__doc__ = method.__doc__ 16 | 17 | def __get__(self, inst, cls): 18 | # self: <__main__.cache object at 0xb781340c> 19 | # inst: <__main__.Foo object at 0xb781348c> 20 | # cls: 21 | if inst is None: 22 | # instance attribute accessed on class, return self 23 | # You get here if you write `Foo.bar` 24 | return self 25 | # compute, cache and return the instance's attribute value 26 | result = self.method(inst) 27 | # setattr redefines the instance's attribute so this doesn't get called again 28 | setattr(inst, self.name, result) 29 | return result 30 | 31 | -------------------------------------------------------------------------------- /fastiqa/vqa.py: -------------------------------------------------------------------------------- 1 | from fastai.vision.all import * 2 | from fastai.distributed import * 3 | 4 | from fastiqa.bunches.vqa.vid2mos import * 5 | from fastiqa.bunches.vqa.vid_sp2mos import * 6 | from fastiqa.bunches.vqa.single_vid2mos import * 7 | from fastiqa.bunches.feat2mos import * 8 | from fastiqa.bunches.vqa.test_videos import * 9 | 10 | from fastiqa.models._body_head import * 11 | from fastiqa.models.resnet_3d import * 12 | from fastiqa.models.inception_head import * 13 | from fastiqa.models._roi_pool import * 14 | 15 | from fastiqa.learn import * 16 | from fastiqa.iqa_exp import * 17 | -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/favicon.ico -------------------------------------------------------------------------------- /file_upload.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
5 | 6 |
7 | 8 |
9 |
10 |
11 | 12 | Click ► 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
22 |
23 | 27 | 30 |
31 | 32 | 35 | 125 | -------------------------------------------------------------------------------- /filepond/filepond_example.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 13 | -------------------------------------------------------------------------------- /font/roboto/Roboto-Bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Bold.eot -------------------------------------------------------------------------------- /font/roboto/Roboto-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Bold.ttf -------------------------------------------------------------------------------- /font/roboto/Roboto-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Bold.woff -------------------------------------------------------------------------------- /font/roboto/Roboto-Bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Bold.woff2 -------------------------------------------------------------------------------- /font/roboto/Roboto-Light.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Light.eot -------------------------------------------------------------------------------- /font/roboto/Roboto-Light.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Light.ttf -------------------------------------------------------------------------------- /font/roboto/Roboto-Light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Light.woff -------------------------------------------------------------------------------- /font/roboto/Roboto-Light.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Light.woff2 -------------------------------------------------------------------------------- /font/roboto/Roboto-Medium.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Medium.eot -------------------------------------------------------------------------------- /font/roboto/Roboto-Medium.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Medium.ttf -------------------------------------------------------------------------------- /font/roboto/Roboto-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Medium.woff -------------------------------------------------------------------------------- /font/roboto/Roboto-Medium.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Medium.woff2 -------------------------------------------------------------------------------- /font/roboto/Roboto-Regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Regular.eot -------------------------------------------------------------------------------- /font/roboto/Roboto-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Regular.ttf -------------------------------------------------------------------------------- /font/roboto/Roboto-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Regular.woff -------------------------------------------------------------------------------- /font/roboto/Roboto-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Regular.woff2 -------------------------------------------------------------------------------- /font/roboto/Roboto-Thin.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Thin.eot -------------------------------------------------------------------------------- /font/roboto/Roboto-Thin.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Thin.ttf -------------------------------------------------------------------------------- /font/roboto/Roboto-Thin.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Thin.woff -------------------------------------------------------------------------------- /font/roboto/Roboto-Thin.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/font/roboto/Roboto-Thin.woff2 -------------------------------------------------------------------------------- /foot.htm: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

LINKS

6 | 7 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 22 |
23 |
24 |
25 |
26 | 27 |
28 |
29 | 30 |
31 |
32 |
33 | 34 | 35 | 36 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /head.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | {{abbr}} 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 38 | 39 | 57 | 58 | 59 |
60 | 61 | 82 | 83 |
84 |
85 |
86 |

{{title}}

87 |

88 |

89 |
90 | {{authors}} 91 |
92 |
93 | {{address}} 94 | 97 | 98 |
99 | *+Equal contribution 100 |
101 | {{icon}} 102 |
103 | 104 | 105 | 106 |
107 | 108 | 109 | 115 | 116 |
117 |

118 |
119 |
120 | 121 | 122 | 141 | -------------------------------------------------------------------------------- /hist.htm: -------------------------------------------------------------------------------- 1 | 61 | 62 |
63 | 64 | 71 |
72 | -------------------------------------------------------------------------------- /images/A021.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/A021.gif -------------------------------------------------------------------------------- /images/A037.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/A037.gif -------------------------------------------------------------------------------- /images/Noisy Lady.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/Noisy Lady.png -------------------------------------------------------------------------------- /images/Picture1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/Picture1.jpg -------------------------------------------------------------------------------- /images/Picture2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/Picture2.png -------------------------------------------------------------------------------- /images/Picture3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/Picture3.jpg -------------------------------------------------------------------------------- /images/demo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/demo.jpg -------------------------------------------------------------------------------- /images/ex1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/ex1.gif -------------------------------------------------------------------------------- /images/ex2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/ex2.gif -------------------------------------------------------------------------------- /images/pvq_db.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/images/pvq_db.png -------------------------------------------------------------------------------- /imgs/CVPR_Logo_Horz2_web.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/imgs/CVPR_Logo_Horz2_web.jpg -------------------------------------------------------------------------------- /js/include_once.js: -------------------------------------------------------------------------------- 1 | if ($(document).attr('title') == "") { // no head 2 | if (window._head_loaded === undefined) { 3 | window._head_loaded = true; 4 | $(function(){ 5 | document.body.innerHTML = "" + document.body.innerHTML + ""; 6 | $("#header").load("head.htm"); 7 | $("#footer").load("foot.htm"); 8 | }); 9 | // the rest of your javascript 10 | } 11 | } 12 | 13 | if (window.isMobile == undefined) { 14 | //global vars 15 | var isMobile = { 16 | Android: function() { 17 | return navigator.userAgent.match(/Android/i); 18 | }, 19 | BlackBerry: function() { 20 | return navigator.userAgent.match(/BlackBerry/i); 21 | }, 22 | iOS: function() { 23 | return navigator.userAgent.match(/iPhone|iPad|iPod/i); 24 | }, 25 | Opera: function() { 26 | return navigator.userAgent.match(/Opera Mini/i); 27 | }, 28 | Windows: function() { 29 | return navigator.userAgent.match(/IEMobile/i) || navigator.userAgent.match(/WPDesktop/i); 30 | }, 31 | any: function() { 32 | return (isMobile.Android() || isMobile.BlackBerry() || isMobile.iOS() || isMobile.Opera() || isMobile.Windows()); 33 | } 34 | }; 35 | } 36 | -------------------------------------------------------------------------------- /json/AVA.json: -------------------------------------------------------------------------------- 1 | { 2 | "__name__": "AVA", 3 | "dir": "/media/zq/DB/db/AVA/", 4 | "csv_labels": "train_val_labels.csv", 5 | "csv_labels_test_set": "test_labels.csv", 6 | "folder": "images", 7 | "fn_col": "ImageID", 8 | "fn_suffix": ".jpg", 9 | "pad": 800, 10 | "label_col": "mu", 11 | "label_col_p": ["p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10"], 12 | "label_range": [1, 10] 13 | } 14 | -------------------------------------------------------------------------------- /json/CLIVE.json: -------------------------------------------------------------------------------- 1 | { 2 | "__name__": "CLIVE", 3 | "dir": "/media/zq/DB/db/CLIVE/", 4 | "csv_labels": "labels.csv", 5 | "folder": "Images", 6 | "height": 500, 7 | "width": 500, 8 | "fn_col": "name", 9 | "label_col": "mos" 10 | } 11 | -------------------------------------------------------------------------------- /json/KoNViD.json: -------------------------------------------------------------------------------- 1 | { 2 | "__name__": "KoNViD", 3 | "dir": "/media/zq/DB/db/KoNViD/", 4 | "csv_labels": "labels.csv", 5 | "folder": "jpg", 6 | "fn_col": "name", 7 | "label_col": "mos", 8 | "width": "960", 9 | "height": "540", 10 | "raw_resolution": [540, 960] 11 | } 12 | -------------------------------------------------------------------------------- /json/KonIQ.json: -------------------------------------------------------------------------------- 1 | { 2 | "__name__": "KonIQ", 3 | "dir": "/media/zq/DB/db/KonIQ/", 4 | "csv_labels": "labels.csv", 5 | "folder": "1024x768", 6 | "height": 768, 7 | "width": 1024, 8 | "fn_col": "image_name", 9 | "label_col": "MOS_zscore" 10 | } 11 | -------------------------------------------------------------------------------- /json/LIVE_FB_IQA.json: -------------------------------------------------------------------------------- 1 | { 2 | "__name__": "LIVE_FB_IQA", 3 | "dir": "/media/zq/DB/db/LIVE_FB_IQA/", 4 | "csv_labels": "labels<=640_padded.csv", 5 | "csv_labels_test_set": "labels>640.csv", 6 | "test_bs": 1, 7 | "fn_col": "name_image", 8 | "label_col": ["mos_image", "mos_patch_1", "mos_patch_2", "mos_patch_3"], 9 | "roi_col": ["left_image", "top_image", "right_image", "bottom_image", 10 | "left_patch_1", "top_patch_1", "right_patch_1", "bottom_patch_1", 11 | "left_patch_2", "top_patch_2", "right_patch_2", "bottom_patch_2", 12 | "left_patch_3", "top_patch_3", "right_patch_3", "bottom_patch_3"], 13 | "folder": "images", 14 | "raw_resolution": ["None", "None"], 15 | "suff": "" 16 | } 17 | -------------------------------------------------------------------------------- /json/LIVE_VQC.json: -------------------------------------------------------------------------------- 1 | { 2 | "__name__": "LIVE_VQC", 3 | "dir": "/media/zq/DB/db/LIVE_VQC/", 4 | "csv_labels": "labels.csv", 5 | "fn_col": "name", 6 | "label_col": "mos" 7 | } 8 | -------------------------------------------------------------------------------- /json/LSVQ.json: -------------------------------------------------------------------------------- 1 | { 2 | "__name__": "LSVQ", 3 | "dir": "/media/zq/DB/db/LSVQ/", 4 | "csv_labels": "labels_train_test.csv", 5 | "csv_labels_test_set": "labels_test_1080p.csv", 6 | "folder": "jpg", 7 | "fn_col": "name", 8 | "label_col": ["mos", "mos_p1", "mos_p2", "mos_p3"], 9 | "roi_col": [ ["left_vid", "top_vid", "right_vid", "bottom_vid"], 10 | ["left_p1", "top_p1", "right_p1", "bottom_p1"], 11 | ["left_p2", "top_p2", "right_p2", "bottom_p2"], 12 | ["left_p3", "top_p3", "right_p3", "bottom_p3"]] 13 | } 14 | -------------------------------------------------------------------------------- /license-free.txt: -------------------------------------------------------------------------------- 1 | Material App 2 | A Material Design App Landing page template built with MDB by TemplateFlip.com 3 | 4 | v1.0.0 5 | Released: Jun 26 2018 6 | Updated: Jun 26 2018 7 | 8 | Demo: https://templateflip.com/demo/?template=material-app 9 | Download: https://templateflip.com/templates/material-app 10 | 11 | License Terms 12 | 13 | ************* 14 | 15 | Licensed under the Creative Commons Attribution 3.0 license. 16 | You can use this template for free in personal projects. 17 | In return you need to keep the credit to https://templateflip.com from the site footer. 18 | 19 | You can buy a premium license to be allowed to remove the footer link and for use in commerical projects. 20 | Buy License - https://templateflip.com/templates/digital-agency/ 21 | 22 | Prohibitions 23 | 24 | ************* 25 | 26 | You do not have the rights to sub-license, sell or distribute any of TemplateFlip item. 27 | If you wish to promote our resources on your site, you must link back to our resource page, 28 | where users can find the download and not directly to the download file. 29 | You must not use the themes and templates from TemplateFlip for displaying unlawful content. 30 | For any further queries regarding terms of use and licensing, feel free to contact us. 31 | 32 | ************* 33 | 34 | Check out our Website templates at https://templateflip.com/templates/ 35 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | """generate html website code""" 2 | # python main.py pvq.ini PatchVQ 3 | 4 | 5 | # %% 6 | import sys 7 | from jinja2 import Environment, BaseLoader, FileSystemLoader, Template 8 | import pyperclip 9 | import random 10 | import pandas as pd 11 | import configparser 12 | import webbrowser 13 | import os 14 | 15 | def main(ini_file, target='DEFAULT'): 16 | c = configparser.ConfigParser() 17 | c.read(ini_file) 18 | # c.read('amt.ini') 19 | cfg = dict(c[target].items()) 20 | 21 | # cfg that ends with 's' (multiple items) 22 | # for x in ['phases', 'include_pages', 'addons','train_vids', 'label_vids', 'sample_vids', 'gold_vids', 'backup_vids']: 23 | # if cfg[x].strip() != '': 24 | # cfg[x] = cfg[x].strip().split('\n') 25 | # else: 26 | # cfg[x] = [] 27 | 28 | for k in cfg.keys(): 29 | cfg[k] = cfg[k].replace('"', '"') 30 | 31 | # with open("./template.htm") as f: template_str = f.read() 32 | # template = Environment(loader=FileSystemLoader("./")).from_string(template_str) 33 | template = Environment(loader=FileSystemLoader("./")).from_string(""" 34 | {% for sec in sections.split(',') %} 35 | {% include sec.strip()+'.htm' %} 36 | {% endfor %} 37 | """) 38 | rendered = template.render(**cfg) 39 | 40 | if cfg['out_html'] == 'clipboard': 41 | pyperclip.copy(rendered) 42 | else: 43 | dir = 'file://' + os.path.dirname(os.path.realpath(__file__)) 44 | html_file = cfg['out_html'] if os.path.isabs(cfg['out_html']) else dir + os.path.sep + cfg['out_html'] 45 | with open(cfg['out_html'], "w") as f: 46 | f.write(rendered) 47 | 48 | 49 | """ 50 | # %% 51 | !pip install pyperclip 52 | !python main.py video debugNoBackup 53 | !python main.py video jerry 54 | # %% 55 | """ 56 | # %% 57 | if __name__ == "__main__": 58 | main(ini_file= sys.argv[1], target=sys.argv[2]) 59 | -------------------------------------------------------------------------------- /paq2piq/abstract.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 33 |
34 |
35 |
36 |

ABSTRACT

37 |

38 | 39 |

40 | {{abstract}} 41 |
42 |
43 |
44 |
45 | 46 | 47 |
48 |
49 | 50 |
51 |
52 | 53 |
54 |
55 | 56 | 61 | 62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 | 70 |
71 |
72 |
73 |
74 |
75 | 76 |
77 |
78 | 79 | 80 | 81 |
82 |
83 | 84 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 |
95 | 96 |
97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 |
105 |
106 |
107 | -------------------------------------------------------------------------------- /paq2piq/applications.htm: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

APPLICATIONS

5 |

6 | Apply on videos in a frame-by-frame manner 7 |

8 | 9 | 10 | 11 | 124 | 125 |
126 |
127 | -------------------------------------------------------------------------------- /paq2piq/download_zone.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
5 |
6 |
7 |

DOWNLOAD

8 |

9 | Please submit an issue here if you encounter any difficulties. 10 |

11 |
12 |
13 |
14 |
15 |
16 |
17 |
Database
18 | 19 |
20 |
    21 |
  • 22 |

    4 M ratings

    23 |
    • 24 |
  • 25 |

    8 k subjects

    26 |
    • 27 |
  • 28 |

    40 k images

    29 |
    • 30 |
  • 31 |

    120 k patches

    32 |
    • 33 |
Download 35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
Code
44 |
45 |
    46 |
  • 47 |

    Pretrained Models

    48 |
    • 49 |
  • 50 |

    Reproducible Results

    51 |
    • 52 |
  • 53 |

    Training/Testing Utils

    54 |
    • 55 |
  • 56 |

    Open Demo In Colab

    57 |
    • 58 |
Download 59 |
60 |
61 |
62 |
63 | 64 |
65 |
66 |
67 |
68 |
Paper
69 |
70 |
    71 |
  • 72 |

    Available on ArXiv

    73 |
    • 74 |
  • 75 |

    Accepted to CVPR 2020

    76 |
    • 77 |
  • 78 |

    79 | 80 |

    81 |
    • 82 | 85 |
86 | Download 87 |
88 |
89 |
90 |
91 | 92 | 93 |
94 |
95 |
96 | -------------------------------------------------------------------------------- /pvq.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | sections=head,abstract,applications,online_demo, download_zone,foot 3 | ; sections=head,online_demo,download_zone,foot 4 | out_html = index.html 5 | 6 | [PatchVQ] 7 | issue_report=https://github.com/baidut/PatchVQ/issues 8 | data_download=https://github.com/baidut/PatchVQ 9 | pdf_download=https://arxiv.org/pdf/2011.13544.pdf 10 | sections=head,pvq/abstract,pvq/download_zone,foot 11 | abbr = PatchVQ 12 | title = Patch-VQ: ‘Patching Up’ the Video Quality Problem 13 | authors = Zhenqiang Ying1*, 14 | Maniratnam Mandal1*, 15 | Deepti Ghadiyaram2+, 16 | Alan Bovik1+, 17 | addresses = 1University of Texas at Austin, 18 | 2Facebook AI 19 | emails = {zqying, mmandal}@utexas.edu, deeptigp@fb.com, bovik@ece.utexas.edu 20 | 21 | abstract = No-reference (NR) perceptual video quality assessment (VQA) is a complex, unsolved, and important problem to social and streaming media applications. Efficient and accurate video quality predictors are needed to monitor and guide the processing of billions of shared, often imperfect, user-generated content (UGC). Unfortunately, current NR models are limited in their prediction capabilities on real-world, "in-the-wild" UGC video data. To advance progress on this problem, we created the largest (by far) subjective video quality dataset, containing 39, 000 real-world distorted videos and 117, 000 space-time localized video patches ("v-patches"), and 5.5M human perceptual quality annotations. Using this, we created two unique NR-VQA models: (a) a local-to-global region-based NR VQA architecture (called PVQ) that learns to predict global video quality and achieves state-of-the-art performance on 3 UGC datasets, and (b) a first-of-a-kind space-time video quality mapping engine (called PVQ Mapper) that helps localize and visualize perceptual distortions in space and time. We will make the new database and prediction models available immediately following the review process. 22 | 23 | 24 | [PaQ2PiQ] 25 | sections=head,paq2piq/abstract,paq2piq/applications,paq2piq/online_demo, paq2piq/download_zone,foot 26 | data_download=https://github.com/niu-haoran/FLIVE_Database/blob/master/database_prep.ipynb 27 | abbr = PaQ-2-PiQ 28 | title = From Patches to Pictures (PaQ-2-PiQ):
Mapping the Perceptual Space of Picture Quality 29 | authors = Zhenqiang Ying1*, 30 | Haoran Niu1*, 31 | Praful Gupta1, 32 | Dhruv Mahajan2, 33 | Deepti Ghadiyaram2+, 34 | Alan Bovik1+, 35 | addresses = 1University of Texas at Austin, 36 | 2Facebook AI 37 | emails = {zqying, haoranniu, praful gupta}@utexas.edu, {dhruvm, deeptigp}@fb.com, bovik@ece.utexas.edu 38 | 39 | abstract = Blind or no-reference (NR) perceptual picture quality prediction is a difficult, unsolved problem of great consequence to the social and streaming media industries that impacts billions of viewers daily. Unfortunately, popular NR prediction models perform poorly on real-world distorted pictures. To advance progress on this problem, we introduce the largest (by far) subjective picture quality database, containing about 40000 real-world distorted pictures and 120000 patches, on which we collected about 4M human judgments of picture quality. Using these picture and patch quality labels, we built deep region-based architectures that learn to produce state-of-the-art global picture quality predictions as well as useful local picture quality maps. Our innovations include picture quality prediction architectures that produce global-to-local inferences as well as local-to-global inferences (via feedback). 40 | 41 | icon = 42 | 43 | out_html = paq2piq.html 44 | -------------------------------------------------------------------------------- /pvq/abstract.htm: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 9 | 10 |
11 |

ABSTRACT

12 |

13 | 14 |

15 | {{abstract}} 16 |
17 |
18 |
19 |
20 | 21 |
22 | -------------------------------------------------------------------------------- /pvq/download_zone.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
5 |
6 |
7 |

DOWNLOAD

8 |

9 | Please submit an issue here if you encounter any difficulties. 10 |

11 |
12 |
13 |
14 |
15 |
16 |
17 |
Database
18 | 19 |
20 |
    21 |
  • 22 |

    5.5 M ratings

    23 |
    • 24 |
  • 25 |

    6.3 k subjects

    26 |
    • 27 |
  • 28 |

    39 k videos

    29 |
    • 30 |
  • 31 |

    117 k patches

    32 |
    • 33 |
Download 34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
Code
43 |
44 |
    45 |
  • 46 |

    Pretrained Models

    47 |
    • 48 |
  • 49 |

    Reproducible Results

    50 |
    • 51 |
  • 52 |

    Training/Testing Utils

    53 |
    • 54 |
  • 55 |
    56 | 57 |
    • 58 |
Download 59 |
60 |
61 |
62 |
63 | 64 |
65 |
66 |
67 |
68 |
Paper
69 |
70 |
71 |
72 |
73 |
74 |
75 |
    76 |
  • 77 |

    Available on ArXiv

    78 |
    • 79 |
  • 80 | 81 |
    • 82 |
  • 83 |

    84 | 85 |

    86 |
    • 87 | 90 |
91 | Download 92 |
93 |
94 |
95 |
96 | 97 | 98 |
99 |
100 |
101 | -------------------------------------------------------------------------------- /styles/column.css: -------------------------------------------------------------------------------- 1 | /* Create four equal columns that sits next to each other */ 2 | .column { 3 | -ms-flex: 25%; /* IE10 */ 4 | flex: 25%; 5 | max-width: 25%; 6 | padding: 0 4px; 7 | } 8 | 9 | .doublecolumn { 10 | -ms-flex: 50%; /* IE10 */ 11 | flex: 50%; 12 | max-width: 50%; 13 | padding: 0 8px; 14 | } 15 | 16 | .column img { 17 | margin-top: 8px; 18 | vertical-align: middle; 19 | width: 100%; 20 | } 21 | 22 | /* Responsive layout - makes a two column-layout instead of four columns */ 23 | @media screen and (max-width: 800px) { 24 | .column { 25 | -ms-flex: 50%; 26 | flex: 50%; 27 | max-width: 50%; 28 | } 29 | .doublecolumn { 30 | -ms-flex: 100%; 31 | flex: 100%; 32 | max-width: 100%; 33 | } 34 | } 35 | 36 | /* Responsive layout - makes the two columns stack on top of each other instead of next to each other */ 37 | @media screen and (max-width: 600px) { 38 | .column { 39 | -ms-flex: 100%; 40 | flex: 100%; 41 | max-width: 100%; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /styles/heatmap.css: -------------------------------------------------------------------------------- 1 | 2 | 3 | /* Create four equal columns that sits next to each other */ 4 | .column { 5 | -ms-flex: 25%; /* IE10 */ 6 | flex: 25%; 7 | max-width: 25%; 8 | padding: 0 4px; 9 | } 10 | 11 | .doublecolumnL { 12 | -ms-flex: 40%; /* IE10 */ 13 | flex: 40%; 14 | max-width: 40%; 15 | padding: 0 8px; 16 | } 17 | 18 | .doublecolumnR { 19 | -ms-flex: 50%; /* IE10 */ 20 | flex: 50%; 21 | padding: 0 8px; 22 | } 23 | 24 | .column img { 25 | margin-top: 8px; 26 | vertical-align: middle; 27 | width: 100%; 28 | } 29 | 30 | /* Responsive layout - makes a two column-layout instead of four columns */ 31 | @media screen and (max-width: 800px) { 32 | .column { 33 | -ms-flex: 50%; 34 | flex: 50%; 35 | max-width: 50%; 36 | } 37 | .doublecolumn { 38 | -ms-flex: 100%; 39 | flex: 100%; 40 | max-width: 100%; 41 | } 42 | } 43 | 44 | /* Responsive layout - makes the two columns stack on top of each other instead of next to each other */ 45 | @media screen and (max-width: 600px) { 46 | .column { 47 | -ms-flex: 100%; 48 | flex: 100%; 49 | max-width: 100%; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /styles/main.css: -------------------------------------------------------------------------------- 1 | video { 2 | width: 100%; 3 | height: auto; 4 | } 5 | 6 | 7 | #intro { 8 | background: url("../img/blue-grey.jpg") no-repeat center center; 9 | background-size: cover; 10 | } 11 | 12 | .top-nav-collapse { 13 | background-color: #263238 !important; 14 | } 15 | 16 | .navbar:not(.top-nav-collapse) { 17 | background: transparent !important; 18 | } 19 | 20 | @media (max-width: 768px) { 21 | .navbar:not(.top-nav-collapse) { 22 | background: #263238 !important; 23 | } 24 | } 25 | 26 | #intro .h6 { 27 | font-weight: 300; 28 | line-height: 1.7; 29 | } 30 | 31 | @media (max-width: 450px) { 32 | .margins { 33 | margin-right: 1rem; 34 | margin-left: 1rem; 35 | } 36 | } 37 | 38 | #pricing .lead { 39 | opacity: 0.7; 40 | } 41 | 42 | #pricing ul li { 43 | font-size: 1.1em; 44 | } 45 | 46 | #client .carousel-indicators li { 47 | background-color: #9e9e9e !important; 48 | } 49 | 50 | #client .carousel-indicators .active { 51 | background-color: #3f51b5 !important; 52 | } 53 | 54 | #contact .md-form input[type=text]:focus:not([readonly]) { 55 | border-bottom: 1px solid #2BBBAD; 56 | -webkit-box-shadow: 0 1px 0 0 #2BBBAD; 57 | box-shadow: 0 1px 0 0 #2BBBAD; 58 | } 59 | 60 | #contact .md-form input[type=text]:focus:not([readonly]) + label { 61 | color: #2BBBAD; 62 | } 63 | 64 | #contact .md-form textarea:focus:not([readonly]) { 65 | border-bottom: 1px solid #2BBBAD; 66 | -webkit-box-shadow: 0 1px 0 0 #2BBBAD; 67 | box-shadow: 0 1px 0 0 #2BBBAD; 68 | } 69 | 70 | #contact .md-form textarea:focus:not([readonly]) + label { 71 | color: #2BBBAD; 72 | } 73 | 74 | #about{ 75 | /*font-family: "Comic Sans MS", cursive, sans-serif;*/ 76 | font-family: Georgia, serif; 77 | } 78 | -------------------------------------------------------------------------------- /styles/reset-this.css: -------------------------------------------------------------------------------- 1 | .reset-this * { 2 | animation : none; 3 | animation-delay : 0; 4 | animation-direction : normal; 5 | animation-duration : 0; 6 | animation-fill-mode : none; 7 | animation-iteration-count : 1; 8 | animation-name : none; 9 | animation-play-state : running; 10 | animation-timing-function : ease; 11 | backface-visibility : visible; 12 | background : 0; 13 | background-attachment : scroll; 14 | background-clip : border-box; 15 | background-color : transparent; 16 | background-image : none; 17 | background-origin : padding-box; 18 | background-position : 0 0; 19 | background-position-x : 0; 20 | background-position-y : 0; 21 | background-repeat : repeat; 22 | background-size : auto auto; 23 | border : 0; 24 | border-style : none; 25 | border-width : medium; 26 | border-color : inherit; 27 | border-bottom : 0; 28 | border-bottom-color : inherit; 29 | border-bottom-left-radius : 0; 30 | border-bottom-right-radius : 0; 31 | border-bottom-style : none; 32 | border-bottom-width : medium; 33 | border-collapse : separate; 34 | border-image : none; 35 | border-left : 0; 36 | border-left-color : inherit; 37 | border-left-style : none; 38 | border-left-width : medium; 39 | border-radius : 0; 40 | border-right : 0; 41 | border-right-color : inherit; 42 | border-right-style : none; 43 | border-right-width : medium; 44 | border-spacing : 0; 45 | border-top : 0; 46 | border-top-color : inherit; 47 | border-top-left-radius : 0; 48 | border-top-right-radius : 0; 49 | border-top-style : none; 50 | border-top-width : medium; 51 | bottom : auto; 52 | box-shadow : none; 53 | box-sizing : content-box; 54 | caption-side : top; 55 | clear : none; 56 | clip : auto; 57 | color : inherit; 58 | columns : auto; 59 | column-count : auto; 60 | column-fill : balance; 61 | column-gap : normal; 62 | column-rule : medium none currentColor; 63 | column-rule-color : currentColor; 64 | column-rule-style : none; 65 | column-rule-width : none; 66 | column-span : 1; 67 | column-width : auto; 68 | content : normal; 69 | counter-increment : none; 70 | counter-reset : none; 71 | cursor : auto; 72 | direction : ltr; 73 | display : inline; 74 | empty-cells : show; 75 | float : none; 76 | font : normal; 77 | font-family : inherit; 78 | font-size : medium; 79 | font-style : normal; 80 | font-variant : normal; 81 | font-weight : normal; 82 | height : auto; 83 | hyphens : none; 84 | left : auto; 85 | letter-spacing : normal; 86 | line-height : normal; 87 | list-style : none; 88 | list-style-image : none; 89 | list-style-position : outside; 90 | list-style-type : disc; 91 | margin : 0; 92 | margin-bottom : 0; 93 | margin-left : 0; 94 | margin-right : 0; 95 | margin-top : 0; 96 | max-height : none; 97 | max-width : none; 98 | min-height : 0; 99 | min-width : 0; 100 | opacity : 1; 101 | orphans : 0; 102 | outline : 0; 103 | outline-color : invert; 104 | outline-style : none; 105 | outline-width : medium; 106 | overflow : visible; 107 | overflow-x : visible; 108 | overflow-y : visible; 109 | padding : 0; 110 | padding-bottom : 0; 111 | padding-left : 0; 112 | padding-right : 0; 113 | padding-top : 0; 114 | page-break-after : auto; 115 | page-break-before : auto; 116 | page-break-inside : auto; 117 | perspective : none; 118 | perspective-origin : 50% 50%; 119 | position : static; 120 | /* May need to alter quotes for different locales (e.g fr) */ 121 | quotes : '\201C' '\201D' '\2018' '\2019'; 122 | right : auto; 123 | tab-size : 8; 124 | table-layout : auto; 125 | text-align : inherit; 126 | text-align-last : auto; 127 | text-decoration : none; 128 | text-decoration-color : inherit; 129 | text-decoration-line : none; 130 | text-decoration-style : solid; 131 | text-indent : 0; 132 | text-shadow : none; 133 | text-transform : none; 134 | top : auto; 135 | transform : none; 136 | transform-style : flat; 137 | transition : none; 138 | transition-delay : 0s; 139 | transition-duration : 0s; 140 | transition-property : none; 141 | transition-timing-function : ease; 142 | unicode-bidi : normal; 143 | vertical-align : baseline; 144 | visibility : visible; 145 | white-space : normal; 146 | widows : 0; 147 | width : auto; 148 | word-spacing : normal; 149 | z-index : auto; 150 | /* basic modern patch */ 151 | all: initial; 152 | all: unset; 153 | } 154 | 155 | /* basic modern patch */ 156 | 157 | #reset-this-root { 158 | all: initial; 159 | * { 160 | all: unset; 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /styles/social_bar.css: -------------------------------------------------------------------------------- 1 | /* .icon-bar { 2 | position: fixed; 3 | top: 50%; 4 | -webkit-transform: translateY(-50%); 5 | -ms-transform: translateY(-50%); 6 | transform: translateY(-50%); 7 | } 8 | 9 | .icon-bar a { 10 | display: block; 11 | text-align: center; 12 | padding: 16px; 13 | transition: all 0.3s ease; 14 | color: white; 15 | font-size: 20px; 16 | } 17 | 18 | .icon-bar a:hover { 19 | background-color: #000; 20 | } */ 21 | 22 | /* .facebook { 23 | background: #3B5998; 24 | color: white; 25 | } 26 | 27 | .twitter { 28 | background: #55ACEE; 29 | color: white; 30 | } 31 | 32 | .google { 33 | background: #dd4b39; 34 | color: white; 35 | } 36 | 37 | .linkedin { 38 | background: #007bb5; 39 | color: white; 40 | } 41 | 42 | .youtube { 43 | background: #bb0000; 44 | color: white; 45 | } 46 | 47 | .content { 48 | margin-left: 75px; 49 | font-size: 30px; 50 | } */ 51 | 52 | .social { 53 | padding: 20px; 54 | font-size: 30px; 55 | width: 30px; 56 | } 57 | -------------------------------------------------------------------------------- /styles/switch.css: -------------------------------------------------------------------------------- 1 | /* https://www.w3schools.com/howto/howto_css_switch.asp */ 2 | /* The switch - the box around the slider */ 3 | .switch { 4 | position: relative; 5 | display: inline-block; 6 | width: 60px; 7 | height: 34px; 8 | } 9 | 10 | /* Hide default HTML checkbox */ 11 | .switch input { 12 | opacity: 0; 13 | width: 0; 14 | height: 0; 15 | } 16 | 17 | /* The slider */ 18 | .slider { 19 | position: absolute; 20 | cursor: pointer; 21 | top: 0; 22 | left: 0; 23 | right: 0; 24 | bottom: 0; 25 | background-color: #ccc; 26 | -webkit-transition: .4s; 27 | transition: .4s; 28 | } 29 | 30 | .slider:before { 31 | position: absolute; 32 | content: ""; 33 | height: 26px; 34 | width: 26px; 35 | left: 4px; 36 | bottom: 4px; 37 | background-color: white; 38 | -webkit-transition: .4s; 39 | transition: .4s; 40 | } 41 | 42 | input:checked + .slider { 43 | background-color: #2196F3; 44 | } 45 | 46 | input:focus + .slider { 47 | box-shadow: 0 0 1px #2196F3; 48 | } 49 | 50 | input:checked + .slider:before { 51 | -webkit-transform: translateX(26px); 52 | -ms-transform: translateX(26px); 53 | transform: translateX(26px); 54 | } 55 | 56 | /* Rounded sliders */ 57 | .slider.round { 58 | border-radius: 34px; 59 | } 60 | 61 | .slider.round:before { 62 | border-radius: 50%; 63 | } 64 | -------------------------------------------------------------------------------- /template.htm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidut/PatchVQ/923499b06e2bd9906be084e702eb98f20c4fa9c4/template.htm --------------------------------------------------------------------------------