%0 Journal Article %1 GotzHahn2021KonVi-53766 %A Götz-Hahn, Franz %A Hosu, Vlad %A Lin, Hanhe %A Saupe, Dietmar %D 2021 %J IEEE Access %K sfbtrr161 a05 from:christinawarren 2021 %P 72139--72160 %R 10.1109/ACCESS.2021.3077642 %T KonVid-150k : A Dataset for No-Reference Video Quality Assessment of Videos in-the-Wild %U https://dx.doi.org/10.1109/ACCESS.2021.3077642 %V 9 %X Video quality assessment (VQA) methods focus on particular degradation types, usually artificially induced on a small set of reference videos. Hence, most traditional VQA methods under-perform in-the-wild. Deep learning approaches have had limited success due to the small size and diversity of existing VQA datasets, either artificial or authentically distorted. We introduce a new in-the-wild VQA dataset that is substantially larger and diverse: KonVid-150k. It consists of a coarsely annotated set of 153,841 videos having five quality ratings each, and 1,596 videos with a minimum of 89 ratings each. Additionally, we propose new efficient VQA approaches (MLSP-VQA) relying on multi-level spatially pooled deep-features (MLSP). They are exceptionally well suited for training at scale, compared to deep transfer learning approaches. Our best method, MLSP-VQA-FF, improves the Spearman rank-order correlation coefficient (SRCC) performance metric on the commonly used KoNViD-1k in-the-wild benchmark dataset to 0.82. It surpasses the best existing deep-learning model (0.80 SRCC) and hand-crafted feature-based method (0.78 SRCC). We further investigate how alternative approaches perform under different levels of label noise, and dataset size, showing that MLSP-VQA-FF is the overall best method for videos in-the-wild. Finally, we show that the MLSP-VQA models trained on KonVid-150k sets the new state-of-the-art for cross-test performance on KoNViD-1k and LIVE-Qualcomm with a 0.83 and 0.64 SRCC, respectively. For KoNViD-1k this inter-dataset testing outperforms intra-dataset experiments, showing excellent generalization.

@article{GotzHahn2021KonVi-53766, abstract = {Video quality assessment (VQA) methods focus on particular degradation types, usually artificially induced on a small set of reference videos. Hence, most traditional VQA methods under-perform in-the-wild. Deep learning approaches have had limited success due to the small size and diversity of existing VQA datasets, either artificial or authentically distorted. We introduce a new in-the-wild VQA dataset that is substantially larger and diverse: KonVid-150k. It consists of a coarsely annotated set of 153,841 videos having five quality ratings each, and 1,596 videos with a minimum of 89 ratings each. Additionally, we propose new efficient VQA approaches (MLSP-VQA) relying on multi-level spatially pooled deep-features (MLSP). They are exceptionally well suited for training at scale, compared to deep transfer learning approaches. Our best method, MLSP-VQA-FF, improves the Spearman rank-order correlation coefficient (SRCC) performance metric on the commonly used KoNViD-1k in-the-wild benchmark dataset to 0.82. It surpasses the best existing deep-learning model (0.80 SRCC) and hand-crafted feature-based method (0.78 SRCC). We further investigate how alternative approaches perform under different levels of label noise, and dataset size, showing that MLSP-VQA-FF is the overall best method for videos in-the-wild. Finally, we show that the MLSP-VQA models trained on KonVid-150k sets the new state-of-the-art for cross-test performance on KoNViD-1k and LIVE-Qualcomm with a 0.83 and 0.64 SRCC, respectively. For KoNViD-1k this inter-dataset testing outperforms intra-dataset experiments, showing excellent generalization.}, added-at = {2021-06-18T09:39:15.000+0200}, author = {Götz-Hahn, Franz and Hosu, Vlad and Lin, Hanhe and Saupe, Dietmar}, biburl = {https://puma.ub.uni-stuttgart.de/bibtex/25b66e546a7a1f81571da16ccef09e233/sfbtrr161}, doi = {10.1109/ACCESS.2021.3077642}, interhash = {ed7535aae27cb92564bd12566cfca830}, intrahash = {5b66e546a7a1f81571da16ccef09e233}, journal = {IEEE Access}, keywords = {sfbtrr161 a05 from:christinawarren 2021}, pages = {72139--72160}, timestamp = {2021-06-18T07:39:15.000+0200}, title = {KonVid-150k : A Dataset for No-Reference Video Quality Assessment of Videos in-the-Wild}, url = {https://dx.doi.org/10.1109/ACCESS.2021.3077642}, volume = 9, year = 2021 }