%0 Conference Paper %1 Schaefer2023 %A Schäfer, Noel %A Künzel, Sebastian %A Munz-Körner, Tanja %A Tilli, Pascal %A Vidyapu, Sandeep %A Thang Vu, Ngoc %A Weiskopf, Daniel %B Proceedings of the 16th International Symposium on Visual Information Communication and Interaction %C New York, NY, USA %D 2023 %I Association for Computing Machinery %K exc2075 myown pn6 vis(us) visus visus:kuenzesn visus:munzta visus:weiskopf %P 1–8 %R 10.1145/3615522.3615547 %T Visual Analysis of Scene-Graph-Based Visual Question Answering %U https://doi.org/10.1145/3615522.3615547 %X Scene-graph-based Visual Question Answering (VQA) has emerged as a burgeoning field in Deep Learning research, with a growing demand for robust and interpretable VQA systems. In this paper, we present a novel visual analysis approach that addresses two critical objectives in VQA: identifying and correcting prediction issues and providing insights into model decision-making processes through visualizing internal information. Our approach builds on the GraphVQA framework, which uses graph neural networks to process scene graphs representing images and which was trained on the widely-used GQA dataset. Our analysis tool aims at users familiar with the basics of graph-based VQA. By leveraging query-based scene analysis and visualization of crucial internal states, we are able to detect and pinpoint reasons for inaccurate predictions, facilitating model refinement and dataset curation. Identifying expressive internal states is a challenge. Through rigorous computer-based evaluations and presentation of a use case, we demonstrate the effectiveness of our analysis tool and model state visualization. %@ 9798400707513

@inproceedings{Schaefer2023, abstract = {Scene-graph-based Visual Question Answering (VQA) has emerged as a burgeoning field in Deep Learning research, with a growing demand for robust and interpretable VQA systems. In this paper, we present a novel visual analysis approach that addresses two critical objectives in VQA: identifying and correcting prediction issues and providing insights into model decision-making processes through visualizing internal information. Our approach builds on the GraphVQA framework, which uses graph neural networks to process scene graphs representing images and which was trained on the widely-used GQA dataset. Our analysis tool aims at users familiar with the basics of graph-based VQA. By leveraging query-based scene analysis and visualization of crucial internal states, we are able to detect and pinpoint reasons for inaccurate predictions, facilitating model refinement and dataset curation. Identifying expressive internal states is a challenge. Through rigorous computer-based evaluations and presentation of a use case, we demonstrate the effectiveness of our analysis tool and model state visualization.}, added-at = {2023-10-23T19:14:40.000+0200}, address = {New York, NY, USA}, author = {Schäfer, Noel and Künzel, Sebastian and Munz-Körner, Tanja and Tilli, Pascal and Vidyapu, Sandeep and Thang Vu, Ngoc and Weiskopf, Daniel}, biburl = {https://puma.ub.uni-stuttgart.de/bibtex/245f80081761d7a4507e7f2b210b79768/tanjamunz}, booktitle = {Proceedings of the 16th International Symposium on Visual Information Communication and Interaction}, day = 20, doi = {10.1145/3615522.3615547}, interhash = {e858fa98d08c5e0e3d2b4baefa5bd60e}, intrahash = {45f80081761d7a4507e7f2b210b79768}, isbn = {9798400707513}, keywords = {exc2075 myown pn6 vis(us) visus visus:kuenzesn visus:munzta visus:weiskopf}, location = {Guangzhou, China}, month = {10}, pages = {1–8}, publisher = {Association for Computing Machinery}, series = {VINCI '23}, timestamp = {2023-10-23T19:14:40.000+0200}, title = {Visual Analysis of Scene-Graph-Based Visual Question Answering}, url = {https://doi.org/10.1145/3615522.3615547}, year = 2023 }