{ "types" : { "Bookmark" : { "pluralLabel" : "Bookmarks" }, "Publication" : { "pluralLabel" : "Publications" }, "GoldStandardPublication" : { "pluralLabel" : "GoldStandardPublications" }, "GoldStandardBookmark" : { "pluralLabel" : "GoldStandardBookmarks" }, "Tag" : { "pluralLabel" : "Tags" }, "User" : { "pluralLabel" : "Users" }, "Group" : { "pluralLabel" : "Groups" }, "Sphere" : { "pluralLabel" : "Spheres" } }, "properties" : { "count" : { "valueType" : "number" }, "date" : { "valueType" : "date" }, "changeDate" : { "valueType" : "date" }, "url" : { "valueType" : "url" }, "id" : { "valueType" : "url" }, "tags" : { "valueType" : "item" }, "user" : { "valueType" : "item" } }, "items" : [ { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/288a5d21da4a41f15c79cfacfc16236b3/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "88a5d21da4a41f15c79cfacfc16236b3", "interHash" : "539e7856c2e2d296175310c13454ce03", "label" : "Impact of Privacy Protection Methods of Lifelogs on Remembered Memories", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:08", "changeDate" : "2025-02-28 16:08:06", "count" : 7, "pub-type": "inproceedings", "booktitle": "Proc. ACM SIGCHI Conference on Human Factors in Computing Systems (CHI)", "year": "2023", "url": "", "author": [ "Passant Elagroudy","Mohamed Khamis","Florian Mathis","Diana Irmscher","Ekta Sood","Andreas Bulling","Albrecht Schmidt" ], "authors": [ {"first" : "Passant", "last" : "Elagroudy"}, {"first" : "Mohamed", "last" : "Khamis"}, {"first" : "Florian", "last" : "Mathis"}, {"first" : "Diana", "last" : "Irmscher"}, {"first" : "Ekta", "last" : "Sood"}, {"first" : "Andreas", "last" : "Bulling"}, {"first" : "Albrecht", "last" : "Schmidt"} ], "pages": "1--10","note": "spotlight","abstract": "Lifelogging is traditionally used for memory augmentation. However, recent research shows that users\u2019 trust in the completeness and accuracy of lifelogs might skew their memories. Privacy-protection alterations such as body blurring and content deletion are commonly applied to photos to circumvent capturing sensitive information. However, their impact on how users remember memories remain unclear. To this end, we conduct a white-hat memory attack and report on an iterative experiment (N=21) to compare the impact of viewing 1) unaltered lifelogs, 2) blurred lifelogs, and 3) a subset of the lifelogs after deleting private ones, on confidently remembering memories. Findings indicate that all the privacy methods impact memories\u2019 quality similarly and that users tend to change their answers in recognition more than recall scenarios. Results also show that users have high confidence in their remembered content across all privacy methods. Our work raises awareness about the mindful designing of technological interventions.", "doi" : "10.1145/3544548.3581565", "bibtexKey": "elagroudy23_chi" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/2f38edfcdce37f45530abaf7145d5a8d9/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "f38edfcdce37f45530abaf7145d5a8d9", "interHash" : "091b4950a1314d694e8a6e2c5d18b823", "label" : "InteRead: An Eye Tracking Dataset of Interrupted Reading", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:08", "changeDate" : "2025-02-28 16:08:06", "count" : 7, "pub-type": "inproceedings", "booktitle": "Proc. 31st Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING)", "year": "2024", "url": "https://aclanthology.org/2024.lrec-main.802/", "author": [ "Francesca Zermiani","Prajit Dhar","Ekta Sood","Fabian Kögel","Andreas Bulling","Maria Wirzberger" ], "authors": [ {"first" : "Francesca", "last" : "Zermiani"}, {"first" : "Prajit", "last" : "Dhar"}, {"first" : "Ekta", "last" : "Sood"}, {"first" : "Fabian", "last" : "Kögel"}, {"first" : "Andreas", "last" : "Bulling"}, {"first" : "Maria", "last" : "Wirzberger"} ], "pages": "9154--9169","abstract": "Eye movements during reading offer a window into cognitive processes and language comprehension, but the scarcity of reading data with interruptions -- which learners frequently encounter in their everyday learning environments -- hampers advances in the development of intelligent learning technologies. We introduce InteRead -- a novel 50-participant dataset of gaze data recorded during self-paced reading of real-world text. InteRead further offers fine-grained annotations of interruptions interspersed throughout the text as well as resumption lags incurred by these interruptions. Interruptions were triggered automatically once readers reached predefined target words. We validate our dataset by reporting interdisciplinary analyses on different measures of gaze behavior. In line with prior research, our analyses show that the interruptions as well as word length and word frequency effects significantly impact eye movements during reading. We also explore individual differences within our dataset, shedding light on the potential for tailored educational solutions. InteRead is accessible from our datasets web-page: https://www.ife.uni-stuttgart.de/en/llis/research/datasets/.", "bibtexKey": "zermiani24_coling" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/2732572932bd773a1d162712578fb9ecd/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "732572932bd773a1d162712578fb9ecd", "interHash" : "4f5017faa6f0f522d719d5ce15205a93", "label" : "Facial Composite Generation with Iterative Human Feedback", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:08", "changeDate" : "2025-02-28 16:08:06", "count" : 6, "pub-type": "inproceedings", "booktitle": "Proc. The 1st Gaze Meets ML workshop, PMLR","series": "Proceedings of Machine Learning Research","publisher":"PMLR", "year": "2023", "url": "https://proceedings.mlr.press/v210/strohm23a.html", "author": [ "Florian Strohm","Ekta Sood","Dominike Thomas","Mihai Bâce","Andreas Bulling" ], "authors": [ {"first" : "Florian", "last" : "Strohm"}, {"first" : "Ekta", "last" : "Sood"}, {"first" : "Dominike", "last" : "Thomas"}, {"first" : "Mihai", "last" : "Bâce"}, {"first" : "Andreas", "last" : "Bulling"} ], "editor": [ "Ismini Lourentzou","Joy Wu","Satyananda Kashyap","Alexandros Karargyris","Leo Anthony Celi","Ban Kawas","Sachin Talathi" ], "editors": [ {"first" : "Ismini", "last" : "Lourentzou"}, {"first" : "Joy", "last" : "Wu"}, {"first" : "Satyananda", "last" : "Kashyap"}, {"first" : "Alexandros", "last" : "Karargyris"}, {"first" : "Leo Anthony", "last" : "Celi"}, {"first" : "Ban", "last" : "Kawas"}, {"first" : "Sachin", "last" : "Talathi"} ], "volume": "210","pages": "165--183","abstract": "We propose the first method in which human and AI collaborate to iteratively reconstruct the human\u2019s mental image of another person\u2019s face only from their eye gaze. Current tools for generating digital human faces involve a tedious and time-consuming manual design process. While gaze-based mental image reconstruction represents a promising alternative, previous methods still assumed prior knowledge about the target face, thereby severely limiting their practical usefulness. The key novelty of our method is a collaborative, it- erative query engine: Based on the user\u2019s gaze behaviour in each iteration, our method predicts which images to show to the user in the next iteration. Results from two human studies (N=12 and N=22) show that our method can visually reconstruct digital faces that are more similar to the mental image, and is more usable compared to other methods. As such, our findings point at the significant potential of human-AI collaboration for recon- structing mental images, potentially also beyond faces, and of human gaze as a rich source of information and a powerful mediator in said collaboration.", "pdf" : "https://proceedings.mlr.press/v210/strohm23a/strohm23a.pdf", "bibtexKey": "strohm23_gmml" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/22332687c0dcf57f8a4e6bfc4adde675c/simtech", "tags" : [ "PN7","PN7-5","curated","exc2075" ], "intraHash" : "2332687c0dcf57f8a4e6bfc4adde675c", "interHash" : "1d3abae0619209fa4857a347bb8db61c", "label" : "VQA-MHUG: A gaze dataset to study multimodal neural attention in VQA", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:08", "changeDate" : "2025-02-28 16:08:06", "count" : 6, "pub-type": "inproceedings", "booktitle": "Proc. ACL SIGNLL Conference on Computational Natural Language Learning (CoNLL)","publisher":"Association for Computational Linguistics", "year": "2021", "url": "", "author": [ "Ekta Sood","Fabian Kögel","Florian Strohm","Prajit Dhar","Andreas Bulling" ], "authors": [ {"first" : "Ekta", "last" : "Sood"}, {"first" : "Fabian", "last" : "Kögel"}, {"first" : "Florian", "last" : "Strohm"}, {"first" : "Prajit", "last" : "Dhar"}, {"first" : "Andreas", "last" : "Bulling"} ], "pages": "27--43","note": "spotlight","abstract": "We present VQA-MHUG - a novel 49-participant dataset of multimodal human gaze on both images and questions during visual question answering (VQA) collected using a high-speed eye tracker. We use our dataset to analyze the similarity between human and neural attentive strategies learned by five state-of-the-art VQA models: Modulated Co-Attention Network (MCAN) with either grid or region features, Pythia, Bilinear Attention Network (BAN), and the Multimodal Factorized Bilinear Pooling Network (MFB). While prior work has focused on studying the image modality, our analyses show - for the first time - that for all models, higher correlation with human attention on text is a significant predictor of VQA performance. This finding points at a potential for improving VQA performance and, at the same time, calls for further research on neural text attention mechanisms and their integration into architectures for vision and language tasks, including but potentially also beyond VQA.", "code" : "https://git.hcics.simtech.uni-stuttgart.de/public-projects/vqa-mhug-interpretability", "award" : "Oral presentation", "dataset" : "https://perceptualui.org/research/datasets/VQA-MHUG/", "doi" : "10.18653/v1/2021.conll-1.3", "bibtexKey": "sood21_conll" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/289a5c7ed25d6ec5d7638bcfd2a68f910/simtech", "tags" : [ "curated","exc2075","pn7","pn7-5" ], "intraHash" : "89a5c7ed25d6ec5d7638bcfd2a68f910", "interHash" : "7e28d355a52f4f5a098453fa16f906d8", "label" : "Gaze-enhanced Crossmodal Embeddings for Emotion Recognition", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:08", "changeDate" : "2025-02-28 16:08:06", "count" : 6, "pub-type": "inproceedings", "booktitle": "Proc. International Symposium on Eye Tracking Research and Applications (ETRA)", "year": "2022", "url": "", "author": [ "Ahmed Abdou","Ekta Sood","Philipp Müller","Andreas Bulling" ], "authors": [ {"first" : "Ahmed", "last" : "Abdou"}, {"first" : "Ekta", "last" : "Sood"}, {"first" : "Philipp", "last" : "Müller"}, {"first" : "Andreas", "last" : "Bulling"} ], "volume": "6","pages": "1--18","abstract": "Emotional expressions are inherently multimodal -- integrating facial behavior, speech, and gaze -- but their automatic recognition is often limited to a single modality, e.g. speech during a phone call. While previous work proposed crossmodal emotion embeddings to improve monomodal recognition performance, despite its importance, a representation of gaze was not included. We propose a new approach to emotion recognition that incorporates an explicit representation of gaze in a crossmodal emotion embedding framework. We show that our method outperforms the previous state of the art for both audio-only and video-only emotion classification on the popular One-Minute Gradual Emotion Recognition dataset. Furthermore, we report extensive ablation experiments and provide insights into the performance of different state-of-the-art gaze representations and integration strategies. Our results not only underline the importance of gaze for emotion recognition but also demonstrate a practical and highly effective approach to leveraging gaze information for this task.", "code" : "https://git.hcics.simtech.uni-stuttgart.de/public-projects/gaze-enhanced-crossmodal-embeddings-for-emotion-recognition", "doi" : "10.1145/3530879", "bibtexKey": "abdou22_etra" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/2c75873ce735b3cc04365c6cc575964c3/simtech", "tags" : [ "PN7","PN7-5","curated","exc2075" ], "intraHash" : "c75873ce735b3cc04365c6cc575964c3", "interHash" : "9c4b0990c6ef60dd915e4c6dd33d8a7b", "label" : "Video Language Co-Attention with Multimodal Fast-Learning Feature Fusion for VideoQA", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:08", "changeDate" : "2025-02-28 16:08:06", "count" : 7, "pub-type": "inproceedings", "booktitle": "Proceedings of the 7th Workshop on Representation Learning for NLP","publisher":"Association for Computational Linguistics","address":"Stroudsburg", "year": "2022", "url": "", "author": [ "Adnen Abdessaied","Ekta Sood","Andreas Bulling" ], "authors": [ {"first" : "Adnen", "last" : "Abdessaied"}, {"first" : "Ekta", "last" : "Sood"}, {"first" : "Andreas", "last" : "Bulling"} ], "pages": "143-155", "venue" : "Dublin, Ireland and Online", "isbn" : "978-1-955917-48-3", "research-areas" : "Computer Science", "language" : "eng", "eventdate" : "2022-05-26", "eventtitle" : "7th Workshop on Representation Learning for NLP (RepL4NLP 2022)", "affiliation" : "Abdessaied, A (Corresponding Author), Univ Stuttgart, Inst Visualizat & Interact Syst VIS, Stuttgart, Germany.\n Abdessaied, Adnen; Sood, Ekta; Bulling, Andreas, Univ Stuttgart, Inst Visualizat & Interact Syst VIS, Stuttgart, Germany.", "unique-id" : "WOS:000847242200013", "doi" : "10.18653/v1/2022.repl4nlp-1.15", "bibtexKey": "abdessaied2022video" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/2d5ddc756c2b5f9a2cf500b26bec8aa2a/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "d5ddc756c2b5f9a2cf500b26bec8aa2a", "interHash" : "af6bd1a5a36a93d625a1f8c83b975539", "label" : "Multimodal Integration of Human-Like Attention in Visual Question Answering", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:07", "changeDate" : "2025-02-28 16:08:06", "count" : 6, "pub-type": "techreport", "year": "2021", "url": "https://arxiv.org/pdf/2109.13139.pdf", "author": [ "Ekta Sood","Fabian Kögel","Philipp Müller","Dominike Thomas","Mihai Bâce","Andreas Bulling" ], "authors": [ {"first" : "Ekta", "last" : "Sood"}, {"first" : "Fabian", "last" : "Kögel"}, {"first" : "Philipp", "last" : "Müller"}, {"first" : "Dominike", "last" : "Thomas"}, {"first" : "Mihai", "last" : "Bâce"}, {"first" : "Andreas", "last" : "Bulling"} ], "pages": "1--11","note": "arxiv:2109.13139","abstract": "Human-like attention as a supervisory signal to guide neural attention has shown significant promise but is currently limited to uni-modal integration \u2013 even for inherently multi-modal tasks such as visual question answering (VQA). We present the Multimodal Human-like Attention Network (MULAN) \u2013 the first method for multimodal integration of human-like attention on image and text during training of VQA models. MULAN integrates attention predictions from two state-of-the-art text and image saliency models into neural self-attention layers of a recent transformer-based VQA model. Through evaluations on the challenging VQAv2 dataset, we show that MULAN achieves a new state-of-the-art performance of 73.98% accuracy on test-std and 73.72% on test-dev and, at the same time, has approximately 80% fewer trainable parameters than prior work. Overall, our work underlines the potential of integrating multimodal human-like and neural attention for VQA.", "bibtexKey": "sood21_arxiv" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/25f0b7060bff3df6fe3393b50d82945b2/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "5f0b7060bff3df6fe3393b50d82945b2", "interHash" : "38ecc86042c56ce4c08a5405bd36ee67", "label" : "Neural Photofit : Gaze-based Mental Image Reconstruction", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:07", "changeDate" : "2025-02-28 16:08:06", "count" : 8, "pub-type": "inproceedings", "booktitle": "2021 IEEE/CVF International Conference on Computer Vision (ICCV)","publisher":"IEEE","address":"Piscataway", "year": "2021", "url": "", "author": [ "Florian Strohm","Ekta Sood","Sven Mayer","Philipp Müller","Mihai Bâce","Andreas Bulling" ], "authors": [ {"first" : "Florian", "last" : "Strohm"}, {"first" : "Ekta", "last" : "Sood"}, {"first" : "Sven", "last" : "Mayer"}, {"first" : "Philipp", "last" : "Müller"}, {"first" : "Mihai", "last" : "Bâce"}, {"first" : "Andreas", "last" : "Bulling"} ], "pages": "245-254", "venue" : "Online", "isbn" : "978-1-6654-2812-5 and 978-1-6654-2813-2", "research-areas" : "Computer Science", "language" : "eng", "eventdate" : "2021-10-10/2021-10-17", "eventtitle" : "2021 IEEE/CVF International Conference on Computer Vision (ICCV)", "affiliation" : "Strohm, F (Corresponding Author), Univ Stuttgart, Stuttgart, Germany.\n Strohm, Florian; Sood, Ekta; Bace, Mihai; Bulling, Andreas, Univ Stuttgart, Stuttgart, Germany.\n Mayer, Sven, Ludwig Maximilians Univ Munchen, Munich, Germany.\n Mueller, Philipp, German Res Ctr Artificial Intelligence DFKI, Saarbrucken, Germany.", "unique-id" : "WOS:000797698900025", "doi" : "10.1109/ICCV48922.2021.00031", "bibtexKey": "strohm2021neural" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/2116525f8369c3bf70a00a497ae4363ef/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "116525f8369c3bf70a00a497ae4363ef", "interHash" : "9039d069dd9d3f940693c0f44272cb3f", "label" : "Improving Neural Saliency Prediction with a Cognitive Model of Human Visual Attention", "user" : "simtech", "description" : "", "date" : "2025-02-17 14:54:07", "changeDate" : "2025-02-28 16:08:06", "count" : 18, "pub-type": "inproceedings", "booktitle": "Proc. the 45th Annual Meeting of the Cognitive Science Society (CogSci)", "year": "2023", "url": "", "author": [ "Ekta Sood","Lei Shi","Matteo Bortoletto","Yao Wang","Philipp Müller","Andreas Bulling" ], "authors": [ {"first" : "Ekta", "last" : "Sood"}, {"first" : "Lei", "last" : "Shi"}, {"first" : "Matteo", "last" : "Bortoletto"}, {"first" : "Yao", "last" : "Wang"}, {"first" : "Philipp", "last" : "Müller"}, {"first" : "Andreas", "last" : "Bulling"} ], "pages": "3639--3646","note": "spotlight","abstract": "We present a novel method for saliency prediction that leverages a cognitive model of visual attention as an inductive bias. This approach is in stark contrast to recent purely data-driven saliency models that achieve performance improvements mainly by increased capacity, resulting in high computational costs and the need for large-scale training datasets. We demonstrate that by using a cognitive model, our method achieves competitive performance to the state of the art across several natural image datasets while only requiring a fraction of the parameters. Furthermore, we set the new state of the art for saliency prediction on information visualizations, demonstrating the effectiveness of our approach for cross-domain generalization. We further provide augmented versions of the full MSCOCO dataset with synthetic gaze data using the cognitive model, which we used to pre-train our method. Our results are highly promising and underline the significant potential of bridging between cognitive and data-driven models, potentially also beyond attention.", "code" : "https://git.hcics.simtech.uni-stuttgart.de/public-projects/neural-saliency-prediction-with-a-cognitive-model/", "supp" : "Yes", "dataset" : "https://perceptualui.org/research/datasets/MSCOCOEMMAFigureQAEMMA/", "bibtexKey": "sood23_cogsci" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/2f97b9935cacc514b3e94fb2318b6f61e/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "f97b9935cacc514b3e94fb2318b6f61e", "interHash" : "b088d82b86d04188aded80d640e5a7cb", "label" : "Anticipating Averted Gaze in Dyadic Interactions", "user" : "simtech", "description" : "", "date" : "2021-12-08 17:10:50", "changeDate" : "2025-02-28 16:08:06", "count" : 9, "pub-type": "inproceedings", "booktitle": "ACM Symposium on Eye Tracking Research and Applications","series": "ETRA '20 Full Papers","publisher":"Association for Computing Machinery","address":"New York, NY, USA", "year": "2020", "url": "https://doi.org/10.1145/3379155.3391332", "author": [ "Philipp Müller","Ekta Sood","Andreas Bulling" ], "authors": [ {"first" : "Philipp", "last" : "Müller"}, {"first" : "Ekta", "last" : "Sood"}, {"first" : "Andreas", "last" : "Bulling"} ], "pages": "1\u201310","abstract": "We present the first method to anticipate averted gaze in natural dyadic interactions.\nThe task of anticipating averted gaze, i.e. that a person will not make eye contact\nin the near future, remains unsolved despite its importance for human social encounters\nas well as a number of applications, including human-robot interaction or conversational\nagents. Our multimodal method is based on a long short-term memory (LSTM) network\nthat analyses non-verbal facial cues and speaking behaviour. We empirically evaluate\nour method for different future time horizons on a novel dataset of 121 YouTube videos\nof dyadic video conferences (74 hours in total). We investigate person-specific and\nperson-independent performance and demonstrate that our method clearly outperforms\nbaselines in both settings. As such, our work sheds light on the tight interplay between\neye contact and other non-verbal signals and underlines the potential of computational\nmodelling and anticipation of averted gaze for interactive applications.", "isbn" : "9781450371339", "location" : "Stuttgart, Germany", "doi" : "10.1145/3379155.3391332", "bibtexKey": "Müller2020" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/271dca1344ec8f44c6e672bafb003df36/simtech", "tags" : [ "EXC2075","PN7","PN7-5","curated" ], "intraHash" : "71dca1344ec8f44c6e672bafb003df36", "interHash" : "c56326210814f816b786c87e5b2b8909", "label" : "Improving Natural Language Processing Tasks with Human Gaze-Guided Neural Attention", "user" : "simtech", "description" : "", "date" : "2021-12-08 17:10:49", "changeDate" : "2025-02-28 16:08:06", "count" : 7, "pub-type": "inproceedings", "booktitle": "Advances in Neural Information Processing Systems","publisher":"Curran Associates, Inc.", "year": "2020", "url": "https://proceedings.neurips.cc/paper/2020/file/460191c72f67e90150a093b4585e7eb4-Paper.pdf", "author": [ "Ekta Sood","Simon Tannert","Philipp Mueller","Andreas Bulling" ], "authors": [ {"first" : "Ekta", "last" : "Sood"}, {"first" : "Simon", "last" : "Tannert"}, {"first" : "Philipp", "last" : "Mueller"}, {"first" : "Andreas", "last" : "Bulling"} ], "editor": [ "H. Larochelle","M. Ranzato","R. Hadsell","M. F. Balcan","H. Lin" ], "editors": [ {"first" : "H.", "last" : "Larochelle"}, {"first" : "M.", "last" : "Ranzato"}, {"first" : "R.", "last" : "Hadsell"}, {"first" : "M. F.", "last" : "Balcan"}, {"first" : "H.", "last" : "Lin"} ], "volume": "33","pages": "6327--6341", "bibtexKey": "NEURIPS2020_460191c7" } , { "type" : "Publication", "id" : "https://puma.ub.uni-stuttgart.de/bibtex/2a6b4a6f7774f1abad2001d0bfaf39f02/simtech", "tags" : [ "EXC2075","PN6","PN7","PN7-5","curated" ], "intraHash" : "a6b4a6f7774f1abad2001d0bfaf39f02", "interHash" : "1772969bd0a2da70ec972ee1754b7f56", "label" : "Interpreting Attention Models with Human Visual Attention in Machine Reading Comprehension", "user" : "simtech", "description" : "", "date" : "2021-12-08 17:10:28", "changeDate" : "2025-02-28 16:08:06", "count" : 12, "pub-type": "inproceedings", "booktitle": "Proceedings of the 24th Conference on Computational Natural Language Learning","publisher":"Association for Computational Linguistics","address":"Online", "year": "2020", "url": "https://aclanthology.org/2020.conll-1.2", "author": [ "Ekta Sood","Simon Tannert","Diego Frassinelli","Andreas Bulling","Ngoc Thang Vu" ], "authors": [ {"first" : "Ekta", "last" : "Sood"}, {"first" : "Simon", "last" : "Tannert"}, {"first" : "Diego", "last" : "Frassinelli"}, {"first" : "Andreas", "last" : "Bulling"}, {"first" : "Ngoc Thang", "last" : "Vu"} ], "pages": "12--25","abstract": "While neural networks with attention mechanisms have achieved superior performance on many natural language processing tasks, it remains unclear to which extent learned attention resembles human visual attention. In this paper, we propose a new method that leverages eye-tracking data to investigate the relationship between human visual attention and neural attention in machine reading comprehension. To this end, we introduce a novel 23 participant eye tracking dataset - MQA-RC, in which participants read movie plots and answered pre-defined questions. We compare state of the art networks based on long short-term memory (LSTM), convolutional neural models (CNN) and XLNet Transformer architectures. We find that higher similarity to human attention and performance significantly correlates to the LSTM and CNN models. However, we show this relationship does not hold true for the XLNet models -- despite the fact that the XLNet performs best on this challenging task. Our results suggest that different architectures seem to learn rather different neural attention strategies and similarity of neural to human attention does not guarantee best performance.", "doi" : "10.18653/v1/2020.conll-1.2", "bibtexKey": "sood-etal-2020-interpreting" } ] }