@article{10.1145/3380744,
author = {Akhtar, Md Shad and Chauhan, Dushyant Singh and Ekbal, Asif},
title = {A Deep Multi-Task Contextual Attention Framework for Multi-Modal Affect Analysis},
year = {2020},
issue_date = {May 2020},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {14},
number = {3},
issn = {1556-4681},
url = {https://doi.org/10.1145/3380744},
doi = {10.1145/3380744},
abstract = {Multi-modal affect analysis (e.g., sentiment and emotion analysis) is an interdisciplinary
study and has been an emerging and prominent field in Natural Language Processing
and Computer Vision. The effective fusion of multiple modalities (e.g., text, acoustic,
or visual frames) is a non-trivial task, as these modalities, often, carry distinct
and diverse information, and do not contribute equally. The issue further escalates
when these data contain noise. In this article, we study the concept of multi-task
learning for multi-modal affect analysis and explore a contextual inter-modal attention
framework that aims to leverage the association among the neighboring utterances and
their multi-modal information. In general, sentiments and emotions have inter-dependence
on each other (e.g., anger → negative or happy → positive). In our current work, we
exploit the relatedness among the participating tasks in the multi-task framework.
We define three different multi-task setups, each having two tasks, i.e., sentiment
8 emotion classification, sentiment classification 8 sentiment intensity prediction,
and emotion classificati on 8 emotion intensity prediction. Our evaluation of the
proposed system on the CMU-Multi-modal Opinion Sentiment and Emotion Intensity benchmark
dataset suggests that, in comparison with the single-task learning framework, our
multi-task framework yields better performance for the inter-related participating
tasks. Further, comparative studies show that our proposed approach attains state-of-the-art
performance for most of the cases.},
journal = {ACM Trans. Knowl. Discov. Data},
month = may,
articleno = {32},
numpages = {27},
keywords = {multi-modal analysis, Multi-task learning, inter-modal attention, sentiment analysis, emotion analysis, sentiment intensity prediction, emotion intensity prediction}
}