@inproceedings{9a5254c1a57b493b8a46e4073c5c0b28,
title = "Hear The Flow: Optical Flow-Based Self-Supervised Visual Sound Source Localization",
abstract = "Learning to localize the sound source in videos without explicit annotations is a novel area of audio-visual research. Existing work in this area focuses on creating attention maps to capture the correlation between the two modalities to localize the source of the sound. In a video, oftentimes, the objects exhibiting movement are the ones generating the sound. In this work, we capture this characteristic by modeling the optical flow in a video as a prior to better aid in localizing the sound source. We further demonstrate that the addition of flow-based attention substantially improves visual sound source localization. Finally, we benchmark our method on standard sound source localization datasets and achieve state-of-the-art performance on the Soundnet Flickr and VGG Sound Source datasets. Code: https://github.com/denfed/heartheflow.",
keywords = "Algorithms: Vision + language and/or other modalities, Machine learning architectures, and algorithms (including transfer, low-shot, semi-, self-, and un-supervised learning), formulations",
author = "Dennis Fedorishin and \{Dayal Mohan\}, Deen and Bhavin Jawade and Srirangaraj Setlur and Venu Govindaraju",
note = "Publisher Copyright: {\textcopyright} 2023 IEEE.; 23rd IEEE/CVF Winter Conference on Applications of Computer Vision, WACV 2023 ; Conference date: 03-01-2023 Through 07-01-2023",
year = "2023",
doi = "10.1109/WACV56688.2023.00231",
language = "English",
series = "Proceedings - 2023 IEEE Winter Conference on Applications of Computer Vision, WACV 2023",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "2277--2286",
booktitle = "Proceedings - 2023 IEEE Winter Conference on Applications of Computer Vision, WACV 2023",
address = "United States",
}