@inproceedings{3c97697877964e81b4c67c172c5f2fdc,
title = "Improving Joint Learning of Chest X-Ray and Radiology Report by Word Region Alignment",
abstract = "Self-supervised learning provides an opportunity to explore unlabeled chest X-rays and their associated free-text reports accumulated in clinical routine without manual supervision. This paper proposes a Joint Image Text Representation Learning Network (JoImTeRNet) for pre-training on chest X-ray images and their radiology reports. The model was pre-trained on both the global image-sentence level and the local image region-word level for visual-textual matching. Both are bidirectionally constrained on Cross-Entropy based and ranking-based Triplet Matching Losses. The region-word matching is calculated using the attention mechanism without direct supervision about their mapping. The pre-trained multi-modal representation learning paves the way for downstream tasks concerning image and/or text encoding. We demonstrate the representation learning quality by cross-modality retrievals and multi-label classifications on two datasets: OpenI-IU and MIMIC-CXR. Our code is available at https://github.com/mshaikh2/JoImTeR\_MLMI\_2021.",
keywords = "Attention, Multi-modality, Self-supervised learning",
author = "Zhanghexuan Ji and Shaikh, \{Mohammad Abuzar\} and Dana Moukheiber and Srihari, \{Sargur N.\} and Yifan Peng and Mingchen Gao",
note = "Publisher Copyright: {\textcopyright} 2021, Springer Nature Switzerland AG.; 12th International Workshop on Machine Learning in Medical Imaging, MLMI 2021, held in conjunction with 24th International Conference on Medical Image Computing and Computer Assisted Intervention, MICCAI 2021 ; Conference date: 27-09-2021 Through 27-09-2021",
year = "2021",
doi = "10.1007/978-3-030-87589-3\_12",
language = "English",
isbn = "9783030875886",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Science and Business Media Deutschland GmbH",
pages = "110--119",
editor = "Chunfeng Lian and Xiaohuan Cao and Islem Rekik and Xuanang Xu and Pingkun Yan",
booktitle = "Machine Learning in Medical Imaging - 12th International Workshop, MLMI 2021, Held in Conjunction with MICCAI 2021, Proceedings",
address = "Germany",
}