@inproceedings{5cf21b4d5b2b4e23a176cd6ceb69efc9,
title = "3DRefTransformer: Fine-Grained Object Identification in Real-World Scenes Using Natural Language",
abstract = "In this paper, we study fine-grained 3D object identification in real-world scenes described by a textual query. The task aims to discriminatively understand an instance of a particular 3D object described by natural language utterances among other instances of 3D objects of the same class appearing in a visual scene. We introduce the 3DRefTransformer net, a transformer-based neural network that identifies 3D objects described by linguistic utterances in real-world scenes. The network's input is 3D object segmented point cloud images representing a real-world scene and a language utterance that refers to one of the scene objects. The goal is to identify the referred object. Compared to the state-of-the-art models that are mostly based on graph convolutions and LSTMs, our 3DRefTrans-former net offers two key advantages. First, it is an end-to-end transformer model that operates both on language and 3D visual objects. Second, it has a natural ability to ground textual terms in the utterance to the learning representation of 3D objects in the scene. We further incorporate object pairwise spatial relation loss and contrastive learning during model training. We show in our experiments that our model improves the performance upon the current SOTA significantly on Referit3D Nr3D and Sr3D datasets. Code and Models will be made publicly available at https://vision-cair.github.io/3dreftransformer/.",
keywords = "3D Computer Vision Large-scale Vision Applications, Analysis and Understanding, Object Detection/Recognition/Categorization, Scene Understanding, Vision and Languages, Visual Reasoning",
author = "Ahmed Abdelreheem and Ujjwal Upadhyay and Ivan Skorokhodov and {Al Yahya}, Rawan and Jun Chen and Mohamed Elhoseiny",
note = "Publisher Copyright: {\textcopyright} 2022 IEEE.; 22nd IEEE/CVF Winter Conference on Applications of Computer Vision, WACV 2022 ; Conference date: 04-01-2022 Through 08-01-2022",
year = "2022",
doi = "10.1109/WACV51458.2022.00068",
language = "English (US)",
series = "Proceedings - 2022 IEEE/CVF Winter Conference on Applications of Computer Vision, WACV 2022",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "607--616",
booktitle = "Proceedings - 2022 IEEE/CVF Winter Conference on Applications of Computer Vision, WACV 2022",
address = "United States",
}