@inproceedings{e28fd4a1d8534571b746ea677831af4a,
title = "Practical Computational Power of Linear Transformers and Their Recurrent and Self-Referential Extensions",
abstract = "Recent studies of the computational power of recurrent neural networks (RNNs) reveal a hierarchy of RNN architectures, given real-time and finite-precision assumptions. Here we study auto-regressive Transformers with linearised attention, a.k.a. linear Transformers (LTs) or Fast Weight Programmers (FWPs). LTs are special in the sense that they are equivalent to RNN-like sequence processors with a fixed-size state, while they can also be expressed as the now-popular self-attention networks. We show that many well-known results for the standard Transformer directly transfer to LTs/FWPs. Our formal language recognition experiments demonstrate how recently proposed FWP extensions such as recurrent FWPs and self-referential weight matrices successfully overcome certain limitations of the LT, e.g., allowing for generalisation on the parity problem. Our code is public.",
author = "Kazuki Irie and R{\'o}bert Csord{\'a}s and J{\"u}rgen Schmidhuber",
note = "Publisher Copyright: {\textcopyright} 2023 Association for Computational Linguistics.; 2023 Conference on Empirical Methods in Natural Language Processing, EMNLP 2023 ; Conference date: 06-12-2023 Through 10-12-2023",
year = "2023",
doi = "10.18653/v1/2023.emnlp-main.588",
language = "English (US)",
series = "EMNLP 2023 - 2023 Conference on Empirical Methods in Natural Language Processing, Proceedings",
publisher = "Association for Computational Linguistics (ACL)",
pages = "9455--9465",
editor = "Houda Bouamor and Juan Pino and Kalika Bali",
booktitle = "EMNLP 2023 - 2023 Conference on Empirical Methods in Natural Language Processing, Proceedings",
address = "United States",
}