---
references:
- id: BahdanauNeural2015
  accessed:
    - year: 2023
      month: 3
      day: 21
  author:
    - family: Bahdanau
      given: Dzmitry
    - family: Cho
      given: Kyunghyun
    - family: Bengio
      given: Yoshua
  citation-key: BahdanauNeural2015
  event-title: ICLR
  issued:
    - year: 2015
  language: en
  publisher: arXiv
  title: Neural Machine Translation by Jointly Learning to Align and Translate
  type: paper-conference
  URL: http://arxiv.org/abs/1409.0473

- id: BenderDangers2021
  accessed:
    - year: 2023
      month: 2
      day: 13
  author:
    - family: Bender
      given: Emily M.
    - family: Gebru
      given: Timnit
    - family: McMillan-Major
      given: Angelina
    - family: Shmitchell
      given: Shmargaret
  citation-key: BenderDangers2021
  container-title: >-
    Proceedings of the 2021 ACM Conference on Fairness, Accountability, and
    Transparency
  DOI: 10.1145/3442188.3445922
  event-place: Virtual Event Canada
  event-title: 'FAccT ''21: 2021 ACM Conference on Fairness, Accountability, and Transparency'
  ISBN: 978-1-4503-8309-7
  issued:
    - year: 2021
      month: 3
      day: 3
  language: en
  page: 610-623
  publisher: ACM
  publisher-place: Virtual Event Canada
  title: 'On the Dangers of Stochastic Parrots: Can Language Models Be Too Big? 🦜'
  type: paper-conference

- id: BeurerKellnerPrompting2022
  accessed:
    - year: 2023
      month: 4
      day: 25
  author:
    - family: Beurer-Kellner
      given: Luca
    - family: Fischer
      given: Marc
    - family: Vechev
      given: Martin
  citation-key: BeurerKellnerPrompting2022
  issued:
    - year: 2022
      month: 12
      day: 12
  language: en
  number: arXiv:2212.06094
  publisher: arXiv
  title: 'Prompting Is Programming: A Query Language For Large Language Models'
  type: article
  URL: http://arxiv.org/abs/2212.06094

- id: BrownLanguage2020
  accessed:
    - year: 2020
      month: 6
      day: 7
  author:
    - family: Brown
      given: Tom B.
    - family: Mann
      given: Benjamin
    - family: Ryder
      given: Nick
    - family: Subbiah
      given: Melanie
    - family: Kaplan
      given: Jared
    - family: Dhariwal
      given: Prafulla
    - family: Neelakantan
      given: Arvind
    - family: Shyam
      given: Pranav
    - family: Sastry
      given: Girish
    - family: Askell
      given: Amanda
    - family: Agarwal
      given: Sandhini
    - family: Herbert-Voss
      given: Ariel
    - family: Krueger
      given: Gretchen
    - family: Henighan
      given: Tom
    - family: Child
      given: Rewon
    - family: Ramesh
      given: Aditya
    - family: Ziegler
      given: Daniel M.
    - family: Wu
      given: Jeffrey
    - family: Winter
      given: Clemens
    - family: Hesse
      given: Christopher
    - family: Chen
      given: Mark
    - family: Sigler
      given: Eric
    - family: Litwin
      given: Mateusz
    - family: Gray
      given: Scott
    - family: Chess
      given: Benjamin
    - family: Clark
      given: Jack
    - family: Berner
      given: Christopher
    - family: McCandlish
      given: Sam
    - family: Radford
      given: Alec
    - family: Sutskever
      given: Ilya
    - family: Amodei
      given: Dario
  citation-key: BrownLanguage2020
  container-title: arXiv:2005.14165 [cs]
  issued:
    - year: 2020
      month: 6
      day: 1
  title: Language Models are Few-Shot Learners
  type: article-journal
  URL: http://arxiv.org/abs/2005.14165

- id: BubeckSparks2023
  accessed:
    - year: 2023
      month: 3
      day: 28
  author:
    - family: Bubeck
      given: Sébastien
    - family: Chandrasekaran
      given: Varun
    - family: Eldan
      given: Ronen
    - family: Gehrke
      given: Johannes
    - family: Horvitz
      given: Eric
    - family: Kamar
      given: Ece
    - family: Lee
      given: Peter
    - family: Lee
      given: Yin Tat
    - family: Li
      given: Yuanzhi
    - family: Lundberg
      given: Scott
    - family: Nori
      given: Harsha
    - family: Palangi
      given: Hamid
    - family: Ribeiro
      given: Marco Tulio
    - family: Zhang
      given: Yi
  citation-key: BubeckSparks2023
  DOI: 10.48550/arXiv.2303.12712
  issued:
    - year: 2023
      month: 3
      day: 24
  number: arXiv:2303.12712
  publisher: arXiv
  title: 'Sparks of Artificial General Intelligence: Early experiments with GPT-4'
  type: article
  URL: http://arxiv.org/abs/2303.12712

- id: CaoChoose2021
  accessed:
    - year: 2023
      month: 8
      day: 11
  author:
    - family: Cao
      given: Shuhao
  citation-key: CaoChoose2021
  container-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2021
  page: 24924–24940
  publisher: Curran Associates, Inc.
  title: 'Choose a Transformer: Fourier or Galerkin'
  type: paper-conference
  URL: http://arxiv.org/abs/2105.14995
  volume: '34'

- id: CelikyilmazScaffolding2017
  author:
    - family: Celikyilmaz
      given: Asli
    - family: Deng
      given: Li
    - family: Li
      given: Lihong
    - family: Wang
      given: Chong
  citation-key: CelikyilmazScaffolding2017
  container-title: arXiv:1702.08653 [cs]
  issued:
    - year: 2017
      month: 2
      day: 28
  title: Scaffolding Networks for Teaching and Learning to Comprehend
  type: article-journal
  URL: http://arxiv.org/abs/1702.08653

- id: ChoyUniversal2016
  author:
    - family: Choy
      given: Christopher B
    - family: Gwak
      given: JunYoung
    - family: Savarese
      given: Silvio
    - family: Chandraker
      given: Manmohan
  citation-key: ChoyUniversal2016
  container-title: Advances in Neural Information Processing Systems 29
  editor:
    - family: Lee
      given: D. D.
    - family: Sugiyama
      given: M.
    - family: Luxburg
      given: U. V.
    - family: Guyon
      given: I.
    - family: Garnett
      given: R.
  issued:
    - year: 2016
  page: 2406–2414
  publisher: Curran Associates, Inc.
  title: Universal Correspondence Network
  type: paper-conference
  URL: http://papers.nips.cc/paper/6487-universal-correspondence-network.pdf

- id: DinJump2023
  accessed:
    - year: 2023
      month: 3
      day: 28
  author:
    - family: Din
      given: Alexander Yom
    - family: Karidi
      given: Taelin
    - family: Choshen
      given: Leshem
    - family: Geva
      given: Mor
  citation-key: DinJump2023
  issued:
    - year: 2023
      month: 3
      day: 16
  language: en
  title: 'Jump to Conclusions: Short-Cutting Transformers With Linear Transformations'
  type: article
  URL: https://arxiv.org/abs/2303.09435v1

- id: ErgenConvexifying2022
  accessed:
    - year: 2023
      month: 5
      day: 28
  author:
    - family: Ergen
      given: Tolga
    - family: Neyshabur
      given: Behnam
    - family: Mehta
      given: Harsh
  citation-key: ErgenConvexifying2022
  DOI: 10.48550/arXiv.2211.11052
  issued:
    - year: 2022
      month: 11
      day: 20
  number: arXiv:2211.11052
  publisher: arXiv
  title: >-
    Convexifying Transformers: Improving optimization and understanding of
    transformer networks
  type: article
  URL: http://arxiv.org/abs/2211.11052

- id: FreemanHow2019
  accessed:
    - year: 2020
      month: 3
      day: 28
  author:
    - family: Freeman
      given: Alexandra L J
  citation-key: FreemanHow2019
  container-title: Drug and Therapeutics Bulletin
  container-title-short: DTB
  DOI: 10.1136/dtb.2019.000008
  ISSN: 0012-6543, 1755-5248
  issue: '8'
  issued:
    - year: 2019
      month: 8
  language: en
  page: 119-124
  title: How to communicate evidence to patients
  type: article-journal
  volume: '57'

- id: HuangMusic2018
  accessed:
    - year: 2019
      month: 9
      day: 26
  author:
    - family: Huang
      given: Cheng-Zhi Anna
    - family: Vaswani
      given: Ashish
    - family: Uszkoreit
      given: Jakob
    - family: Simon
      given: Ian
    - family: Hawthorne
      given: Curtis
    - family: Shazeer
      given: Noam
    - family: Dai
      given: Andrew M.
    - family: Hoffman
      given: Matthew D.
    - family: Dinculescu
      given: Monica
    - family: Eck
      given: Douglas
  citation-key: HuangMusic2018
  issued:
    - year: 2018
      month: 9
      day: 27
  title: 'Music Transformer: Generating Music with Long-Term Structure'
  type: article-journal
  URL: https://openreview.net/forum?id=rJe4ShAcF7

- id: KatharopoulosTransformers2020
  accessed:
    - year: 2020
      month: 9
      day: 16
  author:
    - family: Katharopoulos
      given: Angelos
    - family: Vyas
      given: Apoorv
    - family: Pappas
      given: Nikolaos
    - family: Fleuret
      given: François
  citation-key: KatharopoulosTransformers2020
  container-title: arXiv:2006.16236 [cs, stat]
  issued:
    - year: 2020
      month: 8
      day: 31
  title: >-
    Transformers are RNNs: Fast Autoregressive Transformers with Linear
    Attention
  type: article-journal
  URL: http://arxiv.org/abs/2006.16236

- id: LiTrain2020
  accessed:
    - year: 2020
      month: 3
      day: 28
  author:
    - family: Li
      given: Zhuohan
    - family: Wallace
      given: Eric
    - family: Shen
      given: Sheng
    - family: Lin
      given: Kevin
    - family: Keutzer
      given: Kurt
    - family: Klein
      given: Dan
    - family: Gonzalez
      given: Joseph E.
  citation-key: LiTrain2020
  container-title: arXiv:2002.11794 [cs]
  issued:
    - year: 2020
      month: 2
      day: 26
  title: >-
    Train Large, Then Compress: Rethinking Model Size for Efficient Training and
    Inference of Transformers
  type: article-journal
  URL: http://arxiv.org/abs/2002.11794

- id: MerrillTransformers2022
  accessed:
    - year: 2022
      month: 10
      day: 9
  author:
    - family: Merrill
      given: William
    - family: Sabharwal
      given: Ashish
  citation-key: MerrillTransformers2022
  DOI: 10.48550/arXiv.2210.02671
  issued:
    - year: 2022
      month: 10
      day: 6
  number: arXiv:2210.02671
  publisher: arXiv
  title: Transformers Implement First-Order Logic with Majority Quantifiers
  type: article
  URL: http://arxiv.org/abs/2210.02671

- id: MullerTransformers2022
  accessed:
    - year: 2022
      month: 8
      day: 23
  author:
    - family: Müller
      given: Samuel
    - family: Hollmann
      given: Noah
    - family: Arango
      given: Sebastian Pineda
    - family: Grabocka
      given: Josif
    - family: Hutter
      given: Frank
  citation-key: MullerTransformers2022
  DOI: 10.48550/arXiv.2112.10510
  issued:
    - year: 2022
      month: 8
      day: 12
  number: arXiv:2112.10510
  publisher: arXiv
  title: Transformers Can Do Bayesian Inference
  type: article
  URL: http://arxiv.org/abs/2112.10510

- id: NguyenClimaX2023
  accessed:
    - year: 2023
      month: 2
      day: 7
  author:
    - family: Nguyen
      given: Tung
    - family: Brandstetter
      given: Johannes
    - family: Kapoor
      given: Ashish
    - family: Gupta
      given: Jayesh K.
    - family: Grover
      given: Aditya
  citation-key: NguyenClimaX2023
  DOI: 10.48550/arXiv.2301.10343
  issued:
    - year: 2023
      month: 1
      day: 24
  number: arXiv:2301.10343
  publisher: arXiv
  title: 'ClimaX: A foundation model for weather and climate'
  type: article
  URL: http://arxiv.org/abs/2301.10343

- id: OrtegaShaking2021
  accessed:
    - year: 2021
      month: 10
      day: 26
  author:
    - family: Ortega
      given: Pedro A.
    - family: Kunesch
      given: Markus
    - family: Delétang
      given: Grégoire
    - family: Genewein
      given: Tim
    - family: Grau-Moya
      given: Jordi
    - family: Veness
      given: Joel
    - family: Buchli
      given: Jonas
    - family: Degrave
      given: Jonas
    - family: Piot
      given: Bilal
    - family: Perolat
      given: Julien
    - family: Everitt
      given: Tom
    - family: Tallec
      given: Corentin
    - family: Parisotto
      given: Emilio
    - family: Erez
      given: Tom
    - family: Chen
      given: Yutian
    - family: Reed
      given: Scott
    - family: Hutter
      given: Marcus
    - family: Freitas
      given: Nando
      non-dropping-particle: de
    - family: Legg
      given: Shane
  citation-key: OrtegaShaking2021
  container-title: arXiv:2110.10819 [cs]
  issued:
    - year: 2021
      month: 10
      day: 20
  language: en
  title: >-
    Shaking the foundations: delusions in sequence models for interaction and
    control
  type: article-journal
  URL: http://arxiv.org/abs/2110.10819

- id: PhuongFormal2022
  accessed:
    - year: 2022
      month: 8
      day: 5
  author:
    - family: Phuong
      given: Mary
    - family: Hutter
      given: Marcus
  citation-key: PhuongFormal2022
  issued:
    - year: 2022
      month: 7
      day: 19
  language: en
  number: arXiv:2207.09238
  publisher: arXiv
  title: Formal Algorithms for Transformers
  type: article
  URL: http://arxiv.org/abs/2207.09238

- id: PiantadosiMeaning2022
  accessed:
    - year: 2022
      month: 8
      day: 27
  author:
    - family: Piantadosi
      given: Steven T.
    - family: Hill
      given: Felix
  citation-key: PiantadosiMeaning2022
  DOI: 10.48550/arXiv.2208.02957
  issued:
    - year: 2022
      month: 8
      day: 12
  number: arXiv:2208.02957
  publisher: arXiv
  title: Meaning without reference in large language models
  type: article
  URL: http://arxiv.org/abs/2208.02957

- id: RadfordLanguage2019
  author:
    - family: Radford
      given: Alec
    - family: Wu
      given: Jeffrey
    - family: Child
      given: Rewon
    - family: Luan
      given: David
    - family: Amodei
      given: Dario
    - family: Sutskever
      given: Ilya
  citation-key: RadfordLanguage2019
  issued:
    - year: 2019
  language: en
  page: '24'
  title: Language Models are Unsupervised Multitask Learners
  type: article-journal

- id: RamsauerHopfield2020
  accessed:
    - year: 2020
      month: 8
      day: 13
  author:
    - family: Ramsauer
      given: Hubert
    - family: Schäfl
      given: Bernhard
    - family: Lehner
      given: Johannes
    - family: Seidl
      given: Philipp
    - family: Widrich
      given: Michael
    - family: Gruber
      given: Lukas
    - family: Holzleitner
      given: Markus
    - family: Pavlović
      given: Milena
    - family: Sandve
      given: Geir Kjetil
    - family: Greiff
      given: Victor
    - family: Kreil
      given: David
    - family: Kopp
      given: Michael
    - family: Klambauer
      given: Günter
    - family: Brandstetter
      given: Johannes
    - family: Hochreiter
      given: Sepp
  citation-key: RamsauerHopfield2020
  container-title: arXiv:2008.02217 [cs, stat]
  issued:
    - year: 2020
      month: 7
      day: 16
  title: Hopfield Networks is All You Need
  type: article-journal
  URL: http://arxiv.org/abs/2008.02217

- id: SerranoLanguage2023
  accessed:
    - year: 2024
      month: 1
      day: 11
  author:
    - family: Serrano
      given: Sofia
    - family: Brumbaugh
      given: Zander
    - family: Smith
      given: Noah A.
  citation-key: SerranoLanguage2023
  DOI: 10.48550/arXiv.2311.17301
  issued:
    - year: 2023
      month: 11
      day: 28
  number: arXiv:2311.17301
  publisher: arXiv
  title: 'Language Models: A Guide for the Perplexed'
  type: article
  URL: http://arxiv.org/abs/2311.17301

- id: VardasbiState2023
  accessed:
    - year: 2023
      month: 5
      day: 22
  author:
    - family: Vardasbi
      given: Ali
    - family: Pires
      given: Telmo Pessoa
    - family: Schmidt
      given: Robin M.
    - family: Peitz
      given: Stephan
  citation-key: VardasbiState2023
  DOI: 10.48550/arXiv.2304.12776
  issued:
    - year: 2023
      month: 4
      day: 25
  number: arXiv:2304.12776
  publisher: arXiv
  title: 'State Spaces Aren''t Enough: Machine Translation Needs Attention'
  type: article
  URL: http://arxiv.org/abs/2304.12776

- id: VaswaniAttention2017
  accessed:
    - year: 2017
      month: 9
      day: 6
  author:
    - family: Vaswani
      given: Ashish
    - family: Shazeer
      given: Noam
    - family: Parmar
      given: Niki
    - family: Uszkoreit
      given: Jakob
    - family: Jones
      given: Llion
    - family: Gomez
      given: Aidan N.
    - family: Kaiser
      given: Lukasz
    - family: Polosukhin
      given: Illia
  citation-key: VaswaniAttention2017
  container-title: arXiv:1706.03762 [cs]
  issued:
    - year: 2017
      month: 6
      day: 12
  title: Attention Is All You Need
  type: article-journal
  URL: http://arxiv.org/abs/1706.03762

- id: WangMambaByte2024
  accessed:
    - year: 2024
      month: 3
      day: 5
  author:
    - family: Wang
      given: Junxiong
    - family: Gangavarapu
      given: Tushaar
    - family: Yan
      given: Jing Nathan
    - family: Rush
      given: Alexander M.
  citation-key: WangMambaByte2024
  DOI: 10.48550/arXiv.2401.13660
  issued:
    - year: 2024
      month: 1
      day: 24
  number: arXiv:2401.13660
  publisher: arXiv
  title: 'MambaByte: Token-free Selective State Space Model'
  type: article
  URL: http://arxiv.org/abs/2401.13660

- id: WilligCan2022
  accessed:
    - year: 2023
      month: 8
      day: 1
  author:
    - family: Willig
      given: Moritz
    - family: Zečević
      given: Matej
    - family: Dhami
      given: Devendra Singh
    - family: Kersting
      given: Kristian
  citation-key: WilligCan2022
  DOI: 10.48550/arXiv.2206.10591
  issued:
    - year: 2022
      month: 12
      day: 23
  number: arXiv:2206.10591
  publisher: arXiv
  title: Can Foundation Models Talk Causality?
  type: article
  URL: http://arxiv.org/abs/2206.10591

- id: WuLanguage2024
  accessed:
    - year: 2024
      month: 3
      day: 5
  author:
    - family: Wu
      given: Shangda
    - family: Tan
      given: Xu
    - family: Wang
      given: Zili
    - family: Wang
      given: Rui
    - family: Li
      given: Xiaobing
    - family: Sun
      given: Maosong
  citation-key: WuLanguage2024
  DOI: 10.48550/arXiv.2402.19155
  issued:
    - year: 2024
      month: 2
      day: 29
  number: arXiv:2402.19155
  publisher: arXiv
  title: 'Beyond Language Models: Byte Models are Digital World Simulators'
  type: article
  URL: http://arxiv.org/abs/2402.19155

- id: YangFeature2020
  accessed:
    - year: 2021
      month: 1
      day: 4
  author:
    - family: Yang
      given: Greg
    - family: Hu
      given: Edward J.
  citation-key: YangFeature2020
  container-title: arXiv:2011.14522 [cond-mat]
  issued:
    - year: 2020
      month: 11
      day: 29
  title: Feature Learning in Infinite-Width Neural Networks
  type: article-journal
  URL: http://arxiv.org/abs/2011.14522

- id: ZhaiAttention2021
  accessed:
    - year: 2023
      month: 5
      day: 22
  author:
    - family: Zhai
      given: Shuangfei
    - family: Talbott
      given: Walter
    - family: Srivastava
      given: Nitish
    - family: Huang
      given: Chen
    - family: Goh
      given: Hanlin
    - family: Zhang
      given: Ruixiang
    - family: Susskind
      given: Josh
  citation-key: ZhaiAttention2021
  DOI: 10.48550/arXiv.2105.14103
  issued:
    - year: 2021
      month: 9
      day: 21
  number: arXiv:2105.14103
  publisher: arXiv
  title: An Attention Free Transformer
  type: article
  URL: http://arxiv.org/abs/2105.14103

- id: ZhangUnveiling2022
  accessed:
    - year: 2022
      month: 6
      day: 22
  author:
    - family: Zhang
      given: Yi
    - family: Backurs
      given: Arturs
    - family: Bubeck
      given: Sébastien
    - family: Eldan
      given: Ronen
    - family: Gunasekar
      given: Suriya
    - family: Wagner
      given: Tal
  citation-key: ZhangUnveiling2022
  DOI: 10.48550/arXiv.2206.04301
  issued:
    - year: 2022
      month: 6
      day: 9
  number: arXiv:2206.04301
  publisher: arXiv
  title: 'Unveiling Transformers with LEGO: a synthetic reasoning task'
  type: article
  URL: http://arxiv.org/abs/2206.04301

- id: ZhouRecurrentGPT2023
  accessed:
    - year: 2023
      month: 5
      day: 23
  author:
    - family: Zhou
      given: Wangchunshu
    - family: Jiang
      given: Yuchen Eleanor
    - family: Cui
      given: Peng
    - family: Wang
      given: Tiannan
    - family: Xiao
      given: Zhenxin
    - family: Hou
      given: Yifan
    - family: Cotterell
      given: Ryan
    - family: Sachan
      given: Mrinmaya
  citation-key: ZhouRecurrentGPT2023
  DOI: 10.48550/arXiv.2305.13304
  issued:
    - year: 2023
      month: 5
      day: 22
  number: arXiv:2305.13304
  publisher: arXiv
  title: 'RecurrentGPT: Interactive Generation of (Arbitrarily) Long Text'
  type: article
  URL: http://arxiv.org/abs/2305.13304

- id: ZouUniversal2023
  accessed:
    - year: 2023
      month: 8
      day: 8
  author:
    - family: Zou
      given: Andy
    - family: Wang
      given: Zifan
    - family: Kolter
      given: J. Zico
    - family: Fredrikson
      given: Matt
  citation-key: ZouUniversal2023
  DOI: 10.48550/arXiv.2307.15043
  issued:
    - year: 2023
      month: 7
      day: 27
  number: arXiv:2307.15043
  publisher: arXiv
  title: Universal and Transferable Adversarial Attacks on Aligned Language Models
  type: article
  URL: http://arxiv.org/abs/2307.15043
...
