---
references:
- id: Bengio2025Superintelligent
  accessed:
    - year: 2025
      month: 2
      day: 25
  author:
    - family: Bengio
      given: Yoshua
    - family: Cohen
      given: Michael
    - family: Fornasiere
      given: Damiano
    - family: Ghosn
      given: Joumana
    - family: Greiner
      given: Pietro
    - family: MacDermott
      given: Matt
    - family: Mindermann
      given: Sören
    - family: Oberman
      given: Adam
    - family: Richardson
      given: Jesse
    - family: Richardson
      given: Oliver
    - family: Rondeau
      given: Marc-Antoine
    - family: St-Charles
      given: Pierre-Luc
    - family: Williams-King
      given: David
  citation-key: Bengio2025Superintelligent
  DOI: 10.48550/arXiv.2502.15657
  issued:
    - year: 2025
      month: 2
      day: 21
  number: arXiv:2502.15657
  publisher: arXiv
  title: >-
    Superintelligent Agents Pose Catastrophic Risks: Can Scientist AI Offer a
    Safer Path?
  type: article
  URL: http://arxiv.org/abs/2502.15657

- id: Castelfranchi1998Modelling
  accessed:
    - year: 2025
      month: 2
      day: 25
  author:
    - family: Castelfranchi
      given: Cristiano
  citation-key: Castelfranchi1998Modelling
  collection-title: Artificial Intelligence 40 years later
  container-title: Artificial Intelligence
  container-title-short: Artificial Intelligence
  DOI: 10.1016/S0004-3702(98)00056-3
  ISSN: 0004-3702
  issue: '1'
  issued:
    - year: 1998
      month: 8
      day: 1
  page: 157-182
  title: Modelling social action for AI agents
  type: article-journal
  volume: '103'

- id: Crutchfield2025Agentic
  accessed:
    - year: 2025
      month: 5
      day: 30
  author:
    - family: Crutchfield
      given: James P.
    - family: Jurgens
      given: Alexandra
  citation-key: Crutchfield2025Agentic
  DOI: 10.48550/arXiv.2505.19275
  issued:
    - year: 2025
      month: 5
      day: 25
  number: arXiv:2505.19275
  publisher: arXiv
  title: >-
    Agentic Information Theory: Ergodicity and Intrinsic Semantics of
    Information Processes
  type: article
  URL: http://arxiv.org/abs/2505.19275

- id: Hammond2025MultiAgent
  accessed:
    - year: 2025
      month: 3
      day: 17
  author:
    - family: Hammond
      given: Lewis
    - family: Chan
      given: Alan
    - family: Clifton
      given: Jesse
    - family: Hoelscher-Obermaier
      given: Jason
    - family: Khan
      given: Akbir
    - family: McLean
      given: Euan
    - family: Smith
      given: Chandler
    - family: Barfuss
      given: Wolfram
    - family: Foerster
      given: Jakob
    - family: Gavenčiak
      given: Tomáš
    - family: Han
      given: The Anh
    - family: Hughes
      given: Edward
    - family: Kovařík
      given: Vojtěch
    - family: Kulveit
      given: Jan
    - family: Leibo
      given: Joel Z.
    - family: Oesterheld
      given: Caspar
    - family: Witt
      given: Christian Schroeder
      dropping-particle: de
    - family: Shah
      given: Nisarg
    - family: Wellman
      given: Michael
    - family: Bova
      given: Paolo
    - family: Cimpeanu
      given: Theodor
    - family: Ezell
      given: Carson
    - family: Feuillade-Montixi
      given: Quentin
    - family: Franklin
      given: Matija
    - family: Kran
      given: Esben
    - family: Krawczuk
      given: Igor
    - family: Lamparth
      given: Max
    - family: Lauffer
      given: Niklas
    - family: Meinke
      given: Alexander
    - family: Motwani
      given: Sumeet
    - family: Reuel
      given: Anka
    - family: Conitzer
      given: Vincent
    - family: Dennis
      given: Michael
    - family: Gabriel
      given: Iason
    - family: Gleave
      given: Adam
    - family: Hadfield
      given: Gillian
    - family: Haghtalab
      given: Nika
    - family: Kasirzadeh
      given: Atoosa
    - family: Krier
      given: Sébastien
    - family: Larson
      given: Kate
    - family: Lehman
      given: Joel
    - family: Parkes
      given: David C.
    - family: Piliouras
      given: Georgios
    - family: Rahwan
      given: Iyad
  citation-key: Hammond2025MultiAgent
  DOI: 10.48550/arXiv.2502.14143
  issued:
    - year: 2025
      month: 2
      day: 19
  number: arXiv:2502.14143
  publisher: arXiv
  title: Multi-Agent Risks from Advanced AI
  type: article
  URL: http://arxiv.org/abs/2502.14143

- id: Johnson2019AI
  accessed:
    - year: 2025
      month: 2
      day: 25
  author:
    - family: Johnson
      given: Deborah G.
    - family: Verdicchio
      given: Mario
  citation-key: Johnson2019AI
  container-title: AI & SOCIETY
  container-title-short: AI & Soc
  DOI: 10.1007/s00146-017-0781-9
  ISSN: 1435-5655
  issue: '3'
  issued:
    - year: 2019
      month: 9
      day: 1
  language: en
  page: 639-647
  title: 'AI, agency and responsibility: the VW fraud case and beyond'
  type: article-journal
  volume: '34'

- id: Kang2022AI
  accessed:
    - year: 2025
      month: 2
      day: 25
  author:
    - family: Kang
      given: Hyunjin
    - family: Lou
      given: Chen
  citation-key: Kang2022AI
  container-title: Journal of Computer-Mediated Communication
  container-title-short: Journal of Computer-Mediated Communication
  DOI: 10.1093/jcmc/zmac014
  ISSN: 1083-6101
  issue: '5'
  issued:
    - year: 2022
      month: 9
      day: 1
  page: zmac014
  title: >-
    AI agency vs. human agency: understanding human–AI interactions on TikTok
    and their implications for user engagement
  type: article-journal
  volume: '27'

- id: Kenton2023Discovering
  accessed:
    - year: 2025
      month: 2
      day: 10
  author:
    - family: Kenton
      given: Zachary
    - family: Kumar
      given: Ramana
    - family: Farquhar
      given: Sebastian
    - family: Richens
      given: Jonathan
    - family: MacDermott
      given: Matt
    - family: Everitt
      given: Tom
  citation-key: Kenton2023Discovering
  container-title: Artificial Intelligence
  container-title-short: Artificial Intelligence
  DOI: 10.1016/j.artint.2023.103963
  ISSN: 0004-3702
  issued:
    - year: 2023
      month: 9
      day: 1
  page: '103963'
  title: Discovering agents
  type: article-journal
  URL: http://arxiv.org/abs/2208.08345
  volume: '322'

- id: Kulveit2025Gradual
  accessed:
    - year: 2025
      month: 3
      day: 3
  author:
    - family: Kulveit
      given: Jan
    - family: Douglas
      given: Raymond
    - family: Ammann
      given: Nora
    - family: Turan
      given: Deger
    - family: Krueger
      given: David
    - family: Duvenaud
      given: David
  citation-key: Kulveit2025Gradual
  DOI: 10.48550/arXiv.2501.16946
  issued:
    - year: 2025
      month: 1
      day: 29
  number: arXiv:2501.16946
  publisher: arXiv
  title: >-
    Gradual Disempowerment: Systemic Existential Risks from Incremental AI
    Development
  type: article
  URL: http://arxiv.org/abs/2501.16946

- id: Legaspi2019Synthetic
  accessed:
    - year: 2025
      month: 2
      day: 25
  author:
    - family: Legaspi
      given: Roberto
    - family: He
      given: Zhengqi
    - family: Toyoizumi
      given: Taro
  citation-key: Legaspi2019Synthetic
  collection-title: Artificial Intelligence
  container-title: Current Opinion in Behavioral Sciences
  container-title-short: Current Opinion in Behavioral Sciences
  DOI: 10.1016/j.cobeha.2019.04.004
  ISSN: 2352-1546
  issued:
    - year: 2019
      month: 10
      day: 1
  page: 84-90
  title: 'Synthetic agency: sense of agency in artificial intelligence'
  type: article-journal
  volume: '29'

- id: Liu2024Attaining
  accessed:
    - year: 2025
      month: 2
      day: 10
  author:
    - family: Liu
      given: Anjie
    - family: Wang
      given: Jianhong
    - family: Li
      given: Haoxuan
    - family: Chen
      given: Xu
    - family: Wang
      given: Jun
    - family: Kaski
      given: Samuel
    - family: Yang
      given: Mengyue
  citation-key: Liu2024Attaining
  DOI: 10.48550/arXiv.2405.16588
  issued:
    - year: 2024
      month: 5
      day: 26
  number: arXiv:2405.16588
  publisher: arXiv
  title: >-
    Attaining Human Desirable Outcomes in Human-AI Interaction via Structural
    Causal Games
  type: article
  URL: http://arxiv.org/abs/2405.16588

- id: MacDermott2024Measuring
  accessed:
    - year: 2025
      month: 2
      day: 10
  author:
    - family: MacDermott
      given: Matt
    - family: Fox
      given: James
    - family: Belardinelli
      given: Francesco
    - family: Everitt
      given: Tom
  citation-key: MacDermott2024Measuring
  DOI: 10.48550/arXiv.2412.04758
  issued:
    - year: 2024
      month: 12
      day: 6
  number: arXiv:2412.04758
  publisher: arXiv
  title: Measuring Goal-Directedness
  type: article
  URL: http://arxiv.org/abs/2412.04758

- id: Richens2024Robust
  accessed:
    - year: 2025
      month: 1
      day: 18
  author:
    - family: Richens
      given: Jonathan
    - family: Everitt
      given: Tom
  citation-key: Richens2024Robust
  DOI: 10.48550/arXiv.2402.10877
  issued:
    - year: 2024
      month: 7
      day: 19
  number: arXiv:2402.10877
  publisher: arXiv
  title: Robust agents learn causal world models
  type: article
  URL: http://arxiv.org/abs/2402.10877

- id: vanRijmenam2021Revising
  accessed:
    - year: 2025
      month: 2
      day: 25
  author:
    - family: Rijmenam
      given: Mark
      non-dropping-particle: van
    - family: Logue
      given: Danielle
  citation-key: vanRijmenam2021Revising
  container-title: Innovation
  DOI: 10.1080/14479338.2020.1816833
  ISSN: 1447-9338
  issue: '1'
  issued:
    - year: 2021
      month: 1
      day: 2
  page: 127-144
  publisher: Routledge
  title: >-
    Revising the ‘science of the organisation’: theorising AI agency and
    actorhood
  type: article-journal
  volume: '23'

- id: Ward2023Honesty
  accessed:
    - year: 2025
      month: 2
      day: 2
  author:
    - family: Ward
      given: Francis
    - family: Toni
      given: Francesca
    - family: Belardinelli
      given: Francesco
    - family: Everitt
      given: Tom
  citation-key: Ward2023Honesty
  container-title: Advances in Neural Information Processing Systems
  event-place: Red Hook, NY, USA
  issued:
    - year: 2023
      month: 12
      day: 15
  language: en
  page: 2313-2341
  publisher: Curran Associates Inc.
  publisher-place: Red Hook, NY, USA
  title: 'Honesty Is the Best Policy: Defining and Mitigating AI Deception'
  type: paper-conference
  URL: http://arxiv.org/abs/2312.01350
  volume: '36'

- id: Ward2024Reasons
  accessed:
    - year: 2025
      month: 1
      day: 18
  author:
    - family: Ward
      given: Francis Rhys
    - family: MacDermott
      given: Matt
    - family: Belardinelli
      given: Francesco
    - family: Toni
      given: Francesca
    - family: Everitt
      given: Tom
  citation-key: Ward2024Reasons
  DOI: 10.48550/arXiv.2402.07221
  issued:
    - year: 2024
      month: 2
      day: 15
  number: arXiv:2402.07221
  publisher: arXiv
  title: 'The Reasons that Agents Act: Intention and Instrumental Goals'
  type: article
  URL: http://arxiv.org/abs/2402.07221

- id: Zhuang2021Consequences
  accessed:
    - year: 2023
      month: 4
      day: 30
  author:
    - family: Zhuang
      given: Simon
    - family: Hadfield-Menell
      given: Dylan
  citation-key: Zhuang2021Consequences
  DOI: 10.48550/arXiv.2102.03896
  issued:
    - year: 2021
      month: 2
      day: 7
  number: arXiv:2102.03896
  publisher: arXiv
  title: Consequences of Misaligned AI
  type: article
  URL: http://arxiv.org/abs/2102.03896
...
