---
references:
- id: Aguirre2020AI
  accessed:
    - year: 2025
      month: 1
      day: 26
  author:
    - family: Aguirre
      given: Anthony
    - family: Dempsey
      given: Gaia
    - family: Surden
      given: Harry
    - family: Reiner
      given: Peter B.
  citation-key: Aguirre2020AI
  container-title: IEEE Transactions on Technology and Society
  container-title-short: IEEE Trans. Technol. Soc.
  DOI: 10.1109/TTS.2020.3013490
  ISSN: 2637-6415
  issue: '3'
  issued:
    - year: 2020
      month: 9
  language: en
  page: 128-137
  title: 'AI Loyalty: A New Paradigm for Aligning Stakeholder Interests'
  type: article-journal
  volume: '1'

- id: Aktipis2016Principles
  accessed:
    - year: 2023
      month: 9
      day: 16
  author:
    - family: Aktipis
      given: Athena
  citation-key: Aktipis2016Principles
  container-title: Evolutionary Applications
  DOI: 10.1111/eva.12303
  ISSN: 1752-4571
  issue: '1'
  issued:
    - year: 2016
  language: en
  page: 17-36
  title: >-
    Principles of cooperation across systems: from human sharing to
    multicellularity and cancer
  type: article-journal
  volume: '9'

- id: Bostrom2014Superintelligence
  author:
    - family: Bostrom
      given: Nick
  citation-key: Bostrom2014Superintelligence
  event-place: Oxford, New York
  ISBN: 978-0-19-967811-2
  issued:
    - year: 2014
      month: 7
      day: 3
  number-of-pages: '352'
  publisher: Oxford University Press
  publisher-place: Oxford, New York
  title: 'Superintelligence: Paths, Dangers, Strategies'
  type: book

- id: Conitzer2023Foundations
  accessed:
    - year: 2025
      month: 4
      day: 18
  author:
    - family: Conitzer
      given: Vincent
    - family: Oesterheld
      given: Caspar
  citation-key: Conitzer2023Foundations
  container-title: Proceedings of the AAAI Conference on Artificial Intelligence
  DOI: 10.1609/aaai.v37i13.26791
  ISSN: 2374-3468
  issue: '13'
  issued:
    - year: 2023
  language: en
  number: '13'
  page: 15359-15367
  title: Foundations of Cooperative AI
  type: article-journal
  URL: https://ojs.aaai.org/index.php/AAAI/article/view/26791
  volume: '37'

- id: Critch2022Cooperative
  accessed:
    - year: 2025
      month: 4
      day: 14
  author:
    - family: Critch
      given: Andrew
    - family: Dennis
      given: Michael
    - family: Russell
      given: Stuart
  citation-key: Critch2022Cooperative
  DOI: 10.48550/arXiv.2208.07006
  issued:
    - year: 2022
      month: 8
      day: 15
  number: arXiv:2208.07006
  publisher: arXiv
  title: >-
    Cooperative and uncooperative institution designs: Surprises and problems in
    open-source game theory
  type: article
  URL: http://arxiv.org/abs/2208.07006

- id: Daskalakis2013Mechanism
  accessed:
    - year: 2018
      month: 6
      day: 25
  author:
    - family: Daskalakis
      given: Constantinos
    - family: Deckelbaum
      given: Alan
    - family: Tzamos
      given: Christos
  citation-key: Daskalakis2013Mechanism
  DOI: 10.1145/2492002.2482593
  ISBN: 978-1-4503-1962-1
  issued:
    - year: 2013
  language: en
  page: '269'
  publisher: ACM Press
  title: Mechanism design via optimal transport
  type: paper-conference
  URL: http://people.csail.mit.edu/costis/ec54-daskalakis-full.pdf

- id: Duque2024Advantage
  accessed:
    - year: 2025
      month: 4
      day: 17
  author:
    - family: Duque
      given: Juan Agustin
    - family: Aghajohari
      given: Milad
    - family: Cooijmans
      given: Tim
    - family: Ciuca
      given: Razvan
    - family: Zhang
      given: Tianyu
    - family: Gidel
      given: Gauthier
    - family: Courville
      given: Aaron
  citation-key: Duque2024Advantage
  event-title: The Thirteenth International Conference on Learning Representations
  issued:
    - year: 2024
      month: 10
      day: 4
  language: en
  title: Advantage Alignment Algorithms
  type: paper-conference
  URL: https://openreview.net/forum?id=QFO1asgas2

- id: Ecoffet2021Reinforcement
  accessed:
    - year: 2025
      month: 1
      day: 15
  author:
    - family: Ecoffet
      given: Adrien
    - family: Lehman
      given: Joel
  citation-key: Ecoffet2021Reinforcement
  container-title: Proceedings of the 38th International Conference on Machine Learning
  event-title: International Conference on Machine Learning
  ISSN: 2640-3498
  issued:
    - year: 2021
      month: 7
      day: 1
  language: en
  page: 2926-2936
  publisher: PMLR
  title: Reinforcement Learning Under Moral Uncertainty
  type: paper-conference
  URL: http://arxiv.org/abs/2006.04734

- id: Guha2023AI
  accessed:
    - year: 2024
      month: 2
      day: 27
  author:
    - family: Guha
      given: Neel
    - family: Lawrence
      given: Christie
    - family: Gailmard
      given: Lindsey A.
    - family: Rodolfa
      given: Kit
    - family: Surani
      given: Faiz
    - family: Bommasani
      given: Rishi
    - family: Raji
      given: Inioluwa
    - family: Cuéllar
      given: Mariano-Florentino
    - family: Honigsberg
      given: Colleen
    - family: Liang
      given: Percy
    - family: Ho
      given: Daniel E.
  citation-key: Guha2023AI
  container-title: George Washington Law Review, Forthcoming
  issued:
    - year: 2023
      month: 11
      day: 15
  language: en
  title: >-
    AI Regulation Has Its Own Alignment Problem: The Technical and Institutional
    Feasibility of Disclosure, Registration, Licensing, and Auditing
  type: article-journal
  URL: https://dho.stanford.edu/wp-content/uploads/AI_Regulation.pdf

- id: HadfieldMenell2018Incomplete
  accessed:
    - year: 2025
      month: 4
      day: 14
  author:
    - family: Hadfield-Menell
      given: Dylan
    - family: Hadfield
      given: Gillian
  citation-key: HadfieldMenell2018Incomplete
  DOI: 10.48550/arXiv.1804.04268
  issued:
    - year: 2018
      month: 4
      day: 12
  number: arXiv:1804.04268
  publisher: arXiv
  title: Incomplete Contracting and AI Alignment
  type: article
  URL: http://arxiv.org/abs/1804.04268

- id: Hutson2022Taught
  accessed:
    - year: 2022
      month: 6
      day: 22
  author:
    - family: Hutson
      given: Matthew
  citation-key: Hutson2022Taught
  container-title: Science
  container-title-short: Science
  DOI: 10.1126/science.abq7833
  ISSN: 0036-8075, 1095-9203
  issue: '6593'
  issued:
    - year: 2022
      month: 5
      day: 6
  language: en
  page: 570-573
  title: Taught to the test
  type: article-journal
  volume: '376'

- id: Jackson2014Mechanism
  accessed:
    - year: 2017
      month: 3
      day: 6
  author:
    - family: Jackson
      given: Matthew O.
  citation-key: Jackson2014Mechanism
  event-place: Rochester, NY
  genre: SSRN Scholarly Paper
  issued:
    - year: 2014
  number: ID 2542983
  publisher: Social Science Research Network
  publisher-place: Rochester, NY
  title: Mechanism theory
  type: report
  URL: https://web.stanford.edu/~jacksonm/mechtheo.pdf

- id: KorinekDirect
  author:
    - family: Korinek
      given: Anton
    - family: Fellow
      given: Rubenstein
    - family: Balwit
      given: Avital
    - family: Scholar
      given: Rhodes
  citation-key: KorinekDirect
  language: en
  title: Direct and social goals for AI systems
  type: article-journal

- id: Lambrecht2017Dynamics
  accessed:
    - year: 2023
      month: 12
      day: 22
  author:
    - family: Lambrecht
      given: Bart M.
    - family: Myers
      given: Stewart C.
  citation-key: Lambrecht2017Dynamics
  container-title: The Review of Financial Studies
  DOI: 10.1093/rfs/hhx081
  ISSN: 0893-9454, 1465-7368
  issue: '11'
  issued:
    - year: 2017
      month: 11
      day: 1
  language: en
  page: 3759-3800
  title: The Dynamics of Investment, Payout and Debt
  type: article-journal
  volume: '30'

- id: Manheim2019Categorizing
  accessed:
    - year: 2023
      month: 1
      day: 11
  author:
    - family: Manheim
      given: David
    - family: Garrabrant
      given: Scott
  citation-key: Manheim2019Categorizing
  issued:
    - year: 2019
      month: 2
      day: 24
  language: en
  number: arXiv:1803.04585
  publisher: arXiv
  title: Categorizing Variants of Goodhart's Law
  type: article
  URL: http://arxiv.org/abs/1803.04585

- id: Naude2022Future
  accessed:
    - year: 2024
      month: 11
      day: 20
  author:
    - family: Naudé
      given: Wim
  citation-key: Naude2022Future
  collection-title: IZA Discussion Papers
  container-title: IZA Discussion Papers
  DOI: 10.2139/ssrn.4281268
  issued:
    - year: 2022
      month: 11
  language: en
  number: '15713'
  publisher: Institute of Labor Economics (IZA)
  title: >-
    The Future Economics of Artificial Intelligence: Mythical Agents, a
    Singleton and the Dark Forest
  type: article-journal
  URL: https://docs.iza.org/dp15713.pdf

- id: Ngo2024Alignment
  accessed:
    - year: 2024
      month: 10
      day: 28
  author:
    - family: Ngo
      given: Richard
    - family: Chan
      given: Lawrence
    - family: Mindermann
      given: Sören
  citation-key: Ngo2024Alignment
  DOI: 10.48550/arXiv.2209.00626
  issued:
    - year: 2024
      month: 3
      day: 19
  number: arXiv:2209.00626
  publisher: arXiv
  title: The Alignment Problem from a Deep Learning Perspective
  type: article
  URL: http://arxiv.org/abs/2209.00626

- id: Nowak2006Five
  accessed:
    - year: 2020
      month: 1
      day: 9
  author:
    - family: Nowak
      given: Martin A.
  citation-key: Nowak2006Five
  container-title: Science
  DOI: 10.1126/science.1133755
  ISSN: 0036-8075, 1095-9203
  issue: '5805'
  issued:
    - year: 2006
      month: 12
      day: 8
  language: en
  page: 1560-1563
  PMID: '17158317'
  publisher: American Association for the Advancement of Science
  title: Five Rules for the Evolution of Cooperation
  type: article-journal
  URL: >-
    https://web.archive.org/web/20110518021052/http://www.ped.fas.harvard.edu/people/faculty/publications_nowak/Nowak_Science06.pdf
  volume: '314'

- id: Omohundro2008Basic
  accessed:
    - year: 2023
      month: 9
      day: 12
  author:
    - family: Omohundro
      given: Stephen M.
  citation-key: Omohundro2008Basic
  container-title: >-
    Proceedings of the 2008 conference on Artificial General Intelligence 2008:
    Proceedings of the First AGI Conference
  event-place: NLD
  ISBN: 978-1-58603-833-5
  issued:
    - year: 2008
      month: 6
      day: 20
  page: 483–492
  publisher: IOS Press
  publisher-place: NLD
  title: The Basic AI Drives
  type: paper-conference
  URL: https://selfawaresystems.files.wordpress.com/2008/01/ai_drives_final.pdf

- id: Ringstrom2022Reward
  accessed:
    - year: 2023
      month: 1
      day: 26
  author:
    - family: Ringstrom
      given: Thomas J.
  citation-key: Ringstrom2022Reward
  DOI: 10.48550/arXiv.2211.10851
  issued:
    - year: 2022
      month: 11
      day: 22
  number: arXiv:2211.10851
  publisher: arXiv
  title: >-
    Reward is not Necessary: How to Create a Compositional Self-Preserving Agent
    for Life-Long Learning
  type: article
  URL: http://arxiv.org/abs/2211.10851

- id: Russell2019Human
  author:
    - family: Russell
      given: Stuart
  citation-key: Russell2019Human
  issued:
    - year: 2019
      month: 10
      day: 8
  language: English
  number-of-pages: '349'
  publisher: Penguin Books
  title: 'Human Compatible: Artificial Intelligence and the Problem of Control'
  type: book

- id: Silver2021Reward
  accessed:
    - year: 2022
      month: 5
      day: 27
  author:
    - family: Silver
      given: David
    - family: Singh
      given: Satinder
    - family: Precup
      given: Doina
    - family: Sutton
      given: Richard S.
  citation-key: Silver2021Reward
  container-title: Artificial Intelligence
  container-title-short: Artificial Intelligence
  DOI: 10.1016/j.artint.2021.103535
  ISSN: 0004-3702
  issued:
    - year: 2021
      month: 10
      day: 1
  language: en
  page: '103535'
  title: Reward is enough
  type: article-journal
  volume: '299'

- id: Taylor2020Alignment
  accessed:
    - year: 2023
      month: 9
      day: 7
  author:
    - family: Taylor
      given: Jessica
    - family: Yudkowsky
      given: Eliezer
    - family: LaVictoire
      given: Patrick
    - family: Critch
      given: Andrew
  citation-key: Taylor2020Alignment
  container-author:
    - family: Taylor
      given: Jessica
    - family: Yudkowsky
      given: Eliezer
    - family: LaVictoire
      given: Patrick
    - family: Critch
      given: Andrew
  container-title: Ethics of Artificial Intelligence
  DOI: 10.1093/oso/9780190905033.003.0013
  ISBN: 978-0-19-090503-3 978-0-19-090507-1
  issued:
    - year: 2020
      month: 9
      day: 17
  language: en
  page: 342-382
  publisher: Oxford University Press
  title: Alignment for Advanced Machine Learning Systems
  type: chapter
  URL: https://intelligence.org/files/AlignmentMachineLearning.pdf

- id: Xu2023Decisionaid
  accessed:
    - year: 2023
      month: 4
      day: 6
  author:
    - family: Xu
      given: Ruqing
    - family: Dean
      given: Sarah
  citation-key: Xu2023Decisionaid
  DOI: 10.48550/arXiv.2303.13712
  issued:
    - year: 2023
      month: 3
      day: 23
  number: arXiv:2303.13712
  publisher: arXiv
  title: Decision-aid or Controller? Steering Human Decision Makers with Algorithms
  type: article
  URL: http://arxiv.org/abs/2303.13712

- id: Zhuang2021Consequences
  accessed:
    - year: 2023
      month: 4
      day: 30
  author:
    - family: Zhuang
      given: Simon
    - family: Hadfield-Menell
      given: Dylan
  citation-key: Zhuang2021Consequences
  DOI: 10.48550/arXiv.2102.03896
  issued:
    - year: 2021
      month: 2
      day: 7
  number: arXiv:2102.03896
  publisher: arXiv
  title: Consequences of Misaligned AI
  type: article
  URL: http://arxiv.org/abs/2102.03896
...
