---
references:
- id: AcemogluAI2020
  accessed:
    - year: 2021
      month: 9
      day: 20
  author:
    - family: Acemoglu
      given: Daron
    - family: Autor
      given: David
    - family: Hazell
      given: Jonathon
    - family: Restrepo
      given: Pascual
  citation-key: AcemogluAI2020
  DOI: 10.3386/w28257
  genre: Working Paper
  issued:
    - year: 2020
      month: 12
  number: '28257'
  publisher: National Bureau of Economic Research
  title: 'AI and Jobs: Evidence from Online Vacancies'
  type: report
  URL: https://www.nber.org/papers/w28257

- id: AcemogluArtificial2018
  accessed:
    - year: 2021
      month: 9
      day: 20
  author:
    - family: Acemoglu
      given: Daron
    - family: Restrepo
      given: Pascual
  citation-key: AcemogluArtificial2018
  DOI: 10.3386/w24196
  genre: Working Paper
  issued:
    - year: 2018
      month: 1
  number: '24196'
  publisher: National Bureau of Economic Research
  title: Artificial Intelligence, Automation and Work
  type: report
  URL: https://www.nber.org/papers/w24196

- id: AcemogluWrong2020
  accessed:
    - year: 2021
      month: 9
      day: 20
  author:
    - family: Acemoglu
      given: Daron
    - family: Restrepo
      given: Pascual
  citation-key: AcemogluWrong2020
  container-title: Cambridge Journal of Regions, Economy and Society
  DOI: 10.1093/cjres/rsz022
  ISSN: 1752-1378, 1752-1386
  issue: '1'
  issued:
    - year: 2020
      month: 5
      day: 15
  language: en
  page: 25-35
  title: >-
    The wrong kind of AI? Artificial intelligence and the future of labour
    demand
  type: article-journal
  volume: '13'

- id: BirhaneGames2022
  accessed:
    - year: 2022
      month: 5
      day: 23
  author:
    - family: Birhane
      given: Abeba
    - family: Sumpter
      given: David J. T.
  citation-key: BirhaneGames2022
  DOI: 10.48550/arXiv.2205.08922
  issued:
    - year: 2022
      month: 5
      day: 18
  number: arXiv:2205.08922
  publisher: arXiv
  title: 'The games we play: critical complexity improves machine learning'
  type: article
  URL: http://arxiv.org/abs/2205.08922

- id: BostromSuperintelligence2014
  author:
    - family: Bostrom
      given: Nick
  citation-key: BostromSuperintelligence2014
  event-place: Oxford, New York
  ISBN: 978-0-19-967811-2
  issued:
    - year: 2014
      month: 7
      day: 3
  number-of-pages: '352'
  publisher: Oxford University Press
  publisher-place: Oxford, New York
  title: 'Superintelligence: Paths, Dangers, Strategies'
  type: book

- id: BubeckSparks2023
  accessed:
    - year: 2023
      month: 3
      day: 28
  author:
    - family: Bubeck
      given: Sébastien
    - family: Chandrasekaran
      given: Varun
    - family: Eldan
      given: Ronen
    - family: Gehrke
      given: Johannes
    - family: Horvitz
      given: Eric
    - family: Kamar
      given: Ece
    - family: Lee
      given: Peter
    - family: Lee
      given: Yin Tat
    - family: Li
      given: Yuanzhi
    - family: Lundberg
      given: Scott
    - family: Nori
      given: Harsha
    - family: Palangi
      given: Hamid
    - family: Ribeiro
      given: Marco Tulio
    - family: Zhang
      given: Yi
  citation-key: BubeckSparks2023
  DOI: 10.48550/arXiv.2303.12712
  issued:
    - year: 2023
      month: 3
      day: 24
  number: arXiv:2303.12712
  publisher: arXiv
  title: 'Sparks of Artificial General Intelligence: Early experiments with GPT-4'
  type: article
  URL: http://arxiv.org/abs/2303.12712

- id: ChalmersSingularity2016
  accessed:
    - year: 2023
      month: 11
      day: 9
  author:
    - family: Chalmers
      given: David J.
  citation-key: ChalmersSingularity2016
  container-title: Science Fiction and Philosophy
  DOI: 10.1002/9781118922590.ch16
  ISBN: 978-1-118-92259-0
  issued:
    - year: 2016
  language: en
  page: 171-224
  publisher: John Wiley & Sons, Ltd
  section: '16'
  title: The Singularity
  type: chapter
  URL: https://www.consc.net/papers/singularityjcs.pdf

- id: CholletMeasure2019
  accessed:
    - year: 2021
      month: 6
      day: 10
  author:
    - family: Chollet
      given: François
  citation-key: CholletMeasure2019
  container-title: arXiv:1911.01547 [cs]
  issued:
    - year: 2019
      month: 11
      day: 25
  title: On the Measure of Intelligence
  type: article-journal
  URL: http://arxiv.org/abs/1911.01547

- id: CollisonScience2018
  accessed:
    - year: 2021
      month: 6
      day: 1
  author:
    - family: Collison
      given: Patrick
    - family: Nielsen
      given: Michael
  citation-key: CollisonScience2018
  container-title: The Atlantic
  issued:
    - year: 2018
      month: 11
      day: 16
  language: en
  section: Science
  title: Science Is Getting Less Bang for Its Buck
  type: article-magazine
  URL: >-
    https://www.theatlantic.com/science/archive/2018/11/diminishing-returns-science/575665/

- id: DonohoData2023
  accessed:
    - year: 2023
      month: 11
      day: 19
  author:
    - family: Donoho
      given: David
  citation-key: DonohoData2023
  DOI: 10.48550/arXiv.2310.00865
  issued:
    - year: 2023
      month: 10
      day: 1
  number: arXiv:2310.00865
  publisher: arXiv
  title: Data Science at the Singularity
  type: article
  URL: http://arxiv.org/abs/2310.00865

- id: EverittUniversal2018
  accessed:
    - year: 2022
      month: 8
      day: 9
  author:
    - family: Everitt
      given: Tom
    - family: Hutter
      given: Marcus
  citation-key: EverittUniversal2018
  container-title: Foundations of Trusted Autonomy
  DOI: 10.1007/978-3-319-64816-3_2
  editor:
    - family: Abbass
      given: Hussein A.
    - family: Scholz
      given: Jason
    - family: Reid
      given: Darryn J.
  event-place: Cham
  ISBN: 978-3-319-64815-6 978-3-319-64816-3
  issued:
    - year: 2018
  language: en
  page: 15-46
  publisher: Springer International Publishing
  publisher-place: Cham
  title: >-
    Universal Artificial Intelligence: Practical Agents and Fundamental
    Challenges
  type: chapter
  URL: http://link.springer.com/10.1007/978-3-319-64816-3_2
  volume: '117'

- id: GraceAlgorithmic2013
  author:
    - family: Grace
      given: Katja
  citation-key: GraceAlgorithmic2013
  issued:
    - year: 2013
  language: en
  title: Algorithmic Progress in Six Domains
  type: article-journal
  URL: https://intelligence.org/files/AlgorithmicProgress.pdf

- id: GraceViewpoint2018
  accessed:
    - year: 2023
      month: 10
      day: 10
  author:
    - family: Grace
      given: Katja
    - family: Salvatier
      given: John
    - family: Dafoe
      given: Allan
    - family: Zhang
      given: Baobao
    - family: Evans
      given: Owain
  citation-key: GraceViewpoint2018
  container-title: Journal of Artificial Intelligence Research
  DOI: 10.1613/jair.1.11222
  ISSN: 1076-9757
  issued:
    - year: 2018
      month: 7
      day: 31
  language: en
  page: 729-754
  title: 'Viewpoint: When Will AI Exceed Human Performance? Evidence from AI Experts'
  type: article-journal
  volume: '62'

- id: HarariHomo2018
  author:
    - family: Harari
      given: Yuval Noah
  citation-key: HarariHomo2018
  edition: Illustrated edition
  event-place: New York
  ISBN: 978-0-06-246434-7
  issued:
    - year: 2018
      month: 9
      day: 4
  language: English
  number-of-pages: '464'
  publisher: Harper Perennial
  publisher-place: New York
  title: 'Homo Deus: A Brief History of Tomorrow'
  type: book

- id: HernandezMeasuring2020
  accessed:
    - year: 2024
      month: 3
      day: 6
  author:
    - family: Hernandez
      given: Danny
    - family: Brown
      given: Tom B.
  citation-key: HernandezMeasuring2020
  DOI: 10.48550/arXiv.2005.04305
  issued:
    - year: 2020
      month: 5
      day: 8
  number: arXiv:2005.04305
  publisher: arXiv
  title: Measuring the Algorithmic Efficiency of Neural Networks
  type: article
  URL: http://arxiv.org/abs/2005.04305

- id: HildebrandtSmart2020
  accessed:
    - year: 2022
      month: 5
      day: 10
  author:
    - family: Hildebrandt
      given: Mireille
  citation-key: HildebrandtSmart2020
  container-title: Internet Policy Review
  ISSN: 2197-6775
  issue: '4'
  issued:
    - year: 2020
      month: 12
      day: 17
  title: Smart technologies
  type: article-journal
  URL: https://policyreview.info/concepts/smart-technologies
  volume: '9'

- id: HutsonTaught2022
  accessed:
    - year: 2022
      month: 6
      day: 22
  author:
    - family: Hutson
      given: Matthew
  citation-key: HutsonTaught2022
  container-title: Science
  container-title-short: Science
  DOI: 10.1126/science.abq7833
  ISSN: 0036-8075, 1095-9203
  issue: '6593'
  issued:
    - year: 2022
      month: 5
      day: 6
  language: en
  page: 570-573
  title: Taught to the test
  type: article-journal
  volume: '376'

- id: HutterTheory2000
  accessed:
    - year: 2022
      month: 8
      day: 9
  author:
    - family: Hutter
      given: Marcus
  citation-key: HutterTheory2000
  DOI: 10.48550/arXiv.cs/0004001
  issued:
    - year: 2000
      month: 4
      day: 3
  number: arXiv:cs/0004001
  publisher: arXiv
  title: >-
    A Theory of Universal Artificial Intelligence based on Algorithmic
    Complexity
  type: article
  URL: http://arxiv.org/abs/cs/0004001

- id: JohansenFinitetime2001
  accessed:
    - year: 2013
      month: 2
      day: 27
  author:
    - family: Johansen
      given: Anders
    - family: Sornette
      given: Didier
  citation-key: JohansenFinitetime2001
  container-title: 'Physica A: Statistical Mechanics and its Applications'
  container-title-short: 'Physica A: Statistical Mechanics and its Applications'
  DOI: 10.1016/S0378-4371(01)00105-4
  ISSN: 0378-4371
  issue: 3–4
  issued:
    - year: 2001
      month: 5
      day: 15
  page: 465-502
  title: >-
    Finite-time singularity in the dynamics of the world population, economic
    and financial indices
  type: article-journal
  URL: http://arxiv.org/abs/cond-mat/0002075
  volume: '294'

- id: Lee142020
  accessed:
    - year: 2024
      month: 3
      day: 4
  author:
    - family: Lee
      given: Edward Ashford
  citation-key: Lee142020
  container-title: 'The Coevolution: The Entwined Futures of Humans and Machines'
  ISBN: 978-0-262-35837-8
  issued:
    - year: 2020
  page: 281-311
  publisher: MIT Press
  title: 14 COEVOLUTION
  type: chapter
  URL: https://ieeexplore.ieee.org/document/9072041

- id: LeeCoevolution2020
  author:
    - family: Lee
      given: Edward Ashford
  citation-key: LeeCoevolution2020
  edition: Illustrated edition
  event-place: Cambridge, Massachusetts
  ISBN: 978-0-262-04393-9
  issued:
    - year: 2020
      month: 4
      day: 21
  language: English
  number-of-pages: '384'
  publisher: The MIT Press
  publisher-place: Cambridge, Massachusetts
  title: 'The Coevolution: The Entwined Futures of Humans and Machines'
  type: book

- id: ManheimCategorizing2019
  accessed:
    - year: 2023
      month: 1
      day: 11
  author:
    - family: Manheim
      given: David
    - family: Garrabrant
      given: Scott
  citation-key: ManheimCategorizing2019
  issued:
    - year: 2019
      month: 2
      day: 24
  language: en
  number: arXiv:1803.04585
  publisher: arXiv
  title: Categorizing Variants of Goodhart's Law
  type: article
  URL: http://arxiv.org/abs/1803.04585

- id: MitchellWhy2021
  accessed:
    - year: 2021
      month: 6
      day: 10
  author:
    - family: Mitchell
      given: Melanie
  citation-key: MitchellWhy2021
  container-title: arXiv:2104.12871 [cs]
  issued:
    - year: 2021
      month: 4
      day: 28
  language: en
  title: Why AI is Harder Than We Think
  type: article-journal
  URL: http://arxiv.org/abs/2104.12871

- id: NathanGlobal2021
  accessed:
    - year: 2022
      month: 2
      day: 9
  author:
    - family: Nathan
      given: Christopher
    - family: Hyams
      given: Keith
  citation-key: NathanGlobal2021
  container-title: Policy Sciences
  container-title-short: Policy Sci
  DOI: 10.1007/s11077-021-09444-0
  ISSN: 1573-0891
  issued:
    - year: 2021
      month: 12
      day: 2
  language: en
  title: Global policymakers and catastrophic risk
  type: article-journal

- id: OmohundroBasic2008
  accessed:
    - year: 2023
      month: 9
      day: 12
  author:
    - family: Omohundro
      given: Stephen M.
  citation-key: OmohundroBasic2008
  container-title: >-
    Proceedings of the 2008 conference on Artificial General Intelligence 2008:
    Proceedings of the First AGI Conference
  event-place: NLD
  ISBN: 978-1-58603-833-5
  issued:
    - year: 2008
      month: 6
      day: 20
  page: 483–492
  publisher: IOS Press
  publisher-place: NLD
  title: The Basic AI Drives
  type: paper-conference
  URL: https://selfawaresystems.files.wordpress.com/2008/01/ai_drives_final.pdf

- id: PhilipponAdditive2022
  accessed:
    - year: 2022
      month: 6
      day: 22
  author:
    - family: Philippon
      given: Thomas
  citation-key: PhilipponAdditive2022
  collection-title: Working Paper Series
  DOI: 10.3386/w29950
  genre: Working Paper
  issued:
    - year: 2022
      month: 4
  number: '29950'
  publisher: National Bureau of Economic Research
  title: Additive Growth
  type: article
  URL: https://pages.stern.nyu.edu/~tphilipp/papers/AddGrowth_macro.pdf

- id: RussellHuman2019
  author:
    - family: Russell
      given: Stuart
  citation-key: RussellHuman2019
  issued:
    - year: 2019
      month: 10
      day: 8
  language: English
  number-of-pages: '349'
  publisher: Penguin Books
  title: 'Human Compatible: Artificial Intelligence and the Problem of Control'
  type: book

- id: SastryComputing
  author:
    - family: Sastry
      given: Girish
    - family: Heim
      given: Lennart
    - family: Belfield
      given: Haydn
    - family: Anderljung
      given: Markus
    - family: Brundage
      given: Miles
    - family: Hazell
      given: Julian
    - family: O’Keefe
      given: Cullen
    - family: Hadfield
      given: Gillian K
    - family: Ngo
      given: Richard
    - family: Pilz
      given: Konstantin
    - family: Gor
      given: George
    - family: Bluemke
      given: Emma
    - family: Shoker
      given: Sarah
    - family: Egan
      given: Janet
    - family: Trager
      given: Robert F
    - family: Avin
      given: Shahar
    - family: Weller
      given: Adrian
    - family: Bengio
      given: Yoshua
    - family: Coyle
      given: Diane
  citation-key: SastryComputing
  language: en
  title: Computing Power and the Governance of Artificial Intelligence
  type: article-journal

- id: ScottNot2022
  accessed:
    - year: 2022
      month: 5
      day: 10
  author:
    - family: Scott
      given: Kevin
  citation-key: ScottNot2022
  container-title: American Academy of Arts & Sciences
  issued:
    - year: 2022
  language: en
  title: >-
    I Do Not Think It Means What You Think It Means: Artificial Intelligence,
    Cognitive Work & Scale
  type: article-magazine
  URL: >-
    https://www.amacad.org/publication/i-do-not-think-it-means-what-you-think-it-means-artificial-intelligence-cognitive-work

- id: SilverReward2021
  accessed:
    - year: 2022
      month: 5
      day: 27
  author:
    - family: Silver
      given: David
    - family: Singh
      given: Satinder
    - family: Precup
      given: Doina
    - family: Sutton
      given: Richard S.
  citation-key: SilverReward2021
  container-title: Artificial Intelligence
  container-title-short: Artificial Intelligence
  DOI: 10.1016/j.artint.2021.103535
  ISSN: 0004-3702
  issued:
    - year: 2021
      month: 10
      day: 1
  language: en
  page: '103535'
  title: Reward is enough
  type: article-journal
  volume: '299'

- id: SunehagPrinciples2013
  accessed:
    - year: 2022
      month: 8
      day: 9
  author:
    - family: Sunehag
      given: Peter
    - family: Hutter
      given: Marcus
  citation-key: SunehagPrinciples2013
  collection-title: Lecture Notes in Computer Science
  container-title: >-
    Algorithmic Probability and Friends. Bayesian Prediction and Artificial
    Intelligence: Papers from the Ray Solomonoff 85th Memorial Conference,
    Melbourne, VIC, Australia, November 30 – December 2, 2011
  DOI: 10.1007/978-3-642-44958-1_30
  editor:
    - family: Dowe
      given: David L.
  event-place: Berlin, Heidelberg
  ISBN: 978-3-642-44958-1
  issued:
    - year: 2013
  language: en
  page: 386-398
  publisher: Springer
  publisher-place: Berlin, Heidelberg
  title: Principles of Solomonoff Induction and AIXI
  type: chapter
  URL: https://doi.org/10.1007/978-3-642-44958-1_30

- id: WongAsymptotic2022
  accessed:
    - year: 2022
      month: 5
      day: 15
  author:
    - family: Wong
      given: Michael L.
    - family: Bartlett
      given: Stuart
  citation-key: WongAsymptotic2022
  container-title: Journal of The Royal Society Interface
  DOI: 10.1098/rsif.2022.0029
  issue: '190'
  issued:
    - year: 2022
  page: '20220029'
  publisher: Royal Society
  title: >-
    Asymptotic burnout and homeostatic awakening: a possible solution to the
    Fermi paradox?
  type: article-journal
  volume: '19'
...
