---
references:
- id: Allen-ZhuOptimal2016
  author:
    - family: Allen-Zhu
      given: Zeyuan
    - family: Hazan
      given: Elad
  citation-key: Allen-ZhuOptimal2016
  container-title: Advances in Neural Information Processing Systems 29
  editor:
    - family: Lee
      given: D. D.
    - family: Sugiyama
      given: M.
    - family: Luxburg
      given: U. V.
    - family: Guyon
      given: I.
    - family: Garnett
      given: R.
  issued:
    - year: 2016
  page: 1606–1614
  publisher: Curran Associates, Inc.
  title: Optimal Black-Box Reductions Between Optimization Objectives
  type: paper-conference
  URL: >-
    http://papers.nips.cc/paper/6364-optimal-black-box-reductions-between-optimization-objectives.pdf

- id: AnilScalable2021
  accessed:
    - year: 2023
      month: 4
      day: 17
  author:
    - family: Anil
      given: Rohan
    - family: Gupta
      given: Vineet
    - family: Koren
      given: Tomer
    - family: Regan
      given: Kevin
    - family: Singer
      given: Yoram
  citation-key: AnilScalable2021
  DOI: 10.48550/arXiv.2002.09018
  issued:
    - year: 2021
      month: 3
      day: 5
  number: arXiv:2002.09018
  publisher: arXiv
  title: Scalable Second Order Optimization for Deep Learning
  type: article
  URL: http://arxiv.org/abs/2002.09018

- id: ArnoldAccelerating2017
  accessed:
    - year: 2018
      month: 4
      day: 27
  author:
    - family: Arnold
      given: Sébastien M. R.
    - family: Wang
      given: Chunming
  citation-key: ArnoldAccelerating2017
  container-title: arXiv:1709.05069 [cs]
  event-title: ICLR
  issued:
    - year: 2017
  title: >-
    Accelerating SGD for Distributed Deep-Learning Using Approximated Hessian
    Matrix
  type: paper-conference
  URL: http://arxiv.org/abs/1709.05069

- id: BachOptimization2011
  archive: Calibre
  author:
    - family: Bach
      given: Francis
    - family: Jenatton
      given: Rodolphe
    - family: Mairal
      given: Julien
  citation-key: BachOptimization2011
  collection-number: '1.0'
  collection-title: Foundations and Trends(r) in Machine Learning
  DOI: 10.1561/2200000015
  ISBN: 978-1-60198-510-1
  issued:
    - year: 2011
      month: 12
      day: 23
  language: eng
  publisher: Now Publishers Inc
  title: Optimization With Sparsity-Inducing Penalties
  type: book
  URL: https://hal.inria.fr/hal-00613125v1/document

- id: BaDistributed2016
  accessed:
    - year: 2018
      month: 4
      day: 2
  author:
    - family: Ba
      given: Jimmy
    - family: Grosse
      given: Roger
    - family: Martens
      given: James
  citation-key: BaDistributed2016
  issued:
    - year: 2016
      month: 11
      day: 5
  title: >-
    Distributed Second-Order Optimization using Kronecker-Factored
    Approximations
  type: article-journal
  URL: https://openreview.net/forum?id=SkkTMpjex

- id: BattitiFirstand1992
  accessed:
    - year: 2015
      month: 3
      day: 20
  author:
    - family: Battiti
      given: Roberto
  citation-key: BattitiFirstand1992
  container-title: Neural computation
  container-title-short: Neural Computation
  DOI: 10.1162/neco.1992.4.2.141
  ISSN: 0899-7667
  issue: '2'
  issued:
    - year: 1992
  page: 141–166
  title: >-
    First-and second-order methods for learning: between steepest descent and
    Newton's method
  type: article-journal
  URL: >-
    http://rtm.science.unitn.it/~battiti/archive/FirstSecondOrderMethodsForLearning.PDF
  volume: '4'

- id: BottouOptimization2016
  accessed:
    - year: 2016
      month: 6
      day: 17
  author:
    - family: Bottou
      given: Léon
    - family: Curtis
      given: Frank E.
    - family: Nocedal
      given: Jorge
  citation-key: BottouOptimization2016
  container-title: arXiv:1606.04838 [cs, math, stat]
  issued:
    - year: 2016
      month: 6
      day: 15
  title: Optimization Methods for Large-Scale Machine Learning
  type: article-journal
  URL: http://arxiv.org/abs/1606.04838

- id: BoumalManopt2014
  accessed:
    - year: 2016
      month: 9
      day: 11
  author:
    - family: Boumal
      given: Nicolas
    - family: Mishra
      given: Bamdev
    - family: Absil
      given: P.-A.
    - family: Sepulchre
      given: Rodolphe
  citation-key: BoumalManopt2014
  container-title: Journal of Machine Learning Research
  issued:
    - year: 2014
  page: 1455-1459
  title: Manopt, a Matlab Toolbox for Optimization on Manifolds
  type: article-journal
  URL: http://jmlr.org/papers/v15/boumal14a.html
  volume: '15'

- id: ChoHessianfree2015
  accessed:
    - year: 2018
      month: 4
      day: 2
  author:
    - family: Cho
      given: Minhyung
    - family: Dhir
      given: Chandra Shekhar
    - family: Lee
      given: Jaehyung
  citation-key: ChoHessianfree2015
  container-title: Advances In Neural Information Processing Systems
  event-title: NIPS
  issued:
    - year: 2015
      month: 9
      day: 11
  title: >-
    Hessian-free Optimization for Learning Deep Multidimensional Recurrent
    Neural Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1509.03475

- id: DangelBackPACK2019
  accessed:
    - year: 2021
      month: 8
      day: 5
  author:
    - family: Dangel
      given: Felix
    - family: Kunstner
      given: Frederik
    - family: Hennig
      given: Philipp
  citation-key: DangelBackPACK2019
  container-title: International conference on learning representations
  event-title: International Conference on Learning Representations
  issued:
    - year: 2019
      month: 9
      day: 25
  language: en
  title: 'BackPACK: Packing more into Backprop'
  type: paper-conference
  URL: https://openreview.net/forum?id=BJlrF24twB

- id: DauphinIdentifying2014
  author:
    - family: Dauphin
      given: Yann
    - family: Pascanu
      given: Razvan
    - family: Gulcehre
      given: Caglar
    - family: Cho
      given: Kyunghyun
    - family: Ganguli
      given: Surya
    - family: Bengio
      given: Yoshua
  citation-key: DauphinIdentifying2014
  container-title: Advances in Neural Information Processing Systems 27
  issued:
    - year: 2014
      month: 6
      day: 10
  page: 2933–2941
  publisher: Curran Associates, Inc.
  title: >-
    Identifying and attacking the saddle point problem in high-dimensional
    non-convex optimization
  type: paper-conference
  URL: http://arxiv.org/abs/1406.2572

- id: DennisChapter1989
  accessed:
    - year: 2019
      month: 2
      day: 5
  author:
    - family: Dennis
      given: J. E.
    - family: Schnabel
      given: Robert B.
  citation-key: DennisChapter1989
  collection-title: Optimization
  container-title: Handbooks in Operations Research and Management Science
  DOI: 10.1016/S0927-0507(89)01002-9
  issued:
    - year: 1989
      month: 1
      day: 1
  page: 1-72
  publisher: Elsevier
  title: Chapter I A view of unconstrained optimization
  type: chapter
  URL: https://apps.dtic.mil/dtic/tr/fulltext/u2/a188327.pdf
  volume: '1'

- id: DennisNumerical1996
  accessed:
    - year: 2019
      month: 2
      day: 5
  author:
    - family: Dennis
      given: J.
    - family: Schnabel
      given: R.
  citation-key: DennisNumerical1996
  collection-title: Classics in Applied Mathematics
  DOI: 10.1137/1.9781611971200
  ISBN: 978-0-89871-364-0
  issued:
    - year: 1996
      month: 1
      day: 1
  number-of-pages: '390'
  publisher: Society for Industrial and Applied Mathematics
  title: Numerical Methods for Unconstrained Optimization and Nonlinear Equations
  type: book
  URL: https://epubs.siam.org/doi/book/10.1137/1.9781611971200
  volume: '16'

- id: FoongInBetween2019
  accessed:
    - year: 2021
      month: 8
      day: 23
  author:
    - family: Foong
      given: Andrew Y. K.
    - family: Li
      given: Yingzhen
    - family: Hernández-Lobato
      given: José Miguel
    - family: Turner
      given: Richard E.
  citation-key: FoongInBetween2019
  container-title: arXiv:1906.11537 [cs, stat]
  issued:
    - year: 2019
      month: 6
      day: 27
  title: '''In-Between'' Uncertainty in Bayesian Neural Networks'
  type: article-journal
  URL: http://arxiv.org/abs/1906.11537

- id: GillWhat2004
  accessed:
    - year: 2019
      month: 11
      day: 24
  author:
    - family: Gill
      given: Jeff
    - family: King
      given: Gary
  citation-key: GillWhat2004
  container-title: Sociological Methods & Research
  container-title-short: Sociological Methods & Research
  DOI: 10.1177/0049124103262681
  ISSN: 0049-1241, 1552-8294
  issue: '1'
  issued:
    - year: 2004
      month: 8
  language: en
  page: 54-87
  title: >-
    What to Do When Your Hessian is Not Invertible: Alternatives to Model
    Respecification in Nonlinear Estimation
  type: article-journal
  volume: '33'

- id: GravellApproximate2021
  accessed:
    - year: 2021
      month: 12
      day: 8
  author:
    - family: Gravell
      given: Benjamin
    - family: Shames
      given: Iman
    - family: Summers
      given: Tyler
  citation-key: GravellApproximate2021
  container-title: arXiv:2011.14212 [cs, eess, math]
  issued:
    - year: 2021
      month: 2
      day: 18
  language: en
  title: Approximate Midpoint Policy Iteration for Linear Quadratic Control
  type: article-journal
  URL: http://arxiv.org/abs/2011.14212

- id: GrosseMetrics2021
  accessed:
    - year: 2022
      month: 7
      day: 25
  author:
    - family: Grosse
      given: Roger
  citation-key: GrosseMetrics2021
  container-title: CSC2541 Winter 2021
  issued:
    - year: 2021
  page: Chapter 3
  title: Metrics
  type: chapter
  URL: >-
    https://www.cs.toronto.edu/~rgrosse/courses/csc2541_2021/readings/L03_metrics.pdf

- id: GrosseSecondOrder2021
  author:
    - family: Grosse
      given: Roger
  citation-key: GrosseSecondOrder2021
  container-title: CSC2541 Winter 2021
  issued:
    - year: 2021
  language: en
  page: Chapter 4
  title: Second-Order Optimization
  type: chapter
  URL: >-
    https://www.cs.toronto.edu/~rgrosse/courses/csc2541_2021/readings/L04_second_order.pdf

- id: GrosseTaylor2021
  author:
    - family: Grosse
      given: Roger
  citation-key: GrosseTaylor2021
  container-title: CSC2541 Winter 2021
  issued:
    - year: 2021
  language: en
  page: Chapter 2
  title: Taylor Approximations
  type: chapter
  URL: >-
    https://www.cs.toronto.edu/~rgrosse/courses/csc2541_2021/readings/L02_Taylor_approximations.pdf

- id: GuptaShampoo2018
  accessed:
    - year: 2023
      month: 4
      day: 17
  author:
    - family: Gupta
      given: Vineet
    - family: Koren
      given: Tomer
    - family: Singer
      given: Yoram
  citation-key: GuptaShampoo2018
  container-title: Proceedings of the 35th International Conference on Machine Learning
  event-title: International Conference on Machine Learning
  ISSN: 2640-3498
  issued:
    - year: 2018
      month: 7
      day: 3
  language: en
  page: 1842-1850
  publisher: PMLR
  title: 'Shampoo: Preconditioned Stochastic Tensor Optimization'
  type: paper-conference
  URL: https://proceedings.mlr.press/v80/gupta18a.html

- id: HollPhysical2021
  accessed:
    - year: 2022
      month: 7
      day: 12
  author:
    - family: Holl
      given: Philipp
    - family: Koltun
      given: Vladlen
    - family: Thuerey
      given: Nils
  citation-key: HollPhysical2021
  DOI: 10.48550/arXiv.2109.15048
  issued:
    - year: 2021
      month: 10
      day: 1
  number: arXiv:2109.15048
  publisher: arXiv
  title: Physical Gradients for Deep Learning
  type: article
  URL: http://arxiv.org/abs/2109.15048

- id: ImmerImproving2021
  accessed:
    - year: 2021
      month: 8
      day: 23
  author:
    - family: Immer
      given: Alexander
    - family: Korzepa
      given: Maciej
    - family: Bauer
      given: Matthias
  citation-key: ImmerImproving2021
  container-title: International Conference on Artificial Intelligence and Statistics
  event-title: International Conference on Artificial Intelligence and Statistics
  ISSN: 2640-3498
  issued:
    - year: 2021
      month: 3
      day: 18
  language: en
  page: 703-711
  publisher: PMLR
  title: Improving predictions of Bayesian neural nets via local linearization
  type: paper-conference
  URL: https://proceedings.mlr.press/v130/immer21a.html

- id: LesageLandrySecondorder2021
  accessed:
    - year: 2021
      month: 12
      day: 8
  author:
    - family: Lesage-Landry
      given: Antoine
    - family: Taylor
      given: Joshua A.
    - family: Shames
      given: Iman
  citation-key: LesageLandrySecondorder2021
  container-title: IEEE Transactions on Automatic Control
  container-title-short: IEEE Trans. Automat. Contr.
  DOI: 10.1109/TAC.2020.3040372
  ISSN: 0018-9286, 1558-2523, 2334-3303
  issue: '10'
  issued:
    - year: 2021
      month: 10
  language: en
  page: 4866-4872
  title: Second-order Online Nonconvex Optimization
  type: article-journal
  URL: http://arxiv.org/abs/2001.10114
  volume: '66'

- id: MadsenMethods2004
  author:
    - family: Madsen
      given: K
    - family: Nielsen
      given: H.B.
    - family: Tingleff
      given: O.
  citation-key: MadsenMethods2004
  issued:
    - year: 2004
  title: Methods for non-linear least squares problems
  type: report
  URL: http://www2.imm.dtu.dk/pubdb/views/edoc_download.php/3215/pdf/imm3215.pdf

- id: MartensDeep2010
  accessed:
    - year: 2018
      month: 4
      day: 2
  author:
    - family: Martens
      given: James
  citation-key: MartensDeep2010
  collection-title: ICML'10
  container-title: >-
    Proceedings of the 27th International Conference on International Conference
    on Machine Learning
  event-place: USA
  ISBN: 978-1-60558-907-7
  issued:
    - year: 2010
  page: 735–742
  publisher: Omnipress
  publisher-place: USA
  title: Deep Learning via Hessian-free Optimization
  type: paper-conference
  URL: http://www.cs.utoronto.ca/~jmartens/docs/Deep_HessianFree.pdf

- id: MartensLearning2011
  accessed:
    - year: 2018
      month: 4
      day: 2
  author:
    - family: Martens
      given: James
    - family: Sutskever
      given: Ilya
  citation-key: MartensLearning2011
  collection-title: ICML'11
  container-title: >-
    Proceedings of the 28th International Conference on International Conference
    on Machine Learning
  event-place: USA
  ISBN: 978-1-4503-0619-5
  issued:
    - year: 2011
  page: 1033–1040
  publisher: Omnipress
  publisher-place: USA
  title: Learning Recurrent Neural Networks with Hessian-free Optimization
  type: paper-conference
  URL: http://dl.acm.org/citation.cfm?id=3104482.3104612

- id: MartensNew2020
  accessed:
    - year: 2020
      month: 9
      day: 17
  author:
    - family: Martens
      given: James
  citation-key: MartensNew2020
  container-title: Journal of Machine Learning Research
  ISSN: 1533-7928
  issue: '146'
  issued:
    - year: 2020
  page: 1-76
  title: New Insights and Perspectives on the Natural Gradient Method
  type: article-journal
  URL: http://jmlr.org/papers/v21/17-678.html
  volume: '21'

- id: MartensOptimizing2015
  accessed:
    - year: 2022
      month: 1
      day: 6
  author:
    - family: Martens
      given: James
    - family: Grosse
      given: Roger
  citation-key: MartensOptimizing2015
  container-title: Proceedings of the 32nd International Conference on Machine Learning
  event-title: International Conference on Machine Learning
  ISSN: 1938-7228
  issued:
    - year: 2015
      month: 6
      day: 1
  language: en
  page: 2408-2417
  publisher: PMLR
  title: Optimizing Neural Networks with Kronecker-factored Approximate Curvature
  type: paper-conference
  URL: http://arxiv.org/abs/1503.05671

- id: MartensTraining2012
  author:
    - family: Martens
      given: James
    - family: Sutskever
      given: Ilya
  citation-key: MartensTraining2012
  collection-title: Lecture Notes in Computer Science
  container-title: 'Neural networks: Tricks of the trade'
  ISBN: 978-3-642-35288-1 978-3-642-35289-8
  issued:
    - year: 2012
  page: 479–535
  publisher: Springer
  title: Training deep and recurrent networks with Hessian-free optimization
  type: chapter
  URL: http://www.cs.toronto.edu/~jmartens/docs/HF_book_chapter.pdf

- id: NesterovAccelerating2007
  accessed:
    - year: 2016
      month: 9
      day: 21
  author:
    - family: Nesterov
      given: Yu
  citation-key: NesterovAccelerating2007
  container-title: Mathematical Programming
  container-title-short: Math. Program.
  DOI: 10.1007/s10107-006-0089-x
  ISSN: 0025-5610, 1436-4646
  issue: '1'
  issued:
    - year: 2007
      month: 1
      day: 20
  language: en
  page: 159-181
  title: Accelerating the cubic regularization of Newton’s method on convex problems
  type: article-journal
  volume: '112'

- id: NocedalAnalysis1993
  accessed:
    - year: 2015
      month: 4
      day: 10
  author:
    - family: Nocedal
      given: Jorge
    - family: Yuan
      given: Ya-xiang
  citation-key: NocedalAnalysis1993
  container-title: Mathematical Programming
  container-title-short: Mathematical Programming
  DOI: 10.1007/BF01582136
  ISSN: 0025-5610, 1436-4646
  issue: 1-3
  issued:
    - year: 1993
      month: 8
      day: 1
  language: en
  page: 19-37
  title: Analysis of a self-scaling quasi-Newton method
  type: article-journal
  volume: '61'

- id: NocedalNumerical2006
  accessed:
    - year: 2019
      month: 2
      day: 5
  author:
    - family: Nocedal
      given: Jorge
    - family: Wright
      given: S.
  citation-key: NocedalNumerical2006
  collection-title: Springer Series in Operations Research and Financial Engineering
  edition: '2'
  event-place: New York
  ISBN: 978-0-387-30303-1
  issued:
    - year: 2006
  language: en
  publisher: Springer-Verlag
  publisher-place: New York
  title: Numerical Optimization
  type: book
  URL: https://www.springer.com/gp/book/9780387303031

- id: PavlovInterior2020
  accessed:
    - year: 2021
      month: 12
      day: 8
  author:
    - family: Pavlov
      given: Andrei
    - family: Shames
      given: Iman
    - family: Manzie
      given: Chris
  citation-key: PavlovInterior2020
  container-title: arXiv:2004.12710 [cs, eess, math]
  issued:
    - year: 2020
      month: 10
      day: 20
  language: en
  title: Interior Point Differential Dynamic Programming
  type: article-journal
  URL: http://arxiv.org/abs/2004.12710

- id: PilanciIterative2016
  accessed:
    - year: 2017
      month: 2
      day: 28
  author:
    - family: Pilanci
      given: Mert
    - family: Wainwright
      given: Martin J.
  citation-key: PilanciIterative2016
  container-title: Journal of Machine Learning Research
  issue: '53'
  issued:
    - year: 2016
  page: 1–38
  title: >-
    Iterative Hessian sketch: Fast and accurate solution approximation for
    constrained least-squares
  type: article-journal
  URL: http://www.jmlr.org/papers/volume17/14-460/14-460.pdf
  volume: '17'

- id: RuppertNewtonRaphson1985
  author:
    - family: Ruppert
      given: David
  citation-key: RuppertNewtonRaphson1985
  container-title: The Annals of Statistics
  container-title-short: The Annals of Statistics
  DOI: 10.1214/aos/1176346589
  ISSN: 0090-5364, 2168-8966
  issue: '1'
  issued:
    - year: 1985
  page: 236-245
  title: A Newton-Raphson Version of the Multivariate Robbins-Monro Procedure
  type: article-journal
  volume: '13'

- id: SchmidtOptimization2009
  author:
    - family: Schmidt
      given: Mark
    - family: Fung
      given: Glenn
    - family: Rosales
      given: Romer
  citation-key: SchmidtOptimization2009
  container-title: University of British Columbia, Technical Report TR-2009
  issued:
    - year: 2009
  title: Optimization methods for l1-regularization
  type: article-journal
  volume: '19'

- id: SchraudolphFast2002
  accessed:
    - year: 2018
      month: 4
      day: 2
  author:
    - family: Schraudolph
      given: Nicol N.
  citation-key: SchraudolphFast2002
  container-title: Neural Computation
  container-title-short: Neural Computation
  DOI: 10.1162/08997660260028683
  ISSN: 0899-7667
  issue: '7'
  issued:
    - year: 2002
      month: 7
      day: 1
  page: 1723-1738
  title: Fast Curvature Matrix-Vector Products for Second-Order Gradient Descent
  type: article-journal
  URL: https://nic.schraudolph.org/pubs/Schraudolph02.pdf
  volume: '14'

- id: TranstrumGeometry2011
  author:
    - family: Transtrum
      given: Mark K
    - family: Machta
      given: Benjamin B
    - family: Sethna
      given: James P
  citation-key: TranstrumGeometry2011
  container-title: Physical Review E
  container-title-short: Phys. Rev. E
  DOI: 10.1103/PhysRevE.83.036701
  ISSN: 1539-3755, 1550-2376
  issue: '3'
  issued:
    - year: 2011
      month: 3
      day: 3
  page: '036701'
  title: >-
    The geometry of nonlinear least squares with applications to sloppy models
    and optimization
  type: article-journal
  URL: http://www.arxiv.org/abs/1010.1449
  volume: '83'

- id: WilkinsonBayesNewton2021
  accessed:
    - year: 2022
      month: 7
      day: 28
  author:
    - family: Wilkinson
      given: William J.
    - family: Särkkä
      given: Simo
    - family: Solin
      given: Arno
  citation-key: WilkinsonBayesNewton2021
  DOI: 10.48550/arXiv.2111.01721
  issued:
    - year: 2021
      month: 11
      day: 3
  number: arXiv:2111.01721
  publisher: arXiv
  title: Bayes-Newton Methods for Approximate Bayesian Inference with PSD Guarantees
  type: article
  URL: http://arxiv.org/abs/2111.01721

- id: WrightHighdimensional2022
  author:
    - family: Wright
      given: John
    - family: Ma
      given: Yi
  citation-key: WrightHighdimensional2022
  event-place: S.l.
  ISBN: 978-1-108-48973-7
  issued:
    - year: 2022
  language: English
  publisher: Cambridge University Press
  publisher-place: S.l.
  title: >-
    High-dimensional data analysis with low-dimensional models: Principles,
    computation, and applications
  type: book
  URL: https://book-wright-ma.github.io/

- id: WrightOptimization2021
  author:
    - family: Wright
      given: Stephen J.
    - family: Recht
      given: Benjamin
  call-number: QA76.9.B45 W75 2021
  citation-key: WrightOptimization2021
  DOI: 10.1017/9781009004282
  event-place: New York
  ISBN: 978-1-316-51898-4
  issued:
    - year: 2021
  publisher: Cambridge University Press
  publisher-place: New York
  title: Optimization for data analysis
  type: book
  URL: >-
    https://www.cambridge.org/core/books/optimization-for-data-analysis/C02C3708905D236AA354D1CE1739A6A2

- id: YaoPyHessian2020
  accessed:
    - year: 2021
      month: 8
      day: 5
  author:
    - family: Yao
      given: Zhewei
    - family: Gholami
      given: Amir
    - family: Keutzer
      given: Kurt
    - family: Mahoney
      given: Michael
  citation-key: YaoPyHessian2020
  container-title: arXiv:1912.07145 [cs, math]
  issued:
    - year: 2020
      month: 3
      day: 5
  title: 'PyHessian: Neural Networks Through the Lens of the Hessian'
  type: paper-conference
  URL: http://arxiv.org/abs/1912.07145
...
