---
references:
- id: AmbrosioGradient2008
  accessed:
    - year: 2019
      month: 6
      day: 29
  author:
    - family: Ambrosio
      given: Luigi
    - family: Gigli
      given: Nicola
    - family: Savare
      given: Giuseppe
  citation-key: AmbrosioGradient2008
  collection-title: Lectures in Mathematics. ETH Zürich
  edition: '2'
  ISBN: 978-3-7643-8721-1
  issued:
    - year: 2008
  language: en
  publisher: Birkhäuser Basel
  title: 'Gradient Flows: In Metric Spaces and in the Space of Probability Measures'
  type: book
  URL: https://www.springer.com/gp/book/9783764387211

- id: AmbrosioUser2013
  accessed:
    - year: 2020
      month: 9
      day: 10
  author:
    - family: Ambrosio
      given: Luigi
    - family: Gigli
      given: Nicola
  citation-key: AmbrosioUser2013
  collection-title: Lecture Notes in Mathematics
  container-title: >-
    Modelling and Optimisation of Flows on Networks: Cetraro, Italy 2009,
    Editors: Benedetto Piccoli, Michel Rascle
  DOI: 10.1007/978-3-642-32160-3_1
  editor:
    - family: Ambrosio
      given: Luigi
    - family: Bressan
      given: Alberto
    - family: Helbing
      given: Dirk
    - family: Klar
      given: Axel
    - family: Zuazua
      given: Enrique
  event-place: Berlin, Heidelberg
  ISBN: 978-3-642-32160-3
  issued:
    - year: 2013
  language: en
  page: 1-155
  publisher: Springer
  publisher-place: Berlin, Heidelberg
  title: A User’s Guide to Optimal Transport
  type: chapter
  URL: http://www.calcvar.sns.it/media/doc/paper/195/users_guide-final.pdf

- id: AnconaBetter2017
  accessed:
    - year: 2022
      month: 10
      day: 7
  author:
    - family: Ancona
      given: Marco
    - family: Ceolini
      given: Enea
    - family: Öztireli
      given: Cengiz
    - family: Gross
      given: Markus
  citation-key: AnconaBetter2017
  DOI: 10.48550/arXiv.1711.06104
  issued:
    - year: 2017
      month: 11
      day: 16
  language: en
  title: >-
    Towards better understanding of gradient-based attribution methods for Deep
    Neural Networks
  type: article-journal
  URL: https://arxiv.org/abs/1711.06104v4

- id: BartlettDeep2021
  accessed:
    - year: 2022
      month: 8
      day: 18
  author:
    - family: Bartlett
      given: Peter L.
    - family: Montanari
      given: Andrea
    - family: Rakhlin
      given: Alexander
  citation-key: BartlettDeep2021
  container-title: Acta Numerica
  DOI: 10.1017/S0962492921000027
  ISSN: 0962-4929, 1474-0508
  issued:
    - year: 2021
      month: 5
  language: en
  page: 87-201
  publisher: Cambridge University Press
  title: 'Deep learning: a statistical viewpoint'
  type: article-journal
  URL: https://arxiv.org/abs/2103.09177v1
  volume: '30'

- id: ChizatGlobal2018
  accessed:
    - year: 2021
      month: 3
      day: 14
  author:
    - family: Chizat
      given: Lénaïc
    - family: Bach
      given: Francis
  citation-key: ChizatGlobal2018
  collection-title: NIPS'18
  container-title: >-
    Proceedings of the 32nd International Conference on Neural Information
    Processing Systems
  event-place: Red Hook, NY, USA
  event-title: NeurIPS
  issued:
    - year: 2018
      month: 12
      day: 3
  page: 3040–3050
  publisher: Curran Associates Inc.
  publisher-place: Red Hook, NY, USA
  title: >-
    On the global convergence of gradient descent for over-parameterized models
    using optimal transport
  type: paper-conference
  URL: http://arxiv.org/abs/1805.09545

- id: ChuEquivalence2022
  accessed:
    - year: 2022
      month: 11
      day: 2
  author:
    - family: Chu
      given: Casey
    - family: Minami
      given: Kentaro
    - family: Fukumizu
      given: Kenji
  citation-key: ChuEquivalence2022
  event-title: >-
    ICLR 2020 Workshop on Integration of Deep Neural Models and Differential
    Equations
  issued:
    - year: 2022
      month: 6
      day: 14
  language: en
  page: '5'
  title: >-
    The equivalence between Stein variational gradient descent and black-box
    variational inference
  type: paper-conference
  URL: https://openreview.net/forum?id=M390_F-0o

- id: DiGiovanniGraph2022
  accessed:
    - year: 2022
      month: 9
      day: 6
  author:
    - family: Di Giovanni
      given: Francesco
    - family: Rowbottom
      given: James
    - family: Chamberlain
      given: Benjamin P.
    - family: Markovich
      given: Thomas
    - family: Bronstein
      given: Michael M.
  citation-key: DiGiovanniGraph2022
  DOI: 10.48550/arXiv.2206.10991
  issued:
    - year: 2022
      month: 8
      day: 15
  number: arXiv:2206.10991
  publisher: arXiv
  title: Graph Neural Networks as Gradient Flows
  type: article
  URL: http://arxiv.org/abs/2206.10991

- id: GalyFajouFlexible2021
  accessed:
    - year: 2022
      month: 7
      day: 25
  author:
    - family: Galy-Fajou
      given: Théo
    - family: Perrone
      given: Valerio
    - family: Opper
      given: Manfred
  citation-key: GalyFajouFlexible2021
  container-title: Entropy
  DOI: 10.3390/e23080990
  ISSN: 1099-4300
  issue: '8'
  issued:
    - year: 2021
      month: 8
  language: en
  number: '8'
  page: '990'
  publisher: Multidisciplinary Digital Publishing Institute
  title: >-
    Flexible and Efficient Inference with Particles for the Variational Gaussian
    Approximation
  type: article-journal
  URL: https://www.mdpi.com/1099-4300/23/8/990
  volume: '23'

- id: GarbunoInigoInteracting2020
  accessed:
    - year: 2020
      month: 7
      day: 23
  author:
    - family: Garbuno-Inigo
      given: Alfredo
    - family: Hoffmann
      given: Franca
    - family: Li
      given: Wuchen
    - family: Stuart
      given: Andrew M.
  citation-key: GarbunoInigoInteracting2020
  container-title: SIAM Journal on Applied Dynamical Systems
  container-title-short: SIAM J. Appl. Dyn. Syst.
  DOI: 10.1137/19M1251655
  issue: '1'
  issued:
    - year: 2020
      month: 1
      day: 1
  page: 412-441
  publisher: Society for Industrial and Applied Mathematics
  title: >-
    Interacting Langevin Diffusions: Gradient Structure and Ensemble Kalman
    Sampler
  type: article-journal
  URL: http://arxiv.org/abs/1903.08866
  volume: '19'

- id: GuWhy2022
  accessed:
    - year: 2023
      month: 9
      day: 27
  author:
    - family: Gu
      given: Xinran
    - family: Lyu
      given: Kaifeng
    - family: Huang
      given: Longbo
    - family: Arora
      given: Sanjeev
  citation-key: GuWhy2022
  event-title: The Eleventh International Conference on Learning Representations
  issued:
    - year: 2022
      month: 9
      day: 29
  language: en
  title: Why (and When) does Local SGD Generalize Better than SGD?
  type: paper-conference
  URL: https://openreview.net/forum?id=svCcui6Drl

- id: HinzeMosquito2021
  accessed:
    - year: 2022
      month: 11
      day: 8
  author:
    - family: Hinze
      given: Annika
    - family: Lantz
      given: Jörgen
    - family: Hill
      given: Sharon R.
    - family: Ignell
      given: Rickard
  citation-key: HinzeMosquito2021
  container-title: Frontiers in Behavioral Neuroscience
  container-title-short: Front. Behav. Neurosci.
  DOI: 10.3389/fnbeh.2021.643693
  ISSN: 1662-5153
  issued:
    - year: 2021
      month: 3
      day: 11
  language: en
  page: '643693'
  title: >-
    Mosquito Host Seeking in 3D Using a Versatile Climate-Controlled Wind Tunnel
    System
  type: article-journal
  volume: '15'

- id: HochreiterGradient2001
  author:
    - family: Hochreiter
      given: Sepp
    - family: Bengio
      given: Yoshua
    - family: Frasconi
      given: Paolo
    - family: Schmidhuber
      given: Jürgen
  citation-key: HochreiterGradient2001
  container-title: A field guide to dynamical recurrent neural networks
  issued:
    - year: 2001
  publisher: IEEE Press
  title: >-
    Gradient Flow in Recurrent Nets: the Difficulty of Learning Long-Term
    Dependencies
  type: chapter
  URL: http://www.bioinf.jku.at/publications/older/ch7.pdf

- id: LiStochastic2019
  author:
    - family: Li
      given: Qianxiao
    - family: Tai
      given: Cheng
    - family: Weinan
      given: E.
  citation-key: LiStochastic2019
  container-title: The Journal of Machine Learning Research
  issued:
    - year: 2019
      month: 1
      day: 1
  page: 1474–1520
  title: >-
    Stochastic modified equations and dynamics of stochastic gradient algorithms
    I: mathematical foundations
  type: paper-conference
  URL: http://arxiv.org/abs/1811.01558
  volume: '20'

- id: LiuStein2016
  author:
    - family: Liu
      given: Qiang
  citation-key: LiuStein2016
  issued:
    - year: 2016
  language: en
  page: '6'
  title: 'Stein Variational Gradient Descent: Theory and Applications'
  type: article-journal
  URL: http://approximateinference.org/accepted/Liu2016.pdf

- id: LiuStein2017
  accessed:
    - year: 2022
      month: 11
      day: 2
  author:
    - family: Liu
      given: Qiang
  citation-key: LiuStein2017
  DOI: 10.48550/arXiv.1704.07520
  issued:
    - year: 2017
      month: 11
      day: 14
  number: arXiv:1704.07520
  publisher: arXiv
  title: Stein Variational Gradient Descent as Gradient Flow
  type: article
  URL: http://arxiv.org/abs/1704.07520

- id: LiValidity2021
  accessed:
    - year: 2023
      month: 9
      day: 27
  author:
    - family: Li
      given: Zhiyuan
    - family: Malladi
      given: Sadhika
    - family: Arora
      given: Sanjeev
  citation-key: LiValidity2021
  container-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2021
  page: 12712–12725
  publisher: Curran Associates, Inc.
  title: >-
    On the Validity of Modeling SGD with Stochastic Differential Equations
    (SDEs)
  type: paper-conference
  URL: http://arxiv.org/abs/2102.12470
  volume: '34'

- id: LiWhat2021
  accessed:
    - year: 2023
      month: 9
      day: 27
  author:
    - family: Li
      given: Zhiyuan
    - family: Wang
      given: Tianhao
    - family: Arora
      given: Sanjeev
  citation-key: LiWhat2021
  event-title: International Conference on Learning Representations
  issued:
    - year: 2021
      month: 10
      day: 6
  language: en
  title: What Happens after SGD Reaches Zero Loss? --A Mathematical Framework
  type: paper-conference
  URL: http://arxiv.org/abs/2110.06914

- id: LjungStochastic1992
  accessed:
    - year: 2022
      month: 9
      day: 5
  author:
    - family: Ljung
      given: Lennart
    - family: Pflug
      given: Georg
    - family: Walk
      given: Harro
  citation-key: LjungStochastic1992
  container-title-short: DMV Seminar vol. 17
  DOI: 10.1007/978-3-0348-8609-3
  event-place: Basel
  ISBN: 978-3-7643-2733-0 978-3-0348-8609-3
  issued:
    - year: 1992
  language: en
  publisher: Birkhäuser
  publisher-place: Basel
  title: Stochastic Approximation and Optimization of Random Systems
  type: book
  URL: http://publications.mfo.de/handle/mfo/500

- id: LyuUnderstanding2023
  accessed:
    - year: 2023
      month: 9
      day: 27
  author:
    - family: Lyu
      given: Kaifeng
    - family: Li
      given: Zhiyuan
    - family: Arora
      given: Sanjeev
  citation-key: LyuUnderstanding2023
  DOI: 10.48550/arXiv.2206.07085
  event-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2023
      month: 1
      day: 16
  publisher: arXiv
  title: >-
    Understanding the Generalization Benefit of Normalization Layers: Sharpness
    Reduction
  type: paper-conference
  URL: http://arxiv.org/abs/2206.07085

- id: MalladiSDEs2022
  accessed:
    - year: 2023
      month: 9
      day: 27
  author:
    - family: Malladi
      given: Sadhika
    - family: Lyu
      given: Kaifeng
    - family: Panigrahi
      given: Abhishek
    - family: Arora
      given: Sanjeev
  citation-key: MalladiSDEs2022
  container-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2022
      month: 12
      day: 6
  language: en
  page: 7697-7711
  title: On the SDEs and Scaling Rules for Adaptive Gradient Algorithms
  type: paper-conference
  URL: https://arxiv.org/abs/2205.10287
  volume: '35'

- id: MandtStochastic2017
  accessed:
    - year: 2018
      month: 4
      day: 3
  author:
    - family: Mandt
      given: Stephan
    - family: Hoffman
      given: Matthew D.
    - family: Blei
      given: David M.
  citation-key: MandtStochastic2017
  container-title: JMLR
  issued:
    - year: 2017
      month: 4
      day: 13
  title: Stochastic Gradient Descent as Approximate Bayesian Inference
  type: article-journal
  URL: http://arxiv.org/abs/1704.04289

- id: SchillingsAnalysis2017
  accessed:
    - year: 2022
      month: 8
      day: 23
  author:
    - family: Schillings
      given: Claudia
    - family: Stuart
      given: Andrew M.
  citation-key: SchillingsAnalysis2017
  container-title: SIAM Journal on Numerical Analysis
  container-title-short: SIAM J. Numer. Anal.
  DOI: 10.1137/16M105959X
  ISSN: 0036-1429
  issue: '3'
  issued:
    - year: 2017
      month: 1
      day: 1
  page: 1264–1290
  publisher: Society for Industrial and Applied Mathematics
  title: Analysis of the Ensemble Kalman Filter for Inverse Problems
  type: article-journal
  URL: http://arxiv.org/abs/1602.02020
  volume: '55'

- id: WangMarginal2023
  accessed:
    - year: 2023
      month: 9
      day: 27
  author:
    - family: Wang
      given: Runzhe
    - family: Malladi
      given: Sadhika
    - family: Wang
      given: Tianhao
    - family: Lyu
      given: Kaifeng
    - family: Li
      given: Zhiyuan
  citation-key: WangMarginal2023
  DOI: 10.48550/arXiv.2307.15196
  issued:
    - year: 2023
      month: 7
      day: 27
  number: arXiv:2307.15196
  publisher: arXiv
  title: The Marginal Value of Momentum for Small Learning Rate SGD
  type: article
  URL: http://arxiv.org/abs/2307.15196
...
