---
references:
- id: AhnBayesian2012
  accessed:
    - year: 2022
      month: 8
      day: 16
  author:
    - family: Ahn
      given: Sungjin
    - family: Korattikara
      given: Anoop
    - family: Welling
      given: Max
  citation-key: AhnBayesian2012
  collection-title: ICML'12
  container-title: >-
    Proceedings of the 29th International Coference on International Conference
    on Machine Learning
  event-place: Madison, WI, USA
  event-title: ICML
  ISBN: 978-1-4503-1285-1
  issued:
    - year: 2012
      month: 6
      day: 26
  page: 1771–1778
  publisher: Omnipress
  publisher-place: Madison, WI, USA
  title: Bayesian posterior sampling via stochastic gradient fisher scoring
  type: paper-conference
  URL: https://arxiv.org/abs/1206.6380

- id: AlexosStructured2022
  accessed:
    - year: 2022
      month: 8
      day: 18
  author:
    - family: Alexos
      given: Antonios
    - family: Boyd
      given: Alex J.
    - family: Mandt
      given: Stephan
  citation-key: AlexosStructured2022
  container-title: Proceedings of the 39th International Conference on Machine Learning
  event-title: International Conference on Machine Learning
  ISSN: 2640-3498
  issued:
    - year: 2022
      month: 6
      day: 28
  language: en
  page: 414-434
  publisher: PMLR
  title: Structured Stochastic Gradient MCMC
  type: paper-conference
  URL: https://proceedings.mlr.press/v162/alexos22a.html

- id: AryaAutomatic2022
  accessed:
    - year: 2022
      month: 12
      day: 2
  author:
    - family: Arya
      given: Gaurav
    - family: Schauer
      given: Moritz
    - family: Schäfer
      given: Frank
    - family: Rackauckas
      given: Christopher Vincent
  citation-key: AryaAutomatic2022
  event-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2022
      month: 10
      day: 31
  language: en
  title: Automatic Differentiation of Programs with Discrete Randomness
  type: paper-conference
  URL: https://openreview.net/forum?id=V22VeIZ9QU

- id: BachNonAsymptotic2011
  accessed:
    - year: 2014
      month: 6
      day: 11
  author:
    - family: Bach
      given: Francis
    - family: Moulines
      given: Eric
  citation-key: BachNonAsymptotic2011
  container-title: Advances in Neural Information Processing Systems (NIPS)
  event-place: Spain
  issued:
    - year: 2011
  language: English
  page: '-'
  publisher-place: Spain
  title: >-
    Non-Asymptotic Analysis of Stochastic Approximation Algorithms for Machine
    Learning
  type: paper-conference
  URL: http://hal.archives-ouvertes.fr/hal-00608041

- id: BachNonstronglyconvex2013
  author:
    - family: Bach
      given: Francis R.
    - family: Moulines
      given: Eric
  citation-key: BachNonstronglyconvex2013
  container-title: arXiv:1306.2119 [cs, math, stat]
  issued:
    - year: 2013
      month: 6
      day: 10
  page: 773–781
  title: >-
    Non-strongly-convex smooth stochastic approximation with convergence rate
    O(1/n)
  type: paper-conference
  URL: https://arxiv.org/abs/1306.2119v1

- id: BenaimDynamics1999
  accessed:
    - year: 2020
      month: 2
      day: 2
  author:
    - family: Benaïm
      given: Michel
  citation-key: BenaimDynamics1999
  collection-title: Lecture Notes in Math.
  container-title: Séminaire de probabilités de Strasbourg
  DOI: 10.1007/BFb0096509
  event-place: Berlin
  ISBN: 978-3-540-66342-3
  issued:
    - year: 1999
  language: en
  page: 1-68
  publisher: Springer, Berlin
  publisher-place: Berlin
  title: Dynamics of stochastic approximation algorithms
  type: chapter
  URL: http://www.numdam.org/item/?id=SPS_1999__33__1_0
  volume: '33'

- id: BensoussanMachine2020
  accessed:
    - year: 2020
      month: 11
      day: 30
  author:
    - family: Bensoussan
      given: Alain
    - family: Li
      given: Yiqun
    - family: Nguyen
      given: Dinh Phan Cao
    - family: Tran
      given: Minh-Binh
    - family: Yam
      given: Sheung Chi Phillip
    - family: Zhou
      given: Xiang
  citation-key: BensoussanMachine2020
  container-title: arXiv:2006.05604 [cs, math, stat]
  issued:
    - year: 2020
      month: 6
      day: 9
  title: Machine Learning and Control Theory
  type: article-journal
  URL: http://arxiv.org/abs/2006.05604

- id: BotevImportance2015
  accessed:
    - year: 2018
      month: 3
      day: 5
  author:
    - family: Botev
      given: Zdravko I.
    - family: Lloyd
      given: Chris J.
  citation-key: BotevImportance2015
  container-title: Electronic Journal of Statistics
  container-title-short: Electron. J. Statist.
  DOI: 10.1214/15-EJS1071
  ISSN: 1935-7524
  issue: '2'
  issued:
    - year: 2015
  language: EN
  page: 2058-2075
  title: >-
    Importance accelerated Robbins-Monro recursion with applications to
    parametric confidence limits
  type: article-journal
  volume: '9'

- id: BottouLarge2004
  author:
    - family: Bottou
      given: Léon
    - family: LeCun
      given: Yann
  citation-key: BottouLarge2004
  container-title: Advances in Neural Information Processing Systems 16
  editor:
    - family: Thrun
      given: Sebastian
    - family: Saul
      given: Lawrence
    - family: Schölkopf
      given: Bernhard
  event-place: Cambridge, MA
  issued:
    - year: 2004
  publisher: MIT Press
  publisher-place: Cambridge, MA
  title: Large Scale Online Learning
  type: chapter
  URL: http://leon.bottou.org/papers/bottou-lecun-2004

- id: BottouLargescale2010
  accessed:
    - year: 2014
      month: 4
      day: 21
  author:
    - family: Bottou
      given: Léon
  citation-key: BottouLargescale2010
  container-title: >-
    Proceedings of the 19th International Conference on Computational Statistics
    (COMPSTAT'2010)
  event-place: Paris, France
  ISBN: 978-3-7908-2603-6 978-3-7908-2604-3
  issued:
    - year: 2010
      month: 8
  page: 177–186
  publisher: Springer
  publisher-place: Paris, France
  title: Large-scale machine learning with stochastic gradient descent
  type: paper-conference
  URL: http://leon.bottou.org/papers/bottou-2010

- id: BottouOnline1998
  author:
    - family: Bottou
      given: Léon
  citation-key: BottouOnline1998
  container-title: Online Learning and Neural Networks
  editor:
    - family: Saad
      given: David
  event-place: Cambridge, UK
  issued:
    - year: 1998
  page: '142'
  publisher: Cambridge University Press
  publisher-place: Cambridge, UK
  title: Online Algorithms and Stochastic Approximations
  type: chapter
  URL: http://leon.bottou.org/publications/pdf/online-1998.pdf
  volume: '17'

- id: BottouOptimization2016
  accessed:
    - year: 2016
      month: 6
      day: 17
  author:
    - family: Bottou
      given: Léon
    - family: Curtis
      given: Frank E.
    - family: Nocedal
      given: Jorge
  citation-key: BottouOptimization2016
  container-title: arXiv:1606.04838 [cs, math, stat]
  issued:
    - year: 2016
      month: 6
      day: 15
  title: Optimization Methods for Large-Scale Machine Learning
  type: article-journal
  URL: http://arxiv.org/abs/1606.04838

- id: BottouStochastic1991
  author:
    - family: Bottou
      given: Léon
  citation-key: BottouStochastic1991
  container-title: Proceedings of Neuro-Nîmes 91
  event-place: Nimes, France
  issued:
    - year: 1991
  publisher: EC2
  publisher-place: Nimes, France
  title: Stochastic Gradient Learning in Neural Networks
  type: paper-conference
  URL: http://leon.bottou.org/papers/bottou-91c

- id: BottouTradeoffs2008
  author:
    - family: Bottou
      given: Léon
    - family: Bousquet
      given: Olivier
  citation-key: BottouTradeoffs2008
  container-title: Advances in Neural Information Processing Systems
  editor:
    - family: Platt
      given: J.C.
    - family: Koller
      given: D.
    - family: Singer
      given: Y.
    - family: Roweis
      given: S.
  issued:
    - year: 2008
  page: 161–168
  publisher: NIPS Foundation (http://books.nips.cc)
  title: The Tradeoffs of Large Scale Learning
  type: chapter
  URL: http://leon.bottou.org/papers/bottou-bousquet-2008
  volume: '20'

- id: BubeckConvex2015
  accessed:
    - year: 2015
      month: 12
      day: 2
  author:
    - family: Bubeck
      given: Sébastien
  citation-key: BubeckConvex2015
  collection-title: Foundations and Trends in Machine Learning
  DOI: 10.1561/2200000050
  issued:
    - year: 2015
  language: en
  publisher: Now Publishers
  title: 'Convex Optimization: Algorithms and Complexity'
  type: book
  URL: http://arxiv.org/abs/1405.4980
  volume: '8'

- id: CevherConvex2014
  accessed:
    - year: 2014
      month: 11
      day: 6
  author:
    - family: Cevher
      given: Volkan
    - family: Becker
      given: Stephen
    - family: Schmidt
      given: Mark
  citation-key: CevherConvex2014
  container-title: IEEE Signal Processing Magazine
  DOI: 10.1109/MSP.2014.2329397
  ISSN: 1053-5888
  issue: '5'
  issued:
    - year: 2014
      month: 9
  page: 32-43
  title: Convex Optimization for Big Data
  type: article-journal
  URL: http://arxiv.org/abs/1411.0972
  volume: '31'

- id: ChaudhariEntropySGD2017
  accessed:
    - year: 2023
      month: 9
      day: 6
  author:
    - family: Chaudhari
      given: Pratik
    - family: Choromanska
      given: Anna
    - family: Soatto
      given: Stefano
    - family: LeCun
      given: Yann
    - family: Baldassi
      given: Carlo
    - family: Borgs
      given: Christian
    - family: Chayes
      given: Jennifer
    - family: Sagun
      given: Levent
    - family: Zecchina
      given: Riccardo
  citation-key: ChaudhariEntropySGD2017
  DOI: 10.48550/arXiv.1611.01838
  issued:
    - year: 2017
      month: 4
      day: 21
  number: arXiv:1611.01838
  publisher: arXiv
  title: 'Entropy-SGD: Biasing Gradient Descent Into Wide Valleys'
  type: article
  URL: http://arxiv.org/abs/1611.01838
  version: '5'

- id: ChenSmoothing2012
  accessed:
    - year: 2016
      month: 7
      day: 13
  author:
    - family: Chen
      given: Xiaojun
  citation-key: ChenSmoothing2012
  container-title: Mathematical Programming
  container-title-short: Math. Program.
  DOI: 10.1007/s10107-012-0569-0
  ISSN: 0025-5610, 1436-4646
  issue: '1'
  issued:
    - year: 2012
      month: 6
      day: 22
  language: en
  page: 71-99
  title: Smoothing methods for nonsmooth, nonconvex minimization
  type: article-journal
  URL: https://www.polyu.edu.hk/ama/staff/xjchen/MPB2012_reprint.pdf
  volume: '134'

- id: ChenStationary2021
  accessed:
    - year: 2022
      month: 9
      day: 21
  author:
    - family: Chen
      given: Zaiwei
    - family: Mou
      given: Shancong
    - family: Maguluri
      given: Siva Theja
  citation-key: ChenStationary2021
  issued:
    - year: 2021
      month: 11
      day: 11
  language: en
  number: arXiv:2111.06328
  publisher: arXiv
  title: >-
    Stationary Behavior of Constant Stepsize SGD Type Algorithms: An Asymptotic
    Characterization
  type: article
  URL: http://arxiv.org/abs/2111.06328

- id: ChenStochastic2014
  accessed:
    - year: 2022
      month: 8
      day: 16
  author:
    - family: Chen
      given: Tianqi
    - family: Fox
      given: Emily
    - family: Guestrin
      given: Carlos
  citation-key: ChenStochastic2014
  container-title: Proceedings of the 31st International Conference on Machine Learning
  DOI: 10.48550/arXiv.1402.4102
  event-place: Beijing, China
  event-title: International Conference on Machine Learning
  ISSN: 1938-7228
  issued:
    - year: 2014
      month: 6
      day: 18
  language: en
  page: 1683-1691
  publisher: PMLR
  publisher-place: Beijing, China
  title: Stochastic Gradient Hamiltonian Monte Carlo
  type: paper-conference
  URL: https://proceedings.mlr.press/v32/cheni14.html

- id: DiGiovanniGraph2022
  accessed:
    - year: 2022
      month: 9
      day: 6
  author:
    - family: Di Giovanni
      given: Francesco
    - family: Rowbottom
      given: James
    - family: Chamberlain
      given: Benjamin P.
    - family: Markovich
      given: Thomas
    - family: Bronstein
      given: Michael M.
  citation-key: DiGiovanniGraph2022
  DOI: 10.48550/arXiv.2206.10991
  issued:
    - year: 2022
      month: 8
      day: 15
  number: arXiv:2206.10991
  publisher: arXiv
  title: Graph Neural Networks as Gradient Flows
  type: article
  URL: http://arxiv.org/abs/2206.10991

- id: DomingosEvery2020
  accessed:
    - year: 2020
      month: 12
      day: 12
  author:
    - family: Domingos
      given: Pedro
  citation-key: DomingosEvery2020
  container-title: arXiv:2012.00152 [cs, stat]
  issued:
    - year: 2020
      month: 11
      day: 30
  title: Every Model Learned by Gradient Descent Is Approximately a Kernel Machine
  type: article-journal
  URL: http://arxiv.org/abs/2012.00152

- id: DuchiAdaptive2011
  accessed:
    - year: 2015
      month: 6
      day: 18
  author:
    - family: Duchi
      given: John
    - family: Hazan
      given: Elad
    - family: Singer
      given: Yoram
  citation-key: DuchiAdaptive2011
  container-title: Journal of Machine Learning Research
  ISSN: 1532-4435
  issue: Jul
  issued:
    - year: 2011
      month: 7
  page: 2121–2159
  title: Adaptive Subgradient Methods for Online Learning and Stochastic Optimization
  type: article-journal
  URL: http://www.jmlr.org/papers/v12/duchi11a.html
  volume: '12'

- id: FengInverse2021
  accessed:
    - year: 2023
      month: 9
      day: 6
  author:
    - family: Feng
      given: Yu
    - family: Tu
      given: Yuhai
  citation-key: FengInverse2021
  container-title: Proceedings of the National Academy of Sciences
  DOI: 10.1073/pnas.2015617118
  issue: '9'
  issued:
    - year: 2021
      month: 3
      day: 2
  page: e2015617118
  publisher: Proceedings of the National Academy of Sciences
  title: >-
    The inverse variance–flatness relation in stochastic gradient descent is
    critical for finding flat minima
  type: article-journal
  volume: '118'

- id: FriedlanderHybrid2012
  accessed:
    - year: 2015
      month: 2
      day: 1
  author:
    - family: Friedlander
      given: Michael P.
    - family: Schmidt
      given: Mark
  citation-key: FriedlanderHybrid2012
  container-title: SIAM Journal on Scientific Computing
  container-title-short: SIAM J. Sci. Comput.
  DOI: 10.1137/110830629
  ISSN: 1064-8275, 1095-7197
  issue: '3'
  issued:
    - year: 2012
      month: 1
      day: 1
  page: A1380-A1405
  title: Hybrid Deterministic-Stochastic Methods for Data Fitting
  type: article-journal
  URL: http://arxiv.org/abs/1104.2373
  volume: '34'

- id: GhadimiAccelerated2013
  accessed:
    - year: 2014
      month: 9
      day: 30
  author:
    - family: Ghadimi
      given: Saeed
    - family: Lan
      given: Guanghui
  citation-key: GhadimiAccelerated2013
  container-title: arXiv:1310.3787 [math]
  issued:
    - year: 2013
      month: 10
      day: 14
  title: >-
    Accelerated Gradient Methods for Nonconvex Nonlinear and Stochastic
    Programming
  type: article-journal
  URL: http://arxiv.org/abs/1310.3787

- id: GhadimiStochastic2013
  accessed:
    - year: 2014
      month: 9
      day: 30
  author:
    - family: Ghadimi
      given: Saeed
    - family: Lan
      given: Guanghui
  citation-key: GhadimiStochastic2013
  container-title: SIAM Journal on Optimization
  DOI: 10.1137/120880811
  issue: '4'
  issued:
    - year: 2013
      month: 9
      day: 21
  page: 2341–2368
  title: >-
    Stochastic First- and Zeroth-order Methods for Nonconvex Stochastic
    Programming
  type: article-journal
  URL: http://arxiv.org/abs/1309.5549
  volume: '23'

- id: GohWhy2017
  accessed:
    - year: 2022
      month: 4
      day: 23
  author:
    - family: Goh
      given: Gabriel
  citation-key: GohWhy2017
  container-title: Distill
  container-title-short: Distill
  DOI: 10.23915/distill.00006
  ISSN: 2476-0757
  issue: '4'
  issued:
    - year: 2017
      month: 4
      day: 4
  language: en
  page: e6
  title: Why Momentum Really Works
  type: article-journal
  URL: http://distill.pub/2017/momentum
  volume: '2'

- id: HazanConvexity2015
  accessed:
    - year: 2017
      month: 10
      day: 13
  author:
    - family: Hazan
      given: Elad
    - family: Levy
      given: Kfir
    - family: Shalev-Shwartz
      given: Shai
  citation-key: HazanConvexity2015
  container-title: Advances in Neural Information Processing Systems 28
  editor:
    - family: Cortes
      given: C.
    - family: Lawrence
      given: N. D.
    - family: Lee
      given: D. D.
    - family: Sugiyama
      given: M.
    - family: Garnett
      given: R.
  issued:
    - year: 2015
  page: 1594–1602
  publisher: Curran Associates, Inc.
  title: 'Beyond Convexity: Stochastic Quasi-Convex Optimization'
  type: paper-conference
  URL: >-
    http://papers.nips.cc/paper/5718-beyond-convexity-stochastic-quasi-convex-optimization.pdf

- id: HeydeMartingale1974
  accessed:
    - year: 2017
      month: 10
      day: 12
  author:
    - family: Heyde
      given: C. C.
  citation-key: HeydeMartingale1974
  container-title: Stochastic Processes and their Applications
  container-title-short: Stochastic Processes and their Applications
  DOI: 10.1016/0304-4149(74)90004-0
  ISSN: 0304-4149
  issue: '4'
  issued:
    - year: 1974
      month: 10
      day: 1
  page: 359-370
  title: >-
    On martingale limit theory and strong convergence results for stochastic
    approximation procedures
  type: article-journal
  volume: '2'

- id: HuAccelerated2009
  accessed:
    - year: 2014
      month: 9
      day: 30
  author:
    - family: Hu
      given: Chonghai
    - family: Pan
      given: Weike
    - family: Kwok
      given: James T.
  citation-key: HuAccelerated2009
  container-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2009
  page: 781–789
  publisher: Curran Associates, Inc.
  title: Accelerated gradient methods for stochastic optimization and online learning
  type: paper-conference
  URL: >-
    http://papers.nips.cc/paper/3817-accelerated-gradient-methods-for-stochastic-optimization-and-online-learning

- id: JakoveticConvergence2014
  author:
    - family: Jakovetic
      given: D.
    - family: Freitas Xavier
      given: J.M.
    - family: Moura
      given: J.M.F.
  citation-key: JakoveticConvergence2014
  container-title: IEEE Transactions on Signal Processing
  DOI: 10.1109/TSP.2013.2291221
  ISSN: 1053-587X
  issue: '4'
  issued:
    - year: 2014
      month: 2
  page: 868-882
  title: >-
    Convergence Rates of Distributed Nesterov-Like Gradient Methods on Random
    Networks
  type: article-journal
  URL: http://arxiv.org/abs/1308.0916
  volume: '62'

- id: KidambiInsufficiency2023
  accessed:
    - year: 2023
      month: 5
      day: 18
  author:
    - family: Kidambi
      given: Rahul
    - family: Netrapalli
      given: Praneeth
    - family: Jain
      given: Prateek
    - family: Kakade
      given: Sham M.
  citation-key: KidambiInsufficiency2023
  event-title: International Conference on Learning Representations
  issued:
    - year: 2023
      month: 5
      day: 14
  language: en
  title: >-
    On the insufficiency of existing momentum schemes for Stochastic
    Optimization
  type: paper-conference
  URL: https://openreview.net/forum?id=rJTutzbA-

- id: KingmaAdam2015
  accessed:
    - year: 2016
      month: 3
      day: 30
  author:
    - family: Kingma
      given: Diederik
    - family: Ba
      given: Jimmy
  citation-key: KingmaAdam2015
  container-title: Proceeding of ICLR
  issued:
    - year: 2015
  title: 'Adam: A Method for Stochastic Optimization'
  type: article-journal
  URL: http://arxiv.org/abs/1412.6980

- id: LaiStochastic2003
  author:
    - family: Lai
      given: Tze Leung
  citation-key: LaiStochastic2003
  container-title: The Annals of Statistics
  container-title-short: Ann. Statist.
  DOI: 10.1214/aos/1051027873
  ISSN: 0090-5364, 2168-8966
  issue: '2'
  issued:
    - year: 2003
      month: 4
  page: 391-406
  title: Stochastic Approximation
  type: article-journal
  URL: https://statistics.stanford.edu/sites/default/files/2002-31.pdf
  volume: '31'

- id: LeeFirstorder2017
  accessed:
    - year: 2020
      month: 12
      day: 14
  author:
    - family: Lee
      given: Jason D.
    - family: Panageas
      given: Ioannis
    - family: Piliouras
      given: Georgios
    - family: Simchowitz
      given: Max
    - family: Jordan
      given: Michael I.
    - family: Recht
      given: Benjamin
  citation-key: LeeFirstorder2017
  container-title: arXiv:1710.07406 [cs, math, stat]
  issued:
    - year: 2017
      month: 10
      day: 19
  title: First-order Methods Almost Always Avoid Saddle Points
  type: article-journal
  URL: http://arxiv.org/abs/1710.07406

- id: LeeGradient2016
  accessed:
    - year: 2020
      month: 12
      day: 14
  author:
    - family: Lee
      given: Jason D.
    - family: Simchowitz
      given: Max
    - family: Jordan
      given: Michael I.
    - family: Recht
      given: Benjamin
  citation-key: LeeGradient2016
  container-title: arXiv:1602.04915 [cs, math, stat]
  issued:
    - year: 2016
      month: 3
      day: 4
  title: Gradient Descent Converges to Minimizers
  type: article-journal
  URL: http://arxiv.org/abs/1602.04915

- id: LiuStein2019
  accessed:
    - year: 2019
      month: 11
      day: 11
  author:
    - family: Liu
      given: Qiang
    - family: Wang
      given: Dilin
  citation-key: LiuStein2019
  container-title: Advances In Neural Information Processing Systems
  event-title: NIPS 2016
  issued:
    - year: 2019
      month: 9
      day: 9
  title: >-
    Stein Variational Gradient Descent: A general purpose Bayesian inference
    algorithm
  type: paper-conference
  URL: http://arxiv.org/abs/1608.04471

- id: LjungStochastic1992
  accessed:
    - year: 2022
      month: 9
      day: 5
  author:
    - family: Ljung
      given: Lennart
    - family: Pflug
      given: Georg
    - family: Walk
      given: Harro
  citation-key: LjungStochastic1992
  container-title-short: DMV Seminar vol. 17
  DOI: 10.1007/978-3-0348-8609-3
  event-place: Basel
  ISBN: 978-3-7643-2733-0 978-3-0348-8609-3
  issued:
    - year: 1992
  language: en
  publisher: Birkhäuser
  publisher-place: Basel
  title: Stochastic Approximation and Optimization of Random Systems
  type: book
  URL: http://publications.mfo.de/handle/mfo/500

- id: MaclaurinEarly2015
  accessed:
    - year: 2022
      month: 8
      day: 19
  author:
    - family: Maclaurin
      given: Dougal
    - family: Duvenaud
      given: David
    - family: Adams
      given: Ryan P.
  citation-key: MaclaurinEarly2015
  container-title: >-
    Proceedings of the 19th International Conference on Artificial Intelligence
    and Statistics
  DOI: 10.48550/arXiv.1504.01344
  event-title: Artificial Intelligence and Statistics
  ISSN: 1938-7228
  issued:
    - year: 2015
      month: 4
      day: 6
  page: 1070-1077
  publisher: arXiv
  title: Early Stopping as Nonparametric Variational Inference
  type: paper-conference
  URL: http://arxiv.org/abs/1504.01344

- id: MaDiving2017
  author:
    - family: Ma
      given: Siyuan
    - family: Belkin
      given: Mikhail
  citation-key: MaDiving2017
  container-title: arXiv:1703.10622 [cs, stat]
  issued:
    - year: 2017
      month: 3
      day: 30
  title: >-
    Diving into the shallows: a computational perspective on large-scale shallow
    learning
  type: article-journal
  URL: http://arxiv.org/abs/1703.10622

- id: MairalStochastic2013
  accessed:
    - year: 2016
      month: 4
      day: 2
  author:
    - family: Mairal
      given: Julien
  citation-key: MairalStochastic2013
  container-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2013
  page: 2283–2291
  title: Stochastic majorization-minimization algorithms for large-scale optimization
  type: paper-conference
  URL: >-
    http://papers.nips.cc/paper/5129-stochastic-majorization-minimization-algorithms-for-large-scale-optimization

- id: MandtStochastic2017
  accessed:
    - year: 2018
      month: 4
      day: 3
  author:
    - family: Mandt
      given: Stephan
    - family: Hoffman
      given: Matthew D.
    - family: Blei
      given: David M.
  citation-key: MandtStochastic2017
  container-title: JMLR
  issued:
    - year: 2017
      month: 4
      day: 13
  title: Stochastic Gradient Descent as Approximate Bayesian Inference
  type: article-journal
  URL: http://arxiv.org/abs/1704.04289

- id: McMahanAd2013
  accessed:
    - year: 2016
      month: 3
      day: 30
  author:
    - family: McMahan
      given: H. Brendan
    - family: Holt
      given: Gary
    - family: Sculley
      given: D.
    - family: Young
      given: Michael
    - family: Ebner
      given: Dietmar
    - family: Grady
      given: Julian
    - family: Nie
      given: Lan
    - family: Phillips
      given: Todd
    - family: Davydov
      given: Eugene
    - family: Golovin
      given: Daniel
    - family: Chikkerur
      given: Sharat
    - family: Liu
      given: Dan
    - family: Wattenberg
      given: Martin
    - family: Hrafnkelsson
      given: Arnar Mar
    - family: Boulos
      given: Tom
    - family: Kubica
      given: Jeremy
  citation-key: McMahanAd2013
  collection-title: KDD '13
  container-title: >-
    Proceedings of the 19th ACM SIGKDD International Conference on Knowledge
    Discovery and Data Mining
  DOI: 10.1145/2487575.2488200
  event-place: New York, NY, USA
  ISBN: 978-1-4503-2174-7
  issued:
    - year: 2013
  page: 1222–1230
  publisher: ACM
  publisher-place: New York, NY, USA
  title: 'Ad Click Prediction: A View from the Trenches'
  type: paper-conference
  URL: http://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf

- id: MitliagkasAsynchrony2016
  accessed:
    - year: 2017
      month: 7
      day: 21
  author:
    - family: Mitliagkas
      given: Ioannis
    - family: Zhang
      given: Ce
    - family: Hadjis
      given: Stefan
    - family: Ré
      given: Christopher
  citation-key: MitliagkasAsynchrony2016
  container-title: arXiv:1605.09774 [cs, math, stat]
  issued:
    - year: 2016
      month: 5
      day: 31
  title: Asynchrony begets Momentum, with an Application to Deep Learning
  type: article-journal
  URL: http://arxiv.org/abs/1605.09774

- id: NeuInformationTheoretic2021
  accessed:
    - year: 2022
      month: 2
      day: 14
  author:
    - family: Neu
      given: Gergely
    - family: Dziugaite
      given: Gintare Karolina
    - family: Haghifam
      given: Mahdi
    - family: Roy
      given: Daniel M.
  citation-key: NeuInformationTheoretic2021
  container-title: arXiv:2102.00931 [cs, stat]
  issued:
    - year: 2021
      month: 8
      day: 15
  title: Information-Theoretic Generalization Bounds for Stochastic Gradient Descent
  type: article-journal
  URL: https://arxiv.org/abs/2102.00931v1

- id: NguyenStochastic2017
  accessed:
    - year: 2017
      month: 10
      day: 12
  author:
    - family: Nguyen
      given: Lam M.
    - family: Liu
      given: Jie
    - family: Scheinberg
      given: Katya
    - family: Takáč
      given: Martin
  citation-key: NguyenStochastic2017
  container-title: arXiv:1705.07261 [cs, math, stat]
  issued:
    - year: 2017
      month: 5
      day: 20
  title: Stochastic Recursive Gradient Algorithm for Nonconvex Optimization
  type: article-journal
  URL: http://arxiv.org/abs/1705.07261

- id: PatelSGD2017
  author:
    - family: Patel
      given: Vivak
  citation-key: PatelSGD2017
  container-title: arXiv:1702.00317 [cs, math, stat]
  issued:
    - year: 2017
      month: 2
      day: 1
  title: 'On SGD''s Failure in Practice: Characterizing and Overcoming Stalling'
  type: article-journal
  URL: http://arxiv.org/abs/1702.00317

- id: PolyakAcceleration1992
  accessed:
    - year: 2014
      month: 6
      day: 11
  author:
    - family: Polyak
      given: B. T.
    - family: Juditsky
      given: A. B.
  citation-key: PolyakAcceleration1992
  container-title: SIAM Journal on Control and Optimization
  container-title-short: SIAM J. Control Optim.
  DOI: 10.1137/0330046
  ISSN: 0363-0129
  issue: '4'
  issued:
    - year: 1992
      month: 7
      day: 1
  page: 838–855
  title: Acceleration of Stochastic Approximation by Averaging
  type: article-journal
  URL: >-
    http://www.meyn.ece.ufl.edu/archive/spm_files/Courses/ECE555-2011/555media/poljud92.pdf
  volume: '30'

- id: ReddiStochastic2016
  accessed:
    - year: 2017
      month: 10
      day: 12
  author:
    - family: Reddi
      given: Sashank J.
    - family: Hefny
      given: Ahmed
    - family: Sra
      given: Suvrit
    - family: Poczos
      given: Barnabas
    - family: Smola
      given: Alex
  citation-key: ReddiStochastic2016
  container-title: PMLR
  event-title: International Conference on Machine Learning
  issued:
    - year: 2016
      month: 6
      day: 11
  language: en
  page: 314-323
  title: Stochastic Variance Reduction for Nonconvex Optimization
  type: paper-conference
  URL: http://proceedings.mlr.press/v48/reddi16.html
  volume: '1603'

- id: RobbinsConvergence1971
  accessed:
    - year: 2018
      month: 6
      day: 13
  author:
    - family: Robbins
      given: H.
    - family: Siegmund
      given: D.
  citation-key: RobbinsConvergence1971
  container-title: Optimizing Methods in Statistics
  DOI: 10.1016/B978-0-12-604550-5.50015-8
  editor:
    - family: Rustagi
      given: Jagdish S.
  ISBN: 978-0-12-604550-5
  issued:
    - year: 1971
  page: 233-257
  publisher: Academic Press
  title: >-
    A convergence theorem for non negative almost supermartingales and some
    applications
  type: chapter
  URL: https://www.sciencedirect.com/science/article/pii/B9780126045505500158

- id: RobbinsStochastic1951
  accessed:
    - year: 2017
      month: 10
      day: 12
  author:
    - family: Robbins
      given: Herbert
    - family: Monro
      given: Sutton
  citation-key: RobbinsStochastic1951
  container-title: The Annals of Mathematical Statistics
  container-title-short: Ann. Math. Statist.
  DOI: 10.1214/aoms/1177729586
  ISSN: 0003-4851, 2168-8990
  issue: '3'
  issued:
    - year: 1951
      month: 9
  language: EN
  number: '3'
  page: 400-407
  title: A Stochastic Approximation Method
  type: article-journal
  volume: '22'

- id: RuderOverview2016
  accessed:
    - year: 2017
      month: 3
      day: 2
  author:
    - family: Ruder
      given: Sebastian
  citation-key: RuderOverview2016
  container-title: arXiv:1609.04747 [cs]
  issued:
    - year: 2016
      month: 9
      day: 15
  title: An overview of gradient descent optimization algorithms
  type: article-journal
  URL: http://arxiv.org/abs/1609.04747

- id: SagunExplorations2014
  accessed:
    - year: 2015
      month: 6
      day: 17
  author:
    - family: Sagun
      given: Levent
    - family: Guney
      given: V. Ugur
    - family: Arous
      given: Gerard Ben
    - family: LeCun
      given: Yann
  citation-key: SagunExplorations2014
  container-title: arXiv:1412.6615 [cs, stat]
  issued:
    - year: 2014
      month: 12
      day: 20
  title: Explorations on high dimensional landscapes
  type: article-journal
  URL: http://arxiv.org/abs/1412.6615

- id: SalimansWeight2016
  author:
    - family: Salimans
      given: Tim
    - family: Kingma
      given: Diederik P
  citation-key: SalimansWeight2016
  container-title: Advances in Neural Information Processing Systems 29
  editor:
    - family: Lee
      given: D. D.
    - family: Sugiyama
      given: M.
    - family: Luxburg
      given: U. V.
    - family: Guyon
      given: I.
    - family: Garnett
      given: R.
  issued:
    - year: 2016
  page: 901–901
  publisher: Curran Associates, Inc.
  title: >-
    Weight Normalization: A Simple Reparameterization to Accelerate Training of
    Deep Neural Networks
  type: paper-conference
  URL: >-
    http://papers.nips.cc/paper/6114-weight-normalization-a-simple-reparameterization-to-accelerate-training-of-deep-neural-networks.pdf

- id: Shalev-ShwartzStochastic2011
  accessed:
    - year: 2015
      month: 5
      day: 2
  author:
    - family: Shalev-Shwartz
      given: Shai
    - family: Tewari
      given: Ambuj
  citation-key: Shalev-ShwartzStochastic2011
  container-title: Journal of Machine Learning Research
  ISSN: 1532-4435
  issued:
    - year: 2011
      month: 7
  page: 1865–1892
  title: Stochastic Methods for L1-regularized Loss Minimization
  type: article-journal
  URL: http://www.machinelearning.org/archive/icml2009/papers/262.pdf
  volume: '12'

- id: SimsekliHausdorff2020
  accessed:
    - year: 2020
      month: 6
      day: 19
  author:
    - family: Şimşekli
      given: Umut
    - family: Sener
      given: Ozan
    - family: Deligiannidis
      given: George
    - family: Erdogdu
      given: Murat A.
  citation-key: SimsekliHausdorff2020
  container-title: CoRR
  issued:
    - year: 2020
  title: >-
    Hausdorff Dimension, Stochastic Differential Equations, and Generalization
    in Neural Networks
  type: article-journal
  URL: http://arxiv.org/abs/2006.09313
  volume: abs/2006.09313

- id: SmithOrigin2020
  accessed:
    - year: 2022
      month: 4
      day: 23
  author:
    - family: Smith
      given: Samuel L.
    - family: Dherin
      given: Benoit
    - family: Barrett
      given: David
    - family: De
      given: Soham
  citation-key: SmithOrigin2020
  DOI: 10.48550/arXiv.2101.12176
  event-title: International Conference on Learning Representations
  issued:
    - year: 2020
      month: 9
      day: 28
  language: en
  title: On the Origin of Implicit Regularization in Stochastic Gradient Descent
  type: paper-conference
  URL: http://arxiv.org/abs/2101.12176

- id: SpallAdaptive2000
  author:
    - family: Spall
      given: J. C.
  citation-key: SpallAdaptive2000
  container-title: IEEE Transactions on Automatic Control
  DOI: 10.1109/TAC.2000.880982
  ISSN: 0018-9286
  issue: '10'
  issued:
    - year: 2000
      month: 10
  page: 1839-1853
  title: Adaptive stochastic approximation by the simultaneous perturbation method
  type: article-journal
  volume: '45'

- id: SunScheduling2023
  accessed:
    - year: 2023
      month: 9
      day: 6
  author:
    - family: Sun
      given: Jianhui
    - family: Yang
      given: Ying
    - family: Xun
      given: Guangxu
    - family: Zhang
      given: Aidong
  citation-key: SunScheduling2023
  container-title: ACM Transactions on Knowledge Discovery from Data
  container-title-short: ACM Trans. Knowl. Discov. Data
  DOI: 10.1145/3544782
  ISSN: 1556-4681
  issue: '2'
  issued:
    - year: 2023
      month: 3
      day: 20
  page: 29:1–29:37
  title: >-
    Scheduling Hyperparameters to Improve Generalization: From Centralized SGD
    to Asynchronous SGD
  type: article-journal
  volume: '17'

- id: VishwanathanAccelerated2006
  author:
    - family: Vishwanathan
      given: S.V. N.
    - family: Schraudolph
      given: Nicol N.
    - family: Schmidt
      given: Mark W.
    - family: Murphy
      given: Kevin P.
  citation-key: VishwanathanAccelerated2006
  container-title: Proceedings of the 23rd International Conference on Machine Learning
  issued:
    - year: 2006
  title: >-
    Accelerated Training of Conditional Random Fields with Stochastic Gradient
    Methods
  type: paper-conference

- id: WellingBayesian2011
  accessed:
    - year: 2022
      month: 8
      day: 16
  author:
    - family: Welling
      given: Max
    - family: Teh
      given: Yee Whye
  citation-key: WellingBayesian2011
  collection-title: ICML'11
  container-title: >-
    Proceedings of the 28th International Conference on International Conference
    on Machine Learning
  event-place: Madison, WI, USA
  ISBN: 978-1-4503-0619-5
  issued:
    - year: 2011
      month: 6
      day: 28
  page: 681–688
  publisher: Omnipress
  publisher-place: Madison, WI, USA
  title: Bayesian learning via stochastic gradient Langevin dynamics
  type: paper-conference
  URL: https://icml.cc/2011/papers/398_icmlpaper.pdf

- id: WrightOptimization2021
  author:
    - family: Wright
      given: Stephen J.
    - family: Recht
      given: Benjamin
  call-number: QA76.9.B45 W75 2021
  citation-key: WrightOptimization2021
  DOI: 10.1017/9781009004282
  event-place: New York
  ISBN: 978-1-316-51898-4
  issued:
    - year: 2021
  publisher: Cambridge University Press
  publisher-place: New York
  title: Optimization for data analysis
  type: book
  URL: >-
    https://www.cambridge.org/core/books/optimization-for-data-analysis/C02C3708905D236AA354D1CE1739A6A2

- id: XuOptimal2011
  accessed:
    - year: 2014
      month: 6
      day: 11
  author:
    - family: Xu
      given: Wei
  citation-key: XuOptimal2011
  container-title: arXiv:1107.2490 [cs]
  issued:
    - year: 2011
      month: 7
      day: 13
  title: >-
    Towards Optimal One Pass Large Scale Learning with Averaged Stochastic
    Gradient Descent
  type: article-journal
  URL: http://arxiv.org/abs/1107.2490

- id: ZhangStochastic2017
  author:
    - family: Zhang
      given: Xiao
    - family: Wang
      given: Lingxiao
    - family: Gu
      given: Quanquan
  citation-key: ZhangStochastic2017
  container-title: arXiv:1701.00481 [stat]
  issued:
    - year: 2017
      month: 1
      day: 2
  title: >-
    Stochastic Variance-reduced Gradient Descent for Low-rank Matrix Recovery
    from Linear Measurements
  type: article-journal
  URL: http://arxiv.org/abs/1701.00481

- id: ZinkevichParallelized2010
  accessed:
    - year: 2015
      month: 6
      day: 18
  author:
    - family: Zinkevich
      given: Martin
    - family: Weimer
      given: Markus
    - family: Li
      given: Lihong
    - family: Smola
      given: Alex J.
  citation-key: ZinkevichParallelized2010
  container-title: Advances in Neural Information Processing Systems 23
  editor:
    - family: Lafferty
      given: J. D.
    - family: Williams
      given: C. K. I.
    - family: Shawe-Taylor
      given: J.
    - family: Zemel
      given: R. S.
    - family: Culotta
      given: A.
  issued:
    - year: 2010
  page: 2595–2603
  publisher: Curran Associates, Inc.
  title: Parallelized Stochastic Gradient Descent
  type: paper-conference
  URL: >-
    http://papers.nips.cc/paper/4006-parallelized-stochastic-gradient-descent.pdf
...
