---
references:
- id: AgostinelliLearning2015
  author:
    - family: Agostinelli
      given: Forest
    - family: Hoffman
      given: Matthew
    - family: Sadowski
      given: Peter
    - family: Baldi
      given: Pierre
  citation-key: AgostinelliLearning2015
  container-title: >-
    Proceedings of International Conference on Learning Representations (ICLR)
    2015
  event-title: International Conference on Learning Representations (ICLR) 2015
  issued:
    - year: 2015
  title: Learning Activation Functions to Improve Deep Neural Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1412.6830

- id: AnilSorting2018
  accessed:
    - year: 2019
      month: 1
      day: 22
  author:
    - family: Anil
      given: Cem
    - family: Lucas
      given: James
    - family: Grosse
      given: Roger
  citation-key: AnilSorting2018
  issued:
    - year: 2018
      month: 11
      day: 13
  language: en
  title: Sorting out Lipschitz function approximation
  type: article-journal
  URL: https://arxiv.org/abs/1811.05381v1

- id: ArjovskyUnitary2016
  author:
    - family: Arjovsky
      given: Martin
    - family: Shah
      given: Amar
    - family: Bengio
      given: Yoshua
  citation-key: ArjovskyUnitary2016
  collection-title: ICML'16
  container-title: >-
    Proceedings of the 33rd International Conference on International Conference
    on Machine Learning - Volume 48
  event-place: New York, NY, USA
  event-title: International Conference on Machine Learning
  issued:
    - year: 2016
      month: 6
      day: 11
  language: en
  page: 1120-1128
  publisher: JMLR.org
  publisher-place: New York, NY, USA
  title: Unitary Evolution Recurrent Neural Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1511.06464

- id: BalduzziShattered2017
  author:
    - family: Balduzzi
      given: David
    - family: Frean
      given: Marcus
    - family: Leary
      given: Lennox
    - family: Lewis
      given: J. P.
    - family: Ma
      given: Kurt Wan-Duo
    - family: McWilliams
      given: Brian
  citation-key: BalduzziShattered2017
  container-title: PMLR
  event-title: International Conference on Machine Learning
  issued:
    - year: 2017
      month: 7
      day: 17
  language: en
  page: 342-350
  title: >-
    The Shattered Gradients Problem: If resnets are the answer, then what is the
    question?
  type: paper-conference
  URL: http://proceedings.mlr.press/v70/balduzzi17b.html

- id: ChoKernel2009
  accessed:
    - year: 2021
      month: 1
      day: 5
  author:
    - family: Cho
      given: Youngmin
    - family: Saul
      given: Lawrence K.
  citation-key: ChoKernel2009
  collection-title: NIPS'09
  container-title: >-
    Proceedings of the 22nd International Conference on Neural Information
    Processing Systems
  event-place: Red Hook, NY, USA
  ISBN: 978-1-61567-911-9
  issued:
    - year: 2009
      month: 12
      day: 7
  page: 342–350
  publisher: Curran Associates Inc.
  publisher-place: Red Hook, NY, USA
  title: Kernel methods for deep learning
  type: paper-conference
  URL: >-
    https://papers.nips.cc/paper/2009/hash/5751ec3e9a4feab575962e78e006250d-Abstract.html
  volume: '22'

- id: ClevertFast2016
  author:
    - family: Clevert
      given: Djork-Arné
    - family: Unterthiner
      given: Thomas
    - family: Hochreiter
      given: Sepp
  citation-key: ClevertFast2016
  container-title: Proceedings of ICLR
  event-title: ICLR
  issued:
    - year: 2016
  title: Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
  type: paper-conference
  URL: http://arxiv.org/abs/1511.07289

- id: DuchSurvey1999
  author:
    - family: Duch
      given: Włodzisław
    - family: Jankowski
      given: Norbert
  citation-key: DuchSurvey1999
  issued:
    - year: 1999
  language: en
  page: '50'
  title: Survey of Neural Transfer Functions
  type: report
  URL: ftp://ftp.icsi.berkeley.edu/pub/ai/jagota/vol2_6.pdf

- id: GlorotDeep2011
  accessed:
    - year: 2016
      month: 8
      day: 11
  author:
    - family: Glorot
      given: Xavier
    - family: Bordes
      given: Antoine
    - family: Bengio
      given: Yoshua
  citation-key: GlorotDeep2011
  container-title: Aistats
  issued:
    - year: 2011
  page: '275'
  title: Deep Sparse Rectifier Neural Networks.
  type: paper-conference
  URL: http://www.jmlr.org/proceedings/papers/v15/glorot11a/glorot11a.pdf
  volume: '15'

- id: GlorotUnderstanding2010
  author:
    - family: Glorot
      given: Xavier
    - family: Bengio
      given: Yoshua
  citation-key: GlorotUnderstanding2010
  container-title: Aistats
  issued:
    - year: 2010
  page: 249–256
  title: Understanding the difficulty of training deep feedforward neural networks.
  type: paper-conference
  URL: >-
    http://www.jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf?hc_location=ufi
  volume: '9'

- id: GodfreyEvaluation2019
  accessed:
    - year: 2024
      month: 3
      day: 19
  author:
    - family: Godfrey
      given: Luke B.
  citation-key: GodfreyEvaluation2019
  container-title: 2019 IEEE International Conference on Systems, Man and Cybernetics (SMC)
  DOI: 10.1109/SMC.2019.8913972
  event-title: 2019 IEEE International Conference on Systems, Man and Cybernetics (SMC)
  ISSN: 2577-1655
  issued:
    - year: 2019
      month: 10
  page: 3006-3011
  title: An Evaluation of Parametric Activation Functions for Deep Learning
  type: paper-conference

- id: GoodfellowMaxout2013
  author:
    - family: Goodfellow
      given: Ian J.
    - family: Warde-Farley
      given: David
    - family: Mirza
      given: Mehdi
    - family: Courville
      given: Aaron
    - family: Bengio
      given: Yoshua
  citation-key: GoodfellowMaxout2013
  container-title: ICML (3)
  issued:
    - year: 2013
      month: 2
      day: 18
  page: 1319–1327
  title: Maxout Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1302.4389
  volume: '28'

- id: HayouImpact2019
  accessed:
    - year: 2022
      month: 10
      day: 30
  author:
    - family: Hayou
      given: Soufiane
    - family: Doucet
      given: Arnaud
    - family: Rousseau
      given: Judith
  citation-key: HayouImpact2019
  container-title: Proceedings of the 36th International Conference on Machine Learning
  DOI: 10.48550/arXiv.1902.06853
  event-title: International Conference on Machine Learning
  ISSN: 2640-3498
  issued:
    - year: 2019
      month: 5
      day: 24
  language: en
  page: 2672-2680
  publisher: PMLR
  title: On the Impact of the Activation function on Deep Neural Networks Training
  type: paper-conference
  URL: http://arxiv.org/abs/1902.06853

- id: HeDeep2015
  accessed:
    - year: 2017
      month: 7
      day: 24
  author:
    - family: He
      given: Kaiming
    - family: Zhang
      given: Xiangyu
    - family: Ren
      given: Shaoqing
    - family: Sun
      given: Jian
  citation-key: HeDeep2015
  issued:
    - year: 2015
      month: 12
      day: 10
  title: Deep Residual Learning for Image Recognition
  type: report
  URL: http://arxiv.org/abs/1512.03385

- id: HeDelving2015
  author:
    - family: He
      given: Kaiming
    - family: Zhang
      given: Xiangyu
    - family: Ren
      given: Shaoqing
    - family: Sun
      given: Jian
  citation-key: HeDelving2015
  container-title: arXiv:1502.01852 [cs]
  issued:
    - year: 2015
      month: 2
      day: 6
  title: >-
    Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet
    Classification
  type: article-journal
  URL: http://arxiv.org/abs/1502.01852

- id: HeIdentity2016
  accessed:
    - year: 2017
      month: 7
      day: 24
  author:
    - family: He
      given: Kaiming
    - family: Zhang
      given: Xiangyu
    - family: Ren
      given: Shaoqing
    - family: Sun
      given: Jian
  citation-key: HeIdentity2016
  container-title: arXiv:1603.05027 [cs]
  event-title: ECCV
  issued:
    - year: 2016
      month: 3
      day: 16
  title: Identity Mappings in Deep Residual Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1603.05027

- id: HochreiterGradient2001
  author:
    - family: Hochreiter
      given: Sepp
    - family: Bengio
      given: Yoshua
    - family: Frasconi
      given: Paolo
    - family: Schmidhuber
      given: Jürgen
  citation-key: HochreiterGradient2001
  container-title: A field guide to dynamical recurrent neural networks
  issued:
    - year: 2001
  publisher: IEEE Press
  title: >-
    Gradient Flow in Recurrent Nets: the Difficulty of Learning Long-Term
    Dependencies
  type: chapter
  URL: http://www.bioinf.jku.at/publications/older/ch7.pdf

- id: HochreiterVanishing1998
  accessed:
    - year: 2017
      month: 7
      day: 24
  author:
    - family: Hochreiter
      given: Sepp
  citation-key: HochreiterVanishing1998
  container-title: International Journal of Uncertainty Fuzziness and Knowledge Based Systems
  DOI: 10.1142/S0218488598000094
  issued:
    - year: 1998
  page: 107–115
  title: >-
    The vanishing gradient problem during learning recurrent neural nets and
    problem solutions
  type: article-journal
  volume: '6'

- id: KlambauerSelfNormalizing2017
  accessed:
    - year: 2017
      month: 6
      day: 20
  author:
    - family: Klambauer
      given: Günter
    - family: Unterthiner
      given: Thomas
    - family: Mayr
      given: Andreas
    - family: Hochreiter
      given: Sepp
  citation-key: KlambauerSelfNormalizing2017
  container-title: >-
    Proceedings of the 31st International Conference on Neural Information
    Processing Systems
  event-place: Red Hook, NY, USA
  issued:
    - year: 2017
      month: 12
      day: 4
  language: en
  page: 972–981
  publisher: Curran Associates Inc.
  publisher-place: Red Hook, NY, USA
  title: Self-Normalizing Neural Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1706.02515

- id: LaurentMultilinear
  author:
    - family: Laurent
      given: Thomas
  citation-key: LaurentMultilinear
  language: en
  page: '9'
  title: The Multilinear Structure of ReLU Networks
  type: article-journal

- id: LedererActivation2021
  accessed:
    - year: 2021
      month: 2
      day: 2
  author:
    - family: Lederer
      given: Johannes
  citation-key: LedererActivation2021
  container-title: arXiv:2101.09957 [cs, stat]
  issued:
    - year: 2021
      month: 1
      day: 25
  title: 'Activation Functions in Artificial Neural Networks: A Systematic Overview'
  type: article-journal
  URL: http://arxiv.org/abs/2101.09957

- id: LeeDeep2018
  accessed:
    - year: 2020
      month: 12
      day: 9
  author:
    - family: Lee
      given: Jaehoon
    - family: Bahri
      given: Yasaman
    - family: Novak
      given: Roman
    - family: Schoenholz
      given: Samuel S.
    - family: Pennington
      given: Jeffrey
    - family: Sohl-Dickstein
      given: Jascha
  citation-key: LeeDeep2018
  container-title: ICLR
  issued:
    - year: 2018
      month: 3
      day: 2
  language: en
  title: Deep Neural Networks as Gaussian Processes
  type: paper-conference
  URL: http://arxiv.org/abs/1711.00165

- id: MaasRectifier2013
  accessed:
    - year: 2016
      month: 8
      day: 11
  author:
    - family: Maas
      given: Andrew L.
    - family: Hannun
      given: Awni Y.
    - family: Ng
      given: Andrew Y.
  citation-key: MaasRectifier2013
  container-title: Proceedings of ICML
  issued:
    - year: 2013
  title: Rectifier nonlinearities improve neural network acoustic models
  type: paper-conference
  URL: https://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf
  volume: '30'

- id: PascanuDifficulty2013
  author:
    - family: Pascanu
      given: Razvan
    - family: Mikolov
      given: Tomas
    - family: Bengio
      given: Yoshua
  citation-key: PascanuDifficulty2013
  container-title: arXiv:1211.5063 [cs]
  event-title: Proceedings of The 30th International Conference on Machine Learning
  issued:
    - year: 2013
  page: 1310-1318
  title: On the difficulty of training Recurrent Neural Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1211.5063

- id: RahamanSpectral2019
  accessed:
    - year: 2021
      month: 9
      day: 29
  author:
    - family: Rahaman
      given: Nasim
    - family: Baratin
      given: Aristide
    - family: Arpit
      given: Devansh
    - family: Draxler
      given: Felix
    - family: Lin
      given: Min
    - family: Hamprecht
      given: Fred A.
    - family: Bengio
      given: Yoshua
    - family: Courville
      given: Aaron
  citation-key: RahamanSpectral2019
  container-title: arXiv:1806.08734 [cs, stat]
  issued:
    - year: 2019
      month: 5
      day: 31
  title: On the Spectral Bias of Neural Networks
  type: article-journal
  URL: http://arxiv.org/abs/1806.08734

- id: RamachandranSearching2017
  accessed:
    - year: 2021
      month: 8
      day: 2
  author:
    - family: Ramachandran
      given: Prajit
    - family: Zoph
      given: Barret
    - family: Le
      given: Quoc V.
  citation-key: RamachandranSearching2017
  container-title: arXiv:1710.05941 [cs]
  issued:
    - year: 2017
      month: 10
      day: 27
  language: en
  title: Searching for Activation Functions
  type: article-journal
  URL: http://arxiv.org/abs/1710.05941

- id: SitzmannImplicit2020
  accessed:
    - year: 2020
      month: 8
      day: 20
  author:
    - family: Sitzmann
      given: Vincent
    - family: Martel
      given: Julien N. P.
    - family: Bergman
      given: Alexander W.
    - family: Lindell
      given: David B.
    - family: Wetzstein
      given: Gordon
  citation-key: SitzmannImplicit2020
  container-title: arXiv:2006.09661 [cs, eess]
  issued:
    - year: 2020
      month: 6
      day: 17
  title: Implicit Neural Representations with Periodic Activation Functions
  type: article-journal
  URL: http://arxiv.org/abs/2006.09661

- id: SrivastavaHighway2015
  accessed:
    - year: 2017
      month: 7
      day: 24
  author:
    - family: Srivastava
      given: Rupesh Kumar
    - family: Greff
      given: Klaus
    - family: Schmidhuber
      given: Jürgen
  citation-key: SrivastavaHighway2015
  container-title: arXiv:1505.00387 [cs]
  event-title: ICML
  issued:
    - year: 2015
  title: Highway Networks
  type: paper-conference
  URL: http://arxiv.org/abs/1505.00387

- id: UnserRepresenter2019
  author:
    - family: Unser
      given: Michael
  citation-key: UnserRepresenter2019
  container-title: Journal of Machine Learning Research
  ISSN: 1533-7928
  issue: '110'
  issued:
    - year: 2019
      month: 2
      day: 1
  language: en
  page: '30'
  title: A Representer Theorem for Deep Neural Networks
  type: article-journal
  URL: http://arxiv.org/abs/1802.09210
  volume: '20'

- id: WisdomFullcapacity2016
  accessed:
    - year: 2017
      month: 4
      day: 10
  author:
    - family: Wisdom
      given: Scott
    - family: Powers
      given: Thomas
    - family: Hershey
      given: John
    - family: Le Roux
      given: Jonathan
    - family: Atlas
      given: Les
  citation-key: WisdomFullcapacity2016
  container-title: Advances in Neural Information Processing Systems
  issued:
    - year: 2016
  page: 4880–4888
  title: Full-capacity unitary recurrent neural networks
  type: paper-conference
  URL: >-
    http://papers.nips.cc/paper/6327-full-capacity-unitary-recurrent-neural-networks

- id: YangFineGrained2020
  accessed:
    - year: 2021
      month: 2
      day: 18
  author:
    - family: Yang
      given: Greg
    - family: Salman
      given: Hadi
  citation-key: YangFineGrained2020
  container-title: arXiv:1907.10599 [cs, stat]
  issued:
    - year: 2020
      month: 4
      day: 9
  title: A Fine-Grained Spectral Perspective on Neural Networks
  type: article-journal
  URL: http://arxiv.org/abs/1907.10599
...
