---
references:
- id: AroraRip2021
  accessed:
    - year: 2021
      month: 4
      day: 13
  author:
    - family: Arora
      given: Sanjeev
    - family: Zhang
      given: Yi
  citation-key: AroraRip2021
  container-title: arXiv:2102.13189 [cs, stat]
  issued:
    - year: 2021
      month: 2
      day: 25
  language: en
  title: 'Rip van Winkle''s Razor: A Simple Estimate of Overfit to Test Data'
  type: article-journal
  URL: http://arxiv.org/abs/2102.13189

- id: BlumLadder2015
  accessed:
    - year: 2016
      month: 7
      day: 27
  author:
    - family: Blum
      given: Avrim
    - family: Hardt
      given: Moritz
  citation-key: BlumLadder2015
  container-title: arXiv:1502.04585 [cs]
  issued:
    - year: 2015
      month: 2
      day: 16
  title: 'The Ladder: A Reliable Leaderboard for Machine Learning Competitions'
  type: article-journal
  URL: http://arxiv.org/abs/1502.04585

- id: BrockmanOpenAI2016
  accessed:
    - year: 2022
      month: 5
      day: 10
  author:
    - family: Brockman
      given: Greg
    - family: Cheung
      given: Vicki
    - family: Pettersson
      given: Ludwig
    - family: Schneider
      given: Jonas
    - family: Schulman
      given: John
    - family: Tang
      given: Jie
    - family: Zaremba
      given: Wojciech
  citation-key: BrockmanOpenAI2016
  container-title: arXiv:1606.01540 [cs]
  issued:
    - year: 2016
      month: 6
      day: 5
  language: en
  title: OpenAI Gym
  type: article-journal
  URL: http://arxiv.org/abs/1606.01540

- id: FlemingHow1986
  accessed:
    - year: 2022
      month: 6
      day: 8
  author:
    - family: Fleming
      given: Philip J.
    - family: Wallace
      given: John J.
  citation-key: FlemingHow1986
  container-title: Communications of the ACM
  container-title-short: Commun. ACM
  DOI: 10.1145/5666.5673
  ISSN: 0001-0782
  issue: '3'
  issued:
    - year: 1986
      month: 3
      day: 1
  page: 218–221
  title: >-
    How not to lie with statistics: the correct way to summarize benchmark
    results
  type: article-journal
  volume: '29'

- id: GeirhosShortcut2020
  accessed:
    - year: 2020
      month: 4
      day: 17
  author:
    - family: Geirhos
      given: Robert
    - family: Jacobsen
      given: Jörn-Henrik
    - family: Michaelis
      given: Claudio
    - family: Zemel
      given: Richard
    - family: Brendel
      given: Wieland
    - family: Bethge
      given: Matthias
    - family: Wichmann
      given: Felix A.
  citation-key: GeirhosShortcut2020
  container-title: arXiv:2004.07780 [cs, q-bio]
  issued:
    - year: 2020
      month: 4
      day: 16
  title: Shortcut Learning in Deep Neural Networks
  type: article-journal
  URL: http://arxiv.org/abs/2004.07780

- id: HutsonTaught2022
  accessed:
    - year: 2022
      month: 6
      day: 22
  author:
    - family: Hutson
      given: Matthew
  citation-key: HutsonTaught2022
  container-title: Science
  container-title-short: Science
  DOI: 10.1126/science.abq7833
  ISSN: 0036-8075, 1095-9203
  issue: '6593'
  issued:
    - year: 2022
      month: 5
      day: 6
  language: en
  page: 570-573
  title: Taught to the test
  type: article-journal
  volume: '376'

- id: HyndmanBrief2020
  accessed:
    - year: 2022
      month: 5
      day: 10
  author:
    - family: Hyndman
      given: Rob J.
  citation-key: HyndmanBrief2020
  collection-title: M4 Competition
  container-title: International Journal of Forecasting
  container-title-short: International Journal of Forecasting
  DOI: 10.1016/j.ijforecast.2019.03.015
  ISSN: 0169-2070
  issue: '1'
  issued:
    - year: 2020
      month: 1
      day: 1
  language: en
  page: 7-14
  title: "A brief history of forecasting\_competitions"
  type: article-journal
  volume: '36'

- id: LathuiliereComprehensive2020
  author:
    - family: Lathuilière
      given: Stéphane
    - family: Mesejo
      given: Pablo
    - family: Alameda-Pineda
      given: Xavier
    - family: Horaud
      given: Radu
  citation-key: LathuiliereComprehensive2020
  container-title: IEEE Transactions on Pattern Analysis and Machine Intelligence
  DOI: 10.1109/TPAMI.2019.2910523
  ISSN: 1939-3539
  issue: '9'
  issued:
    - year: 2020
      month: 9
  page: 2065-2081
  title: A Comprehensive Analysis of Deep Regression
  type: article-journal
  volume: '42'

- id: LonesHow2021
  accessed:
    - year: 2022
      month: 10
      day: 7
  author:
    - family: Lones
      given: Michael A.
  citation-key: LonesHow2021
  DOI: 10.48550/arXiv.2108.02497
  issued:
    - year: 2021
      month: 8
      day: 5
  language: en
  title: 'How to avoid machine learning pitfalls: a guide for academic researchers'
  type: article-journal
  URL: https://arxiv.org/abs/2108.02497v2

- id: MakridakisM42020
  accessed:
    - year: 2022
      month: 5
      day: 10
  author:
    - family: Makridakis
      given: Spyros
    - family: Spiliotis
      given: Evangelos
    - family: Assimakopoulos
      given: Vassilios
  citation-key: MakridakisM42020
  collection-title: M4 Competition
  container-title: International Journal of Forecasting
  container-title-short: International Journal of Forecasting
  DOI: 10.1016/j.ijforecast.2019.04.014
  ISSN: 0169-2070
  issue: '1'
  issued:
    - year: 2020
      month: 1
      day: 1
  language: en
  page: 54-74
  title: 'The M4 Competition: 100,000 time series and 61 forecasting methods'
  type: article-journal
  volume: '36'

- id: MusgraveMetric2020
  accessed:
    - year: 2020
      month: 8
      day: 15
  author:
    - family: Musgrave
      given: Kevin
    - family: Belongie
      given: Serge
    - family: Lim
      given: Ser-Nam
  citation-key: MusgraveMetric2020
  container-title: arXiv:2003.08505 [cs]
  issued:
    - year: 2020
      month: 7
      day: 23
  title: A Metric Learning Reality Check
  type: article-journal
  URL: http://arxiv.org/abs/2003.08505

- id: MytkowiczProducing2009
  accessed:
    - year: 2022
      month: 6
      day: 8
  author:
    - family: Mytkowicz
      given: Todd
    - family: Diwan
      given: Amer
    - family: Hauswirth
      given: Matthias
    - family: Sweeney
      given: Peter F.
  citation-key: MytkowiczProducing2009
  collection-title: ASPLOS XIV
  container-title: >-
    Proceedings of the 14th international conference on Architectural support
    for programming languages and operating systems
  DOI: 10.1145/1508244.1508275
  event-place: New York, NY, USA
  ISBN: 978-1-60558-406-5
  issued:
    - year: 2009
      month: 3
      day: 7
  page: 265–276
  publisher: Association for Computing Machinery
  publisher-place: New York, NY, USA
  title: Producing wrong data without doing anything obviously wrong!
  type: paper-conference

- id: OlsonPMLB2017
  accessed:
    - year: 2022
      month: 5
      day: 11
  author:
    - family: Olson
      given: Randal S.
    - family: La Cava
      given: William
    - family: Orzechowski
      given: Patryk
    - family: Urbanowicz
      given: Ryan J.
    - family: Moore
      given: Jason H.
  citation-key: OlsonPMLB2017
  container-title: BioData Mining
  container-title-short: BioData Mining
  DOI: 10.1186/s13040-017-0154-4
  ISSN: 1756-0381
  issue: '1'
  issued:
    - year: 2017
      month: 12
      day: 11
  page: '36'
  title: 'PMLB: a large benchmark suite for machine learning evaluation and comparison'
  type: article-journal
  volume: '10'

- id: vKistowskiHow2015
  accessed:
    - year: 2022
      month: 6
      day: 8
  author:
    - family: Kistowski
      given: Jóakim
      non-dropping-particle: v.
    - family: Arnold
      given: Jeremy A.
    - family: Huppler
      given: Karl
    - family: Lange
      given: Klaus-Dieter
    - family: Henning
      given: John L.
    - family: Cao
      given: Paul
  citation-key: vKistowskiHow2015
  collection-title: ICPE '15
  container-title: >-
    Proceedings of the 6th ACM/SPEC International Conference on Performance
    Engineering
  DOI: 10.1145/2668930.2688819
  event-place: New York, NY, USA
  ISBN: 978-1-4503-3248-4
  issued:
    - year: 2015
      month: 1
      day: 31
  page: 333–336
  publisher: Association for Computing Machinery
  publisher-place: New York, NY, USA
  title: How to Build a Benchmark
  type: paper-conference
...
