test_timeit.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401
  1. import timeit
  2. import unittest
  3. import sys
  4. import io
  5. from textwrap import dedent
  6. from test.support import captured_stdout
  7. from test.support import captured_stderr
  8. # timeit's default number of iterations.
  9. DEFAULT_NUMBER = 1000000
  10. # timeit's default number of repetitions.
  11. DEFAULT_REPEAT = 5
  12. # XXX: some tests are commented out that would improve the coverage but take a
  13. # long time to run because they test the default number of loops, which is
  14. # large. The tests could be enabled if there was a way to override the default
  15. # number of loops during testing, but this would require changing the signature
  16. # of some functions that use the default as a default argument.
  17. class FakeTimer:
  18. BASE_TIME = 42.0
  19. def __init__(self, seconds_per_increment=1.0):
  20. self.count = 0
  21. self.setup_calls = 0
  22. self.seconds_per_increment=seconds_per_increment
  23. timeit._fake_timer = self
  24. def __call__(self):
  25. return self.BASE_TIME + self.count * self.seconds_per_increment
  26. def inc(self):
  27. self.count += 1
  28. def setup(self):
  29. self.setup_calls += 1
  30. def wrap_timer(self, timer):
  31. """Records 'timer' and returns self as callable timer."""
  32. self.saved_timer = timer
  33. return self
  34. class TestTimeit(unittest.TestCase):
  35. def tearDown(self):
  36. try:
  37. del timeit._fake_timer
  38. except AttributeError:
  39. pass
  40. def test_reindent_empty(self):
  41. self.assertEqual(timeit.reindent("", 0), "")
  42. self.assertEqual(timeit.reindent("", 4), "")
  43. def test_reindent_single(self):
  44. self.assertEqual(timeit.reindent("pass", 0), "pass")
  45. self.assertEqual(timeit.reindent("pass", 4), "pass")
  46. def test_reindent_multi_empty(self):
  47. self.assertEqual(timeit.reindent("\n\n", 0), "\n\n")
  48. self.assertEqual(timeit.reindent("\n\n", 4), "\n \n ")
  49. def test_reindent_multi(self):
  50. self.assertEqual(timeit.reindent(
  51. "print()\npass\nbreak", 0),
  52. "print()\npass\nbreak")
  53. self.assertEqual(timeit.reindent(
  54. "print()\npass\nbreak", 4),
  55. "print()\n pass\n break")
  56. def test_timer_invalid_stmt(self):
  57. self.assertRaises(ValueError, timeit.Timer, stmt=None)
  58. self.assertRaises(SyntaxError, timeit.Timer, stmt='return')
  59. self.assertRaises(SyntaxError, timeit.Timer, stmt='yield')
  60. self.assertRaises(SyntaxError, timeit.Timer, stmt='yield from ()')
  61. self.assertRaises(SyntaxError, timeit.Timer, stmt='break')
  62. self.assertRaises(SyntaxError, timeit.Timer, stmt='continue')
  63. self.assertRaises(SyntaxError, timeit.Timer, stmt='from timeit import *')
  64. self.assertRaises(SyntaxError, timeit.Timer, stmt=' pass')
  65. self.assertRaises(SyntaxError, timeit.Timer,
  66. setup='while False:\n pass', stmt=' break')
  67. def test_timer_invalid_setup(self):
  68. self.assertRaises(ValueError, timeit.Timer, setup=None)
  69. self.assertRaises(SyntaxError, timeit.Timer, setup='return')
  70. self.assertRaises(SyntaxError, timeit.Timer, setup='yield')
  71. self.assertRaises(SyntaxError, timeit.Timer, setup='yield from ()')
  72. self.assertRaises(SyntaxError, timeit.Timer, setup='break')
  73. self.assertRaises(SyntaxError, timeit.Timer, setup='continue')
  74. self.assertRaises(SyntaxError, timeit.Timer, setup='from timeit import *')
  75. self.assertRaises(SyntaxError, timeit.Timer, setup=' pass')
  76. def test_timer_empty_stmt(self):
  77. timeit.Timer(stmt='')
  78. timeit.Timer(stmt=' \n\t\f')
  79. timeit.Timer(stmt='# comment')
  80. fake_setup = "import timeit\ntimeit._fake_timer.setup()"
  81. fake_stmt = "import timeit\ntimeit._fake_timer.inc()"
  82. def fake_callable_setup(self):
  83. self.fake_timer.setup()
  84. def fake_callable_stmt(self):
  85. self.fake_timer.inc()
  86. def timeit(self, stmt, setup, number=None, globals=None):
  87. self.fake_timer = FakeTimer()
  88. t = timeit.Timer(stmt=stmt, setup=setup, timer=self.fake_timer,
  89. globals=globals)
  90. kwargs = {}
  91. if number is None:
  92. number = DEFAULT_NUMBER
  93. else:
  94. kwargs['number'] = number
  95. delta_time = t.timeit(**kwargs)
  96. self.assertEqual(self.fake_timer.setup_calls, 1)
  97. self.assertEqual(self.fake_timer.count, number)
  98. self.assertEqual(delta_time, number)
  99. # Takes too long to run in debug build.
  100. #def test_timeit_default_iters(self):
  101. # self.timeit(self.fake_stmt, self.fake_setup)
  102. def test_timeit_zero_iters(self):
  103. self.timeit(self.fake_stmt, self.fake_setup, number=0)
  104. def test_timeit_few_iters(self):
  105. self.timeit(self.fake_stmt, self.fake_setup, number=3)
  106. def test_timeit_callable_stmt(self):
  107. self.timeit(self.fake_callable_stmt, self.fake_setup, number=3)
  108. def test_timeit_callable_setup(self):
  109. self.timeit(self.fake_stmt, self.fake_callable_setup, number=3)
  110. def test_timeit_callable_stmt_and_setup(self):
  111. self.timeit(self.fake_callable_stmt,
  112. self.fake_callable_setup, number=3)
  113. # Takes too long to run in debug build.
  114. #def test_timeit_function(self):
  115. # delta_time = timeit.timeit(self.fake_stmt, self.fake_setup,
  116. # timer=FakeTimer())
  117. # self.assertEqual(delta_time, DEFAULT_NUMBER)
  118. def test_timeit_function_zero_iters(self):
  119. delta_time = timeit.timeit(self.fake_stmt, self.fake_setup, number=0,
  120. timer=FakeTimer())
  121. self.assertEqual(delta_time, 0)
  122. def test_timeit_globals_args(self):
  123. global _global_timer
  124. _global_timer = FakeTimer()
  125. t = timeit.Timer(stmt='_global_timer.inc()', timer=_global_timer)
  126. self.assertRaises(NameError, t.timeit, number=3)
  127. timeit.timeit(stmt='_global_timer.inc()', timer=_global_timer,
  128. globals=globals(), number=3)
  129. local_timer = FakeTimer()
  130. timeit.timeit(stmt='local_timer.inc()', timer=local_timer,
  131. globals=locals(), number=3)
  132. def repeat(self, stmt, setup, repeat=None, number=None):
  133. self.fake_timer = FakeTimer()
  134. t = timeit.Timer(stmt=stmt, setup=setup, timer=self.fake_timer)
  135. kwargs = {}
  136. if repeat is None:
  137. repeat = DEFAULT_REPEAT
  138. else:
  139. kwargs['repeat'] = repeat
  140. if number is None:
  141. number = DEFAULT_NUMBER
  142. else:
  143. kwargs['number'] = number
  144. delta_times = t.repeat(**kwargs)
  145. self.assertEqual(self.fake_timer.setup_calls, repeat)
  146. self.assertEqual(self.fake_timer.count, repeat * number)
  147. self.assertEqual(delta_times, repeat * [float(number)])
  148. # Takes too long to run in debug build.
  149. #def test_repeat_default(self):
  150. # self.repeat(self.fake_stmt, self.fake_setup)
  151. def test_repeat_zero_reps(self):
  152. self.repeat(self.fake_stmt, self.fake_setup, repeat=0)
  153. def test_repeat_zero_iters(self):
  154. self.repeat(self.fake_stmt, self.fake_setup, number=0)
  155. def test_repeat_few_reps_and_iters(self):
  156. self.repeat(self.fake_stmt, self.fake_setup, repeat=3, number=5)
  157. def test_repeat_callable_stmt(self):
  158. self.repeat(self.fake_callable_stmt, self.fake_setup,
  159. repeat=3, number=5)
  160. def test_repeat_callable_setup(self):
  161. self.repeat(self.fake_stmt, self.fake_callable_setup,
  162. repeat=3, number=5)
  163. def test_repeat_callable_stmt_and_setup(self):
  164. self.repeat(self.fake_callable_stmt, self.fake_callable_setup,
  165. repeat=3, number=5)
  166. # Takes too long to run in debug build.
  167. #def test_repeat_function(self):
  168. # delta_times = timeit.repeat(self.fake_stmt, self.fake_setup,
  169. # timer=FakeTimer())
  170. # self.assertEqual(delta_times, DEFAULT_REPEAT * [float(DEFAULT_NUMBER)])
  171. def test_repeat_function_zero_reps(self):
  172. delta_times = timeit.repeat(self.fake_stmt, self.fake_setup, repeat=0,
  173. timer=FakeTimer())
  174. self.assertEqual(delta_times, [])
  175. def test_repeat_function_zero_iters(self):
  176. delta_times = timeit.repeat(self.fake_stmt, self.fake_setup, number=0,
  177. timer=FakeTimer())
  178. self.assertEqual(delta_times, DEFAULT_REPEAT * [0.0])
  179. def assert_exc_string(self, exc_string, expected_exc_name):
  180. exc_lines = exc_string.splitlines()
  181. self.assertGreater(len(exc_lines), 2)
  182. self.assertTrue(exc_lines[0].startswith('Traceback'))
  183. self.assertTrue(exc_lines[-1].startswith(expected_exc_name))
  184. def test_print_exc(self):
  185. s = io.StringIO()
  186. t = timeit.Timer("1/0")
  187. try:
  188. t.timeit()
  189. except:
  190. t.print_exc(s)
  191. self.assert_exc_string(s.getvalue(), 'ZeroDivisionError')
  192. MAIN_DEFAULT_OUTPUT = "1 loop, best of 5: 1 sec per loop\n"
  193. def run_main(self, seconds_per_increment=1.0, switches=None, timer=None):
  194. if timer is None:
  195. timer = FakeTimer(seconds_per_increment=seconds_per_increment)
  196. if switches is None:
  197. args = []
  198. else:
  199. args = switches[:]
  200. args.append(self.fake_stmt)
  201. # timeit.main() modifies sys.path, so save and restore it.
  202. orig_sys_path = sys.path[:]
  203. with captured_stdout() as s:
  204. timeit.main(args=args, _wrap_timer=timer.wrap_timer)
  205. sys.path[:] = orig_sys_path[:]
  206. return s.getvalue()
  207. def test_main_bad_switch(self):
  208. s = self.run_main(switches=['--bad-switch'])
  209. self.assertEqual(s, dedent("""\
  210. option --bad-switch not recognized
  211. use -h/--help for command line help
  212. """))
  213. def test_main_seconds(self):
  214. s = self.run_main(seconds_per_increment=5.5)
  215. self.assertEqual(s, "1 loop, best of 5: 5.5 sec per loop\n")
  216. def test_main_milliseconds(self):
  217. s = self.run_main(seconds_per_increment=0.0055)
  218. self.assertEqual(s, "50 loops, best of 5: 5.5 msec per loop\n")
  219. def test_main_microseconds(self):
  220. s = self.run_main(seconds_per_increment=0.0000025, switches=['-n100'])
  221. self.assertEqual(s, "100 loops, best of 5: 2.5 usec per loop\n")
  222. def test_main_fixed_iters(self):
  223. s = self.run_main(seconds_per_increment=2.0, switches=['-n35'])
  224. self.assertEqual(s, "35 loops, best of 5: 2 sec per loop\n")
  225. def test_main_setup(self):
  226. s = self.run_main(seconds_per_increment=2.0,
  227. switches=['-n35', '-s', 'print("CustomSetup")'])
  228. self.assertEqual(s, "CustomSetup\n" * DEFAULT_REPEAT +
  229. "35 loops, best of 5: 2 sec per loop\n")
  230. def test_main_multiple_setups(self):
  231. s = self.run_main(seconds_per_increment=2.0,
  232. switches=['-n35', '-s', 'a = "CustomSetup"', '-s', 'print(a)'])
  233. self.assertEqual(s, "CustomSetup\n" * DEFAULT_REPEAT +
  234. "35 loops, best of 5: 2 sec per loop\n")
  235. def test_main_fixed_reps(self):
  236. s = self.run_main(seconds_per_increment=60.0, switches=['-r9'])
  237. self.assertEqual(s, "1 loop, best of 9: 60 sec per loop\n")
  238. def test_main_negative_reps(self):
  239. s = self.run_main(seconds_per_increment=60.0, switches=['-r-5'])
  240. self.assertEqual(s, "1 loop, best of 1: 60 sec per loop\n")
  241. @unittest.skipIf(sys.flags.optimize >= 2, "need __doc__")
  242. def test_main_help(self):
  243. s = self.run_main(switches=['-h'])
  244. # Note: It's not clear that the trailing space was intended as part of
  245. # the help text, but since it's there, check for it.
  246. self.assertEqual(s, timeit.__doc__ + ' ')
  247. def test_main_verbose(self):
  248. s = self.run_main(switches=['-v'])
  249. self.assertEqual(s, dedent("""\
  250. 1 loop -> 1 secs
  251. raw times: 1 sec, 1 sec, 1 sec, 1 sec, 1 sec
  252. 1 loop, best of 5: 1 sec per loop
  253. """))
  254. def test_main_very_verbose(self):
  255. s = self.run_main(seconds_per_increment=0.000_030, switches=['-vv'])
  256. self.assertEqual(s, dedent("""\
  257. 1 loop -> 3e-05 secs
  258. 2 loops -> 6e-05 secs
  259. 5 loops -> 0.00015 secs
  260. 10 loops -> 0.0003 secs
  261. 20 loops -> 0.0006 secs
  262. 50 loops -> 0.0015 secs
  263. 100 loops -> 0.003 secs
  264. 200 loops -> 0.006 secs
  265. 500 loops -> 0.015 secs
  266. 1000 loops -> 0.03 secs
  267. 2000 loops -> 0.06 secs
  268. 5000 loops -> 0.15 secs
  269. 10000 loops -> 0.3 secs
  270. raw times: 300 msec, 300 msec, 300 msec, 300 msec, 300 msec
  271. 10000 loops, best of 5: 30 usec per loop
  272. """))
  273. def test_main_with_time_unit(self):
  274. unit_sec = self.run_main(seconds_per_increment=0.003,
  275. switches=['-u', 'sec'])
  276. self.assertEqual(unit_sec,
  277. "100 loops, best of 5: 0.003 sec per loop\n")
  278. unit_msec = self.run_main(seconds_per_increment=0.003,
  279. switches=['-u', 'msec'])
  280. self.assertEqual(unit_msec,
  281. "100 loops, best of 5: 3 msec per loop\n")
  282. unit_usec = self.run_main(seconds_per_increment=0.003,
  283. switches=['-u', 'usec'])
  284. self.assertEqual(unit_usec,
  285. "100 loops, best of 5: 3e+03 usec per loop\n")
  286. # Test invalid unit input
  287. with captured_stderr() as error_stringio:
  288. invalid = self.run_main(seconds_per_increment=0.003,
  289. switches=['-u', 'parsec'])
  290. self.assertEqual(error_stringio.getvalue(),
  291. "Unrecognized unit. Please select nsec, usec, msec, or sec.\n")
  292. def test_main_exception(self):
  293. with captured_stderr() as error_stringio:
  294. s = self.run_main(switches=['1/0'])
  295. self.assert_exc_string(error_stringio.getvalue(), 'ZeroDivisionError')
  296. def test_main_exception_fixed_reps(self):
  297. with captured_stderr() as error_stringio:
  298. s = self.run_main(switches=['-n1', '1/0'])
  299. self.assert_exc_string(error_stringio.getvalue(), 'ZeroDivisionError')
  300. def autorange(self, seconds_per_increment=1/1024, callback=None):
  301. timer = FakeTimer(seconds_per_increment=seconds_per_increment)
  302. t = timeit.Timer(stmt=self.fake_stmt, setup=self.fake_setup, timer=timer)
  303. return t.autorange(callback)
  304. def test_autorange(self):
  305. num_loops, time_taken = self.autorange()
  306. self.assertEqual(num_loops, 500)
  307. self.assertEqual(time_taken, 500/1024)
  308. def test_autorange_second(self):
  309. num_loops, time_taken = self.autorange(seconds_per_increment=1.0)
  310. self.assertEqual(num_loops, 1)
  311. self.assertEqual(time_taken, 1.0)
  312. def test_autorange_with_callback(self):
  313. def callback(a, b):
  314. print("{} {:.3f}".format(a, b))
  315. with captured_stdout() as s:
  316. num_loops, time_taken = self.autorange(callback=callback)
  317. self.assertEqual(num_loops, 500)
  318. self.assertEqual(time_taken, 500/1024)
  319. expected = ('1 0.001\n'
  320. '2 0.002\n'
  321. '5 0.005\n'
  322. '10 0.010\n'
  323. '20 0.020\n'
  324. '50 0.049\n'
  325. '100 0.098\n'
  326. '200 0.195\n'
  327. '500 0.488\n')
  328. self.assertEqual(s.getvalue(), expected)
  329. if __name__ == '__main__':
  330. unittest.main()