refleak.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195
  1. import os
  2. import sys
  3. import warnings
  4. from inspect import isabstract
  5. from test import support
  6. from test.support import os_helper
  7. from test.libregrtest.utils import clear_caches
  8. try:
  9. from _abc import _get_dump
  10. except ImportError:
  11. import weakref
  12. def _get_dump(cls):
  13. # Reimplement _get_dump() for pure-Python implementation of
  14. # the abc module (Lib/_py_abc.py)
  15. registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
  16. return (registry_weakrefs, cls._abc_cache,
  17. cls._abc_negative_cache, cls._abc_negative_cache_version)
  18. def dash_R(ns, test_name, test_func):
  19. """Run a test multiple times, looking for reference leaks.
  20. Returns:
  21. False if the test didn't leak references; True if we detected refleaks.
  22. """
  23. # This code is hackish and inelegant, but it seems to do the job.
  24. import copyreg
  25. import collections.abc
  26. if not hasattr(sys, 'gettotalrefcount'):
  27. raise Exception("Tracking reference leaks requires a debug build "
  28. "of Python")
  29. # Avoid false positives due to various caches
  30. # filling slowly with random data:
  31. warm_caches()
  32. # Save current values for dash_R_cleanup() to restore.
  33. fs = warnings.filters[:]
  34. ps = copyreg.dispatch_table.copy()
  35. pic = sys.path_importer_cache.copy()
  36. try:
  37. import zipimport
  38. except ImportError:
  39. zdc = None # Run unmodified on platforms without zipimport support
  40. else:
  41. zdc = zipimport._zip_directory_cache.copy()
  42. abcs = {}
  43. for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
  44. if not isabstract(abc):
  45. continue
  46. for obj in abc.__subclasses__() + [abc]:
  47. abcs[obj] = _get_dump(obj)[0]
  48. # bpo-31217: Integer pool to get a single integer object for the same
  49. # value. The pool is used to prevent false alarm when checking for memory
  50. # block leaks. Fill the pool with values in -1000..1000 which are the most
  51. # common (reference, memory block, file descriptor) differences.
  52. int_pool = {value: value for value in range(-1000, 1000)}
  53. def get_pooled_int(value):
  54. return int_pool.setdefault(value, value)
  55. nwarmup, ntracked, fname = ns.huntrleaks
  56. fname = os.path.join(os_helper.SAVEDCWD, fname)
  57. repcount = nwarmup + ntracked
  58. # Pre-allocate to ensure that the loop doesn't allocate anything new
  59. rep_range = list(range(repcount))
  60. rc_deltas = [0] * repcount
  61. alloc_deltas = [0] * repcount
  62. fd_deltas = [0] * repcount
  63. getallocatedblocks = sys.getallocatedblocks
  64. gettotalrefcount = sys.gettotalrefcount
  65. _getquickenedcount = sys._getquickenedcount
  66. fd_count = os_helper.fd_count
  67. # initialize variables to make pyflakes quiet
  68. rc_before = alloc_before = fd_before = 0
  69. if not ns.quiet:
  70. print("beginning", repcount, "repetitions", file=sys.stderr)
  71. print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
  72. flush=True)
  73. dash_R_cleanup(fs, ps, pic, zdc, abcs)
  74. support.gc_collect()
  75. for i in rep_range:
  76. test_func()
  77. dash_R_cleanup(fs, ps, pic, zdc, abcs)
  78. support.gc_collect()
  79. # Read memory statistics immediately after the garbage collection
  80. alloc_after = getallocatedblocks() - _getquickenedcount()
  81. rc_after = gettotalrefcount()
  82. fd_after = fd_count()
  83. if not ns.quiet:
  84. print('.', end='', file=sys.stderr, flush=True)
  85. rc_deltas[i] = get_pooled_int(rc_after - rc_before)
  86. alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
  87. fd_deltas[i] = get_pooled_int(fd_after - fd_before)
  88. alloc_before = alloc_after
  89. rc_before = rc_after
  90. fd_before = fd_after
  91. if not ns.quiet:
  92. print(file=sys.stderr)
  93. # These checkers return False on success, True on failure
  94. def check_rc_deltas(deltas):
  95. # Checker for reference counters and memory blocks.
  96. #
  97. # bpo-30776: Try to ignore false positives:
  98. #
  99. # [3, 0, 0]
  100. # [0, 1, 0]
  101. # [8, -8, 1]
  102. #
  103. # Expected leaks:
  104. #
  105. # [5, 5, 6]
  106. # [10, 1, 1]
  107. return all(delta >= 1 for delta in deltas)
  108. def check_fd_deltas(deltas):
  109. return any(deltas)
  110. failed = False
  111. for deltas, item_name, checker in [
  112. (rc_deltas, 'references', check_rc_deltas),
  113. (alloc_deltas, 'memory blocks', check_rc_deltas),
  114. (fd_deltas, 'file descriptors', check_fd_deltas)
  115. ]:
  116. # ignore warmup runs
  117. deltas = deltas[nwarmup:]
  118. if checker(deltas):
  119. msg = '%s leaked %s %s, sum=%s' % (
  120. test_name, deltas, item_name, sum(deltas))
  121. print(msg, file=sys.stderr, flush=True)
  122. with open(fname, "a", encoding="utf-8") as refrep:
  123. print(msg, file=refrep)
  124. refrep.flush()
  125. failed = True
  126. return failed
  127. def dash_R_cleanup(fs, ps, pic, zdc, abcs):
  128. import copyreg
  129. import collections.abc
  130. # Restore some original values.
  131. warnings.filters[:] = fs
  132. copyreg.dispatch_table.clear()
  133. copyreg.dispatch_table.update(ps)
  134. sys.path_importer_cache.clear()
  135. sys.path_importer_cache.update(pic)
  136. try:
  137. import zipimport
  138. except ImportError:
  139. pass # Run unmodified on platforms without zipimport support
  140. else:
  141. zipimport._zip_directory_cache.clear()
  142. zipimport._zip_directory_cache.update(zdc)
  143. # Clear ABC registries, restoring previously saved ABC registries.
  144. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
  145. abs_classes = filter(isabstract, abs_classes)
  146. for abc in abs_classes:
  147. for obj in abc.__subclasses__() + [abc]:
  148. for ref in abcs.get(obj, set()):
  149. if ref() is not None:
  150. obj.register(ref())
  151. obj._abc_caches_clear()
  152. # Clear caches
  153. clear_caches()
  154. # Clear type cache at the end: previous function calls can modify types
  155. sys._clear_type_cache()
  156. def warm_caches():
  157. # char cache
  158. s = bytes(range(256))
  159. for i in range(256):
  160. s[i:i+1]
  161. # unicode cache
  162. [chr(i) for i in range(256)]
  163. # int cache
  164. list(range(-5, 257))