test_modulefinder.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438
  1. import os
  2. import errno
  3. import importlib.machinery
  4. import py_compile
  5. import shutil
  6. import unittest
  7. import tempfile
  8. from test import support
  9. import modulefinder
  10. # Each test description is a list of 5 items:
  11. #
  12. # 1. a module name that will be imported by modulefinder
  13. # 2. a list of module names that modulefinder is required to find
  14. # 3. a list of module names that modulefinder should complain
  15. # about because they are not found
  16. # 4. a list of module names that modulefinder should complain
  17. # about because they MAY be not found
  18. # 5. a string specifying packages to create; the format is obvious imo.
  19. #
  20. # Each package will be created in test_dir, and test_dir will be
  21. # removed after the tests again.
  22. # Modulefinder searches in a path that contains test_dir, plus
  23. # the standard Lib directory.
  24. maybe_test = [
  25. "a.module",
  26. ["a", "a.module", "sys",
  27. "b"],
  28. ["c"], ["b.something"],
  29. """\
  30. a/__init__.py
  31. a/module.py
  32. from b import something
  33. from c import something
  34. b/__init__.py
  35. from sys import *
  36. """,
  37. ]
  38. maybe_test_new = [
  39. "a.module",
  40. ["a", "a.module", "sys",
  41. "b", "__future__"],
  42. ["c"], ["b.something"],
  43. """\
  44. a/__init__.py
  45. a/module.py
  46. from b import something
  47. from c import something
  48. b/__init__.py
  49. from __future__ import absolute_import
  50. from sys import *
  51. """]
  52. package_test = [
  53. "a.module",
  54. ["a", "a.b", "a.c", "a.module", "mymodule", "sys"],
  55. ["blahblah", "c"], [],
  56. """\
  57. mymodule.py
  58. a/__init__.py
  59. import blahblah
  60. from a import b
  61. import c
  62. a/module.py
  63. import sys
  64. from a import b as x
  65. from a.c import sillyname
  66. a/b.py
  67. a/c.py
  68. from a.module import x
  69. import mymodule as sillyname
  70. from sys import version_info
  71. """]
  72. absolute_import_test = [
  73. "a.module",
  74. ["a", "a.module",
  75. "b", "b.x", "b.y", "b.z",
  76. "__future__", "sys", "gc"],
  77. ["blahblah", "z"], [],
  78. """\
  79. mymodule.py
  80. a/__init__.py
  81. a/module.py
  82. from __future__ import absolute_import
  83. import sys # sys
  84. import blahblah # fails
  85. import gc # gc
  86. import b.x # b.x
  87. from b import y # b.y
  88. from b.z import * # b.z.*
  89. a/gc.py
  90. a/sys.py
  91. import mymodule
  92. a/b/__init__.py
  93. a/b/x.py
  94. a/b/y.py
  95. a/b/z.py
  96. b/__init__.py
  97. import z
  98. b/unused.py
  99. b/x.py
  100. b/y.py
  101. b/z.py
  102. """]
  103. relative_import_test = [
  104. "a.module",
  105. ["__future__",
  106. "a", "a.module",
  107. "a.b", "a.b.y", "a.b.z",
  108. "a.b.c", "a.b.c.moduleC",
  109. "a.b.c.d", "a.b.c.e",
  110. "a.b.x",
  111. "gc"],
  112. [], [],
  113. """\
  114. mymodule.py
  115. a/__init__.py
  116. from .b import y, z # a.b.y, a.b.z
  117. a/module.py
  118. from __future__ import absolute_import # __future__
  119. import gc # gc
  120. a/gc.py
  121. a/sys.py
  122. a/b/__init__.py
  123. from ..b import x # a.b.x
  124. #from a.b.c import moduleC
  125. from .c import moduleC # a.b.moduleC
  126. a/b/x.py
  127. a/b/y.py
  128. a/b/z.py
  129. a/b/g.py
  130. a/b/c/__init__.py
  131. from ..c import e # a.b.c.e
  132. a/b/c/moduleC.py
  133. from ..c import d # a.b.c.d
  134. a/b/c/d.py
  135. a/b/c/e.py
  136. a/b/c/x.py
  137. """]
  138. relative_import_test_2 = [
  139. "a.module",
  140. ["a", "a.module",
  141. "a.sys",
  142. "a.b", "a.b.y", "a.b.z",
  143. "a.b.c", "a.b.c.d",
  144. "a.b.c.e",
  145. "a.b.c.moduleC",
  146. "a.b.c.f",
  147. "a.b.x",
  148. "a.another"],
  149. [], [],
  150. """\
  151. mymodule.py
  152. a/__init__.py
  153. from . import sys # a.sys
  154. a/another.py
  155. a/module.py
  156. from .b import y, z # a.b.y, a.b.z
  157. a/gc.py
  158. a/sys.py
  159. a/b/__init__.py
  160. from .c import moduleC # a.b.c.moduleC
  161. from .c import d # a.b.c.d
  162. a/b/x.py
  163. a/b/y.py
  164. a/b/z.py
  165. a/b/c/__init__.py
  166. from . import e # a.b.c.e
  167. a/b/c/moduleC.py
  168. #
  169. from . import f # a.b.c.f
  170. from .. import x # a.b.x
  171. from ... import another # a.another
  172. a/b/c/d.py
  173. a/b/c/e.py
  174. a/b/c/f.py
  175. """]
  176. relative_import_test_3 = [
  177. "a.module",
  178. ["a", "a.module"],
  179. ["a.bar"],
  180. [],
  181. """\
  182. a/__init__.py
  183. def foo(): pass
  184. a/module.py
  185. from . import foo
  186. from . import bar
  187. """]
  188. relative_import_test_4 = [
  189. "a.module",
  190. ["a", "a.module"],
  191. [],
  192. [],
  193. """\
  194. a/__init__.py
  195. def foo(): pass
  196. a/module.py
  197. from . import *
  198. """]
  199. bytecode_test = [
  200. "a",
  201. ["a"],
  202. [],
  203. [],
  204. ""
  205. ]
  206. syntax_error_test = [
  207. "a.module",
  208. ["a", "a.module", "b"],
  209. ["b.module"], [],
  210. """\
  211. a/__init__.py
  212. a/module.py
  213. import b.module
  214. b/__init__.py
  215. b/module.py
  216. ? # SyntaxError: invalid syntax
  217. """]
  218. same_name_as_bad_test = [
  219. "a.module",
  220. ["a", "a.module", "b", "b.c"],
  221. ["c"], [],
  222. """\
  223. a/__init__.py
  224. a/module.py
  225. import c
  226. from b import c
  227. b/__init__.py
  228. b/c.py
  229. """]
  230. coding_default_utf8_test = [
  231. "a_utf8",
  232. ["a_utf8", "b_utf8"],
  233. [], [],
  234. """\
  235. a_utf8.py
  236. # use the default of utf8
  237. print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
  238. import b_utf8
  239. b_utf8.py
  240. # use the default of utf8
  241. print('Unicode test B code point 2090 \u2090 that is not valid in cp1252')
  242. """]
  243. coding_explicit_utf8_test = [
  244. "a_utf8",
  245. ["a_utf8", "b_utf8"],
  246. [], [],
  247. """\
  248. a_utf8.py
  249. # coding=utf8
  250. print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
  251. import b_utf8
  252. b_utf8.py
  253. # use the default of utf8
  254. print('Unicode test B code point 2090 \u2090 that is not valid in cp1252')
  255. """]
  256. coding_explicit_cp1252_test = [
  257. "a_cp1252",
  258. ["a_cp1252", "b_utf8"],
  259. [], [],
  260. b"""\
  261. a_cp1252.py
  262. # coding=cp1252
  263. # 0xe2 is not allowed in utf8
  264. print('CP1252 test P\xe2t\xe9')
  265. import b_utf8
  266. """ + """\
  267. b_utf8.py
  268. # use the default of utf8
  269. print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
  270. """.encode('utf-8')]
  271. def open_file(path):
  272. dirname = os.path.dirname(path)
  273. try:
  274. os.makedirs(dirname)
  275. except OSError as e:
  276. if e.errno != errno.EEXIST:
  277. raise
  278. return open(path, 'wb')
  279. def create_package(test_dir, source):
  280. ofi = None
  281. try:
  282. for line in source.splitlines():
  283. if type(line) != bytes:
  284. line = line.encode('utf-8')
  285. if line.startswith(b' ') or line.startswith(b'\t'):
  286. ofi.write(line.strip() + b'\n')
  287. else:
  288. if ofi:
  289. ofi.close()
  290. if type(line) == bytes:
  291. line = line.decode('utf-8')
  292. ofi = open_file(os.path.join(test_dir, line.strip()))
  293. finally:
  294. if ofi:
  295. ofi.close()
  296. class ModuleFinderTest(unittest.TestCase):
  297. def setUp(self):
  298. self.test_dir = tempfile.mkdtemp()
  299. self.test_path = [self.test_dir, os.path.dirname(tempfile.__file__)]
  300. def tearDown(self):
  301. shutil.rmtree(self.test_dir)
  302. def _do_test(self, info, report=False, debug=0, replace_paths=[], modulefinder_class=modulefinder.ModuleFinder):
  303. import_this, modules, missing, maybe_missing, source = info
  304. create_package(self.test_dir, source)
  305. mf = modulefinder_class(path=self.test_path, debug=debug,
  306. replace_paths=replace_paths)
  307. mf.import_hook(import_this)
  308. if report:
  309. mf.report()
  310. ## # This wouldn't work in general when executed several times:
  311. ## opath = sys.path[:]
  312. ## sys.path = self.test_path
  313. ## try:
  314. ## __import__(import_this)
  315. ## except:
  316. ## import traceback; traceback.print_exc()
  317. ## sys.path = opath
  318. ## return
  319. modules = sorted(set(modules))
  320. found = sorted(mf.modules)
  321. # check if we found what we expected, not more, not less
  322. self.assertEqual(found, modules)
  323. # check for missing and maybe missing modules
  324. bad, maybe = mf.any_missing_maybe()
  325. self.assertEqual(bad, missing)
  326. self.assertEqual(maybe, maybe_missing)
  327. def test_package(self):
  328. self._do_test(package_test)
  329. def test_maybe(self):
  330. self._do_test(maybe_test)
  331. def test_maybe_new(self):
  332. self._do_test(maybe_test_new)
  333. def test_absolute_imports(self):
  334. self._do_test(absolute_import_test)
  335. def test_relative_imports(self):
  336. self._do_test(relative_import_test)
  337. def test_relative_imports_2(self):
  338. self._do_test(relative_import_test_2)
  339. def test_relative_imports_3(self):
  340. self._do_test(relative_import_test_3)
  341. def test_relative_imports_4(self):
  342. self._do_test(relative_import_test_4)
  343. def test_syntax_error(self):
  344. self._do_test(syntax_error_test)
  345. def test_same_name_as_bad(self):
  346. self._do_test(same_name_as_bad_test)
  347. def test_bytecode(self):
  348. base_path = os.path.join(self.test_dir, 'a')
  349. source_path = base_path + importlib.machinery.SOURCE_SUFFIXES[0]
  350. bytecode_path = base_path + importlib.machinery.BYTECODE_SUFFIXES[0]
  351. with open_file(source_path) as file:
  352. file.write('testing_modulefinder = True\n'.encode('utf-8'))
  353. py_compile.compile(source_path, cfile=bytecode_path)
  354. os.remove(source_path)
  355. self._do_test(bytecode_test)
  356. def test_replace_paths(self):
  357. old_path = os.path.join(self.test_dir, 'a', 'module.py')
  358. new_path = os.path.join(self.test_dir, 'a', 'spam.py')
  359. with support.captured_stdout() as output:
  360. self._do_test(maybe_test, debug=2,
  361. replace_paths=[(old_path, new_path)])
  362. output = output.getvalue()
  363. expected = "co_filename %r changed to %r" % (old_path, new_path)
  364. self.assertIn(expected, output)
  365. def test_extended_opargs(self):
  366. extended_opargs_test = [
  367. "a",
  368. ["a", "b"],
  369. [], [],
  370. """\
  371. a.py
  372. %r
  373. import b
  374. b.py
  375. """ % list(range(2**16))] # 2**16 constants
  376. self._do_test(extended_opargs_test)
  377. def test_coding_default_utf8(self):
  378. self._do_test(coding_default_utf8_test)
  379. def test_coding_explicit_utf8(self):
  380. self._do_test(coding_explicit_utf8_test)
  381. def test_coding_explicit_cp1252(self):
  382. self._do_test(coding_explicit_cp1252_test)
  383. def test_load_module_api(self):
  384. class CheckLoadModuleApi(modulefinder.ModuleFinder):
  385. def __init__(self, *args, **kwds):
  386. super().__init__(*args, **kwds)
  387. def load_module(self, fqname, fp, pathname, file_info):
  388. # confirm that the fileinfo is a tuple of 3 elements
  389. suffix, mode, type = file_info
  390. return super().load_module(fqname, fp, pathname, file_info)
  391. self._do_test(absolute_import_test, modulefinder_class=CheckLoadModuleApi)
  392. if __name__ == "__main__":
  393. unittest.main()