_bootstrap.py 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367
  1. """Core implementation of import.
  2. This module is NOT meant to be directly imported! It has been designed such
  3. that it can be bootstrapped into Python as the implementation of import. As
  4. such it requires the injection of specific modules and attributes in order to
  5. work. One should use importlib as the public-facing version of this module.
  6. """
  7. #
  8. # IMPORTANT: Whenever making changes to this module, be sure to run a top-level
  9. # `make regen-importlib` followed by `make` in order to get the frozen version
  10. # of the module updated. Not doing so will result in the Makefile to fail for
  11. # all others who don't have a ./python around to freeze the module
  12. # in the early stages of compilation.
  13. #
  14. # See importlib._setup() for what is injected into the global namespace.
  15. # When editing this code be aware that code executed at import time CANNOT
  16. # reference any injected objects! This includes not only global code but also
  17. # anything specified at the class level.
  18. def _object_name(obj):
  19. try:
  20. return obj.__qualname__
  21. except AttributeError:
  22. return type(obj).__qualname__
  23. # Bootstrap-related code ######################################################
  24. # Modules injected manually by _setup()
  25. _thread = None
  26. _warnings = None
  27. _weakref = None
  28. # Import done by _install_external_importers()
  29. _bootstrap_external = None
  30. def _wrap(new, old):
  31. """Simple substitute for functools.update_wrapper."""
  32. for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
  33. if hasattr(old, replace):
  34. setattr(new, replace, getattr(old, replace))
  35. new.__dict__.update(old.__dict__)
  36. def _new_module(name):
  37. return type(sys)(name)
  38. # Module-level locking ########################################################
  39. # A dict mapping module names to weakrefs of _ModuleLock instances
  40. # Dictionary protected by the global import lock
  41. _module_locks = {}
  42. # A dict mapping thread ids to _ModuleLock instances
  43. _blocking_on = {}
  44. class _DeadlockError(RuntimeError):
  45. pass
  46. class _ModuleLock:
  47. """A recursive lock implementation which is able to detect deadlocks
  48. (e.g. thread 1 trying to take locks A then B, and thread 2 trying to
  49. take locks B then A).
  50. """
  51. def __init__(self, name):
  52. self.lock = _thread.allocate_lock()
  53. self.wakeup = _thread.allocate_lock()
  54. self.name = name
  55. self.owner = None
  56. self.count = 0
  57. self.waiters = 0
  58. def has_deadlock(self):
  59. # Deadlock avoidance for concurrent circular imports.
  60. me = _thread.get_ident()
  61. tid = self.owner
  62. seen = set()
  63. while True:
  64. lock = _blocking_on.get(tid)
  65. if lock is None:
  66. return False
  67. tid = lock.owner
  68. if tid == me:
  69. return True
  70. if tid in seen:
  71. # bpo 38091: the chain of tid's we encounter here
  72. # eventually leads to a fixpoint or a cycle, but
  73. # does not reach 'me'. This means we would not
  74. # actually deadlock. This can happen if other
  75. # threads are at the beginning of acquire() below.
  76. return False
  77. seen.add(tid)
  78. def acquire(self):
  79. """
  80. Acquire the module lock. If a potential deadlock is detected,
  81. a _DeadlockError is raised.
  82. Otherwise, the lock is always acquired and True is returned.
  83. """
  84. tid = _thread.get_ident()
  85. _blocking_on[tid] = self
  86. try:
  87. while True:
  88. with self.lock:
  89. if self.count == 0 or self.owner == tid:
  90. self.owner = tid
  91. self.count += 1
  92. return True
  93. if self.has_deadlock():
  94. raise _DeadlockError('deadlock detected by %r' % self)
  95. if self.wakeup.acquire(False):
  96. self.waiters += 1
  97. # Wait for a release() call
  98. self.wakeup.acquire()
  99. self.wakeup.release()
  100. finally:
  101. del _blocking_on[tid]
  102. def release(self):
  103. tid = _thread.get_ident()
  104. with self.lock:
  105. if self.owner != tid:
  106. raise RuntimeError('cannot release un-acquired lock')
  107. assert self.count > 0
  108. self.count -= 1
  109. if self.count == 0:
  110. self.owner = None
  111. if self.waiters:
  112. self.waiters -= 1
  113. self.wakeup.release()
  114. def __repr__(self):
  115. return '_ModuleLock({!r}) at {}'.format(self.name, id(self))
  116. class _DummyModuleLock:
  117. """A simple _ModuleLock equivalent for Python builds without
  118. multi-threading support."""
  119. def __init__(self, name):
  120. self.name = name
  121. self.count = 0
  122. def acquire(self):
  123. self.count += 1
  124. return True
  125. def release(self):
  126. if self.count == 0:
  127. raise RuntimeError('cannot release un-acquired lock')
  128. self.count -= 1
  129. def __repr__(self):
  130. return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self))
  131. class _ModuleLockManager:
  132. def __init__(self, name):
  133. self._name = name
  134. self._lock = None
  135. def __enter__(self):
  136. self._lock = _get_module_lock(self._name)
  137. self._lock.acquire()
  138. def __exit__(self, *args, **kwargs):
  139. self._lock.release()
  140. # The following two functions are for consumption by Python/import.c.
  141. def _get_module_lock(name):
  142. """Get or create the module lock for a given module name.
  143. Acquire/release internally the global import lock to protect
  144. _module_locks."""
  145. _imp.acquire_lock()
  146. try:
  147. try:
  148. lock = _module_locks[name]()
  149. except KeyError:
  150. lock = None
  151. if lock is None:
  152. if _thread is None:
  153. lock = _DummyModuleLock(name)
  154. else:
  155. lock = _ModuleLock(name)
  156. def cb(ref, name=name):
  157. _imp.acquire_lock()
  158. try:
  159. # bpo-31070: Check if another thread created a new lock
  160. # after the previous lock was destroyed
  161. # but before the weakref callback was called.
  162. if _module_locks.get(name) is ref:
  163. del _module_locks[name]
  164. finally:
  165. _imp.release_lock()
  166. _module_locks[name] = _weakref.ref(lock, cb)
  167. finally:
  168. _imp.release_lock()
  169. return lock
  170. def _lock_unlock_module(name):
  171. """Acquires then releases the module lock for a given module name.
  172. This is used to ensure a module is completely initialized, in the
  173. event it is being imported by another thread.
  174. """
  175. lock = _get_module_lock(name)
  176. try:
  177. lock.acquire()
  178. except _DeadlockError:
  179. # Concurrent circular import, we'll accept a partially initialized
  180. # module object.
  181. pass
  182. else:
  183. lock.release()
  184. # Frame stripping magic ###############################################
  185. def _call_with_frames_removed(f, *args, **kwds):
  186. """remove_importlib_frames in import.c will always remove sequences
  187. of importlib frames that end with a call to this function
  188. Use it instead of a normal call in places where including the importlib
  189. frames introduces unwanted noise into the traceback (e.g. when executing
  190. module code)
  191. """
  192. return f(*args, **kwds)
  193. def _verbose_message(message, *args, verbosity=1):
  194. """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
  195. if sys.flags.verbose >= verbosity:
  196. if not message.startswith(('#', 'import ')):
  197. message = '# ' + message
  198. print(message.format(*args), file=sys.stderr)
  199. def _requires_builtin(fxn):
  200. """Decorator to verify the named module is built-in."""
  201. def _requires_builtin_wrapper(self, fullname):
  202. if fullname not in sys.builtin_module_names:
  203. raise ImportError('{!r} is not a built-in module'.format(fullname),
  204. name=fullname)
  205. return fxn(self, fullname)
  206. _wrap(_requires_builtin_wrapper, fxn)
  207. return _requires_builtin_wrapper
  208. def _requires_frozen(fxn):
  209. """Decorator to verify the named module is frozen."""
  210. def _requires_frozen_wrapper(self, fullname):
  211. if not _imp.is_frozen(fullname):
  212. raise ImportError('{!r} is not a frozen module'.format(fullname),
  213. name=fullname)
  214. return fxn(self, fullname)
  215. _wrap(_requires_frozen_wrapper, fxn)
  216. return _requires_frozen_wrapper
  217. # Typically used by loader classes as a method replacement.
  218. def _load_module_shim(self, fullname):
  219. """Load the specified module into sys.modules and return it.
  220. This method is deprecated. Use loader.exec_module() instead.
  221. """
  222. msg = ("the load_module() method is deprecated and slated for removal in "
  223. "Python 3.12; use exec_module() instead")
  224. _warnings.warn(msg, DeprecationWarning)
  225. spec = spec_from_loader(fullname, self)
  226. if fullname in sys.modules:
  227. module = sys.modules[fullname]
  228. _exec(spec, module)
  229. return sys.modules[fullname]
  230. else:
  231. return _load(spec)
  232. # Module specifications #######################################################
  233. def _module_repr(module):
  234. """The implementation of ModuleType.__repr__()."""
  235. loader = getattr(module, '__loader__', None)
  236. if spec := getattr(module, "__spec__", None):
  237. return _module_repr_from_spec(spec)
  238. elif hasattr(loader, 'module_repr'):
  239. try:
  240. return loader.module_repr(module)
  241. except Exception:
  242. pass
  243. # Fall through to a catch-all which always succeeds.
  244. try:
  245. name = module.__name__
  246. except AttributeError:
  247. name = '?'
  248. try:
  249. filename = module.__file__
  250. except AttributeError:
  251. if loader is None:
  252. return '<module {!r}>'.format(name)
  253. else:
  254. return '<module {!r} ({!r})>'.format(name, loader)
  255. else:
  256. return '<module {!r} from {!r}>'.format(name, filename)
  257. class ModuleSpec:
  258. """The specification for a module, used for loading.
  259. A module's spec is the source for information about the module. For
  260. data associated with the module, including source, use the spec's
  261. loader.
  262. `name` is the absolute name of the module. `loader` is the loader
  263. to use when loading the module. `parent` is the name of the
  264. package the module is in. The parent is derived from the name.
  265. `is_package` determines if the module is considered a package or
  266. not. On modules this is reflected by the `__path__` attribute.
  267. `origin` is the specific location used by the loader from which to
  268. load the module, if that information is available. When filename is
  269. set, origin will match.
  270. `has_location` indicates that a spec's "origin" reflects a location.
  271. When this is True, `__file__` attribute of the module is set.
  272. `cached` is the location of the cached bytecode file, if any. It
  273. corresponds to the `__cached__` attribute.
  274. `submodule_search_locations` is the sequence of path entries to
  275. search when importing submodules. If set, is_package should be
  276. True--and False otherwise.
  277. Packages are simply modules that (may) have submodules. If a spec
  278. has a non-None value in `submodule_search_locations`, the import
  279. system will consider modules loaded from the spec as packages.
  280. Only finders (see importlib.abc.MetaPathFinder and
  281. importlib.abc.PathEntryFinder) should modify ModuleSpec instances.
  282. """
  283. def __init__(self, name, loader, *, origin=None, loader_state=None,
  284. is_package=None):
  285. self.name = name
  286. self.loader = loader
  287. self.origin = origin
  288. self.loader_state = loader_state
  289. self.submodule_search_locations = [] if is_package else None
  290. self._uninitialized_submodules = []
  291. # file-location attributes
  292. self._set_fileattr = False
  293. self._cached = None
  294. def __repr__(self):
  295. args = ['name={!r}'.format(self.name),
  296. 'loader={!r}'.format(self.loader)]
  297. if self.origin is not None:
  298. args.append('origin={!r}'.format(self.origin))
  299. if self.submodule_search_locations is not None:
  300. args.append('submodule_search_locations={}'
  301. .format(self.submodule_search_locations))
  302. return '{}({})'.format(self.__class__.__name__, ', '.join(args))
  303. def __eq__(self, other):
  304. smsl = self.submodule_search_locations
  305. try:
  306. return (self.name == other.name and
  307. self.loader == other.loader and
  308. self.origin == other.origin and
  309. smsl == other.submodule_search_locations and
  310. self.cached == other.cached and
  311. self.has_location == other.has_location)
  312. except AttributeError:
  313. return NotImplemented
  314. @property
  315. def cached(self):
  316. if self._cached is None:
  317. if self.origin is not None and self._set_fileattr:
  318. if _bootstrap_external is None:
  319. raise NotImplementedError
  320. self._cached = _bootstrap_external._get_cached(self.origin)
  321. return self._cached
  322. @cached.setter
  323. def cached(self, cached):
  324. self._cached = cached
  325. @property
  326. def parent(self):
  327. """The name of the module's parent."""
  328. if self.submodule_search_locations is None:
  329. return self.name.rpartition('.')[0]
  330. else:
  331. return self.name
  332. @property
  333. def has_location(self):
  334. return self._set_fileattr
  335. @has_location.setter
  336. def has_location(self, value):
  337. self._set_fileattr = bool(value)
  338. def spec_from_loader(name, loader, *, origin=None, is_package=None):
  339. """Return a module spec based on various loader methods."""
  340. if origin is None:
  341. origin = getattr(loader, '_ORIGIN', None)
  342. if not origin and hasattr(loader, 'get_filename'):
  343. if _bootstrap_external is None:
  344. raise NotImplementedError
  345. spec_from_file_location = _bootstrap_external.spec_from_file_location
  346. if is_package is None:
  347. return spec_from_file_location(name, loader=loader)
  348. search = [] if is_package else None
  349. return spec_from_file_location(name, loader=loader,
  350. submodule_search_locations=search)
  351. if is_package is None:
  352. if hasattr(loader, 'is_package'):
  353. try:
  354. is_package = loader.is_package(name)
  355. except ImportError:
  356. is_package = None # aka, undefined
  357. else:
  358. # the default
  359. is_package = False
  360. return ModuleSpec(name, loader, origin=origin, is_package=is_package)
  361. def _spec_from_module(module, loader=None, origin=None):
  362. # This function is meant for use in _setup().
  363. try:
  364. spec = module.__spec__
  365. except AttributeError:
  366. pass
  367. else:
  368. if spec is not None:
  369. return spec
  370. name = module.__name__
  371. if loader is None:
  372. try:
  373. loader = module.__loader__
  374. except AttributeError:
  375. # loader will stay None.
  376. pass
  377. try:
  378. location = module.__file__
  379. except AttributeError:
  380. location = None
  381. if origin is None:
  382. if loader is not None:
  383. origin = getattr(loader, '_ORIGIN', None)
  384. if not origin and location is not None:
  385. origin = location
  386. try:
  387. cached = module.__cached__
  388. except AttributeError:
  389. cached = None
  390. try:
  391. submodule_search_locations = list(module.__path__)
  392. except AttributeError:
  393. submodule_search_locations = None
  394. spec = ModuleSpec(name, loader, origin=origin)
  395. spec._set_fileattr = False if location is None else (origin == location)
  396. spec.cached = cached
  397. spec.submodule_search_locations = submodule_search_locations
  398. return spec
  399. def _init_module_attrs(spec, module, *, override=False):
  400. # The passed-in module may be not support attribute assignment,
  401. # in which case we simply don't set the attributes.
  402. # __name__
  403. if (override or getattr(module, '__name__', None) is None):
  404. try:
  405. module.__name__ = spec.name
  406. except AttributeError:
  407. pass
  408. # __loader__
  409. if override or getattr(module, '__loader__', None) is None:
  410. loader = spec.loader
  411. if loader is None:
  412. # A backward compatibility hack.
  413. if spec.submodule_search_locations is not None:
  414. if _bootstrap_external is None:
  415. raise NotImplementedError
  416. NamespaceLoader = _bootstrap_external.NamespaceLoader
  417. loader = NamespaceLoader.__new__(NamespaceLoader)
  418. loader._path = spec.submodule_search_locations
  419. spec.loader = loader
  420. # While the docs say that module.__file__ is not set for
  421. # built-in modules, and the code below will avoid setting it if
  422. # spec.has_location is false, this is incorrect for namespace
  423. # packages. Namespace packages have no location, but their
  424. # __spec__.origin is None, and thus their module.__file__
  425. # should also be None for consistency. While a bit of a hack,
  426. # this is the best place to ensure this consistency.
  427. #
  428. # See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module
  429. # and bpo-32305
  430. module.__file__ = None
  431. try:
  432. module.__loader__ = loader
  433. except AttributeError:
  434. pass
  435. # __package__
  436. if override or getattr(module, '__package__', None) is None:
  437. try:
  438. module.__package__ = spec.parent
  439. except AttributeError:
  440. pass
  441. # __spec__
  442. try:
  443. module.__spec__ = spec
  444. except AttributeError:
  445. pass
  446. # __path__
  447. if override or getattr(module, '__path__', None) is None:
  448. if spec.submodule_search_locations is not None:
  449. # XXX We should extend __path__ if it's already a list.
  450. try:
  451. module.__path__ = spec.submodule_search_locations
  452. except AttributeError:
  453. pass
  454. # __file__/__cached__
  455. if spec.has_location:
  456. if override or getattr(module, '__file__', None) is None:
  457. try:
  458. module.__file__ = spec.origin
  459. except AttributeError:
  460. pass
  461. if override or getattr(module, '__cached__', None) is None:
  462. if spec.cached is not None:
  463. try:
  464. module.__cached__ = spec.cached
  465. except AttributeError:
  466. pass
  467. return module
  468. def module_from_spec(spec):
  469. """Create a module based on the provided spec."""
  470. # Typically loaders will not implement create_module().
  471. module = None
  472. if hasattr(spec.loader, 'create_module'):
  473. # If create_module() returns `None` then it means default
  474. # module creation should be used.
  475. module = spec.loader.create_module(spec)
  476. elif hasattr(spec.loader, 'exec_module'):
  477. raise ImportError('loaders that define exec_module() '
  478. 'must also define create_module()')
  479. if module is None:
  480. module = _new_module(spec.name)
  481. _init_module_attrs(spec, module)
  482. return module
  483. def _module_repr_from_spec(spec):
  484. """Return the repr to use for the module."""
  485. # We mostly replicate _module_repr() using the spec attributes.
  486. name = '?' if spec.name is None else spec.name
  487. if spec.origin is None:
  488. if spec.loader is None:
  489. return '<module {!r}>'.format(name)
  490. else:
  491. return '<module {!r} ({!r})>'.format(name, spec.loader)
  492. else:
  493. if spec.has_location:
  494. return '<module {!r} from {!r}>'.format(name, spec.origin)
  495. else:
  496. return '<module {!r} ({})>'.format(spec.name, spec.origin)
  497. # Used by importlib.reload() and _load_module_shim().
  498. def _exec(spec, module):
  499. """Execute the spec's specified module in an existing module's namespace."""
  500. name = spec.name
  501. with _ModuleLockManager(name):
  502. if sys.modules.get(name) is not module:
  503. msg = 'module {!r} not in sys.modules'.format(name)
  504. raise ImportError(msg, name=name)
  505. try:
  506. if spec.loader is None:
  507. if spec.submodule_search_locations is None:
  508. raise ImportError('missing loader', name=spec.name)
  509. # Namespace package.
  510. _init_module_attrs(spec, module, override=True)
  511. else:
  512. _init_module_attrs(spec, module, override=True)
  513. if not hasattr(spec.loader, 'exec_module'):
  514. msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
  515. "falling back to load_module()")
  516. _warnings.warn(msg, ImportWarning)
  517. spec.loader.load_module(name)
  518. else:
  519. spec.loader.exec_module(module)
  520. finally:
  521. # Update the order of insertion into sys.modules for module
  522. # clean-up at shutdown.
  523. module = sys.modules.pop(spec.name)
  524. sys.modules[spec.name] = module
  525. return module
  526. def _load_backward_compatible(spec):
  527. # It is assumed that all callers have been warned about using load_module()
  528. # appropriately before calling this function.
  529. try:
  530. spec.loader.load_module(spec.name)
  531. except:
  532. if spec.name in sys.modules:
  533. module = sys.modules.pop(spec.name)
  534. sys.modules[spec.name] = module
  535. raise
  536. # The module must be in sys.modules at this point!
  537. # Move it to the end of sys.modules.
  538. module = sys.modules.pop(spec.name)
  539. sys.modules[spec.name] = module
  540. if getattr(module, '__loader__', None) is None:
  541. try:
  542. module.__loader__ = spec.loader
  543. except AttributeError:
  544. pass
  545. if getattr(module, '__package__', None) is None:
  546. try:
  547. # Since module.__path__ may not line up with
  548. # spec.submodule_search_paths, we can't necessarily rely
  549. # on spec.parent here.
  550. module.__package__ = module.__name__
  551. if not hasattr(module, '__path__'):
  552. module.__package__ = spec.name.rpartition('.')[0]
  553. except AttributeError:
  554. pass
  555. if getattr(module, '__spec__', None) is None:
  556. try:
  557. module.__spec__ = spec
  558. except AttributeError:
  559. pass
  560. return module
  561. def _load_unlocked(spec):
  562. # A helper for direct use by the import system.
  563. if spec.loader is not None:
  564. # Not a namespace package.
  565. if not hasattr(spec.loader, 'exec_module'):
  566. msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
  567. "falling back to load_module()")
  568. _warnings.warn(msg, ImportWarning)
  569. return _load_backward_compatible(spec)
  570. module = module_from_spec(spec)
  571. # This must be done before putting the module in sys.modules
  572. # (otherwise an optimization shortcut in import.c becomes
  573. # wrong).
  574. spec._initializing = True
  575. try:
  576. sys.modules[spec.name] = module
  577. try:
  578. if spec.loader is None:
  579. if spec.submodule_search_locations is None:
  580. raise ImportError('missing loader', name=spec.name)
  581. # A namespace package so do nothing.
  582. else:
  583. spec.loader.exec_module(module)
  584. except:
  585. try:
  586. del sys.modules[spec.name]
  587. except KeyError:
  588. pass
  589. raise
  590. # Move the module to the end of sys.modules.
  591. # We don't ensure that the import-related module attributes get
  592. # set in the sys.modules replacement case. Such modules are on
  593. # their own.
  594. module = sys.modules.pop(spec.name)
  595. sys.modules[spec.name] = module
  596. _verbose_message('import {!r} # {!r}', spec.name, spec.loader)
  597. finally:
  598. spec._initializing = False
  599. return module
  600. # A method used during testing of _load_unlocked() and by
  601. # _load_module_shim().
  602. def _load(spec):
  603. """Return a new module object, loaded by the spec's loader.
  604. The module is not added to its parent.
  605. If a module is already in sys.modules, that existing module gets
  606. clobbered.
  607. """
  608. with _ModuleLockManager(spec.name):
  609. return _load_unlocked(spec)
  610. # Loaders #####################################################################
  611. class BuiltinImporter:
  612. """Meta path import for built-in modules.
  613. All methods are either class or static methods to avoid the need to
  614. instantiate the class.
  615. """
  616. _ORIGIN = "built-in"
  617. @staticmethod
  618. def module_repr(module):
  619. """Return repr for the module.
  620. The method is deprecated. The import machinery does the job itself.
  621. """
  622. _warnings.warn("BuiltinImporter.module_repr() is deprecated and "
  623. "slated for removal in Python 3.12", DeprecationWarning)
  624. return f'<module {module.__name__!r} ({BuiltinImporter._ORIGIN})>'
  625. @classmethod
  626. def find_spec(cls, fullname, path=None, target=None):
  627. if path is not None:
  628. return None
  629. if _imp.is_builtin(fullname):
  630. return spec_from_loader(fullname, cls, origin=cls._ORIGIN)
  631. else:
  632. return None
  633. @classmethod
  634. def find_module(cls, fullname, path=None):
  635. """Find the built-in module.
  636. If 'path' is ever specified then the search is considered a failure.
  637. This method is deprecated. Use find_spec() instead.
  638. """
  639. _warnings.warn("BuiltinImporter.find_module() is deprecated and "
  640. "slated for removal in Python 3.12; use find_spec() instead",
  641. DeprecationWarning)
  642. spec = cls.find_spec(fullname, path)
  643. return spec.loader if spec is not None else None
  644. @staticmethod
  645. def create_module(spec):
  646. """Create a built-in module"""
  647. if spec.name not in sys.builtin_module_names:
  648. raise ImportError('{!r} is not a built-in module'.format(spec.name),
  649. name=spec.name)
  650. return _call_with_frames_removed(_imp.create_builtin, spec)
  651. @staticmethod
  652. def exec_module(module):
  653. """Exec a built-in module"""
  654. _call_with_frames_removed(_imp.exec_builtin, module)
  655. @classmethod
  656. @_requires_builtin
  657. def get_code(cls, fullname):
  658. """Return None as built-in modules do not have code objects."""
  659. return None
  660. @classmethod
  661. @_requires_builtin
  662. def get_source(cls, fullname):
  663. """Return None as built-in modules do not have source code."""
  664. return None
  665. @classmethod
  666. @_requires_builtin
  667. def is_package(cls, fullname):
  668. """Return False as built-in modules are never packages."""
  669. return False
  670. load_module = classmethod(_load_module_shim)
  671. class FrozenImporter:
  672. """Meta path import for frozen modules.
  673. All methods are either class or static methods to avoid the need to
  674. instantiate the class.
  675. """
  676. _ORIGIN = "frozen"
  677. @staticmethod
  678. def module_repr(m):
  679. """Return repr for the module.
  680. The method is deprecated. The import machinery does the job itself.
  681. """
  682. _warnings.warn("FrozenImporter.module_repr() is deprecated and "
  683. "slated for removal in Python 3.12", DeprecationWarning)
  684. return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
  685. @classmethod
  686. def _fix_up_module(cls, module):
  687. spec = module.__spec__
  688. state = spec.loader_state
  689. if state is None:
  690. # The module is missing FrozenImporter-specific values.
  691. # Fix up the spec attrs.
  692. origname = vars(module).pop('__origname__', None)
  693. assert origname, 'see PyImport_ImportFrozenModuleObject()'
  694. ispkg = hasattr(module, '__path__')
  695. assert _imp.is_frozen_package(module.__name__) == ispkg, ispkg
  696. filename, pkgdir = cls._resolve_filename(origname, spec.name, ispkg)
  697. spec.loader_state = type(sys.implementation)(
  698. filename=filename,
  699. origname=origname,
  700. )
  701. __path__ = spec.submodule_search_locations
  702. if ispkg:
  703. assert __path__ == [], __path__
  704. if pkgdir:
  705. spec.submodule_search_locations.insert(0, pkgdir)
  706. else:
  707. assert __path__ is None, __path__
  708. # Fix up the module attrs (the bare minimum).
  709. assert not hasattr(module, '__file__'), module.__file__
  710. if filename:
  711. try:
  712. module.__file__ = filename
  713. except AttributeError:
  714. pass
  715. if ispkg:
  716. if module.__path__ != __path__:
  717. assert module.__path__ == [], module.__path__
  718. module.__path__.extend(__path__)
  719. else:
  720. # These checks ensure that _fix_up_module() is only called
  721. # in the right places.
  722. __path__ = spec.submodule_search_locations
  723. ispkg = __path__ is not None
  724. # Check the loader state.
  725. assert sorted(vars(state)) == ['filename', 'origname'], state
  726. if state.origname:
  727. # The only frozen modules with "origname" set are stdlib modules.
  728. (__file__, pkgdir,
  729. ) = cls._resolve_filename(state.origname, spec.name, ispkg)
  730. assert state.filename == __file__, (state.filename, __file__)
  731. if pkgdir:
  732. assert __path__ == [pkgdir], (__path__, pkgdir)
  733. else:
  734. assert __path__ == ([] if ispkg else None), __path__
  735. else:
  736. __file__ = None
  737. assert state.filename is None, state.filename
  738. assert __path__ == ([] if ispkg else None), __path__
  739. # Check the file attrs.
  740. if __file__:
  741. assert hasattr(module, '__file__')
  742. assert module.__file__ == __file__, (module.__file__, __file__)
  743. else:
  744. assert not hasattr(module, '__file__'), module.__file__
  745. if ispkg:
  746. assert hasattr(module, '__path__')
  747. assert module.__path__ == __path__, (module.__path__, __path__)
  748. else:
  749. assert not hasattr(module, '__path__'), module.__path__
  750. assert not spec.has_location
  751. @classmethod
  752. def _resolve_filename(cls, fullname, alias=None, ispkg=False):
  753. if not fullname or not getattr(sys, '_stdlib_dir', None):
  754. return None, None
  755. try:
  756. sep = cls._SEP
  757. except AttributeError:
  758. sep = cls._SEP = '\\' if sys.platform == 'win32' else '/'
  759. if fullname != alias:
  760. if fullname.startswith('<'):
  761. fullname = fullname[1:]
  762. if not ispkg:
  763. fullname = f'{fullname}.__init__'
  764. else:
  765. ispkg = False
  766. relfile = fullname.replace('.', sep)
  767. if ispkg:
  768. pkgdir = f'{sys._stdlib_dir}{sep}{relfile}'
  769. filename = f'{pkgdir}{sep}__init__.py'
  770. else:
  771. pkgdir = None
  772. filename = f'{sys._stdlib_dir}{sep}{relfile}.py'
  773. return filename, pkgdir
  774. @classmethod
  775. def find_spec(cls, fullname, path=None, target=None):
  776. info = _call_with_frames_removed(_imp.find_frozen, fullname)
  777. if info is None:
  778. return None
  779. # We get the marshaled data in exec_module() (the loader
  780. # part of the importer), instead of here (the finder part).
  781. # The loader is the usual place to get the data that will
  782. # be loaded into the module. (For example, see _LoaderBasics
  783. # in _bootstra_external.py.) Most importantly, this importer
  784. # is simpler if we wait to get the data.
  785. # However, getting as much data in the finder as possible
  786. # to later load the module is okay, and sometimes important.
  787. # (That's why ModuleSpec.loader_state exists.) This is
  788. # especially true if it avoids throwing away expensive data
  789. # the loader would otherwise duplicate later and can be done
  790. # efficiently. In this case it isn't worth it.
  791. _, ispkg, origname = info
  792. spec = spec_from_loader(fullname, cls,
  793. origin=cls._ORIGIN,
  794. is_package=ispkg)
  795. filename, pkgdir = cls._resolve_filename(origname, fullname, ispkg)
  796. spec.loader_state = type(sys.implementation)(
  797. filename=filename,
  798. origname=origname,
  799. )
  800. if pkgdir:
  801. spec.submodule_search_locations.insert(0, pkgdir)
  802. return spec
  803. @classmethod
  804. def find_module(cls, fullname, path=None):
  805. """Find a frozen module.
  806. This method is deprecated. Use find_spec() instead.
  807. """
  808. _warnings.warn("FrozenImporter.find_module() is deprecated and "
  809. "slated for removal in Python 3.12; use find_spec() instead",
  810. DeprecationWarning)
  811. return cls if _imp.is_frozen(fullname) else None
  812. @staticmethod
  813. def create_module(spec):
  814. """Set __file__, if able."""
  815. module = _new_module(spec.name)
  816. try:
  817. filename = spec.loader_state.filename
  818. except AttributeError:
  819. pass
  820. else:
  821. if filename:
  822. module.__file__ = filename
  823. return module
  824. @staticmethod
  825. def exec_module(module):
  826. spec = module.__spec__
  827. name = spec.name
  828. code = _call_with_frames_removed(_imp.get_frozen_object, name)
  829. exec(code, module.__dict__)
  830. @classmethod
  831. def load_module(cls, fullname):
  832. """Load a frozen module.
  833. This method is deprecated. Use exec_module() instead.
  834. """
  835. # Warning about deprecation implemented in _load_module_shim().
  836. module = _load_module_shim(cls, fullname)
  837. info = _imp.find_frozen(fullname)
  838. assert info is not None
  839. _, ispkg, origname = info
  840. module.__origname__ = origname
  841. vars(module).pop('__file__', None)
  842. if ispkg:
  843. module.__path__ = []
  844. cls._fix_up_module(module)
  845. return module
  846. @classmethod
  847. @_requires_frozen
  848. def get_code(cls, fullname):
  849. """Return the code object for the frozen module."""
  850. return _imp.get_frozen_object(fullname)
  851. @classmethod
  852. @_requires_frozen
  853. def get_source(cls, fullname):
  854. """Return None as frozen modules do not have source code."""
  855. return None
  856. @classmethod
  857. @_requires_frozen
  858. def is_package(cls, fullname):
  859. """Return True if the frozen module is a package."""
  860. return _imp.is_frozen_package(fullname)
  861. # Import itself ###############################################################
  862. class _ImportLockContext:
  863. """Context manager for the import lock."""
  864. def __enter__(self):
  865. """Acquire the import lock."""
  866. _imp.acquire_lock()
  867. def __exit__(self, exc_type, exc_value, exc_traceback):
  868. """Release the import lock regardless of any raised exceptions."""
  869. _imp.release_lock()
  870. def _resolve_name(name, package, level):
  871. """Resolve a relative module name to an absolute one."""
  872. bits = package.rsplit('.', level - 1)
  873. if len(bits) < level:
  874. raise ImportError('attempted relative import beyond top-level package')
  875. base = bits[0]
  876. return '{}.{}'.format(base, name) if name else base
  877. def _find_spec_legacy(finder, name, path):
  878. msg = (f"{_object_name(finder)}.find_spec() not found; "
  879. "falling back to find_module()")
  880. _warnings.warn(msg, ImportWarning)
  881. loader = finder.find_module(name, path)
  882. if loader is None:
  883. return None
  884. return spec_from_loader(name, loader)
  885. def _find_spec(name, path, target=None):
  886. """Find a module's spec."""
  887. meta_path = sys.meta_path
  888. if meta_path is None:
  889. # PyImport_Cleanup() is running or has been called.
  890. raise ImportError("sys.meta_path is None, Python is likely "
  891. "shutting down")
  892. if not meta_path:
  893. _warnings.warn('sys.meta_path is empty', ImportWarning)
  894. # We check sys.modules here for the reload case. While a passed-in
  895. # target will usually indicate a reload there is no guarantee, whereas
  896. # sys.modules provides one.
  897. is_reload = name in sys.modules
  898. for finder in meta_path:
  899. with _ImportLockContext():
  900. try:
  901. find_spec = finder.find_spec
  902. except AttributeError:
  903. spec = _find_spec_legacy(finder, name, path)
  904. if spec is None:
  905. continue
  906. else:
  907. spec = find_spec(name, path, target)
  908. if spec is not None:
  909. # The parent import may have already imported this module.
  910. if not is_reload and name in sys.modules:
  911. module = sys.modules[name]
  912. try:
  913. __spec__ = module.__spec__
  914. except AttributeError:
  915. # We use the found spec since that is the one that
  916. # we would have used if the parent module hadn't
  917. # beaten us to the punch.
  918. return spec
  919. else:
  920. if __spec__ is None:
  921. return spec
  922. else:
  923. return __spec__
  924. else:
  925. return spec
  926. else:
  927. return None
  928. def _sanity_check(name, package, level):
  929. """Verify arguments are "sane"."""
  930. if not isinstance(name, str):
  931. raise TypeError('module name must be str, not {}'.format(type(name)))
  932. if level < 0:
  933. raise ValueError('level must be >= 0')
  934. if level > 0:
  935. if not isinstance(package, str):
  936. raise TypeError('__package__ not set to a string')
  937. elif not package:
  938. raise ImportError('attempted relative import with no known parent '
  939. 'package')
  940. if not name and level == 0:
  941. raise ValueError('Empty module name')
  942. _ERR_MSG_PREFIX = 'No module named '
  943. _ERR_MSG = _ERR_MSG_PREFIX + '{!r}'
  944. def _find_and_load_unlocked(name, import_):
  945. path = None
  946. parent = name.rpartition('.')[0]
  947. parent_spec = None
  948. if parent:
  949. if parent not in sys.modules:
  950. _call_with_frames_removed(import_, parent)
  951. # Crazy side-effects!
  952. if name in sys.modules:
  953. return sys.modules[name]
  954. parent_module = sys.modules[parent]
  955. try:
  956. path = parent_module.__path__
  957. except AttributeError:
  958. msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent)
  959. raise ModuleNotFoundError(msg, name=name) from None
  960. parent_spec = parent_module.__spec__
  961. child = name.rpartition('.')[2]
  962. spec = _find_spec(name, path)
  963. if spec is None:
  964. raise ModuleNotFoundError(_ERR_MSG.format(name), name=name)
  965. else:
  966. if parent_spec:
  967. # Temporarily add child we are currently importing to parent's
  968. # _uninitialized_submodules for circular import tracking.
  969. parent_spec._uninitialized_submodules.append(child)
  970. try:
  971. module = _load_unlocked(spec)
  972. finally:
  973. if parent_spec:
  974. parent_spec._uninitialized_submodules.pop()
  975. if parent:
  976. # Set the module as an attribute on its parent.
  977. parent_module = sys.modules[parent]
  978. try:
  979. setattr(parent_module, child, module)
  980. except AttributeError:
  981. msg = f"Cannot set an attribute on {parent!r} for child module {child!r}"
  982. _warnings.warn(msg, ImportWarning)
  983. return module
  984. _NEEDS_LOADING = object()
  985. def _find_and_load(name, import_):
  986. """Find and load the module."""
  987. # Optimization: we avoid unneeded module locking if the module
  988. # already exists in sys.modules and is fully initialized.
  989. module = sys.modules.get(name, _NEEDS_LOADING)
  990. if (module is _NEEDS_LOADING or
  991. getattr(getattr(module, "__spec__", None), "_initializing", False)):
  992. with _ModuleLockManager(name):
  993. module = sys.modules.get(name, _NEEDS_LOADING)
  994. if module is _NEEDS_LOADING:
  995. return _find_and_load_unlocked(name, import_)
  996. # Optimization: only call _bootstrap._lock_unlock_module() if
  997. # module.__spec__._initializing is True.
  998. # NOTE: because of this, initializing must be set *before*
  999. # putting the new module in sys.modules.
  1000. _lock_unlock_module(name)
  1001. if module is None:
  1002. message = ('import of {} halted; '
  1003. 'None in sys.modules'.format(name))
  1004. raise ModuleNotFoundError(message, name=name)
  1005. return module
  1006. def _gcd_import(name, package=None, level=0):
  1007. """Import and return the module based on its name, the package the call is
  1008. being made from, and the level adjustment.
  1009. This function represents the greatest common denominator of functionality
  1010. between import_module and __import__. This includes setting __package__ if
  1011. the loader did not.
  1012. """
  1013. _sanity_check(name, package, level)
  1014. if level > 0:
  1015. name = _resolve_name(name, package, level)
  1016. return _find_and_load(name, _gcd_import)
  1017. def _handle_fromlist(module, fromlist, import_, *, recursive=False):
  1018. """Figure out what __import__ should return.
  1019. The import_ parameter is a callable which takes the name of module to
  1020. import. It is required to decouple the function from assuming importlib's
  1021. import implementation is desired.
  1022. """
  1023. # The hell that is fromlist ...
  1024. # If a package was imported, try to import stuff from fromlist.
  1025. for x in fromlist:
  1026. if not isinstance(x, str):
  1027. if recursive:
  1028. where = module.__name__ + '.__all__'
  1029. else:
  1030. where = "``from list''"
  1031. raise TypeError(f"Item in {where} must be str, "
  1032. f"not {type(x).__name__}")
  1033. elif x == '*':
  1034. if not recursive and hasattr(module, '__all__'):
  1035. _handle_fromlist(module, module.__all__, import_,
  1036. recursive=True)
  1037. elif not hasattr(module, x):
  1038. from_name = '{}.{}'.format(module.__name__, x)
  1039. try:
  1040. _call_with_frames_removed(import_, from_name)
  1041. except ModuleNotFoundError as exc:
  1042. # Backwards-compatibility dictates we ignore failed
  1043. # imports triggered by fromlist for modules that don't
  1044. # exist.
  1045. if (exc.name == from_name and
  1046. sys.modules.get(from_name, _NEEDS_LOADING) is not None):
  1047. continue
  1048. raise
  1049. return module
  1050. def _calc___package__(globals):
  1051. """Calculate what __package__ should be.
  1052. __package__ is not guaranteed to be defined or could be set to None
  1053. to represent that its proper value is unknown.
  1054. """
  1055. package = globals.get('__package__')
  1056. spec = globals.get('__spec__')
  1057. if package is not None:
  1058. if spec is not None and package != spec.parent:
  1059. _warnings.warn("__package__ != __spec__.parent "
  1060. f"({package!r} != {spec.parent!r})",
  1061. ImportWarning, stacklevel=3)
  1062. return package
  1063. elif spec is not None:
  1064. return spec.parent
  1065. else:
  1066. _warnings.warn("can't resolve package from __spec__ or __package__, "
  1067. "falling back on __name__ and __path__",
  1068. ImportWarning, stacklevel=3)
  1069. package = globals['__name__']
  1070. if '__path__' not in globals:
  1071. package = package.rpartition('.')[0]
  1072. return package
  1073. def __import__(name, globals=None, locals=None, fromlist=(), level=0):
  1074. """Import a module.
  1075. The 'globals' argument is used to infer where the import is occurring from
  1076. to handle relative imports. The 'locals' argument is ignored. The
  1077. 'fromlist' argument specifies what should exist as attributes on the module
  1078. being imported (e.g. ``from module import <fromlist>``). The 'level'
  1079. argument represents the package location to import from in a relative
  1080. import (e.g. ``from ..pkg import mod`` would have a 'level' of 2).
  1081. """
  1082. if level == 0:
  1083. module = _gcd_import(name)
  1084. else:
  1085. globals_ = globals if globals is not None else {}
  1086. package = _calc___package__(globals_)
  1087. module = _gcd_import(name, package, level)
  1088. if not fromlist:
  1089. # Return up to the first dot in 'name'. This is complicated by the fact
  1090. # that 'name' may be relative.
  1091. if level == 0:
  1092. return _gcd_import(name.partition('.')[0])
  1093. elif not name:
  1094. return module
  1095. else:
  1096. # Figure out where to slice the module's name up to the first dot
  1097. # in 'name'.
  1098. cut_off = len(name) - len(name.partition('.')[0])
  1099. # Slice end needs to be positive to alleviate need to special-case
  1100. # when ``'.' not in name``.
  1101. return sys.modules[module.__name__[:len(module.__name__)-cut_off]]
  1102. elif hasattr(module, '__path__'):
  1103. return _handle_fromlist(module, fromlist, _gcd_import)
  1104. else:
  1105. return module
  1106. def _builtin_from_name(name):
  1107. spec = BuiltinImporter.find_spec(name)
  1108. if spec is None:
  1109. raise ImportError('no built-in module named ' + name)
  1110. return _load_unlocked(spec)
  1111. def _setup(sys_module, _imp_module):
  1112. """Setup importlib by importing needed built-in modules and injecting them
  1113. into the global namespace.
  1114. As sys is needed for sys.modules access and _imp is needed to load built-in
  1115. modules, those two modules must be explicitly passed in.
  1116. """
  1117. global _imp, sys
  1118. _imp = _imp_module
  1119. sys = sys_module
  1120. # Set up the spec for existing builtin/frozen modules.
  1121. module_type = type(sys)
  1122. for name, module in sys.modules.items():
  1123. if isinstance(module, module_type):
  1124. if name in sys.builtin_module_names:
  1125. loader = BuiltinImporter
  1126. elif _imp.is_frozen(name):
  1127. loader = FrozenImporter
  1128. else:
  1129. continue
  1130. spec = _spec_from_module(module, loader)
  1131. _init_module_attrs(spec, module)
  1132. if loader is FrozenImporter:
  1133. loader._fix_up_module(module)
  1134. # Directly load built-in modules needed during bootstrap.
  1135. self_module = sys.modules[__name__]
  1136. for builtin_name in ('_thread', '_warnings', '_weakref'):
  1137. if builtin_name not in sys.modules:
  1138. builtin_module = _builtin_from_name(builtin_name)
  1139. else:
  1140. builtin_module = sys.modules[builtin_name]
  1141. setattr(self_module, builtin_name, builtin_module)
  1142. def _install(sys_module, _imp_module):
  1143. """Install importers for builtin and frozen modules"""
  1144. _setup(sys_module, _imp_module)
  1145. sys.meta_path.append(BuiltinImporter)
  1146. sys.meta_path.append(FrozenImporter)
  1147. def _install_external_importers():
  1148. """Install importers that require external filesystem access"""
  1149. global _bootstrap_external
  1150. import _frozen_importlib_external
  1151. _bootstrap_external = _frozen_importlib_external
  1152. _frozen_importlib_external._install(sys.modules[__name__])