Change venv
This commit is contained in:
@@ -8,10 +8,9 @@ import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
from pip._vendor.pkg_resources import Distribution
|
||||
|
||||
from pip._internal.distributions import make_distribution_for_install_requirement
|
||||
from pip._internal.distributions.installed import InstalledDistribution
|
||||
@@ -20,11 +19,14 @@ from pip._internal.exceptions import (
|
||||
HashMismatch,
|
||||
HashUnpinned,
|
||||
InstallationError,
|
||||
MetadataInconsistent,
|
||||
NetworkConnectionError,
|
||||
PreviousBuildDirError,
|
||||
VcsHashUnsupported,
|
||||
)
|
||||
from pip._internal.index.package_finder import PackageFinder
|
||||
from pip._internal.metadata import BaseDistribution, get_metadata_distribution
|
||||
from pip._internal.models.direct_url import ArchiveInfo
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.models.wheel import Wheel
|
||||
from pip._internal.network.download import BatchDownloader, Downloader
|
||||
@@ -33,13 +35,20 @@ from pip._internal.network.lazy_wheel import (
|
||||
dist_from_wheel_url,
|
||||
)
|
||||
from pip._internal.network.session import PipSession
|
||||
from pip._internal.operations.build.build_tracker import BuildTracker
|
||||
from pip._internal.req.req_install import InstallRequirement
|
||||
from pip._internal.req.req_tracker import RequirementTracker
|
||||
from pip._internal.utils.deprecation import deprecated
|
||||
from pip._internal.utils.filesystem import copy2_fixed
|
||||
from pip._internal.utils.direct_url_helpers import (
|
||||
direct_url_for_editable,
|
||||
direct_url_from_link,
|
||||
)
|
||||
from pip._internal.utils.hashes import Hashes, MissingHashes
|
||||
from pip._internal.utils.logging import indent_log
|
||||
from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree
|
||||
from pip._internal.utils.misc import (
|
||||
display_path,
|
||||
hash_file,
|
||||
hide_url,
|
||||
is_installable_dir,
|
||||
)
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
from pip._internal.utils.unpacking import unpack_file
|
||||
from pip._internal.vcs import vcs
|
||||
@@ -48,30 +57,29 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_prepared_distribution(
|
||||
req, # type: InstallRequirement
|
||||
req_tracker, # type: RequirementTracker
|
||||
finder, # type: PackageFinder
|
||||
build_isolation, # type: bool
|
||||
):
|
||||
# type: (...) -> Distribution
|
||||
req: InstallRequirement,
|
||||
build_tracker: BuildTracker,
|
||||
finder: PackageFinder,
|
||||
build_isolation: bool,
|
||||
check_build_deps: bool,
|
||||
) -> BaseDistribution:
|
||||
"""Prepare a distribution for installation."""
|
||||
abstract_dist = make_distribution_for_install_requirement(req)
|
||||
with req_tracker.track(req):
|
||||
abstract_dist.prepare_distribution_metadata(finder, build_isolation)
|
||||
return abstract_dist.get_pkg_resources_distribution()
|
||||
with build_tracker.track(req):
|
||||
abstract_dist.prepare_distribution_metadata(
|
||||
finder, build_isolation, check_build_deps
|
||||
)
|
||||
return abstract_dist.get_metadata_distribution()
|
||||
|
||||
|
||||
def unpack_vcs_link(link, location):
|
||||
# type: (Link, str) -> None
|
||||
def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
|
||||
vcs_backend = vcs.get_backend_for_scheme(link.scheme)
|
||||
assert vcs_backend is not None
|
||||
vcs_backend.unpack(location, url=hide_url(link.url))
|
||||
vcs_backend.unpack(location, url=hide_url(link.url), verbosity=verbosity)
|
||||
|
||||
|
||||
class File:
|
||||
|
||||
def __init__(self, path, content_type):
|
||||
# type: (str, Optional[str]) -> None
|
||||
def __init__(self, path: str, content_type: Optional[str]) -> None:
|
||||
self.path = path
|
||||
if content_type is None:
|
||||
self.content_type = mimetypes.guess_type(path)[0]
|
||||
@@ -80,19 +88,16 @@ class File:
|
||||
|
||||
|
||||
def get_http_url(
|
||||
link, # type: Link
|
||||
download, # type: Downloader
|
||||
download_dir=None, # type: Optional[str]
|
||||
hashes=None, # type: Optional[Hashes]
|
||||
):
|
||||
# type: (...) -> File
|
||||
link: Link,
|
||||
download: Downloader,
|
||||
download_dir: Optional[str] = None,
|
||||
hashes: Optional[Hashes] = None,
|
||||
) -> File:
|
||||
temp_dir = TempDirectory(kind="unpack", globally_managed=True)
|
||||
# If a download dir is specified, is the file already downloaded there?
|
||||
already_downloaded_path = None
|
||||
if download_dir:
|
||||
already_downloaded_path = _check_download_dir(
|
||||
link, download_dir, hashes
|
||||
)
|
||||
already_downloaded_path = _check_download_dir(link, download_dir, hashes)
|
||||
|
||||
if already_downloaded_path:
|
||||
from_path = already_downloaded_path
|
||||
@@ -106,72 +111,14 @@ def get_http_url(
|
||||
return File(from_path, content_type)
|
||||
|
||||
|
||||
def _copy2_ignoring_special_files(src, dest):
|
||||
# type: (str, str) -> None
|
||||
"""Copying special files is not supported, but as a convenience to users
|
||||
we skip errors copying them. This supports tools that may create e.g.
|
||||
socket files in the project source directory.
|
||||
"""
|
||||
try:
|
||||
copy2_fixed(src, dest)
|
||||
except shutil.SpecialFileError as e:
|
||||
# SpecialFileError may be raised due to either the source or
|
||||
# destination. If the destination was the cause then we would actually
|
||||
# care, but since the destination directory is deleted prior to
|
||||
# copy we ignore all of them assuming it is caused by the source.
|
||||
logger.warning(
|
||||
"Ignoring special file error '%s' encountered copying %s to %s.",
|
||||
str(e),
|
||||
src,
|
||||
dest,
|
||||
)
|
||||
|
||||
|
||||
def _copy_source_tree(source, target):
|
||||
# type: (str, str) -> None
|
||||
target_abspath = os.path.abspath(target)
|
||||
target_basename = os.path.basename(target_abspath)
|
||||
target_dirname = os.path.dirname(target_abspath)
|
||||
|
||||
def ignore(d, names):
|
||||
# type: (str, List[str]) -> List[str]
|
||||
skipped = [] # type: List[str]
|
||||
if d == source:
|
||||
# Pulling in those directories can potentially be very slow,
|
||||
# exclude the following directories if they appear in the top
|
||||
# level dir (and only it).
|
||||
# See discussion at https://github.com/pypa/pip/pull/6770
|
||||
skipped += ['.tox', '.nox']
|
||||
if os.path.abspath(d) == target_dirname:
|
||||
# Prevent an infinite recursion if the target is in source.
|
||||
# This can happen when TMPDIR is set to ${PWD}/...
|
||||
# and we copy PWD to TMPDIR.
|
||||
skipped += [target_basename]
|
||||
return skipped
|
||||
|
||||
shutil.copytree(
|
||||
source,
|
||||
target,
|
||||
ignore=ignore,
|
||||
symlinks=True,
|
||||
copy_function=_copy2_ignoring_special_files,
|
||||
)
|
||||
|
||||
|
||||
def get_file_url(
|
||||
link, # type: Link
|
||||
download_dir=None, # type: Optional[str]
|
||||
hashes=None # type: Optional[Hashes]
|
||||
):
|
||||
# type: (...) -> File
|
||||
"""Get file and optionally check its hash.
|
||||
"""
|
||||
link: Link, download_dir: Optional[str] = None, hashes: Optional[Hashes] = None
|
||||
) -> File:
|
||||
"""Get file and optionally check its hash."""
|
||||
# If a download dir is specified, is the file already there and valid?
|
||||
already_downloaded_path = None
|
||||
if download_dir:
|
||||
already_downloaded_path = _check_download_dir(
|
||||
link, download_dir, hashes
|
||||
)
|
||||
already_downloaded_path = _check_download_dir(link, download_dir, hashes)
|
||||
|
||||
if already_downloaded_path:
|
||||
from_path = already_downloaded_path
|
||||
@@ -189,13 +136,13 @@ def get_file_url(
|
||||
|
||||
|
||||
def unpack_url(
|
||||
link, # type: Link
|
||||
location, # type: str
|
||||
download, # type: Downloader
|
||||
download_dir=None, # type: Optional[str]
|
||||
hashes=None, # type: Optional[Hashes]
|
||||
):
|
||||
# type: (...) -> Optional[File]
|
||||
link: Link,
|
||||
location: str,
|
||||
download: Downloader,
|
||||
verbosity: int,
|
||||
download_dir: Optional[str] = None,
|
||||
hashes: Optional[Hashes] = None,
|
||||
) -> Optional[File]:
|
||||
"""Unpack link into location, downloading if required.
|
||||
|
||||
:param hashes: A Hashes object, one of whose embedded hashes must match,
|
||||
@@ -205,30 +152,10 @@ def unpack_url(
|
||||
"""
|
||||
# non-editable vcs urls
|
||||
if link.is_vcs:
|
||||
unpack_vcs_link(link, location)
|
||||
unpack_vcs_link(link, location, verbosity=verbosity)
|
||||
return None
|
||||
|
||||
# Once out-of-tree-builds are no longer supported, could potentially
|
||||
# replace the below condition with `assert not link.is_existing_dir`
|
||||
# - unpack_url does not need to be called for in-tree-builds.
|
||||
#
|
||||
# As further cleanup, _copy_source_tree and accompanying tests can
|
||||
# be removed.
|
||||
if link.is_existing_dir():
|
||||
deprecated(
|
||||
"A future pip version will change local packages to be built "
|
||||
"in-place without first copying to a temporary directory. "
|
||||
"We recommend you use --use-feature=in-tree-build to test "
|
||||
"your packages with this new behavior before it becomes the "
|
||||
"default.\n",
|
||||
replacement=None,
|
||||
gone_in="21.3",
|
||||
issue=7555
|
||||
)
|
||||
if os.path.isdir(location):
|
||||
rmtree(location)
|
||||
_copy_source_tree(link.file_path, location)
|
||||
return None
|
||||
assert not link.is_existing_dir()
|
||||
|
||||
# file urls
|
||||
if link.is_file:
|
||||
@@ -251,10 +178,14 @@ def unpack_url(
|
||||
return file
|
||||
|
||||
|
||||
def _check_download_dir(link, download_dir, hashes):
|
||||
# type: (Link, str, Optional[Hashes]) -> Optional[str]
|
||||
""" Check download_dir for previously downloaded file with correct hash
|
||||
If a correct file is found return its path else None
|
||||
def _check_download_dir(
|
||||
link: Link,
|
||||
download_dir: str,
|
||||
hashes: Optional[Hashes],
|
||||
warn_on_hash_mismatch: bool = True,
|
||||
) -> Optional[str]:
|
||||
"""Check download_dir for previously downloaded file with correct hash
|
||||
If a correct file is found return its path else None
|
||||
"""
|
||||
download_path = os.path.join(download_dir, link.filename)
|
||||
|
||||
@@ -262,46 +193,45 @@ def _check_download_dir(link, download_dir, hashes):
|
||||
return None
|
||||
|
||||
# If already downloaded, does its hash match?
|
||||
logger.info('File was already downloaded %s', download_path)
|
||||
logger.info("File was already downloaded %s", download_path)
|
||||
if hashes:
|
||||
try:
|
||||
hashes.check_against_path(download_path)
|
||||
except HashMismatch:
|
||||
logger.warning(
|
||||
'Previously-downloaded file %s has bad hash. '
|
||||
'Re-downloading.',
|
||||
download_path
|
||||
)
|
||||
if warn_on_hash_mismatch:
|
||||
logger.warning(
|
||||
"Previously-downloaded file %s has bad hash. Re-downloading.",
|
||||
download_path,
|
||||
)
|
||||
os.unlink(download_path)
|
||||
return None
|
||||
return download_path
|
||||
|
||||
|
||||
class RequirementPreparer:
|
||||
"""Prepares a Requirement
|
||||
"""
|
||||
"""Prepares a Requirement"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
build_dir, # type: str
|
||||
download_dir, # type: Optional[str]
|
||||
src_dir, # type: str
|
||||
build_isolation, # type: bool
|
||||
req_tracker, # type: RequirementTracker
|
||||
session, # type: PipSession
|
||||
progress_bar, # type: str
|
||||
finder, # type: PackageFinder
|
||||
require_hashes, # type: bool
|
||||
use_user_site, # type: bool
|
||||
lazy_wheel, # type: bool
|
||||
in_tree_build, # type: bool
|
||||
):
|
||||
# type: (...) -> None
|
||||
build_dir: str,
|
||||
download_dir: Optional[str],
|
||||
src_dir: str,
|
||||
build_isolation: bool,
|
||||
check_build_deps: bool,
|
||||
build_tracker: BuildTracker,
|
||||
session: PipSession,
|
||||
progress_bar: str,
|
||||
finder: PackageFinder,
|
||||
require_hashes: bool,
|
||||
use_user_site: bool,
|
||||
lazy_wheel: bool,
|
||||
verbosity: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.src_dir = src_dir
|
||||
self.build_dir = build_dir
|
||||
self.req_tracker = req_tracker
|
||||
self.build_tracker = build_tracker
|
||||
self._session = session
|
||||
self._download = Downloader(session, progress_bar)
|
||||
self._batch_download = BatchDownloader(session, progress_bar)
|
||||
@@ -314,6 +244,9 @@ class RequirementPreparer:
|
||||
# Is build isolation allowed?
|
||||
self.build_isolation = build_isolation
|
||||
|
||||
# Should check build dependencies?
|
||||
self.check_build_deps = check_build_deps
|
||||
|
||||
# Should hash-checking be required?
|
||||
self.require_hashes = require_hashes
|
||||
|
||||
@@ -323,35 +256,45 @@ class RequirementPreparer:
|
||||
# Should wheels be downloaded lazily?
|
||||
self.use_lazy_wheel = lazy_wheel
|
||||
|
||||
# Should in-tree builds be used for local paths?
|
||||
self.in_tree_build = in_tree_build
|
||||
# How verbose should underlying tooling be?
|
||||
self.verbosity = verbosity
|
||||
|
||||
# Memoized downloaded files, as mapping of url: (path, mime type)
|
||||
self._downloaded = {} # type: Dict[str, Tuple[str, str]]
|
||||
# Memoized downloaded files, as mapping of url: path.
|
||||
self._downloaded: Dict[str, str] = {}
|
||||
|
||||
# Previous "header" printed for a link-based InstallRequirement
|
||||
self._previous_requirement_header = ("", "")
|
||||
|
||||
def _log_preparing_link(self, req):
|
||||
# type: (InstallRequirement) -> None
|
||||
def _log_preparing_link(self, req: InstallRequirement) -> None:
|
||||
"""Provide context for the requirement being prepared."""
|
||||
if req.link.is_file and not req.original_link_is_in_wheel_cache:
|
||||
if req.link.is_file and not req.is_wheel_from_cache:
|
||||
message = "Processing %s"
|
||||
information = str(display_path(req.link.file_path))
|
||||
else:
|
||||
message = "Collecting %s"
|
||||
information = str(req.req or req)
|
||||
|
||||
# If we used req.req, inject requirement source if available (this
|
||||
# would already be included if we used req directly)
|
||||
if req.req and req.comes_from:
|
||||
if isinstance(req.comes_from, str):
|
||||
comes_from: Optional[str] = req.comes_from
|
||||
else:
|
||||
comes_from = req.comes_from.from_path()
|
||||
if comes_from:
|
||||
information += f" (from {comes_from})"
|
||||
|
||||
if (message, information) != self._previous_requirement_header:
|
||||
self._previous_requirement_header = (message, information)
|
||||
logger.info(message, information)
|
||||
|
||||
if req.original_link_is_in_wheel_cache:
|
||||
if req.is_wheel_from_cache:
|
||||
with indent_log():
|
||||
logger.info("Using cached %s", req.link.filename)
|
||||
|
||||
def _ensure_link_req_src_dir(self, req, parallel_builds):
|
||||
# type: (InstallRequirement, bool) -> None
|
||||
def _ensure_link_req_src_dir(
|
||||
self, req: InstallRequirement, parallel_builds: bool
|
||||
) -> None:
|
||||
"""Ensure source_dir of a linked InstallRequirement."""
|
||||
# Since source_dir is only set for editable requirements.
|
||||
if req.link.is_wheel:
|
||||
@@ -359,7 +302,7 @@ class RequirementPreparer:
|
||||
# directory.
|
||||
return
|
||||
assert req.source_dir is None
|
||||
if req.link.is_existing_dir() and self.in_tree_build:
|
||||
if req.link.is_existing_dir():
|
||||
# build local directories in-tree
|
||||
req.source_dir = req.link.file_path
|
||||
return
|
||||
@@ -376,6 +319,7 @@ class RequirementPreparer:
|
||||
# installation.
|
||||
# FIXME: this won't upgrade when there's an existing
|
||||
# package unpacked in `req.source_dir`
|
||||
# TODO: this check is now probably dead code
|
||||
if is_installable_dir(req.source_dir):
|
||||
raise PreviousBuildDirError(
|
||||
"pip can't proceed with requirements '{}' due to a"
|
||||
@@ -385,8 +329,7 @@ class RequirementPreparer:
|
||||
"Please delete it and try again.".format(req, req.source_dir)
|
||||
)
|
||||
|
||||
def _get_linked_req_hashes(self, req):
|
||||
# type: (InstallRequirement) -> Hashes
|
||||
def _get_linked_req_hashes(self, req: InstallRequirement) -> Hashes:
|
||||
# By the time this is called, the requirement's link should have
|
||||
# been checked so we can tell what kind of requirements req is
|
||||
# and raise some more informative errors than otherwise.
|
||||
@@ -418,18 +361,72 @@ class RequirementPreparer:
|
||||
# showing the user what the hash should be.
|
||||
return req.hashes(trust_internet=False) or MissingHashes()
|
||||
|
||||
def _fetch_metadata_using_lazy_wheel(self, link):
|
||||
# type: (Link) -> Optional[Distribution]
|
||||
"""Fetch metadata using lazy wheel, if possible."""
|
||||
if not self.use_lazy_wheel:
|
||||
return None
|
||||
def _fetch_metadata_only(
|
||||
self,
|
||||
req: InstallRequirement,
|
||||
) -> Optional[BaseDistribution]:
|
||||
if self.require_hashes:
|
||||
logger.debug('Lazy wheel is not used as hash checking is required')
|
||||
logger.debug(
|
||||
"Metadata-only fetching is not used as hash checking is required",
|
||||
)
|
||||
return None
|
||||
# Try PEP 658 metadata first, then fall back to lazy wheel if unavailable.
|
||||
return self._fetch_metadata_using_link_data_attr(
|
||||
req
|
||||
) or self._fetch_metadata_using_lazy_wheel(req.link)
|
||||
|
||||
def _fetch_metadata_using_link_data_attr(
|
||||
self,
|
||||
req: InstallRequirement,
|
||||
) -> Optional[BaseDistribution]:
|
||||
"""Fetch metadata from the data-dist-info-metadata attribute, if possible."""
|
||||
# (1) Get the link to the metadata file, if provided by the backend.
|
||||
metadata_link = req.link.metadata_link()
|
||||
if metadata_link is None:
|
||||
return None
|
||||
assert req.req is not None
|
||||
logger.info(
|
||||
"Obtaining dependency information for %s from %s",
|
||||
req.req,
|
||||
metadata_link,
|
||||
)
|
||||
# (2) Download the contents of the METADATA file, separate from the dist itself.
|
||||
metadata_file = get_http_url(
|
||||
metadata_link,
|
||||
self._download,
|
||||
hashes=metadata_link.as_hashes(),
|
||||
)
|
||||
with open(metadata_file.path, "rb") as f:
|
||||
metadata_contents = f.read()
|
||||
# (3) Generate a dist just from those file contents.
|
||||
metadata_dist = get_metadata_distribution(
|
||||
metadata_contents,
|
||||
req.link.filename,
|
||||
req.req.name,
|
||||
)
|
||||
# (4) Ensure the Name: field from the METADATA file matches the name from the
|
||||
# install requirement.
|
||||
#
|
||||
# NB: raw_name will fall back to the name from the install requirement if
|
||||
# the Name: field is not present, but it's noted in the raw_name docstring
|
||||
# that that should NEVER happen anyway.
|
||||
if metadata_dist.raw_name != req.req.name:
|
||||
raise MetadataInconsistent(
|
||||
req, "Name", req.req.name, metadata_dist.raw_name
|
||||
)
|
||||
return metadata_dist
|
||||
|
||||
def _fetch_metadata_using_lazy_wheel(
|
||||
self,
|
||||
link: Link,
|
||||
) -> Optional[BaseDistribution]:
|
||||
"""Fetch metadata using lazy wheel, if possible."""
|
||||
# --use-feature=fast-deps must be provided.
|
||||
if not self.use_lazy_wheel:
|
||||
return None
|
||||
if link.is_file or not link.is_wheel:
|
||||
logger.debug(
|
||||
'Lazy wheel is not used as '
|
||||
'%r does not points to a remote wheel',
|
||||
"Lazy wheel is not used as %r does not point to a remote wheel",
|
||||
link,
|
||||
)
|
||||
return None
|
||||
@@ -437,22 +434,22 @@ class RequirementPreparer:
|
||||
wheel = Wheel(link.filename)
|
||||
name = canonicalize_name(wheel.name)
|
||||
logger.info(
|
||||
'Obtaining dependency information from %s %s',
|
||||
name, wheel.version,
|
||||
"Obtaining dependency information from %s %s",
|
||||
name,
|
||||
wheel.version,
|
||||
)
|
||||
url = link.url.split('#', 1)[0]
|
||||
url = link.url.split("#", 1)[0]
|
||||
try:
|
||||
return dist_from_wheel_url(name, url, self._session)
|
||||
except HTTPRangeRequestUnsupported:
|
||||
logger.debug('%s does not support range requests', url)
|
||||
logger.debug("%s does not support range requests", url)
|
||||
return None
|
||||
|
||||
def _complete_partial_requirements(
|
||||
self,
|
||||
partially_downloaded_reqs, # type: Iterable[InstallRequirement]
|
||||
parallel_builds=False, # type: bool
|
||||
):
|
||||
# type: (...) -> None
|
||||
partially_downloaded_reqs: Iterable[InstallRequirement],
|
||||
parallel_builds: bool = False,
|
||||
) -> None:
|
||||
"""Download any requirements which were only fetched by metadata."""
|
||||
# Download to a temporary directory. These will be copied over as
|
||||
# needed for downstream 'download', 'wheel', and 'install' commands.
|
||||
@@ -461,7 +458,7 @@ class RequirementPreparer:
|
||||
# Map each link to the requirement that owns it. This allows us to set
|
||||
# `req.local_file_path` on the appropriate requirement after passing
|
||||
# all the links at once into BatchDownloader.
|
||||
links_to_fully_download = {} # type: Dict[Link, InstallRequirement]
|
||||
links_to_fully_download: Dict[Link, InstallRequirement] = {}
|
||||
for req in partially_downloaded_reqs:
|
||||
assert req.link
|
||||
links_to_fully_download[req.link] = req
|
||||
@@ -480,35 +477,47 @@ class RequirementPreparer:
|
||||
for req in partially_downloaded_reqs:
|
||||
self._prepare_linked_requirement(req, parallel_builds)
|
||||
|
||||
def prepare_linked_requirement(self, req, parallel_builds=False):
|
||||
# type: (InstallRequirement, bool) -> Distribution
|
||||
def prepare_linked_requirement(
|
||||
self, req: InstallRequirement, parallel_builds: bool = False
|
||||
) -> BaseDistribution:
|
||||
"""Prepare a requirement to be obtained from req.link."""
|
||||
assert req.link
|
||||
link = req.link
|
||||
self._log_preparing_link(req)
|
||||
with indent_log():
|
||||
# Check if the relevant file is already available
|
||||
# in the download directory
|
||||
file_path = None
|
||||
if self.download_dir is not None and link.is_wheel:
|
||||
if self.download_dir is not None and req.link.is_wheel:
|
||||
hashes = self._get_linked_req_hashes(req)
|
||||
file_path = _check_download_dir(req.link, self.download_dir, hashes)
|
||||
file_path = _check_download_dir(
|
||||
req.link,
|
||||
self.download_dir,
|
||||
hashes,
|
||||
# When a locally built wheel has been found in cache, we don't warn
|
||||
# about re-downloading when the already downloaded wheel hash does
|
||||
# not match. This is because the hash must be checked against the
|
||||
# original link, not the cached link. It that case the already
|
||||
# downloaded file will be removed and re-fetched from cache (which
|
||||
# implies a hash check against the cache entry's origin.json).
|
||||
warn_on_hash_mismatch=not req.is_wheel_from_cache,
|
||||
)
|
||||
|
||||
if file_path is not None:
|
||||
# The file is already available, so mark it as downloaded
|
||||
self._downloaded[req.link.url] = file_path, None
|
||||
self._downloaded[req.link.url] = file_path
|
||||
else:
|
||||
# The file is not available, attempt to fetch only metadata
|
||||
wheel_dist = self._fetch_metadata_using_lazy_wheel(link)
|
||||
if wheel_dist is not None:
|
||||
metadata_dist = self._fetch_metadata_only(req)
|
||||
if metadata_dist is not None:
|
||||
req.needs_more_preparation = True
|
||||
return wheel_dist
|
||||
return metadata_dist
|
||||
|
||||
# None of the optimizations worked, fully prepare the requirement
|
||||
return self._prepare_linked_requirement(req, parallel_builds)
|
||||
|
||||
def prepare_linked_requirements_more(self, reqs, parallel_builds=False):
|
||||
# type: (Iterable[InstallRequirement], bool) -> None
|
||||
def prepare_linked_requirements_more(
|
||||
self, reqs: Iterable[InstallRequirement], parallel_builds: bool = False
|
||||
) -> None:
|
||||
"""Prepare linked requirements more, if needed."""
|
||||
reqs = [req for req in reqs if req.needs_more_preparation]
|
||||
for req in reqs:
|
||||
@@ -517,12 +526,12 @@ class RequirementPreparer:
|
||||
hashes = self._get_linked_req_hashes(req)
|
||||
file_path = _check_download_dir(req.link, self.download_dir, hashes)
|
||||
if file_path is not None:
|
||||
self._downloaded[req.link.url] = file_path, None
|
||||
self._downloaded[req.link.url] = file_path
|
||||
req.needs_more_preparation = False
|
||||
|
||||
# Prepare requirements we found were already downloaded for some
|
||||
# reason. The other downloads will be completed separately.
|
||||
partially_downloaded_reqs = [] # type: List[InstallRequirement]
|
||||
partially_downloaded_reqs: List[InstallRequirement] = []
|
||||
for req in reqs:
|
||||
if req.needs_more_preparation:
|
||||
partially_downloaded_reqs.append(req)
|
||||
@@ -532,35 +541,87 @@ class RequirementPreparer:
|
||||
# TODO: separate this part out from RequirementPreparer when the v1
|
||||
# resolver can be removed!
|
||||
self._complete_partial_requirements(
|
||||
partially_downloaded_reqs, parallel_builds=parallel_builds,
|
||||
partially_downloaded_reqs,
|
||||
parallel_builds=parallel_builds,
|
||||
)
|
||||
|
||||
def _prepare_linked_requirement(self, req, parallel_builds):
|
||||
# type: (InstallRequirement, bool) -> Distribution
|
||||
def _prepare_linked_requirement(
|
||||
self, req: InstallRequirement, parallel_builds: bool
|
||||
) -> BaseDistribution:
|
||||
assert req.link
|
||||
link = req.link
|
||||
|
||||
self._ensure_link_req_src_dir(req, parallel_builds)
|
||||
hashes = self._get_linked_req_hashes(req)
|
||||
|
||||
if link.is_existing_dir() and self.in_tree_build:
|
||||
if hashes and req.is_wheel_from_cache:
|
||||
assert req.download_info is not None
|
||||
assert link.is_wheel
|
||||
assert link.is_file
|
||||
# We need to verify hashes, and we have found the requirement in the cache
|
||||
# of locally built wheels.
|
||||
if (
|
||||
isinstance(req.download_info.info, ArchiveInfo)
|
||||
and req.download_info.info.hashes
|
||||
and hashes.has_one_of(req.download_info.info.hashes)
|
||||
):
|
||||
# At this point we know the requirement was built from a hashable source
|
||||
# artifact, and we verified that the cache entry's hash of the original
|
||||
# artifact matches one of the hashes we expect. We don't verify hashes
|
||||
# against the cached wheel, because the wheel is not the original.
|
||||
hashes = None
|
||||
else:
|
||||
logger.warning(
|
||||
"The hashes of the source archive found in cache entry "
|
||||
"don't match, ignoring cached built wheel "
|
||||
"and re-downloading source."
|
||||
)
|
||||
req.link = req.cached_wheel_source_link
|
||||
link = req.link
|
||||
|
||||
self._ensure_link_req_src_dir(req, parallel_builds)
|
||||
|
||||
if link.is_existing_dir():
|
||||
local_file = None
|
||||
elif link.url not in self._downloaded:
|
||||
try:
|
||||
local_file = unpack_url(
|
||||
link, req.source_dir, self._download,
|
||||
self.download_dir, hashes
|
||||
link,
|
||||
req.source_dir,
|
||||
self._download,
|
||||
self.verbosity,
|
||||
self.download_dir,
|
||||
hashes,
|
||||
)
|
||||
except NetworkConnectionError as exc:
|
||||
raise InstallationError(
|
||||
'Could not install requirement {} because of HTTP '
|
||||
'error {} for URL {}'.format(req, exc, link)
|
||||
"Could not install requirement {} because of HTTP "
|
||||
"error {} for URL {}".format(req, exc, link)
|
||||
)
|
||||
else:
|
||||
file_path, content_type = self._downloaded[link.url]
|
||||
file_path = self._downloaded[link.url]
|
||||
if hashes:
|
||||
hashes.check_against_path(file_path)
|
||||
local_file = File(file_path, content_type)
|
||||
local_file = File(file_path, content_type=None)
|
||||
|
||||
# If download_info is set, we got it from the wheel cache.
|
||||
if req.download_info is None:
|
||||
# Editables don't go through this function (see
|
||||
# prepare_editable_requirement).
|
||||
assert not req.editable
|
||||
req.download_info = direct_url_from_link(link, req.source_dir)
|
||||
# Make sure we have a hash in download_info. If we got it as part of the
|
||||
# URL, it will have been verified and we can rely on it. Otherwise we
|
||||
# compute it from the downloaded file.
|
||||
# FIXME: https://github.com/pypa/pip/issues/11943
|
||||
if (
|
||||
isinstance(req.download_info.info, ArchiveInfo)
|
||||
and not req.download_info.info.hashes
|
||||
and local_file
|
||||
):
|
||||
hash = hash_file(local_file.path)[0].hexdigest()
|
||||
# We populate info.hash for backward compatibility.
|
||||
# This will automatically populate info.hashes.
|
||||
req.download_info.info.hash = f"sha256={hash}"
|
||||
|
||||
# For use in later processing,
|
||||
# preserve the file path on the requirement.
|
||||
@@ -568,12 +629,15 @@ class RequirementPreparer:
|
||||
req.local_file_path = local_file.path
|
||||
|
||||
dist = _get_prepared_distribution(
|
||||
req, self.req_tracker, self.finder, self.build_isolation,
|
||||
req,
|
||||
self.build_tracker,
|
||||
self.finder,
|
||||
self.build_isolation,
|
||||
self.check_build_deps,
|
||||
)
|
||||
return dist
|
||||
|
||||
def save_linked_requirement(self, req):
|
||||
# type: (InstallRequirement) -> None
|
||||
def save_linked_requirement(self, req: InstallRequirement) -> None:
|
||||
assert self.download_dir is not None
|
||||
assert req.link is not None
|
||||
link = req.link
|
||||
@@ -584,8 +648,9 @@ class RequirementPreparer:
|
||||
|
||||
if link.is_existing_dir():
|
||||
logger.debug(
|
||||
'Not copying link to destination directory '
|
||||
'since it is a directory: %s', link,
|
||||
"Not copying link to destination directory "
|
||||
"since it is a directory: %s",
|
||||
link,
|
||||
)
|
||||
return
|
||||
if req.local_file_path is None:
|
||||
@@ -596,31 +661,35 @@ class RequirementPreparer:
|
||||
if not os.path.exists(download_location):
|
||||
shutil.copy(req.local_file_path, download_location)
|
||||
download_path = display_path(download_location)
|
||||
logger.info('Saved %s', download_path)
|
||||
logger.info("Saved %s", download_path)
|
||||
|
||||
def prepare_editable_requirement(
|
||||
self,
|
||||
req, # type: InstallRequirement
|
||||
):
|
||||
# type: (...) -> Distribution
|
||||
"""Prepare an editable requirement
|
||||
"""
|
||||
req: InstallRequirement,
|
||||
) -> BaseDistribution:
|
||||
"""Prepare an editable requirement."""
|
||||
assert req.editable, "cannot prepare a non-editable req as editable"
|
||||
|
||||
logger.info('Obtaining %s', req)
|
||||
logger.info("Obtaining %s", req)
|
||||
|
||||
with indent_log():
|
||||
if self.require_hashes:
|
||||
raise InstallationError(
|
||||
'The editable requirement {} cannot be installed when '
|
||||
'requiring hashes, because there is no single file to '
|
||||
'hash.'.format(req)
|
||||
"The editable requirement {} cannot be installed when "
|
||||
"requiring hashes, because there is no single file to "
|
||||
"hash.".format(req)
|
||||
)
|
||||
req.ensure_has_source_dir(self.src_dir)
|
||||
req.update_editable()
|
||||
assert req.source_dir
|
||||
req.download_info = direct_url_for_editable(req.unpacked_source_directory)
|
||||
|
||||
dist = _get_prepared_distribution(
|
||||
req, self.req_tracker, self.finder, self.build_isolation,
|
||||
req,
|
||||
self.build_tracker,
|
||||
self.finder,
|
||||
self.build_isolation,
|
||||
self.check_build_deps,
|
||||
)
|
||||
|
||||
req.check_if_exists(self.use_user_site)
|
||||
@@ -629,27 +698,24 @@ class RequirementPreparer:
|
||||
|
||||
def prepare_installed_requirement(
|
||||
self,
|
||||
req, # type: InstallRequirement
|
||||
skip_reason # type: str
|
||||
):
|
||||
# type: (...) -> Distribution
|
||||
"""Prepare an already-installed requirement
|
||||
"""
|
||||
req: InstallRequirement,
|
||||
skip_reason: str,
|
||||
) -> BaseDistribution:
|
||||
"""Prepare an already-installed requirement."""
|
||||
assert req.satisfied_by, "req should have been satisfied but isn't"
|
||||
assert skip_reason is not None, (
|
||||
"did not get skip reason skipped but req.satisfied_by "
|
||||
"is set to {}".format(req.satisfied_by)
|
||||
)
|
||||
logger.info(
|
||||
'Requirement %s: %s (%s)',
|
||||
skip_reason, req, req.satisfied_by.version
|
||||
"Requirement %s: %s (%s)", skip_reason, req, req.satisfied_by.version
|
||||
)
|
||||
with indent_log():
|
||||
if self.require_hashes:
|
||||
logger.debug(
|
||||
'Since it is already installed, we are trusting this '
|
||||
'package without checking its hash. To ensure a '
|
||||
'completely repeatable environment, install into an '
|
||||
'empty virtualenv.'
|
||||
"Since it is already installed, we are trusting this "
|
||||
"package without checking its hash. To ensure a "
|
||||
"completely repeatable environment, install into an "
|
||||
"empty virtualenv."
|
||||
)
|
||||
return InstalledDistribution(req).get_pkg_resources_distribution()
|
||||
return InstalledDistribution(req).get_metadata_distribution()
|
||||
|
||||
Reference in New Issue
Block a user