CLERC/Reskreen
6
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects 1 Releases 1 Wiki Activity

Change venv

Browse Source
This commit is contained in:
Ambulance Clerc
2023-05-31 08:31:22 +02:00
parent fb6f579089
commit fdbb52c96f
466 changed files with 25899 additions and 64721 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
venv/Lib/site-packages/pip/_internal/operations/build/metadata.py
View File

@@ -3,33 +3,37 @@
import os
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import (
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory
def generate_metadata(build_env, backend):
# type: (BuildEnvironment, Pep517HookCaller) -> str
def generate_metadata(
build_env: BuildEnvironment, backend: BuildBackendHookCaller, details: str
) -> str:
"""Generate metadata using mechanisms described in PEP 517.
Returns the generated metadata directory.
"""
metadata_tmpdir = TempDirectory(
kind="modern-metadata", globally_managed=True
)
metadata_tmpdir = TempDirectory(kind="modern-metadata", globally_managed=True)
metadata_dir = metadata_tmpdir.path
with build_env:
# Note that Pep517HookCaller implements a fallback for
# Note that BuildBackendHookCaller implements a fallback for
# prepare_metadata_for_build_wheel, so we don't have to
# consider the possibility that this hook doesn't exist.
runner = runner_with_spinner_message("Preparing wheel metadata")
runner = runner_with_spinner_message("Preparing metadata (pyproject.toml)")
with backend.subprocess_runner(runner):
distinfo_dir = backend.prepare_metadata_for_build_wheel(
metadata_dir
)
try:
distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
return os.path.join(metadata_dir, distinfo_dir)

62
venv/Lib/site-packages/pip/_internal/operations/build/metadata_legacy.py
View File

@@ -5,7 +5,12 @@ import logging
import os
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import InstallationError
from pip._internal.cli.spinners import open_spinner
from pip._internal.exceptions import (
InstallationError,
InstallationSubprocessError,
MetadataGenerationFailed,
)
from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args
from pip._internal.utils.subprocess import call_subprocess
from pip._internal.utils.temp_dir import TempDirectory
@@ -13,49 +18,39 @@ from pip._internal.utils.temp_dir import TempDirectory
logger = logging.getLogger(__name__)
def _find_egg_info(directory):
# type: (str) -> str
"""Find an .egg-info subdirectory in `directory`.
"""
filenames = [
f for f in os.listdir(directory) if f.endswith(".egg-info")
]
def _find_egg_info(directory: str) -> str:
"""Find an .egg-info subdirectory in `directory`."""
filenames = [f for f in os.listdir(directory) if f.endswith(".egg-info")]
if not filenames:
raise InstallationError(
f"No .egg-info directory found in {directory}"
)
raise InstallationError(f"No .egg-info directory found in {directory}")
if len(filenames) > 1:
raise InstallationError(
"More than one .egg-info directory found in {}".format(
directory
)
"More than one .egg-info directory found in {}".format(directory)
)
return os.path.join(directory, filenames[0])
def generate_metadata(
build_env, # type: BuildEnvironment
setup_py_path, # type: str
source_dir, # type: str
isolated, # type: bool
details, # type: str
):
# type: (...) -> str
build_env: BuildEnvironment,
setup_py_path: str,
source_dir: str,
isolated: bool,
details: str,
) -> str:
"""Generate metadata using setup.py-based defacto mechanisms.
Returns the generated metadata directory.
"""
logger.debug(
'Running setup.py (path:%s) egg_info for package %s',
setup_py_path, details,
"Running setup.py (path:%s) egg_info for package %s",
setup_py_path,
details,
)
egg_info_dir = TempDirectory(
kind="pip-egg-info", globally_managed=True
).path
egg_info_dir = TempDirectory(kind="pip-egg-info", globally_managed=True).path
args = make_setuptools_egg_info_args(
setup_py_path,
@@ -64,11 +59,16 @@ def generate_metadata(
)
with build_env:
call_subprocess(
args,
cwd=source_dir,
command_desc='python setup.py egg_info',
)
with open_spinner("Preparing metadata (setup.py)") as spinner:
try:
call_subprocess(
args,
cwd=source_dir,
command_desc="python setup.py egg_info",
spinner=spinner,
)
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
# Return the .egg-info directory.
return _find_egg_info(egg_info_dir)

19
venv/Lib/site-packages/pip/_internal/operations/build/wheel.py
View File

@@ -2,7 +2,7 @@ import logging
import os
from typing import Optional
from pip._vendor.pep517.wrappers import Pep517HookCaller
from pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pip._internal.utils.subprocess import runner_with_spinner_message
@@ -10,22 +10,21 @@ logger = logging.getLogger(__name__)
def build_wheel_pep517(
name, # type: str
backend, # type: Pep517HookCaller
metadata_directory, # type: str
tempd, # type: str
):
# type: (...) -> Optional[str]
name: str,
backend: BuildBackendHookCaller,
metadata_directory: str,
tempd: str,
) -> Optional[str]:
"""Build one InstallRequirement using the PEP 517 build process.
Returns path to wheel if successfully built. Otherwise, returns None.
"""
assert metadata_directory is not None
try:
logger.debug('Destination directory: %s', tempd)
logger.debug("Destination directory: %s", tempd)
runner = runner_with_spinner_message(
f'Building wheel for {name} (PEP 517)'
f"Building wheel for {name} (pyproject.toml)"
)
with backend.subprocess_runner(runner):
wheel_name = backend.build_wheel(
@@ -33,6 +32,6 @@ def build_wheel_pep517(
metadata_directory=metadata_directory,
)
except Exception:
logger.error('Failed building wheel for %s', name)
logger.error("Failed building wheel for %s", name)
return None
return os.path.join(tempd, wheel_name)

68
venv/Lib/site-packages/pip/_internal/operations/build/wheel_legacy.py
View File

@@ -4,59 +4,51 @@ from typing import List, Optional
from pip._internal.cli.spinners import open_spinner
from pip._internal.utils.setuptools_build import make_setuptools_bdist_wheel_args
from pip._internal.utils.subprocess import (
LOG_DIVIDER,
call_subprocess,
format_command_args,
)
from pip._internal.utils.subprocess import call_subprocess, format_command_args
logger = logging.getLogger(__name__)
def format_command_result(
command_args, # type: List[str]
command_output, # type: str
):
# type: (...) -> str
command_args: List[str],
command_output: str,
) -> str:
"""Format command information for logging."""
command_desc = format_command_args(command_args)
text = f'Command arguments: {command_desc}\n'
text = f"Command arguments: {command_desc}\n"
if not command_output:
text += 'Command output: None'
text += "Command output: None"
elif logger.getEffectiveLevel() > logging.DEBUG:
text += 'Command output: [use --verbose to show]'
text += "Command output: [use --verbose to show]"
else:
if not command_output.endswith('\n'):
command_output += '\n'
text += f'Command output:\n{command_output}{LOG_DIVIDER}'
if not command_output.endswith("\n"):
command_output += "\n"
text += f"Command output:\n{command_output}"
return text
def get_legacy_build_wheel_path(
names, # type: List[str]
temp_dir, # type: str
name, # type: str
command_args, # type: List[str]
command_output, # type: str
):
# type: (...) -> Optional[str]
names: List[str],
temp_dir: str,
name: str,
command_args: List[str],
command_output: str,
) -> Optional[str]:
"""Return the path to the wheel in the temporary build directory."""
# Sort for determinism.
names = sorted(names)
if not names:
msg = (
'Legacy build of wheel for {!r} created no files.\n'
).format(name)
msg = ("Legacy build of wheel for {!r} created no files.\n").format(name)
msg += format_command_result(command_args, command_output)
logger.warning(msg)
return None
if len(names) > 1:
msg = (
'Legacy build of wheel for {!r} created more than one file.\n'
'Filenames (choosing first): {}\n'
"Legacy build of wheel for {!r} created more than one file.\n"
"Filenames (choosing first): {}\n"
).format(name, names)
msg += format_command_result(command_args, command_output)
logger.warning(msg)
@@ -65,14 +57,13 @@ def get_legacy_build_wheel_path(
def build_wheel_legacy(
name, # type: str
setup_py_path, # type: str
source_dir, # type: str
global_options, # type: List[str]
build_options, # type: List[str]
tempd, # type: str
):
# type: (...) -> Optional[str]
name: str,
setup_py_path: str,
source_dir: str,
global_options: List[str],
build_options: List[str],
tempd: str,
) -> Optional[str]:
"""Build one unpacked package using the "legacy" build process.
Returns path to wheel if successfully built. Otherwise, returns None.
@@ -84,19 +75,20 @@ def build_wheel_legacy(
destination_dir=tempd,
)
spin_message = f'Building wheel for {name} (setup.py)'
spin_message = f"Building wheel for {name} (setup.py)"
with open_spinner(spin_message) as spinner:
logger.debug('Destination directory: %s', tempd)
logger.debug("Destination directory: %s", tempd)
try:
output = call_subprocess(
wheel_args,
command_desc="python setup.py bdist_wheel",
cwd=source_dir,
spinner=spinner,
)
except Exception:
spinner.finish("error")
logger.error('Failed building wheel for %s', name)
logger.error("Failed building wheel for %s", name)
return None
names = os.listdir(tempd)

54
venv/Lib/site-packages/pip/_internal/operations/check.py
View File

@@ -2,19 +2,16 @@
"""
import logging
from typing import TYPE_CHECKING, Callable, Dict, List, NamedTuple, Optional, Set, Tuple
from typing import Callable, Dict, List, NamedTuple, Optional, Set, Tuple
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._internal.distributions import make_distribution_for_install_requirement
from pip._internal.metadata import get_default_environment
from pip._internal.metadata.base import DistributionVersion
from pip._internal.req.req_install import InstallRequirement
if TYPE_CHECKING:
from pip._vendor.packaging.utils import NormalizedName
logger = logging.getLogger(__name__)
@@ -24,12 +21,12 @@ class PackageDetails(NamedTuple):
# Shorthands
PackageSet = Dict['NormalizedName', PackageDetails]
Missing = Tuple['NormalizedName', Requirement]
Conflicting = Tuple['NormalizedName', DistributionVersion, Requirement]
PackageSet = Dict[NormalizedName, PackageDetails]
Missing = Tuple[NormalizedName, Requirement]
Conflicting = Tuple[NormalizedName, DistributionVersion, Requirement]
MissingDict = Dict['NormalizedName', List[Missing]]
ConflictingDict = Dict['NormalizedName', List[Conflicting]]
MissingDict = Dict[NormalizedName, List[Missing]]
ConflictingDict = Dict[NormalizedName, List[Conflicting]]
CheckResult = Tuple[MissingDict, ConflictingDict]
ConflictDetails = Tuple[PackageSet, CheckResult]
@@ -51,8 +48,9 @@ def create_package_set_from_installed() -> Tuple[PackageSet, bool]:
return package_set, problems
def check_package_set(package_set, should_ignore=None):
# type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult
def check_package_set(
package_set: PackageSet, should_ignore: Optional[Callable[[str], bool]] = None
) -> CheckResult:
"""Check if a package set is consistent
If should_ignore is passed, it should be a callable that takes a
@@ -64,8 +62,8 @@ def check_package_set(package_set, should_ignore=None):
for package_name, package_detail in package_set.items():
# Info about dependencies of package_name
missing_deps = set() # type: Set[Missing]
conflicting_deps = set() # type: Set[Conflicting]
missing_deps: Set[Missing] = set()
conflicting_deps: Set[Conflicting] = set()
if should_ignore and should_ignore(package_name):
continue
@@ -77,7 +75,7 @@ def check_package_set(package_set, should_ignore=None):
if name not in package_set:
missed = True
if req.marker is not None:
missed = req.marker.evaluate()
missed = req.marker.evaluate({"extra": ""})
if missed:
missing_deps.add((name, req))
continue
@@ -95,8 +93,7 @@ def check_package_set(package_set, should_ignore=None):
return missing, conflicting
def check_install_conflicts(to_install):
# type: (List[InstallRequirement]) -> ConflictDetails
def check_install_conflicts(to_install: List[InstallRequirement]) -> ConflictDetails:
"""For checking if the dependency graph would be consistent after \
installing given requirements
"""
@@ -112,33 +109,32 @@ def check_install_conflicts(to_install):
package_set,
check_package_set(
package_set, should_ignore=lambda name: name not in whitelist
)
),
)
def _simulate_installation_of(to_install, package_set):
# type: (List[InstallRequirement], PackageSet) -> Set[NormalizedName]
"""Computes the version of packages after installing to_install.
"""
def _simulate_installation_of(
to_install: List[InstallRequirement], package_set: PackageSet
) -> Set[NormalizedName]:
"""Computes the version of packages after installing to_install."""
# Keep track of packages that were installed
installed = set()
# Modify it as installing requirement_set would (assuming no errors)
for inst_req in to_install:
abstract_dist = make_distribution_for_install_requirement(inst_req)
dist = abstract_dist.get_pkg_resources_distribution()
assert dist is not None
name = canonicalize_name(dist.project_name)
package_set[name] = PackageDetails(dist.parsed_version, dist.requires())
dist = abstract_dist.get_metadata_distribution()
name = dist.canonical_name
package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))
installed.add(name)
return installed
def _create_whitelist(would_be_installed, package_set):
# type: (Set[NormalizedName], PackageSet) -> Set[NormalizedName]
def _create_whitelist(
would_be_installed: Set[NormalizedName], package_set: PackageSet
) -> Set[NormalizedName]:
packages_affected = set(would_be_installed)
for package_name in package_set:

200
venv/Lib/site-packages/pip/_internal/operations/freeze.py
View File

@@ -1,19 +1,8 @@
import collections
import logging
import os
from typing import (
Container,
Dict,
Iterable,
Iterator,
List,
NamedTuple,
Optional,
Set,
Union,
)
from typing import Container, Dict, Generator, Iterable, List, NamedTuple, Optional, Set
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
@@ -30,22 +19,20 @@ logger = logging.getLogger(__name__)
class _EditableInfo(NamedTuple):
requirement: Optional[str]
editable: bool
requirement: str
comments: List[str]
def freeze(
requirement=None, # type: Optional[List[str]]
local_only=False, # type: bool
user_only=False, # type: bool
paths=None, # type: Optional[List[str]]
isolated=False, # type: bool
exclude_editable=False, # type: bool
skip=() # type: Container[str]
):
# type: (...) -> Iterator[str]
installations = {} # type: Dict[str, FrozenRequirement]
requirement: Optional[List[str]] = None,
local_only: bool = False,
user_only: bool = False,
paths: Optional[List[str]] = None,
isolated: bool = False,
exclude_editable: bool = False,
skip: Container[str] = (),
) -> Generator[str, None, None]:
installations: Dict[str, FrozenRequirement] = {}
dists = get_environment(paths).iter_installed_distributions(
local_only=local_only,
@@ -63,42 +50,50 @@ def freeze(
# should only be emitted once, even if the same option is in multiple
# requirements files, so we need to keep track of what has been emitted
# so that we don't emit it again if it's seen again
emitted_options = set() # type: Set[str]
emitted_options: Set[str] = set()
# keep track of which files a requirement is in so that we can
# give an accurate warning if a requirement appears multiple times.
req_files = collections.defaultdict(list) # type: Dict[str, List[str]]
req_files: Dict[str, List[str]] = collections.defaultdict(list)
for req_file_path in requirement:
with open(req_file_path) as req_file:
for line in req_file:
if (not line.strip() or
line.strip().startswith('#') or
line.startswith((
'-r', '--requirement',
'-f', '--find-links',
'-i', '--index-url',
'--pre',
'--trusted-host',
'--process-dependency-links',
'--extra-index-url',
'--use-feature'))):
if (
not line.strip()
or line.strip().startswith("#")
or line.startswith(
(
"-r",
"--requirement",
"-f",
"--find-links",
"-i",
"--index-url",
"--pre",
"--trusted-host",
"--process-dependency-links",
"--extra-index-url",
"--use-feature",
)
)
):
line = line.rstrip()
if line not in emitted_options:
emitted_options.add(line)
yield line
continue
if line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
if line.startswith("-e") or line.startswith("--editable"):
if line.startswith("-e"):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line = line[len("--editable") :].strip().lstrip("=")
line_req = install_req_from_editable(
line,
isolated=isolated,
)
else:
line_req = install_req_from_line(
COMMENT_RE.sub('', line).strip(),
COMMENT_RE.sub("", line).strip(),
isolated=isolated,
)
@@ -106,15 +101,15 @@ def freeze(
logger.info(
"Skipping line in requirement file [%s] because "
"it's not clear what it would install: %s",
req_file_path, line.strip(),
req_file_path,
line.strip(),
)
logger.info(
" (add #egg=PackageName to the URL to avoid"
" this warning)"
)
else:
line_req_canonical_name = canonicalize_name(
line_req.name)
line_req_canonical_name = canonicalize_name(line_req.name)
if line_req_canonical_name not in installations:
# either it's not installed, or it is installed
# but has been processed already
@@ -123,14 +118,13 @@ def freeze(
"Requirement file [%s] contains %s, but "
"package %r is not installed",
req_file_path,
COMMENT_RE.sub('', line).strip(),
line_req.name
COMMENT_RE.sub("", line).strip(),
line_req.name,
)
else:
req_files[line_req.name].append(req_file_path)
else:
yield str(installations[
line_req_canonical_name]).rstrip()
yield str(installations[line_req_canonical_name]).rstrip()
del installations[line_req_canonical_name]
req_files[line_req.name].append(req_file_path)
@@ -138,42 +132,33 @@ def freeze(
# single requirements file or in different requirements files).
for name, files in req_files.items():
if len(files) > 1:
logger.warning("Requirement %s included multiple times [%s]",
name, ', '.join(sorted(set(files))))
logger.warning(
"Requirement %s included multiple times [%s]",
name,
", ".join(sorted(set(files))),
)
yield(
'## The following requirements were added by '
'pip freeze:'
)
for installation in sorted(
installations.values(), key=lambda x: x.name.lower()):
yield ("## The following requirements were added by pip freeze:")
for installation in sorted(installations.values(), key=lambda x: x.name.lower()):
if installation.canonical_name not in skip:
yield str(installation).rstrip()
def _format_as_name_version(dist: BaseDistribution) -> str:
if isinstance(dist.version, Version):
return f"{dist.raw_name}=={dist.version}"
return f"{dist.raw_name}==={dist.version}"
dist_version = dist.version
if isinstance(dist_version, Version):
return f"{dist.raw_name}=={dist_version}"
return f"{dist.raw_name}==={dist_version}"
def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
"""
Compute and return values (req, editable, comments) for use in
Compute and return values (req, comments) for use in
FrozenRequirement.from_dist().
"""
if not dist.editable:
return _EditableInfo(requirement=None, editable=False, comments=[])
if dist.location is None:
display = _format_as_name_version(dist)
logger.warning("Editable requirement not found on disk: %s", display)
return _EditableInfo(
requirement=None,
editable=True,
comments=[f"# Editable install not found ({display})"],
)
location = os.path.normcase(os.path.abspath(dist.location))
editable_project_location = dist.editable_project_location
assert editable_project_location
location = os.path.normcase(os.path.abspath(editable_project_location))
from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs
@@ -182,13 +167,13 @@ def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
if vcs_backend is None:
display = _format_as_name_version(dist)
logger.debug(
'No VCS found for editable requirement "%s" in: %r', display,
'No VCS found for editable requirement "%s" in: %r',
display,
location,
)
return _EditableInfo(
requirement=location,
editable=True,
comments=[f'# Editable install with no version control ({display})'],
comments=[f"# Editable install with no version control ({display})"],
)
vcs_name = type(vcs_backend).__name__
@@ -199,50 +184,47 @@ def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
display = _format_as_name_version(dist)
return _EditableInfo(
requirement=location,
editable=True,
comments=[f'# Editable {vcs_name} install with no remote ({display})'],
comments=[f"# Editable {vcs_name} install with no remote ({display})"],
)
except RemoteNotValidError as ex:
display = _format_as_name_version(dist)
return _EditableInfo(
requirement=location,
editable=True,
comments=[
f"# Editable {vcs_name} install ({display}) with either a deleted "
f"local remote or invalid URI:",
f"# '{ex.url}'",
],
)
except BadCommand:
logger.warning(
'cannot determine version of editable source in %s '
'(%s command not found in path)',
"cannot determine version of editable source in %s "
"(%s command not found in path)",
location,
vcs_backend.name,
)
return _EditableInfo(requirement=None, editable=True, comments=[])
return _EditableInfo(requirement=location, comments=[])
except InstallationError as exc:
logger.warning(
"Error when trying to get requirement for VCS system %s, "
"falling back to uneditable format", exc
)
logger.warning("Error when trying to get requirement for VCS system %s", exc)
else:
return _EditableInfo(requirement=req, editable=True, comments=[])
return _EditableInfo(requirement=req, comments=[])
logger.warning('Could not determine repository location of %s', location)
logger.warning("Could not determine repository location of %s", location)
return _EditableInfo(
requirement=None,
editable=False,
comments=['## !! Could not determine repository location'],
requirement=location,
comments=["## !! Could not determine repository location"],
)
class FrozenRequirement:
def __init__(self, name, req, editable, comments=()):
# type: (str, Union[str, Requirement], bool, Iterable[str]) -> None
def __init__(
self,
name: str,
req: str,
editable: bool,
comments: Iterable[str] = (),
) -> None:
self.name = name
self.canonical_name = canonicalize_name(name)
self.req = req
@@ -251,27 +233,23 @@ class FrozenRequirement:
@classmethod
def from_dist(cls, dist: BaseDistribution) -> "FrozenRequirement":
# TODO `get_requirement_info` is taking care of editable requirements.
# TODO This should be refactored when we will add detection of
# editable that provide .dist-info metadata.
req, editable, comments = _get_editable_info(dist)
if req is None and not editable:
# if PEP 610 metadata is present, attempt to use it
editable = dist.editable
if editable:
req, comments = _get_editable_info(dist)
else:
comments = []
direct_url = dist.direct_url
if direct_url:
req = direct_url_as_pep440_direct_reference(
direct_url, dist.raw_name
)
comments = []
if req is None:
# name==version requirement
req = _format_as_name_version(dist)
# if PEP 610 metadata is present, use it
req = direct_url_as_pep440_direct_reference(direct_url, dist.raw_name)
else:
# name==version requirement
req = _format_as_name_version(dist)
return cls(dist.raw_name, req, editable, comments=comments)
def __str__(self):
# type: () -> str
def __str__(self) -> str:
req = self.req
if self.editable:
req = f'-e {req}'
return '\n'.join(list(self.comments) + [str(req)]) + '\n'
req = f"-e {req}"
return "\n".join(list(self.comments) + [str(req)]) + "\n"

29
venv/Lib/site-packages/pip/_internal/operations/install/editable_legacy.py
View File

@@ -1,7 +1,7 @@
"""Legacy editable installation process, i.e. `setup.py develop`.
"""
import logging
from typing import List, Optional, Sequence
from typing import Optional, Sequence
from pip._internal.build_env import BuildEnvironment
from pip._internal.utils.logging import indent_log
@@ -12,27 +12,25 @@ logger = logging.getLogger(__name__)
def install_editable(
install_options, # type: List[str]
global_options, # type: Sequence[str]
prefix, # type: Optional[str]
home, # type: Optional[str]
use_user_site, # type: bool
name, # type: str
setup_py_path, # type: str
isolated, # type: bool
build_env, # type: BuildEnvironment
unpacked_source_directory, # type: str
):
# type: (...) -> None
*,
global_options: Sequence[str],
prefix: Optional[str],
home: Optional[str],
use_user_site: bool,
name: str,
setup_py_path: str,
isolated: bool,
build_env: BuildEnvironment,
unpacked_source_directory: str,
) -> None:
"""Install a package in editable mode. Most arguments are pass-through
to setuptools.
"""
logger.info('Running setup.py develop for %s', name)
logger.info("Running setup.py develop for %s", name)
args = make_setuptools_develop_args(
setup_py_path,
global_options=global_options,
install_options=install_options,
no_user_config=isolated,
prefix=prefix,
home=home,
@@ -43,5 +41,6 @@ def install_editable(
with build_env:
call_subprocess(
args,
command_desc="python setup.py develop",
cwd=unpacked_source_directory,
)

132
venv/Lib/site-packages/pip/_internal/operations/install/legacy.py
View File

@@ -1,132 +0,0 @@
"""Legacy installation process, i.e. `setup.py install`.
"""
import logging
import os
import sys
from distutils.util import change_root
from typing import List, Optional, Sequence
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import InstallationError
from pip._internal.models.scheme import Scheme
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import ensure_dir
from pip._internal.utils.setuptools_build import make_setuptools_install_args
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory
logger = logging.getLogger(__name__)
class LegacyInstallFailure(Exception):
def __init__(self):
# type: () -> None
self.parent = sys.exc_info()
def write_installed_files_from_setuptools_record(
record_lines: List[str],
root: Optional[str],
req_description: str,
) -> None:
def prepend_root(path):
# type: (str) -> str
if root is None or not os.path.isabs(path):
return path
else:
return change_root(root, path)
for line in record_lines:
directory = os.path.dirname(line)
if directory.endswith('.egg-info'):
egg_info_dir = prepend_root(directory)
break
else:
message = (
"{} did not indicate that it installed an "
".egg-info directory. Only setup.py projects "
"generating .egg-info directories are supported."
).format(req_description)
raise InstallationError(message)
new_lines = []
for line in record_lines:
filename = line.strip()
if os.path.isdir(filename):
filename += os.path.sep
new_lines.append(
os.path.relpath(prepend_root(filename), egg_info_dir)
)
new_lines.sort()
ensure_dir(egg_info_dir)
inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
with open(inst_files_path, 'w') as f:
f.write('\n'.join(new_lines) + '\n')
def install(
install_options, # type: List[str]
global_options, # type: Sequence[str]
root, # type: Optional[str]
home, # type: Optional[str]
prefix, # type: Optional[str]
use_user_site, # type: bool
pycompile, # type: bool
scheme, # type: Scheme
setup_py_path, # type: str
isolated, # type: bool
req_name, # type: str
build_env, # type: BuildEnvironment
unpacked_source_directory, # type: str
req_description, # type: str
):
# type: (...) -> bool
header_dir = scheme.headers
with TempDirectory(kind="record") as temp_dir:
try:
record_filename = os.path.join(temp_dir.path, 'install-record.txt')
install_args = make_setuptools_install_args(
setup_py_path,
global_options=global_options,
install_options=install_options,
record_filename=record_filename,
root=root,
prefix=prefix,
header_dir=header_dir,
home=home,
use_user_site=use_user_site,
no_user_config=isolated,
pycompile=pycompile,
)
runner = runner_with_spinner_message(
f"Running setup.py install for {req_name}"
)
with indent_log(), build_env:
runner(
cmd=install_args,
cwd=unpacked_source_directory,
)
if not os.path.exists(record_filename):
logger.debug('Record file %s not found', record_filename)
# Signal to the caller that we didn't install the new package
return False
except Exception:
# Signal to the caller that we didn't install the new package
raise LegacyInstallFailure
# At this point, we have successfully installed the requirement.
# We intentionally do not use any encoding to read the file because
# setuptools writes the file using distutils.file_util.write_file,
# which does not specify an encoding.
with open(record_filename) as f:
record_lines = f.read().splitlines()
write_installed_files_from_setuptools_record(record_lines, root, req_description)
return True

391
venv/Lib/site-packages/pip/_internal/operations/install/wheel.py
View File

@@ -22,6 +22,7 @@ from typing import (
BinaryIO,
Callable,
Dict,
Generator,
Iterable,
Iterator,
List,
@@ -38,11 +39,14 @@ from zipfile import ZipFile, ZipInfo
from pip._vendor.distlib.scripts import ScriptMaker
from pip._vendor.distlib.util import get_export_entry
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.six import ensure_str, ensure_text, reraise
from pip._internal.exceptions import InstallationError
from pip._internal.locations import get_major_minor_version
from pip._internal.metadata import BaseDistribution, get_wheel_distribution
from pip._internal.metadata import (
BaseDistribution,
FilesystemWheel,
get_wheel_distribution,
)
from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl
from pip._internal.models.scheme import SCHEME_KEYS, Scheme
from pip._internal.utils.filesystem import adjacent_tmp_file, replace
@@ -59,62 +63,55 @@ if TYPE_CHECKING:
from typing import Protocol
class File(Protocol):
src_record_path = None # type: RecordPath
dest_path = None # type: str
changed = None # type: bool
src_record_path: "RecordPath"
dest_path: str
changed: bool
def save(self):
# type: () -> None
def save(self) -> None:
pass
logger = logging.getLogger(__name__)
RecordPath = NewType('RecordPath', str)
RecordPath = NewType("RecordPath", str)
InstalledCSVRow = Tuple[RecordPath, str, Union[int, str]]
def rehash(path, blocksize=1 << 20):
# type: (str, int) -> Tuple[str, str]
def rehash(path: str, blocksize: int = 1 << 20) -> Tuple[str, str]:
"""Return (encoded_digest, length) for path using hashlib.sha256()"""
h, length = hash_file(path, blocksize)
digest = 'sha256=' + urlsafe_b64encode(
h.digest()
).decode('latin1').rstrip('=')
digest = "sha256=" + urlsafe_b64encode(h.digest()).decode("latin1").rstrip("=")
return (digest, str(length))
def csv_io_kwargs(mode):
# type: (str) -> Dict[str, Any]
def csv_io_kwargs(mode: str) -> Dict[str, Any]:
"""Return keyword arguments to properly open a CSV file
in the given mode.
"""
return {'mode': mode, 'newline': '', 'encoding': 'utf-8'}
return {"mode": mode, "newline": "", "encoding": "utf-8"}
def fix_script(path):
# type: (str) -> bool
def fix_script(path: str) -> bool:
"""Replace #!python with #!/path/to/python
Return True if file was changed.
"""
# XXX RECORD hashes will need to be updated
assert os.path.isfile(path)
with open(path, 'rb') as script:
with open(path, "rb") as script:
firstline = script.readline()
if not firstline.startswith(b'#!python'):
if not firstline.startswith(b"#!python"):
return False
exename = sys.executable.encode(sys.getfilesystemencoding())
firstline = b'#!' + exename + os.linesep.encode("ascii")
firstline = b"#!" + exename + os.linesep.encode("ascii")
rest = script.read()
with open(path, 'wb') as script:
with open(path, "wb") as script:
script.write(firstline)
script.write(rest)
return True
def wheel_root_is_purelib(metadata):
# type: (Message) -> bool
def wheel_root_is_purelib(metadata: Message) -> bool:
return metadata.get("Root-Is-Purelib", "").lower() == "true"
@@ -129,8 +126,7 @@ def get_entrypoints(dist: BaseDistribution) -> Tuple[Dict[str, str], Dict[str, s
return console_scripts, gui_scripts
def message_about_scripts_not_on_PATH(scripts):
# type: (Sequence[str]) -> Optional[str]
def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> Optional[str]:
"""Determine if any scripts are not on PATH and format a warning.
Returns a warning message if one or more scripts are not on PATH,
otherwise None.
@@ -139,7 +135,7 @@ def message_about_scripts_not_on_PATH(scripts):
return None
# Group scripts by the path they were installed in
grouped_by_dir = collections.defaultdict(set) # type: Dict[str, Set[str]]
grouped_by_dir: Dict[str, Set[str]] = collections.defaultdict(set)
for destfile in scripts:
parent_dir = os.path.dirname(destfile)
script_name = os.path.basename(destfile)
@@ -147,23 +143,26 @@ def message_about_scripts_not_on_PATH(scripts):
# We don't want to warn for directories that are on PATH.
not_warn_dirs = [
os.path.normcase(i).rstrip(os.sep) for i in
os.environ.get("PATH", "").split(os.pathsep)
os.path.normcase(os.path.normpath(i)).rstrip(os.sep)
for i in os.environ.get("PATH", "").split(os.pathsep)
]
# If an executable sits with sys.executable, we don't warn for it.
# This covers the case of venv invocations without activating the venv.
not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable)))
warn_for = {
parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items()
if os.path.normcase(parent_dir) not in not_warn_dirs
} # type: Dict[str, Set[str]]
not_warn_dirs.append(
os.path.normcase(os.path.normpath(os.path.dirname(sys.executable)))
)
warn_for: Dict[str, Set[str]] = {
parent_dir: scripts
for parent_dir, scripts in grouped_by_dir.items()
if os.path.normcase(os.path.normpath(parent_dir)) not in not_warn_dirs
}
if not warn_for:
return None
# Format a message
msg_lines = []
for parent_dir, dir_scripts in warn_for.items():
sorted_scripts = sorted(dir_scripts) # type: List[str]
sorted_scripts: List[str] = sorted(dir_scripts)
if len(sorted_scripts) == 1:
start_text = "script {} is".format(sorted_scripts[0])
else:
@@ -172,8 +171,9 @@ def message_about_scripts_not_on_PATH(scripts):
)
msg_lines.append(
"The {} installed in '{}' which is not on PATH."
.format(start_text, parent_dir)
"The {} installed in '{}' which is not on PATH.".format(
start_text, parent_dir
)
)
last_line_fmt = (
@@ -200,8 +200,9 @@ def message_about_scripts_not_on_PATH(scripts):
return "\n".join(msg_lines)
def _normalized_outrows(outrows):
# type: (Iterable[InstalledCSVRow]) -> List[Tuple[str, str, str]]
def _normalized_outrows(
outrows: Iterable[InstalledCSVRow],
) -> List[Tuple[str, str, str]]:
"""Normalize the given rows of a RECORD file.
Items in each row are converted into str. Rows are then sorted to make
@@ -221,69 +222,57 @@ def _normalized_outrows(outrows):
# For additional background, see--
# https://github.com/pypa/pip/issues/5868
return sorted(
(ensure_str(record_path, encoding='utf-8'), hash_, str(size))
for record_path, hash_, size in outrows
(record_path, hash_, str(size)) for record_path, hash_, size in outrows
)
def _record_to_fs_path(record_path):
# type: (RecordPath) -> str
return record_path
def _record_to_fs_path(record_path: RecordPath, lib_dir: str) -> str:
return os.path.join(lib_dir, record_path)
def _fs_to_record_path(path, relative_to=None):
# type: (str, Optional[str]) -> RecordPath
if relative_to is not None:
# On Windows, do not handle relative paths if they belong to different
# logical disks
if os.path.splitdrive(path)[0].lower() == \
os.path.splitdrive(relative_to)[0].lower():
path = os.path.relpath(path, relative_to)
path = path.replace(os.path.sep, '/')
return cast('RecordPath', path)
def _fs_to_record_path(path: str, lib_dir: str) -> RecordPath:
# On Windows, do not handle relative paths if they belong to different
# logical disks
if os.path.splitdrive(path)[0].lower() == os.path.splitdrive(lib_dir)[0].lower():
path = os.path.relpath(path, lib_dir)
def _parse_record_path(record_column):
# type: (str) -> RecordPath
p = ensure_text(record_column, encoding='utf-8')
return cast('RecordPath', p)
path = path.replace(os.path.sep, "/")
return cast("RecordPath", path)
def get_csv_rows_for_installed(
old_csv_rows, # type: List[List[str]]
installed, # type: Dict[RecordPath, RecordPath]
changed, # type: Set[RecordPath]
generated, # type: List[str]
lib_dir, # type: str
):
# type: (...) -> List[InstalledCSVRow]
old_csv_rows: List[List[str]],
installed: Dict[RecordPath, RecordPath],
changed: Set[RecordPath],
generated: List[str],
lib_dir: str,
) -> List[InstalledCSVRow]:
"""
:param installed: A map from archive RECORD path to installation RECORD
path.
"""
installed_rows = [] # type: List[InstalledCSVRow]
installed_rows: List[InstalledCSVRow] = []
for row in old_csv_rows:
if len(row) > 3:
logger.warning('RECORD line has more than three elements: %s', row)
old_record_path = _parse_record_path(row[0])
logger.warning("RECORD line has more than three elements: %s", row)
old_record_path = cast("RecordPath", row[0])
new_record_path = installed.pop(old_record_path, old_record_path)
if new_record_path in changed:
digest, length = rehash(_record_to_fs_path(new_record_path))
digest, length = rehash(_record_to_fs_path(new_record_path, lib_dir))
else:
digest = row[1] if len(row) > 1 else ''
length = row[2] if len(row) > 2 else ''
digest = row[1] if len(row) > 1 else ""
length = row[2] if len(row) > 2 else ""
installed_rows.append((new_record_path, digest, length))
for f in generated:
path = _fs_to_record_path(f, lib_dir)
digest, length = rehash(f)
installed_rows.append((path, digest, length))
for installed_record_path in installed.values():
installed_rows.append((installed_record_path, '', ''))
installed_rows.append((installed_record_path, "", ""))
return installed_rows
def get_console_script_specs(console):
# type: (Dict[str, str]) -> List[str]
def get_console_script_specs(console: Dict[str, str]) -> List[str]:
"""
Given the mapping from entrypoint name to callable, return the relevant
console script specs.
@@ -326,62 +315,57 @@ def get_console_script_specs(console):
# DEFAULT
# - The default behavior is to install pip, pipX, pipX.Y, easy_install
# and easy_install-X.Y.
pip_script = console.pop('pip', None)
pip_script = console.pop("pip", None)
if pip_script:
if "ENSUREPIP_OPTIONS" not in os.environ:
scripts_to_generate.append('pip = ' + pip_script)
scripts_to_generate.append("pip = " + pip_script)
if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
scripts_to_generate.append(
'pip{} = {}'.format(sys.version_info[0], pip_script)
"pip{} = {}".format(sys.version_info[0], pip_script)
)
scripts_to_generate.append(
f'pip{get_major_minor_version()} = {pip_script}'
)
scripts_to_generate.append(f"pip{get_major_minor_version()} = {pip_script}")
# Delete any other versioned pip entry points
pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)]
pip_ep = [k for k in console if re.match(r"pip(\d+(\.\d+)?)?$", k)]
for k in pip_ep:
del console[k]
easy_install_script = console.pop('easy_install', None)
easy_install_script = console.pop("easy_install", None)
if easy_install_script:
if "ENSUREPIP_OPTIONS" not in os.environ:
scripts_to_generate.append(
'easy_install = ' + easy_install_script
)
scripts_to_generate.append("easy_install = " + easy_install_script)
scripts_to_generate.append(
'easy_install-{} = {}'.format(
"easy_install-{} = {}".format(
get_major_minor_version(), easy_install_script
)
)
# Delete any other versioned easy_install entry points
easy_install_ep = [
k for k in console if re.match(r'easy_install(-\d\.\d)?$', k)
k for k in console if re.match(r"easy_install(-\d+\.\d+)?$", k)
]
for k in easy_install_ep:
del console[k]
# Generate the console entry points specified in the wheel
scripts_to_generate.extend(starmap('{} = {}'.format, console.items()))
scripts_to_generate.extend(starmap("{} = {}".format, console.items()))
return scripts_to_generate
class ZipBackedFile:
def __init__(self, src_record_path, dest_path, zip_file):
# type: (RecordPath, str, ZipFile) -> None
def __init__(
self, src_record_path: RecordPath, dest_path: str, zip_file: ZipFile
) -> None:
self.src_record_path = src_record_path
self.dest_path = dest_path
self._zip_file = zip_file
self.changed = False
def _getinfo(self):
# type: () -> ZipInfo
def _getinfo(self) -> ZipInfo:
return self._zip_file.getinfo(self.src_record_path)
def save(self):
# type: () -> None
def save(self) -> None:
# directory creation is lazy and after file filtering
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
@@ -410,22 +394,19 @@ class ZipBackedFile:
class ScriptFile:
def __init__(self, file):
# type: (File) -> None
def __init__(self, file: "File") -> None:
self._file = file
self.src_record_path = self._file.src_record_path
self.dest_path = self._file.dest_path
self.changed = False
def save(self):
# type: () -> None
def save(self) -> None:
self._file.save()
self.changed = fix_script(self.dest_path)
class MissingCallableSuffix(InstallationError):
def __init__(self, entry_point):
# type: (str) -> None
def __init__(self, entry_point: str) -> None:
super().__init__(
"Invalid script entry point: {} - A callable "
"suffix is required. Cf https://packaging.python.org/"
@@ -434,31 +415,30 @@ class MissingCallableSuffix(InstallationError):
)
def _raise_for_invalid_entrypoint(specification):
# type: (str) -> None
def _raise_for_invalid_entrypoint(specification: str) -> None:
entry = get_export_entry(specification)
if entry is not None and entry.suffix is None:
raise MissingCallableSuffix(str(entry))
class PipScriptMaker(ScriptMaker):
def make(self, specification, options=None):
# type: (str, Dict[str, Any]) -> List[str]
def make(
self, specification: str, options: Optional[Dict[str, Any]] = None
) -> List[str]:
_raise_for_invalid_entrypoint(specification)
return super().make(specification, options)
def _install_wheel(
name, # type: str
wheel_zip, # type: ZipFile
wheel_path, # type: str
scheme, # type: Scheme
pycompile=True, # type: bool
warn_script_location=True, # type: bool
direct_url=None, # type: Optional[DirectUrl]
requested=False, # type: bool
):
# type: (...) -> None
name: str,
wheel_zip: ZipFile,
wheel_path: str,
scheme: Scheme,
pycompile: bool = True,
warn_script_location: bool = True,
direct_url: Optional[DirectUrl] = None,
requested: bool = False,
) -> None:
"""Install a wheel.
:param name: Name of the project to install
@@ -485,33 +465,23 @@ def _install_wheel(
# installed = files copied from the wheel to the destination
# changed = files changed while installing (scripts #! line typically)
# generated = files newly generated during the install (script wrappers)
installed = {} # type: Dict[RecordPath, RecordPath]
changed = set() # type: Set[RecordPath]
generated = [] # type: List[str]
installed: Dict[RecordPath, RecordPath] = {}
changed: Set[RecordPath] = set()
generated: List[str] = []
def record_installed(srcfile, destfile, modified=False):
# type: (RecordPath, str, bool) -> None
def record_installed(
srcfile: RecordPath, destfile: str, modified: bool = False
) -> None:
"""Map archive RECORD paths to installation RECORD paths."""
newpath = _fs_to_record_path(destfile, lib_dir)
installed[srcfile] = newpath
if modified:
changed.add(_fs_to_record_path(destfile))
changed.add(newpath)
def all_paths():
# type: () -> Iterable[RecordPath]
names = wheel_zip.namelist()
# If a flag is set, names may be unicode in Python 2. We convert to
# text explicitly so these are valid for lookup in RECORD.
decoded_names = map(ensure_text, names)
for name in decoded_names:
yield cast("RecordPath", name)
def is_dir_path(path):
# type: (RecordPath) -> bool
def is_dir_path(path: RecordPath) -> bool:
return path.endswith("/")
def assert_no_path_traversal(dest_dir_path, target_path):
# type: (str, str) -> None
def assert_no_path_traversal(dest_dir_path: str, target_path: str) -> None:
if not is_within_directory(dest_dir_path, target_path):
message = (
"The wheel {!r} has a file {!r} trying to install"
@@ -521,10 +491,10 @@ def _install_wheel(
message.format(wheel_path, target_path, dest_dir_path)
)
def root_scheme_file_maker(zip_file, dest):
# type: (ZipFile, str) -> Callable[[RecordPath], File]
def make_root_scheme_file(record_path):
# type: (RecordPath) -> File
def root_scheme_file_maker(
zip_file: ZipFile, dest: str
) -> Callable[[RecordPath], "File"]:
def make_root_scheme_file(record_path: RecordPath) -> "File":
normed_path = os.path.normpath(record_path)
dest_path = os.path.join(dest, normed_path)
assert_no_path_traversal(dest, dest_path)
@@ -532,17 +502,12 @@ def _install_wheel(
return make_root_scheme_file
def data_scheme_file_maker(zip_file, scheme):
# type: (ZipFile, Scheme) -> Callable[[RecordPath], File]
scheme_paths = {}
for key in SCHEME_KEYS:
encoded_key = ensure_text(key)
scheme_paths[encoded_key] = ensure_text(
getattr(scheme, key), encoding=sys.getfilesystemencoding()
)
def data_scheme_file_maker(
zip_file: ZipFile, scheme: Scheme
) -> Callable[[RecordPath], "File"]:
scheme_paths = {key: getattr(scheme, key) for key in SCHEME_KEYS}
def make_data_scheme_file(record_path):
# type: (RecordPath) -> File
def make_data_scheme_file(record_path: RecordPath) -> "File":
normed_path = os.path.normpath(record_path)
try:
_, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2)
@@ -561,9 +526,7 @@ def _install_wheel(
"Unknown scheme key used in {}: {} (for file {!r}). .data"
" directory contents should be in subdirectories named"
" with a valid scheme key ({})"
).format(
wheel_path, scheme_key, record_path, valid_scheme_keys
)
).format(wheel_path, scheme_key, record_path, valid_scheme_keys)
raise InstallationError(message)
dest_path = os.path.join(scheme_path, dest_subpath)
@@ -572,30 +535,19 @@ def _install_wheel(
return make_data_scheme_file
def is_data_scheme_path(path):
# type: (RecordPath) -> bool
def is_data_scheme_path(path: RecordPath) -> bool:
return path.split("/", 1)[0].endswith(".data")
paths = all_paths()
paths = cast(List[RecordPath], wheel_zip.namelist())
file_paths = filterfalse(is_dir_path, paths)
root_scheme_paths, data_scheme_paths = partition(
is_data_scheme_path, file_paths
)
root_scheme_paths, data_scheme_paths = partition(is_data_scheme_path, file_paths)
make_root_scheme_file = root_scheme_file_maker(
wheel_zip,
ensure_text(lib_dir, encoding=sys.getfilesystemencoding()),
)
files = map(make_root_scheme_file, root_scheme_paths)
make_root_scheme_file = root_scheme_file_maker(wheel_zip, lib_dir)
files: Iterator[File] = map(make_root_scheme_file, root_scheme_paths)
def is_script_scheme_path(path):
# type: (RecordPath) -> bool
def is_script_scheme_path(path: RecordPath) -> bool:
parts = path.split("/", 2)
return (
len(parts) > 2 and
parts[0].endswith(".data") and
parts[1] == "scripts"
)
return len(parts) > 2 and parts[0].endswith(".data") and parts[1] == "scripts"
other_scheme_paths, script_scheme_paths = partition(
is_script_scheme_path, data_scheme_paths
@@ -606,30 +558,32 @@ def _install_wheel(
files = chain(files, other_scheme_files)
# Get the defined entry points
distribution = get_wheel_distribution(wheel_path, canonicalize_name(name))
distribution = get_wheel_distribution(
FilesystemWheel(wheel_path),
canonicalize_name(name),
)
console, gui = get_entrypoints(distribution)
def is_entrypoint_wrapper(file):
# type: (File) -> bool
def is_entrypoint_wrapper(file: "File") -> bool:
# EP, EP.exe and EP-script.py are scripts generated for
# entry point EP by setuptools
path = file.dest_path
name = os.path.basename(path)
if name.lower().endswith('.exe'):
if name.lower().endswith(".exe"):
matchname = name[:-4]
elif name.lower().endswith('-script.py'):
elif name.lower().endswith("-script.py"):
matchname = name[:-10]
elif name.lower().endswith(".pya"):
matchname = name[:-4]
else:
matchname = name
# Ignore setuptools-generated scripts
return (matchname in console or matchname in gui)
return matchname in console or matchname in gui
script_scheme_files = map(make_data_scheme_file, script_scheme_paths)
script_scheme_files = filterfalse(
is_entrypoint_wrapper, script_scheme_files
script_scheme_files: Iterator[File] = map(
make_data_scheme_file, script_scheme_paths
)
script_scheme_files = filterfalse(is_entrypoint_wrapper, script_scheme_files)
script_scheme_files = map(ScriptFile, script_scheme_files)
files = chain(files, script_scheme_files)
@@ -637,8 +591,7 @@ def _install_wheel(
file.save()
record_installed(file.src_record_path, file.dest_path, file.changed)
def pyc_source_file_paths():
# type: () -> Iterator[str]
def pyc_source_file_paths() -> Generator[str, None, None]:
# We de-duplicate installation paths, since there can be overlap (e.g.
# file in .data maps to same location as file in wheel root).
# Sorting installation paths makes it easier to reproduce and debug
@@ -647,30 +600,21 @@ def _install_wheel(
full_installed_path = os.path.join(lib_dir, installed_path)
if not os.path.isfile(full_installed_path):
continue
if not full_installed_path.endswith('.py'):
if not full_installed_path.endswith(".py"):
continue
yield full_installed_path
def pyc_output_path(path):
# type: (str) -> str
"""Return the path the pyc file would have been written to.
"""
def pyc_output_path(path: str) -> str:
"""Return the path the pyc file would have been written to."""
return importlib.util.cache_from_source(path)
# Compile all of the pyc files for the installed files
if pycompile:
with captured_stdout() as stdout:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
warnings.filterwarnings("ignore")
for path in pyc_source_file_paths():
# Python 2's `compileall.compile_file` requires a str in
# error cases, so we must convert to the native type.
path_arg = ensure_str(
path, encoding=sys.getfilesystemencoding()
)
success = compileall.compile_file(
path_arg, force=True, quiet=True
)
success = compileall.compile_file(path, force=True, quiet=True)
if success:
pyc_path = pyc_output_path(path)
assert os.path.exists(pyc_path)
@@ -689,7 +633,7 @@ def _install_wheel(
# Ensure we don't generate any variants for scripts because this is almost
# never what somebody wants.
# See https://bitbucket.org/pypa/distlib/issue/35/
maker.variants = {''}
maker.variants = {""}
# This is required because otherwise distlib creates scripts that are not
# executable.
@@ -699,14 +643,12 @@ def _install_wheel(
# Generate the console and GUI entry points specified in the wheel
scripts_to_generate = get_console_script_specs(console)
gui_scripts_to_generate = list(starmap('{} = {}'.format, gui.items()))
gui_scripts_to_generate = list(starmap("{} = {}".format, gui.items()))
generated_console_scripts = maker.make_multiple(scripts_to_generate)
generated.extend(generated_console_scripts)
generated.extend(
maker.make_multiple(gui_scripts_to_generate, {'gui': True})
)
generated.extend(maker.make_multiple(gui_scripts_to_generate, {"gui": True}))
if warn_script_location:
msg = message_about_scripts_not_on_PATH(generated_console_scripts)
@@ -716,8 +658,7 @@ def _install_wheel(
generated_file_mode = 0o666 & ~current_umask()
@contextlib.contextmanager
def _generate_file(path, **kwargs):
# type: (str, **Any) -> Iterator[BinaryIO]
def _generate_file(path: str, **kwargs: Any) -> Generator[BinaryIO, None, None]:
with adjacent_tmp_file(path, **kwargs) as f:
yield f
os.chmod(f.name, generated_file_mode)
@@ -726,9 +667,9 @@ def _install_wheel(
dest_info_dir = os.path.join(lib_dir, info_dir)
# Record pip as the installer
installer_path = os.path.join(dest_info_dir, 'INSTALLER')
installer_path = os.path.join(dest_info_dir, "INSTALLER")
with _generate_file(installer_path) as installer_file:
installer_file.write(b'pip\n')
installer_file.write(b"pip\n")
generated.append(installer_path)
# Record the PEP 610 direct URL reference
@@ -740,12 +681,12 @@ def _install_wheel(
# Record the REQUESTED file
if requested:
requested_path = os.path.join(dest_info_dir, 'REQUESTED')
requested_path = os.path.join(dest_info_dir, "REQUESTED")
with open(requested_path, "wb"):
pass
generated.append(requested_path)
record_text = distribution.read_text('RECORD')
record_text = distribution.read_text("RECORD")
record_rows = list(csv.reader(record_text.splitlines()))
rows = get_csv_rows_for_installed(
@@ -753,42 +694,38 @@ def _install_wheel(
installed=installed,
changed=changed,
generated=generated,
lib_dir=lib_dir)
lib_dir=lib_dir,
)
# Record details of all files installed
record_path = os.path.join(dest_info_dir, 'RECORD')
record_path = os.path.join(dest_info_dir, "RECORD")
with _generate_file(record_path, **csv_io_kwargs('w')) as record_file:
# The type mypy infers for record_file is different for Python 3
# (typing.IO[Any]) and Python 2 (typing.BinaryIO). We explicitly
# cast to typing.IO[str] as a workaround.
writer = csv.writer(cast('IO[str]', record_file))
with _generate_file(record_path, **csv_io_kwargs("w")) as record_file:
# Explicitly cast to typing.IO[str] as a workaround for the mypy error:
# "writer" has incompatible type "BinaryIO"; expected "_Writer"
writer = csv.writer(cast("IO[str]", record_file))
writer.writerows(_normalized_outrows(rows))
@contextlib.contextmanager
def req_error_context(req_description):
# type: (str) -> Iterator[None]
def req_error_context(req_description: str) -> Generator[None, None, None]:
try:
yield
except InstallationError as e:
message = "For req: {}. {}".format(req_description, e.args[0])
reraise(
InstallationError, InstallationError(message), sys.exc_info()[2]
)
raise InstallationError(message) from e
def install_wheel(
name, # type: str
wheel_path, # type: str
scheme, # type: Scheme
req_description, # type: str
pycompile=True, # type: bool
warn_script_location=True, # type: bool
direct_url=None, # type: Optional[DirectUrl]
requested=False, # type: bool
):
# type: (...) -> None
name: str,
wheel_path: str,
scheme: Scheme,
req_description: str,
pycompile: bool = True,
warn_script_location: bool = True,
direct_url: Optional[DirectUrl] = None,
requested: bool = False,
) -> None:
with ZipFile(wheel_path, allowZip64=True) as z:
with req_error_context(req_description):
_install_wheel(

528
venv/Lib/site-packages/pip/_internal/operations/prepare.py
View File

@@ -8,10 +8,9 @@ import logging
import mimetypes
import os
import shutil
from typing import Dict, Iterable, List, Optional, Tuple
from typing import Dict, Iterable, List, Optional
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.pkg_resources import Distribution
from pip._internal.distributions import make_distribution_for_install_requirement
from pip._internal.distributions.installed import InstalledDistribution
@@ -20,11 +19,14 @@ from pip._internal.exceptions import (
HashMismatch,
HashUnpinned,
InstallationError,
MetadataInconsistent,
NetworkConnectionError,
PreviousBuildDirError,
VcsHashUnsupported,
)
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import BaseDistribution, get_metadata_distribution
from pip._internal.models.direct_url import ArchiveInfo
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
from pip._internal.network.download import BatchDownloader, Downloader
@@ -33,13 +35,20 @@ from pip._internal.network.lazy_wheel import (
dist_from_wheel_url,
)
from pip._internal.network.session import PipSession
from pip._internal.operations.build.build_tracker import BuildTracker
from pip._internal.req.req_install import InstallRequirement
from pip._internal.req.req_tracker import RequirementTracker
from pip._internal.utils.deprecation import deprecated
from pip._internal.utils.filesystem import copy2_fixed
from pip._internal.utils.direct_url_helpers import (
direct_url_for_editable,
direct_url_from_link,
)
from pip._internal.utils.hashes import Hashes, MissingHashes
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree
from pip._internal.utils.misc import (
display_path,
hash_file,
hide_url,
is_installable_dir,
)
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.unpacking import unpack_file
from pip._internal.vcs import vcs
@@ -48,30 +57,29 @@ logger = logging.getLogger(__name__)
def _get_prepared_distribution(
req, # type: InstallRequirement
req_tracker, # type: RequirementTracker
finder, # type: PackageFinder
build_isolation, # type: bool
):
# type: (...) -> Distribution
req: InstallRequirement,
build_tracker: BuildTracker,
finder: PackageFinder,
build_isolation: bool,
check_build_deps: bool,
) -> BaseDistribution:
"""Prepare a distribution for installation."""
abstract_dist = make_distribution_for_install_requirement(req)
with req_tracker.track(req):
abstract_dist.prepare_distribution_metadata(finder, build_isolation)
return abstract_dist.get_pkg_resources_distribution()
with build_tracker.track(req):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
return abstract_dist.get_metadata_distribution()
def unpack_vcs_link(link, location):
# type: (Link, str) -> None
def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
vcs_backend = vcs.get_backend_for_scheme(link.scheme)
assert vcs_backend is not None
vcs_backend.unpack(location, url=hide_url(link.url))
vcs_backend.unpack(location, url=hide_url(link.url), verbosity=verbosity)
class File:
def __init__(self, path, content_type):
# type: (str, Optional[str]) -> None
def __init__(self, path: str, content_type: Optional[str]) -> None:
self.path = path
if content_type is None:
self.content_type = mimetypes.guess_type(path)[0]
@@ -80,19 +88,16 @@ class File:
def get_http_url(
link, # type: Link
download, # type: Downloader
download_dir=None, # type: Optional[str]
hashes=None, # type: Optional[Hashes]
):
# type: (...) -> File
link: Link,
download: Downloader,
download_dir: Optional[str] = None,
hashes: Optional[Hashes] = None,
) -> File:
temp_dir = TempDirectory(kind="unpack", globally_managed=True)
# If a download dir is specified, is the file already downloaded there?
already_downloaded_path = None
if download_dir:
already_downloaded_path = _check_download_dir(
link, download_dir, hashes
)
already_downloaded_path = _check_download_dir(link, download_dir, hashes)
if already_downloaded_path:
from_path = already_downloaded_path
@@ -106,72 +111,14 @@ def get_http_url(
return File(from_path, content_type)
def _copy2_ignoring_special_files(src, dest):
# type: (str, str) -> None
"""Copying special files is not supported, but as a convenience to users
we skip errors copying them. This supports tools that may create e.g.
socket files in the project source directory.
"""
try:
copy2_fixed(src, dest)
except shutil.SpecialFileError as e:
# SpecialFileError may be raised due to either the source or
# destination. If the destination was the cause then we would actually
# care, but since the destination directory is deleted prior to
# copy we ignore all of them assuming it is caused by the source.
logger.warning(
"Ignoring special file error '%s' encountered copying %s to %s.",
str(e),
src,
dest,
)
def _copy_source_tree(source, target):
# type: (str, str) -> None
target_abspath = os.path.abspath(target)
target_basename = os.path.basename(target_abspath)
target_dirname = os.path.dirname(target_abspath)
def ignore(d, names):
# type: (str, List[str]) -> List[str]
skipped = [] # type: List[str]
if d == source:
# Pulling in those directories can potentially be very slow,
# exclude the following directories if they appear in the top
# level dir (and only it).
# See discussion at https://github.com/pypa/pip/pull/6770
skipped += ['.tox', '.nox']
if os.path.abspath(d) == target_dirname:
# Prevent an infinite recursion if the target is in source.
# This can happen when TMPDIR is set to ${PWD}/...
# and we copy PWD to TMPDIR.
skipped += [target_basename]
return skipped
shutil.copytree(
source,
target,
ignore=ignore,
symlinks=True,
copy_function=_copy2_ignoring_special_files,
)
def get_file_url(
link, # type: Link
download_dir=None, # type: Optional[str]
hashes=None # type: Optional[Hashes]
):
# type: (...) -> File
"""Get file and optionally check its hash.
"""
link: Link, download_dir: Optional[str] = None, hashes: Optional[Hashes] = None
) -> File:
"""Get file and optionally check its hash."""
# If a download dir is specified, is the file already there and valid?
already_downloaded_path = None
if download_dir:
already_downloaded_path = _check_download_dir(
link, download_dir, hashes
)
already_downloaded_path = _check_download_dir(link, download_dir, hashes)
if already_downloaded_path:
from_path = already_downloaded_path
@@ -189,13 +136,13 @@ def get_file_url(
def unpack_url(
link, # type: Link
location, # type: str
download, # type: Downloader
download_dir=None, # type: Optional[str]
hashes=None, # type: Optional[Hashes]
):
# type: (...) -> Optional[File]
link: Link,
location: str,
download: Downloader,
verbosity: int,
download_dir: Optional[str] = None,
hashes: Optional[Hashes] = None,
) -> Optional[File]:
"""Unpack link into location, downloading if required.
:param hashes: A Hashes object, one of whose embedded hashes must match,
@@ -205,30 +152,10 @@ def unpack_url(
"""
# non-editable vcs urls
if link.is_vcs:
unpack_vcs_link(link, location)
unpack_vcs_link(link, location, verbosity=verbosity)
return None
# Once out-of-tree-builds are no longer supported, could potentially
# replace the below condition with `assert not link.is_existing_dir`
# - unpack_url does not need to be called for in-tree-builds.
#
# As further cleanup, _copy_source_tree and accompanying tests can
# be removed.
if link.is_existing_dir():
deprecated(
"A future pip version will change local packages to be built "
"in-place without first copying to a temporary directory. "
"We recommend you use --use-feature=in-tree-build to test "
"your packages with this new behavior before it becomes the "
"default.\n",
replacement=None,
gone_in="21.3",
issue=7555
)
if os.path.isdir(location):
rmtree(location)
_copy_source_tree(link.file_path, location)
return None
assert not link.is_existing_dir()
# file urls
if link.is_file:
@@ -251,10 +178,14 @@ def unpack_url(
return file
def _check_download_dir(link, download_dir, hashes):
# type: (Link, str, Optional[Hashes]) -> Optional[str]
""" Check download_dir for previously downloaded file with correct hash
If a correct file is found return its path else None
def _check_download_dir(
link: Link,
download_dir: str,
hashes: Optional[Hashes],
warn_on_hash_mismatch: bool = True,
) -> Optional[str]:
"""Check download_dir for previously downloaded file with correct hash
If a correct file is found return its path else None
"""
download_path = os.path.join(download_dir, link.filename)
@@ -262,46 +193,45 @@ def _check_download_dir(link, download_dir, hashes):
return None
# If already downloaded, does its hash match?
logger.info('File was already downloaded %s', download_path)
logger.info("File was already downloaded %s", download_path)
if hashes:
try:
hashes.check_against_path(download_path)
except HashMismatch:
logger.warning(
'Previously-downloaded file %s has bad hash. '
'Re-downloading.',
download_path
)
if warn_on_hash_mismatch:
logger.warning(
"Previously-downloaded file %s has bad hash. Re-downloading.",
download_path,
)
os.unlink(download_path)
return None
return download_path
class RequirementPreparer:
"""Prepares a Requirement
"""
"""Prepares a Requirement"""
def __init__(
self,
build_dir, # type: str
download_dir, # type: Optional[str]
src_dir, # type: str
build_isolation, # type: bool
req_tracker, # type: RequirementTracker
session, # type: PipSession
progress_bar, # type: str
finder, # type: PackageFinder
require_hashes, # type: bool
use_user_site, # type: bool
lazy_wheel, # type: bool
in_tree_build, # type: bool
):
# type: (...) -> None
build_dir: str,
download_dir: Optional[str],
src_dir: str,
build_isolation: bool,
check_build_deps: bool,
build_tracker: BuildTracker,
session: PipSession,
progress_bar: str,
finder: PackageFinder,
require_hashes: bool,
use_user_site: bool,
lazy_wheel: bool,
verbosity: int,
) -> None:
super().__init__()
self.src_dir = src_dir
self.build_dir = build_dir
self.req_tracker = req_tracker
self.build_tracker = build_tracker
self._session = session
self._download = Downloader(session, progress_bar)
self._batch_download = BatchDownloader(session, progress_bar)
@@ -314,6 +244,9 @@ class RequirementPreparer:
# Is build isolation allowed?
self.build_isolation = build_isolation
# Should check build dependencies?
self.check_build_deps = check_build_deps
# Should hash-checking be required?
self.require_hashes = require_hashes
@@ -323,35 +256,45 @@ class RequirementPreparer:
# Should wheels be downloaded lazily?
self.use_lazy_wheel = lazy_wheel
# Should in-tree builds be used for local paths?
self.in_tree_build = in_tree_build
# How verbose should underlying tooling be?
self.verbosity = verbosity
# Memoized downloaded files, as mapping of url: (path, mime type)
self._downloaded = {} # type: Dict[str, Tuple[str, str]]
# Memoized downloaded files, as mapping of url: path.
self._downloaded: Dict[str, str] = {}
# Previous "header" printed for a link-based InstallRequirement
self._previous_requirement_header = ("", "")
def _log_preparing_link(self, req):
# type: (InstallRequirement) -> None
def _log_preparing_link(self, req: InstallRequirement) -> None:
"""Provide context for the requirement being prepared."""
if req.link.is_file and not req.original_link_is_in_wheel_cache:
if req.link.is_file and not req.is_wheel_from_cache:
message = "Processing %s"
information = str(display_path(req.link.file_path))
else:
message = "Collecting %s"
information = str(req.req or req)
# If we used req.req, inject requirement source if available (this
# would already be included if we used req directly)
if req.req and req.comes_from:
if isinstance(req.comes_from, str):
comes_from: Optional[str] = req.comes_from
else:
comes_from = req.comes_from.from_path()
if comes_from:
information += f" (from {comes_from})"
if (message, information) != self._previous_requirement_header:
self._previous_requirement_header = (message, information)
logger.info(message, information)
if req.original_link_is_in_wheel_cache:
if req.is_wheel_from_cache:
with indent_log():
logger.info("Using cached %s", req.link.filename)
def _ensure_link_req_src_dir(self, req, parallel_builds):
# type: (InstallRequirement, bool) -> None
def _ensure_link_req_src_dir(
self, req: InstallRequirement, parallel_builds: bool
) -> None:
"""Ensure source_dir of a linked InstallRequirement."""
# Since source_dir is only set for editable requirements.
if req.link.is_wheel:
@@ -359,7 +302,7 @@ class RequirementPreparer:
# directory.
return
assert req.source_dir is None
if req.link.is_existing_dir() and self.in_tree_build:
if req.link.is_existing_dir():
# build local directories in-tree
req.source_dir = req.link.file_path
return
@@ -376,6 +319,7 @@ class RequirementPreparer:
# installation.
# FIXME: this won't upgrade when there's an existing
# package unpacked in `req.source_dir`
# TODO: this check is now probably dead code
if is_installable_dir(req.source_dir):
raise PreviousBuildDirError(
"pip can't proceed with requirements '{}' due to a"
@@ -385,8 +329,7 @@ class RequirementPreparer:
"Please delete it and try again.".format(req, req.source_dir)
)
def _get_linked_req_hashes(self, req):
# type: (InstallRequirement) -> Hashes
def _get_linked_req_hashes(self, req: InstallRequirement) -> Hashes:
# By the time this is called, the requirement's link should have
# been checked so we can tell what kind of requirements req is
# and raise some more informative errors than otherwise.
@@ -418,18 +361,72 @@ class RequirementPreparer:
# showing the user what the hash should be.
return req.hashes(trust_internet=False) or MissingHashes()
def _fetch_metadata_using_lazy_wheel(self, link):
# type: (Link) -> Optional[Distribution]
"""Fetch metadata using lazy wheel, if possible."""
if not self.use_lazy_wheel:
return None
def _fetch_metadata_only(
self,
req: InstallRequirement,
) -> Optional[BaseDistribution]:
if self.require_hashes:
logger.debug('Lazy wheel is not used as hash checking is required')
logger.debug(
"Metadata-only fetching is not used as hash checking is required",
)
return None
# Try PEP 658 metadata first, then fall back to lazy wheel if unavailable.
return self._fetch_metadata_using_link_data_attr(
req
) or self._fetch_metadata_using_lazy_wheel(req.link)
def _fetch_metadata_using_link_data_attr(
self,
req: InstallRequirement,
) -> Optional[BaseDistribution]:
"""Fetch metadata from the data-dist-info-metadata attribute, if possible."""
# (1) Get the link to the metadata file, if provided by the backend.
metadata_link = req.link.metadata_link()
if metadata_link is None:
return None
assert req.req is not None
logger.info(
"Obtaining dependency information for %s from %s",
req.req,
metadata_link,
)
# (2) Download the contents of the METADATA file, separate from the dist itself.
metadata_file = get_http_url(
metadata_link,
self._download,
hashes=metadata_link.as_hashes(),
)
with open(metadata_file.path, "rb") as f:
metadata_contents = f.read()
# (3) Generate a dist just from those file contents.
metadata_dist = get_metadata_distribution(
metadata_contents,
req.link.filename,
req.req.name,
)
# (4) Ensure the Name: field from the METADATA file matches the name from the
# install requirement.
#
# NB: raw_name will fall back to the name from the install requirement if
# the Name: field is not present, but it's noted in the raw_name docstring
# that that should NEVER happen anyway.
if metadata_dist.raw_name != req.req.name:
raise MetadataInconsistent(
req, "Name", req.req.name, metadata_dist.raw_name
)
return metadata_dist
def _fetch_metadata_using_lazy_wheel(
self,
link: Link,
) -> Optional[BaseDistribution]:
"""Fetch metadata using lazy wheel, if possible."""
# --use-feature=fast-deps must be provided.
if not self.use_lazy_wheel:
return None
if link.is_file or not link.is_wheel:
logger.debug(
'Lazy wheel is not used as '
'%r does not points to a remote wheel',
"Lazy wheel is not used as %r does not point to a remote wheel",
link,
)
return None
@@ -437,22 +434,22 @@ class RequirementPreparer:
wheel = Wheel(link.filename)
name = canonicalize_name(wheel.name)
logger.info(
'Obtaining dependency information from %s %s',
name, wheel.version,
"Obtaining dependency information from %s %s",
name,
wheel.version,
)
url = link.url.split('#', 1)[0]
url = link.url.split("#", 1)[0]
try:
return dist_from_wheel_url(name, url, self._session)
except HTTPRangeRequestUnsupported:
logger.debug('%s does not support range requests', url)
logger.debug("%s does not support range requests", url)
return None
def _complete_partial_requirements(
self,
partially_downloaded_reqs, # type: Iterable[InstallRequirement]
parallel_builds=False, # type: bool
):
# type: (...) -> None
partially_downloaded_reqs: Iterable[InstallRequirement],
parallel_builds: bool = False,
) -> None:
"""Download any requirements which were only fetched by metadata."""
# Download to a temporary directory. These will be copied over as
# needed for downstream 'download', 'wheel', and 'install' commands.
@@ -461,7 +458,7 @@ class RequirementPreparer:
# Map each link to the requirement that owns it. This allows us to set
# `req.local_file_path` on the appropriate requirement after passing
# all the links at once into BatchDownloader.
links_to_fully_download = {} # type: Dict[Link, InstallRequirement]
links_to_fully_download: Dict[Link, InstallRequirement] = {}
for req in partially_downloaded_reqs:
assert req.link
links_to_fully_download[req.link] = req
@@ -480,35 +477,47 @@ class RequirementPreparer:
for req in partially_downloaded_reqs:
self._prepare_linked_requirement(req, parallel_builds)
def prepare_linked_requirement(self, req, parallel_builds=False):
# type: (InstallRequirement, bool) -> Distribution
def prepare_linked_requirement(
self, req: InstallRequirement, parallel_builds: bool = False
) -> BaseDistribution:
"""Prepare a requirement to be obtained from req.link."""
assert req.link
link = req.link
self._log_preparing_link(req)
with indent_log():
# Check if the relevant file is already available
# in the download directory
file_path = None
if self.download_dir is not None and link.is_wheel:
if self.download_dir is not None and req.link.is_wheel:
hashes = self._get_linked_req_hashes(req)
file_path = _check_download_dir(req.link, self.download_dir, hashes)
file_path = _check_download_dir(
req.link,
self.download_dir,
hashes,
# When a locally built wheel has been found in cache, we don't warn
# about re-downloading when the already downloaded wheel hash does
# not match. This is because the hash must be checked against the
# original link, not the cached link. It that case the already
# downloaded file will be removed and re-fetched from cache (which
# implies a hash check against the cache entry's origin.json).
warn_on_hash_mismatch=not req.is_wheel_from_cache,
)
if file_path is not None:
# The file is already available, so mark it as downloaded
self._downloaded[req.link.url] = file_path, None
self._downloaded[req.link.url] = file_path
else:
# The file is not available, attempt to fetch only metadata
wheel_dist = self._fetch_metadata_using_lazy_wheel(link)
if wheel_dist is not None:
metadata_dist = self._fetch_metadata_only(req)
if metadata_dist is not None:
req.needs_more_preparation = True
return wheel_dist
return metadata_dist
# None of the optimizations worked, fully prepare the requirement
return self._prepare_linked_requirement(req, parallel_builds)
def prepare_linked_requirements_more(self, reqs, parallel_builds=False):
# type: (Iterable[InstallRequirement], bool) -> None
def prepare_linked_requirements_more(
self, reqs: Iterable[InstallRequirement], parallel_builds: bool = False
) -> None:
"""Prepare linked requirements more, if needed."""
reqs = [req for req in reqs if req.needs_more_preparation]
for req in reqs:
@@ -517,12 +526,12 @@ class RequirementPreparer:
hashes = self._get_linked_req_hashes(req)
file_path = _check_download_dir(req.link, self.download_dir, hashes)
if file_path is not None:
self._downloaded[req.link.url] = file_path, None
self._downloaded[req.link.url] = file_path
req.needs_more_preparation = False
# Prepare requirements we found were already downloaded for some
# reason. The other downloads will be completed separately.
partially_downloaded_reqs = [] # type: List[InstallRequirement]
partially_downloaded_reqs: List[InstallRequirement] = []
for req in reqs:
if req.needs_more_preparation:
partially_downloaded_reqs.append(req)
@@ -532,35 +541,87 @@ class RequirementPreparer:
# TODO: separate this part out from RequirementPreparer when the v1
# resolver can be removed!
self._complete_partial_requirements(
partially_downloaded_reqs, parallel_builds=parallel_builds,
partially_downloaded_reqs,
parallel_builds=parallel_builds,
)
def _prepare_linked_requirement(self, req, parallel_builds):
# type: (InstallRequirement, bool) -> Distribution
def _prepare_linked_requirement(
self, req: InstallRequirement, parallel_builds: bool
) -> BaseDistribution:
assert req.link
link = req.link
self._ensure_link_req_src_dir(req, parallel_builds)
hashes = self._get_linked_req_hashes(req)
if link.is_existing_dir() and self.in_tree_build:
if hashes and req.is_wheel_from_cache:
assert req.download_info is not None
assert link.is_wheel
assert link.is_file
# We need to verify hashes, and we have found the requirement in the cache
# of locally built wheels.
if (
isinstance(req.download_info.info, ArchiveInfo)
and req.download_info.info.hashes
and hashes.has_one_of(req.download_info.info.hashes)
):
# At this point we know the requirement was built from a hashable source
# artifact, and we verified that the cache entry's hash of the original
# artifact matches one of the hashes we expect. We don't verify hashes
# against the cached wheel, because the wheel is not the original.
hashes = None
else:
logger.warning(
"The hashes of the source archive found in cache entry "
"don't match, ignoring cached built wheel "
"and re-downloading source."
)
req.link = req.cached_wheel_source_link
link = req.link
self._ensure_link_req_src_dir(req, parallel_builds)
if link.is_existing_dir():
local_file = None
elif link.url not in self._downloaded:
try:
local_file = unpack_url(
link, req.source_dir, self._download,
self.download_dir, hashes
link,
req.source_dir,
self._download,
self.verbosity,
self.download_dir,
hashes,
)
except NetworkConnectionError as exc:
raise InstallationError(
'Could not install requirement {} because of HTTP '
'error {} for URL {}'.format(req, exc, link)
"Could not install requirement {} because of HTTP "
"error {} for URL {}".format(req, exc, link)
)
else:
file_path, content_type = self._downloaded[link.url]
file_path = self._downloaded[link.url]
if hashes:
hashes.check_against_path(file_path)
local_file = File(file_path, content_type)
local_file = File(file_path, content_type=None)
# If download_info is set, we got it from the wheel cache.
if req.download_info is None:
# Editables don't go through this function (see
# prepare_editable_requirement).
assert not req.editable
req.download_info = direct_url_from_link(link, req.source_dir)
# Make sure we have a hash in download_info. If we got it as part of the
# URL, it will have been verified and we can rely on it. Otherwise we
# compute it from the downloaded file.
# FIXME: https://github.com/pypa/pip/issues/11943
if (
isinstance(req.download_info.info, ArchiveInfo)
and not req.download_info.info.hashes
and local_file
):
hash = hash_file(local_file.path)[0].hexdigest()
# We populate info.hash for backward compatibility.
# This will automatically populate info.hashes.
req.download_info.info.hash = f"sha256={hash}"
# For use in later processing,
# preserve the file path on the requirement.
@@ -568,12 +629,15 @@ class RequirementPreparer:
req.local_file_path = local_file.path
dist = _get_prepared_distribution(
req, self.req_tracker, self.finder, self.build_isolation,
req,
self.build_tracker,
self.finder,
self.build_isolation,
self.check_build_deps,
)
return dist
def save_linked_requirement(self, req):
# type: (InstallRequirement) -> None
def save_linked_requirement(self, req: InstallRequirement) -> None:
assert self.download_dir is not None
assert req.link is not None
link = req.link
@@ -584,8 +648,9 @@ class RequirementPreparer:
if link.is_existing_dir():
logger.debug(
'Not copying link to destination directory '
'since it is a directory: %s', link,
"Not copying link to destination directory "
"since it is a directory: %s",
link,
)
return
if req.local_file_path is None:
@@ -596,31 +661,35 @@ class RequirementPreparer:
if not os.path.exists(download_location):
shutil.copy(req.local_file_path, download_location)
download_path = display_path(download_location)
logger.info('Saved %s', download_path)
logger.info("Saved %s", download_path)
def prepare_editable_requirement(
self,
req, # type: InstallRequirement
):
# type: (...) -> Distribution
"""Prepare an editable requirement
"""
req: InstallRequirement,
) -> BaseDistribution:
"""Prepare an editable requirement."""
assert req.editable, "cannot prepare a non-editable req as editable"
logger.info('Obtaining %s', req)
logger.info("Obtaining %s", req)
with indent_log():
if self.require_hashes:
raise InstallationError(
'The editable requirement {} cannot be installed when '
'requiring hashes, because there is no single file to '
'hash.'.format(req)
"The editable requirement {} cannot be installed when "
"requiring hashes, because there is no single file to "
"hash.".format(req)
)
req.ensure_has_source_dir(self.src_dir)
req.update_editable()
assert req.source_dir
req.download_info = direct_url_for_editable(req.unpacked_source_directory)
dist = _get_prepared_distribution(
req, self.req_tracker, self.finder, self.build_isolation,
req,
self.build_tracker,
self.finder,
self.build_isolation,
self.check_build_deps,
)
req.check_if_exists(self.use_user_site)
@@ -629,27 +698,24 @@ class RequirementPreparer:
def prepare_installed_requirement(
self,
req, # type: InstallRequirement
skip_reason # type: str
):
# type: (...) -> Distribution
"""Prepare an already-installed requirement
"""
req: InstallRequirement,
skip_reason: str,
) -> BaseDistribution:
"""Prepare an already-installed requirement."""
assert req.satisfied_by, "req should have been satisfied but isn't"
assert skip_reason is not None, (
"did not get skip reason skipped but req.satisfied_by "
"is set to {}".format(req.satisfied_by)
)
logger.info(
'Requirement %s: %s (%s)',
skip_reason, req, req.satisfied_by.version
"Requirement %s: %s (%s)", skip_reason, req, req.satisfied_by.version
)
with indent_log():
if self.require_hashes:
logger.debug(
'Since it is already installed, we are trusting this '
'package without checking its hash. To ensure a '
'completely repeatable environment, install into an '
'empty virtualenv.'
"Since it is already installed, we are trusting this "
"package without checking its hash. To ensure a "
"completely repeatable environment, install into an "
"empty virtualenv."
)
return InstalledDistribution(req).get_pkg_resources_distribution()
return InstalledDistribution(req).get_metadata_distribution()