req_file.py 18.6 KB
Newer Older
xuebingbing's avatar
xuebingbing committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
"""
Requirements file parsing
"""

# The following comment should be removed at some point in the future.
# mypy: strict-optional=False

from __future__ import absolute_import

import optparse
import os
import re
import shlex
import sys

from pip._vendor.six.moves.urllib import parse as urllib_parse

from pip._internal.cli import cmdoptions
from pip._internal.exceptions import (
    InstallationError,
    RequirementsFileParseError,
)
from pip._internal.models.search_scope import SearchScope
from pip._internal.utils.encoding import auto_decode
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.urls import get_url_scheme

if MYPY_CHECK_RUNNING:
    from optparse import Values
    from typing import (
xuebingbing's avatar
xuebingbing committed
31
        Any, Callable, Dict, Iterator, List, NoReturn, Optional, Text, Tuple,
xuebingbing's avatar
xuebingbing committed
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
    )

    from pip._internal.index.package_finder import PackageFinder
    from pip._internal.network.session import PipSession

    ReqFileLines = Iterator[Tuple[int, Text]]

    LineParser = Callable[[Text], Tuple[str, Values]]


__all__ = ['parse_requirements']

SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
COMMENT_RE = re.compile(r'(^|\s+)#.*$')

# Matches environment variable-style values in '${MY_VARIABLE_1}' with the
# variable name consisting of only uppercase letters, digits or the '_'
# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
# 2013 Edition.
ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')

SUPPORTED_OPTIONS = [
    cmdoptions.index_url,
    cmdoptions.extra_index_url,
    cmdoptions.no_index,
    cmdoptions.constraints,
    cmdoptions.requirements,
    cmdoptions.editable,
    cmdoptions.find_links,
    cmdoptions.no_binary,
    cmdoptions.only_binary,
    cmdoptions.require_hashes,
    cmdoptions.pre,
    cmdoptions.trusted_host,
    cmdoptions.always_unzip,  # Deprecated
]  # type: List[Callable[..., optparse.Option]]

# options to be passed to requirements
SUPPORTED_OPTIONS_REQ = [
    cmdoptions.install_options,
    cmdoptions.global_options,
    cmdoptions.hash,
]  # type: List[Callable[..., optparse.Option]]

# the 'dest' string values
SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ]


xuebingbing's avatar
xuebingbing committed
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
class ParsedRequirement(object):
    def __init__(
        self,
        requirement,  # type:str
        is_editable,  # type: bool
        comes_from,  # type: str
        constraint,  # type: bool
        options=None,  # type: Optional[Dict[str, Any]]
        line_source=None,  # type: Optional[str]
    ):
        # type: (...) -> None
        self.requirement = requirement
        self.is_editable = is_editable
        self.comes_from = comes_from
        self.options = options
        self.constraint = constraint
        self.line_source = line_source


xuebingbing's avatar
xuebingbing committed
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
class ParsedLine(object):
    def __init__(
        self,
        filename,  # type: str
        lineno,  # type: int
        comes_from,  # type: str
        args,  # type: str
        opts,  # type: Values
        constraint,  # type: bool
    ):
        # type: (...) -> None
        self.filename = filename
        self.lineno = lineno
        self.comes_from = comes_from
        self.opts = opts
        self.constraint = constraint

xuebingbing's avatar
xuebingbing committed
116 117 118 119 120 121 122 123 124 125 126 127
        if args:
            self.is_requirement = True
            self.is_editable = False
            self.requirement = args
        elif opts.editables:
            self.is_requirement = True
            self.is_editable = True
            # We don't support multiple -e on one line
            self.requirement = opts.editables[0]
        else:
            self.is_requirement = False

xuebingbing's avatar
xuebingbing committed
128 129 130 131 132 133 134 135 136

def parse_requirements(
    filename,  # type: str
    session,  # type: PipSession
    finder=None,  # type: Optional[PackageFinder]
    comes_from=None,  # type: Optional[str]
    options=None,  # type: Optional[optparse.Values]
    constraint=False,  # type: bool
):
xuebingbing's avatar
xuebingbing committed
137
    # type: (...) -> Iterator[ParsedRequirement]
xuebingbing's avatar
xuebingbing committed
138 139 140 141 142 143 144 145 146 147 148
    """Parse a requirements file and yield InstallRequirement instances.

    :param filename:    Path or url of requirements file.
    :param session:     PipSession instance.
    :param finder:      Instance of pip.index.PackageFinder.
    :param comes_from:  Origin description of requirements.
    :param options:     cli options.
    :param constraint:  If true, parsing a constraint file rather than
        requirements file.
    """
    line_parser = get_line_parser(finder)
xuebingbing's avatar
xuebingbing committed
149
    parser = RequirementsFileParser(session, line_parser, comes_from)
xuebingbing's avatar
xuebingbing committed
150 151

    for parsed_line in parser.parse(filename, constraint):
xuebingbing's avatar
xuebingbing committed
152 153 154 155 156
        parsed_req = handle_line(
            parsed_line,
            options=options,
            finder=finder,
            session=session
xuebingbing's avatar
xuebingbing committed
157
        )
xuebingbing's avatar
xuebingbing committed
158 159
        if parsed_req is not None:
            yield parsed_req
xuebingbing's avatar
xuebingbing committed
160 161


xuebingbing's avatar
xuebingbing committed
162 163
def preprocess(content):
    # type: (Text) -> ReqFileLines
xuebingbing's avatar
xuebingbing committed
164 165 166 167 168 169 170 171 172 173 174
    """Split, filter, and join lines, and return a line iterator

    :param content: the content of the requirements file
    """
    lines_enum = enumerate(content.splitlines(), start=1)  # type: ReqFileLines
    lines_enum = join_lines(lines_enum)
    lines_enum = ignore_comments(lines_enum)
    lines_enum = expand_env_variables(lines_enum)
    return lines_enum


xuebingbing's avatar
xuebingbing committed
175
def handle_requirement_line(
xuebingbing's avatar
xuebingbing committed
176 177 178
    line,  # type: ParsedLine
    options=None,  # type: Optional[optparse.Values]
):
xuebingbing's avatar
xuebingbing committed
179
    # type: (...) -> ParsedRequirement
xuebingbing's avatar
xuebingbing committed
180 181

    # preserve for the nested code path
xuebingbing's avatar
xuebingbing committed
182
    line_comes_from = '{} {} (line {})'.format(
xuebingbing's avatar
xuebingbing committed
183 184 185
        '-c' if line.constraint else '-r', line.filename, line.lineno,
    )

xuebingbing's avatar
xuebingbing committed
186 187 188 189 190 191 192 193 194 195 196 197
    assert line.is_requirement

    if line.is_editable:
        # For editable requirements, we don't support per-requirement
        # options, so just return the parsed requirement.
        return ParsedRequirement(
            requirement=line.requirement,
            is_editable=line.is_editable,
            comes_from=line_comes_from,
            constraint=line.constraint,
        )
    else:
xuebingbing's avatar
xuebingbing committed
198
        if options:
xuebingbing's avatar
xuebingbing committed
199
            # Disable wheels if the user has specified build options
xuebingbing's avatar
xuebingbing committed
200
            cmdoptions.check_install_build_global(options, line.opts)
xuebingbing's avatar
xuebingbing committed
201

xuebingbing's avatar
xuebingbing committed
202 203 204 205 206
        # get the options that apply to requirements
        req_options = {}
        for dest in SUPPORTED_OPTIONS_REQ_DEST:
            if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
                req_options[dest] = line.opts.__dict__[dest]
xuebingbing's avatar
xuebingbing committed
207

xuebingbing's avatar
xuebingbing committed
208
        line_source = 'line {} of {}'.format(line.lineno, line.filename)
xuebingbing's avatar
xuebingbing committed
209 210 211
        return ParsedRequirement(
            requirement=line.requirement,
            is_editable=line.is_editable,
xuebingbing's avatar
xuebingbing committed
212 213
            comes_from=line_comes_from,
            constraint=line.constraint,
xuebingbing's avatar
xuebingbing committed
214
            options=req_options,
xuebingbing's avatar
xuebingbing committed
215 216 217
            line_source=line_source,
        )

xuebingbing's avatar
xuebingbing committed
218 219 220 221 222 223 224 225 226 227

def handle_option_line(
    opts,  # type: Values
    filename,  # type: str
    lineno,  # type: int
    finder=None,  # type: Optional[PackageFinder]
    options=None,  # type: Optional[optparse.Values]
    session=None,  # type: Optional[PipSession]
):
    # type:  (...) -> None
xuebingbing's avatar
xuebingbing committed
228 229

    # percolate hash-checking option upward
xuebingbing's avatar
xuebingbing committed
230 231
    if opts.require_hashes:
        options.require_hashes = opts.require_hashes
xuebingbing's avatar
xuebingbing committed
232 233 234 235 236

    # set finder options
    elif finder:
        find_links = finder.find_links
        index_urls = finder.index_urls
xuebingbing's avatar
xuebingbing committed
237 238 239
        if opts.index_url:
            index_urls = [opts.index_url]
        if opts.no_index is True:
xuebingbing's avatar
xuebingbing committed
240
            index_urls = []
xuebingbing's avatar
xuebingbing committed
241 242 243
        if opts.extra_index_urls:
            index_urls.extend(opts.extra_index_urls)
        if opts.find_links:
xuebingbing's avatar
xuebingbing committed
244 245 246
            # FIXME: it would be nice to keep track of the source
            # of the find_links: support a find-links local path
            # relative to a requirements file.
xuebingbing's avatar
xuebingbing committed
247 248
            value = opts.find_links[0]
            req_dir = os.path.dirname(os.path.abspath(filename))
xuebingbing's avatar
xuebingbing committed
249 250 251 252 253 254 255 256 257 258 259
            relative_to_reqs_file = os.path.join(req_dir, value)
            if os.path.exists(relative_to_reqs_file):
                value = relative_to_reqs_file
            find_links.append(value)

        search_scope = SearchScope(
            find_links=find_links,
            index_urls=index_urls,
        )
        finder.search_scope = search_scope

xuebingbing's avatar
xuebingbing committed
260
        if opts.pre:
xuebingbing's avatar
xuebingbing committed
261 262 263
            finder.set_allow_all_prereleases()

        if session:
xuebingbing's avatar
xuebingbing committed
264 265
            for host in opts.trusted_hosts or []:
                source = 'line {} of {}'.format(lineno, filename)
xuebingbing's avatar
xuebingbing committed
266 267
                session.add_trusted_host(host, source=source)

xuebingbing's avatar
xuebingbing committed
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311

def handle_line(
    line,  # type: ParsedLine
    options=None,  # type: Optional[optparse.Values]
    finder=None,  # type: Optional[PackageFinder]
    session=None,  # type: Optional[PipSession]
):
    # type: (...) -> Optional[ParsedRequirement]
    """Handle a single parsed requirements line; This can result in
    creating/yielding requirements, or updating the finder.

    :param line:        The parsed line to be processed.
    :param options:     CLI options.
    :param finder:      The finder - updated by non-requirement lines.
    :param session:     The session - updated by non-requirement lines.

    Returns a ParsedRequirement object if the line is a requirement line,
    otherwise returns None.

    For lines that contain requirements, the only options that have an effect
    are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
    requirement. Other options from SUPPORTED_OPTIONS may be present, but are
    ignored.

    For lines that do not contain requirements, the only options that have an
    effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
    be present, but are ignored. These lines may contain multiple options
    (although our docs imply only one is supported), and all our parsed and
    affect the finder.
    """

    if line.is_requirement:
        parsed_req = handle_requirement_line(line, options)
        return parsed_req
    else:
        handle_option_line(
            line.opts,
            line.filename,
            line.lineno,
            finder,
            options,
            session,
        )
        return None
xuebingbing's avatar
xuebingbing committed
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336


class RequirementsFileParser(object):
    def __init__(
        self,
        session,  # type: PipSession
        line_parser,  # type: LineParser
        comes_from,  # type: str
    ):
        # type: (...) -> None
        self._session = session
        self._line_parser = line_parser
        self._comes_from = comes_from

    def parse(self, filename, constraint):
        # type: (str, bool) -> Iterator[ParsedLine]
        """Parse a given file, yielding parsed lines.
        """
        for line in self._parse_and_recurse(filename, constraint):
            yield line

    def _parse_and_recurse(self, filename, constraint):
        # type: (str, bool) -> Iterator[ParsedLine]
        for line in self._parse_file(filename, constraint):
            if (
xuebingbing's avatar
xuebingbing committed
337
                not line.is_requirement and
xuebingbing's avatar
xuebingbing committed
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
                (line.opts.requirements or line.opts.constraints)
            ):
                # parse a nested requirements file
                if line.opts.requirements:
                    req_path = line.opts.requirements[0]
                    nested_constraint = False
                else:
                    req_path = line.opts.constraints[0]
                    nested_constraint = True

                # original file is over http
                if SCHEME_RE.search(filename):
                    # do a url join so relative paths work
                    req_path = urllib_parse.urljoin(filename, req_path)
                # original file and nested file are paths
                elif not SCHEME_RE.search(req_path):
                    # do a join so relative paths work
                    req_path = os.path.join(
                        os.path.dirname(filename), req_path,
                    )

                for inner_line in self._parse_and_recurse(
                    req_path, nested_constraint,
                ):
                    yield inner_line
            else:
                yield line

    def _parse_file(self, filename, constraint):
        # type: (str, bool) -> Iterator[ParsedLine]
        _, content = get_file_content(
            filename, self._session, comes_from=self._comes_from
        )

xuebingbing's avatar
xuebingbing committed
372
        lines_enum = preprocess(content)
xuebingbing's avatar
xuebingbing committed
373 374 375 376 377 378

        for line_number, line in lines_enum:
            try:
                args_str, opts = self._line_parser(line)
            except OptionParsingError as e:
                # add offending line
xuebingbing's avatar
xuebingbing committed
379
                msg = 'Invalid requirement: {}\n{}'.format(line, e.msg)
xuebingbing's avatar
xuebingbing committed
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
                raise RequirementsFileParseError(msg)

            yield ParsedLine(
                filename,
                line_number,
                self._comes_from,
                args_str,
                opts,
                constraint,
            )


def get_line_parser(finder):
    # type: (Optional[PackageFinder]) -> LineParser
    def parse_line(line):
        # type: (Text) -> Tuple[str, Values]
        # Build new parser for each line since it accumulates appendable
        # options.
        parser = build_parser()
        defaults = parser.get_default_values()
        defaults.index_url = None
        if finder:
            defaults.format_control = finder.format_control

        args_str, options_str = break_args_options(line)
        # Prior to 2.7.3, shlex cannot deal with unicode entries
        if sys.version_info < (2, 7, 3):
            # https://github.com/python/mypy/issues/1174
            options_str = options_str.encode('utf8')  # type: ignore

        # https://github.com/python/mypy/issues/1174
        opts, _ = parser.parse_args(
            shlex.split(options_str), defaults)  # type: ignore

        return args_str, opts

    return parse_line


def break_args_options(line):
    # type: (Text) -> Tuple[str, Text]
    """Break up the line into an args and options string.  We only want to shlex
    (and then optparse) the options, not the args.  args can contain markers
    which are corrupted by shlex.
    """
    tokens = line.split(' ')
    args = []
    options = tokens[:]
    for token in tokens:
        if token.startswith('-') or token.startswith('--'):
            break
        else:
            args.append(token)
            options.pop(0)
    return ' '.join(args), ' '.join(options)  # type: ignore


class OptionParsingError(Exception):
    def __init__(self, msg):
        # type: (str) -> None
        self.msg = msg


def build_parser():
    # type: () -> optparse.OptionParser
    """
    Return a parser for parsing requirement lines
    """
    parser = optparse.OptionParser(add_help_option=False)

    option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
    for option_factory in option_factories:
        option = option_factory()
        parser.add_option(option)

    # By default optparse sys.exits on parsing errors. We want to wrap
    # that in our own exception.
    def parser_exit(self, msg):
        # type: (Any, str) -> NoReturn
        raise OptionParsingError(msg)
    # NOTE: mypy disallows assigning to a method
    #       https://github.com/python/mypy/issues/2427
    parser.exit = parser_exit  # type: ignore

    return parser


def join_lines(lines_enum):
    # type: (ReqFileLines) -> ReqFileLines
    """Joins a line ending in '\' with the previous line (except when following
    comments).  The joined line takes on the index of the first line.
    """
    primary_line_number = None
    new_line = []  # type: List[Text]
    for line_number, line in lines_enum:
        if not line.endswith('\\') or COMMENT_RE.match(line):
            if COMMENT_RE.match(line):
                # this ensures comments are always matched later
                line = ' ' + line
            if new_line:
                new_line.append(line)
                yield primary_line_number, ''.join(new_line)
                new_line = []
            else:
                yield line_number, line
        else:
            if not new_line:
                primary_line_number = line_number
            new_line.append(line.strip('\\'))

    # last line contains \
    if new_line:
        yield primary_line_number, ''.join(new_line)

    # TODO: handle space after '\'.


def ignore_comments(lines_enum):
    # type: (ReqFileLines) -> ReqFileLines
    """
    Strips comments and filter empty lines.
    """
    for line_number, line in lines_enum:
        line = COMMENT_RE.sub('', line)
        line = line.strip()
        if line:
            yield line_number, line


def expand_env_variables(lines_enum):
    # type: (ReqFileLines) -> ReqFileLines
    """Replace all environment variables that can be retrieved via `os.getenv`.

    The only allowed format for environment variables defined in the
    requirement file is `${MY_VARIABLE_1}` to ensure two things:

    1. Strings that contain a `$` aren't accidentally (partially) expanded.
    2. Ensure consistency across platforms for requirement files.

    These points are the result of a discussion on the `github pull
    request #3514 <https://github.com/pypa/pip/pull/3514>`_.

    Valid characters in variable names follow the `POSIX standard
    <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
    to uppercase letter, digits and the `_` (underscore).
    """
    for line_number, line in lines_enum:
        for env_var, var_name in ENV_VAR_RE.findall(line):
            value = os.getenv(var_name)
            if not value:
                continue

            line = line.replace(env_var, value)

        yield line_number, line


def get_file_content(url, session, comes_from=None):
    # type: (str, PipSession, Optional[str]) -> Tuple[str, Text]
    """Gets the content of a file; it may be a filename, file: URL, or
    http: URL.  Returns (location, content).  Content is unicode.
    Respects # -*- coding: declarations on the retrieved files.

    :param url:         File path or url.
    :param session:     PipSession instance.
    :param comes_from:  Origin description of requirements.
    """
    scheme = get_url_scheme(url)

    if scheme in ['http', 'https']:
        # FIXME: catch some errors
        resp = session.get(url)
        resp.raise_for_status()
        return resp.url, resp.text

    elif scheme == 'file':
        if comes_from and comes_from.startswith('http'):
            raise InstallationError(
xuebingbing's avatar
xuebingbing committed
558 559 560
                'Requirements file {} references URL {}, '
                'which is local'.format(comes_from, url)
            )
xuebingbing's avatar
xuebingbing committed
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576

        path = url.split(':', 1)[1]
        path = path.replace('\\', '/')
        match = _url_slash_drive_re.match(path)
        if match:
            path = match.group(1) + ':' + path.split('|', 1)[1]
        path = urllib_parse.unquote(path)
        if path.startswith('/'):
            path = '/' + path.lstrip('/')
        url = path

    try:
        with open(url, 'rb') as f:
            content = auto_decode(f.read())
    except IOError as exc:
        raise InstallationError(
xuebingbing's avatar
xuebingbing committed
577
            'Could not open requirements file: {}'.format(exc)
xuebingbing's avatar
xuebingbing committed
578 579 580 581 582
        )
    return url, content


_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)