Server IP : 85.214.239.14 / Your IP : 18.117.232.108 Web Server : Apache/2.4.62 (Debian) System : Linux h2886529.stratoserver.net 4.9.0 #1 SMP Tue Jan 9 19:45:01 MSK 2024 x86_64 User : www-data ( 33) PHP Version : 7.4.18 Disable Function : pcntl_alarm,pcntl_fork,pcntl_waitpid,pcntl_wait,pcntl_wifexited,pcntl_wifstopped,pcntl_wifsignaled,pcntl_wifcontinued,pcntl_wexitstatus,pcntl_wtermsig,pcntl_wstopsig,pcntl_signal,pcntl_signal_get_handler,pcntl_signal_dispatch,pcntl_get_last_error,pcntl_strerror,pcntl_sigprocmask,pcntl_sigwaitinfo,pcntl_sigtimedwait,pcntl_exec,pcntl_getpriority,pcntl_setpriority,pcntl_async_signals,pcntl_unshare, MySQL : OFF | cURL : OFF | WGET : ON | Perl : ON | Python : ON | Sudo : ON | Pkexec : OFF Directory : /proc/2/task/2/root/proc/self/root/proc/2/root/usr/share/dh-python/dhpython/ |
Upload File : |
# Copyright © 2010-2020 Piotr Ożarowski <piotr@debian.org> # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. import email import logging import platform import os import re from functools import partial from os.path import exists, isdir, join from subprocess import PIPE, Popen if __name__ == '__main__': import sys sys.path.append(os.path.abspath(join(os.path.dirname(__file__), '..'))) from dhpython import PKG_PREFIX_MAP, PUBLIC_DIR_RE,\ PYDIST_DIRS, PYDIST_OVERRIDES_FNAMES, PYDIST_DPKG_SEARCH_TPLS from dhpython.markers import ComplexEnvironmentMarker, parse_environment_marker from dhpython.tools import memoize from dhpython.version import get_requested_versions, Version log = logging.getLogger('dhpython') PYDIST_RE = re.compile(r""" (?P<name>[A-Za-z][A-Za-z0-9_.-]*) # Python distribution name \s* (?P<vrange>(?:-?\d\.\d+(?:-(?:\d\.\d+)?)?)?) # version range \s* (?P<dependency>(?:[a-z][^;]*)?) # Debian dependency (?: # optional upstream version -> Debian version translator ;\s* (?P<standard>PEP386)? # PEP-386 mode \s* (?P<rules>(?:s|tr|y).*)? # translator rules )? """, re.VERBOSE) REQUIRES_RE = re.compile(r''' (?P<name>[A-Za-z][A-Za-z0-9_.-]*) # Python distribution name \s* (?P<enabled_extras>(?:\[[^\]]*\])?) # ignored for now \s* \(? # optional parenthesis (?: # optional minimum/maximum version (?P<operator><=?|>=?|==|!=|~=) \s* (?P<version>(\w|[-.*])+) (?: # optional interval minimum/maximum version \s* , \s* (?P<operator2><=?|>=?|==|!=) \s* (?P<version2>(\w|[-.])+) )? )? \)? # optional closing parenthesis \s* (?:; # optional environment markers (?P<environment_marker>.+) )? ''', re.VERBOSE) EXTRA_RE = re.compile(r''' ; \s* extra \s* == \s* (?P<quote>['"]) (?P<section>[a-zA-Z0-9-_.]+) (?P=quote) ''', re.VERBOSE) REQ_SECTIONS_RE = re.compile(r''' ^ \[ (?P<section>[a-zA-Z0-9-_.]+)? \s* (?:: (?P<environment_marker>.+) )? \] \s* $ ''', re.VERBOSE) DEB_VERS_OPS = { '==': '=', '<': '<<', '>': '>>', '~=': '>=', } def validate(fpath): """Check if pydist file looks good.""" with open(fpath, encoding='utf-8') as fp: for line in fp: line = line.strip() if line.startswith('#') or not line: continue if not PYDIST_RE.match(line): log.error('invalid pydist data in file %s: %s', fpath.rsplit('/', 1)[-1], line) return False return True @memoize def load(impl): """Load information about installed Python distributions. :param impl: interpreter implementation, f.e. cpython2, cpython3, pypy :type impl: str """ fname = PYDIST_OVERRIDES_FNAMES.get(impl) if exists(fname): to_check = [fname] # first one! else: to_check = [] dname = PYDIST_DIRS.get(impl) if isdir(dname): to_check.extend(join(dname, i) for i in os.listdir(dname)) fbdir = os.environ.get('DH_PYTHON_DIST', '/usr/share/dh-python/dist/') fbname = join(fbdir, '{}_fallback'.format(impl)) if exists(fbname): # fall back generated at dh-python build time to_check.append(fbname) # last one! result = {} for fpath in to_check: with open(fpath, encoding='utf-8') as fp: for line in fp: line = line.strip() if line.startswith('#') or not line: continue dist = PYDIST_RE.search(line) if not dist: raise Exception('invalid pydist line: %s (in %s)' % (line, fpath)) dist = dist.groupdict() name = safe_name(dist['name']) dist['versions'] = get_requested_versions(impl, dist['vrange']) dist['dependency'] = dist['dependency'].strip() if dist['rules']: dist['rules'] = dist['rules'].split(';') else: dist['rules'] = [] result.setdefault(name, []).append(dist) return result def guess_dependency(impl, req, version=None, bdep=None, accept_upstream_versions=False): bdep = bdep or {} log.debug('trying to find dependency for %s (python=%s)', req, version) if isinstance(version, str): version = Version(version) # some upstreams have weird ideas for distribution name... name, rest = re.compile('([^!><=~ \(\)\[;]+)(.*)').match(req).groups() # TODO: check stdlib and dist-packaged for name.py and name.so files req = safe_name(name) + rest data = load(impl) req_d = REQUIRES_RE.match(req) if not req_d: log.info('please ask dh_python3 author to fix REQUIRES_RE ' 'or your upstream author to fix requires.txt') raise Exception('requirement is not valid: %s' % req) req_d = req_d.groupdict() env_marker_alts = '' if req_d['environment_marker']: action = check_environment_marker_restrictions( req, req_d['environment_marker'], impl) if action is False: return elif action is True: pass else: env_marker_alts = ' ' + action name = req_d['name'] details = data.get(name.lower()) if details: log.debug("dependency: module seems to be installed") for item in details: if version and version not in item.get('versions', version): # rule doesn't match version, try next one continue if not item['dependency']: log.debug("dependency: requirement ignored") return # this requirement should be ignored if item['dependency'].endswith(')'): # no need to translate versions if version is hardcoded in # Debian dependency log.debug("dependency: requirement already has hardcoded version") return item['dependency'] + env_marker_alts if req_d['operator'] == '==' and req_d['version'].endswith('*'): # Translate "== 1.*" to "~= 1.0" req_d['operator'] = '~=' req_d['version'] = req_d['version'].replace('*', '0') log.debug("dependency: translated wildcard version to semver limit") if req_d['version'] and (item['standard'] or item['rules']) and\ req_d['operator'] not in (None, '!='): o = _translate_op(req_d['operator']) v = _translate(req_d['version'], item['rules'], item['standard']) d = "%s (%s %s)%s" % ( item['dependency'], o, v, env_marker_alts) if req_d['version2'] and req_d['operator2'] not in (None,'!='): o2 = _translate_op(req_d['operator2']) v2 = _translate(req_d['version2'], item['rules'], item['standard']) d += ", %s (%s %s)%s" % ( item['dependency'], o2, v2, env_marker_alts) elif req_d['operator'] == '~=': o2 = '<<' v2 = _translate(_max_compatible(req_d['version']), item['rules'], item['standard']) d += ", %s (%s %s)%s" % ( item['dependency'], o2, v2, env_marker_alts) log.debug("dependency: constructed version") return d elif accept_upstream_versions and req_d['version'] and \ req_d['operator'] not in (None,'!='): o = _translate_op(req_d['operator']) d = "%s (%s %s)%s" % ( item['dependency'], o, req_d['version'], env_marker_alts) if req_d['version2'] and req_d['operator2'] not in (None,'!='): o2 = _translate_op(req_d['operator2']) d += ", %s (%s %s)%s" % ( item['dependency'], o2, req_d['version2'], env_marker_alts) elif req_d['operator'] == '~=': o2 = '<<' d += ", %s (%s %s)%s" % ( item['dependency'], o2, _max_compatible(req_d['version']), env_marker_alts) log.debug("dependency: constructed upstream version") return d else: if item['dependency'] in bdep: if None in bdep[item['dependency']] and bdep[item['dependency']][None]: log.debug("dependency: included in build-deps with limits ") return "{} ({}){}".format( item['dependency'], bdep[item['dependency']][None], env_marker_alts) # if arch in bdep[item['dependency']]: # TODO: handle architecture specific dependencies from build depends # (current architecture is needed here) log.debug("dependency: included in build-deps") return item['dependency'] + env_marker_alts # search for Egg metadata file or directory (using dpkg -S) dpkg_query_tpl, regex_filter = PYDIST_DPKG_SEARCH_TPLS[impl] dpkg_query = dpkg_query_tpl.format(ci_regexp(safe_name(name))) log.debug("invoking dpkg -S %s", dpkg_query) process = Popen(('/usr/bin/dpkg', '-S', dpkg_query), stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() if process.returncode == 0: result = set() stdout = str(stdout, 'utf-8') for line in stdout.split('\n'): if not line.strip(): continue pkg, path = line.split(':', 1) if regex_filter and not re.search(regex_filter, path): continue result.add(pkg) if len(result) > 1: log.error('more than one package name found for %s dist', name) elif not result: log.debug('dpkg -S did not find package for %s', name) else: log.debug('dependency: found a result with dpkg -S') return result.pop() + env_marker_alts else: log.debug('dpkg -S did not find package for %s: %s', name, stderr) pname = sensible_pname(impl, name) log.info('Cannot find package that provides %s. ' 'Please add package that provides it to Build-Depends or ' 'add "%s %s" line to %s or add proper ' 'dependency to Depends by hand and ignore this info.', name, safe_name(name), pname, PYDIST_OVERRIDES_FNAMES[impl]) # return pname def check_environment_marker_restrictions(req, marker_str, impl): """Check wither we should include or skip a dependency based on its environment markers. Returns: True - to keep a dependency False - to skip it str - to append "| foo" to generated dependencies """ if impl != 'cpython3': log.info('Ignoring environment markers for non-Python 3.x: %s', req) return False try: marker, op, value = parse_environment_marker(marker_str) except ComplexEnvironmentMarker: log.info('Ignoring complex environment marker: %s', req) return False # TODO: Use dynamic values when building arch-dependent # binaries, otherwise static values # TODO: Hurd values? supported_values = { 'implementation_name': ('cpython', 'pypy'), 'os_name': ('posix',), 'platform_system': ('GNU/kFreeBSD', 'Linux'), 'platform_machine': (platform.machine(),), 'platform_python_implementation': ('CPython', 'PyPy'), 'sys_platform': ( 'gnukfreebsd8', 'gnukfreebsd9', 'gnukfreebsd10', 'gnukfreebsd11', 'gnukfreebsd12', 'gnukfreebsd13', 'linux'), } if marker in supported_values: sv = supported_values[marker] if op in ('==', '!='): if ((op == '==' and value not in sv) or (op == '!=' and value in sv)): log.debug('Skipping requirement (%s != %s): %s', value, sv, req) return False else: log.info( 'Skipping requirement with unhandled environment marker ' 'comparison: %s', req) return False elif marker in ('python_version', 'python_full_version', 'implementation_version'): # TODO: Replace with full PEP-440 parser env_ver = value split_ver = value.split('.') if marker == 'python_version': version_parts = 2 elif marker == 'python_full_version': version_parts = 3 else: version_parts = len(split_ver) if '*' in env_ver: if split_ver.index('*') != len(split_ver) -1: log.info('Skipping requirement with intermediate wildcard: %s', req) return False split_ver.pop() env_ver = '.'.join(split_ver) if op == '==': if marker == 'python_full_version': marker = 'python_version' version_parts = 2 else: op == '=~' elif op == '!=': if marker == 'python_full_version': marker = 'python_version' version_parts = 2 else: log.info('Ignoring wildcard != requirement, not ' 'representable in Debian: %s', req) return True else: log.info('Skipping requirement with %s on a wildcard: %s', op, req) return False int_ver = [] for ver_part in split_ver: if ver_part.isdigit(): int_ver.append(int(ver_part)) else: env_ver = '.'.join(str(x) for x in int_ver) log.info('Truncating unparseable version %s to %s in %s', value, env_ver, req) break if len(int_ver) < version_parts: int_ver.append(0) env_ver += '.0' next_ver = int_ver.copy() next_ver[version_parts - 1] += 1 next_ver = '.'.join(str(x) for x in next_ver) prev_ver = int_ver.copy() prev_ver[version_parts - 1] -= 1 prev_ver = '.'.join(str(x) for x in prev_ver) if op == '<': if int_ver <= [3, 0, 0]: return False return '| python3 (>> {})'.format(env_ver) elif op == '<=': return '| python3 (>> {})'.format(next_ver) elif op == '>=': if int_ver < [3, 0, 0]: return True return '| python3 (<< {})'.format(env_ver) elif op == '>': if int_ver < [3, 0, 0]: return True return '| python3 (<< {})'.format(next_ver) elif op in ('==', '==='): # === is arbitrary equality (PEP 440) if marker == 'python_version' or op == '==': return '| python3 (<< {}) | python3 (>> {})'.format( env_ver, next_ver) else: log.info( 'Skipping requirement with %s environment marker, cannot ' 'model in Debian deps: %s', op, req) return False elif op == '~=': # Compatible equality (PEP 440) ceq_next_ver = int_ver[:2] ceq_next_ver[1] += 1 ceq_next_ver = '.'.join(str(x) for x in ceq_next_ver) return '| python3 (<< {}) | python3 (>> {})'.format( env_ver, ceq_next_ver) elif op == '!=': log.info('Ignoring != comparison in environment marker, cannot ' 'model in Debian deps: %s', req) return True elif marker == 'extra': # Handled in section logic of parse_requires_dist() return True else: log.info('Skipping requirement with unknown environment marker: %s', marker) return False return True def parse_pydep(impl, fname, bdep=None, options=None, depends_sec=None, recommends_sec=None, suggests_sec=None): depends_sec = depends_sec or [] recommends_sec = recommends_sec or [] suggests_sec = suggests_sec or [] public_dir = PUBLIC_DIR_RE[impl].match(fname) ver = None if public_dir and public_dir.groups() and len(public_dir.group(1)) != 1: ver = public_dir.group(1) guess_deps = partial(guess_dependency, impl=impl, version=ver, bdep=bdep, accept_upstream_versions=getattr( options, 'accept_upstream_versions', False)) result = {'depends': [], 'recommends': [], 'suggests': []} modified = section = False env_action = True processed = [] with open(fname, 'r', encoding='utf-8') as fp: for line in fp: line = line.strip() if not line or line.startswith('#'): processed.append(line) continue if line.startswith('['): m = REQ_SECTIONS_RE.match(line) if not m: log.info('Skipping section %s, unable to parse header', line) processed.append(line) section = object() continue section = m.group('section') env_action = True if m.group('environment_marker'): env_action = check_environment_marker_restrictions( line, m.group('environment_marker'), impl) processed.append(line) continue if section: if section in depends_sec: result_key = 'depends' elif section in recommends_sec: result_key = 'recommends' elif section in suggests_sec: result_key = 'suggests' else: processed.append(line) continue else: result_key = 'depends' dependency = None if env_action: dependency = guess_deps(req=line) if dependency and isinstance(env_action, str): dependency = ', '.join( part.strip() + ' ' + env_action for part in dependency.split(',')) if dependency: result[result_key].append(dependency) modified = True else: processed.append(line) if modified and public_dir: with open(fname, 'w', encoding='utf-8') as fp: fp.writelines(i + '\n' for i in processed) return result def parse_requires_dist(impl, fname, bdep=None, options=None, depends_sec=None, recommends_sec=None, suggests_sec=None): """Extract dependencies from a dist-info/METADATA file""" depends_sec = depends_sec or [] recommends_sec = recommends_sec or [] suggests_sec = suggests_sec or [] public_dir = PUBLIC_DIR_RE[impl].match(fname) ver = None if public_dir and public_dir.groups() and len(public_dir.group(1)) != 1: ver = public_dir.group(1) guess_deps = partial(guess_dependency, impl=impl, version=ver, bdep=bdep, accept_upstream_versions=getattr( options, 'accept_upstream_versions', False)) result = {'depends': [], 'recommends': [], 'suggests': []} section = None with open(fname, 'r', encoding='utf-8') as fp: metadata = email.message_from_string(fp.read()) requires = metadata.get_all('Requires-Dist', []) for req in requires: m = EXTRA_RE.search(req) result_key = 'depends' if m: section = m.group('section') if section: if section in depends_sec: result_key = 'depends' elif section in recommends_sec: result_key = 'recommends' elif section in suggests_sec: result_key = 'suggests' else: continue dependency = guess_deps(req=req) if dependency: result[result_key].append(dependency) return result def safe_name(name): """Emulate distribute's safe_name.""" return re.compile('[^A-Za-z0-9.]+').sub('_', name).lower() def sensible_pname(impl, egg_name): """Guess Debian package name from Egg name.""" egg_name = safe_name(egg_name).replace('_', '-') if egg_name.startswith('python-'): egg_name = egg_name[7:] return '{}-{}'.format(PKG_PREFIX_MAP[impl], egg_name.lower()) def ci_regexp(name): """Return case insensitive dpkg -S regexp.""" return ''.join("[%s%s]" % (i.upper(), i) if i.isalpha() else i for i in name.lower()) PRE_VER_RE = re.compile(r'[-.]?(alpha|beta|rc|dev|a|b|c)') GROUP_RE = re.compile(r'\$(\d+)') def _pl2py(pattern): """Convert Perl RE patterns used in uscan to Python's >>> print(_pl2py('foo$3')) foo\g<3> """ return GROUP_RE.sub(r'\\g<\1>', pattern) def _max_compatible(version): """Return the maximum version compatible with `version` in PEP440 terms, used by ~= requires version specifiers. https://www.python.org/dev/peps/pep-0440/#compatible-release >>> _max_compatible('2.2') '3' >>> _max_compatible('1.4.5') '1.5' >>> _max_compatible('1.3.alpha4') '2' >>> _max_compatible('2.1.3.post5') '2.2' """ v = Version(version) v.serial = None v.releaselevel = None if v.micro is not None: v.micro = None return str(v + 1) v.minor = None return str(v + 1) def _translate(version, rules, standard): """Translate Python version into Debian one. >>> _translate('1.C2betac', ['s/c//gi'], None) '1.2beta' >>> _translate('5-fooa1.2beta3-fooD', ... ['s/^/1:/', 's/-foo//g', 's:([A-Z]):+$1:'], 'PEP386') '1:5~a1.2~beta3+D' >>> _translate('x.y.x.z', ['tr/xy/ab/', 'y,z,Z,'], None) 'a.b.a.Z' """ for rule in rules: # uscan supports s, tr and y operations if rule.startswith(('tr', 'y')): # Note: no support for escaped separator in the pattern pos = 1 if rule.startswith('y') else 2 tmp = rule[pos + 1:].split(rule[pos]) version = version.translate(str.maketrans(tmp[0], tmp[1])) elif rule.startswith('s'): # uscan supports: g, u and x flags tmp = rule[2:].split(rule[1]) pattern = re.compile(tmp[0]) count = 1 if tmp[2:]: flags = tmp[2] if 'g' in flags: count = 0 if 'i' in flags: pattern = re.compile(tmp[0], re.I) version = pattern.sub(_pl2py(tmp[1]), version, count) else: log.warn('unknown rule ignored: %s', rule) if standard == 'PEP386': version = PRE_VER_RE.sub(r'~\g<1>', version) return version def _translate_op(operator): """Translate Python version operator into Debian one. >>> _translate_op('==') '=' >>> _translate_op('<') '<<' >>> _translate_op('<=') '<=' """ return DEB_VERS_OPS.get(operator, operator) if __name__ == '__main__': impl = os.environ.get('IMPL', 'cpython3') for i in sys.argv[1:]: if os.path.isfile(i): try: print(', '.join(parse_pydep(impl, i)['depends'])) except Exception as err: log.error('%s: cannot guess (%s)', i, err) else: try: print(guess_dependency(impl, i) or '') except Exception as err: log.error('%s: cannot guess (%s)', i, err)