469 lines
15 KiB
Python
469 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# SPDX-FileCopyrightText: Florian Bruhin (The Compiler) <mail@qutebrowser.org>
|
|
#
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
"""Script to regenerate requirements files in misc/requirements."""
|
|
|
|
import re
|
|
import sys
|
|
import os.path
|
|
import glob
|
|
import json
|
|
import subprocess
|
|
import tempfile
|
|
import argparse
|
|
import shutil
|
|
import pathlib
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
|
|
os.pardir))
|
|
|
|
from scripts import utils
|
|
|
|
REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
'..', '..') # /scripts/dev -> /scripts -> /
|
|
REQ_DIR = os.path.join(REPO_DIR, 'misc', 'requirements')
|
|
|
|
CHANGELOG_URLS_PATH = pathlib.Path(__file__).parent / "changelog_urls.json"
|
|
CHANGELOG_URLS = json.loads(CHANGELOG_URLS_PATH.read_text())
|
|
|
|
|
|
def convert_line(line, comments):
|
|
"""Convert the given requirement line to place into the output."""
|
|
for pattern, repl in comments['replace'].items():
|
|
line = re.sub(pattern, repl, line)
|
|
|
|
pkgname = line.split('=')[0]
|
|
|
|
if pkgname in comments['ignore']:
|
|
line = '# ' + line
|
|
|
|
try:
|
|
line += ' # ' + comments['comment'][pkgname]
|
|
except KeyError:
|
|
pass
|
|
|
|
try:
|
|
line += ' # rq.filter: {}'.format(comments['filter'][pkgname])
|
|
except KeyError:
|
|
pass
|
|
|
|
try:
|
|
line += ' ; {}'.format(comments['markers'][pkgname])
|
|
except KeyError:
|
|
pass
|
|
|
|
return line
|
|
|
|
|
|
def read_comments(fobj):
|
|
"""Find special comments in the config.
|
|
|
|
Args:
|
|
fobj: A file object for the config.
|
|
|
|
Return:
|
|
A dict with the parsed comment data.
|
|
"""
|
|
comments = {
|
|
'filter': {},
|
|
'markers': {},
|
|
'comment': {},
|
|
'ignore': [],
|
|
'add': [],
|
|
'replace': {},
|
|
'pre': False,
|
|
'pip_args': [],
|
|
}
|
|
for line in fobj:
|
|
if line.startswith('#@'):
|
|
if ':' in line:
|
|
command, args = line[2:].split(':', maxsplit=1)
|
|
command = command.strip()
|
|
args = args.strip()
|
|
else:
|
|
command = line[2:].strip()
|
|
args = None
|
|
|
|
if command == 'filter':
|
|
pkg, filt = args.split(' ', maxsplit=1)
|
|
comments['filter'][pkg] = filt
|
|
elif command == 'comment':
|
|
pkg, comment = args.split(' ', maxsplit=1)
|
|
comments['comment'][pkg] = comment
|
|
elif command == 'ignore':
|
|
comments['ignore'] += args.split(', ')
|
|
elif command == 'replace':
|
|
pattern, replacement = args.split(' ', maxsplit=1)
|
|
comments['replace'][pattern] = replacement
|
|
elif command == 'markers':
|
|
pkg, markers = args.split(' ', maxsplit=1)
|
|
comments['markers'][pkg] = markers
|
|
elif command == 'add':
|
|
comments['add'].append(args)
|
|
elif command == 'pre':
|
|
comments['pre'] = True
|
|
elif command == 'pip_args':
|
|
comments['pip_args'] += args.split()
|
|
return comments
|
|
|
|
|
|
def get_all_names():
|
|
"""Get all requirement names based on filenames."""
|
|
for filename in glob.glob(os.path.join(REQ_DIR, 'requirements-*.txt-raw')):
|
|
basename = os.path.basename(filename)
|
|
yield basename.removeprefix('requirements-').removesuffix('.txt-raw')
|
|
|
|
|
|
def run_pip(venv_dir, *args, quiet=False, **kwargs):
|
|
"""Run pip inside the virtualenv."""
|
|
args = list(args)
|
|
if quiet:
|
|
args.insert(1, '-q')
|
|
|
|
arg_str = ' '.join(str(arg) for arg in args)
|
|
utils.print_col('venv$ pip {}'.format(arg_str), 'blue')
|
|
|
|
venv_python = get_venv_python(venv_dir)
|
|
return subprocess.run([venv_python, '-m', 'pip'] + args, check=True, **kwargs)
|
|
|
|
|
|
def init_venv(venv_dir, requirements, pre=False, pip_args=None):
|
|
"""Initialize a new virtualenv and install the given packages."""
|
|
with utils.gha_group('Creating virtualenv'):
|
|
utils.print_col('$ python3 -m venv {}'.format(venv_dir), 'blue')
|
|
subprocess.run([sys.executable, '-m', 'venv', venv_dir], check=True)
|
|
|
|
run_pip(venv_dir, 'install', '-U', 'pip', quiet=not utils.ON_CI)
|
|
run_pip(venv_dir, 'install', '-U', 'setuptools', 'wheel', quiet=not utils.ON_CI)
|
|
|
|
install_command = ['install', '-r', requirements]
|
|
if pre:
|
|
install_command.append('--pre')
|
|
if pip_args:
|
|
install_command += pip_args
|
|
|
|
with utils.gha_group('Installing requirements'):
|
|
run_pip(venv_dir, *install_command)
|
|
run_pip(venv_dir, 'check')
|
|
|
|
|
|
def parse_args():
|
|
"""Parse commandline arguments via argparse."""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--force-test', help="Force running environment tests",
|
|
action='store_true')
|
|
parser.add_argument('names', nargs='*')
|
|
return parser.parse_args()
|
|
|
|
|
|
def git_diff(*args):
|
|
"""Run a git diff command."""
|
|
command = (
|
|
["git", "--no-pager", "-c", "diff.mnemonicPrefix=false", "diff"]
|
|
+ list(args)
|
|
+ ["--", "requirements.txt", "misc/requirements/requirements-*.txt"]
|
|
)
|
|
proc = subprocess.run(command,
|
|
stdout=subprocess.PIPE,
|
|
encoding='utf-8',
|
|
check=True)
|
|
return proc.stdout.splitlines()
|
|
|
|
|
|
class Change:
|
|
|
|
"""A single requirements change from a git diff output."""
|
|
|
|
def __init__(self, name: str, base_path: pathlib.Path) -> None:
|
|
self.name = name
|
|
self.old = None
|
|
self.new = None
|
|
self.base = extract_requirement_name(base_path)
|
|
if CHANGELOG_URLS.get(name):
|
|
self.url = CHANGELOG_URLS[name]
|
|
self.link = '[{}]({})'.format(self.name, self.url)
|
|
else:
|
|
self.url = '(no changelog)'
|
|
self.link = self.name
|
|
|
|
def __lt__(self, other):
|
|
return (self.base, self.name.lower()) < (other.base, other.name.lower())
|
|
|
|
def __str__(self):
|
|
prefix = f"- [{self.base}] {self.name}"
|
|
suffix = f" {self.url}"
|
|
if self.old is None:
|
|
return f"{prefix} new: {self.new} {suffix}"
|
|
elif self.new is None:
|
|
return f"{prefix} removed: {self.old} {suffix}"
|
|
else:
|
|
return f"{prefix} {self.old} -> {self.new} {suffix}"
|
|
|
|
def table_str(self):
|
|
"""Generate a markdown table."""
|
|
if self.old is None:
|
|
return f'| {self.base} | {self.link} | -- | {self.new} |'
|
|
elif self.new is None:
|
|
return f'| {self.base} | {self.link} | {self.old} | -- |'
|
|
else:
|
|
return f'| {self.base} | {self.link} | {self.old} | {self.new} |'
|
|
|
|
|
|
def _get_changed_files():
|
|
"""Get a list of changed files via git."""
|
|
changed_files = set()
|
|
filenames = git_diff('--name-only')
|
|
for filename in filenames:
|
|
requirement_name = extract_requirement_name(pathlib.Path(filename))
|
|
changed_files.add(requirement_name)
|
|
|
|
return sorted(changed_files)
|
|
|
|
|
|
def extract_requirement_name(path: pathlib.Path) -> str:
|
|
"""Get a requirement name from a file path.
|
|
|
|
e.g. "pylint" from "misc/requirements/requirements-pylint.txt"
|
|
"""
|
|
if path == pathlib.Path("requirements.txt"):
|
|
return "qutebrowser"
|
|
|
|
prefix = "requirements-"
|
|
assert path.suffix == ".txt", path
|
|
assert path.stem.startswith(prefix), path
|
|
return path.stem.removeprefix(prefix)
|
|
|
|
|
|
def parse_versioned_line(line):
|
|
"""Parse a requirements.txt line into name/version."""
|
|
if line[0] == '#': # ignored dependency
|
|
line = line[1:].strip()
|
|
|
|
# Strip comments and pip environment markers
|
|
line = line.rsplit('#', maxsplit=1)[0]
|
|
line = line.split(';')[0].strip()
|
|
|
|
ops = ["==", "~=", "!=", ">", "<", ">=", "<="]
|
|
|
|
if any(op in line for op in ops):
|
|
# strictly speaking, this version isn't necessarily correct, but it's
|
|
# enough for the table.
|
|
for op in ops:
|
|
if op in line:
|
|
name, version = line.split(op)
|
|
elif line.startswith('-e'):
|
|
rest, name = line.split('#egg=')
|
|
version = rest.split('@')[1][:7]
|
|
else:
|
|
name = line
|
|
version = '?'
|
|
|
|
if name.startswith('#'): # duplicate requirements
|
|
name = name[1:].strip()
|
|
|
|
return name, version
|
|
|
|
|
|
def _get_changes(diff):
|
|
"""Get a list of changed versions from git."""
|
|
changes_dict = {}
|
|
current_path = None
|
|
|
|
for line in diff:
|
|
if not line.startswith('-') and not line.startswith('+'):
|
|
continue
|
|
elif line.startswith('--- '):
|
|
prefix = '--- a/'
|
|
current_path = pathlib.Path(line.removeprefix(prefix))
|
|
continue
|
|
elif line.startswith('+++ '):
|
|
prefix = '+++ b/'
|
|
new_path = pathlib.Path(line.removeprefix(prefix))
|
|
assert current_path == new_path, (current_path, new_path)
|
|
continue
|
|
elif not line.strip():
|
|
# Could be newline changes on Windows
|
|
continue
|
|
elif line[1:].startswith('# This file is automatically'):
|
|
# Could be newline changes on Windows
|
|
continue
|
|
|
|
name, version = parse_versioned_line(line[1:])
|
|
|
|
if name not in changes_dict:
|
|
changes_dict[name] = Change(name, base_path=current_path)
|
|
|
|
if line.startswith('-'):
|
|
changes_dict[name].old = version
|
|
elif line.startswith('+'):
|
|
changes_dict[name].new = version
|
|
|
|
return sorted(changes_dict.values())
|
|
|
|
|
|
def print_changed_files():
|
|
"""Output all changed files from this run."""
|
|
diff = git_diff()
|
|
if utils.ON_CI:
|
|
with utils.gha_group('Raw diff'):
|
|
print('\n'.join(diff))
|
|
|
|
changed_files = _get_changed_files()
|
|
files_text = '\n'.join('- ' + line for line in changed_files)
|
|
|
|
changes = _get_changes(diff)
|
|
changes_text = '\n'.join(str(change) for change in changes)
|
|
|
|
utils.print_subtitle('Files')
|
|
print(files_text)
|
|
print()
|
|
utils.print_subtitle('Changes')
|
|
print(changes_text)
|
|
|
|
if utils.ON_CI:
|
|
print()
|
|
print('::set-output name=changed::' +
|
|
files_text.replace('\n', '%0A'))
|
|
table_header = [
|
|
'| File | Requirement | old | new |',
|
|
'|------|-------------|-----|-----|',
|
|
]
|
|
diff_table = '%0A'.join(table_header +
|
|
[change.table_str() for change in changes])
|
|
print('::set-output name=diff::' + diff_table)
|
|
|
|
|
|
def get_venv_python(venv_dir):
|
|
"""Get the path to Python inside a virtualenv."""
|
|
subdir = 'Scripts' if os.name == 'nt' else 'bin'
|
|
return os.path.join(venv_dir, subdir, 'python')
|
|
|
|
|
|
def get_outfile(name):
|
|
"""Get the path to the output requirements.txt file."""
|
|
if name == 'qutebrowser':
|
|
return os.path.join(REPO_DIR, 'requirements.txt')
|
|
return os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
|
|
|
|
|
|
def build_requirements(name):
|
|
"""Build a requirements file."""
|
|
utils.print_subtitle("Building")
|
|
filename = os.path.join(REQ_DIR, 'requirements-{}.txt-raw'.format(name))
|
|
|
|
with open(filename, 'r', encoding='utf-8') as f:
|
|
comments = read_comments(f)
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
init_venv(venv_dir=tmpdir,
|
|
requirements=filename,
|
|
pre=comments['pre'],
|
|
pip_args=comments['pip_args'])
|
|
with utils.gha_group('Freezing requirements'):
|
|
args = ['--all'] if name == 'tox' else []
|
|
proc = run_pip(tmpdir, 'freeze', *args, stdout=subprocess.PIPE)
|
|
reqs = proc.stdout.decode('utf-8')
|
|
if utils.ON_CI:
|
|
print(reqs.strip())
|
|
|
|
outfile = get_outfile(name)
|
|
|
|
with open(outfile, 'w', encoding='utf-8') as f:
|
|
f.write("# This file is automatically generated by "
|
|
"scripts/dev/recompile_requirements.py\n\n")
|
|
for line in reqs.splitlines():
|
|
if line.startswith('qutebrowser=='):
|
|
continue
|
|
f.write(convert_line(line, comments) + '\n')
|
|
|
|
for line in comments['add']:
|
|
f.write(line + '\n')
|
|
|
|
return outfile
|
|
|
|
|
|
def test_tox():
|
|
"""Test requirements via tox."""
|
|
req_path = os.path.join(REQ_DIR, 'requirements-tox.txt')
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
venv_dir = os.path.join(tmpdir, 'venv')
|
|
tox_workdir = os.path.join(tmpdir, 'tox-workdir')
|
|
venv_python = get_venv_python(venv_dir)
|
|
init_venv(venv_dir, req_path)
|
|
list_proc = subprocess.run([venv_python, '-m', 'tox', '--listenvs'],
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
text=True)
|
|
environments = list_proc.stdout.strip().split('\n')
|
|
for env in environments:
|
|
with utils.gha_group('tox for {}'.format(env)):
|
|
utils.print_subtitle(env)
|
|
utils.print_col('venv$ tox -e {} --notest'.format(env), 'blue')
|
|
subprocess.run([venv_python, '-m', 'tox',
|
|
'--workdir', tox_workdir,
|
|
'-e', env,
|
|
'--notest'],
|
|
check=True)
|
|
|
|
|
|
def install_requirements(name, outfile, *, force=False):
|
|
"""Test install a resulting requirements file."""
|
|
print()
|
|
utils.print_subtitle("Testing")
|
|
|
|
if name not in _get_changed_files() and not force:
|
|
print(f"Skipping test as there were no changes for {name}.")
|
|
return
|
|
|
|
in_file = os.path.join(REQ_DIR, 'requirements-{}.txt-raw'.format(name))
|
|
with open(in_file, 'r', encoding='utf-8') as f:
|
|
comments = read_comments(f)
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
init_venv(tmpdir, outfile, pip_args=comments['pip_args'])
|
|
|
|
|
|
def cleanup_pylint_build():
|
|
"""Clean up pylint_checkers build files."""
|
|
path = pathlib.Path(__file__).parent / 'pylint_checkers' / 'build'
|
|
utils.print_col(f'$ rm -r {path}', 'blue')
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def main():
|
|
"""Re-compile the given (or all) requirement files."""
|
|
args = parse_args()
|
|
if args.names:
|
|
names = args.names
|
|
else:
|
|
names = sorted(get_all_names())
|
|
|
|
utils.print_col('Rebuilding requirements: ' + ', '.join(names), 'green')
|
|
for name in names:
|
|
utils.print_title(name)
|
|
outfile = build_requirements(name)
|
|
install_requirements(name, outfile, force=args.force_test)
|
|
if name == 'pylint':
|
|
cleanup_pylint_build()
|
|
|
|
utils.print_title('Testing via tox')
|
|
if args.names and not args.force_test:
|
|
# If we selected a subset, let's not go through the trouble of testing
|
|
# via tox.
|
|
print("Skipping: Selected a subset only")
|
|
elif not _get_changed_files() and not args.force_test:
|
|
print("Skipping: No changes")
|
|
else:
|
|
test_tox()
|
|
|
|
utils.print_title('Changed')
|
|
print_changed_files()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|