mirror
/
qutebrowser
mirror of https://github.com/qutebrowser/qutebrowser.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update/enforce changelog URLs

This commit is contained in:
Florian Bruhin 2020-11-09 12:39:23 +01:00
parent a0656b78da
commit ad2067bf54
2 changed files with 133 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
scripts/dev/misc_checks.py
View File

@ -31,10 +31,11 @@ import traceback
import pathlib
from typing import List, Iterator, Optional
sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
os.pardir))
REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO_ROOT))
from scripts import utils
from scripts.dev import recompile_requirements
BINARY_EXTS = {'.png', '.icns', '.ico', '.bmp', '.gz', '.bin', '.pdf',
'.sqlite', '.woff2', '.whl'}
@ -77,8 +78,46 @@ def _get_files(
yield path
def check_changelog_urls(_args: argparse.Namespace = None) -> bool:
"""Ensure we have changelog URLs for all requirements."""
ok = True
all_requirements = set()
for name in recompile_requirements.get_all_names():
outfile = recompile_requirements.get_outfile(name)
missing = set()
with open(outfile, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line.startswith('#') or not line:
continue
req, _version = recompile_requirements.parse_versioned_line(line)
if req.startswith('./'):
continue
all_requirements.add(req)
if req not in recompile_requirements.CHANGELOG_URLS:
missing.add(req)
if missing:
ok = False
req_str = ', '.join(sorted(missing))
utils.print_col(
f"Missing changelog URLs in {name} requirements: {req_str}", 'red')
extra = set(recompile_requirements.CHANGELOG_URLS) - all_requirements
if extra:
ok = False
req_str = ', '.join(sorted(extra))
utils.print_col(f"Extra changelog URLs: {req_str}", 'red')
if not ok:
print("Hint: Changelog URLs are in scripts/dev/recompile_requirements.py")
return ok
def check_git(_args: argparse.Namespace = None) -> bool:
"""Check for uncommitted git files.."""
"""Check for uncommitted git files."""
if not os.path.isdir(".git"):
print("No .git dir, ignoring")
print()
@ -241,6 +280,7 @@ def main() -> int:
'vcs': check_vcs_conflict,
'spelling': check_spelling,
'userscripts': check_userscripts_descriptions,
'changelog-urls': check_changelog_urls,
}
parser = argparse.ArgumentParser()

129
scripts/dev/recompile_requirements.py
View File

@ -39,57 +39,89 @@ REQ_DIR = os.path.join(REPO_DIR, 'misc', 'requirements')
CHANGELOG_URLS = {
'pyparsing': 'https://github.com/pyparsing/pyparsing/blob/master/CHANGES',
'cherrypy': 'https://github.com/cherrypy/cherrypy/blob/master/CHANGES.rst',
'pylint': 'http://pylint.pycqa.org/en/latest/whatsnew/changelog.html',
'setuptools': 'https://github.com/pypa/setuptools/blob/master/CHANGES.rst',
'isort': 'https://pycqa.github.io/isort/CHANGELOG/',
'lazy-object-proxy': 'https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst',
'mccabe': 'https://github.com/PyCQA/mccabe#changes',
'pytest-cov': 'https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst',
'pytest-xdist': 'https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst',
'pytest-forked': 'https://github.com/pytest-dev/pytest-forked/blob/master/CHANGELOG',
'pytest-xvfb': 'https://github.com/The-Compiler/pytest-xvfb/blob/master/CHANGELOG.rst',
'EasyProcess': 'https://github.com/ponty/EasyProcess/commits/master',
'PyVirtualDisplay': 'https://github.com/ponty/PyVirtualDisplay/commits/master',
'execnet': 'https://execnet.readthedocs.io/en/latest/changelog.html',
'apipkg': 'https://github.com/pytest-dev/apipkg/blob/master/CHANGELOG',
'pytest-rerunfailures': 'https://github.com/pytest-dev/pytest-rerunfailures/blob/master/CHANGES.rst',
'pytest-repeat': 'https://github.com/pytest-dev/pytest-repeat/blob/master/CHANGES.rst',
'requests': 'https://github.com/psf/requests/blob/master/HISTORY.md',
'requests-file': 'https://github.com/dashea/requests-file/blob/master/CHANGES.rst',
'werkzeug': 'https://github.com/pallets/werkzeug/blob/master/CHANGES.rst',
'Werkzeug': 'https://github.com/pallets/werkzeug/blob/master/CHANGES.rst',
'click': 'https://click.palletsprojects.com/en/7.x/changelog/',
'glob2': 'https://github.com/miracle2k/python-glob2/blob/master/CHANGES',
'itsdangerous': 'https://itsdangerous.palletsprojects.com/en/1.1.x/changes/',
'parse-type': 'https://github.com/jenisys/parse_type/blob/master/CHANGES.txt',
'sortedcontainers': 'https://github.com/grantjenks/python-sortedcontainers/blob/master/HISTORY.rst',
'soupsieve': 'https://facelessuser.github.io/soupsieve/about/changelog/',
'Flask': 'https://flask.palletsprojects.com/en/1.1.x/changelog/',
'Mako': 'https://docs.makotemplates.org/en/latest/changelog.html',
'glob2': 'https://github.com/miracle2k/python-glob2/blob/master/CHANGES',
'hypothesis': 'https://hypothesis.readthedocs.io/en/latest/changes.html',
'mypy': 'https://mypy-lang.blogspot.com/',
'pytest': 'https://docs.pytest.org/en/latest/changelog.html',
'iniconfig': 'https://github.com/RonnyPfannschmidt/iniconfig/blob/master/CHANGELOG',
'tox': 'https://tox.readthedocs.io/en/latest/changelog.html',
'pyyaml': 'https://github.com/yaml/pyyaml/blob/master/CHANGES',
'PyYAML': 'https://github.com/yaml/pyyaml/blob/master/CHANGES',
'pytest-bdd': 'https://github.com/pytest-dev/pytest-bdd/blob/master/CHANGES.rst',
'snowballstemmer': 'https://github.com/snowballstem/snowball/blob/master/NEWS',
'virtualenv': 'https://virtualenv.pypa.io/en/latest/changelog.html',
'pip': 'https://pip.pypa.io/en/stable/news/',
'packaging': 'https://pypi.org/project/packaging/',
'build': 'https://github.com/pypa/build/commits/master',
'flake8-docstrings': 'https://pypi.org/project/flake8-docstrings/',
'attrs': 'http://www.attrs.org/en/stable/changelog.html',
'jinja2': 'https://github.com/pallets/jinja/blob/master/CHANGES.rst',
'Jinja2': 'https://github.com/pallets/jinja/blob/master/CHANGES.rst',
'MarkupSafe': 'https://markupsafe.palletsprojects.com/en/1.1.x/changes/',
'flake8': 'https://gitlab.com/pycqa/flake8/tree/master/docs/source/release-notes',
'cffi': 'https://cffi.readthedocs.io/en/latest/whatsnew.html',
'flake8-docstrings': 'https://pypi.org/project/flake8-docstrings/',
'flake8-debugger': 'https://github.com/JBKahn/flake8-debugger/',
'flake8-builtins': 'https://github.com/gforcada/flake8-builtins/blob/master/CHANGES.rst',
'flake8-bugbear': 'https://github.com/PyCQA/flake8-bugbear',
'flake8-tidy-imports': 'https://github.com/adamchainz/flake8-tidy-imports/blob/master/HISTORY.rst',
'flake8-tuple': 'https://github.com/ar4s/flake8_tuple/blob/master/HISTORY.rst',
'flake8-comprehensions': 'https://github.com/adamchainz/flake8-comprehensions/blob/master/HISTORY.rst',
'flake8-copyright': 'https://github.com/savoirfairelinux/flake8-copyright/blob/master/CHANGELOG.rst',
'flake8-deprecated': 'https://github.com/gforcada/flake8-deprecated/blob/master/CHANGES.rst',
'flake8-future-import': 'https://github.com/xZise/flake8-future-import#changes',
'flake8-mock': 'https://github.com/aleGpereira/flake8-mock#changes',
'flake8-polyfill': 'https://gitlab.com/pycqa/flake8-polyfill/-/blob/master/CHANGELOG.rst',
'flake8-string-format': 'https://github.com/xZise/flake8-string-format#changes',
'pep8-naming': 'https://github.com/PyCQA/pep8-naming/blob/master/CHANGELOG.rst',
'pycodestyle': 'https://github.com/PyCQA/pycodestyle/blob/master/CHANGES.txt',
'pyflakes': 'https://github.com/PyCQA/pyflakes/blob/master/NEWS.rst',
'cffi': 'https://cffi.readthedocs.io/en/latest/whatsnew.html',
'astroid': 'https://github.com/PyCQA/astroid/blob/2.4/ChangeLog',
'pytest-instafail': 'https://github.com/pytest-dev/pytest-instafail/blob/master/CHANGES.rst',
'coverage': 'https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst',
'colorama': 'https://github.com/tartley/colorama/blob/master/CHANGELOG.rst',
'hunter': 'https://github.com/ionelmc/python-hunter/blob/master/CHANGELOG.rst',
'uritemplate': 'https://pypi.org/project/uritemplate/',
'flake8-builtins': 'https://github.com/gforcada/flake8-builtins/blob/master/CHANGES.rst',
'flake8-bugbear': 'https://github.com/PyCQA/flake8-bugbear',
'flake8-tidy-imports': 'https://github.com/adamchainz/flake8-tidy-imports/blob/master/HISTORY.rst',
'flake8-tuple': 'https://github.com/ar4s/flake8_tuple/blob/master/HISTORY.rst',
'uritemplate': 'https://github.com/python-hyper/uritemplate/blob/master/HISTORY.rst',
'more-itertools': 'https://github.com/erikrose/more-itertools/blob/master/docs/versions.rst',
'pydocstyle': 'http://www.pydocstyle.org/en/latest/release_notes.html',
'sphinx': 'https://www.sphinx-doc.org/en/master/changes.html',
'Sphinx': 'https://www.sphinx-doc.org/en/master/changes.html',
'Babel': 'http://babel.pocoo.org/en/latest/changelog.html',
'alabaster': 'https://alabaster.readthedocs.io/en/latest/changelog.html',
'imagesize': 'https://github.com/shibukawa/imagesize_py/commits/master',
'pytz': 'https://mm.icann.org/pipermail/tz-announce/',
'sphinxcontrib-applehelp': 'https://www.sphinx-doc.org/en/master/changes.html',
'sphinxcontrib-devhelp': 'https://www.sphinx-doc.org/en/master/changes.html',
'sphinxcontrib-htmlhelp': 'https://www.sphinx-doc.org/en/master/changes.html',
'sphinxcontrib-jsmath': 'https://www.sphinx-doc.org/en/master/changes.html',
'sphinxcontrib-qthelp': 'https://www.sphinx-doc.org/en/master/changes.html',
'sphinxcontrib-serializinghtml': 'https://www.sphinx-doc.org/en/master/changes.html',
'jaraco.functools': 'https://github.com/jaraco/jaraco.functools/blob/master/CHANGES.rst',
'parse': 'https://github.com/r1chardj0n3s/parse#potential-gotchas',
'py': 'https://py.readthedocs.io/en/latest/changelog.html#changelog',
'Pympler': 'https://github.com/pympler/pympler/blob/master/CHANGELOG.md',
'pytest-mock': 'https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst',
'pytest-qt': 'https://github.com/pytest-dev/pytest-qt/blob/master/CHANGELOG.rst',
'wcwidth': 'https://github.com/jquast/wcwidth#history',
'pyinstaller': 'https://pyinstaller.readthedocs.io/en/stable/CHANGES.html',
'pyinstaller-hooks-contrib': 'https://github.com/pyinstaller/pyinstaller-hooks-contrib/blob/master/CHANGELOG.rst',
'pytest-benchmark': 'https://pytest-benchmark.readthedocs.io/en/stable/changelog.html',
@ -97,11 +129,8 @@ CHANGELOG_URLS = {
'docutils': 'https://docutils.sourceforge.io/RELEASE-NOTES.html',
'bump2version': 'https://github.com/c4urself/bump2version/blob/master/CHANGELOG.md',
'six': 'https://github.com/benjaminp/six/blob/master/CHANGES',
'flake8-comprehensions': 'https://github.com/adamchainz/flake8-comprehensions/blob/master/HISTORY.rst',
'altgraph': 'https://github.com/ronaldoussoren/altgraph/blob/master/doc/changelog.rst',
'urllib3': 'https://github.com/urllib3/urllib3/blob/master/CHANGES.rst',
'wheel': 'https://github.com/pypa/wheel/blob/master/docs/news.rst',
'mako': 'https://docs.makotemplates.org/en/latest/changelog.html',
'lxml': 'https://lxml.de/4.6/changes-4.6.0.html',
'jwcrypto': 'https://github.com/latchset/jwcrypto/commits/master',
'tox-pip-version': 'https://github.com/pglass/tox-pip-version/commits/master',
@ -124,8 +153,8 @@ CHANGELOG_URLS = {
'chardet': 'https://github.com/chardet/chardet/releases',
'idna': 'https://github.com/kjd/idna/blob/master/HISTORY.rst',
'tldextract': 'https://github.com/john-kurkowski/tldextract/blob/master/CHANGELOG.md',
'typing_extensions': 'https://github.com/python/typing/commits/master/typing_extensions',
'diff_cover': 'https://github.com/Bachmann1234/diff_cover/blob/master/CHANGELOG',
'typing-extensions': 'https://github.com/python/typing/commits/master/typing_extensions',
'diff-cover': 'https://github.com/Bachmann1234/diff_cover/blob/master/CHANGELOG',
'pytest-clarity': 'https://github.com/darrenburns/pytest-clarity/commits/master',
'pytest-icdiff': 'https://github.com/hjwp/pytest-icdiff/blob/master/HISTORY.rst',
'icdiff': 'https://github.com/jeffkaufman/icdiff/blob/master/ChangeLog',
@ -134,7 +163,21 @@ CHANGELOG_URLS = {
'beautifulsoup4': 'https://bazaar.launchpad.net/~leonardr/beautifulsoup/bs4/view/head:/CHANGELOG',
'check-manifest': 'https://github.com/mgedmin/check-manifest/blob/master/CHANGES.rst',
'yamllint': 'https://github.com/adrienverge/yamllint/blob/master/CHANGELOG.rst',
'pathspec': 'https://github.com/cpburnz/python-path-specification/blob/master/CHANGES.rst',
'filelock': 'https://github.com/benediktschmitt/py-filelock/commits/master',
'cssutils': 'https://pythonhosted.org/cssutils/CHANGELOG.html',
'github3.py': 'https://github3py.readthedocs.io/en/master/release-notes/index.html',
'manhole': 'https://github3py.readthedocs.io/en/master/release-notes/index.html',
'pycparser': 'https://github.com/eliben/pycparser/blob/master/CHANGES',
'python-dateutil': 'https://dateutil.readthedocs.io/en/stable/changelog.html',
'appdirs': 'https://github.com/ActiveState/appdirs/blob/master/CHANGES.rst',
'pluggy': 'https://github.com/pytest-dev/pluggy/blob/master/CHANGELOG.rst',
'tox-venv': 'https://github.com/tox-dev/tox-venv/blob/master/CHANGELOG.rst',
'inflect': 'https://github.com/jazzband/inflect/blob/master/CHANGES.rst',
'jinja2-pluralize': 'https://github.com/audreyfeldroy/jinja2_pluralize/blob/master/HISTORY.rst',
'mypy-extensions': 'https://github.com/python/mypy_extensions/commits/master',
'pyroma': 'https://github.com/regebro/pyroma/blob/master/HISTORY.txt',
'pyPEG2': None,
}
@ -275,7 +318,7 @@ class Change:
self.name = name
self.old = None
self.new = None
if name in CHANGELOG_URLS:
if CHANGELOG_URLS.get(name):
self.url = CHANGELOG_URLS[name]
self.link = '[{}]({})'.format(self.name, self.url)
else:
@ -315,6 +358,25 @@ def _get_changed_files():
return sorted(changed_files)
def parse_versioned_line(line):
"""Parse a requirements.txt line into name/version."""
if '==' in line:
name, version = line.split('==')
if ';' in version: # pip environment markers
version = version.split(';')[0].strip()
elif line.startswith('-e'):
rest, name = line.split('#egg=')
version = rest.split('@')[1][:7]
else:
name = line
version = '?'
if name.startswith('#'): # duplicate requirements
name = name[1:].strip()
return name, version
def _get_changes():
"""Get a list of changed versions from git."""
changes_dict = {}
@ -325,19 +387,7 @@ def _get_changes():
if line.startswith('+++ ') or line.startswith('--- '):
continue
if '==' in line:
name, version = line[1:].split('==')
if ';' in version: # pip environment markers
version = version.split(';')[0].strip()
elif line[1:].startswith('-e'):
rest, name = line.split('#egg=')
version = rest.split('@')[1][:7]
else:
name = line[1:]
version = '?'
if name.startswith('#'): # duplicate requirements
name = name[1:].strip()
name, version = parse_versioned_line(line[1:])
if name not in changes_dict:
changes_dict[name] = Change(name)
@ -389,6 +439,12 @@ def get_host_python(name):
return sys.executable
def get_outfile(name):
if name == 'qutebrowser':
return os.path.join(REPO_DIR, 'requirements.txt')
return os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
def build_requirements(name):
"""Build a requirements file."""
utils.print_subtitle("Building")
@ -409,11 +465,6 @@ def build_requirements(name):
if utils.ON_CI:
print(reqs.strip())
if name == 'qutebrowser':
outfile = os.path.join(REPO_DIR, 'requirements.txt')
else:
outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
with open(outfile, 'w', encoding='utf-8') as f:
f.write("# This file is automatically generated by "
"scripts/dev/recompile_requirements.py\n\n")