Merge pull request #8243 from feat/e2e_screenshots
This commit is contained in:
toofar
commit
3bc30e748d
|
|
@ -27,6 +27,7 @@ jobs:
|
|||
PY_COLORS: "1"
|
||||
DOCKER: "${{ matrix.image }}"
|
||||
CI: true
|
||||
TMPDIR: "${{ runner.temp }}"
|
||||
volumes:
|
||||
# Hardcoded because we can't use ${{ runner.temp }} here apparently.
|
||||
- /home/runner/work/_temp/:/home/runner/work/_temp/
|
||||
|
|
@ -42,6 +43,21 @@ jobs:
|
|||
if: "endsWith(matrix.image, '-qt6')"
|
||||
- name: Run tox
|
||||
run: dbus-run-session tox -e ${{ matrix.testenv }}
|
||||
- name: Gather info
|
||||
id: info
|
||||
run: |
|
||||
echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
|
||||
echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
|
||||
shell: bash
|
||||
if: failure()
|
||||
- name: Upload screenshots
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "end2end-screenshots-${{ steps.info.outputs.date }}-${{ steps.info.outputs.sha_short }}-${{ matrix.image }}"
|
||||
path: |
|
||||
${{ runner.temp }}/pytest-of-user/pytest-current/pytest-screenshots/*.png
|
||||
if-no-files-found: ignore
|
||||
if: failure()
|
||||
irc:
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ jobs:
|
|||
DOCKER: "${{ matrix.image }}"
|
||||
CI: true
|
||||
PYTEST_ADDOPTS: "--color=yes"
|
||||
TMPDIR: "${{ runner.temp }}"
|
||||
volumes:
|
||||
# Hardcoded because we can't use ${{ runner.temp }} here apparently.
|
||||
- /home/runner/work/_temp/:/home/runner/work/_temp/
|
||||
|
|
@ -119,6 +120,21 @@ jobs:
|
|||
run: "python scripts/dev/ci/problemmatchers.py tests ${{ runner.temp }}"
|
||||
- name: Run tox
|
||||
run: "dbus-run-session -- tox -e ${{ matrix.testenv }}"
|
||||
- name: Gather info
|
||||
id: info
|
||||
run: |
|
||||
echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
|
||||
echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
|
||||
shell: bash
|
||||
if: failure()
|
||||
- name: Upload screenshots
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "end2end-screenshots-${{ steps.info.outputs.date }}-${{ steps.info.outputs.sha_short }}-${{ matrix.image }}"
|
||||
path: |
|
||||
${{ runner.temp }}/pytest-of-user/pytest-current/pytest-screenshots/*.png
|
||||
if-no-files-found: ignore
|
||||
if: failure()
|
||||
|
||||
tests:
|
||||
if: "!contains(github.event.head_commit.message, '[ci skip]')"
|
||||
|
|
@ -222,6 +238,8 @@ jobs:
|
|||
- name: Upgrade 3rd party assets
|
||||
run: "tox exec -e ${{ matrix.testenv }} -- python scripts/dev/update_3rdparty.py --gh-token ${{ secrets.GITHUB_TOKEN }}"
|
||||
if: "startsWith(matrix.os, 'windows-')"
|
||||
- name: "Set TMPDIR for pytest"
|
||||
run: 'echo "TMPDIR=${{ runner.temp }}" >> "$GITHUB_ENV"'
|
||||
- name: "Run ${{ matrix.testenv }}"
|
||||
run: "dbus-run-session -- tox -e ${{ matrix.testenv }} -- ${{ matrix.args }}"
|
||||
if: "startsWith(matrix.os, 'ubuntu-')"
|
||||
|
|
@ -236,6 +254,21 @@ jobs:
|
|||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
name: "${{ matrix.testenv }}"
|
||||
- name: Gather info
|
||||
id: info
|
||||
run: |
|
||||
echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
|
||||
echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
|
||||
shell: bash
|
||||
if: failure()
|
||||
- name: Upload screenshots
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "end2end-screenshots-${{ steps.info.outputs.date }}-${{ steps.info.outputs.sha_short }}-${{ matrix.testenv }}-${{ matrix.os }}"
|
||||
path: |
|
||||
${{ runner.temp }}/pytest-of-runner/pytest-current/pytest-screenshots/*.png
|
||||
if-no-files-found: ignore
|
||||
if: failure()
|
||||
|
||||
codeql:
|
||||
if: "!contains(github.event.head_commit.message, '[ci skip]')"
|
||||
|
|
|
|||
|
|
@ -30,6 +30,10 @@ Removed
|
|||
|
||||
- Support for macOS 11 Big Sur is dropped. Binaries are now built on macOS 12
|
||||
Monterey and are unlikely to still run on older macOS versions.
|
||||
- Failed end2end tests will now save screenshots of the browser window when
|
||||
run under xvfb (the default on linux). Screenshots will be under
|
||||
`$TEMP/pytest-current/pytest-screenshots/` or attached to the GitHub actions
|
||||
run as an artifact. (#7625)
|
||||
|
||||
Changed
|
||||
~~~~~~~
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ git+https://github.com/pygments/pygments.git
|
|||
git+https://github.com/pytest-dev/pytest-repeat.git
|
||||
git+https://github.com/pytest-dev/pytest-cov.git
|
||||
git+https://github.com/The-Compiler/pytest-xvfb.git
|
||||
git+https://github.com/python-pillow/Pillow.git
|
||||
git+https://github.com/pytest-dev/pytest-xdist.git
|
||||
git+https://github.com/john-kurkowski/tldextract
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ packaging==24.1
|
|||
parse==1.20.2
|
||||
parse_type==0.6.3
|
||||
platformdirs==4.3.6
|
||||
pillow==10.4.0
|
||||
pluggy==1.5.0
|
||||
py-cpuinfo==9.0.0
|
||||
Pygments==2.18.0
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ pytest-cov
|
|||
# To avoid windows from popping up
|
||||
pytest-xvfb
|
||||
PyVirtualDisplay
|
||||
pillow
|
||||
# To run on multiple cores with -n
|
||||
pytest-xdist
|
||||
|
||||
|
|
|
|||
|
|
@ -86,3 +86,4 @@ filterwarnings =
|
|||
# Python 3.12: https://github.com/ionelmc/pytest-benchmark/issues/240 (fixed but not released)
|
||||
ignore:(datetime\.)?datetime\.utcnow\(\) is deprecated and scheduled for removal in a future version\. Use timezone-aware objects to represent datetimes in UTC. (datetime\.)?datetime\.now\(datetime\.UTC\)\.:DeprecationWarning:pytest_benchmark\.utils
|
||||
faulthandler_timeout = 90
|
||||
xvfb_colordepth = 24
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import sys
|
|||
import contextlib
|
||||
import enum
|
||||
import argparse
|
||||
import tempfile
|
||||
from typing import Iterator, Optional, Dict
|
||||
|
||||
from qutebrowser.qt.core import QStandardPaths
|
||||
|
|
@ -311,14 +312,15 @@ def _create(path: str) -> None:
|
|||
0700. If the destination directory exists already the permissions
|
||||
should not be changed.
|
||||
"""
|
||||
if APPNAME == 'qute_test' and path.startswith('/home'): # pragma: no cover
|
||||
for k, v in os.environ.items():
|
||||
if k == 'HOME' or k.startswith('XDG_'):
|
||||
log.init.debug(f"{k} = {v}")
|
||||
raise AssertionError(
|
||||
"Trying to create directory inside /home during "
|
||||
"tests, this should not happen."
|
||||
)
|
||||
if APPNAME == 'qute_test':
|
||||
if path.startswith('/home') and not path.startswith(tempfile.gettempdir()): # pragma: no cover
|
||||
for k, v in os.environ.items():
|
||||
if k == 'HOME' or k.startswith('XDG_'):
|
||||
log.init.debug(f"{k} = {v}")
|
||||
raise AssertionError(
|
||||
"Trying to create directory inside /home during "
|
||||
"tests, this should not happen."
|
||||
)
|
||||
os.makedirs(path, 0o700, exist_ok=True)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -168,5 +168,6 @@
|
|||
"mdurl": "https://github.com/executablebooks/mdurl/commits/master",
|
||||
"blinker": "https://blinker.readthedocs.io/en/stable/#changes",
|
||||
"exceptiongroup": "https://github.com/agronholm/exceptiongroup/blob/main/CHANGES.rst",
|
||||
"nh3": "https://github.com/messense/nh3/commits/main"
|
||||
"nh3": "https://github.com/messense/nh3/commits/main",
|
||||
"pillow": "https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -372,7 +372,8 @@ def pytest_runtest_makereport(item, call):
|
|||
|
||||
@pytest.hookimpl(hookwrapper=True)
|
||||
def pytest_terminal_summary(terminalreporter):
|
||||
"""Group benchmark results on CI."""
|
||||
"""Add custom pytest summary sections."""
|
||||
# Group benchmark results on CI.
|
||||
if testutils.ON_CI:
|
||||
terminalreporter.write_line(
|
||||
testutils.gha_group_begin('Benchmark results'))
|
||||
|
|
@ -380,3 +381,21 @@ def pytest_terminal_summary(terminalreporter):
|
|||
terminalreporter.write_line(testutils.gha_group_end())
|
||||
else:
|
||||
yield
|
||||
|
||||
# List any screenshots of failed end2end tests that were generated during
|
||||
# the run. Screenshots are captured from QuteProc.after_test()
|
||||
properties = lambda report: dict(report.user_properties)
|
||||
reports = [
|
||||
report
|
||||
for report in terminalreporter.getreports("")
|
||||
if "screenshot" in properties(report)
|
||||
]
|
||||
screenshots = [
|
||||
pathlib.Path(properties(report)["screenshot"])
|
||||
for report in reports
|
||||
]
|
||||
|
||||
if screenshots:
|
||||
terminalreporter.ensure_newline()
|
||||
screenshot_dir = screenshots[0].parent
|
||||
terminalreporter.section(f"End2end screenshots available in: {screenshot_dir}", sep="-", blue=True, bold=True)
|
||||
|
|
|
|||
|
|
@ -18,10 +18,15 @@ from qutebrowser.qt.core import PYQT_VERSION, QCoreApplication
|
|||
pytest.register_assert_rewrite('end2end.fixtures')
|
||||
|
||||
# pylint: disable=unused-import
|
||||
# Import fixtures that the bdd tests rely on.
|
||||
from end2end.fixtures.notificationserver import notification_server
|
||||
from end2end.fixtures.webserver import server, server_per_test, server2, ssl_server
|
||||
from end2end.fixtures.quteprocess import (quteproc_process, quteproc,
|
||||
quteproc_new)
|
||||
from end2end.fixtures.quteprocess import (
|
||||
quteproc_process, quteproc,
|
||||
quteproc_new,
|
||||
screenshot_dir,
|
||||
take_x11_screenshot,
|
||||
)
|
||||
from end2end.fixtures.testprocess import pytest_runtest_makereport
|
||||
# pylint: enable=unused-import
|
||||
from qutebrowser.utils import qtutils, utils, version
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
"""Fixtures to run qutebrowser in a QProcess and communicate."""
|
||||
|
||||
import pathlib
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
|
@ -18,6 +19,7 @@ import json
|
|||
|
||||
import yaml
|
||||
import pytest
|
||||
from PIL.ImageGrab import grab
|
||||
from qutebrowser.qt.core import pyqtSignal, QUrl, QPoint
|
||||
from qutebrowser.qt.gui import QImage, QColor
|
||||
|
||||
|
|
@ -416,7 +418,8 @@ class QuteProc(testprocess.Process):
|
|||
'--debug-flag', 'werror',
|
||||
'--debug-flag', 'test-notification-service',
|
||||
'--debug-flag', 'caret',
|
||||
'--qt-flag', 'disable-features=PaintHoldingCrossOrigin']
|
||||
'--qt-flag', 'disable-features=PaintHoldingCrossOrigin',
|
||||
'--qt-arg', 'geometry', '800x600+0+0']
|
||||
|
||||
if self.request.config.webengine and testutils.disable_seccomp_bpf_sandbox():
|
||||
args += testutils.DISABLE_SECCOMP_BPF_ARGS
|
||||
|
|
@ -545,6 +548,8 @@ class QuteProc(testprocess.Process):
|
|||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
if call.failed:
|
||||
self._take_x11_screenshot_of_failed_test()
|
||||
if call.failed or hasattr(call, 'wasxfail') or call.skipped:
|
||||
super().after_test()
|
||||
return
|
||||
|
|
@ -872,6 +877,10 @@ class QuteProc(testprocess.Process):
|
|||
self.send_cmd(cmd.format('no-scroll-filtering'))
|
||||
self.send_cmd(cmd.format('log-scroll-pos'))
|
||||
|
||||
def _take_x11_screenshot_of_failed_test(self):
|
||||
fixture = self.request.getfixturevalue('take_x11_screenshot')
|
||||
fixture()
|
||||
|
||||
|
||||
class YamlLoader(yaml.SafeLoader):
|
||||
|
||||
|
|
@ -912,6 +921,39 @@ def _xpath_escape(text):
|
|||
return 'concat({})'.format(', '.join(parts))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def screenshot_dir(request, tmp_path_factory):
|
||||
"""Return the path of a directory to save e2e screenshots in."""
|
||||
path = tmp_path_factory.getbasetemp()
|
||||
if "PYTEST_XDIST_WORKER" in os.environ:
|
||||
# If we are running under xdist remove the per-worker directory
|
||||
# (like "popen-gw0") so the user doesn't have to search through
|
||||
# multiple folders for the screenshot they are looking for.
|
||||
path = path.parent
|
||||
path /= "pytest-screenshots"
|
||||
path.mkdir(exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def take_x11_screenshot(request, screenshot_dir, record_property, xvfb):
|
||||
"""Take a screenshot of the current pytest-xvfb display.
|
||||
|
||||
Screenshots are saved to the location of the `screenshot_dir` fixture.
|
||||
"""
|
||||
def doit():
|
||||
if not xvfb:
|
||||
# Likely we are being run with --no-xvfb
|
||||
return
|
||||
|
||||
img = grab(xdisplay=f":{xvfb.display}")
|
||||
fpath = screenshot_dir / f"{request.node.name}.png"
|
||||
img.save(fpath)
|
||||
|
||||
record_property("screenshot", str(fpath))
|
||||
return doit
|
||||
|
||||
|
||||
@pytest.fixture(scope='module')
|
||||
def quteproc_process(qapp, server, request):
|
||||
"""Fixture for qutebrowser process which is started once per file."""
|
||||
|
|
@ -923,7 +965,7 @@ def quteproc_process(qapp, server, request):
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def quteproc(quteproc_process, server, request):
|
||||
def quteproc(quteproc_process, server, request, take_x11_screenshot):
|
||||
"""Per-test qutebrowser fixture which uses the per-file process."""
|
||||
request.node._quteproc_log = quteproc_process.captured_log
|
||||
quteproc_process.before_test()
|
||||
|
|
|
|||
|
|
@ -98,8 +98,9 @@ def test_quteproc_error_message(qtbot, quteproc, cmd, request_mock):
|
|||
quteproc.after_test()
|
||||
|
||||
|
||||
def test_quteproc_error_message_did_fail(qtbot, quteproc, request_mock):
|
||||
def test_quteproc_error_message_did_fail(qtbot, quteproc, request_mock, monkeypatch):
|
||||
"""Make sure the test does not fail on teardown if the main test failed."""
|
||||
monkeypatch.setattr(quteproc, "_take_x11_screenshot_of_failed_test", lambda: None)
|
||||
request_mock.node.rep_call.failed = True
|
||||
with qtbot.wait_signal(quteproc.got_error):
|
||||
quteproc.send_cmd(':message-error test')
|
||||
|
|
@ -108,6 +109,17 @@ def test_quteproc_error_message_did_fail(qtbot, quteproc, request_mock):
|
|||
quteproc.after_test()
|
||||
|
||||
|
||||
def test_quteproc_screenshot_on_fail(qtbot, quteproc, request_mock, monkeypatch, mocker):
|
||||
"""Make sure we call the method to take a screenshot to test failure."""
|
||||
take_screenshot_spy = mocker.Mock()
|
||||
monkeypatch.setattr(
|
||||
quteproc, "_take_x11_screenshot_of_failed_test", take_screenshot_spy
|
||||
)
|
||||
request_mock.node.rep_call.failed = True
|
||||
quteproc.after_test()
|
||||
take_screenshot_spy.assert_called_once()
|
||||
|
||||
|
||||
def test_quteproc_skip_via_js(qtbot, quteproc):
|
||||
with pytest.raises(pytest.skip.Exception, match='test'):
|
||||
quteproc.send_cmd(':jseval console.log("[SKIP] test");')
|
||||
|
|
|
|||
Loading…
Reference in New Issue