diff --git a/misc/requirements/requirements-tests.txt-raw b/misc/requirements/requirements-tests.txt-raw
index d6f389457..779078021 100644
--- a/misc/requirements/requirements-tests.txt-raw
+++ b/misc/requirements/requirements-tests.txt-raw
@@ -6,13 +6,10 @@ hypothesis
 pytest
 pytest-bdd
 pytest-benchmark
-pytest-cov
 pytest-instafail
 pytest-mock
 pytest-qt
 pytest-rerunfailures
-pytest-xvfb
-PyVirtualDisplay
 
 ## optional:
 # To test :debug-trace, gets skipped if hunter is not installed
@@ -23,6 +20,11 @@ vulture
 pygments
 # --repeat switch (used to manually repeat tests)
 pytest-repeat
+# For coverage tests
+pytest-cov
+# To avoid windows from popping up
+pytest-xvfb
+PyVirtualDisplay
 
 # Needed to test misc/userscripts/qute-lastpass
 tldextract
diff --git a/tests/conftest.py b/tests/conftest.py
index 4c6f969fb..d4d06c6bc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -234,11 +234,6 @@ def pytest_configure(config):
 
 @pytest.fixture(scope='session', autouse=True)
 def check_display(request):
-    if (not request.config.getoption('--no-xvfb') and
-            'QUTE_BUILDBOT' in os.environ and
-            request.config.xvfb is not None):
-        raise Exception("Xvfb is running on buildbot!")
-
     if utils.is_linux and not os.environ.get('DISPLAY', ''):
         raise Exception("No display and no Xvfb available!")
 
diff --git a/tests/unit/scripts/test_check_coverage.py b/tests/unit/scripts/test_check_coverage.py
index 8dadc4a29..c924ec4f3 100644
--- a/tests/unit/scripts/test_check_coverage.py
+++ b/tests/unit/scripts/test_check_coverage.py
@@ -94,6 +94,15 @@ def covtest(testdir, monkeypatch):
         def test_module():
             func()
     """)
+
+    # Check if coverage plugin is available
+    res = testdir.runpytest('--version', '--version')
+    assert res.ret == 0
+    output = res.stderr.str()
+    assert 'This is pytest version' in output
+    if 'pytest-cov' not in output:
+        pytest.skip("cov plugin not available")
+
     return CovtestHelper(testdir, monkeypatch)