diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py
index 2e44197a2..ee0ac2c53 100755
--- a/scripts/dev/build_release.py
+++ b/scripts/dev/build_release.py
@@ -275,12 +275,12 @@ def build_windows():
 
     utils.print_title("Running pyinstaller 32bit")
     _maybe_remove(out_32)
-    call_tox('pyinstaller32', '-r', python=python_x86)
+    call_tox('pyinstaller-32', '-r', python=python_x86)
     shutil.move(out_pyinstaller, out_32)
 
     utils.print_title("Running pyinstaller 64bit")
     _maybe_remove(out_64)
-    call_tox('pyinstaller', '-r', python=python_x64)
+    call_tox('pyinstaller-64', '-r', python=python_x64)
     shutil.move(out_pyinstaller, out_64)
 
     utils.print_title("Running 32bit smoke test")
diff --git a/tox.ini b/tox.ini
index 0cd722f1a..309584355 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,6 +7,7 @@
 envlist = py38-pyqt515-cov,mypy,misc,vulture,flake8,pylint,pyroma,check-manifest,eslint,yamllint
 distshare = {toxworkdir}
 skipsdist = true
+minversion = 3.15
 
 [testenv]
 setenv =
@@ -158,7 +159,7 @@ commands =
     {envpython} scripts/dev/check_doc_changes.py {posargs}
     {envpython} scripts/asciidoc2html.py {posargs}
 
-[testenv:pyinstaller]
+[testenv:pyinstaller-{64,32}]
 basepython = {env:PYTHON:python3}
 pip_version = pip
 passenv = APPDATA HOME PYINSTALLER_DEBUG
@@ -169,17 +170,6 @@ deps =
 commands =
     {envbindir}/pyinstaller --noconfirm misc/qutebrowser.spec
 
-[testenv:pyinstaller32]
-# A copy of the pyinstaller environment above, to be used for 32-bit on Windows
-# to make sure the both installations are separated.
-# This doesn't actually do anything 32-bit specific, that part happens by
-# setting the PYTHON environment variable accordingly.
-basepython = {[testenv:pyinstaller]basepython}
-pip_version = {[testenv:pyinstaller]pip_version}
-passenv = {[testenv:pyinstaller]passenv}
-deps = {[testenv:pyinstaller]deps}
-commands = {[testenv:pyinstaller]commands}
-
 [testenv:eslint]
 basepython = python3
 deps =