Change configuration options for host- and adblock

This commit is contained in:
Árni Dagur 2020-08-08 03:51:56 +00:00 committed by Árni Dagur
parent 68b9960a67
commit 188bd533a4
6 changed files with 366 additions and 192 deletions

View File

@ -137,10 +137,10 @@
|<<completion.web_history.max_items,completion.web_history.max_items>>|Number of URLs to show in the web history.
|<<confirm_quit,confirm_quit>>|Require a confirmation before quitting the application.
|<<content.autoplay,content.autoplay>>|Automatically start playing `<video>` elements.
|<<content.blocking.adblock.enabled,content.blocking.adblock.enabled>>|Enable Brave AdBlock.
|<<content.blocking.adblock.lists,content.blocking.adblock.lists>>|List of URLs of lists which contain adblock rules.
|<<content.blocking.hosts.enabled,content.blocking.hosts.enabled>>|Enable host blocking.
|<<content.blocking.enabled,content.blocking.enabled>>|Enable the ad blocker
|<<content.blocking.hosts.lists,content.blocking.hosts.lists>>|List of URLs of lists which contain hosts to block.
|<<content.blocking.method,content.blocking.method>>|Which method of blocking ads should be used.
|<<content.blocking.whitelist,content.blocking.whitelist>>|A list of patterns that should always be loaded, despite being ad-blocked. Local domains are always exempt from adblocking.
|<<content.cache.appcache,content.cache.appcache>>|Enable support for the HTML 5 web application cache feature.
|<<content.cache.maximum_pages,content.cache.maximum_pages>>|Maximum number of pages to hold in the global memory page cache.
@ -1901,16 +1901,6 @@ On QtWebEngine, this setting requires Qt 5.10 or newer.
On QtWebKit, this setting is unavailable.
[[content.blocking.adblock.enabled]]
=== content.blocking.adblock.enabled
Enable Brave AdBlock.
This setting supports URL patterns.
Type: <<types,Bool>>
Default: +pass:[true]+
[[content.blocking.adblock.lists]]
=== content.blocking.adblock.lists
List of URLs of lists which contain adblock rules.
@ -1923,9 +1913,9 @@ Default:
- +pass:[https://easylist.to/easylist/easylist.txt]+
- +pass:[https://easylist.to/easylist/easyprivacy.txt]+
[[content.blocking.hosts.enabled]]
=== content.blocking.hosts.enabled
Enable host blocking.
[[content.blocking.enabled]]
=== content.blocking.enabled
Enable the ad blocker
This setting supports URL patterns.
@ -1956,6 +1946,27 @@ Default:
- +pass:[https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts]+
[[content.blocking.method]]
=== content.blocking.method
Which method of blocking ads should be used.
Support for Adblock Plus syntax blocklists requires the `adblock` Python
package to be installed, which is an optional dependency of qutebrowser.
It can be installed using the command `pip install --user adblock`. This
is required when either `adblock` or `both` are selected.
Type: <<types,String>>
Valid values:
* +auto+: Use Brave's ABP-style adblocker if available, host blocking otherwise
* +adblock+: Use Brave's ABP-style adblocker
* +hosts+: Use hosts blocking
* +both+: Use both hosts blocking and Brave's ABP-style adblocker
Default: +pass:[auto]+
[[content.blocking.whitelist]]
=== content.blocking.whitelist
A list of patterns that should always be loaded, despite being ad-blocked. Local domains are always exempt from adblocking.

View File

@ -38,6 +38,7 @@ from qutebrowser.api import (
qtutils,
)
from qutebrowser.components.utils import blockutils
from qutebrowser.components.braveadblock import ad_blocker
logger = logging.getLogger("network")
@ -78,11 +79,21 @@ def _is_whitelisted_url(url: QUrl) -> bool:
return False
def _should_be_used() -> bool:
"""Whether the hostblocker should be used or not"""
method = config.val.content.blocking.method
adblock_dependency_satisfied = ad_blocker is None
return method in ("both", "hosts") or (
method == "auto" and not adblock_dependency_satisfied
)
class HostBlocker:
"""Manage blocked hosts based from /etc/hosts-like files.
Attributes:
enabled: Given the current blocking method, should the host blocker be enabled?
_blocked_hosts: A set of blocked hosts.
_config_blocked_hosts: A set of blocked hosts from ~/.config.
_in_progress: The DownloadItems which are currently downloading.
@ -99,6 +110,7 @@ class HostBlocker:
config_dir: pathlib.Path,
has_basedir: bool = False
) -> None:
self.enabled = _should_be_used()
self._has_basedir = has_basedir
self._blocked_hosts = set() # type: typing.Set[str]
self._config_blocked_hosts = set() # type: typing.Set[str]
@ -112,12 +124,15 @@ class HostBlocker:
def _is_blocked(self, request_url: QUrl, first_party_url: QUrl = None) -> bool:
"""Check whether the given request is blocked."""
if not self.enabled:
return False
if first_party_url is not None and not first_party_url.isValid():
first_party_url = None
qtutils.ensure_valid(request_url)
if not config.get("content.blocking.hosts.enabled", url=first_party_url):
if not config.get("content.blocking.enabled", url=first_party_url):
return False
host = request_url.host()
@ -206,7 +221,8 @@ class HostBlocker:
if (
config.val.content.blocking.hosts.lists
and not self._has_basedir
and config.val.content.blocking.hosts.enabled
and config.val.content.blocking.enabled
and self.enabled
):
message.info("Run :adblock-update to get adblock lists.")
@ -300,6 +316,11 @@ def on_config_changed() -> None:
host_blocker.update_files()
@hook.config_changed("content.blocking.method")
def on_method_changed():
host_blocker.enabled = _should_be_used()
@hook.init()
def init(context: apitypes.InitContext) -> None:
"""Initialize the host blocker."""

View File

@ -60,6 +60,25 @@ def _is_whitelisted_url(url: QUrl) -> bool:
return False
def _should_be_used() -> bool:
"""Whether the Brave adblocker should be used or not, assuming the
dependency is satisfied."""
method = config.val.content.blocking.method
return method in ("auto", "both", "adblock")
def _possibly_show_missing_dependency_warning() -> None:
"""If appropriate, show the user a message warning them that the 'adblock'
dependency is missing."""
method = config.val.content.blocking.method
if method in ("adblock", "both"):
message.warning(
"Ad blocking method is set to '{}' but 'adblock' dependency is not installed.".format(
method
)
)
_RESOURCE_TYPE_STRINGS = {
ResourceType.main_frame: "main_frame",
ResourceType.sub_frame: "sub_frame",
@ -95,6 +114,7 @@ class BraveAdBlocker:
"""Manage blocked hosts based from /etc/hosts-like files.
Attributes:
enabled: Should we block ads or not
_has_basedir: Whether a custom --basedir is set.
_cache_path: The path of the adblock engine cache file
_in_progress: The DownloadItems which are currently downloading.
@ -111,6 +131,7 @@ class BraveAdBlocker:
"""
def __init__(self, *, data_dir: pathlib.Path, has_basedir: bool = False) -> None:
self.enabled = _should_be_used()
self._has_basedir = has_basedir
self._cache_path = data_dir / "adblock-cache.dat"
self._in_progress = [] # type: typing.List[downloads.TempDownload]
@ -126,6 +147,9 @@ class BraveAdBlocker:
resource_type: typing.Optional[interceptor.ResourceType] = None,
) -> bool:
"""Check whether the given request is blocked."""
if not self.enabled:
return False
if first_party_url is not None and not first_party_url.isValid():
first_party_url = None
@ -134,7 +158,7 @@ class BraveAdBlocker:
qtutils.ensure_valid(request_url)
if not config.get("content.blocking.adblock.enabled", url=first_party_url):
if not config.get("content.blocking.enabled", url=first_party_url):
# Do nothing if adblocking is disabled.
return False
@ -180,7 +204,8 @@ class BraveAdBlocker:
if (
config.val.content.blocking.adblock.lists
and not self._has_basedir
and config.val.content.blocking.adblock.enabled
and config.val.content.blocking.enabled
and self.enabled
):
message.info("Run :adblock-update to get adblock lists.")
@ -254,11 +279,20 @@ class BraveAdBlocker:
@hook.config_changed("content.blocking.adblock.lists")
def on_config_changed() -> None:
def on_lists_changed() -> None:
if ad_blocker is not None:
ad_blocker.update_files()
@hook.config_changed("content.blocking.method")
def on_method_changed():
if ad_blocker is not None:
# This implies the 'adblock' dependency is satisfied
ad_blocker.enabled = _should_be_used()
else:
_possibly_show_missing_dependency_warning()
@hook.init()
def init(context: apitypes.InitContext) -> None:
"""Initialize the Brave ad blocker."""
@ -266,9 +300,10 @@ def init(context: apitypes.InitContext) -> None:
if adblock is None:
# We want 'adblock' to be an optional dependency. If the module is
# not found, we simply set the `_ad_blocker` global to `None`. Always
# remember to check the case where `_ad_blocker` is `None`!
# not found, we simply set the `ad_blocker` global to `None`. Always
# remember to check the case where `ad_blocker` is `None`!
ad_blocker = None
_possibly_show_missing_dependency_warning()
return
ad_blocker = BraveAdBlocker(

View File

@ -605,7 +605,7 @@ content.headers.user_agent:
to JavaScript requires a restart.
content.host_blocking.enabled:
renamed: content.blocking.hosts.enabled
renamed: content.blocking.enabled
content.host_blocking.lists:
renamed: content.blocking.hosts.lists
@ -613,11 +613,11 @@ content.host_blocking.lists:
content.host_blocking.whitelist:
renamed: content.blocking.whitelist
content.blocking.hosts.enabled:
content.blocking.enabled:
default: true
supports_pattern: true
type: Bool
desc: Enable host blocking.
desc: Enable the ad blocker
content.blocking.hosts.lists:
default:
@ -641,15 +641,22 @@ content.blocking.hosts.lists:
The file `~/.config/qutebrowser/blocked-hosts` is always read if it exists.
content.blocking.adblock.enabled:
default: true
supports_pattern: true
type: Bool
desc: >-
Enable Adblock Plus syntax adblock lists using Brave Browsers's adblocking
library. This feature requires the "adblock" Python package, which is an
optional dependency of qutebrowser. It can be installed using the command
`pip install --user adblock`.
content.blocking.method:
default: auto
type:
name: String
valid_values:
- auto: Use Brave's ABP-style adblocker if available, host blocking otherwise
- adblock: Use Brave's ABP-style adblocker
- hosts: Use hosts blocking
- both: Use both hosts blocking and Brave's ABP-style adblocker
desc: |
Which method of blocking ads should be used.
Support for Adblock Plus syntax blocklists requires the `adblock` Python
package to be installed, which is an optional dependency of qutebrowser.
It can be installed using the command `pip install --user adblock`. This
is required when either `adblock` or `both` are selected.
content.blocking.adblock.lists:
default:

View File

@ -32,41 +32,44 @@ from qutebrowser.utils import urlmatch
from helpers import utils
pytestmark = pytest.mark.usefixtures('qapp')
pytestmark = pytest.mark.usefixtures("qapp")
# TODO See ../utils/test_standarddirutils for OSError and caplog assertion
WHITELISTED_HOSTS = ('qutebrowser.org', 'mediumhost.io', 'http://*.edu')
WHITELISTED_HOSTS = ("qutebrowser.org", "mediumhost.io", "http://*.edu")
BLOCKLIST_HOSTS = ('localhost',
'mediumhost.io',
'malware.badhost.org',
'4-verybadhost.com',
'ads.worsthostever.net')
BLOCKLIST_HOSTS = (
"localhost",
"mediumhost.io",
"malware.badhost.org",
"4-verybadhost.com",
"ads.worsthostever.net",
)
CLEAN_HOSTS = ('goodhost.gov', 'verygoodhost.com')
CLEAN_HOSTS = ("goodhost.gov", "verygoodhost.com")
URLS_TO_CHECK = ('http://localhost',
'http://mediumhost.io',
'ftp://malware.badhost.org',
'http://4-verybadhost.com',
'http://ads.worsthostever.net',
'http://goodhost.gov',
'ftp://verygoodhost.com',
'http://qutebrowser.org',
'http://veryverygoodhost.edu')
URLS_TO_CHECK = (
"http://localhost",
"http://mediumhost.io",
"ftp://malware.badhost.org",
"http://4-verybadhost.com",
"http://ads.worsthostever.net",
"http://goodhost.gov",
"ftp://verygoodhost.com",
"http://qutebrowser.org",
"http://veryverygoodhost.edu",
)
@pytest.fixture
def host_blocker_factory(config_tmpdir, data_tmpdir, download_stub,
config_stub):
def host_blocker_factory(config_tmpdir, data_tmpdir, download_stub, config_stub):
def factory():
return adblock.HostBlocker(config_dir=config_tmpdir,
data_dir=data_tmpdir)
return adblock.HostBlocker(config_dir=config_tmpdir, data_dir=data_tmpdir)
return factory
def create_zipfile(directory, files, zipname='test'):
def create_zipfile(directory, files, zipname="test"):
"""Return a path to a newly created zip file.
Args:
@ -74,17 +77,19 @@ def create_zipfile(directory, files, zipname='test'):
files: list of filenames (relative to directory) to each file to add.
zipname: name to give to the zip file.
"""
zipfile_path = directory / zipname + '.zip'
with zipfile.ZipFile(str(zipfile_path), 'w') as new_zipfile:
zipfile_path = directory / zipname + ".zip"
with zipfile.ZipFile(str(zipfile_path), "w") as new_zipfile:
for file_path in files:
new_zipfile.write(str(directory / file_path),
arcname=os.path.basename(str(file_path)))
new_zipfile.write(
str(directory / file_path), arcname=os.path.basename(str(file_path))
)
# Removes path from file name
return str(zipname + '.zip')
return str(zipname + ".zip")
def create_blocklist(directory, blocked_hosts=BLOCKLIST_HOSTS,
name='hosts', line_format='one_per_line'):
def create_blocklist(
directory, blocked_hosts=BLOCKLIST_HOSTS, name="hosts", line_format="one_per_line"
):
"""Return a path to a blocklist file.
Args:
@ -96,25 +101,29 @@ def create_blocklist(directory, blocked_hosts=BLOCKLIST_HOSTS,
'not_correct' --> Not a correct hosts file format.
"""
blocklist_file = directory / name
with blocklist_file.open('w', encoding='UTF-8') as blocklist:
with blocklist_file.open("w", encoding="UTF-8") as blocklist:
# ensure comments are ignored when processing blocklist
blocklist.write('# Blocked Hosts List #\n\n')
if line_format == 'etc_hosts': # /etc/hosts like format
blocklist.write("# Blocked Hosts List #\n\n")
if line_format == "etc_hosts": # /etc/hosts like format
for host in blocked_hosts:
blocklist.write('127.0.0.1 ' + host + '\n')
elif line_format == 'one_per_line':
blocklist.write("127.0.0.1 " + host + "\n")
elif line_format == "one_per_line":
for host in blocked_hosts:
blocklist.write(host + '\n')
elif line_format == 'not_correct':
blocklist.write(host + "\n")
elif line_format == "not_correct":
for host in blocked_hosts:
blocklist.write(host + ' This is not a correct hosts file\n')
blocklist.write(host + " This is not a correct hosts file\n")
else:
raise ValueError('Incorrect line_format argument')
raise ValueError("Incorrect line_format argument")
return name
def assert_urls(host_blocker, blocked=BLOCKLIST_HOSTS,
whitelisted=WHITELISTED_HOSTS, urls_to_check=URLS_TO_CHECK):
def assert_urls(
host_blocker,
blocked=BLOCKLIST_HOSTS,
whitelisted=WHITELISTED_HOSTS,
urls_to_check=URLS_TO_CHECK,
):
"""Test if Urls to check are blocked or not by HostBlocker.
Ensure URLs in 'blocked' and not in 'whitelisted' are blocked.
@ -122,7 +131,7 @@ def assert_urls(host_blocker, blocked=BLOCKLIST_HOSTS,
localhost is an example of a special case that shouldn't be blocked.
"""
whitelisted = list(whitelisted) + ['localhost']
whitelisted = list(whitelisted) + ["localhost"]
for str_url in urls_to_check:
url = QUrl(str_url)
host = url.host()
@ -135,8 +144,8 @@ def assert_urls(host_blocker, blocked=BLOCKLIST_HOSTS,
def blocklist_to_url(filename):
"""Get an example.com-URL with the given filename as path."""
assert not os.path.isabs(filename), filename
url = QUrl('http://example.com/')
url.setPath('/' + filename)
url = QUrl("http://example.com/")
url.setPath("/" + filename)
assert url.isValid(), url.errorString()
return url
@ -153,54 +162,100 @@ def generic_blocklists(directory):
- a remote text file without valid hosts format.
"""
# remote zip file with 1 hosts file and 2 useless files
file1 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS,
name='README', line_format='not_correct')
file2 = create_blocklist(directory, blocked_hosts=BLOCKLIST_HOSTS[:3],
name='hosts', line_format='etc_hosts')
file3 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS,
name='false_positive', line_format='one_per_line')
file1 = create_blocklist(
directory, blocked_hosts=CLEAN_HOSTS, name="README", line_format="not_correct"
)
file2 = create_blocklist(
directory,
blocked_hosts=BLOCKLIST_HOSTS[:3],
name="hosts",
line_format="etc_hosts",
)
file3 = create_blocklist(
directory,
blocked_hosts=CLEAN_HOSTS,
name="false_positive",
line_format="one_per_line",
)
files_to_zip = [file1, file2, file3]
blocklist1 = blocklist_to_url(
create_zipfile(directory, files_to_zip, 'block1'))
blocklist1 = blocklist_to_url(create_zipfile(directory, files_to_zip, "block1"))
# remote zip file without file named hosts
# (Should raise a FileNotFoundError)
file1 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS,
name='md5sum', line_format='etc_hosts')
file2 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS,
name='README', line_format='not_correct')
file3 = create_blocklist(directory, blocked_hosts=CLEAN_HOSTS,
name='false_positive', line_format='one_per_line')
file1 = create_blocklist(
directory, blocked_hosts=CLEAN_HOSTS, name="md5sum", line_format="etc_hosts"
)
file2 = create_blocklist(
directory, blocked_hosts=CLEAN_HOSTS, name="README", line_format="not_correct"
)
file3 = create_blocklist(
directory,
blocked_hosts=CLEAN_HOSTS,
name="false_positive",
line_format="one_per_line",
)
files_to_zip = [file1, file2, file3]
blocklist2 = blocklist_to_url(
create_zipfile(directory, files_to_zip, 'block2'))
blocklist2 = blocklist_to_url(create_zipfile(directory, files_to_zip, "block2"))
# remote zip file with only one valid hosts file inside
file1 = create_blocklist(directory, blocked_hosts=[BLOCKLIST_HOSTS[3]],
name='malwarelist', line_format='etc_hosts')
blocklist3 = blocklist_to_url(create_zipfile(directory, [file1], 'block3'))
file1 = create_blocklist(
directory,
blocked_hosts=[BLOCKLIST_HOSTS[3]],
name="malwarelist",
line_format="etc_hosts",
)
blocklist3 = blocklist_to_url(create_zipfile(directory, [file1], "block3"))
# local text file with valid hosts
blocklist4 = QUrl.fromLocalFile(str(directory / create_blocklist(
directory, blocked_hosts=[BLOCKLIST_HOSTS[4]],
name='mycustomblocklist', line_format='one_per_line')))
blocklist4 = QUrl.fromLocalFile(
str(
directory
/ create_blocklist(
directory,
blocked_hosts=[BLOCKLIST_HOSTS[4]],
name="mycustomblocklist",
line_format="one_per_line",
)
)
)
assert blocklist4.isValid(), blocklist4.errorString()
# remote text file without valid hosts format
blocklist5 = blocklist_to_url(create_blocklist(
directory, blocked_hosts=CLEAN_HOSTS, name='notcorrectlist',
line_format='not_correct'))
blocklist5 = blocklist_to_url(
create_blocklist(
directory,
blocked_hosts=CLEAN_HOSTS,
name="notcorrectlist",
line_format="not_correct",
)
)
return [blocklist1.toString(), blocklist2.toString(),
blocklist3.toString(), blocklist4.toString(),
blocklist5.toString()]
return [
blocklist1.toString(),
blocklist2.toString(),
blocklist3.toString(),
blocklist4.toString(),
blocklist5.toString(),
]
def test_disabled_blocking_update(config_stub, tmpdir, caplog,
host_blocker_factory):
"""Ensure no URL is blocked when host blocking is disabled."""
@pytest.mark.parametrize(
"blocking_enabled, method",
[
# Assuming the adblock dependency is installed
(True, "auto"),
(True, "adblock"),
(False, "auto"),
(False, "adblock"),
(False, "both"),
(False, "hosts"),
],
)
def test_disabled_blocking_update(config_stub, tmpdir, caplog, host_blocker_factory, blocking_enabled, method):
"""Ensure no URL is blocked when host blocking should be disabled."""
config_stub.val.content.blocking.hosts.lists = generic_blocklists(tmpdir)
config_stub.val.content.blocking.hosts.enabled = False
config_stub.val.content.blocking.enabled = blocking_enabled
config_stub.val.content.blocking.method = method
host_blocker = host_blocker_factory()
host_blocker.adblock_update()
@ -215,14 +270,14 @@ def test_disabled_blocking_update(config_stub, tmpdir, caplog,
def test_disabled_blocking_per_url(config_stub, host_blocker_factory):
example_com = 'https://www.example.com/'
example_com = "https://www.example.com/"
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.hosts.lists = []
pattern = urlmatch.UrlPattern(example_com)
config_stub.set_obj('content.blocking.hosts.enabled', False,
pattern=pattern)
config_stub.set_obj("content.blocking.enabled", False, pattern=pattern)
url = QUrl('blocked.example.com')
url = QUrl("blocked.example.com")
host_blocker = host_blocker_factory()
host_blocker._blocked_hosts.add(url.host())
@ -234,7 +289,8 @@ def test_disabled_blocking_per_url(config_stub, host_blocker_factory):
def test_no_blocklist_update(config_stub, download_stub, host_blocker_factory):
"""Ensure no URL is blocked when no block list exists."""
config_stub.val.content.blocking.hosts.lists = None
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.enabled = True
host_blocker = host_blocker_factory()
host_blocker.adblock_update()
@ -248,7 +304,8 @@ def test_no_blocklist_update(config_stub, download_stub, host_blocker_factory):
def test_successful_update(config_stub, tmpdir, caplog, host_blocker_factory):
"""Ensure hosts from host_blocking.lists are blocked after an update."""
config_stub.val.content.blocking.hosts.lists = generic_blocklists(tmpdir)
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.whitelist = None
host_blocker = host_blocker_factory()
@ -263,37 +320,43 @@ def test_successful_update(config_stub, tmpdir, caplog, host_blocker_factory):
assert_urls(host_blocker, whitelisted=[])
def test_parsing_multiple_hosts_on_line(host_blocker_factory):
def test_parsing_multiple_hosts_on_line(config_stub, host_blocker_factory):
"""Ensure multiple hosts on a line get parsed correctly."""
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.enabled = True
host_blocker = host_blocker_factory()
bytes_host_line = ' '.join(BLOCKLIST_HOSTS).encode('utf-8')
bytes_host_line = " ".join(BLOCKLIST_HOSTS).encode("utf-8")
parsed_hosts = host_blocker._read_hosts_line(bytes_host_line)
host_blocker._blocked_hosts |= parsed_hosts
assert_urls(host_blocker, whitelisted=[])
@pytest.mark.parametrize('ip, host', [
('127.0.0.1', 'localhost'),
('27.0.0.1', 'localhost.localdomain'),
('27.0.0.1', 'local'),
('55.255.255.255', 'broadcasthost'),
(':1', 'localhost'),
(':1', 'ip6-localhost'),
(':1', 'ip6-loopback'),
('e80::1%lo0', 'localhost'),
('f00::0', 'ip6-localnet'),
('f00::0', 'ip6-mcastprefix'),
('f02::1', 'ip6-allnodes'),
('f02::2', 'ip6-allrouters'),
('ff02::3', 'ip6-allhosts'),
('.0.0.0', '0.0.0.0'),
('127.0.1.1', 'myhostname'),
('127.0.0.53', 'myhostname'),
])
@pytest.mark.parametrize(
"ip, host",
[
("127.0.0.1", "localhost"),
("27.0.0.1", "localhost.localdomain"),
("27.0.0.1", "local"),
("55.255.255.255", "broadcasthost"),
(":1", "localhost"),
(":1", "ip6-localhost"),
(":1", "ip6-loopback"),
("e80::1%lo0", "localhost"),
("f00::0", "ip6-localnet"),
("f00::0", "ip6-mcastprefix"),
("f02::1", "ip6-allnodes"),
("f02::2", "ip6-allrouters"),
("ff02::3", "ip6-allhosts"),
(".0.0.0", "0.0.0.0"),
("127.0.1.1", "myhostname"),
("127.0.0.53", "myhostname"),
],
)
def test_whitelisted_lines(host_blocker_factory, ip, host):
"""Make sure we don't block hosts we don't want to."""
host_blocker = host_blocker_factory()
line = ('{} {}'.format(ip, host)).encode('ascii')
line = ("{} {}".format(ip, host)).encode("ascii")
parsed_hosts = host_blocker._read_hosts_line(line)
assert host not in parsed_hosts
@ -303,13 +366,18 @@ def test_failed_dl_update(config_stub, tmpdir, caplog, host_blocker_factory):
Ensure hosts from this list are not blocked.
"""
dl_fail_blocklist = blocklist_to_url(create_blocklist(
tmpdir, blocked_hosts=CLEAN_HOSTS, name='download_will_fail',
line_format='one_per_line'))
hosts_to_block = (generic_blocklists(tmpdir) +
[dl_fail_blocklist.toString()])
dl_fail_blocklist = blocklist_to_url(
create_blocklist(
tmpdir,
blocked_hosts=CLEAN_HOSTS,
name="download_will_fail",
line_format="one_per_line",
)
)
hosts_to_block = generic_blocklists(tmpdir) + [dl_fail_blocklist.toString()]
config_stub.val.content.blocking.hosts.lists = hosts_to_block
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.whitelist = None
host_blocker = host_blocker_factory()
@ -327,37 +395,36 @@ def test_failed_dl_update(config_stub, tmpdir, caplog, host_blocker_factory):
assert_urls(host_blocker, whitelisted=[])
@pytest.mark.parametrize('location', ['content', 'comment'])
def test_invalid_utf8(config_stub, tmpdir, caplog, host_blocker_factory,
location):
@pytest.mark.parametrize("location", ["content", "comment"])
def test_invalid_utf8(config_stub, tmpdir, caplog, host_blocker_factory, location):
"""Make sure invalid UTF-8 is handled correctly.
See https://github.com/qutebrowser/qutebrowser/issues/2301
"""
blocklist = tmpdir / 'blocklist'
if location == 'comment':
blocklist.write_binary(b'# nbsp: \xa0\n')
blocklist = tmpdir / "blocklist"
if location == "comment":
blocklist.write_binary(b"# nbsp: \xa0\n")
else:
assert location == 'content'
blocklist.write_binary(b'https://www.example.org/\xa0')
assert location == "content"
blocklist.write_binary(b"https://www.example.org/\xa0")
for url in BLOCKLIST_HOSTS:
blocklist.write(url + '\n', mode='a')
blocklist.write(url + "\n", mode="a")
url = blocklist_to_url('blocklist')
url = blocklist_to_url("blocklist")
config_stub.val.content.blocking.hosts.lists = [url.toString()]
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.whitelist = None
host_blocker = host_blocker_factory()
host_blocker.adblock_update()
current_download = host_blocker._in_progress[0]
if location == 'content':
if location == "content":
with caplog.at_level(logging.ERROR):
current_download.successful = True
current_download.finished.emit()
expected = (r"Failed to decode: "
r"b'https://www.example.org/\xa0localhost")
expected = r"Failed to decode: " r"b'https://www.example.org/\xa0localhost"
assert caplog.messages[-2].startswith(expected)
else:
current_download.successful = True
@ -367,18 +434,17 @@ def test_invalid_utf8(config_stub, tmpdir, caplog, host_blocker_factory,
assert_urls(host_blocker, whitelisted=[])
def test_invalid_utf8_compiled(config_stub, config_tmpdir, data_tmpdir,
monkeypatch, caplog, host_blocker_factory):
def test_invalid_utf8_compiled(
config_stub, config_tmpdir, data_tmpdir, monkeypatch, caplog, host_blocker_factory
):
"""Make sure invalid UTF-8 in the compiled file is handled."""
config_stub.val.content.blocking.hosts.lists = []
# Make sure the HostBlocker doesn't delete blocked-hosts in __init__
monkeypatch.setattr(adblock.HostBlocker, 'update_files',
lambda _self: None)
monkeypatch.setattr(adblock.HostBlocker, "update_files", lambda _self: None)
(config_tmpdir / 'blocked-hosts').write_binary(
b'https://www.example.org/\xa0')
(data_tmpdir / 'blocked-hosts').ensure()
(config_tmpdir / "blocked-hosts").write_binary(b"https://www.example.org/\xa0")
(data_tmpdir / "blocked-hosts").ensure()
host_blocker = host_blocker_factory()
with caplog.at_level(logging.ERROR):
@ -392,12 +458,15 @@ def test_blocking_with_whitelist(config_stub, data_tmpdir, host_blocker_factory)
# by creating a file named blocked-hosts,
# Exclude localhost from it as localhost is never blocked via list
filtered_blocked_hosts = BLOCKLIST_HOSTS[1:]
blocklist = create_blocklist(data_tmpdir,
blocked_hosts=filtered_blocked_hosts,
name='blocked-hosts',
line_format='one_per_line')
blocklist = create_blocklist(
data_tmpdir,
blocked_hosts=filtered_blocked_hosts,
name="blocked-hosts",
line_format="one_per_line",
)
config_stub.val.content.blocking.hosts.lists = [blocklist]
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.whitelist = list(WHITELISTED_HOSTS)
host_blocker = host_blocker_factory()
@ -413,10 +482,15 @@ def test_config_change_initial(config_stub, tmpdir, host_blocker_factory):
- User quits qutebrowser, empties host_blocking.lists from his config
- User restarts qutebrowser, does adblock-update
"""
create_blocklist(tmpdir, blocked_hosts=BLOCKLIST_HOSTS,
name='blocked-hosts', line_format='one_per_line')
create_blocklist(
tmpdir,
blocked_hosts=BLOCKLIST_HOSTS,
name="blocked-hosts",
line_format="one_per_line",
)
config_stub.val.content.blocking.hosts.lists = None
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.whitelist = None
host_blocker = host_blocker_factory()
@ -428,11 +502,17 @@ def test_config_change_initial(config_stub, tmpdir, host_blocker_factory):
def test_config_change(config_stub, tmpdir, host_blocker_factory):
"""Ensure blocked-hosts resets if host-block-list is changed to None."""
filtered_blocked_hosts = BLOCKLIST_HOSTS[1:] # Exclude localhost
blocklist = blocklist_to_url(create_blocklist(
tmpdir, blocked_hosts=filtered_blocked_hosts, name='blocked-hosts',
line_format='one_per_line'))
blocklist = blocklist_to_url(
create_blocklist(
tmpdir,
blocked_hosts=filtered_blocked_hosts,
name="blocked-hosts",
line_format="one_per_line",
)
)
config_stub.val.content.blocking.hosts.lists = [blocklist.toString()]
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.method = "hosts"
config_stub.val.content.blocking.whitelist = None
host_blocker = host_blocker_factory()
@ -447,26 +527,34 @@ def test_add_directory(config_stub, tmpdir, host_blocker_factory):
"""Ensure adblocker can import all files in a directory."""
blocklist_hosts2 = []
for i in BLOCKLIST_HOSTS[1:]:
blocklist_hosts2.append('1' + i)
blocklist_hosts2.append("1" + i)
create_blocklist(tmpdir, blocked_hosts=BLOCKLIST_HOSTS,
name='blocked-hosts', line_format='one_per_line')
create_blocklist(tmpdir, blocked_hosts=blocklist_hosts2,
name='blocked-hosts2', line_format='one_per_line')
create_blocklist(
tmpdir,
blocked_hosts=BLOCKLIST_HOSTS,
name="blocked-hosts",
line_format="one_per_line",
)
create_blocklist(
tmpdir,
blocked_hosts=blocklist_hosts2,
name="blocked-hosts2",
line_format="one_per_line",
)
config_stub.val.content.blocking.hosts.lists = [tmpdir.strpath]
config_stub.val.content.blocking.hosts.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.method = "hosts"
host_blocker = host_blocker_factory()
host_blocker.adblock_update()
assert len(host_blocker._blocked_hosts) == len(blocklist_hosts2) * 2
def test_adblock_benchmark(data_tmpdir, benchmark, host_blocker_factory):
blocked_hosts = data_tmpdir / 'blocked-hosts'
blocked_hosts.write_text('\n'.join(utils.blocked_hosts()),
encoding='utf-8')
blocked_hosts = data_tmpdir / "blocked-hosts"
blocked_hosts.write_text("\n".join(utils.blocked_hosts()), encoding="utf-8")
url = QUrl('https://www.example.org/')
url = QUrl("https://www.example.org/")
blocker = host_blocker_factory()
blocker.read_hosts()
assert blocker._blocked_hosts

View File

@ -163,7 +163,7 @@ def easylist_easyprivacy(tmpdir):
@pytest.fixture
def ad_blocker(data_tmpdir):
def ad_blocker(config_stub, data_tmpdir):
return braveadblock.BraveAdBlocker(data_dir=pathlib.Path(data_tmpdir))
@ -182,7 +182,17 @@ def assert_urls(
@pytest.mark.parametrize(
"blocking_enabled, should_be_blocked", [(True, True), (False, False)]
"blocking_enabled, method, should_be_blocked",
[
(True, "auto", True),
(True, "adblock", True),
(True, "both", True),
(True, "hosts", False),
(False, "auto", False),
(False, "adblock", False),
(False, "both", False),
(False, "hosts", False),
],
)
def test_blocking_enabled(
config_stub,
@ -190,11 +200,13 @@ def test_blocking_enabled(
caplog,
ad_blocker,
blocking_enabled,
method,
should_be_blocked,
):
"""Tests that the ads are blocked when the adblocker is enabled, and vice versa."""
config_stub.val.content.blocking.adblock.lists = easylist_easyprivacy
config_stub.val.content.blocking.adblock.enabled = blocking_enabled
config_stub.val.content.blocking.enabled = blocking_enabled
config_stub.val.content.blocking.method = method
ad_blocker.adblock_update()
while ad_blocker._in_progress:
@ -208,7 +220,7 @@ def test_blocking_enabled(
def test_adblock_cache(config_stub, easylist_easyprivacy, caplog, ad_blocker):
config_stub.val.content.blocking.adblock.lists = easylist_easyprivacy
config_stub.val.content.blocking.adblock.enabled = True
config_stub.val.content.blocking.enabled = True
for i in range(3):
print("At cache test iteration {}".format(i))
@ -250,7 +262,7 @@ def test_adblock_cache(config_stub, easylist_easyprivacy, caplog, ad_blocker):
def test_invalid_utf8(ad_blocker, config_stub, blocklist_invalid_utf8, caplog):
"""Test that the adblocker handles invalid utf-8 correctly."""
config_stub.val.content.blocking.adblock.lists = [blocklist_invalid_utf8]
config_stub.val.content.blocking.adblock.enabled = True
config_stub.val.content.blocking.enabled = True
with caplog.at_level(logging.INFO):
ad_blocker.adblock_update()
@ -260,7 +272,7 @@ def test_invalid_utf8(ad_blocker, config_stub, blocklist_invalid_utf8, caplog):
def test_config_changed(ad_blocker, config_stub, easylist_easyprivacy, caplog):
"""Ensure blocked-hosts resets if host-block-list is changed to None."""
config_stub.val.content.blocking.adblock.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.whitelist = None
for _ in range(2):
@ -295,7 +307,7 @@ def test_config_changed(ad_blocker, config_stub, easylist_easyprivacy, caplog):
def test_whitelist_on_dataset(config_stub, easylist_easyprivacy):
config_stub.val.content.blocking.adblock.lists = easylist_easyprivacy
config_stub.val.content.blocking.adblock.enabled = True
config_stub.val.content.blocking.enabled = True
config_stub.val.content.blocking.whitelist = None
def assert_whitelisted(url, source_url, resource_type):