mirror
/
sitespeed.io
mirror of https://github.com/sitespeedio/sitespeed.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added fake useragent

This commit is contained in:
Peter Hedenskog 2012-10-08 08:56:51 +02:00
parent 373a81ed3d
commit 69d3550b31
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sitespeed.io
View File

@ -65,10 +65,12 @@ RETRIES=1
index=0
isVerified=false
isHTML=false
#Faking firefox as useragent
USERAGENT='Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2'
echo "Will start fetching all a links ..."
wget -r -l $DEPTH -nd -t $RETRIES -e robots=off --no-check-certificate --follow-tags=a --spider $URL 2>&1 | while read line
wget -r -l $DEPTH -nd -t $RETRIES -e robots=off --no-check-certificate --follow-tags=a --spider -U $USERAGENT $URL 2>&1 | while read line
do
# The spider option checks if a file exist, only fetch only existing