TheCloneArmy/Unified_Porn_Blocklist
1
0
Fork 0
mirror of https://github.com/columndeeply/hosts.git synced 2025-10-27 11:31:09 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add sed command to remove all domains not containing a dot (should get rid of all invalid domains)

Browse Source
This commit is contained in:
columndeeply
2025-08-10 15:15:05 +02:00
parent 65a0727c19
commit d0275f952f
1 changed files with 20 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
scripts/cleanup.sh
View File

@ -7,28 +7,30 @@
#
for file in "$@"; do
[ ! -f "$file" ] && echo "$file does not exist" && continue
[ ! -f "$file" ] && echo "$file does not exist" && continue
# Remove comments, stuff pointing to localhost, whitespaces, tabs, etc.
sed -i 's/#.*$//g;/^$/d;/localhost$/d;/::/d;/local$/d;/localdomain$/d;/broadcasthost$/d;/0.0.0.0$/d;/^[[:space:]]*$/d;s/[ \t]*$//g;s/^[ \t]*//g;s/[[:blank:]]/ /g;s/https\?:\/\///g;s/\/.*$//g;s/\?.*$//g' "$file"
# Remove comments, stuff pointing to localhost, whitespaces, tabs, etc.
sed -i 's/#.*$//g;/^$/d;/localhost$/d;/::/d;/local$/d;/localdomain$/d;/broadcasthost$/d;/0.0.0.0$/d;/^[[:space:]]*$/d;s/[ \t]*$//g;s/^[ \t]*//g;s/[[:blank:]]/ /g;s/https\?:\/\///g;s/\/.*$//g;s/\?.*$//g' "$file"
# All domains should point to 127.0.0.1
## Add it to all lines not starting with an IP address
sed -i -r '/^([0-9]{1,3}\.){3}[0-9]{1,3} /! s/^/127.0.0.1 /' "$file"
## Replace all IPs with 127.0.0.1
sed -i -r 's/^([0-9]{1,3}\.){3}[0-9]{1,3} /127.0.0.1 /g' "$file"
## Remove any lines pointing to an IP address, hosts file only works with domains or hostnames
sed -i -r '/ ([0-9]{1,3}\.){3}[0-9]{1,3}$/d' "$file"
# All domains should point to 127.0.0.1
## Add it to all lines not starting with an IP address
sed -i -r '/^([0-9]{1,3}\.){3}[0-9]{1,3} /! s/^/127.0.0.1 /' "$file"
## Replace all IPs with 127.0.0.1
sed -i -r 's/^([0-9]{1,3}\.){3}[0-9]{1,3} /127.0.0.1 /g' "$file"
## Remove any lines pointing to an IP address, hosts file only works with domains or hostnames
sed -i -r '/ ([0-9]{1,3}\.){3}[0-9]{1,3}$/d' "$file"
## Remove any lines that don't have a dot after the 127.0.0.1
sed -i '/127\.0\.0\.1 [^ ]*\.[^ ]*/!d' "$file"
# Remove multiple spaces and return lines (If anybody knows how to do this with sed please let me know)
tr -s ' ' < "$file" | tr -d '\r' > "$file.tmp"
# Remove multiple spaces and return lines (If anybody knows how to do this with sed please let me know)
tr -s ' ' < "$file" | tr -d '\r' > "$file.tmp"
# Remove any domain that exists in the whitelist
grep -v -x -f ../whitelist "$file.tmp" > "$file.tmp2"
# Remove any domain that exists in the whitelist
grep -v -x -f ../whitelist "$file.tmp" > "$file.tmp2"
# Remove duplicates
sort -u -o "$file"_clean "$file.tmp2"
# Remove duplicates
sort -u -o "$file"_clean "$file.tmp2"
# Remove the old files
rm "$file.tmp" "$file.tmp2" "$file"
# Remove the old files
rm "$file.tmp" "$file.tmp2" "$file"
done