From d0275f952fd697924cfa1a7738da790444261eb4 Mon Sep 17 00:00:00 2001 From: columndeeply Date: Sun, 10 Aug 2025 15:15:05 +0200 Subject: [PATCH] Add sed command to remove all domains not containing a dot (should get rid of all invalid domains) --- scripts/cleanup.sh | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index babb2b1..7ee5330 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -7,28 +7,30 @@ # for file in "$@"; do - [ ! -f "$file" ] && echo "$file does not exist" && continue + [ ! -f "$file" ] && echo "$file does not exist" && continue - # Remove comments, stuff pointing to localhost, whitespaces, tabs, etc. - sed -i 's/#.*$//g;/^$/d;/localhost$/d;/::/d;/local$/d;/localdomain$/d;/broadcasthost$/d;/0.0.0.0$/d;/^[[:space:]]*$/d;s/[ \t]*$//g;s/^[ \t]*//g;s/[[:blank:]]/ /g;s/https\?:\/\///g;s/\/.*$//g;s/\?.*$//g' "$file" + # Remove comments, stuff pointing to localhost, whitespaces, tabs, etc. + sed -i 's/#.*$//g;/^$/d;/localhost$/d;/::/d;/local$/d;/localdomain$/d;/broadcasthost$/d;/0.0.0.0$/d;/^[[:space:]]*$/d;s/[ \t]*$//g;s/^[ \t]*//g;s/[[:blank:]]/ /g;s/https\?:\/\///g;s/\/.*$//g;s/\?.*$//g' "$file" - # All domains should point to 127.0.0.1 - ## Add it to all lines not starting with an IP address - sed -i -r '/^([0-9]{1,3}\.){3}[0-9]{1,3} /! s/^/127.0.0.1 /' "$file" - ## Replace all IPs with 127.0.0.1 - sed -i -r 's/^([0-9]{1,3}\.){3}[0-9]{1,3} /127.0.0.1 /g' "$file" - ## Remove any lines pointing to an IP address, hosts file only works with domains or hostnames - sed -i -r '/ ([0-9]{1,3}\.){3}[0-9]{1,3}$/d' "$file" + # All domains should point to 127.0.0.1 + ## Add it to all lines not starting with an IP address + sed -i -r '/^([0-9]{1,3}\.){3}[0-9]{1,3} /! s/^/127.0.0.1 /' "$file" + ## Replace all IPs with 127.0.0.1 + sed -i -r 's/^([0-9]{1,3}\.){3}[0-9]{1,3} /127.0.0.1 /g' "$file" + ## Remove any lines pointing to an IP address, hosts file only works with domains or hostnames + sed -i -r '/ ([0-9]{1,3}\.){3}[0-9]{1,3}$/d' "$file" + ## Remove any lines that don't have a dot after the 127.0.0.1 + sed -i '/127\.0\.0\.1 [^ ]*\.[^ ]*/!d' "$file" - # Remove multiple spaces and return lines (If anybody knows how to do this with sed please let me know) - tr -s ' ' < "$file" | tr -d '\r' > "$file.tmp" + # Remove multiple spaces and return lines (If anybody knows how to do this with sed please let me know) + tr -s ' ' < "$file" | tr -d '\r' > "$file.tmp" - # Remove any domain that exists in the whitelist - grep -v -x -f ../whitelist "$file.tmp" > "$file.tmp2" + # Remove any domain that exists in the whitelist + grep -v -x -f ../whitelist "$file.tmp" > "$file.tmp2" - # Remove duplicates - sort -u -o "$file"_clean "$file.tmp2" + # Remove duplicates + sort -u -o "$file"_clean "$file.tmp2" - # Remove the old files - rm "$file.tmp" "$file.tmp2" "$file" + # Remove the old files + rm "$file.tmp" "$file.tmp2" "$file" done