🦖Yabba-Dabba Do! | Bashing out a Threat Intelligence Script☄️

Ronald BartelsRonald Bartels
6 min read

Linux hosts of various types can be protected using Threat Intelligence feeds and various blocking techniques. He is a variant I’ve used that is done in bash. The script uses nftables and has the package iprange as a dependency.

/usr/local/sbin/extrathreatblock.sh

#!/bin/bash
# Extra Threatblock: A threat intelligence script by r00igev@@r/mybroadband
# Version 0.0.2 October 2023
# usage extra-threatblock.sh <configuration file>
# eg: extra-threatblock.sh /etc/extra/threatblock/extra-threatblock.conf
#
PATH=/usr/sbin:/usr/local/bin:/usr/bin:/bin:

function exists() { command -v "$1" >/dev/null 2>&1 ; }

# Check that configuration file has been specified
if [[ -z "$1" ]]; then
  echo "Error: please specify a configuration file, e.g. $0 /etc/extra/threatblock/extra-threatblock.conf"
  exit 1
fi

# Shellcheck source=extra-threatblock.conf
if ! source "$1"; then
  echo "Error: can't load configuration file $1"
  exit 1
fi

# Check if all commands exist
if ! exists curl && exists egrep && exists grep && exists nft && exists sed && exists sort && exists wc && exists iprange && figlet ; then
  echo >&2 "Error: searching PATH fails to find executables among: curl egrep grep nft sed sort wc iprange figlet"
  exit 1
fi

figlet -f slant Extra Threatblock
echo "Extra enabled gateway. Driving SD-WAN adoption in South Africa! https://fusionsdwan.co.za"

# Check if BLOCK_POLICY is defined
if [ -z "$BLOCK_POLICY" ]; then
  echo "Error: BLOCK_POLICY is not defined. Please set the BLOCK_POLICY variable."
  exit 1
fi

# Check if WAN_IF is defined
if [ -z "$WAN_IF" ]; then
  echo "Error: WAN_IF is not defined. Please set the WAN_IF variable."
  exit 1
fi

# Do CIDR optimization if set
DO_OPTIMIZE_CIDR=no
if exists iprange && [[ ${OPTIMIZE_CIDR:-yes} != no ]]; then
  DO_OPTIMIZE_CIDR=yes
fi

# Remove comments from exceptions files
if [ -f "$INTERNAL_BLOCKLIST_EXCEPTIONS" ]; then
   INTERNAL_EXCEPTIONS_TMP=$(mktemp)
   for exception in $(sed -r -e 's/\s*#.*$//;/^$/d' "$INTERNAL_BLOCKLIST_EXCEPTIONS")
      do
         exception_array+=( "$exception" )
         echo $exception >> $INTERNAL_EXCEPTIONS_TMP
   done
fi
if [ -f "$EXTERNAL_BLOCKLIST_EXCEPTIONS" ]; then
   EXTERNAL_EXCEPTIONS_TMP=$(mktemp)
   for exception in $(sed -r -e 's/\s*#.*$//;/^$/d' "$EXTERNAL_BLOCKLIST_EXCEPTIONS")
      do
         exception_array+=( "$exception" )
         echo $exception >> $EXTERNAL_EXCEPTIONS_TMP
   done
fi

# Create the nftables set if needed (or abort if does not exist and FORCE=no)
# Check if the template file exists
if [ ! -f "$NFT_TEMPLATE" ]; then
  echo "Error: Template file not found at $NFT_TEMPLATE."
  exit 1
fi

# Use the nft command to delete the table
nft delete table inet "$BLOCKLIST_NAME" 2>/dev/null

echo "nftables table '$BLOCKLIST_NAME' dropped."

# Read the template file, replaces ${WAN_IF} and ${BLOCK_POLICY} with the actual values, and create the nftables table
nft -f <(sed -e "s/\${BLOCK_POLICY}/$BLOCK_POLICY/g" -e "s/\${WAN_IF}/$WAN_IF/g" "$NFT_TEMPLATE")

echo "nftables table '$BLOCKLIST_NAME' created with with WAN interface $WAN_IF and block policy $BLOCK_POLICY."

BLOCKLIST_TMP=$(mktemp)
echo -n "Downloading blacklists:"
for i in "${BLOCKLISTS[@]}"
do
  IP_TMP=$(mktemp)
  (( HTTP_RC=$(curl -L -A "extra-threatblock/script/mybroadband" --connect-timeout 10 --max-time 10 -o "$IP_TMP" -s -w "%{http_code}" "$i") ))
  if (( HTTP_RC == 200 || HTTP_RC == 302 || HTTP_RC == 0 )); then # "0" because file:/// returns 000
    command grep -Po '^(?:\d{1,3}\.){3}\d{1,3}(?:/\d{1,2})?' "$IP_TMP" | sed -r 's/^0*([0-9]+)\.0*([0-9]+)\.0*([0-9]+)\.0*([0-9]+)$/\1.\2.\3.\4/' >> "$BLOCKLIST_TMP"
    [[ ${VERBOSE:-yes} == yes ]] && echo -n "."
  elif (( HTTP_RC == 503 )); then
    echo -e "\\nUnavailable (${HTTP_RC}): $i"
  else
    echo >&2 -e "\\nWarning: curl returned HTTP response code $HTTP_RC for URL $i"
  fi
  rm -f "$IP_TMP"
done

# Optimize CIDR
sed -r -e '/^(0\.0\.0\.0|10\.|127\.|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[0-1]\.|192\.168\.|22[4-9]\.|23[0-9]\.)/d' "$BLOCKLIST_TMP"|sort -n|sort -mu >| "$BLOCKLIST"
if [[ ${OPTIMIZE_CIDR} == yes ]]; then
  if [[ ${VERBOSE:-no} == yes ]]; then
    echo -e "\\nAddresses before CIDR optimization: $(wc -l "$BLOCKLIST" | cut -d' ' -f1)"
  fi
  < "$BLOCKLIST" iprange --optimize - > "$BLOCKLIST_TMP" 2>/dev/null
  if [[ ${#exception_array[@]} > 0 ]]; then
    echo "Allowing for ${#exception_array[@]} exclusions from threatblock"
    echo "Addresses before removing exclusions: $(wc -l "$BLOCKLIST_TMP" | cut -d' ' -f1)"
    BLOCKLIST_WITH_EXCEPT_TMP=$(mktemp)
    iprange "$BLOCKLIST_TMP" --except "$EXTERNAL_EXCEPTIONS_TMP" > "$BLOCKLIST_WITH_EXCEPT_TMP" 2>/dev/null
    cp "$BLOCKLIST_WITH_EXCEPT_TMP" "$BLOCKLIST_TMP"
  fi
  if [[ ${VERBOSE:-no} == yes ]]; then
    echo "Addresses after CIDR optimization:  $(wc -l "$BLOCKLIST_TMP" | cut -d' ' -f1)"
  fi
  cp "$BLOCKLIST_TMP" "$BLOCKLIST"
fi

rm -f "$BLOCKLIST_TMP"

echo -n "Populating nft threatblock:"

# Define the name of the set and the filename containing IPs/subnets
set_name="hexceptions-v4"
blocklist_file="$EXTERNAL_EXCEPTIONS_TMP"
# Read the internal exceptions file line by line
while IFS= read -r line; do
  # Add the IP/subnet to the nftables set 2>/dev/null
  nft add element inet threatblock "$set_name" { $line }
  # Increment the count
done < "$blocklist_file"
set_name="iexceptions-v4"
blocklist_file="$INTERNAL_EXCEPTIONS_TMP"
# Read the internal exceptions file line by line
while IFS= read -r line; do
  # Add the IP/subnet to the nftables set 2>/dev/null
  nft add element inet threatblock "$set_name" { $line }
  # Increment the count
done < "$blocklist_file"

# Define the name of the set and the filename containing IPs/subnets
set_name="blocklist-v4"
blocklist_file="$BLOCKLIST"
# Count for printing dots
count=0
# Number of IPs to add in each batch - too big and nft will not load it
batch_size=512
# Initialize a string to store IPs with commas
ip_batch=""
# Read the blocklist file line by line
while IFS= read -r line; do
  # Add the IP to the batch with a comma separator
  if [ -z "$ip_batch" ]; then
    ip_batch="$line"
  else
    ip_batch="$ip_batch,$line"
  fi
  # If the batch size is reached, add IPs and print a dot
  if (( count % batch_size == batch_size - 1 )); then
    nft add element inet threatblock "$set_name" { $ip_batch } 2>/dev/null
    ((count += 1))
    echo -n "."
    ip_batch="" # Clear the batch
  fi
  ((count++))
done < "$blocklist_file"
# Add any remaining IPs in the last batch
if [ -n "$ip_batch" ]; then
  nft add element inet threatblock "$set_name" { $ip_batch } 2>/dev/null
  ((count++))
  echo -n "."
fi
echo

if [[ ${VERBOSE:-no} == yes ]]; then
   echo "Threatblock completed. $count IPs/subnets added to $set_name."
fi

/etc/extra/threatblock/extra-threatblock.conf

BLOCKLIST_NAME=threatblock # Table used within nftables
BLOCKLIST_TMP=${BLOCKLIST_NAME}-tmp
BLOCKLIST_DIR=/etc/extra/threatblock
BLOCKLIST=${BLOCKLIST_DIR}/extra-threatblock.list
INTERNAL_BLOCKLIST_EXCEPTIONS=${BLOCKLIST_DIR}/internal-${BLOCKLIST_NAME}.exceptions
EXTERNAL_BLOCKLIST_EXCEPTIONS=${BLOCKLIST_DIR}/external-${BLOCKLIST_NAME}.exceptions
NFT_TEMPLATE=${BLOCKLIST_DIR}/nft-blocklist.template # Template file for nftables rules

VERBOSE=yes # Set to no for cron jobs, default to yes
FORCE=yes # Will create the nft table if it does not already exist
OPTIMIZE_CIDR=yes # Optimize block list by aggregating overlapping subnets

# Block policy: 'drop' or 'reject', default: 'drop'
BLOCK_POLICY=drop

# WAN interface
WAN_IF=ens192
# LAN interface
LAN_IF=ens224.900

# Blocklists of IPs being dropped
BLOCKLISTS=(
    "file://${BLOCKLIST_DIR}/${BLOCKLIST_NAME}-custom.list" # Optional, for your personal nemeses
    "https://reputation.alienvault.com/reputation.generic" # Alienvault
    "https://iplists.firehol.org/files/talosintel_ipfilter.ipset" # Talos
    "https://iplists.firehol.org/files/tor_exits_7d.ipset" # TOR exit nodes
    "https://cinsscore.com/list/ci-badguys.txt" # C.I. Army Malicious IP List
    "https://lists.blocklist.de/lists/all.txt" # blocklist.de attackers
    "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt" # Emerging threats
    "https://rules.emergingthreats.net/blockrules/compromised-ips.txt" # Emerging threats
    "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset" # Firehol Level 1
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_shadowserver.txt" # Shadow server
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_shodan.txt" # Shodan
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_sogou.txt" # Sogou search engine
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_internet_cens.txt" # Internet census
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_diverseenvironment.txt" # Diverse environment
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_netsysres.txt" # Net Systems Research
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_rwth-aachen.txt" # AAChen
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_onyphe.txt" # onephe
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_stretchoid.txt" # Strechoid
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_openportsstats.txt" # OpenPortStats
    "https://iplists.firehol.org/files/normshield_high_webscan.ipset" # Normshield high risk scanners
    "https://raw.githubusercontent.com/stamparm/ipsum/master/levels/3.txt" # IPSUM
    "https://raw.githubusercontent.com/ipverse/rir-ip/master/country/ru/ipv4-aggregated.txt" # Russia se push
    "https://raw.githubusercontent.com/ipverse/rir-ip/master/country/ng/ipv4-aggregated.txt" # Nigeria se push
)
MAXELEM=131072

/etc/extra/threatblock/nft-blocklist.template

table inet threatblock {

        set canary_ports {
                type inet_service
                elements = { 23, 25, 110, 139, 445, 554, 1080, 3389, 5900, 7547, 8291, 8080, 31337 }
        }
        set trap {
                type ipv4_addr
                flags dynamic, timeout
                timeout 8h
        }
        set hexceptions-v4 {
                type ipv4_addr
                flags interval
                auto-merge
        }
        set hexceptions-v6 {
                type ipv6_addr
                flags interval
                auto-merge
        }
        set iexceptions-v4 {
                type ipv4_addr
                flags interval
                auto-merge
        }
        set iexceptions-v6 {
                type ipv6_addr
                flags interval
                auto-merge
        }
        set blocklist-v4 {
                type ipv4_addr
                flags interval
                auto-merge
        }
        set blocklist-v6 {
                type ipv6_addr
                flags interval
                auto-merge
        }
        chain forward {
                type filter hook forward priority -1; policy accept;
                iifname lo accept
                ct state established,related accept
                ip daddr @blocklist-v4 counter ${BLOCK_POLICY}
                ip6 daddr @blocklist-v6 counter ${BLOCK_POLICY}
                ip saddr @blocklist-v4 counter ${BLOCK_POLICY}
                ip6 saddr @blocklist-v6 counter ${BLOCK_POLICY}
                counter
        }

        chain input {
                type filter hook input priority -1; policy accept;
                iifname lo accept
                ct state established,related accept
                ip saddr @hexceptions-v4 counter accept
                ip6 saddr @hexceptions-v6 counter accept
                iifname ${WAN_IF} tcp dport @canary_ports add @trap { ip saddr }
                ip saddr @blocklist-v4 counter ${BLOCK_POLICY}
                ip6 saddr @blocklist-v6 counter ${BLOCK_POLICY}
                ip saddr @trap drop
                counter
        }
        chain output {
                type filter hook output priority -1; policy accept;
                iifname lo accept
                ct state established,related accept
                ip daddr @blocklist-v4 counter ${BLOCK_POLICY}
                ip6 daddr @blocklist-v6 counter ${BLOCK_POLICY}
                counter
        }

}

/etc/extra/threatblock/external-threatblock.exceptions

9.9.9.9
1.1.1.2

/etc/extra/threatblock/internal-threatblock.exceptions

192.168.0.0/16

The following Linux package is a good additional tool to have on your host:

https://hubandspoke.amastelek.com/protect-your-linux-system-with-sshguard-a-better-alternative-to-fail2ban
10
Subscribe to my newsletter

Read articles from Ronald Bartels directly inside your inbox. Subscribe to the newsletter, and don't miss out.

LinuxBashscriptsthreat intelligencecybersecuritynftables

Written by

Ronald Bartels
Ronald Bartels

Driving SD-WAN Adoption in South Africa