source: nutchez-0.2/src/test/duplicate_del @ 169

#!/bin/bash
# Program:
#   Delete duplicating ip $ hostname in file (for nutchez management interface).
# Author: 
#   Waue, Shunfa, Rock {waue, shunfa, rock}@nchc.org.tw
# Version:
#    1.0
# History:
#   2010/06/07  Rock    First release (1.0)

# Delete duplicating ip addresss in file
IPs=$(cat $1 | grep -v 'ip6' | grep -v '#' | grep -v '^$' | awk '{print $1}')
cp -f $1 ${1}.bak

for ip_add in $(echo $IPs)
do
    ip_nu=$(cat -n $1 | grep $ip_add | awk '{print $1}')
    ip_count=$(echo $ip_nu | wc -w)
  
    for (( i=1; i<${ip_count}; i++ ))
    do
        del_line=$(echo $ip_nu | cut -d " " -f${i})
        sed -i "${del_line}d" $1
    done
done
  

# Dlete duplicating hostname in file
hostnames=$(cat $1 | grep -v ip6 | grep -v '#' | grep -v '^$' |awk '{print $2}')
for host in $(echo $hostnames)
do
    # line numbers
    host_nu=$(cat -n $1 | grep "$host\$" | awk '{print $1}')
    host_count=$(echo $host_nu | wc -w)
  
    for (( i=1; i<${host_count}; i++ ))
    do
        del_line=$(echo $host_nu | cut -d " " -f${i})
        sed -i "${del_line}d" $1
    done                                                                                                                                                             
done