#!/bin/sh
#author:Wos
#scrape and test proxy available and speed
PXY="http://127.0.0.1:7891"
PROXYFILE=/tmp/proxyfile.txt

function logo()
{
    echo -ne "\033[96;40m"
    base64 -d <<<"H4sIAAAAAAAAA62TS24EIQxE95zCysabxuzhKkjNQXz42C73bzLRSFEYIQFTPIoyTfQvrbxb3Ko1
kU97tyohfOVILrbsE2uS0peDNvH/XTifnCYkTYHAxu5jnxLfSRubgEk1JDiwnFZcyjTocDQkBCZs
S0A7vRiFwLFuvnpy2r5LI5+rYwZQywcm1FrX5SZ3myOBH7Vdxjmt2PFMMiAbRh0+MOHY79l09H4R
w49Z2d2KRgzEeWkNCk+kSEsenDC5QYocykBOc6RKIOP4dVpHZef9Xm6yKazrquw5TwSSCShjatzm
qn0neNKf+WAEAwU1jtP8QYwYWufN7XrdzRLTVTCGH8YNOQ8oCZmxNhFxPotGjBrQDXP40SPxfnIS
7PX9opMbb8TzMVjlZ937eVW9c3LnOKQe7qJfGnb7m3YZy3Uvd1PTy+cWFeTXA99+739o5RuifJOQ
gAQAAA==
"|gunzip
    echo -ne "\033[0m"

}


function usage()
{
    echo "Get and test a list of proxy servers(http and socks5 type)"
    echo ""
    echo "      `basename $0` -5 [filename] (get proxy list from local file)"
    echo "      `basename $0` -5 [url] (get proxy list from url)"
    echo "      `basename $0` -3 [filename] (get proxy list from local file)"
    echo "      `basename $0` -3 [url] (get proxy list from url)"
    exit  -1
}

logo
function testping()
{

CNT=$(cat /tmp/hostlist|wc -l)
if [ ${CNT} == 0 ]; then
    exit -1
fi
cat /tmp/hostlist|awk '{print $1}'|xargs -P8 -I % sh -c 'fping -c1 -t180 %'|grep bytes|awk '{print $1,$6}'|sort -k2 -g|uniq|awk '{print $1}'|xargs -P8 -I % sh -c 'cat /tmp/hostlist|grep %'|sort|uniq > /tmp/fastproxylist
}



function filter_ip_port()
{
    cat ${PROXYFILE}|grep -oE "([0-9]{1,3}\.){3}[0-9]{1,3}:[0-9]{1,5}"|awk -F: '{print $1,$2}' > /tmp/hostlist
    cat ${PROXYFILE}|grep -oE "([0-9]{1,3}\.){3}[0-9]{1,3}\s+[0-9]{1,5}"|awk '{print $1,$2}' >> /tmp/hostlist
    cat ${PROXYFILE}|awk -F\" '{print $2,$16}'|grep -oE "([0-9]{1,3}\.){3}[0-9]{1,3}\s+[0-9]{1,5}" >> /tmp/hostlist
}


if [ $# == 2 ] && [ ${1} == '-5' ] && [ -f "${2}" ]; then
    cp ${2} ${PROXYFILE}
    filter_ip_port
elif [ $# == 2 ] && [ ${1} == '-3' ] && [ -f "${2}" ]; then
    cp ${2} ${PROXYFILE}
    filter_ip_port
elif [ $# == 2 ] && [ ${1} == '-5' ] && [[ "${2}" =~ ^(http(s)?:\/\/).* ]]; then
    while :;
    do
    > ${PROXYFILE}
    curl -x ${PXY} -s ${2} > ${PROXYFILE} || curl -s ${2} > ${PROXYFILE} 
    CNT=$(cat ${PROXYFILE}|wc -l)
    if [ $CNT == 0 ]; then
        sleep 1
        echo -e "\033[96;40mwait a moment...\033[0m"
    else
        echo -e "\033[96;40msocks5 proxy it's ok\033[0m"
	break
    fi
    done
    filter_ip_port
elif [ $# == 2 ] && [ ${1} == '-3' ] && [[ "${2}" =~ ^(http(s)?:\/\/).* ]]; then
    while :;
    do
    > ${PROXYFILE}
    curl -x ${PXY} -s ${2} > ${PROXYFILE} || curl -s ${2} > ${PROXYFILE} 
    CNT=$(cat ${PROXYFILE}|wc -l)
    if [ $CNT == 0 ]; then
        sleep 1
        echo -e "\033[96;40mwait a moment...\033[0m"
    else
        echo -e "\033[96;40mhttp proxy it's ok\033[0m"
	break
    fi
    done
    filter_ip_port
else
    usage
fi

echo -e "\033[96;40mPing Test...\033[0m"

testping
> /tmp/proxy.txt
> /tmp/proxy2.txt
> /tmp/proxy3.txt
if [ ${1} == '-5' ];then
    echo -e "\033[96;40m"
    echo "################"
    echo "#filter country#"
    echo "################"
    echo -e "\033[0m"
    cat /tmp/fastproxylist|awk '{print $1}'|uniq|xargs -P8 -I % sh -c 'curl --connect-timeout 3 -s http://ipinfo.io/%/country|grep -v -E "CN|RU|IN|JP|KR|TW|SG" >/dev/null && (echo %|tee -a /tmp/proxy.txt)'
    cat /tmp/proxy.txt|xargs -I % sh -c 'cat /tmp/fastproxylist|grep % >> /tmp/proxy2.txt'
    echo -e "\033[96;40m"
    echo "#############################"
    echo "#socks5 proxy available test#"
    echo "#############################"
    echo -e "\033[0m"
    cat /tmp/proxy2.txt|awk '{print $1":"$2}'|xargs -P8 -I % sh -c 'curl --connect-timeout 3 -x socks5://% -s https://www.gstatic.com/generate_204 >/dev/null && (echo %|tee -a /tmp/proxy3.txt)'
    cat /tmp/proxy3.txt|awk -F: '{print "socks5",$1,$2}' > socks5.txt
    echo "result save as socks5.txt in current directory"
else
    echo -e "\033[96;40m"
    echo "################"
    echo "#filter country#"
    echo "################"
    echo -e "\033[0m"
    cat /tmp/fastproxylist|awk '{print $1}'|uniq|xargs -P8 -I % sh -c 'curl --connect-timeout 3 -s http://ipinfo.io/%/country|grep  -v -E "CN|RU|IN|JP|KR|TW|SG" >/dev/null && (echo %|tee -a /tmp/proxy.txt)'
    cat /tmp/proxy.txt|xargs -I % sh -c 'cat /tmp/fastproxylist|grep % >> /tmp/proxy2.txt'
    echo -e "\033[96;40m"
    echo "###########################"
    echo "#http proxy available test#"
    echo "###########################"
    echo -e "\033[0m"
    cat /tmp/proxy2.txt|awk '{print $1":"$2}'|xargs -P8 -I % sh -c 'curl --connect-timeout 3 -x http://% -s https://www.gstatic.com/generate_204 >/dev/null && (echo %|tee -a /tmp/proxy3.txt)'
    cat /tmp/proxy3.txt|awk -F: '{print "http",$1,$2}' > http.txt
    echo "result save as http.txt in current directory"
fi