快网CDN怎么样统计脚本

内容分发网络（Content Delivery Network，简称CDN）是建立并覆盖在承载网之上，由分布在不同区域的边缘节点服务器群组成的分布式网络。CDN分担源站压力，避免网络拥塞，确保在不同区域、不同场景下加速网站内容的分发，提高资源访问速度。许多用户在实际运用CDN的过程中都会遇到一些CDN常见的问题，今天小编跟大家聊一下公司使用快网的CDN的问题。

公司使用的快网的CDN，但是快网本身并不提供日志的分析统计，我这里还没有用elk，无奈只能先使用shell对日志进行分析统计，作为运维组对业务情况的一个大致的了解。

脚本是对前一天的CDNLOG进行解压缩分析，最后将结果按类输出到指定目录，再将这些结果打包扔回CDNLOG目录，因为CDNLOG目录是要上传云存储的，一并保存。其实还有一个动作，是将这些内容作为参数传给python脚本给一个监控系统，出图便于查看，这里没有贴上。

同样作为留底，不喜勿喷。

#!/bin/bashfunction line(){    cols=`tput cols`        for t in `seq 1 ${cols}`        do            echo -n "="        done}function global_define(){    yesterday=`date -d "-1day" +%F`    comp_path="/srv/cdnlog/downloads/${yesterday}/"    cdnlog_list=`ls ${comp_path} | sed 's#.gz##g'`    uncomp_path="/srv/cdnlog/uncomp/${yesterday}/"    user=`whoami`    backup_path="/srv/cdnlog/downloads/${yesterday}/"}function global_check(){    if [ $user != root ];then        echo "Please use root ."        exit 1    fi}function cycle(){    for n in $cdnlog_list    do        check        define        uncomp        segmentation        source_address        method        http_code        http_size        referer        retime        platform        tag        back        cdn_node        response_size        delete    done}function check(){    if [ -d ${uncomp_path}${n}/ ];then        rm -rf ${uncomp_path}${n}/    fi}function define(){    gz_file="${n}.gz"    log_file="${n}.log"    log_path="/srv/cdnlog/uncomp/${yesterday}/${n}/"}function uncomp(){    mkdir -p ${log_path}    gunzip -c ${comp_path}${gz_file} > ${log_path}${log_file}}function segmentation(){    log_file_count=`cat ${log_path}${log_file} | wc -l`        awk -vsour_addr=${log_path}sour_addr.tmp -vmethod=${log_path}method.tmp -vhttp_code=${log_path}http_code.tmp -vhttp_size=${log_path}http_size.tmp -vreferer=${log_path}referer.tmp  -vplatform=${log_path}platform.tmp -vcdn_node=${log_path}cdn_node.tmp -vresponse_size=${log_path}response_size.tmp '{print $1 > sour_addr ;print $6,$7 > method ;print $9 > http_code ;print $10 > http_size ;print $11 > referer ;print $(NF) > cdn_node ;print $(NF-1) > response_size}' ${log_path}${log_file}        cat ${log_path}${log_file} | egrep "FCACHE_HIT_DISK|FCACHE_HIT|FCACHE_MISS" | awk -F 'FCACHE_HIT_DISK|FCACHE_HIT|FCACHE_MISS' '{print $2}' > ${log_path}more.tmp        awk -vretime1=${log_path}retime1.tmp -vretime2=${log_path}retime2.tmp -vretime3=${log_path}retime3.tmp -vretime4=${log_path}retime4.tmp -vretime5=${log_path}retime5.tmp -vretime6=${log_path}retime6.tmp -vretime7=${log_path}retime7.tmp -vretime8=${log_path}retime8.tmp -vretime9=${log_path}retime9.tmp -vtag=${log_path}tag.tmp -vback=${log_path}back.tmp '{print $1 > retime1 ;print $2 > retime2 ;print $3 > retime3 ;print $4 > retime4 ;print $5 > retime5 ;print $6 > retime6 ;print $7 > retime7 ;print $8 > retime8 ;print $9 > retime9 ;print $10 > tag ;print $12 > back }' ${log_path}more.tmp        cat ${log_path}${log_file} | awk -F '[(|)]' '{print $2}' > ${log_path}platform.tmp}function source_address(){    line >> ${log_path}sour_addr.log    echo "Source Address Top 30 :" >> ${log_path}sour_addr.log    echo "" >> ${log_path}sour_addr.log    cat ${log_path}sour_addr.tmp | sort -r | uniq -c | sort -nr | head -30 >> ${log_path}sour_addr.log    sour_addr_total=`cat ${log_path}sour_addr.tmp | sort -r | uniq -c | wc -l `    echo "" >> ${log_path}sour_addr.log    echo "Source Address Total : $sour_addr_total" >> ${log_path}sour_addr.log    line >> ${log_path}sour_addr.log}function method(){    line >> ${log_path}method.log    echo "Method & URL Top 30 :" >> ${log_path}method.log    echo "" >> ${log_path}method.log    cat ${log_path}method.tmp | sed 's#"##g' | sort -r | uniq -c | sort -nr | head -30 >> ${log_path}method.log    line >> ${log_path}method.log}function http_code(){    line >> ${log_path}http_code.log    echo "Http Code Top :" >> ${log_path}http_code.log    echo "" >> ${log_path}http_code.log    http_code_list=`cat ${log_path}http_code.tmp | sort -r | uniq -c | sort -nr | awk '{print $2}'`    cat ${log_path}http_code.tmp | sort -r | uniq -c | sort -nr >> ${log_path}http_code.log    echo "" >> ${log_path}http_code.log        for y in $http_code_list    do        count=`cat ${log_path}http_code.log | awk -vncode=$y '$2==ncode{print $1}'`        rate=`echo "$count $log_file_count" | awk '{printf ("%.2f\n",$1*100/$2)}'`        echo -e "HTTP $y Count: \t$count" >> ${log_path}http_code.log        echo -e "HTTP $y Rate: \t\t${rate}%" >> ${log_path}http_code.log        echo "" >> ${log_path}http_code.log    done        line >> ${log_path}http_code.log}function http_size(){    size_max=`cat ${log_path}http_size.tmp | sort -nr | head -1`    size_min=`cat ${log_path}http_size.tmp | sort -nr | tail -1`    size_total_tmp=`cat ${log_path}http_size.tmp | awk '{sum+=$1}END{print sum}'`    size_total=`echo $size_total_tmp | awk '{printf ("%.0f\n",$1)}'`    size_avg=`echo "$size_total $log_file_count" | awk '{printf ("%.2f\n",$1/$2)}'`        line >> ${log_path}http_size.log    echo "" >> ${log_path}http_size.log    echo -e "Http Size Max : \t$size_max" >> ${log_path}http_size.log    echo -e "Http Size Min : \t$size_min" >> ${log_path}http_size.log    echo -e "Http Size Total : \t$size_total" >> ${log_path}http_size.log    echo -e "Http Size Avg : \t$size_avg" >> ${log_path}http_size.log    line >> ${log_path}http_size.log}function referer(){    line >> ${log_path}referer.log    echo "Referer Top 30 :" >> ${log_path}referer.log    echo "" >> ${log_path}referer.log    cat ${log_path}referer.tmp | sed 's#"##g' | sort -r | uniq -c | sort -nr | head -30 >> ${log_path}referer.log    line >> ${log_path}referer.log}function retime(){    sed -ie '/-/d' ${log_path}retime{1..9}.tmp        for i in `seq 1 9`    do        eval retime${i}=`cat  ${log_path}retime${i}.tmp | wc -l`        eval sum${i}=`cat ${log_path}retime${i}.tmp | awk '{sum+=$1}END{print sum}' | awk '{printf ("%.2f\n",$1)}'`        eval avg${i}=`eval echo "\\${retime${i}} \\${sum$i}" | awk '{printf ("%.2f\n",$2/$1)}'`        eval max${i}=`cat ${log_path}retime${i}.tmp | sort -nr | head -1`        eval min${i}=`cat ${log_path}retime${i}.tmp | sort -nr | tail -1`    done        for k in `seq 1 9`    do        line >> ${log_path}retime${k}.log        echo "Retime${k} :" >> ${log_path}retime${k}.log        echo "" >> ${log_path}retime${k}.log        echo -en "Max \t: \t" >> ${log_path}retime${k}.log        eval echo "\${max$k}" >> ${log_path}retime${k}.log        echo -en "Min \t: \t" >> ${log_path}retime${k}.log        eval echo "\${min$k}" >> ${log_path}retime${k}.log        echo -en "Avg \t: \t" >> ${log_path}retime${k}.log        eval echo "\${avg$k}" >> ${log_path}retime${k}.log        echo -en "Total : \t" >> ${log_path}retime${k}.log        eval echo "\${sum$k}" >> ${log_path}retime${k}.log        line >> ${log_path}retime${k}.log    done        sed -i 's#-nan#0.00#g' ${log_path}retime{1..9}.log}function platform(){    iphone_count=`cat ${log_path}platform.tmp | egrep "iPhone" | wc -l`    ipad_count=`cat ${log_path}platform.tmp | egrep "iPad" | wc -l`    mac_count=`cat ${log_path}platform.tmp | egrep "Macintosh" | wc -l`    android_count=`cat ${log_path}platform.tmp | egrep "Android|Adr" | wc -l`    windows_count=`cat ${log_path}platform.tmp | egrep "Windows NT|Windows 98|Windows XP" | wc -l`    windows_mobile_count=`cat ${log_path}platform.tmp | egrep "Windows Phone" | wc -l`    java_count=`cat ${log_path}platform.tmp | egrep "java" | wc -l`    linux_count=`cat ${log_path}platform.tmp | egrep "X11; Linux|linux-gnu" | wc -l`        iphone_rate=`echo "$iphone_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    ipad_rate=`echo "$ipad_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    mac_rate=`echo "$mac_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    android_rate=`echo "$android_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    windows_rate=`echo "$windows_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    windows_mobile_rate=`echo "$windows_mobile_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    java_rate=`echo "$java_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    linux_rate=`echo "$linux_count $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`    line >> ${log_path}platform.log    echo "Platform Info :" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "iPhone count \t\t: $iphone_count" >> ${log_path}platform.log    echo -e "iPhone rate \t\t: ${iphone_rate}%" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "iPad count \t\t: $ipad_count" >> ${log_path}platform.log    echo -e "iPad rate \t\t: ${ipad_rate}%" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "Mac count \t\t: $mac_count" >> ${log_path}platform.log    echo -e "Mac rate \t\t: ${mac_rate}%" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "Android count \t\t: $android_count" >> ${log_path}platform.log    echo -e "Android rate \t\t: ${android_rate}%" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "Windows count \t\t: $windows_count" >> ${log_path}platform.log    echo -e "Windows rate \t\t: ${windows_rate}%" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "Windows mobile count \t: $windows_mobile_count" >> ${log_path}platform.log    echo -e "Windows mobile rate \t: ${windows_mobile_rate}%" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "Java count \t\t: $java_count" >> ${log_path}platform.log    echo -e "Java rate \t\t: ${java_rate}%" >> ${log_path}platform.log    echo "" >> ${log_path}platform.log    echo -e "Linux count \t\t: $linux_count" >> ${log_path}platform.log    echo -e "Linux rate \t\t: ${linux_rate}%" >> ${log_path}platform.log    line >> ${log_path}platform.log}function tag(){    tag_total=`cat ${log_path}more.tmp | wc -l`    tag_suc=`cat ${log_path}more.tmp | awk '{print $10}' | grep "1" | wc -l`    tag_fail=`cat ${log_path}more.tmp | awk '{print $10}' | grep "0" | wc -l`    tag_suc_rate=`echo "$tag_suc $tag_total" | awk '{printf ("%.2f\n",$1/$2*100)}'`    tag_fal_rate=`echo "$tag_fail $tag_total" | awk '{printf ("%.2f\n",$1/$2*100)}'`        line >> ${log_path}tag.log    echo "Tag of Download Info :" >> ${log_path}tag.log    echo "" >> ${log_path}tag.log    echo -e "Download Success Count \t\t: $tag_suc" >> ${log_path}tag.log    echo -e "Download Success Rate \t\t: ${tag_suc_rate}%" >> ${log_path}tag.log    echo -e "Download Fail Count \t\t: $tag_fail" >> ${log_path}tag.log    echo -e "Download Fail Rate \t\t: ${tag_fal_rate}%" >> ${log_path}tag.log    line >> ${log_path}tag.log}function back(){    back_total=`cat ${log_path}more.tmp | awk '{print $12}' | egrep -v "-"| wc -l`    back_rate=`echo "$back_total $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`        line >> ${log_path}back.log    echo "Back Source Info :" >> ${log_path}back.log    echo "" >> ${log_path}back.log    echo -e "Back Source Count \t: $back_total" >> ${log_path}back.log    echo -e "Back Source Rate \t: ${back_rate}%" >> ${log_path}back.log    line >> ${log_path}back.log}function cdn_node(){    line >> ${log_path}cdn_node.log    echo "CDN Node Top 30 :" >> ${log_path}cdn_node.log    echo "" >> ${log_path}cdn_node.log    cat ${log_path}cdn_node.tmp | sort -n | uniq -c | sort -nr | head -30 > ${log_path}cdn_node.log    line >> ${log_path}cdn_node.log}function response_size(){    response_size_total=`cat ${log_path}response_size.tmp | awk '{sum+=$1}END{print sum}' | awk '{printf ("%.0f\n",$1)}'`    response_size_avg=`echo "$response_size_total $log_file_count" | awk '{printf ("%.2f\n",$1/$2*100)}'`        line  >> ${log_path}response.log    echo "Response Size Info :" >> ${log_path}response.log    echo "" >> ${log_path}response.log    echo -e "Response Size Total \t: $response_size_total" >> ${log_path}response.log    echo -e "Response Size Avg \t: $response_size_avg" >> ${log_path}response.log    line >> ${log_path}response.log}function delete(){    rm -f ${log_path}*.tmp    rm -f ${log_path}*.tmpe    rm -f ${log_path}${log_file}}function backup(){    tar zcf ${backup_path}statistics_${yesterday}.tar.gz ${log_path}*.log}function main(){    global_define    global_check    cycle    backup}main

如果大家还有什么不明白的地方可以在官网找我们的CDN技术工程师的，CDN技术工程师在行业内拥有十几年的经验了，所以会比小编回答的更加详细专业。官网链接www.yisu.com