shell diff + kettle = mail 预警

 1 #!bin/bash
 2 kettleConfigHome=/home/vmuser/linbo/kettleDemo/job/config
 3 apiInfos=/home/vmuser/linbo/kettleDemo/job/data/apiInfo
 4 api_csv_path_home=/home/vmuser/linbo/test_upload/data_file
 5 sendMailData=/home/vmuser/linbo/kettleDemo/job/sendData
 6 dt=`date +"%Y%m%d"`
 7 lastDay=`date -d "1 day ago" +"%Y%m%d"`
 8 dmt=`date -d "1 day ago" +"%Y-%m-%d"`
 9 declare -A diffResult_eq
10 declare -A diffResult_nt
11 #先将pord 接口生成的文件load test的情况告知相关人 ;
12 sh /home/vmuser/linbo/kettleDemo/job/execSH/fileCpCase.sh
13 apiCnt=(`ls -rt $api_csv_path_home`)
14 #apiCnt=(unilever_three_shop_sales_retrun)
15 for((i=0;i<${#apiCnt[*]} ;i++)); 
16 do 
17     strInfo=""
18     #默认不发送邮件;
19     sendMail=false 
20     apiName=${apiCnt[$i]}        
21     #到prod 目录下查将今天生成的文件筛选出来
22     cd $api_csv_path_home/$apiName/prod
23     cmmd00="ls --full-time ./ | sed -n '/$(date +"%Y-%m-%d")/p' | awk '{print \$9}'"
24 #    cmmd00="ls --full-time ./ | sed -n '/${dmt}/p' | awk '{print \$9}'"
25     prod_currentDateFile=(`eval $cmmd00`)
26     cd $api_csv_path_home/$apiName/test
27     cmmd02="ls --full-time ./ | sed -n '/$(date +"%Y-%m-%d")/p' | awk '{print \$9}'"
28 #    cmmd02="ls --full-time ./ | sed -n '/${dmt}/p' | awk '{print \$9}'"
29     test_currentDateFile=(`eval $cmmd02`)
30     #echo "${i}_$apiName.prod.[${#prod_currentDateFile[*]}] ：${prod_currentDateFile[*]} "
31     #echo "${i}_$apiName.test.[${#test_currentDateFile[*]}] ：${test_currentDateFile[*]} "
32     if (( ${#prod_currentDateFile[*]} == 0 )); then 
33     echo "**************** $apiName 今天没有生成文件 不需要对比 continue ! ***************"
34         continue 
35     fi         
36     for((j=0;j<${#prod_currentDateFile[*]};j++));
37     do 
38         apiCsvPath=${api_csv_path_home}/${apiName}
39         prod_csvName=${prod_currentDateFile[$j]}
40         #假如prod接口下今天只生成了1个文件 并且test环境也有文件生成 就直接使用test环境的文件进行比较,否则使用生产环境的文件与测试环境的文件进行绝对匹配 ;
41         if [[ ${#prod_currentDateFile[*]} = 1 &&  ${#test_currentDateFile[*]} = 1 ]]; then
42             uat_csvName=${test_currentDateFile[0]}
43         else 
44             uat_csvName="uat_$prod_csvName"
45         fi    
46         #获取prod 和 test 环境当天生成文件的大小和行数 ;注意是文件创建日期为今天并不是文件名称中的日期;
47         prod_fileSize=`ls -l ${apiCsvPath}/prod/${prod_csvName} | awk '{print \$5}'`
48         test_fileSize=`ls -l ${apiCsvPath}/test/${uat_csvName} | awk '{print \$5}'`
49         prod_fileRowCnt=`cat  ${apiCsvPath}/prod/${prod_csvName} | wc -l`
50         test_fileRowCnt=`cat  ${apiCsvPath}/test/${uat_csvName} | wc -l`
51 
52         if (( $prod_fileSize != 0 )); then
53 #            echo "删除上一天的发送容  rm -rf ${sendMailData}/${apiName}/${lastDay} "
54             rm -rf ${sendMailData}/${apiName}/${lastDay}
55             if [[ -d "${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}" ]]; then
56                 #echo "${prod_csvName%.*} 文件存在 则删除，保持每次都是最新需要发送的内容 ！ "
57                 rm -rf ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}/*
58             fi
59             #"创建 ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*} 目录"
60             mkdir -p ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}
61             #"将 ${prod_csvName} 和 ${uat_csvName} 文件cp 到 $sendMailData/${apiName}/${dt}/${prod_csvName%.*} 目录下 "
62             cp $apiCsvPath/prod/${prod_csvName} $sendMailData/${apiName}/$dt/${prod_csvName%.*}
63             cp $apiCsvPath/test/${uat_csvName} $sendMailData/${apiName}/$dt/${prod_csvName%.*}
64             cd $sendMailData/${apiName}/$dt/${prod_csvName%.*}/
65             sort ${prod_csvName} -o ${prod_csvName}
66             sort ${uat_csvName} -o ${uat_csvName}
67             diff -iHB ${prod_csvName} ${uat_csvName} >./${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log
68             diffFlag=`cat ./${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log | wc -l`             
69             if (( $diffFlag != 0 )); then
70                 strInfo="
  • $prod_csvName    【 fileSize=${prod_fileSize}B ,  rowCnt=${prod_fileRowCnt}行 】; 
    •  
  • $uat_csvName  【 fileSize=${test_fileSize}B , rowCnt=${test_fileRowCnt}行 】;
    • $strInfo"
    71                 diffResult_nt["${apiName}_${j}"]="${strInfo}"
72             #    echo "--------修改发送邮件标志--------"
73                 sendMail=true            
74             else                 
75                 rm -rf ${prod_csvName} ${uat_csvName} ${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log
76                 diffResult_eq["${apiName}_${j}"]="${apiName}.${prod_csvName}和${uat_csvName}[${prod_fileSize}B : ${prod_fileRowCnt}行]\n"
77             fi
78         : '
79         else 
80             echo " $prod_csvName 和 $uat_csvName  文件大小为 0  无需比较 ！"
81             '
82         fi
83     done 
84 if [[ $sendMail = true ]]; then 
85     #发送邮件前需要修改的配置 ;
86     sed -i "s#^attachment_filePath=.*#attachment_filePath=//home//vmuser//linbo//kettleDemo//job//sendData//${apiName}//${dt}#g" $kettleConfigHome/sandMailVariate.properties
87     sed -i "s#^var1=.*#var1=
      ${strInfo}
    #g" $kettleConfigHome/sandMailVariate.properties
    88     sed -i "s#^var2=.*#var2=$apiName#g" $kettleConfigHome/sandMailVariate.properties
89     echo ">>>>>>>>>>>>>>>>>>>>>>>> 开始发送邮件 >>>>>>>>>>>>>>>>>>>>>"
90     sh /home/vmuser/linbo/kettleDemo/job/kjb/execSendMail.sh
91 fi
92 done
93 echo -e "${diffResult_eq[*]}" >/home/vmuser/linbo/kettleDemo/job/log/${dt}/diffResult_eq_$(date +"%Y%m%d_%H").log
94 echo "${diffResult_nt[*]}" >/home/vmuser/linbo/kettleDemo/job/log/${dt}/diffResult_nt_$(date +"%Y%m%d_%H").log
      1 #!/bin/bash
  2 kettleConfigHome=/home/vmuser/linbo/kettleDemo/job/config
  3 api_csv_path_home=/home/vmuser/linbo/test_upload/data_file
  4 prod_tableHead=""  5 test_tableHead="
    Noevn.prodfileNamefileSizefileRowCnt
    

    "  6 evnArr=(prod test)
  7 declare -A prod_tableRowInfo
  8 declare -A test_tableRowInfo
  9 #dtformat=`date -d "4 day ago" +"%Y-%m-%d"`
 10 #lastday=`date -d "4 day ago" +"%Y-%m-%d"`
 11 dtformat=`date +"%Y-%m-%d"`
 12 index=0 13 k=0 14 apiNames=(`ls -rt $api_csv_path_home`)
 15#t_burberry_sales_daily bz_return_order bz_return_order_line unilever_three_shop_sales_retrun ca_order_return unilever_sales_and_retrun mk_refund nike_gc_dgtl_hk_demand nike_odi_hk nike_odi_tmall nike_gc_dgtl_gp_order nike_hk_china_demand mk_bzsales disney_sales_return
 16 #apiNames=(t_burberry_sales_daily bz_return_order bz_return_order_line unilever_three_shop_sales_retrun ca_order_return unilever_sales_and_retrun mk_refund nike_gc_dgtl_hk_demand nike_odi_hk nike_odi_tmall nike_gc_dgtl_gp_order nike_hk_china_demand mk_bzsales disney_sales_return)
 17for apiName in ${apiNames[*]}
 18do 19for ((i=0;i<${#evnArr[*]};i++));
 20do 21         cd $api_csv_path_home/$apiName/${evnArr[i]}
 22         cmmd0="ls --full-time ./ | sed -n '/${dtformat}/p' | awk '{print \$9}'" 23         cmmd1="ls --full-time ./ | sed -n '/${dtformat}/p' | awk '{print \$5\"B\"}'" 24 25if [[ ${evnArr[i]} = prod ]]; then 26             prod_fileArr=(`eval $cmmd0`)
 27             prod_fileSizeArr=(`eval $cmmd1`)
 28             prod_EvnAndDataArr[k]="${apiName}.${evnArr[i]}.[${#prod_fileArr[*]}]" 29            #获取每个文件的行数 ;
 30             prod_fizeRowCntArr=()
 31for (( a=0;a<${#prod_fileArr[*]};a++));
 32do 33                 rowCnt0=`cat ${prod_fileArr[$a]} | wc -l`
 34                 prod_fizeRowCntArr[$a]="${rowCnt0}行" 35done 36# 将数据按一定的格式放入prod_tableRowInfo 中            
 37if [[ ${#prod_fileArr[*]} -le 1 ]]; then 38 #                    echo" ${#prod_fileArr[*]} -eq 1 " 39                     row="" 40elif [[ ${#prod_fileArr[*]} -gt 1 ]]; then 41 #                echo" ${#prod_fileArr[*]} -gt 1 " 42for ((b=0;b<${#prod_fileArr[*]};b++));
 43do 44if [ $b -eq 0 ]; then 45                         row="" 46else 47                         row="${row}" 48fi 49done 50fi 51             prod_tableRowInfo["$index"]="$row" 52 53else 54             test_fileArr=(`eval $cmmd0`)
 55             test_fileSizeArr=(`eval $cmmd1`)
 56             test_EvnAndDataArr[k]="${apiName}.${evnArr[i]}.[${#test_fileArr[*]}]" 57            #获取每个文件的行数 ;
 58             test_fizeRowCntArr=()
 59for (( a=0 ;a<${#test_fileArr[*]} ;a++ ));
 60do 61             rowCnt1=`cat ${test_fileArr[$a]} | wc -l`
 62             test_fizeRowCntArr[$a]="${rowCnt1}行" 63done 64 # -------------------------------------------------------------------------------------------------------------------------------------------            
 65if [[ ${#test_fileArr[*]} -le 1 ]]; then 66 #                    echo" ${#test_fileArr[*]} -eq 1 " 67                     row="" 68elif [[ ${#test_fileArr[*]} -gt 1 ]]; then 69 #                echo" ${#test_fileArr[*]} -gt 1 " 70for ((b=0;b<${#test_fileArr[*]};b++));
 71do 72if [ $b -eq 0 ]; then 73                         row="" 74else 75                         row="${row}" 76fi 77done 78fi 79             test_tableRowInfo["$index"]="$row" 80 # -------------------------------------------------------------------------------------------------------------------------------------------            
 81 82fi 83done 84 index=`expr $index + 1`
 85done 86 87#保证顺序
 88for ((x=0;x<${#prod_tableRowInfo[*]};x++));
 89do 90     str1="${str1}${prod_tableRowInfo[$x]}" 91     str2="${str2}${test_tableRowInfo[$x]}" 92done 93 94 #var3="${prod_tableHead}${prod_tableRowInfo[*]}
    Noevn.testfileNamefileSizefileRowCnt
    


    
          


     
         

                            

              
               
 

    ${index}${prod_EvnAndDataArr[0]}${prod_fileArr[0]}${prod_fileSizeArr[0]}${prod_fizeRowCntArr[0]}
    
              
 
                                   
                       

    ${index} ${prod_EvnAndDataArr[$b]}${prod_fileArr[$b]}${prod_fileSizeArr[$b]}${prod_fizeRowCntArr[$b]}
    
                          

     ${prod_fileArr[$b]} ${prod_fileSizeArr[$b]}${prod_fizeRowCntArr[$b]}
    
                       
                 
              
            
             
           

                            

              
              
 

    ${index}${test_EvnAndDataArr[0]}${test_fileArr[0]}${test_fileSizeArr[0]}${test_fizeRowCntArr[0]}
    
              
 
                                   
                       

    ${index} ${test_EvnAndDataArr[$b]}${test_fileArr[$b]}${test_fileSizeArr[$b]}${test_fizeRowCntArr[$b]}
    
                          

     ${test_fileArr[$b]} ${test_fileSizeArr[$b]}${test_fizeRowCntArr[$b]}
    
                       
                 
              
                
         
          
         
  
 
    


  
 

    "
 95 #var4="${test_tableHead}${test_tableRowInfo[*]}"
 96 var3="${prod_tableHead}${str1}"
 97 var4="${test_tableHead}${str2}"
 98 
 99 #echo ${!prod_tableRowInfo[*]}
100 #echo "${var3}"
101 #echo "*************************************************************"
102 #echo "${var4}"
103 #echo ${!test_tableRowInfo[*]}
104 sed -i "s#^var3=.*#var3=${var3}#g" $kettleConfigHome/sandMailVariate.properties
105 sed -i "s#^var4=.*#var4=${var4}#g" $kettleConfigHome/sandMailVariate.properties
106 # 发送邮件
107 sh /home/vmuser/linbo/kettleDemo/job/kjb/fileCpEvnEmailSend.sh

    Hadoop

    相关

    04 Hadoop思想与原理，Hbase原理

    zz手把手教你在腾讯云上搭建hadoop3.x伪集群的方法（参考该文建立伪集群）

    Hadoop与大数据平台

    hadoop安装教程

    hadoop系列整理---PySpark

    hadoop3.2.1下MapReduce操作出现错误: 找不到或无法加载主类org.apache.hadoop.mapred

    为hadoop集群设置静态IP

    hadoop-eclipse-plugin-2.x.x 插件编译

    Apache Hadoop 完全分布式集群搭建

    大数据学习之旅2——从零开始搭hadoop完全分布式集群

    七、Hadoop3.3.1 HA 高可用集群QJM （基于Zookeeper，NameNode高可用+Yarn高可用）

    5-Hadoop学习笔记2

    标签