线上日志收集是很正常的业务，一般由运营来处理，现在给一段代码来实现日志的收集，打包以及上传到s3备份存储，请看代码:
基本上已经够用了，需要定义自己的变量就好！
shell脚本日志收集与打包
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
#!/bin/sh
function send_sms()
{
. /data/se_scripts/conf/setting_new.conf
phone=$phone_number
phone=$(echo $phone |sed 's/\n//g')
phone=$(echo $phone |sed 's/\r//g')
sh /data/se_scripts/conf/send_msg_new.sh ${phone} 701 "upload $1 failed" &
}
function exit_if_timeout()
{
SLEEP_SECONDS=$1
CMD=$2
#set -x
$CMD &
pidApp=$!
( sleep $SLEEP_SECONDS ; kill $pidApp ) &
killerPid=$!
wait $pidApp
status=$?
(kill -0 $killerPid && kill $killerPid) || true
#set +x
return $status
}
function exec_cmd_with_retry()
{
cmd=$1
cur_retry=$2
while [ $cur_retry -gt 0 ]; do
$cmd
ret=$?
if [ $ret -eq 0 ]; then
return
fi
((cur_retry -= 1))
sleep 10
done
send_sms "Failed to exec ${cmd} IP:${local_ip}"
exit 1;
}
function upload_s3path()
{
retry_cnt=$1
src_path=$2
file=$3
arr=(${file//./ })
day=${file:0:8}
hour=${file:8:2}
minute=${file:10:2}
exit_if_timeout 600 "aws s3 cp --profile $s3_profile $src_path/$file $s3_path/$day/$hour/$minute/$file.$local_ip.log"
ret=$?
if [ $ret -eq 0 ] ; then
rm -rf $src_path/$file
fi
if [ $ret -eq 0 ] && [ "${file}" != "${ln_file}" ]; then
send_sms "$src_path/$file uploads3 retry ok ${local_ip}"
elif [ $ret -ne 0 ] && [ $retry_cnt -eq 0 ]; then
send_sms "$src_path/$file uploads3 failed ${local_ip}"
fi
}
function upload_files()
{
for f in `ls $ln_path`
do
upload_s3path $1 $ln_path $f
done
}
function upload_empty()
{
for f in `ls $lock_touch`
do
upload_s3path $1 $lock_touch $f
done
}
function upload_logs()
{
cur_retry=5
while [ $cur_retry -gt 0 ]; do
upload_empty $cur_retry
upload_files $cur_retry
((cur_retry -= 1))
sleep 10
done
}
#down 机 肯定不会有原始日志，所以不检查原始日志
function check_empty()
{
for f in `ls $lock_touch`
do
check=`ls $ln_path | grep $f | grep -v grep | wc -l`
if [ $check -ne 0 ]; then
rm -rf $lock_touch/$f
fi
done
}
function check_pid()
{
if [ ! -d ${lock_pid} ];then
mkdir -p $lock_pid
echo $$ > $lock_pid/PID
return
fi
check=0
running=`cat $lock_pid/PID`
if [ ! -z $running ]; then
check=`ps -ef | grep $running | grep -v grep | wc -l`
fi
if [ "$check" -eq "1" ]; then
exit
fi
echo $$ > $lock_pid/PID
}
#args
log_path=$1
log_file=$2
s3_path=$3
ln_path=$4
ln_file=$6
s3_profile=$7
lock_pid=$5/pid
lock_touch=$5/empty
local_ip=`cat /var/tmp/ifconfigme`
#soft link
exec_cmd_with_retry "ln -s $log_path/$log_file $ln_path/$ln_file" 5
check_pid
check_empty
upload_logs
rm -rf $lock_pid