Transfer Data To Redis

• Tweet
• 

        // Script to dump data to redis server

#!/bin/sh

redis_server='localhost or Amazon server like ec2-11-22-33-44.compute-1.amazonaws.com'
redis_port='6379'
# pem file if using Amazon EC2 service
pem_file='abc.pem'

# provide path to the directory where all the raw files are stored
mypath='/data'
cd $mypath

cat > foo.awk << "my_heredoc"
BEGIN {
    # all fields are separated by ^
    FS = "^";
}
{
    # $7 is the date and time in the form yyyy-mm-dd hh:mm:ss.
    # Split at colons to get hours minutes and seconds into a[1]
    # through a[3].  Round minutes to nearest 5.
    split($7, a, ":");
    a[2] = int(a[2] / 5) * 5;
    # set 852:14776:f37c310bd4 2012-03-10~12:55:00
    # print first, second and forth field, then rounded time.
    printf "set %s:%s:%s %s:%02d:00\\r\\n\n", $1, $2, $4, a[1], a[2];
}

my_heredoc

for file in `ls  r_*`
do
time sed -e 's/\ /~/g' $mypath/$file > todel.txt
time awk -f foo.awk < todel.txt > final.txt
echo "`date` file ready line count `wc -l $file`" >> invalid_lines.txt
time awk 'NF != 3 || !/\\n$/' final.txt >> invalid_lines.txt

process_cnt_before=`/root/redis-2.4.8/src/redis-cli -h $redis_server -p $redis_port info | grep total_commands_processed | awk -F':' '{print $2}'  | sed 's/\\r$//'`

keys_cnt_before=`/root/redis-2.4.8/src/redis-cli -h $redis_server -p $redis_port info | grep 'db0:keys=' | awk -F',' '{print $1}' | awk -F'=' '{print $2}'  | sed 's/\\r$//'`

if [[ "$redis_server" == "localhost" ]];then
time (echo -en `cat final.txt`) | nc $redis_server $redis_port > /dev/null 2> /dev/null
else
# connect from remote
0</dev/null echo -en `cat final.txt` |  gzip | ssh -i $pem_file $redis_server "gunzip | nc $redis_server $redis_port "; > /dev/null 2>&1
fi

if [ $? -eq 0 ];then
echo "`date` file $file successfully processed by redis" >> invalid_lines.txt
else
echo "file $file failed to process by redis " >> invalid_lines.txt
fi

done


process_cnt_after=`/root/redis-2.4.8/src/redis-cli -h $redis_server -p $redis_port info | grep total_commands_processed | awk -F':' '{print $2}' | sed 's/\\r$//'`

process_cnt_diff=`expr $process_cnt_after - $process_cnt_before`
echo "`date` commands processed are $process_cnt_diff " >> invalid_lines.txt

keys_cnt_after=`/root/redis-2.4.8/src/redis-cli -h $redis_server -p $redis_port info | grep 'db0:keys=' | awk -F',' '{print $1}' | awk -F'=' '{print $2}' | sed 's/\\r$//'`

keys_cnt_diff=`expr $keys_cnt_after - $keys_cnt_before`
echo "`date` keys added are $keys_cnt_diff " >> invalid_lines.txt
echo "==================" >> invalid_lines.txt


exit
# other ways to use awk directly
#awk -F'^' '{printf "set %s %s\n",$4,$7}' todel.txt | sed 's/$/\\r\\n/' >> final.txt

#myvar=`echo $file | awk -F"_" '{print $4}'`
#awk -F'^' -v myvar=$myvar '{printf "incr %s:%s:%s\n",$1,$2,myvar}' todel.txt | sed 's/$/\\r\\n/' >> final.txt

Tags:
• Redis