# SccsId[] = "%W% (USL function) %G%"
CBD_name="COMPRESS_BY_DATE"
if [ ".${SECONDS}" = "." ]; then # Bourne function already loaded?
[ ."`set|egrep '^$CBD_name\(\)\{$'`" != . ] && CBD_loaded=1
else # Korn or Bash shell and function already loaded?
if [ `expr "\`uname -s\`" : "[Ll][Ii][Nn][Uu][Xx]"` -eq 0 ]; then
[ ."`typeset +f|awk '/^'$CBD_name'[=\(]?/'`" != . ] && CBD_loaded=1
else # Linux
[ ."`typeset -F|awk '/^'$CBD_name'[=\(]?/'`" != . ] && CBD_loaded=1
fi
fi
if [ 0${CBD_loaded} -eq 0 ]; then
#----------------------------------------------------------------------#
COMPRESS_BY_DATE() # Function documentation located at bottom. #
#----------------------------------------------------------------------#
{ [ ."${AWK}" = . ] && { { [ -x /usr/bin/nawk ] && AWK=/usr/bin/nawk; } \
|| { [ -x /bin/gawk ] && AWK=/bin/gawk ; } \
|| { [ -x /usr/bin/awk ] && AWK=/usr/bin/awk ; }; }
if [ .${SHLIB} = . ]; then SHLIB=/usr/local/scripts; export SHLIB; fi
if [ .${SHBIN} = . ]; then SHBIN=/usr/local/bin; export SHBIN; fi
. $SHLIB/email_msg.sh # Calls $SHLIB/exit.sh
#----------------------------------------------------------------#
# If the following variables are not set, use these as defaults. #
#----------------------------------------------------------------#
: ${id_num=`id|sed 's/^\(uid=\)\([0-9]*\)\(.*\)/\2/'`}
: ${id_hex=`echo "obase=16;$id_num"|bc`}
: ${script_name:=`basename $0`}
: ${name_root:=`echo $script_name|$AWK '{sub(/^\.+/,"");sub(/\..*/,"");print}'`}
: ${tmp:=/var/tmp}
: ${sp:=" "}
: ${yymmddhhmiss:=`date '+%y''%m%d%H''%M''%S'`}
: ${Xtimestamp:=`echo "obase=16;$yymmddhhmiss+$$"|bc`}
CBD_ID="$script_name($CBD_name)"
#--------------------------------------------------------------------#
# Must have either dateplus (compiled C executable) or dateplus.awk. #
#--------------------------------------------------------------------#
if [ ".$dateplus" = "." ]; then
if [ -x $SHBIN/dateplus ]; then
dateplus=$SHBIN/dateplus
elif [ -f $SHLIB/dateplus.awk ]; then
dateplus="$AWK -f $SHLIB/dateplus.awk -- "
else
EMAIL_MSG "ERROR: $CBD_ID" \
"Unable to locate $SHBIN/dateplus (C executable)" \
"${sp}or $SHLIB/dateplus(.awk)* !" \
"${sp}Skipping file compress today ;-)." \
"${sp}$CPD_ID terminated."
return 1
fi
fi
if [ ."$log" = . ]; then
log=$name_root.log
[ ."$teelog" = . ] && teelog="cat"
fi
echo "`date '+%Y-%m-%d %T'` $CBD_name $@" | $teelog
#--------------------------------#
# Parse this function's options. #
#--------------------------------#
CBD_opt_S=0 # Sudo
CBD_opt_T=0 # Test
CBD_opt_f=0 # Date/sort field
CBD_opt_l=0 # Length date/sort field
CBD_opt_n=0 # No sort
CBD_opt_t=0 # Separator char
CBD_opt_v=0 # Verbose
CBD_opt_w=0 # Debug or double-Verbose
CBD_option=0
CBD_sep_char="."
CBD_fld_len=6 # Default
CBD_field_n="last"
CBD_root=$tmp/$name_root"_CBD_go_"$id_hex
CBD_err=$CBD_root"."$Xtimestamp
while getopts STvwsnf:l:t: CBD_opt 2>> $CBD_err
do
case $CBD_opt in
S ) CBD_opt_S=1
CBD_option=1
;;
T ) CBD_opt_T=1
CBD_option=1
;;
v ) CBD_opt_v=1
CBD_option=1
;;
w ) CBD_opt_w=1
CBD_option=1
;;
s ) CBD_opt_s=1
CBD_follow="-follow"
CBD_option=1
;;
n ) CBD_opt_n=1
CBD_option=1
;;
f ) CBD_opt_f=1
CBD_field_n="$OPTARG"
CBD_option=1
;;
l ) CBD_opt_l=1
CBD_fld_len="$OPTARG"
CBD_option=1
;;
t ) CBD_opt_t=1
CBD_sep_char="$OPTARG"
CBD_option=1
;;
\? ) echo "$CBD_ID" \
"Invalid option: -`sed 's/^.*-- //' $CBD_err`" 1>&2
;;
* ) ;;
esac
done
#--------------------------------------------#
# Shift past options to remaining arguments. #
#--------------------------------------------#
[ $CBD_option -ne 0 ] && shift `expr $OPTIND - 1`
[ ."$CBD_root" != . ] && \rm -f $CBD_root* > /dev/null 2>&1
#----------------------------------------------------------------#
# Must reset this dog if this function is apt to be called again #
OPTIND=1 # (try and find this fact documented anywhere else). #
#----------------------------------------------------------------#
#----------------------------------------------------------#
# Validate option and 'n' and 'file_root' arguments. #
# Nawk returns nonzero on failure or zero if all is well. #
#----------------------------------------------------------#
CBD_error_txt=`$AWK -v sp="$sp" \
-v CBD_ID="$CBD_ID" \
-v CBD_name="$CBD_name" \
-v opt_f=$CBD_opt_f \
-v opt_l=$CBD_opt_l \
-v opt_t=$CBD_opt_t \
-v len=$CBD_fld_len \
-v char=$CBD_sep_char \
-v field=$CBD_field_n \
'BEGIN \
{
if (ARGV[2] == "")
exit_usage(CBD_ID": Insufficient args.")
else if (! match(ARGV[1],/^[0-9]+$/))
exit_usage(CBD_ID": Non-numeric keep days number ("ARGV[1]").")
else if (opt_l == 1 && (! match(len,/^[0-9]+$/)))
exit_usage(CBD_ID": Non-numeric field length ("opt_l").\n")
else if (opt_t == 1 && char == "")
{
msg="Separator option (-t) supplied without its"
msg=msg"\n"sp"corresponding field separator character.\n"
exit_usage(CBD_ID": "msg)
}
else if (! (field=="last"||match(field,/^[0-9]+$/)))
exit_usage(CBD_ID": Non-numeric field number ("field").\n")
else if (field == 0)
{
msg="Invalid field number, "field" (must be greater than zero).\n"
exit_usage(CBD_ID": "msg)
}
exit 0
} # E.O.BEGIN
#--------------------------------------------------------#
function exit_usage(MSG)
#--------------------------------------------------------#
{
print sp""MSG,"\n"sp,"Usage: "CBD_name ,
"-STvw -n -f field -l len -t char nn file_root\n" ,
"\n"sp,sp"-S = Use sudo to compress files.\n" ,
"\n"sp,sp"-T = Test only, compress nothing--simply" ,
"\n"sp,sp" pretend we are." ,
"\n"sp,sp"-v = Verbose.\n" ,
"\n"sp,sp"-w = Debug or Double-Verbose.\n" ,
"\n"sp,sp"-s = Follow symbolic links.\n" ,
"\n"sp,sp"-n = Do NOT sort the fileids being compressed",
"\n"sp,sp" (sort is primarily for readability).\n",
"\n"sp,sp"-f = Field specified the test/sort field" ,
"\n"sp,sp" -- defaults to the last field.\n" ,
"\n"sp,sp"-l = Specifies the field length (6|8)" ,
"\n"sp,sp" --default is 6 (for yymmdd).\n" ,
"\n"sp,sp"-t = Specifies separation character, char" ,
"\n"sp,sp" --defaults to a period.\n" ,
"\n"sp,sp"nn = Number of days we keep" ,
"\n"sp,sp" --compress files older than \047n\047 days.\n",
"\n"sp,sp"file_root = Fully-qualified fileid" ,
"\n"sp,sp" --up to the numeric sort data" ,
"\n"sp,sp" (e.g. \047$HOME/logs/daily_log\047)."
exit 1
}' "$1" "$2" 2>&1` # $1 is our target file_root argument.
#----------------------------------------------------------------#
# Test the returns status and fuss at the user if it is nonzero. #
#----------------------------------------------------------------#
if [ $? -ne 0 ]; then
EMAIL_MSG "ERROR: $CBD_ID" \
"$CBD_error_txt" \
"${sp}Skipping compress today ;-)." \
"${sp}$script_name (PID=$$) continues."
return 1
fi
#----------------------------------------------------------#
# Passed validation tests OK, now get about our business. #
#----------------------------------------------------------#
CBD_last_n=$CBD_field_n
CBD_yyyymmdd=`$dateplus -$1`
CBD_dir=`dirname "$2"`
[ $CBD_dir = "." ] && CBD_dir=`pwd`
CBD_file=`basename "$2"`
#--------------------------------------------------------------#
# Isolate the desired (possibly default) 8-digit numeric date #
# field in each line (sort_key) that follows $CBD_sep_char and #
# use that field (as a sort key). Prefix it to the line, then #
# sort that new composite by that prefix (sort -inr -k 1,1). #
# Finally, use nawk to remove the prefix (sub(/^.* +/,"")) and #
# print the original (now sorted) lines. Whew! #
#--------------------------------------------------------------#
[ $CBD_opt_v -gt 0 ] \
&& echo "${sp}find $CBD_dir -type f -name \"$CBD_file*\" -print" \
| $teelog
find $CBD_dir -type f -name "$CBD_file*" -print 2> /dev/null \
| $AWK -F$CBD_sep_char -v sp="$sp" \
-v nosort=$CBD_opt_n \
-v dir=$CBD_dir \
-v field=$CBD_field_n \
-v len=$CBD_fld_len \
-v yyyymmdd=$CBD_yyyymmdd \
-v verbose=$CBD_opt_v \
-v debug=$CBD_opt_w \
-v opt_S=$CBD_opt_S \
-v opt_T=$CBD_opt_T \
'BEGIN \
{ #-------------------------------------------------#
# Build sort key ([0-9]) that handles len digits. #
#-------------------------------------------------#
if (len == "") len = 6 # Assign default len if omitted
for (i = 1; i <= len; i++)
sort_key=sort_key"[0-9]"
sort_field = field
i = 0 # Initialize before Action section
#-------------------------------------------------#
# Truncate yyyymmdd to yymmdd if len value = 6. #
#-------------------------------------------------#
if (len < 8) yyyymmdd=substr(yyyymmdd,3,len)
gzip="/usr/local/bin/gzip"
status=system("test -f "gzip)
if (status == 0)
pr_cmp_prog = cmp_prog = gzip
else
{
gzip="/usr/bin/gzip" # As of Solaris 2.9
status=system("test -f "gzip)
if (status == 0)
pr_cmp_prog = cmp_prog = gzip
else
pr_cmp_prog = cmp_prog = "compress"
}
if (opt_S == 1) # Prefix with "sudo"?
pr_cmp_prog = cmp_prog = "sudo "cmp_prog
sub(/^.*\//,"",pr_cmp_prog)
if (debug) print "1. nosort =|"nosort"|" \
"\n1. dir =|"dir"|" \
"\n1. field =|"field"|" \
"\n1. sort_field =|"sort_field"|" \
"\n1. sort_key =|"sort_key"|" \
"\n1. len =|"len"|" \
"\n1. yyyymmdd =|"yyyymmdd"|" \
"\n1. opt_S =|"opt_S"|" \
"\n1. opt_T =|"opt_T"|" \
"\n1. cmp_prog =|"cmp_prog"|" \
"\n1. pr_cmp_prog=|"pr_cmp_prog"|" | "cat 1>&2"
} # E.O. BEGIN ..
#---------------------------------------------------------------#
# Action section (MAIN) isolates the sort field, prefixing each #
# line with that key, and assigning it to line[] array. The #
# END section will sort the array by that key, compressing #
# those files (via system call) that are older our target date. #
#---------------------------------------------------------------#
{ # Only if we are to sort on #
if (sort_field == "last") # the last field, do we check #
for (n = 1; n <= NF; n++) # each field (first to last) #
if (match($n,sort_key)) # to get last suitable field #
field=n # (prevents a .Z from hosing #
# up this entire process). #
file_line = $0 #-----------------------------#
if (debug) print "1. file_line =|"file_line"|" | "cat 1>&2"
#-----------------------------------------------------------#
# If field is still "last", then change "last" into NF. #
#-----------------------------------------------------------#
if (field == "last") field == NF
if (debug) print "2. field =|"field"|" | "cat 1>&2"
#-----------------------------------------------------------#
# Ensure that we, indeed, have a sort field somewhere in #
# the line (fileid). #
#-----------------------------------------------------------#
if (! match($0,sort_key))
{
if (verbose)
print sp"Skipping "$0". No "len"-digit numeric sort key."
next;
}
if (match($0,/\.zip$/))
{
print sp"Skipping "$0" (zipped file)."
next;
}
#-----------------------------------------------------------#
# Following basename and dirname assignments and "sub()s" #
# isolate those variables from each fileid. The test #
# following that, restricts processing to only those files #
# found in the target directory (excludes subdirectories). #
#-----------------------------------------------------------#
dirname = basename = $0
if (debug) print "1. dirname =|"dirname"|" | "cat 1>&2"
sub(/^.*\//, "",basename)
if (debug) print "1. basename =|"basename"|" | "cat 1>&2"
sub("/"basename,"",dirname)
if (debug) print "2. dirname =|"dirname"|" | "cat 1>&2"
if (debug) print "2. basename =|"basename"|" | "cat 1>&2"
if (dirname == dir)
{
filename = basename
if (debug) print "2. sort_key =|"sort_key"|" | "cat 1>&2"
sub(sort_key"*","((&))",filename)
if (debug) print "3. filename =|"filename"|" | "cat 1>&2"
gsub(/^.*\(++/, "",filename) # Strip all before date field
if (debug) print "4. filename =|"filename"|" | "cat 1>&2"
gsub(/\)++.*$/, "",filename) # and everything after it, then
if (debug) print "5. filename =|"filename"|" | "cat 1>&2"
$field=substr(filename,1,len) # substring it to length.
if (debug) print "5. $field =|"$field"|" | "cat 1>&2"
if (! match($field,"^"sort_key"*$"))
{
print_err_msg()
i = 0 # Reset so nothing is sorted and compressed in the
exit 1 # END section (exit breaks to the END section).
}
#-------------------------------------------------------#
# Only put those in the array that we want to compress. #
#-------------------------------------------------------#
if (substr($field,1,len) < yyyymmdd)
line[++i]=$field" "file_line # Prefix line with sort key
} # if (dirname == dir)
} # E.O.Action section
END \
{ if (i == 0) exit
#-------------------------------------------------------#
# If we are really compressing files, we probably do #
# not have to sort them (since sorting is slow). #
# However, since "find" does not present its output in #
# order, sorting does improve output readability. #
# ----------------------------------------------------- #
# Sort line[] array (built in the preceding action #
# section) in descending order by the first 8+ digits). #
#-------------------------------------------------------#
if (nosort == 0)
{
yes = 1; no = 0; swap = no
for (cursor = 1; cursor < i; cursor++)
{
lowest = cursor
for (n=cursor+1; n <= i ; n++)
if (substr(line[lowest],1,len) < substr(line[n],1,len))
lowest = n
#-------------------------------------------#
# Only if we find a lower value do we swap. #
#-------------------------------------------#
if (lowest != cursor)
{
temp = line[cursor]
line[cursor] = line[lowest]
line[lowest] = temp
}
} # for (cursor = 1 ; cursor < i ; cursor++)
} # if (sort_opt == 1)
#-----------------------------------------------------#
# Compress every element in line[] array. #
#-----------------------------------------------------#
for (n=1 ; n <= i ; n++)
{
if (verbose) print sp""line[n]
sub(/^.* +/,"", line[n]) # Remove sort key,
#---------------------------------------------------#
# Only if the dog exists, do we try to compress it. #
#---------------------------------------------------#
status=system("test -f "line[n])
close( system("test -f "line[n]))
if (status != 0) continue
if (match(line[n], /\.(Z|gz)$/)) continue # Skip this dog.
print sp""pr_cmp_prog"ing "line[n] # Inform the user
if (! opt_T) # and if NOT a test,
{ # then go ahead and
status=system(cmp_prog" -f "line[n]" 2> /dev/null")
close( system(cmp_prog" -f "line[n]" 2> /dev/null"))
if (status != 0)
{ #-------------------------------------------------#
# On error, fuss at the user but keep on truckin. #
#-------------------------------------------------#
print sp""cmp_prog" ERROR: "line[n]"!" | "cat 1>&2"
print sp""cmp_prog" ERROR: "line[n]"!"
}
}
} # for (n=1 ; n <= i ; n++)
} # E.O. END ..
#-------------------------------------------------------#
function print_err_msg() # Globals: len
#-------------------------------------------------------#
{
print sp"Nothing compressed. No "len"-digit numeric key",
"in sort target fileid!",
"\n"sp"Use \047-f\047 option to specify a different",
"sort field",
"\n"sp"or \047-l\047 option to specify a different",
"sort field length." | "cat 1>&2"
}' | $teelog
CBD_status=$?
if [ $CBD_status -ne 0 ]; then
EMAIL_MSG "ERROR: $CBD_ID" \
"${sp}Problem compressing $2. Check out the log." \
"${sp}Skipping file compression today." \
"${sp}$script_name ($$) continues."
fi
return $CBD_status
} # "CBD_" prefix identifies this function's variables
fi
#======================================================================#
# D O C U M E N T A T I O N #
#======================================================================#
# #
# Author: Bob Orlando #
# #
# Date: July 25, 2001 #
# #
# Program ID: compress_by_date.sh #
# #
# Usage: COMPRESS_BY_DATE -STvw -s -n -l 6|8 -f field -t char \ #
# nn file_root #
# #
# -S = Use sudo to compress files. #
# #
# -T = Test only, compress nothing--simply #
# pretend we are. #
# #
# -v = Verbose. #
# #
# -w = Debug or Double-Verbose. #
# #
# -s = Follow symbolic links. #
# #
# -n = Do NOT sort the fileids being compressed #
# (sort is primarily for readability). #
# #
# -f = Field specified the test/sort field #
# -- defaults to the last field. #
# #
# -l = specifies the sort field length, len #
# -- default is 6 (for yymmdd). #
# #
# -t = Specifies separation character, char #
# -- defaults to a period. #
# #
# nn = Number of days we keep #
# -- compress files older than 'n' days. #
# #
# file_root = Fully-qualified fileid #
# -- up to the numeric sort data #
# (e.g. "$HOME/logs/daily_log"). #
# #
# Purpose: Compress files having yymmdd or yyyymmdd date strings #
# in their fileids that are older than a date nn days #
# old. #
# #
# Description: Borrowing from REMOVE_BY_DATE and PRUNE_FILES, this #
# function requires that the files to be compressed have #
# either yymmdd or yyyymmdd date string in their fileids. #
# This is critical because we compress these files by #
# sorting the most recent files to the top of our list #
# and compressing those with date stamps older that a #
# date from is nn days back. #
# #
# The ability to specify a field on which to key is #
# useful because files may have other numeric strings #
# in their fileids, or they may have multiple date #
# strings (e.g. a "data" date and a process date -- #
# 'deletes.000101_000401' -- data dated January 1, #
# but processed/created on April 1). #
# #
# Examples: Assume the following files have two periods and two #
# underscores which result in three fields each). #
# #
# +----------+------------ Underscores #
# | +-- | -----+----- Periods #
# | | | | #
# v v v v #
# test_010101.out_010120.10 #
# test_010102.out_010119.11 #
# test_010103.out_010118.12 #
# test_010104.out_010117.13 #
# test_010105.out_010116.14 #
# test_010106.out_010115.15 #
# test_010107.out_010114.16 #
# test_010108.out_010113.17 #
# test_010109.out_010112.18 #
# test_010110.out_010111.19 #
# #
# Using the first line, our period-delimited fields #
# are: #
# f1 = test_010101 #
# f2 = out_010120 #
# f3 = 10 -- has no 6|8-digit (date) string. #
# #
# Using the first line, unserscore-delimited fields #
# are: #
# f1 = test -- has no 6|8-digit (date) string. #
# f2 = 010101.out #
# f3 = 010120.10 #
# #
# 1. Compresses files based on the last 6-digit (date- #
# -like), period-delimited field. Example 'a' uses #
# both defaults. Example 'b' uses '-f2' (which is #
# the last period-delimited field containing a #
# datelike numeric string--keeping 7 days). #
# #
# a. COMPRESS_BY_DATE 7 test_01 #
# b. COMPRESS_BY_DATE -t. -f2 7 test_01 #
# #
# Compressing test_010108.out_010113.17 #
# Compressing test_010109.out_010112.18 #
# Compressing test_010110.out_010111.19 #
# #
# 2. Compresses files based on the last 8-digit (date- #
# -like), period-delimited field. Example 'a' uses #
# the default field delimiter (.), but requests first #
# field (-f1). Example 'b' is functionally identical. #
# #
# a. COMPRESS_BY_DATE -f1 5 test_01 #
# b. COMPRESS_BY_DATE -t. -f1 5 test_01 #
# #
# Compressing test_010105.out_010116.14 #
# Compressing test_010104.out_010117.13 #
# Compressing test_010103.out_010118.12 #
# Compressing test_010102.out_010119.11 #
# Compressing test_010101.out_010120.10 #
# #
# 3. Assuming we have 8-digit dates in our fileids, we #
# compresses files based on the last 8-digit (date- #
# -like), underscore-delimited field. Example uses #
# '-t_' to specify an underscore as the field #
# delimiter (it can be any character). The last #
# field is the default. #
# #
# COMPRESS_BY_DATE -l 8 -t_ 8 test_2001 #
# #
# Compressing test_20010109.out_20010112.18 #
# Compressing test_20010110.out_20010111.19 #
# #
# 4. This example shows a common error as it attempts to #
# Compress files from the first underscore-delimited #
# field. (The 'test_' portion of the line is the #
# the first underscore-delimited field. The user #
# should have specified -f2 (indicating the 2nd field) #
# to isolate the date following 'test_' and before #
# '.out_19...'.) Since the first field has no date #
# string, the following error message is printed. #
# #
# COMPRESS_BY_DATE -t_ -f1 5 test_01 #
# #
# No date string in field 1! #
# Nothing compressed. #
# #
# Globals: No global variables assigned from this function. #
# "CBD_" prefix identifies local function variables. #
# #
# Calls: EMAIL_MSG and EXIT library functions, and dateplus. #
# #
# Exit_status: 0 = Success #
# 1 = Any failure #
# #
# Exits with failure (1) on severe error (e.g. user #
# provides insufficient or invalid arguments). #
# #
# Notes: ...................................................... #
# ...................................................... #
# #
# Modified: 2006-06-06 Bob Orlando #
# v1.16 * Add sudo (-S), test (-T), and debug (-w) #
# options. (Test compresses nothing--it just #
# shows what it WOULD compress.) #
# #
#----------------------------------------------------------------------#
|