#!/opt/freeware/bin/perl
use FindBin::libs;
use PostEdge::Config::site_perl;
use PostEdge::script;

use File::Basename;
use File::Copy;
use Getopt::Long;

use PE_Date;
use PE_Logging;
use PE_Notify  qw( email );

use PostEdge::Config::Host qw( curr_host );
use PostEdge::Util::File   qw( open_read open_write );

use PostEdge::Globals qw(
    $PRIMARY_PROD_HOST
    $SECONDARY_PROD_HOST
    $SIT_HOST
    $UAT_HOST
    $QA_HOST
    $HOST_ALTS
    $PROD_HOSTS

    $ARS_ROOT

    %EMAILS
);

my $curr_host      = curr_host;
my $curr_host_name = $curr_host->name;

#-------------------------------------------------------------------------------
# Script: dailyrecon
# Author: Daniel Goetz
# Date  : 05/31/2009
#
# Purpose:   Postedge OD daily recon  all-in-one daily status report
#
#
# Parms:
#
# -ag_string    -Optional.  Limit search to appgroups matching parameter.  Defaults to all app groups
# -details      -Optional.  Display all files loaded 
# -end_ago      -Optional.  Until number of days back to search OD system log. Default is 0 [today]
# -hour         -Optional.  The cutoff hour is an integer from 0 to 23. Default is 6 [for 6am]
# -local        -Optional.  First host to compare
# -mailgroup    -Optional.  Send out email to people to PE recon distrib.  Default goes to on-screen display.
# -mailme       -Optional.  Override pre-defined recipients with users PM_GLobal's assocaited email addr
# -noclean      -Optional.  We will delete the report file afterwards unless this option is specified
# -padding      -Optional.  Extra hours before and after to fine tune the query period
# -raw          -Optional.  No substitution of numbers-only filename segments.
# -remote       -Optional.  Second host to compare
# -start_ago    -Optional.  Number of days back to start search OD system log. Default is 1 [day ago]
#
#-------------------------------------------------------------------------------
my $me = basename($0);
my $started = time();
logMsg("starting...");


my $ag_string='';
my $details=0;
my $end_ago=0;
my $hour=6;
my $local_host_name=$PRIMARY_PROD_HOST;  # Do not change hostname to accomodate cluster name. Will default to active EDG server 
my $mailgroup=0;
my $mailme=0;
my $noclean=0;
my $padding=0;
my $raw=0;
my $remote_host_name;
my $start_ago=1;
my $host_name = $curr_host_name;



GetOptions(
           "ag_string=s" => \$ag_string,
           "details"     => \$details,
           "end_ago=i"   => \$end_ago,
           "hour=i"      => \$hour,
           "local=s"     => \$local_host_name,
           "mailgroup"   => \$mailgroup,
           "mailme"      => \$mailme,
           "noclean"     => \$noclean,
           "padding=i"   => \$padding,
           "raw"         => \$raw,
           "remote=s"    => \$remote_host_name,
           "start_ago=i" => \$start_ago
          ) or die "Could not accept parms: $! 
            ----- arg values shown here (all are optional)  ------
                        ag_string is: $ag_string
                        details   is: $details
                        end_ago   is: $end_ago
                        hour      is: $hour
                        local     is: $local_host_name
                        mailgroup is: $mailgroup
                        mailme    is: $mailme
                        noclean   is: $noclean
                        padding   is: $padding
                        raw       is: $raw
                        remote    is: $remote_host_name
                        start_ago is: $start_ago\n\n";

my $local_host = host( $local_host_name );

# Determine optional number of hours earlier & later around cutoff time, and format for report heading
  
my $start_hour = $hour - $padding ; 
my $end_hour   = $hour + $padding ;

$start_hour < 0 and do {
    $start_ago  = $start_ago + 1 + int ($start_hour / 24 );
    $start_hour = $start_hour % 24;
};

$end_hour > 23 and do {
    $end_ago    = $end_ago - 1  - int ($end_hour / 24 );
    $end_hour   = $end_hour % 24;
};


$start_hour = sprintf "%02d" , $start_hour;
$end_hour   = sprintf "%02d" , $end_hour;



# Establish query date ranges by converting query delimiters into seconds

my $start_time      = fmtDate ( time() - $start_ago * 86400 , "MM/DD/YY $start_hour:00:00");
my $end_time        = fmtDate ( time() - $end_ago   * 86400 , "MM/DD/YY $end_hour:00:00");

my (undef , $start) =  split ('->', `arsdate -z "$start_time"`);
my (undef , $end)   =  split ('->', `arsdate -z "$end_time"`);
chomp($start, $end);


# Establsh remote host for recon.  Leave of last char (number) in case running on eppspstd02, jpp..2, etc

if    ( $local_host->is_secondary_prod ) { $remote_host_name = $PRIMARY_PROD_HOST;   }
elsif ( $local_host->is_primary_prod   ) { $remote_host_name = $SECONDARY_PROD_HOST; }
elsif ( $local_host->is_sit            ) { $remote_host_name = $UAT_HOST;            }
elsif ( $local_host->is_uat            ) { $remote_host_name = $SIT_HOST;            }

die "Error" if !$remote_host_name;

# Create report file 

my $report_file="/postedge/tmp/$me.report.$started.txt";
my $rpt_fh = open_write($report_file)
   or die "Cannot open $report_file for writing";

my $arsload_cfg="$ARS_ROOT/config/arsload.cfg";
! -r $arsload_cfg and $arsload_cfg = "~$ENV{LOGNAME}/.arsload.cfg";
-e $arsload_cfg or die "\n
                        No arsload username/password config file found in your home directory.
                        To run this program as a non-odadmin user, please
                        1) Create a file in your home directory called .arsload.cfg containing just these 
                           two lines, substituting your OD ID & Password as shown:
                           USERID=yourID
                           PASSWD=yourPASSWORD
                           
                        2) Set permissions so that only you have full permissions by typing:
                               chmod 700 .arsload.cfg 
                     
                        
                        Once that's been completed, you should be able to run this program.\n\n";
                               



my %arsdoc_info;

my $cfg_fh = open_read($arsload_cfg)
   or dieWhining "Could not open config file for reading!";
while (<$cfg_fh>) 
{
   /^\w/ or next;
   chomp;
   my ($key, $value) = split ( /=/, $_ );
   $arsdoc_info{$key}=$value;
}
close ($cfg_fh) or dieWhining "Could not close config file after reading!";


# Begin main part of program - check for recon file(s) for working date for each type


my $buffer='';         # buffer for temp workspace throughout the program
my %loaded=();         # filename is double hash key of filename and hostname.
my %failed=();         # filename is double hash key of filename and hostname.  value is just datetime.
my @good_files=();     # all filenames that loaded
my @bad_files=();      # all filenames that failed
my @temp_array=();    



# Get local host stats on both servers for comparison


foreach my $host ($local_host_name,$remote_host_name)
{
   logMsg("Checking $host stats from  $start_time to  $end_time ...");

   $buffer=`arsdoc query -S "06032009,01172038,%m%d%Y" -u $arsdoc_info{USERID} -p $arsdoc_info{PASSWD} -h $host -f "System Log"  -i "where msg_text like '%Name(${ag_string}%' and msg_num in (87,88) and TIME_STAMP between $start and $end" 2>/dev/null`;

   my @lines = split("\n",$buffer);

   foreach my $line (@lines)
   {
      my ($datetime,$msg_num) =  (split(",", $line))[0,3];

      my $size="";
      $line =~ /InputSize/ and $size =  substr($line,index($line,' InputSize(')  + 11) ;
      $size =~ s/InputSize\((\S+)\)/$1/;
      $size =~ s/(\S+)\).*$/$1/;
      $size =~ s/--UNKNOWN--/ n\/a     /;
      1 while ($size =~ s/(.*\d)(\d\d\d)/$1,$2/g);


      my $appgroup =  substr($line,index($line,' Name(')  + 6) ;
      $appgroup =~ s/Name\((\S+)\)/$1/;
      $appgroup =~ s/(\S+)\).*$/$1/;


      my $file =  substr($line,index($line,' File(')  + 6) ;
      $file  =~ s/File\((\S+)\)/$1/;
      $file  =~ s/(\S+)\).*$/$1/;
      $file =   basename("$file");



      # Eliminate files which we do not balance and substitute process numbers, timestamps, etc - 

      $file  =~ m/000VIF[RPS]/ and next;
      $file  =~ m/ODVerify/    and next; 

      # Don't substitute numbers in IWS jobs.  Any other filename segment containing only numbers is a timestamp or process ID stamp, and is different on both boxes.  Make these filenames equal.

      # Remove server specific name from any MLIG pulled void reload filename
      state $prod_hosts_re = do {
          my $alt = join( "|", map quotemeta, @$PROD_HOSTS );
          qr/$alt/i
      };

      $file =~ /^MLIG/ and $file =~ s/$prod_hosts_re/ "x" . substr( $&, 1 ) /eg;
      $file =~ /^LPLGENPLETT.920.LPL/ and $file =~ s/^(LPLGENPLETT.920.LPL)([EJ])(.+)$/$1x$3/g;
      $file =~ s/^(000VIFNOAC.000.VIFNOAC\w).(\d\d\d)\d\d\d(.+)$/$1.$2nnn$3/g;

      if (! $raw and $file !~ /^(?:TDBANKPREV|IBMCM|\d\d\d(?:1099|5498))/ )
      {

         $file =~ s/^(\d\d\d\d\d\d)\d\d\d\d\.\d\d\d\d\d+\./${1}epoch.pid./g ;
         $file =~ s/^(\d\d\d\d+)\./num./g ;
         1 while ($file =~ s/\.(\d\d\d\d+)\./.num./g) ;
      }



      if ( $msg_num == 87 )
      {
         push @good_files,$file;

         $loaded{$host}{$file}{SIZE}=$size;
         $loaded{$host}{$file}{APPGROUP}=$appgroup;


         # Keep track of latest time if there are duplicates
         if (( exists $loaded{$local_host_name}{$file}{DATETIME} ) && ( $loaded{$host}{$file}{DATETIME}  lt  $datetime ))
         {  
            $loaded{$host}{$file}{DATETIME}=$datetime; 
         }

         if ( !  exists $loaded{$host}{$file}{DATETIME} ) {  $loaded{$host}{$file}{DATETIME}=$datetime; }


         # Keep track of duplicates
         if (   exists $loaded{$host}{$file}{COUNT} )  {  $loaded{$host}{$file}{COUNT} +=  1; }
         if ( ! exists $loaded{$host}{$file}{COUNT} )  {  $loaded{$host}{$file}{COUNT}  =  1; }

      }


      if ( $msg_num == 88 )
      {
         push @bad_files,$file;

         $failed{$host}{$file}{APPGROUP}=$appgroup;


         # Keep track of latest time if there are duplicates
         if (( exists $failed{$local_host_name}{$file}{DATETIME} ) && ( $failed{$host}{$file}{DATETIME}  lt  $datetime ))
         {  
            $failed{$host}{$file}{DATETIME}=$datetime; 
         }

         if ( ! exists $failed{$host}{$file}{DATETIME} ) {  $failed{$host}{$file}{DATETIME}=$datetime; }



         # Keep track of duplicates
         if (   exists $failed{$host}{$file}{COUNT} )  {  $failed{$host}{$file}{COUNT} +=  1; }
         if ( ! exists $failed{$host}{$file}{COUNT} )  {  $failed{$host}{$file}{COUNT}  =  1; }

      }
   }
}





# Sort array of all uniq files on all hosts - this will be used to generate an array of all hosts per file
# Also build any variables used to make the report subheading clearer

$ag_string and $ag_string=" for OD app groups matching pattern \'$ag_string\%\'";

my $previous_element = 'no-such-file-name';
@good_files = sort @good_files;
@temp_array = grep($_ ne $previous_element && (($previous_element) = $_), @good_files);
@good_files = sort @temp_array;


print $rpt_fh "\n      Recon Report between $local_host_name and $remote_host_name from $start_time to $end_time$ag_string\n";
print $rpt_fh   "      ------------------------------------------------------------------------------------------\n\n";

my $now=`date "+%a %m/%d/%Y  %I:%M %p"`;
print $rpt_fh "                                Report Run Time: $now\n\n";


print $rpt_fh "\n\n                 -=-=-=-  On Demand Exceptions Report$ag_string  -=-=-=- \n\n";
printf $rpt_fh "  %-17s  %15s     %-17s  %-17s   %-60s \n","AppGroup","Size in bytes",$local_host_name,$remote_host_name,"Filename\n\n";

my $missing_count=0;
my $recon_errors=0;

foreach my $file (@good_files) 
{
    $file =~ /^(?:000VIF)/ and next;    # Don't flag balances for this appgroup   

    my $recon_code=0;

    if  ( ! exists $loaded{$local_host_name}{$file}{DATETIME} )   { $recon_code +=  1 ; }
    if  ( ! exists $loaded{$remote_host_name}{$file}{DATETIME} )  { $recon_code +=  2 ; }


    $recon_code >= 3 and scream "$file appears to not have been found on either site yet " .
                                "it was detected as having loaded to either $local_host_name or $remote_host_name.\n" .
                                "Please investigate.   recon code was $recon_code.";  


    $recon_code == 1 and printf $rpt_fh "  %-17s  %15s  %-17s  %-17s   %-60s\n", 
                               $loaded{$remote_host_name}{$file}{APPGROUP},
                               $loaded{$remote_host_name}{$file}{SIZE},
                               "**** MISSING ****",
                               $loaded{$remote_host_name}{$file}{DATETIME},
                               $file ;


    $recon_code == 2 and printf $rpt_fh "  %-17s  %15s  %-17s  %-17s   %-60s\n", 
                               $loaded{$local_host_name}{$file}{APPGROUP},
                               $loaded{$local_host_name}{$file}{SIZE},
                               $loaded{$local_host_name}{$file}{DATETIME},
                               "**** MISSING ****",
                               $file ;


    $recon_code > 0  and $missing_count++;
}


# Now check for mismatches and duplicates

print $rpt_fh "\n";  # Print carriage return to separate between mismatches and warnings below

foreach my $file (@good_files) 
{

    $file =~ /^(?:000VIF)/ and next;    # Don't flag balances for this appgroup   

    if ( exists $loaded{$local_host_name}{$file}{SIZE}  &&  exists $loaded{$remote_host_name}{$file}{SIZE} )
    {

       if  ( $loaded{$local_host_name}{$file}{SIZE} != $loaded{$remote_host_name}{$file}{SIZE}  )
       { 
           print $rpt_fh " ** WARNING ==>  $file loaded with size mismatch: $local_host_name = $loaded{$local_host_name}{$file}{SIZE} bytes at $loaded{$local_host_name}{$file}{DATETIME}, $remote_host_name = $loaded{$remote_host_name}{$file}{SIZE} bytes at $loaded{$remote_host_name}{$file}{DATETIME}\n\n"; 
           $recon_errors++;
       }

       # Flag duplicates
       if  (( $loaded{$local_host_name}{$file}{COUNT} > 1  ||  $loaded{$remote_host_name}{$file}{COUNT} > 1 ) 
                                                 &&
            ( $loaded{$local_host_name}{$file}{COUNT}      !=  $loaded{$remote_host_name}{$file}{COUNT} )) 
       { 
           # Format values for reporting
           exists $loaded{$local_host_name}{$file}{COUNT}  or $loaded{$local_host_name}{$file}{COUNT}  = 0;
           exists $loaded{$remote_host_name}{$file}{COUNT} or $loaded{$remote_host_name}{$file}{COUNT} = 0;

           #print $rpt_fh " ** WARNING ==>  Duplicate count mismatch:  $loaded{$local_host_name}{$file}{COUNT}x on $local_host_name,  $loaded{$remote_host_name}{$file}{COUNT}x on $remote_host_name --- $file\n";
           printf $rpt_fh " ** WARNING ==>  Duplicate count mismatch:  %3dx on %s,  %3dx on %s --- %s\n", $loaded{$local_host_name}{$file}{COUNT}, $local_host_name,  $loaded{$remote_host_name}{$file}{COUNT}, $remote_host_name, $file;

           $recon_errors++;
       }

       if  ( $loaded{$local_host_name}{$file}{APPGROUP} != $loaded{$remote_host_name}{$file}{APPGROUP}  )
       { 
           print $rpt_fh "\n ** WARNING ==>  App Group mismatch!! filenames are the same, but loaded to different appgroups.  $file loaded to $local_host_name app group $loaded{$local_host_name}{$file}{APPGROUP},  and to $remote_host_name app group $loaded{$remote_host_name}{$file}{APPGROUP}!!\n"; 
           $recon_errors++;
       }
    }
}


if (  $recon_errors == 0   &&   $missing_count == 0  )  # Then we have no errors - everything is balanced!
{
   print $rpt_fh "   <<<<<<<  NO EXCEPTIONS OR MISSING FILES TO REPORT FOR THIS PERIOD  >>>>>>> \n\n";
}





print $rpt_fh "\n\n\n                 =-=-=-=  On Demand Ingest Failure Report$ag_string  =-=-=-= \n\n";
printf $rpt_fh "    %-17s%-17s   %-17s   %-60s \n","AppGroup",$local_host_name,$remote_host_name,"Filename\n\n";

$previous_element = 'no-such-file-name';
@bad_files = sort @bad_files;
@temp_array = grep($_ ne $previous_element && (($previous_element) = $_), @bad_files);
@bad_files = sort @temp_array;
my $fails = 0;

foreach my $file (@bad_files) 
{
   
   my $local_failed  = 0;
   my $remote_failed = 0;
   my $local_recovered  = 0;
   my $remote_recovered = 0;
   my $extra_recovery_phrase=" \& has not been recovered";



   printf $rpt_fh "    %-12s  %-17s   %-17s    %-60s\n",
                              $failed{$local_host_name}{$file}{APPGROUP},
                              $failed{$local_host_name}{$file}{DATETIME},
                              $failed{$remote_host_name}{$file}{DATETIME},
                              $file;
    

   exists $failed{$local_host_name}{$file}{DATETIME}   and   $local_failed   = 1;
   exists $failed{$remote_host_name}{$file}{DATETIME}  and   $remote_failed  = 1;


   if  (( exists $failed{$local_host_name}{$file}{DATETIME}  &&  exists $loaded{$local_host_name}{$file}{DATETIME}  ) 
                                                   && 
       (        $failed{$local_host_name}{$file}{DATETIME}   lt        $loaded{$local_host_name}{$file}{DATETIME}  ))

       { $local_recovered = 1 ; }


   if  (( exists $failed{$remote_host_name}{$file}{DATETIME}  &&  exists $loaded{$remote_host_name}{$file}{DATETIME}  ) 
                                                    && 
       (        $failed{$remote_host_name}{$file}{DATETIME}   lt        $loaded{$remote_host_name}{$file}{DATETIME}  ))

       { $remote_recovered = 1; }

    
    
   # Build report line to show where the file failed, and if & when it was recovered

   $fails++;

   if (( $local_failed == 1    and   $local_recovered   == 1  ) 
                                or
      (  $remote_failed == 1   and   $remote_recovered  == 1 )) 
   {
      my $print_line = sprintf " **RECOVERED**   (%-17s) (%-17s)   %-60s\n",
                              $loaded{$local_host_name}{$file}{DATETIME},
                              $loaded{$remote_host_name}{$file}{DATETIME},
                              $file;


      # remove empty parens that are shown if the file didn't file and get receovered 
      $print_line =~ s/ [\(\)] /   /g;  
     
      print $rpt_fh $print_line;

      $extra_recovery_phrase = " prior to being recovered";
   }

    $file =~ /^(?:000VIF)/ and next;    # Don't flag duplicates for this appgroup

    # Flag duplicates
    if  ( $failed{$local_host_name}{$file}{COUNT}   > 1 or  $failed{$remote_host_name}{$file}{COUNT}  > 1 )
    { 
        # Print zeroes on report instead of blanks

        exists $failed{$local_host_name}{$file}{COUNT}  or $failed{$local_host_name}{$file}{COUNT} =0;  
        exists $failed{$remote_host_name}{$file}{COUNT} or $failed{$remote_host_name}{$file}{COUNT}=0;  

        print $rpt_fh "       The file above had multiple unsuccessful load attempts" 
              . $extra_recovery_phrase 
              . ": $local_host_name=$failed{$local_host_name}{$file}{COUNT} fails , $remote_host_name=$failed{$remote_host_name}{$file}{COUNT} fails.\n";
        $fails++;
    }

    # Add carriage return to separate between files if recovered on one or both sites

    ( $remote_failed  or  $local_failed ) and print $rpt_fh "\n";  
}


if (  $fails == 0  )  # Then we have no errors - everything is balanced!
{
   print $rpt_fh "     <<<<<<<  NO FAILURES TO REPORT FOR THIS PERIOD  >>>>>>> \n\n";
}


$details and do 
{
   print $rpt_fh "\n\n\n    -=-=-=-=-=-  On Demand Load Balancing Detail Report$ag_string  -=-=-=-=-=- \n\n";
   printf $rpt_fh "   %-17s  %17s  | %12s   |      %-60s \n\n","$local_host_name time","$remote_host_name time","Load Count","Filename";

   foreach my $file (sort @good_files)
   {
      $file =~ /^(?:000VIF)/ and next;    # Don't display this appgroup

      printf $rpt_fh "  %17s -- %-17s | %5d -- %-5d |  %-60s\n", 
                               $loaded{$local_host_name}{$file}{DATETIME},
                               $loaded{$remote_host_name}{$file}{DATETIME},
                               $loaded{$local_host_name}{$file}{COUNT},
                               $loaded{$remote_host_name}{$file}{COUNT},
                               $file ;
   }
};


close ($rpt_fh) or die "Could not close $report_file after writing";



my $recipients='';

$mailgroup and $recipients = $EMAILS{dailyrecon};
$mailme    and do 
{
   $recipients = $EMAILS{$ENV{LOGNAME}};
   $recipients !~ /\w/ and $recipients = $EMAILS{goetzd}; #default to me if not found
};


$recipients or do   # If not emailing the file, then display to screen
{
   my $rptr_fh = open_read($report_file)
      or dieWhining "Cannot open $report_file for reading";
   $buffer='';
   while (<$rptr_fh>) { $buffer=$buffer . $_ ; }
   close ($rptr_fh) or die "Could not close $report_file after reading";

   print "$buffer\n\n\n" ;    # Display entire report to screen and pad with extra lines at end
};
 

$recipients and do   # Email the report
{

   logMsg "Emailing recon report to $recipients....";

   # Build Subject of email
   $details or  $details = '';
   $details and $details = "/Ingestion";

   email(SUBJECT=>"Recon Exception/Failure$details Report$ag_string attached",
      FILES=>[$report_file],
      TO=>["$recipients"],
      CC=>[''],
      TYPE=>'TEXT',
      MSG=><<EOM
Daily Postedge Recon Report is attached
EOM
      , FRIENDLY=>"$ENV{LOGNAME}.$host_name.$EMAILS{pe_noreply}");
};



$noclean or do 
{
   logMsg "cleaning up file $report_file\n";
   unlink ($report_file) or whine "Could not delete $report_file during post-run cleanup: $!.\nThis can wait until the next business day for investigation.";

};




logMsg("completed. Duration: ", time() - $started, " sec.");