my $usage = qq(Usage:\n\tperl $0 reptask_xxx.log  [ > redirected_output_file ]\n\t-h for help\n);
my $help = << "EOF";

  load_log_statistics.pl    Hein van den Heuvel, Attunity, 
	15-Feb-2017 - Add Options
	01-may-2017 - Allow for out-of-order wildcarded log files.
	15-dec-2018 - print file name above error sections.
	05-feb-2019 - add records/second
  -O  suppress OWNER in output.
 
$usage

EOF
# 00004372: 2013-07-23T23:15:07 [TASK_MANAGER    ]I:  Start loading table 'OE10'.'ORDER_ITEMS' (Id = 40) by subtask 11  (replicationtask_util.c:895)
# 00004184: 2013-07-23T23:15:08 [METADATA_MANAGER]W:  Target table 'OE10.ORDER_ITEMS' does not have a primary key or a unique index. Processing changes on this table can affect performance.  (metadatamanager.c:2868)
# 00004372: 2013-07-23T23:16:24 [TASK_MANAGER    ]I:  Table 'OE05'.'ORDERS' (Id = 19) Loading finished by subtask 19. 225000000 records transferred.  (replicationtask.c:1079)

#00003832: 2015-06-10T00:10:22 [SORTER          ]I:  Start collecting changes for table id = 142  (sorter_transaction.c:1497)
#00011540: 2015-06-10T00:10:22 [TASK_MANAGER    ]I:  Start loading table 'dbo'.'capvoucher' (Id = 142) by subtask 1. Start load timestamp 00051824916E3B83  (replicationtask_util.c:1022)
#00006868: 2015-06-10T00:10:22 [TARGET_LOAD     ]I:  Going to connect to ODBC connection string: DRIVER={NetezzaSQL};SERVERNAME=172.23.13.177;PORT=5480;DATABASE=QNXT_PLANDATA_CA_REB;UID=EDW;  (odbc_endpoint_imp.c:1765)
#00006868: 2015-06-10T00:10:22 [INFRASTRUCTURE  ]I:  Allocated 20 ODBC environment handles  (ar_odbc_func.c:181)

#00013660: 2015-06-10T06:10:52 [TARGET_LOAD     ]I:  Load finished for table 'dbo'.'capvoucher' (Id = 142). 91823741 rows received. 0 rows skipped. Volume transfered 102727745176  (streamcomponent.c:2768)
#00013660: 2015-06-10T06:10:54 [METADATA_MANAGER]W:  Target table 'EDW.CAPVOUCHER' does not have a primary key or a unique index. Processing changes on this table can affect performance  (metadatamanager.c:3620)
#00011540: 2015-06-10T06:10:54 [TASK_MANAGER    ]I:  Table 'dbo'.'capvoucher' (Id = 142) Loading finished by subtask 1. 91823741 records transferred.  (replicationtask.c:1474)
# 300 rows received. 296 rows skipped. 

# 00009272: 2016-10-04T21:03:03 [TASK_MANAGER    ]W:  Table 'SWDBA09'.'SESPF50_VIND_WMI' (subtask 3 thread 0) is suspended.  (replicationtask.c:1764)

use warnings;
use strict;
use Time::Local;

use Getopt::Std;
my %opt;
if (!getopts ('OShH',\%opt)) { print STDERR $usage; exit; };
if ($opt{h} || $opt{H}) { print STDERR $help; exit; };
my $no_Server   = $opt{S};
my $no_Owner    = $opt{O};
my $verbose     = $opt{v};


my (%start, %end, %records, %skipped, %tables, %ids, %rank, %suspended,%r,%apply_errors,%thread);
my ($first, $last, $logs, $oracle_errors, $id, $r, $owner, $tables, $records, $skipped, $rank, $time) = (9876543210,0,0,0,0,0,0,0,0,0,0,0);
my $last_file_with_error = '';
$oracle_errors = -1000;
while (my $files = shift) {
  for my $file (glob $files) {


#00013808: 2016-10-05T15:05:09 [AT_GLOBAL       ]I:  Task Server Log - DB2_C1_DB23_T1 (V5.1.0.334 fcvas2669.md2.ford.com Microsoft Windows Server 2012  (build 9200) 64-bit, PID: 17576) started at Wed Oct 05 15:05:09 2016  (at_logger.c:2366)
#00002560: 2016-07-07T10:23:33 [SERVER          ]I:  Task Server Log- FIS_KCAP_FINT (V5.0.2.33
#
#Task 'MVPRD_LM_CDC2' running CDC only in resume mode 
#
	my ($ident, $rolled) = ('<No Ident seen>', 'Rolled');
	my $last_log_end = $end{$logs};
	open LOG, "<$file" or die "Could not open log <$file>\n$!";
	$logs--;
    while (<LOG>) { #Pick up header line for time, name and version, and/or restart line
		if (/(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d).*.*Log.*?(\(\S+)\s+(\S+)/) {
			$start{$logs} = timelocal($6,$5,$4,$3,$2-1,$1);
			$ident = ($no_Server) ? $7 : qq($7 $8);
			$ident =~ s/\(/- /;
		}
#[TASK_MANAGER    ]I:  Task 'MVPRD_LM_CDC2' running CDC only in resume mode  (replicationtask.c:1006)
#[TASK_MANAGER    ]W:  Task 'MVPRD_LM_CDC2' encountered a recoverable error  (repository.c:4711)

		if (/:  Task.*(full load|CDC).*replication/) { # not a ROLLED log, but a fresh start.  position: 'timestamp:2017-04-10T20:52:36'
			$rolled = q(Error ) if /error/i;
			$rolled = q(Resume) if /resume/i;
			if (/fresh/i) {
				$r{$_} = 0 for (keys %r);
				$rolled = q(Fresh );
			}
			$rolled = $1 if /position: '([^']+)/;
			$rolled = $1 if /retry (#\d+)/i;
			if ($last_log_end) {
				for (keys %start) { # Cancel any active load in our book-keeping.
					$suspended{$_} = $last_log_end unless ($end{$_} || $suspended{$_}); 
				}
			}
		}
		if ( $. > 20) { # Seen enough?
			$tables{$logs} = qq( $rolled $ident $file);
			last;
		}

	}
	printf STDERR "-- First lines in log <$file> not as expected.\n" unless $start{$logs};
#	print STDERR "== $file\n";
#
# Scan each log looking for lines refering to tables being loaded 
# Load all data into arrays to be processed for printout later.
#
	seek LOG,0,0;  # rewind
	while (<LOG>) {

	if (/\][EW]:/) {
		if ( $last_file_with_error ne $file ) {
			$last_file_with_error = $file;
			print STDERR qq(\n*** $file ***\n\n);
		}
		print STDERR;
	}
	
			my $time = timelocal($6,$5,$4,$3,$2-1,$1) if /(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)/;

		if (/..(ORA-.+:)(.{50})/) { # avoid ""ORA-00334: archived log..." on line
			$oracle_errors--;
			$tables{$oracle_errors} = qq( $1 $2);
			$start{$oracle_errors} = $time;
			next;
		}
		
#		[TARGET_APPLY    ]T:  Apply conflict error was ignored based on error behaviour policy. table 'dbo'.'T_LIABILITY'.

		if (/TARGET_APPLY.*error.*table\s+(\S+)\.\s/) {
			my $table = $1;
			$table =~ s/'//g;  #'
			$apply_errors{$table}++;
		}
		
		next unless   /(TASK_MANAGER|TARGET_LOAD).*able (\S+)/;
		my $table = $2;
		$last = $time if $last < $time;  # Allow for out-of-order files.
		$table =~ s/'//g;  #'
		($owner,$table) = split (/\./, $table) if $no_Owner;

		if (/I:  Start loading.*Id =\s+(\d+).\s+.*?subtask\s+(\d+)/) {
			$id = $1;
			$thread{$id} = $2;
			if (defined($r{$id})) {  # Did we see this table ID before? Increment repeat count.
				$r{$id}++;
			} else {
				$tables++;
				$r{$id} = 1;
				$tables{$id} = $table; # Map ID to NAME
				$ids{$table} = $id;    # Map NAME to ID, for SUSPEND info.
				$first = $time if $time < $first;  # Allow for out-of-order files.
			}
			$start{qq($id,$r{$id})} = $time;
			next;
		}

		if (/suspended/) {
			$id = $ids{$table};
			printf STDERR qq(-- no ID found for Suspended Table <$table>\n) unless $id;
#			printf STDERR qq(== $id,$r{$id} == $table == \n);
			$suspended{qq($id,$r{$id})} = $time;
			next;
		}


	#     Load finished for table ''.'PC_HIS_SVC_EXP_CLSVC_NONALL' (Id = 29). 140 rows received. 0 rows skipped. Volume transfered 95640  (streamcomponent.c:2768)

		if (/I:  Load finished.*Id =\s+(\d+)..\s+(\d+)\s+rows received.\s+(\d+)\s+rows skip/) {
			$id = qq($1,$r{$1});
			$end{$id} = $time;
			$records{$id} = $2;
			$skipped{$id} = $3;
			$records += $2;
			$skipped += $3;
			next;
		}
		
		if (/(ORA-.+:)(.{50})/) {
		print STDERR;
			$oracle_errors--;
			$tables{$oracle_errors} = qq($1 $2);
			$start{$oracle_errors} = $time;
		}

##00011540: 2015-06-10T00:10:22 [TASK_MANAGER    ]I:  Start loading table 'dbo'.'capvoucher' (Id = 142) by subtask 1. Start load timestamp 00051824916E3B83  (replicationtask_util.c:1022)

	} # log loop
	close LOG;
	$end{$logs} = $time;
  }
}

#
# Collected all the data, time to report from the Arrays.
# 
	my ($duration, $dot, $dots, $marks, $spaces, $start, $hh, $mm, $ss, $hhmmss, $star);

	$duration = $last - $first;
	$dot = $duration / 60;

	# print "$first - $last - $duration - dot=$dot\n\n";

	if ($tables) {
		print qq(\n Timeline Legend: '.' = in queue, '*' = normal load, '!' = Failed load, <number>=repeat load, '~' = Suspended\n\n);
		printf qq(Date  Time    Duration Rows-Loaded %sTable\n\n), ($skipped)? q(Skipped) : q();
	}
	# printf qq(%6d %6d\n),$_,$thread{$_} for (keys %thread);
	for my $id (sort {$start{$a} <=> $start{$b}} keys %start) {

		my ($i,$r) = split /,/,$id;
		if ($i < 0) { # log file switch ?
			;
			printf qq(%s -------- %s\n), &my_time($start{$id}), defined(($tables{$id})) ? $tables{$id} : "? $id";
			next
		}

		if ($end{$id}) {
#			print STDERR "-- $id -- $end{$id} -- $tables{$i}\n" unless $r{$i};
			$star = ($r == 1 && $r{$i} == 1) ? q(*) : $r; 	#  Marker for NORMAL load, or REPEAT
			$star = q(~) if $suspended{$id};
		} else {
			$records{$id} = 0;
			$skipped{$id} = 0;
			my $suspended = $suspended{$id};
			if ($suspended) {
				$end{$id} = $suspended;
				$star = q(!);    #  Marker for FAILED load
			} else {
				$end{$id} = $last;
				$star = q{?};    #  Marker for UNFINISHED load
			}
		}

		$time = $end{$id} - $start{$id};
		$hh = int($time/3600);
		$mm = int(($time - $hh*3600)/60);
		$ss = $time % 60;

		$start = $start{$id} - $first;
		$marks = int($time/$dot);
		$marks = 1 unless $marks > 0;
		$dots = int($start/$dot);
		$spaces = 60 - $dots - $marks;
		$spaces = 1 unless $spaces > 0;
		my ($orig_id) = split /,/,$id;
		printf qq(%s %02d:%02d:%02d %2d %10d %5d %s%-30s %s%s%s|\n), &my_time($start{$id}), $hh, $mm, $ss,
			$thread{$orig_id}, $records{$id} - $skipped{$id}, ($time)? ($records{$id} - $skipped{$id})/$time : 0,
				($skipped)? sprintf(q(%7d ),$skipped{$id}):'', substr( $tables{$i},0,30), '.' x $dots, $star x $marks, ' ' x $spaces;
		}
		# printf STDOUT "$start_hhmmss{$id} s=$start{$id} e=$end{$id} t=$time, m=$marks $tables{$id}\n" if $records{$id} > 1000000;
		$time = $duration;
		$hh = int($time/3600);
		$mm = int(($time - $hh*3600)/60);
		$hhmmss = sprintf("%02dh%02dm%02ds", $hh, $mm, $time%60);

		$hh = int($dot/3600);
		$mm = int(($dot - $hh*3600)/60);
		my $dot_hhmmss = sprintf (qq(%02dh%02dm%02ds), $hh, $mm, $dot%60); 
		my $loaded = $records - $skipped;
		if ($records) {
#                        02/03 16:56:41 07:00:16  2  687513936 27264 dbo.CER_ATXR_ATTACH_U          ..........................................
			print qq(\n\nStart DateTime  Elapsed  #  Rows  Rows/sec. Schema.TableName               < TimeLine - One mark is $dot_hhmmss >\n);
			printf(qq(\n\nTotal %d rows in %d tables. %s Elapsed. %d Records/second overall, %d rows skipped.\n\n), 
				$loaded, $tables, $hhmmss, ($duration)? ($loaded - $skipped)/$duration: 0, $skipped  );
		}
if (keys %apply_errors){
	print qq(\n\nThere were apply errors\n\n);
	for my $table (sort {$apply_errors{$b} <=> $apply_errors{$a}} keys %apply_errors) {
		print qq($apply_errors{$table}\t$table\n);
	}
}
		
sub my_time() {
		my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(shift @_);
        return sprintf( q(%02d/%02d %02d:%02d:%02d), $mon+1, $mday,$hour,$min,$sec);
}