#!/usr/bin/perl -w
#--------------------------------------------------------------------------------------------------------------
#
# Convert overnet raw data to the tabbed format.
#
# usage: 
# ./overnet2tab.pl -input ~/research/fta/data/overnet03_raw/probe -output ~/research/fta/data/overnet03_tab
#
# Contact: Derrick Kondo derrick.kondo :: inria fr
#
# Copyright 2009 by Failure Trace Archive site (http://fta.inria.fr)
#
#--------------------------------------------------------------------------------------------------------------


# ASSUMES LAST TIME STAMP FOR ALL HOSTS IS THE SAME, WHICH IS TRUE WITH THE LAST PROBE pd_503.  I COUNTED ALL HOSTS IN THE LAST PROBE AND IT MATCHES THAT OF THE NUMBER OF HOSTS IN THE COMPONENT FILE

############################################################################
# TESTS:

# total number of hosts is correct

# avail intv,unavail right for 05000000000000000000000000000000
# some hosts like 017fd554864f8428a179daea50cef491 don't exist at first
# test: node "017c24d109177f7e0f1e5a587a306d8f" is always unavailable but unavail intv is correct

# checked component

# have about 3000 hosts in node.tab

# limit scope of variables
use strict;
 
# Module to help handle command line switches
use Getopt::Long;
use FTA_Format;

# method headers
sub main();
sub extract_parameters();

sub create_platform ($);
sub create_node ($);
sub create_node_perf ($);
sub create_component ($);
sub create_event_trace ($$);
sub create_event_state ($);
sub process_probe ($$$$);

# platform globals
my ($PLATFORM_ID, $PLATFORM_NAME, $PLATFORM_LOCATION, $PLATFORM_TYPE);

$PLATFORM_ID = 0;
$PLATFORM_NAME = "overnet";
$PLATFORM_LOCATION = "worldwide";
$PLATFORM_TYPE = "p2p"; 

# node globals

# DATA STRUCTURES

# %NODE is a hash table that points to a array reference,
# %which points to an array of events.

# node_name -> events arr (node_id, reference to event_traces array of
# (event_id, event_type, start time, stop time))

my (%NODE, $NODE_ID_COUNTER);

$NODE_ID_COUNTER = 0;

# component globals
my ($COMPONENT_ID, $COMPONENT_CITE, $RESOLUTION);
$COMPONENT_ID = 0;
$RESOLUTION = 20*60; # 20 minutes


# creator globals
my ($CREATOR_ID, $CREATOR, $CITE, $COPYRIGHT);
$CREATOR_ID = 0;
# assume no spaces here
$CREATOR = "Ranjita_Bhagwan";
# bibtex entry difficult to parse in tabbed format. Thus put
# bibtex citation instead.
$CITE = "iptps_bhagwan";
$COPYRIGHT = "Ranjita_Bhagwan";

# event_trace globals

main;

sub main (){
    my (@args, $start, $stop, $cwd);

    my ($inputdir, $outputdir, $a, $b);

    ($inputdir, $outputdir) = extract_parameters();
    
    createDir ($outputdir);
    create_platform ($outputdir);
    create_node_perf ($outputdir);
    create_event_trace ($inputdir, $outputdir);
    create_event_state ($outputdir);
# must be called after create_event_trace
    create_node ($outputdir);
    create_component ($outputdir);
    create_creator ($outputdir);

}

sub create_platform ($){
    my ($outputdir) = @_;

    open (OUT, ">$outputdir/platform.tab") or die "could not open $outputdir/platform.tab";

    print_header (*OUT, "platform");

    print_ln (*OUT, "platform", [$PLATFORM_ID, $PLATFORM_NAME,
		     $PLATFORM_LOCATION, $PLATFORM_TYPE, "NULL"]);

    close (OUT);
}
 

sub create_creator ($){
  my ($outputdir) = @_;
  
  open (OUT, ">$outputdir/creator.tab") or die "could not open $outputdir/creator.tab";
  
  print_header (*OUT, "creator");
  
  my (@node_names) = keys (%NODE);
  @node_names = sort @node_names;
  
  @node_names > 0 or die "no nodes\n";
  
  my ($node_name);
  foreach $node_name (@node_names){
    my ($events, $node_id, $event_traces, $event_info);
    
    print_ln (*OUT, "creator", [$CREATOR_ID, $COMPONENT_ID,
		$NODE{$node_name}->[0], $PLATFORM_ID, $CREATOR, $CITE, $COPYRIGHT]);
    
  }
  
  close (OUT);
}


# create_event_trace must be called before
sub create_node ($){
    my ($outputdir) = @_;


    my ($node_name);

    open (NODE, ">$outputdir/node.tab") or die "could not open $outputdir/node.tab";

    print_header (*NODE, "node");

    my (@node_names) = keys (%NODE);
    @node_names = sort @node_names;

    @node_names > 0 or die "no nodes\n";

    foreach $node_name (@node_names){
	my ($events, $node_id, $event_traces, $event_info);
	
	print_ln (*NODE, "node",
		  [$NODE{$node_name}->[0], $PLATFORM_ID,
		   $node_name, "NULL", "NULL", "NULL", "NULL", "NULL",
		   "NULL", "NULL", "NULL", "NULL", "NULL", "NULL",
		   "NULL", "NULL", "NULL"]);
	
    }

    close (NODE);
}

# create_node_perf
sub create_node_perf($){
    my ($outputdir) = @_;

    open (NODE_PERF, ">$outputdir/node_perf.tab") or die "could not open $outputdir/node_perf.tab";

    print_header (*NODE_PERF, "node_perf");

    close (NODE_PERF);

}

# create_event_state
sub create_event_state($){
    my ($outputdir) = @_;

    open (EVENT_STATE, ">$outputdir/event_state.tab") or die "could not open $outputdir/event_state.tab";

    print_header (*EVENT_STATE, "event_state");

    close (EVENT_STATE);

}

# create_event_trace must be called before

sub create_component ($){
    my ($outputdir) = @_;
    
    my ($component_type) = "0"; # 0 -> host availability
# should have crawler availability
#     my ($component_type2_crawler) = "crawler host availability ";

    open (COMPONENT, ">$outputdir/component.tab") or die "could not open $outputdir/component.tab";
    
    print_header (*COMPONENT, "component");

    my (@node_names) = keys (%NODE);
    @node_names = sort @node_names;

    @node_names > 0 or die "no nodes\n";
    
    my ($node_name);
    foreach $node_name (@node_names){
	my ($events, $node_id, $event_traces, $event_info);
	
	$events = $NODE{$node_name};

	$node_id = $events->[0];
	$event_traces = $events->[1];

	my ($trace_start, $trace_end);
# get start time of first event
	$trace_start = $event_traces->[0]->[2];
# get stop time of last event
	$trace_end = $event_traces->[-1]->[3];

	print_ln (*COMPONENT, "component", [$COMPONENT_ID,
					    $node_id,
					    $PLATFORM_ID,
					    $CREATOR_ID,
					    $node_name,
					    $component_type,
					    $trace_start,
					    $trace_end,
					    $RESOLUTION]);
	
    }

    close (COMPONENT);
    
}



sub create_event_trace ($$){
    my ($inputdir, $outputdir) = @_;
    
    my ($event_id, $component_id, $node_id,
	$event_type, $event_state, $event_start_time,
	$event_end_time, $event_end_reason);

    my (@files, $f);

    $event_id = 0;
 
# open input directory   
    opendir (INPUTDIR, $inputdir);
    
# read in inputs
    @files = grep { /^pd\_\d+$/ } readdir (INPUTDIR);
    
    print "Number of trace files: " . @files . "\n";

    @files = sort by_pb_num @files;
    
    my ($node_name, $timestamp);
    foreach $f (@files){
	
	print "\tProcessing file $f\n";
	
	open (IN, "$inputdir/$f") or die "$inputdir/$f could not be opened";
	
	my (@info, $line); 
	    
	while ($line = <IN>){
#	    print "\t$line\n";
	    chomp ($line);

	    next if $line =~ /^(\s)*$/;  # skip blank lines

# read in timestamp
	    if($line =~ m/^\#/){
		
		@info = split /\s+/, $line;
		
		$timestamp = $info[-1]; 
		
# 		print "\t$timestamp\n";

# process node availability
	    } elsif ($line =~ m/^\w+/){
		
		my ($location, $event_type);
		($node_name, $location, $event_type) = split /\s+/, $line;

# 		print "line: $line\n" if $node_name eq "017c24d109177f7e0f1e5a587a306d8f";
	
		defined ($node_name) or die "node_name undef";
		defined ($location) or die "location undef";
		defined ($event_type) or die "event_type undef";
		defined ($timestamp) or die "timestamp undef";

# 		print "event_type: $event_type\n" if $node_name eq "017c24d109177f7e0f1e5a587a306d8f";
		process_probe ($node_name, $location,
			       $event_type, $timestamp);
	    }
	    
	} # while
	
	close (IN);
	
    } # foreach
    close INPUTDIR;

# at end of parsing last probe file, insert stop time for
# last interval of each node

    my (@node_names) ;
    @node_names = keys (%NODE);
    
#     print "num node names: " . @node_names . "\n";

    my ($nn);
    foreach $nn (@node_names) {
	my ($events, $event_info);

	$events = $NODE{$nn}->[1];
	
# get most recent event type
	!defined($events->[-1]->[3]) or die "last intv shouldn't be defined";
	$event_info = $events->[-1];

	defined ($event_info->[1]) or die "event_type undef";

	# ASSUMES LAST TIME STAMP FOR ALL HOSTS IS THE SAME, WHICH IS TRUE WITH THE LAST PROBE pd_503.  I COUNTED ALL HOSTS IN THE LAST PROBE AND IT MATCHES THAT OF THE NUMBER OF HOSTS IN THE COMPONENT FILE
	# add stop time to last event: stop = timestamp
	$event_info->[3] = $timestamp;

# 	print "last timestamp"  if $node_name eq "017c24d109177f7e0f1e5a587a306d8f";
    }
    
    print_event_trace($outputdir);

}

sub print_event_trace($){
    my ($outputdir)=@_;
 
    my (@node_names) = keys (%NODE);
    @node_names = sort @node_names;

    my ($node_name);

    open (EVENT_TRACE, ">$outputdir/event_trace.tab") or die
	"file $outputdir/event_trace.tab could not be used for output";

    print_header (*EVENT_TRACE, "event_trace");

    foreach $node_name (@node_names){
	my ($events, $node_id, $event_traces, $event_info);
	
	$events = $NODE{$node_name};
	defined ($events) or die "events undef";
	$node_id = $events->[0];

	$event_traces = $events->[1];
	
# 	print "node_name: $node_name\n";

	foreach $event_info (@$event_traces){
	    my ($event_id, $event_type, $start_time,
		$stop_time);

	    defined ($event_info) or die "event_info undef";
# 	    print "event_info: " . @$event_info . "\n";
# 	    print "event_id: " . $event_info->[1] . "\n";

	    $event_id = $event_info->[0];
	    $event_type = $event_info->[1];
	    $start_time = $event_info->[2];
	    $stop_time = $event_info->[3];

# 	    print "\tevent_id: $event_id, COMPONENT_ID: $COMPONENT_ID, node_id: $node_id, platform_id: $PLATFORM_ event_type: $event_type, start_time: $start_time, stop_time: $stop_time\n";

	    defined ($event_id) or die "event id undef";
	    defined ($COMPONENT_ID) or die "COMPONENT_ID undef";
	    defined ($node_id) or die "node_id undef";
	    defined ($PLATFORM_ID) or die "platform_id undef";
	    defined ($event_type) or die "event_type undef";
	    defined ($start_time) or die "start_time undef";
	    defined ($stop_time) or die "stop_time undef";

	    print_ln (*EVENT_TRACE, "event_trace", 
		 [$event_id, $COMPONENT_ID,
		 $node_id, $PLATFORM_ID, $node_name, $event_type, 
		 $start_time, $stop_time,
		 "NULL"]);
	}
    }
    close (EVENT_TRACE);
}


sub process_probe ($$$$){
    my ($node_name, $location, $event_type, $timestamp) = @_;

    my ($event_info, $events);

    defined ($event_type) or die "event_type undef";

#     print "node_name: $node_name\n" if $node_name eq "017c24d109177f7e0f1e5a587a306d8f";

# Process first event
    if (!exists($NODE{$node_name})){
# 	print "\tnew node\n" if $node_name eq "017c24d109177f7e0f1e5a587a306d8f";

# Allocate array dynamically and store reference
	$events = [];
# pointer to array
	$event_info = [0, $event_type, $timestamp, undef];
	push (@$events, $event_info);
# 	print "\t\tid: $NODE_ID_COUNTER\n";
	$NODE{$node_name} = [$NODE_ID_COUNTER, $events];
	$NODE_ID_COUNTER++; 

# Process later event
    } else {
	
	$events = $NODE{$node_name}->[1];

# get most recent event type
	$event_info = $events->[-1];

	my ($prev_event_type) = $event_info->[1];

# went from unavail -> avail or vice versa
	if ($prev_event_type != $event_type){
# 	    print "\tstate change\n" if $node_name eq "017c24d109177f7e0f1e5a587a306d8f";

	    # add end time to last event: stop = timestamp
	    $event_info->[3] = $timestamp;

	    # add new event
	    $event_info = [$event_info->[0]+1, $event_type, $timestamp, undef];
	    push (@$events, $event_info);
	}
    }
}



# If the dir doesn't exist, create it                                                                                         
                                                                                                                              
sub createDir ($){
    my ($dir) = @_;                                                                                                           
                                                                                                                              
    if (! -e $dir){                                                                                                           
        print "Creating directory $dir\n";                                                                                    
        my $retval = mkdir($dir, 0777);                                                                                       
        if ($retval == 0)                                                                                                     
        {                                                                                                                     
            die "Fatal Error: $dir could not be created\n Message is: $!\n. Program Terminating....\n";                       
        }                                                                                                                     
    }                                                                                                                         
                                                                                                                              
}                                                                                                                             


# sort by probe number
sub by_pb_num {                                                                                                                                  
    my ($text_a, $pb_num_a) = split /\_+/, $a;
    my ($text_b, $pb_num_b) = split /\_+/, $b;
    
   if ($pb_num_a  > $pb_num_b){                                                                                                                              
        return 1;                                                                                                                                        
    } elsif ($pb_num_a == $pb_num_b){                                                                                                                        
        return 0;                                                                                                                                        
    } else {                                                                                                                      
        return -1;                                                                                                                                       
    }                                                                                                                                                    
}

sub extract_parameters()
{
	my %params = (
		inputdir => \(my $inputdir = undef),
		outputdir => \(my $outputdir = undef),

	);

	# Call Library function which will extract and remove all switches and
	# their corresponding values.
	# These parameters will be removed from @ARGV

	my $errorval = &GetOptions(\%params, "inputdir=s", "outputdir=s");
                        # the exclaimation point allows for the negation
                        # of the switch (i.e. -nobackup/-nobody is a switch)

	# Error value should have returned 1 if nothing went wrong
	# Otherwise, an unlisted parameter was specified.
	if($errorval !=1)
	{
		# Expecting GetOptions to state error.

		print "Exiting Program...\n";
		exit 0;
	}



	defined ($inputdir) or die "inputdir not defined\n";
	defined ($outputdir) or die "outputdir not defined\n";
	(!-e $outputdir) or die "dir $outputdir already exists";
	return ($inputdir, $outputdir);
	
}
