suppressMessages(library(CORElearn))
  suppressMessages(library(dplyr))
  suppressMessages(library(plyr))
  suppressMessages(library(data.table))
  suppressMessages(library(randomForest))
  suppressMessages(library(ggplot2))
  suppressMessages(library(grid))
  suppressMessages(library(argparser))
  suppressMessages(library(stringr))

  suppressMessages(library(lubridate))

  
  export_ds_for_spm <- function(target_event,episodes_list,output){
    if (file.exists(output)) {
      file.remove(output)
    }
    #output for HirateYamana
    for(ep_index in (1:length(episodes_list))){
      ep = episodes_list[[ep_index]][ , !(names(episodes_list[[ep_index]]) %in% c("Timestamps"))]
      ep_list = list()
      for(i in (1:nrow(ep))){
        matches = which(ep[i,] %in% c(1))
        if(length(matches) == 0){
          next
        }
        line=paste(matches,collapse=" ")
        ep_list[i] = line
      }
      if(length(ep_list) == 0){
        next
      }
      ep_list[length(ep_list)+1] = target_event
      episode = ""
      for(ep_lli in (1:length(ep_list))){
        if(length(ep_list[[ep_lli]]) > 0){
          index = paste(paste("<",ep_lli,sep=""),">",sep="")
          if(episode == ""){
            episode = paste(index,ep_list[[ep_lli]],sep=" ")
          } else {
            episode = paste(episode,paste(index,ep_list[[ep_lli]],sep=" "),sep=" -1 ")
          }
        }
      }
      write(paste(episode,"-1 -2"),file=output,append=TRUE)
    }
  }
  
  remove_rare_events <- function(ds,target_event_frequency_proportion_rare){
    if(!csv){
      print("~~~~~~~APPLYING PREPROCESSING REMOVE RARE EVENTS~~~~~~~")
    }
    a = table(ds$Event_id)
    target_event_frequency = a[names(a)==target_event][[1]]
    rare_events = as.integer(names(a[a < target_event_frequency*target_event_frequency_proportion_rare]))
    return(ds[!(ds$Event_id %in% rare_events),])
  }
  
  remove_frequent_events <- function(ds,max_event_frequency_proportion_frequent){
    if(!csv){
      print("~~~~~~~APPLYING PREPROCESSING REMOVE FREQUENT EVENTS~~~~~~~")
    }
    a = table(ds[!(ds$Event_id == target_event),]$Event_id)
    max_freq = sort(a,decreasing = TRUE)[[1]]
    frequent_events = as.integer(names(a[a > max_freq*max_event_frequency_proportion_frequent]))
    #print(frequent_events)
    return(ds[!(ds$Event_id %in% frequent_events),])
  }
  
  keep_only_first_occureness <- function(episodes_list){
    if(!csv){
      print("~~~~~~~APPLYING PREPROCESSING KEEP ONLY FIRST OCCURENESS~~~~~~~")
    }
    #for every episode in the episodes_list
    for(ep_index in (1:length(episodes_list))){
      ep = episodes_list[[ep_index]]
      #For every segment of each episode starting from the end up to the second segment. 
      #We need to keep only the 1st occurness of consequtive events, hence starting from the end is the easy way.
      if(nrow(ep) < 2){
        next
      }
      for(i in (nrow(ep):2)){
        #as we deal with binary vectors, to find the indeces that both vectors have "1" we sum them and check for "2"s in the result
        matches = which((ep[i,]+ep[i-1,]) %in% c(2))
        #replace the 1s with 0s in the matching positions of the segment that is closer to the end of the episode
        ep[i,][c(matches)] = 0
      }
      episodes_list[[ep_index]] = ep
    }
    return(episodes_list)
  }
  
  mil_text <- function(milw,F_thres,episodes_list,b_length){
    if(!csv){
      print("~~~~~~~APPLYING PREPROCESSING MULTI INSTANCE LEARNING~~~~~~~")
    }
    window_df = data.frame(matrix(ncol = b_length+1, nrow = 0))
    
    #for every episode in the episodes_list
    for(ep_index in (1:length(episodes_list))){
      ep = episodes_list[[ep_index]]
      if(nrow(ep) < 1){
        next
      }
      new_ep = data.frame(matrix(ncol = b_length+1, nrow = 0))
      i = 1
      while(i <= nrow(ep)){
        new_ep = rbind(new_ep,ep[i,])

        if(ep[i,][b_length+1] >= F_thres && nrow(window_df) < milw){
          window_df = rbind(window_df,ep[i,])
        }
        if((nrow(window_df) == milw || i == nrow(ep)) && nrow(window_df) > 0){
          mean = colMeans(window_df)
          mean[mean > 0] = 1
          mf <- data.frame(as.list(mean))
          mf[1] = ep[i,][1]
          mf[b_length+1] = ep[i,][b_length+1]
          colnames(mf) = colnames(new_ep)
          new_ep = rbind(new_ep,mf)
          if(nrow(window_df) > 1){
            i = i - (nrow(window_df)-2)
          }
          window_df = data.frame(matrix(ncol = b_length+1, nrow = 0))
        }
        i = i + 1
      }
      episodes_list[[ep_index]] = new_ep
    }
    return(episodes_list)
  }
  
  mil_image <- function(milw,F_thres,episodes_list,b_length){
    if(!csv){
      print("~~~~~~~APPLYING PREPROCESSING MULTI INSTANCE LEARNING~~~~~~~")
    }
    
    #for every episode in the episodes_list
    for(ep_index in (1:length(episodes_list))){
      ep = episodes_list[[ep_index]]
      if(nrow(ep) < 1){
        next
      }
      new_ep = data.frame(matrix(ncol = b_length+1, nrow = 0))
      #a data.frame with the vectors that need to be averaged
      window_df = data.frame(matrix(ncol = b_length+1, nrow = 0))
      i = 1
      while(i <= nrow(ep)){
        #new_ep = rbind(new_ep,ep[i,])
        if(nrow(window_df) < milw){
          window_df = rbind(window_df,ep[i,])
        }
        if((nrow(window_df) == milw || i == nrow(ep)) && nrow(window_df) > 0){
          mean = colMeans(window_df)
          mean[mean > 0] = 1
          mf = data.frame(as.list(mean))
          mf[1] = ep[i,][1]
          mf[b_length+1] = ep[i,][b_length+1]
          #colnames(mf) = colnames(new_ep)
          new_ep = rbind(new_ep,mf)
          if(window_df[1,][b_length+1] >= F_thres && nrow(window_df) > 1){
            i = i - (nrow(window_df)-1)
          }
          window_df = data.frame(matrix(ncol = b_length+1, nrow = 0))
        }
        i = i + 1
      }
      episodes_list[[ep_index]] = new_ep
    }
    return(episodes_list)
  }
  
  #the Risk function
  compute_F <- function(s,midpoint,t,ep_length){
    #s affects the steepness
    # s <- 0.9
    return(1/(1+exp(s*(ep_length-midpoint-t))))
  }
  
  #convert event vectors to binary vectors
  compute_frequency_vectors <- function(aggr_episode_df,b_length,s,midpoint){
    freq_aggr_episode_df <- data.frame(matrix(ncol = b_length+2, nrow = 0))
    x <- c(c("Timestamps"), c(paste("e_",c(1:b_length),sep = "")), c("Risk_F"))
    # colnames(bin_aggr_episode_df) <- x

    for(i in 1:nrow(aggr_episode_df)) {

      #init a vector with target_event 0s

      freq_vector = as.vector(integer(b_length))
      seg <- aggr_episode_df[i,]
      #if segment contains the j number, replace the 0 in the bin_vector with 1
      for(value in seg$x[[1]]){

        if(is.na(value)){
          next
        }

        freq_vector[[value]] = length(which(seg$x[[1]] == value))
      }
      #add a new line to the bin_aggr_epissode_df
      #we use a matrix holding the elements of the new_data.frame as matrix is able to store variable of different data types
      F = compute_F(s,midpoint,i-1,nrow(aggr_episode_df))

      if(midpoint >= nrow(aggr_episode_df) || sum(freq_vector) == 0 ){

        F = 0
      }
      date = seg$Timeframe[[1]]
      new_df = data.frame(matrix(c(date, freq_vector,F),nrow=1,ncol=b_length+2))
      freq_aggr_episode_df <- rbind(freq_aggr_episode_df,new_df)
    }
    # x <- c(c("Timestamps"), c(paste("e_",c(1:3405))), c("Risk_F"))
    colnames(freq_aggr_episode_df) <- x
    return(freq_aggr_episode_df)
  }
  
  create_episodes_list <- function(ds,target_event,b_length,s,midpoint,test_mode){
    if(!csv){
      print("~~~~~~~CREATING FREQUENCY VECTORS AND BINARIZE THEM~~~~~~~")
    }
    #devide in episodes
    target_event_spotted = FALSE
    #a list with data.frames for the episodes (each episode one data.frame)
    episodes_list = list()
    #data.frame for episodes
    episode_df <- data.frame(Timestamps=as.POSIXct(character()),Event_id=integer())
    #iterate over every line of the original dataset
    for(i in 1:nrow(ds)) {
      #get the current row of the ds
      meas <- ds[i,]
      #If it is the target event enable the appropriate flag
      if((meas$Event_id == target_event)){
        target_event_spotted = TRUE
      }
      #fill the episode data.frame with the events that are between two target events
      if(meas$Event_id != target_event && target_event_spotted){
        episode_df <- rbind(episode_df,data.frame(Timestamps=meas$Timestamps, Event_id=meas$Event_id))
      } else if(meas$Event_id == target_event && target_event_spotted && is.data.frame(episode_df) && nrow(episode_df) != 0){

        episode_df <- rbind(episode_df,data.frame(Timestamps=meas$Timestamps, Event_id=NA))
        

        #a second occurness of the target event is spotted, close the episode
        #target_event_spotted = FALSE
        #aggregate by day all the events to form the segments inside the episodes

        labels_seq = seq(floor_date(as.POSIXct(head(episode_df,1)$Timestamps, format="%Y-%m-%dT%H:%M:%OSZ"),'hour'), floor_date(as.POSIXct(tail(episode_df,1)$Timestamps, format="%Y-%m-%dT%H:%M:%OSZ"),'hour'), by="1 hour")

        aggr_episode_df = aggregate(episode_df[ ,2], FUN=function(x){return(x)}, by=list(Timeframe=cut(as.POSIXct(episode_df$Timestamps, format="%Y-%m-%dT%H:%M:%OSZ"),"hour"))) #%Y-%m-%dT%H:%M:%OSZ

        
        aggr_episode_df[, 1]  <- as.POSIXct(aggr_episode_df[, 1])
        full = data.frame(Timeframe=labels_seq)
        full[, 1]  <- as.POSIXct(full[, 1])
        aggr_episode_df = merge(full,aggr_episode_df,by='Timeframe',all.x=TRUE)
        
        #aggr_episode_df = head(aggr_episode_df, -1) 
        
        

        #binarize the frequncy vector
        bin_aggr_episode_df = compute_frequency_vectors(aggr_episode_df,b_length,s,midpoint)
        
        #Remove event 0, which does not provide any info KOUGKA
        bin_aggr_episode_df = bin_aggr_episode_df[ , !(names(bin_aggr_episode_df) %in% c("e_1"))]

        #add the episode to the episodes_list
        episodes_list[[length(episodes_list)+1]] = bin_aggr_episode_df
        #reset episode_df to en empty data.frame
        episode_df <- data.frame(Timestamps=as.POSIXct(character()),Event_id=integer())

      } else if(meas$Event_id == target_event && target_event_spotted && is.data.frame(episode_df) && nrow(episode_df) == 0 && length(episodes_list) > 0 && test_mode){

        freq_vector = as.vector(integer(b_length))
        new_df = data.frame(matrix(c(0, freq_vector,0),nrow=1,ncol=b_length+2))
        episode_df <- rbind(episode_df,new_df)
        
        x <- c(c("Timestamps"), c(paste("e_",c(1:b_length),sep = "")), c("Risk_F"))
        colnames(episode_df) <- x
        episode_df = episode_df[ , !(names(episode_df) %in% c("e_1"))]

        #add the episode to the episodes_list
        episodes_list[[length(episodes_list)+1]] = episode_df
        #reset episode_df to en empty data.frame
        episode_df <- data.frame(Timestamps=as.POSIXct(character()),Event_id=integer())
      }
    }
    return(episodes_list)
  }
  
  preprocess <- function(ds,TEST_DATA,REMOVE_RARE_EVENTS,REMOVE_FREQUENT_EVENTS,KEEP_ONLY_FIRST_OCCURENESS,MULTI_INSTANCE_LEARNING_TEXT,MULTI_INSTANCE_LEARNING_IMAGE,FEATURE_SELECTION,top_features,s,midpoint,b_length,target_event,target_event_frequency_proportion_rare,max_event_frequency_proportion_frequent,w,F_thres,test_mode){
    
    #Remove events that appear < n times. We consider n = (target event frequency)/2
    if(REMOVE_RARE_EVENTS){
      ds<-remove_rare_events(ds,target_event_frequency_proportion_rare)
    }
    
    #Remove events that appear < n times. We consider n = (target event frequency)/2
    if(REMOVE_FREQUENT_EVENTS){
      ds<-remove_frequent_events(ds,max_event_frequency_proportion_frequent)
    }
    
    episodes_list = create_episodes_list(ds,target_event,b_length,s,midpoint,test_mode)
    #if(length(episodes_list) == 0){
    # return()
    #}
    
    #binarize the vector
    for(ep_index in (1:length(episodes_list))){
      ep = episodes_list[[ep_index]]
      ep[2:(ncol(ep)-1)][ep[2:(ncol(ep)-1)] > 0] = 1
      episodes_list[[ep_index]] = ep
    }
    
    # keep only the first occurness of event in consecutive segments
    if(KEEP_ONLY_FIRST_OCCURENESS){
      episodes_list <- keep_only_first_occureness(episodes_list)
    }
    
    # Multi-instance learning to increase the pattern frequency
    if(MULTI_INSTANCE_LEARNING_TEXT){
      episodes_list <- mil_text(w,F_thres,episodes_list,b_length)
    } else if(MULTI_INSTANCE_LEARNING_IMAGE){
      episodes_list <- mil_image(w,F_thres,episodes_list,b_length)
    }
    return(episodes_list)
  }
  
  feature_selection <- function(merged_episodes,top_features){

    if(!csv){
      print("~~~~~~~APPLYING FEATURE SELECTION~~~~~~~")
    }

    estReliefF <- attrEval(Risk_F ~ ., merged_episodes, estimator="RReliefFexpRank", ReliefIterations=50)
    sorted_indeces = order(estReliefF, decreasing = TRUE)

    merged_episodes = merged_episodes %>% select(sorted_indeces[1:min(top_features,length(sorted_indeces))],ncol(merged_episodes))

    return(merged_episodes)
  }
  
  read_dataset <- function(path){
    dataset = read.table(path, header = TRUE, sep = ",", dec = ".", comment.char = "#")
    dataset[, 2]  <- as.numeric(dataset[, 2])
    return(dataset)
  }
  
  read_dataset_cross <- function(path){
    dataset = read.table(path, header = TRUE, sep = ",", dec = ".", comment.char = "#")
    dataset[, 2]  <- as.numeric(dataset[, 2])
    
    aggr_episode_df = aggregate(dataset, FUN=function(x){return(x)}, by=list(cut(as.POSIXct(dataset$Timestamps, format="%Y-%m-%dT%H:%M:%OSZ"),"day"))) #%Y-%m-%dT%H:%M:%OSZ
    
    training = sample_n(aggr_episode_df,(2*nrow(aggr_episode_df)/3))
    
    testing = setDT(aggr_episode_df)[!training, on="Group.1"]
    
    testing = testing[, !"Group.1"]
    testing_df = data.frame(Timestamps=unlist(testing[,1][1][[1]]),Event_id=unlist(testing[,2][1][[1]]))
    for(i in 2:nrow(testing)){
      testing_df = rbind(testing_df,data.frame(Timestamps=unlist(testing[,1][i][[1]]),Event_id=unlist(testing[,2][i][[1]])))
    }
    
    training = as.data.table(training)[, !"Group.1"]
    training_df = data.frame(Timestamps=unlist(training[,1][1][[1]]),Event_id=unlist(training[,2][1][[1]]))
    for(i in 2:nrow(training)){
      training_df = rbind(training_df,data.frame(Timestamps=unlist(training[,1][i][[1]]),Event_id=unlist(training[,2][i][[1]])))
    }
    
    return(list("training" = training_df, "testing"=testing_df))
  }
  
  read_dataset_cross2 <- function(path,form){
    dataset = read.table(path, header = TRUE, sep = ",", dec = ".", comment.char = "#")
    dataset[, 2]  <- as.numeric(dataset[, 2])
    if(form == 1){
      training_df = dataset[1:(2*nrow(dataset)/3),]
      testing_df = dataset[(2*nrow(dataset)/3):nrow(dataset),]
    } else if (form == 2){
      training_df = dataset[(nrow(dataset)/3):nrow(dataset),]
      testing_df = dataset[1:(nrow(dataset)/3),]
    } else {
      training_df = rbind(dataset[1:(nrow(dataset)/3),],dataset[(2*nrow(dataset)/3):nrow(dataset),])
      testing_df = dataset[(nrow(dataset)/3+1):(2*nrow(dataset)/3),]
    }
    return(list("training" = training_df, "testing"=testing_df))
  }
  
  find_next_fake_episodes <- function(ep_index,test_episodes_list){
    fake_episodes_vec = vector()
    cnt = 0
    if(ep_index < length(test_episodes_list)){
      for(i in ((ep_index+1):length(test_episodes_list))){
        n_ep = test_episodes_list[i]
        if(nrow(n_ep[[1]]) == 1 && n_ep[[1]]$Timestamps == 0){
          cnt = cnt + 1
          fake_episodes_vec <- c(fake_episodes_vec, i)
        } else {
          return(fake_episodes_vec)
        }
      }
    }
    return(fake_episodes_vec)
  }
  
  find_next_close_episodes <- function(ep_index,test_episodes_list,pred_index){
    ep_length = nrow(test_episodes_list[ep_index][[1]])
    remaining_hours = max_warning_interval-(ep_length-pred_index)

    nc_episodes_vec = vector()
    cnt = 0
    while (remaining_hours > 0 && ep_index < length(test_episodes_list)) {
      ep_index = ep_index + 1
      n_ep = test_episodes_list[ep_index]
      if(nrow(n_ep[[1]]) == 1 && n_ep[[1]]$Timestamps == 0){
        next
      }
      n_ep_length = nrow(n_ep[[1]])
      if(n_ep_length <= remaining_hours){
        remaining_hours = remaining_hours-n_ep_length
        if(remaining_hours >= 0){
          cnt = cnt + 1
          nc_episodes_vec <- c(nc_episodes_vec, ep_index)
        }
      } else{
        return(nc_episodes_vec)
      }
    }
    return(nc_episodes_vec)
  }

  eval <- function(train_episodes,test_episodes_list,seed,r){

    set.seed(seed)
    my.rf = randomForest(Risk_F ~ .,data=train_episodes,importance=TRUE)
    #varImpPlot(my.rf)
    false_positives = 0
    tp_vec = vector()
    fp_vec = vector()
    fn_vec = vector()
    true_positives = 0
    false_negatives = 0
    ep_index = 1
    while (ep_index <= length(test_episodes_list)) {
      ep = test_episodes_list[[ep_index]]
      fake_episodes_vec = find_next_fake_episodes(ep_index,test_episodes_list)
      fake_ep_cnt = length(fake_episodes_vec)
      
      ep = ep[ , !(names(ep) %in% c("Timestamps"))]
      Prediction <- predict(my.rf, ep)
      ep_legth = length(Prediction)
      pred_indeces = as.numeric(names(Prediction[Prediction >= acceptance_threshold]))
      predicted_next_episodes = 0
      if(length(pred_indeces) > 0){
        pred_index = tail(pred_indeces, n=1)
        nc_episodes_vec = find_next_close_episodes(ep_index,test_episodes_list,pred_index)

        predicted_next_episodes = length(nc_episodes_vec)
      }
      
      
      if(length(pred_indeces[pred_indeces < (ep_legth-(max_warning_interval))]) > 0){
        fp_reps = length(pred_indeces[pred_indeces < (ep_legth-(max_warning_interval))])
        false_positives = false_positives + fp_reps + (fake_ep_cnt*fp_reps)
        fp_vec = c(fp_vec,rep(ep_index,fp_reps),rep(fake_episodes_vec,fp_reps))
      } 
      if(length(pred_indeces[pred_indeces >= (ep_legth-(max_warning_interval)) & pred_indeces <= (ep_legth-min_warning_interval)]) > 0){
        true_positives = true_positives + 1 + fake_ep_cnt + predicted_next_episodes
        tp_vec = c(tp_vec,ep_index,fake_episodes_vec,nc_episodes_vec)
      } else {

        if(predicted_next_episodes > 0){
          true_positives = true_positives + predicted_next_episodes
          tp_vec = c(tp_vec,nc_episodes_vec)
        }

        false_negatives = false_negatives + 1 + fake_ep_cnt
        fn_vec = c(fn_vec,ep_index,fake_episodes_vec)
      }
      ep_index = ep_index + 1 + fake_ep_cnt + predicted_next_episodes
    }
    
    precision = true_positives/(true_positives+false_positives)
    if((true_positives+false_positives) == 0){
      precision = 0
    }
    recall = true_positives/length(test_episodes_list)
    
    F1 = 2*((precision*recall)/(precision+recall))
    if((precision+recall) == 0){
      F1 = 0
    }
    if(!csv){
      cat(paste("dataset:",argv$test,"
  true_positives:", true_positives,"
  false_positives:", false_positives,"
  false_negatives:", false_negatives,"
  precision:", precision,"
  recall:", recall,"
  F1:", F1, "
  tp:",paste(as.character(tp_vec), sep="", collapse=","), "
  fp:",paste(as.character(fp_vec), sep="", collapse=","), "
  fn:",paste(as.character(fn_vec), sep="", collapse=",")))
    } else {

      cat(paste(argv$test,",",true_positives,",",false_positives,",",false_negatives,",",precision,",",recall,",",F1,",",argv$fet,",",argv$tet,",",argv$rre,",",argv$rfe,",",argv$kofe,",",argv$mili,",",argv$milt,",",argv$fs,",",argv$top,",",argv$rer,",",argv$fer,",",argv$seed,",",argv$steepness,",",argv$pthres,",",argv$milw,",",argv$milthres,",",argv$midpoint,",",argv$minwint,",",argv$maxwint,",",r,",tp,",paste(as.character(tp_vec), sep="", collapse=","),",fp,",paste(as.character(fp_vec), sep="", collapse=","),",fn,",paste(as.character(fn_vec), sep="", collapse=","),"
  ",sep=""))

    }
    return(my.rf)
  }
  
  plot <- function(test_episodes_list, episode_index, my.rf){
    test_episodes = test_episodes_list[[episode_index]][ , !(names(test_episodes_list[[episode_index]]) %in% c("Timestamps"))]
    Prediction <- predict(my.rf, test_episodes)
    results = data.frame(Risk_F=test_episodes$Risk_F,num_Prediction=as.numeric(Prediction))
    mse = mean((Prediction-test_episodes$Risk_F)^2)
    
    chart =ggplot(results,aes((1:nrow(results)))) +
      # geom_rect(aes(xmin = ceiling(nrow(df_test)/2), xmax = nrow(df_test), ymin = -Inf, ymax = Inf),
      #           fill = "yellow", alpha = 0.003) +
      geom_line(aes(y = Risk_F, colour = "Actual")) +
      geom_line(aes(y = num_Prediction, colour="Predicted")) +
      labs(colour="Lines") +
      xlab("Segments") +
      ylab('Risk (F)') +
      ggtitle("Risk Prediction") + # (RR_KF_2YEARS_PAT08)
      theme(plot.title = element_text(hjust = 0.5)) +
      geom_text(aes(label = paste("MSE=",round(mse,3)), x = 20, y = 1), hjust = -2, vjust = 6, color="black", size=4) #add MSE label
    
    # Disable clip-area so that the MSE is shown in the plot
    gt <- ggplot_gtable(ggplot_build(chart))
    gt$layout$clip[gt$layout$name == "panel"] <- "off"
    grid.draw(gt)
  }
  
  
  p <- arg_parser("Implementation of the AIRBUS Predictor")
  # Add a positional argument
  p <- add_argument(p, "id", help="experiment ID")
  p <- add_argument(p, "train", help="training dataset")
  p <- add_argument(p, "test", help="test dataset")
  p <- add_argument(p, "fet", help="different types of the fault events",default=151)
  p <- add_argument(p, "tet", help="type of the target fault events",default=151)
  p <- add_argument(p, "--rre", help="remove rare events", default=FALSE)
  p <- add_argument(p, "--rfe", help="remove frequent events", default=FALSE)
  p <- add_argument(p, "--kofe", help="keep only first event", default=FALSE)

  p <- add_argument(p, "--milt", help="MIL as written in the text of the paper", default=TRUE)

  p <- add_argument(p, "--mili", help="MIL as shonw in the Figure of the paper", default=FALSE)

  p <- add_argument(p, "--milthres", help="MIL threshold to the sigmoid function for over-sampling", default=0.3)

  p <- add_argument(p, "--steepness", help="steepness of the sigmoid function", default=0.8)
  p <- add_argument(p, "--midpoint", help="midpoint of the sigmoid function (in days)", default=4)

  p <- add_argument(p, "--fs", help="apply feature selection", default=TRUE)
  p <- add_argument(p, "--top", help="# of features to keep in feature selection", default=50)

  p <- add_argument(p, "--rer", help="rare events ratio of the target event frequency", default=0.2)
  p <- add_argument(p, "--fer", help="frequent events ratio of the frequency of the most frequent event", default=0.8)

  p <- add_argument(p, "--milw", help="MIL window size (in days)", default=2)
  p <- add_argument(p, "--pthres", help="prediction threshold to the Risk value for a true positive episode", default=0.35)

  p <- add_argument(p, "--seed", help="seed for RF", default=500)
  p <- add_argument(p, "--csv", help="output for csv", default=FALSE)
  
  
  p <- add_argument(p, "--spme", help="export datasets for sequential pattern minning", default=FALSE)
  p <- add_argument(p, "--java", help="the java path", default="/usr/bin/java")
  p <- add_argument(p, "--python", help="the java path", default="/usr/bin/python")
  p <- add_argument(p, "--cep", help="the java path", default="/media/thanasis/Storage/ATLANTIS/0_Ensembled_Predictive_Solution_EPS/spm_rules.py")
  p <- add_argument(p, "--spmf", help="the spmf path", default="/media/thanasis/Storage/ATLANTIS/0_Ensembled_Predictive_Solution_EPS/spmf.jar")
  p <- add_argument(p, "--conf", help="minimum support (minsup)", default="20%")
  p <- add_argument(p, "--minti", help="minimum time interval allowed between two succesive itemsets of a sequential pattern", default=1)
  p <- add_argument(p, "--maxti", help="maximum time interval allowed between two succesive itemsets of a sequential pattern", default=5)
  p <- add_argument(p, "--minwi", help="minimum time interval allowed between the first itemset and the last itemset of a sequential pattern", default=1)
  p <- add_argument(p, "--maxwi", help="maximum time interval allowed between the first itemset and the last itemset of a sequential pattern", default=11)
  p <- add_argument(p, "--minwint", help="min # of days before failure to expect a warning for true positive decision", default=1)
  p <- add_argument(p, "--maxwint", help="max # of days before failure to expect a warning for true positive decision", default=8)

  p <- add_argument(p, "--cross", help="cross validation", default=5)

  p <- add_argument(p, "--form", help="form", default=1)
  
  
  
  argv = data.frame()
  if( length(commandArgs(trailingOnly = TRUE)) != 0){
    argv <- parse_args(p)
  } else {

    argv <- parse_args(p,c(1,"/home/thanasis/Desktop/Atlantis/zBreak/Philips_AE_prediction/full_stops/all_channels/training/events1P5_LouvainPhilips_ch2_6396te_training.csv","/home/thanasis/Desktop/Atlantis/zBreak/Philips_AE_prediction/full_stops/all_channels/training/events1P5_LouvainPhilips_ch2_6396te_training.csv",6396,6396))

  }
  
  
  TEST_DATA = FALSE
  id = argv$id
  REMOVE_RARE_EVENTS = argv$rre
  REMOVE_FREQUENT_EVENTS = argv$rfe
  KEEP_ONLY_FIRST_OCCURENESS = argv$kofe
  MULTI_INSTANCE_LEARNING_TEXT = argv$milt #MIL as explained in the text
  MULTI_INSTANCE_LEARNING_IMAGE = argv$mili #MIL as presented in the figure
  FEATURE_SELECTION = argv$fs
  top_features = argv$top
  target_event_frequency_proportion_rare = argv$rer
  max_event_frequency_proportion_frequent = argv$fer
  milw = argv$milw
  F_thres = argv$milthres
  s = argv$steepness
  midpoint = argv$midpoint
  target_event = argv$tet
  b_length = argv$fet
  acceptance_threshold = argv$pthres
  export_spm = argv$spme
  seed = argv$seed
  csv = argv$csv
  max_warning_interval = argv$maxwint
  min_warning_interval = argv$minwint
  
  CROSS = argv$cross

  training_set = read_dataset(argv$train)
  test_set =  read_dataset(argv$test)

  
  episodes_list <- preprocess(training_set,TEST_DATA,REMOVE_RARE_EVENTS,REMOVE_FREQUENT_EVENTS,KEEP_ONLY_FIRST_OCCURENESS,MULTI_INSTANCE_LEARNING_TEXT,MULTI_INSTANCE_LEARNING_IMAGE,FEATURE_SELECTION,top_features,s,midpoint,b_length,target_event,target_event_frequency_proportion_rare,max_event_frequency_proportion_frequent,milw,F_thres,FALSE)

  TEST_DATA = TRUE
  REMOVE_RARE_EVENTS = FALSE
  REMOVE_FREQUENT_EVENTS = FALSE
  KEEP_ONLY_FIRST_OCCURENESS = FALSE
  MULTI_INSTANCE_LEARNING_TEXT = FALSE #MIL as explained in the text
  MULTI_INSTANCE_LEARNING_IMAGE = FALSE #MIL as presented in the figure
  FEATURE_SELECTION = FALSE

  test_episodes_list <- preprocess(test_set,TEST_DATA,REMOVE_RARE_EVENTS,REMOVE_FREQUENT_EVENTS,KEEP_ONLY_FIRST_OCCURENESS,MULTI_INSTANCE_LEARNING_TEXT,MULTI_INSTANCE_LEARNING_IMAGE,FEATURE_SELECTION,top_features,s,midpoint,b_length,target_event,target_event_frequency_proportion_rare,max_event_frequency_proportion_frequent,milw,F_thres,FALSE)
  
  
  #episodes_list = episodes_list[-which(lapply(episodes_list, nrow)<midpoint)]
  #test_episodes_list = test_episodes_list[-which(lapply(test_episodes_list, nrow)<midpoint)]
  
  FEATURE_SELECTION = argv$fs
  if(!csv){
    print("~~~~~~~CROSS~~~~~~~")
  }
  flds = list()
  k = argv$cross
  ep_length = length(episodes_list)
  for (r in (1:k)) {
    for (p in (1:2)){
      cross_name = r
      if(p == 1){
        flds = list(1:round(r*10*ep_length/100))
      } else {
        cross_name = r+k
        flds = list(ceiling(((100-(r*10))*ep_length)/100):ep_length)
      }
      train = episodes_list[-flds[[1]]]
      test = test_episodes_list[flds[[1]]]
      
      #merge episodes
      merged_episodes = ldply(train, data.frame)
      merged_episodes = merged_episodes[ , !(names(merged_episodes) %in% c("Timestamps"))]
      #remove columns with all values equal to zero
      merged_episodes = merged_episodes[, colSums(merged_episodes != 0) > 0]
      
      if(FEATURE_SELECTION){
        merged_episodes = feature_selection(merged_episodes,top_features)
      }
      
      my.rf = eval(merged_episodes,test,seed,cross_name)
    }

  }

  # flds = list()
  # k = argv$cross
  # for (r in (1:k)) {
  #   ep_length = length(episodes_list)
  #   if (r == 1) {
  #     flds[r] = list(1:(r*ep_length/k))
  #   } else {
  #     flds[r] = list(((r-1)*ep_length/k):(r*ep_length/k))
  #   }
  #   
  #   train = episodes_list[-flds[[r]]]
  #   test = test_episodes_list[flds[[r]]]
  #   
  #   #merge episodes
  #   merged_episodes = ldply(train, data.frame)
  #   merged_episodes = merged_episodes[ , !(names(merged_episodes) %in% c("Timestamps"))]
  #   #remove columns with all values equal to zero
  #   merged_episodes = merged_episodes[, colSums(merged_episodes != 0) > 0]
  #   
  #   if(FEATURE_SELECTION){
  #     merged_episodes = feature_selection(merged_episodes,top_features)
  #   }
  #   
  #   my.rf = eval(merged_episodes,test,seed,r)
  # }

  
  # for(s in (0:6)){
  #   my.rf = eval(merged_episodes,test_episodes_list,seed)
  #   seed = seed + 1
  # }
  
  # for(ep in 1:length(test_episodes_list)){
  #   jpeg(paste(ep,'_rplot.jpg'))
  #   plot(test_episodes_list,ep,my.rf)
  #   dev.off()
  # }
  
  if(export_spm){
    if(!csv){
      print("~~~~~~~SEQUENTIAL PATTERN MINING~~~~~~~")
    }
    spm_train_path = gsub(".csv",paste("_spm_",id,".csv",sep=""),argv$train)
    spm_test_path = gsub(".csv",paste("_spm_",id,".csv",sep=""),argv$test)
    spm_results_path = gsub(".csv",paste("_results_",id,".csv",sep=""),argv$train)
    confidence = argv$conf
    min_dist_seq = argv$minti
    max_dist_seq = argv$maxti
    min_dist_first_last = argv$minwi
    max_dist_first_last = argv$maxwi
    java_path = argv$java
    jspmf_path = argv$spmf
    python_path = argv$python
    cep_path = argv$cep
    max_warning_interval = argv$maxwint
    min_warning_interval = argv$minwint 
    export_ds_for_spm(target_event,episodes_list,spm_train_path)
    export_ds_for_spm(target_event,test_episodes_list,spm_test_path)
    
    if (file.exists(spm_results_path)) {
      invisible(file.remove(spm_results_path))
    }
    
    javaOutput <- system(paste(java_path,"-jar",jspmf_path,"run HirateYamana",spm_train_path,spm_results_path,confidence,min_dist_seq,max_dist_seq,min_dist_first_last,max_dist_first_last), intern = TRUE)
    #print(javaOutput)
    
    pythonOutput <- system(paste(python_path,cep_path,spm_results_path,spm_test_path,target_event), intern = TRUE)
    #print(pythonOutput)
    true_positives = 0
    false_positives = 0
    false_negatives = 0
    total_failures = 0
    d = 0
    
    warnings = list()
    for(w in pythonOutput){
      d = as.integer(str_extract(w, "\\-*\\d+\\.*\\d*"))
      if(!grepl("Failure",w,fixed=TRUE)){
        warnings = c(warnings,d)
      } else {
        total_failures = total_failures + 1
        if(length(warnings) == 0){
          false_negatives = false_negatives + 1
        } else {
          if(length(warnings[warnings < d-max_warning_interval]) > 0){
            false_positives = false_positives + length(warnings[warnings < d-max_warning_interval])
          }
          if(length(warnings[warnings >= (d-max_warning_interval)]) > 0 & length(warnings[warnings <= (d-min_warning_interval)]) > 0){
            true_positives = true_positives + 1
          } else {
            false_negatives = false_negatives + 1
          }
        }
        warnings = list()
      }
    }
    
    precision = true_positives/(true_positives+false_positives)
    if((true_positives+false_positives) == 0){
      precision = 0
    }
    
    recall = true_positives/total_failures
    
    F1 = 2*((precision*recall)/(precision+recall))
    if((precision+recall) == 0){
      F1 = 0
    }
    
    if(!csv){
      cat(paste("dataset:",argv$test,"
  true_positives:", true_positives,"
  false_positives:", false_positives,"
  false_negatives:", false_negatives,"
  precision:", precision,"
  recall:", recall,"
  F1:", F1, "
  "))
    } else {
      cat(paste(argv$test,",", true_positives,",", false_positives,",", false_negatives,",", precision,",", recall,",", F1,",",argv$conf,",",argv$minti,",",argv$maxti,",",argv$minwi,",",argv$maxwi,",",argv$minwint,",",argv$maxwint, "
  ",sep=""))
    }
  }