Read from start option + extract date from pattern to put in influxdb

src/main/java/com/airfrance/diqmqs/logparser/Cli.java

@@ -22,6 +22,7 @@ public class Cli {
 	public static Option patternFilePath 	= new Option( "pattern", true , "Input pattern path file" );
 	public static Option logFile 	 		= new Option( "logfile", true , "Input log path files" );
 	public static Option regex 	 			= new Option( "regex", true , "Name of the regex to apply" );
+	public static Option fromStart 	 		= new Option( "fromStart", "Read the file from start" );
 	public static Option fileParam 			= new Option( "paramfile", true , "Input a param file" );
 	public static Option debugOption 	 	= new Option( "debug", "Active debug output message" );
 	public static Option infoOption 	 	= new Option( "info", "Active info output message" );
@@ -47,8 +48,10 @@ public class Cli {
 		options.addOption(debugOption);
 		options.addOption(infoOption);
 		options.addOption(fileParam);
+		options.addOption(fromStart);
 		
 		HelpFormatter formatter = new HelpFormatter();
+		boolean readFromStart = false;
 		CommandLineParser parser = new DefaultParser();
 		CommandLine cmd = null;
 		try {
@@ -141,13 +144,17 @@ public class Cli {
 				System.err.println("regex application missing");
 				System.exit(0);
 			}
+			
+			if(cmd.hasOption("fromStart")) {
+				readFromStart = true;
+			}
 
 			if(cmd.hasOption("logfile")) {
 				ParserEngine engine = new ParserEngine(patternPath);
 				String[] filesName = cmd.getOptionValues("logfile");
 				for (int i = 0; i < filesName.length ; i++) {
 					File f = new File (filesName[i]);
-					engine.addNewParser(f, regexName[i], applicationName);
+					engine.addNewParser(f, regexName[i], applicationName, readFromStart);
 				}
 			}
 		}
@@ -158,16 +165,20 @@ public class Cli {
 	static void startParserFromFile(File file, ParserEngine engine) throws FileNotFoundException {
 		
 		Scanner scanner = new Scanner(file);
+		
 		while (scanner.hasNextLine()) {
 			 String line = scanner.nextLine();
+			 boolean readFromStart = false;
 			 if (!line.startsWith("#"))
 			 {
 				String[] arguments =  line.split("\\t");
 				String applicationName = arguments[0].toLowerCase();
 				Paths paths = new Paths("/", arguments[1]);
 				String regexName = arguments[2];
+				if (arguments[3] != null)
+					readFromStart = true;
 				for (File f : paths.getFiles()) {
-					engine.addNewParser(f, regexName, applicationName);
+					engine.addNewParser(f, regexName, applicationName, readFromStart);
 			 	}
 		
 			 }

src/main/java/com/airfrance/diqmqs/logparser/Log.java

@@ -6,11 +6,12 @@ import java.util.Date;
 public class Log {
 	
 	public static final int NONE = -1; 
-	public static final int INFO = 0;
+	public static final int ERROR = 0;
-	public static final int DEBUG = 1; 
+	public static final int INFO = 1;
+	public static final int DEBUG = 2; 
 	
 	private String className = this.getClass().getName();
-	private static int level = -1;
+	private static int level = 1;
 	SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); 
 	Date now;
 	
@@ -28,6 +29,13 @@ public class Log {
 		Log.level = level;
 	}
 	
+	public void error(String message)
+	{
+		if ( Log.level >= ERROR ) {
+			writeLogError("[ERROR]:" + message);
+		}	
+	}
+	
 	public void info(String message)
 	{
 		if ( Log.level >= INFO ) {
@@ -48,5 +56,12 @@ public class Log {
 		System.out.println(sdfDate.format(now) + " ["  + className + "] " + message );
 	}
 	
+	private void writeLogError(String message)
+	{
+		now = new Date();
+		System.out.println(sdfDate.format(now) + " ["  + className + "] " + message );
+		System.err.println(sdfDate.format(now) + " ["  + className + "] " + message );
+	}
+	
 	
 }

src/main/java/com/airfrance/diqmqs/logparser/Parser.java

@@ -1,7 +1,9 @@
 package com.airfrance.diqmqs.logparser;
 
+import java.util.Calendar;
 import java.util.Map;
 import java.util.Random;
+import java.util.TimeZone;
 
 import org.apache.commons.io.input.Tailer;
 import org.apache.commons.io.input.TailerListenerAdapter;
@@ -53,6 +55,7 @@ public class Parser extends TailerListenerAdapter {
 	 public void handle(String line) {
 		 
 		Match gm = grok.match(line);
+		TimeZone UTC = TimeZone.getTimeZone("UTC");
 		gm.captures();
 		Map<String,Object> result = gm.toMap();
 		if ( result.size() > 0 )
@@ -64,6 +67,8 @@ public class Parser extends TailerListenerAdapter {
 			value = null;
 
 			// iterate on all match pattern 
+			boolean hasDate = false;
+			Calendar cal = Calendar.getInstance(UTC);
 			for (Map.Entry<String,Object> entry : result.entrySet() )
 			{
 				tagName = entry.getKey();
@@ -86,6 +91,43 @@ public class Parser extends TailerListenerAdapter {
 						}
 					}
 				}
+
+				try {
+					if ( tagName.equalsIgnoreCase("year") ) {
+						hasDate = true;
+						cal.set(Calendar.YEAR,  Integer.parseInt(String.valueOf(entry.getValue()) ) );
+					}
+					if ( tagName.equalsIgnoreCase("mounth") ) {
+						hasDate = true;
+						cal.set(Calendar.MONTH,  Integer.parseInt(String.valueOf(entry.getValue()) ) );
+					}
+					if ( tagName.equalsIgnoreCase("day") ) {
+						hasDate = true;
+						cal.set(Calendar.DAY_OF_MONTH,  Integer.parseInt(String.valueOf(entry.getValue()) ) );
+					}
+					if ( tagName.equalsIgnoreCase("hour") ) {
+						hasDate = true;
+						cal.set(Calendar.HOUR_OF_DAY,  Integer.parseInt(String.valueOf(entry.getValue()) ) );
+					}
+					if ( tagName.equalsIgnoreCase("minute") ) {
+						hasDate = true;
+						cal.set(Calendar.MINUTE,  Integer.parseInt(String.valueOf(entry.getValue()) ) );
+					}
+					if ( tagName.equalsIgnoreCase("second") ) {
+						hasDate = true;
+						cal.set(Calendar.SECOND,  Integer.parseInt(String.valueOf(entry.getValue()) ) );
+					}
+					if ( tagName.equalsIgnoreCase("msecond") ) {
+						hasDate = true;
+						cal.set(Calendar.MILLISECOND,  Integer.parseInt(String.valueOf(entry.getValue()) ) );
+					}
+				}catch(Exception e)
+				{
+					log.error("Impossible de parser la date " + e.getMessage());
+				}
+				
+				
+				
 			}
 			sb.append(" ");
 			//Field
@@ -98,9 +140,15 @@ public class Parser extends TailerListenerAdapter {
 				sb.append("value=1");
 			}
 			// TimeStamp
-			sb.append(" ")
+			sb.append(" ");
-				.append(System.currentTimeMillis())
+			if 	(hasDate){
-				.append(String.format("%06d",randomGenerator.nextInt(999999)));
+				sb.append(cal.getTimeInMillis());
+			}
+			else {
+				sb.append(System.currentTimeMillis());
+			}
+			// Time is in nano second from epoch
+			sb.append(String.format("%06d",randomGenerator.nextInt(999999)));
 			
 			log.debug("Line => " + line + "  : Match add it to Queue ");
 			// Add to the queue

src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java

@@ -50,9 +50,9 @@ public class ParserEngine implements Runnable{
 		log.info("Schedule sender is set to " + 1 + " secs" );
 	}
 	
-	void addNewParser(File f , String regexName, String application ) {
+	void addNewParser(File f , String regexName, String application , boolean readFromStart) {
 		TailerListener listener = new Parser(application, regexName, this);
-		Tailer tailer = new Tailer(f, listener, delay, true, true, 8192);
+		Tailer tailer = new Tailer(f, listener, delay, !readFromStart, true, 8192);
 	    Thread thread = new Thread(tailer);
 	    thread.setDaemon(true); 
 	    thread.setName("Parser - " + threadsParser.size() );