Improve log and performance

.classpath

@@ -11,7 +11,7 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
@@ -21,5 +21,11 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
+	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

pom.xml

@@ -7,8 +7,8 @@
   
   <description>logParser - send data to influxdb</description>
    <properties>
-          <maven.compiler.source>1.7</maven.compiler.source>
+          <maven.compiler.source>1.6</maven.compiler.source>
-        <maven.compiler.target>1.7</maven.compiler.target>
+        <maven.compiler.target>1.6</maven.compiler.target>
    </properties>
   	<dependencies>
 		<dependency>
@@ -27,7 +27,6 @@
 			<version>2.4</version>
 		</dependency>
 	</dependencies>
-  
   <build>
 		<plugins>
 			<plugin>

src/main/java/com/airfrance/diqmqs/logparser/Cli.java

@@ -24,6 +24,7 @@ public class Cli {
 	public static Option regex 	 			= new Option( "regex", true , "Name of the regex to apply" );
 	public static Option fileParam 			= new Option( "paramfile", true , "Input a param file" );
 	public static Option debugOption 	 	= new Option( "debug", "Active debug output message" );
+	public static Option infoOption 	 	= new Option( "info", "Active info output message" );
 	private static  long lastModifiedTime	= 0L;
 	public Cli() {
 		// TODO Auto-generated constructor stub
@@ -44,6 +45,7 @@ public class Cli {
 		options.addOption(logFile);
 		options.addOption(regex);
 		options.addOption(debugOption);
+		options.addOption(infoOption);
 		options.addOption(fileParam);
 		
 		HelpFormatter formatter = new HelpFormatter();
@@ -68,9 +70,12 @@ public class Cli {
 			System.exit(0);
 		}
 		
-		boolean debug = false;
 		if(cmd.hasOption("debug")) {
-			debug = true;
+			Log.setLevel(Log.DEBUG);
+		}
+		
+		if(cmd.hasOption("info")) {
+			Log.setLevel(Log.INFO);
 		}
 		
 		String patternPath = "";
@@ -86,13 +91,12 @@ public class Cli {
 		if(cmd.hasOption("paramfile")) {
 			String fileParamName = cmd.getOptionValue("paramfile");
 			File param = new File(fileParamName);
-			ParserEngine engine = new ParserEngine(patternPath, debug);
+			ParserEngine engine = new ParserEngine(patternPath);
+			Log log = Log.getLogger(Cli.class.getName());
 			try {
 				startParserFromFile(param, engine );
 				lastModifiedTime = param.lastModified();
-				if (debug) {
+				log.info("All parsers started");
-					System.out.println("All parsers started");
-				}
 				while (true) {
 		            try {
 						Thread.sleep(5000);
@@ -139,15 +143,12 @@ public class Cli {
 			}
 
 			if(cmd.hasOption("logfile")) {
-				
+				ParserEngine engine = new ParserEngine(patternPath);
-				ParserEngine engine = new ParserEngine(patternPath, debug);
 				String[] filesName = cmd.getOptionValues("logfile");
 				for (int i = 0; i < filesName.length ; i++) {
 					File f = new File (filesName[i]);
 					engine.addNewParser(f, regexName[i], applicationName);
 				}
-				
-				
 			}
 		}
 

src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java

@@ -32,7 +32,7 @@ class GlobScanner {
 		for (String include : includes)
 			includePatterns.add(new Pattern(include, ignoreCase));
 
-		List<Pattern> allExcludePatterns = new ArrayList(excludes.size());
+		List<Pattern> allExcludePatterns = new ArrayList<Pattern>(excludes.size());
 		for (String exclude : excludes)
 			allExcludePatterns.add(new Pattern(exclude, ignoreCase));
 

src/main/java/com/airfrance/diqmqs/logparser/Log.java

@@ -0,0 +1,52 @@
+package com.airfrance.diqmqs.logparser;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+public class Log {
+	
+	public static final int NONE = -1; 
+	public static final int INFO = 0;
+	public static final int DEBUG = 1; 
+	
+	private String className = this.getClass().getName();
+	private static int level = -1;
+	SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); 
+	Date now;
+	
+	private Log(String className) {
+		this.className = className;
+	}
+	
+	public static Log getLogger(String s)
+	{	
+		return new Log(s);
+	}
+	
+	static void setLevel(int level)
+	{
+		Log.level = level;
+	}
+	
+	public void info(String message)
+	{
+		if ( Log.level >= 0 ) {
+			writeLog("[INFO]:" + message);
+		}	
+	}
+	
+	public void debug(String message)
+	{
+		if ( Log.level >= 1 ) {
+			writeLog("[DEBUG]:" + message);
+		}	
+	}
+	
+	private void writeLog(String message)
+	{
+		now = new Date();
+		System.out.println(sdfDate.format(now) + " ["  + className + "] " + message );
+	}
+	
+	
+}

src/main/java/com/airfrance/diqmqs/logparser/Parser.java

@@ -1,8 +1,7 @@
 package com.airfrance.diqmqs.logparser;
 
-import java.net.InetAddress;
-import java.net.UnknownHostException;
 import java.util.Map;
+import java.util.Random;
 
 import org.apache.commons.io.input.Tailer;
 import org.apache.commons.io.input.TailerListenerAdapter;
@@ -21,25 +20,23 @@ public class Parser extends TailerListenerAdapter {
 	private String hostname 		= "";
 	private String application 		= "";
 	private ParserEngine engine		= null;
-
+	private String metricLine 		= ""; 
+	String tagName 					= null;
+	String tagValue					= null;
+	String value 					= null;
+	private Log log 				= Log.getLogger(Parser.class.getName());
+	private Random randomGenerator 	= new Random();
 	
 	public Parser(String application, String regexName, ParserEngine engine) {
 		
 		try {
-			if (engine.isDebug()) {
+			log.info("Create Grok with regex name : " + regexName);
-				System.out.println("Create Grok with regex name : " + regexName);
-			}
 			grok = Grok.create(engine.getPatternPath());
 			grok.compile("%{" + regexName + "}");
 		} catch (GrokException e) {
 			e.printStackTrace();
 		}
-		
+		this.hostname = engine.getHostname();
-		try {
-			hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0];
-		} catch (UnknownHostException e) {
-			e.printStackTrace();
-		}
 		this.application = application;
 		this.engine 	 = engine;
 	}
@@ -49,9 +46,8 @@ public class Parser extends TailerListenerAdapter {
 		super.init(tailer);
 		fileName = tailer.getFile().getName();
 		pathName = tailer.getFile().getParent();
-		if (engine.isDebug()) {
+		log.info("Init tailer on file : " + pathName + "/" + fileName );
-			System.out.println("Init tailer on file : " + pathName + "/" + fileName );
+		metricLine = "log,host=" + hostname + ",application=" + application + ",file=" + sanitizeString(fileName) + ",path=" + sanitizeString(pathName);
-		}
 	}
 	
 	 public void handle(String line) {
@@ -61,21 +57,11 @@ public class Parser extends TailerListenerAdapter {
 		Map<String,Object> result = gm.toMap();
 		if ( result.size() > 0 )
 		{		
-			if (engine.isDebug()) {
-				System.out.println("Match OK ");
-			}
-				
 			StringBuffer sb = new StringBuffer();
-			sb.append("log");
+			sb.append(metricLine);
-			//Tag 
+			tagName = null;
-			sb.append(",host=" + hostname);
+			tagValue = null;
-			sb.append(",application=" + application);
+			value = null;
-			sb.append(",file=" + sanitizeString(fileName) );
-			sb.append(",path=" + sanitizeString(pathName) );
-			
-			String tagName = null;
-			String tagValue = null;
-			String value = null;
 
 			// iterate on all match pattern 
 			for (Map.Entry<String,Object> entry : result.entrySet() )
@@ -86,37 +72,41 @@ public class Parser extends TailerListenerAdapter {
 				{
 					tagValue = entry.getValue().toString();
 					// if key name is 'value', it's not a tag, but a field 
-					if ( ! tagName.equalsIgnoreCase("value") ) {
+					if ( tagName.equalsIgnoreCase("value") ) {
-						// check if tag value is not empty 
+						value = tagValue;
-						if (!tagValue.equalsIgnoreCase("") ) {
-							sb.append("," + tagName + "=" + sanitizeString( tagValue ) );
-						}
 					}
 					else
 					{
-						value = tagValue;
+						// check if tag value is not empty 
+						if (!tagValue.equalsIgnoreCase("") ) {
+							sb.append(",")
+								.append( sanitizeString(tagName))
+								.append("=")
+								.append( sanitizeString(tagValue));		
+						}
 					}
 				}
-				
 			}
 			sb.append(" ");
 			//Field
 			if ( StringUtils.isNumeric(value) ) {
-				sb.append("value=" + value);
+				sb.append("value=")
+				.append(value);
 			}
 			else
 			{
 				sb.append("value=1");
 			}
-			// Time
+			// TimeStamp
-			sb.append(" " + System.currentTimeMillis() + "000000");
+			sb.append(" ")
+				.append(System.currentTimeMillis())
+				.append(String.format("%06d",randomGenerator.nextInt(999999)));
 			
+			log.debug("Line => " + line + "  : Match add it to Queue ");
 			// Add to the queue
 			engine.addToQueue(sb.toString());
 			
-			if (engine.isDebug()) {
+			log.debug(gm.toJson());
-				System.out.println(gm.toJson());
-			}
 		}   
      }
 

src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java

@@ -4,7 +4,9 @@ import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
+import java.net.InetAddress;
 import java.net.URL;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.concurrent.BlockingDeque;
 import java.util.concurrent.Executors;
@@ -19,52 +21,49 @@ import org.apache.commons.io.input.TailerListener;
 public class ParserEngine implements Runnable{
 
 	final private String database						= "qualif";	
-	final long delay 									= 100; 
+	final long delay 									= 200; 
 	private ArrayList<Thread> threadsParser				= null;
 	static BlockingDeque<String> queue					= new LinkedBlockingDeque<String>(2000);
-	private boolean debug 								= false;
 	private String patternPath 							= "";
-	
+	ArrayList<String> res = new ArrayList<String>(2000);
+	Log log = Log.getLogger(ParserEngine.class.getName());
 	private ScheduledExecutorService scheduler;
+	private String hostname;
+
+
+
 	@SuppressWarnings("unused")
 	private ScheduledFuture<?> timerHandle;
 	
-	public ParserEngine(String patternPath, boolean debug) {
+	public ParserEngine(String patternPath) {
 		threadsParser = new ArrayList<Thread>();
-		this.debug = debug;
 		this.patternPath = patternPath;	
 		scheduler = Executors.newScheduledThreadPool(1);
-        // Don't change this as metrics are per second
+		try {
-        this.timerHandle = scheduler.scheduleWithFixedDelay(this, 1, 1, TimeUnit.SECONDS);
+			hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0];
-        
+		} catch (UnknownHostException e) {
-        
+			e.printStackTrace();
-		if (debug) {
-			System.out.println("Create ParserEngine with pattern file : " + patternPath);
-			System.out.println("Schedule sender is set to " + 1 + " sec " );
 		}
+        this.timerHandle = scheduler.scheduleWithFixedDelay(this, 1, 1, TimeUnit.SECONDS);
+		log.info("Create ParserEngine with pattern file : " + patternPath);
+		log.info("Schedule sender is set to " + 1 + " secs" );
 	}
 	
 	void addNewParser(File f , String regexName, String application ) {
-
 		TailerListener listener = new Parser(application, regexName, this);
-		Tailer tailer = new Tailer(f, listener, delay, true);
+		Tailer tailer = new Tailer(f, listener, delay, true, true, 8192);
 	    Thread thread = new Thread(tailer);
 	    thread.setDaemon(true); 
 	    thread.setName("Parser - " + threadsParser.size() );
 	    thread.start();
 	    threadsParser.add(thread);
-	    if (debug) {
+	    log.info("Thread Parser - " + threadsParser.size() + " started on file : " + f.getName());
-			System.out.println("Thread Parser - " + threadsParser.size() + " started - file : " + f.getName());
-		}
-		
 	}
 	
 	void stopAllParser() {
 		for (Thread t : threadsParser) {
 			t.interrupt();
-			 if (debug) {
+			log.info("Stop Thread " + t.getName() );
-				System.out.println("Stop Thread " + t.getName() );
-			 }
 		}
 		threadsParser.clear();
 	}
@@ -73,10 +72,7 @@ public class ParserEngine implements Runnable{
 	public void run() {
 		
 		StringBuilder postData = new StringBuilder();
-		if (debug) {
+		log.info("Actual queue size  " + queue.size() );
-			System.out.println("Actual queue size  " + queue.size() );
-		}
-		ArrayList<String> res = new ArrayList<String>(2000);
 		int nbElement = queue.drainTo(res, 2000);
 		if (nbElement > 0) {
 			for ( String s : res) {
@@ -85,7 +81,13 @@ public class ParserEngine implements Runnable{
 		}
 		try {
 			if (postData.length() > 0) {
-				sendMetricToInfluxdb(postData.toString());
+				if (sendMetricToInfluxdb(postData.toString()))
+				{
+					res.clear();
+				}
+				else{
+					
+				}
 			}
 		} catch (IOException e) {
 			e.printStackTrace();
@@ -95,6 +97,7 @@ public class ParserEngine implements Runnable{
 	 public boolean sendMetricToInfluxdb(String postData) throws IOException {
 
 		 	URL url = new URL("http://diqmqs.airfrance.fr/influxdb_query/write?rp=one_week&db=" + database);
+			long start = System.currentTimeMillis();
 			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
 			connection.setDoOutput(true);
 			connection.setRequestMethod("POST");
@@ -104,18 +107,16 @@ public class ParserEngine implements Runnable{
 
 			// Write data
 			OutputStream os = connection.getOutputStream();
-			if (this.isDebug()) {
+			log.info("Message to send : \n" + postData );
-				System.out.println("Send : " + postData );
-			}
 			os.write(postData.getBytes());
-
 			int HttpResult = connection.getResponseCode();
+			long end = System.currentTimeMillis();
+			long elapseTime  = end - start;
 			if (HttpResult == 204) {
+				log.debug("Message sended in " + elapseTime + " ms");
 				return true;
 			} else {
-				if (this.isDebug()) {
+				log.debug(connection.getResponseCode() + " " + connection.getResponseMessage());
-					System.err.println(connection.getResponseCode() + " " + connection.getResponseMessage());
-				}
 				return false;
 			}
 	}
@@ -130,16 +131,6 @@ public class ParserEngine implements Runnable{
 		 
 	 }
 
-	public boolean isDebug() {
-		return debug;
-	}
-
-
-	public void setDebug(boolean debug) {
-		this.debug = debug;
-	}
-
-
 	public String getPatternPath() {
 		return patternPath;
 	}
@@ -149,4 +140,12 @@ public class ParserEngine implements Runnable{
 		this.patternPath = patternPath;
 	}
 	
+	public String getHostname() {
+		return hostname;
+	}
+
+	public void setHostname(String hostname) {
+		this.hostname = hostname;
+	}
+
 }