From 99006bc76c2bcd1ea2f610cf2336b8a07edf2900 Mon Sep 17 00:00:00 2001 From: Maxime Chassagneux <4163013@airfrance.fr> Date: Thu, 21 Apr 2016 10:55:11 +0200 Subject: [PATCH] Improve log and performance --- .classpath | 8 +- pom.xml | 5 +- .../com/airfrance/diqmqs/logparser/Cli.java | 21 ++--- .../diqmqs/logparser/GlobScanner.java | 2 +- .../com/airfrance/diqmqs/logparser/Log.java | 52 +++++++++++ .../airfrance/diqmqs/logparser/Parser.java | 78 +++++++--------- .../diqmqs/logparser/ParserEngine.java | 89 +++++++++---------- 7 files changed, 151 insertions(+), 104 deletions(-) create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/Log.java diff --git a/.classpath b/.classpath index c9ef8f9..f67a1d3 100644 --- a/.classpath +++ b/.classpath @@ -11,7 +11,7 @@ - + @@ -21,5 +21,11 @@ + + + + + + diff --git a/pom.xml b/pom.xml index 46f88d3..0ae0575 100644 --- a/pom.xml +++ b/pom.xml @@ -7,8 +7,8 @@ logParser - send data to influxdb - 1.7 - 1.7 + 1.6 + 1.6 @@ -27,7 +27,6 @@ 2.4 - diff --git a/src/main/java/com/airfrance/diqmqs/logparser/Cli.java b/src/main/java/com/airfrance/diqmqs/logparser/Cli.java index c67bf9d..60b4bf0 100644 --- a/src/main/java/com/airfrance/diqmqs/logparser/Cli.java +++ b/src/main/java/com/airfrance/diqmqs/logparser/Cli.java @@ -24,6 +24,7 @@ public class Cli { public static Option regex = new Option( "regex", true , "Name of the regex to apply" ); public static Option fileParam = new Option( "paramfile", true , "Input a param file" ); public static Option debugOption = new Option( "debug", "Active debug output message" ); + public static Option infoOption = new Option( "info", "Active info output message" ); private static long lastModifiedTime = 0L; public Cli() { // TODO Auto-generated constructor stub @@ -44,6 +45,7 @@ public class Cli { options.addOption(logFile); options.addOption(regex); options.addOption(debugOption); + options.addOption(infoOption); options.addOption(fileParam); HelpFormatter formatter = new HelpFormatter(); @@ -68,9 +70,12 @@ public class Cli { System.exit(0); } - boolean debug = false; if(cmd.hasOption("debug")) { - debug = true; + Log.setLevel(Log.DEBUG); + } + + if(cmd.hasOption("info")) { + Log.setLevel(Log.INFO); } String patternPath = ""; @@ -86,13 +91,12 @@ public class Cli { if(cmd.hasOption("paramfile")) { String fileParamName = cmd.getOptionValue("paramfile"); File param = new File(fileParamName); - ParserEngine engine = new ParserEngine(patternPath, debug); + ParserEngine engine = new ParserEngine(patternPath); + Log log = Log.getLogger(Cli.class.getName()); try { startParserFromFile(param, engine ); lastModifiedTime = param.lastModified(); - if (debug) { - System.out.println("All parsers started"); - } + log.info("All parsers started"); while (true) { try { Thread.sleep(5000); @@ -139,15 +143,12 @@ public class Cli { } if(cmd.hasOption("logfile")) { - - ParserEngine engine = new ParserEngine(patternPath, debug); + ParserEngine engine = new ParserEngine(patternPath); String[] filesName = cmd.getOptionValues("logfile"); for (int i = 0; i < filesName.length ; i++) { File f = new File (filesName[i]); engine.addNewParser(f, regexName[i], applicationName); } - - } } diff --git a/src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java b/src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java index 13c0e2f..b627b34 100644 --- a/src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java +++ b/src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java @@ -32,7 +32,7 @@ class GlobScanner { for (String include : includes) includePatterns.add(new Pattern(include, ignoreCase)); - List allExcludePatterns = new ArrayList(excludes.size()); + List allExcludePatterns = new ArrayList(excludes.size()); for (String exclude : excludes) allExcludePatterns.add(new Pattern(exclude, ignoreCase)); diff --git a/src/main/java/com/airfrance/diqmqs/logparser/Log.java b/src/main/java/com/airfrance/diqmqs/logparser/Log.java new file mode 100644 index 0000000..4a345ed --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/Log.java @@ -0,0 +1,52 @@ +package com.airfrance.diqmqs.logparser; + +import java.text.SimpleDateFormat; +import java.util.Date; + +public class Log { + + public static final int NONE = -1; + public static final int INFO = 0; + public static final int DEBUG = 1; + + private String className = this.getClass().getName(); + private static int level = -1; + SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + Date now; + + private Log(String className) { + this.className = className; + } + + public static Log getLogger(String s) + { + return new Log(s); + } + + static void setLevel(int level) + { + Log.level = level; + } + + public void info(String message) + { + if ( Log.level >= 0 ) { + writeLog("[INFO]:" + message); + } + } + + public void debug(String message) + { + if ( Log.level >= 1 ) { + writeLog("[DEBUG]:" + message); + } + } + + private void writeLog(String message) + { + now = new Date(); + System.out.println(sdfDate.format(now) + " [" + className + "] " + message ); + } + + +} diff --git a/src/main/java/com/airfrance/diqmqs/logparser/Parser.java b/src/main/java/com/airfrance/diqmqs/logparser/Parser.java index 23c0fea..f4c12e3 100644 --- a/src/main/java/com/airfrance/diqmqs/logparser/Parser.java +++ b/src/main/java/com/airfrance/diqmqs/logparser/Parser.java @@ -1,8 +1,7 @@ package com.airfrance.diqmqs.logparser; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.Map; +import java.util.Random; import org.apache.commons.io.input.Tailer; import org.apache.commons.io.input.TailerListenerAdapter; @@ -21,25 +20,23 @@ public class Parser extends TailerListenerAdapter { private String hostname = ""; private String application = ""; private ParserEngine engine = null; - - + private String metricLine = ""; + String tagName = null; + String tagValue = null; + String value = null; + private Log log = Log.getLogger(Parser.class.getName()); + private Random randomGenerator = new Random(); + public Parser(String application, String regexName, ParserEngine engine) { try { - if (engine.isDebug()) { - System.out.println("Create Grok with regex name : " + regexName); - } + log.info("Create Grok with regex name : " + regexName); grok = Grok.create(engine.getPatternPath()); grok.compile("%{" + regexName + "}"); } catch (GrokException e) { e.printStackTrace(); } - - try { - hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0]; - } catch (UnknownHostException e) { - e.printStackTrace(); - } + this.hostname = engine.getHostname(); this.application = application; this.engine = engine; } @@ -49,9 +46,8 @@ public class Parser extends TailerListenerAdapter { super.init(tailer); fileName = tailer.getFile().getName(); pathName = tailer.getFile().getParent(); - if (engine.isDebug()) { - System.out.println("Init tailer on file : " + pathName + "/" + fileName ); - } + log.info("Init tailer on file : " + pathName + "/" + fileName ); + metricLine = "log,host=" + hostname + ",application=" + application + ",file=" + sanitizeString(fileName) + ",path=" + sanitizeString(pathName); } public void handle(String line) { @@ -60,22 +56,12 @@ public class Parser extends TailerListenerAdapter { gm.captures(); Map result = gm.toMap(); if ( result.size() > 0 ) - { - if (engine.isDebug()) { - System.out.println("Match OK "); - } - + { StringBuffer sb = new StringBuffer(); - sb.append("log"); - //Tag - sb.append(",host=" + hostname); - sb.append(",application=" + application); - sb.append(",file=" + sanitizeString(fileName) ); - sb.append(",path=" + sanitizeString(pathName) ); - - String tagName = null; - String tagValue = null; - String value = null; + sb.append(metricLine); + tagName = null; + tagValue = null; + value = null; // iterate on all match pattern for (Map.Entry entry : result.entrySet() ) @@ -86,37 +72,41 @@ public class Parser extends TailerListenerAdapter { { tagValue = entry.getValue().toString(); // if key name is 'value', it's not a tag, but a field - if ( ! tagName.equalsIgnoreCase("value") ) { - // check if tag value is not empty - if (!tagValue.equalsIgnoreCase("") ) { - sb.append("," + tagName + "=" + sanitizeString( tagValue ) ); - } + if ( tagName.equalsIgnoreCase("value") ) { + value = tagValue; } else { - value = tagValue; + // check if tag value is not empty + if (!tagValue.equalsIgnoreCase("") ) { + sb.append(",") + .append( sanitizeString(tagName)) + .append("=") + .append( sanitizeString(tagValue)); + } } } - } sb.append(" "); //Field if ( StringUtils.isNumeric(value) ) { - sb.append("value=" + value); + sb.append("value=") + .append(value); } else { sb.append("value=1"); } - // Time - sb.append(" " + System.currentTimeMillis() + "000000"); + // TimeStamp + sb.append(" ") + .append(System.currentTimeMillis()) + .append(String.format("%06d",randomGenerator.nextInt(999999))); + log.debug("Line => " + line + " : Match add it to Queue "); // Add to the queue engine.addToQueue(sb.toString()); - if (engine.isDebug()) { - System.out.println(gm.toJson()); - } + log.debug(gm.toJson()); } } diff --git a/src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java b/src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java index 382775d..c6955c9 100644 --- a/src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java +++ b/src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java @@ -4,7 +4,9 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.net.HttpURLConnection; +import java.net.InetAddress; import java.net.URL; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.concurrent.BlockingDeque; import java.util.concurrent.Executors; @@ -19,52 +21,49 @@ import org.apache.commons.io.input.TailerListener; public class ParserEngine implements Runnable{ final private String database = "qualif"; - final long delay = 100; + final long delay = 200; private ArrayList threadsParser = null; static BlockingDeque queue = new LinkedBlockingDeque(2000); - private boolean debug = false; private String patternPath = ""; - + ArrayList res = new ArrayList(2000); + Log log = Log.getLogger(ParserEngine.class.getName()); private ScheduledExecutorService scheduler; + private String hostname; + + + @SuppressWarnings("unused") private ScheduledFuture timerHandle; - public ParserEngine(String patternPath, boolean debug) { + public ParserEngine(String patternPath) { threadsParser = new ArrayList(); - this.debug = debug; this.patternPath = patternPath; scheduler = Executors.newScheduledThreadPool(1); - // Don't change this as metrics are per second - this.timerHandle = scheduler.scheduleWithFixedDelay(this, 1, 1, TimeUnit.SECONDS); - - - if (debug) { - System.out.println("Create ParserEngine with pattern file : " + patternPath); - System.out.println("Schedule sender is set to " + 1 + " sec " ); + try { + hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0]; + } catch (UnknownHostException e) { + e.printStackTrace(); } + this.timerHandle = scheduler.scheduleWithFixedDelay(this, 1, 1, TimeUnit.SECONDS); + log.info("Create ParserEngine with pattern file : " + patternPath); + log.info("Schedule sender is set to " + 1 + " secs" ); } void addNewParser(File f , String regexName, String application ) { - TailerListener listener = new Parser(application, regexName, this); - Tailer tailer = new Tailer(f, listener, delay, true); + Tailer tailer = new Tailer(f, listener, delay, true, true, 8192); Thread thread = new Thread(tailer); thread.setDaemon(true); thread.setName("Parser - " + threadsParser.size() ); thread.start(); threadsParser.add(thread); - if (debug) { - System.out.println("Thread Parser - " + threadsParser.size() + " started - file : " + f.getName()); - } - + log.info("Thread Parser - " + threadsParser.size() + " started on file : " + f.getName()); } void stopAllParser() { for (Thread t : threadsParser) { t.interrupt(); - if (debug) { - System.out.println("Stop Thread " + t.getName() ); - } + log.info("Stop Thread " + t.getName() ); } threadsParser.clear(); } @@ -73,10 +72,7 @@ public class ParserEngine implements Runnable{ public void run() { StringBuilder postData = new StringBuilder(); - if (debug) { - System.out.println("Actual queue size " + queue.size() ); - } - ArrayList res = new ArrayList(2000); + log.info("Actual queue size " + queue.size() ); int nbElement = queue.drainTo(res, 2000); if (nbElement > 0) { for ( String s : res) { @@ -85,7 +81,13 @@ public class ParserEngine implements Runnable{ } try { if (postData.length() > 0) { - sendMetricToInfluxdb(postData.toString()); + if (sendMetricToInfluxdb(postData.toString())) + { + res.clear(); + } + else{ + + } } } catch (IOException e) { e.printStackTrace(); @@ -94,7 +96,8 @@ public class ParserEngine implements Runnable{ public boolean sendMetricToInfluxdb(String postData) throws IOException { - URL url = new URL("http://diqmqs.airfrance.fr/influxdb_query/write?rp=one_week&db=" + database); + URL url = new URL("http://diqmqs.airfrance.fr/influxdb_query/write?rp=one_week&db=" + database); + long start = System.currentTimeMillis(); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setDoOutput(true); connection.setRequestMethod("POST"); @@ -104,21 +107,19 @@ public class ParserEngine implements Runnable{ // Write data OutputStream os = connection.getOutputStream(); - if (this.isDebug()) { - System.out.println("Send : " + postData ); - } + log.info("Message to send : \n" + postData ); os.write(postData.getBytes()); - int HttpResult = connection.getResponseCode(); + long end = System.currentTimeMillis(); + long elapseTime = end - start; if (HttpResult == 204) { + log.debug("Message sended in " + elapseTime + " ms"); return true; } else { - if (this.isDebug()) { - System.err.println(connection.getResponseCode() + " " + connection.getResponseMessage()); - } + log.debug(connection.getResponseCode() + " " + connection.getResponseMessage()); return false; } - } + } public void addToQueue(String s) { @@ -130,16 +131,6 @@ public class ParserEngine implements Runnable{ } - public boolean isDebug() { - return debug; - } - - - public void setDebug(boolean debug) { - this.debug = debug; - } - - public String getPatternPath() { return patternPath; } @@ -148,5 +139,13 @@ public class ParserEngine implements Runnable{ public void setPatternPath(String patternPath) { this.patternPath = patternPath; } + + public String getHostname() { + return hostname; + } + + public void setHostname(String hostname) { + this.hostname = hostname; + } }