fred/logparser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve log and performance

Browse Source
This commit is contained in:
Maxime Chassagneux
2016-04-21 10:55:11 +02:00
parent 9b5aa30043
commit 99006bc76c
7 changed files with 151 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.classpath
View File

@@ -11,7 +11,7 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
@@ -21,5 +21,11 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

5
pom.xml
View File

@@ -7,8 +7,8 @@
<description>logParser - send data to influxdb</description>
<properties>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<maven.compiler.source>1.6</maven.compiler.source>
<maven.compiler.target>1.6</maven.compiler.target>
</properties>
<dependencies>
<dependency>
@@ -27,7 +27,6 @@
<version>2.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>

21
src/main/java/com/airfrance/diqmqs/logparser/Cli.java
View File

@@ -24,6 +24,7 @@ public class Cli {
public static Option regex = new Option( "regex", true , "Name of the regex to apply" );
public static Option fileParam = new Option( "paramfile", true , "Input a param file" );
public static Option debugOption = new Option( "debug", "Active debug output message" );
public static Option infoOption = new Option( "info", "Active info output message" );
private static long lastModifiedTime = 0L;
public Cli() {
// TODO Auto-generated constructor stub
@@ -44,6 +45,7 @@ public class Cli {
options.addOption(logFile);
options.addOption(regex);
options.addOption(debugOption);
options.addOption(infoOption);
options.addOption(fileParam);
HelpFormatter formatter = new HelpFormatter();
@@ -68,9 +70,12 @@ public class Cli {
System.exit(0);
}
boolean debug = false;
if(cmd.hasOption("debug")) {
debug = true;
Log.setLevel(Log.DEBUG);
}
if(cmd.hasOption("info")) {
Log.setLevel(Log.INFO);
}
String patternPath = "";
@@ -86,13 +91,12 @@ public class Cli {
if(cmd.hasOption("paramfile")) {
String fileParamName = cmd.getOptionValue("paramfile");
File param = new File(fileParamName);
ParserEngine engine = new ParserEngine(patternPath, debug);
ParserEngine engine = new ParserEngine(patternPath);
Log log = Log.getLogger(Cli.class.getName());
try {
startParserFromFile(param, engine );
lastModifiedTime = param.lastModified();
if (debug) {
System.out.println("All parsers started");
}
log.info("All parsers started");
while (true) {
try {
Thread.sleep(5000);
@@ -139,15 +143,12 @@ public class Cli {
}
if(cmd.hasOption("logfile")) {
ParserEngine engine = new ParserEngine(patternPath, debug);
ParserEngine engine = new ParserEngine(patternPath);
String[] filesName = cmd.getOptionValues("logfile");
for (int i = 0; i < filesName.length ; i++) {
File f = new File (filesName[i]);
engine.addNewParser(f, regexName[i], applicationName);
}
}
}

2
src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java
View File

@@ -32,7 +32,7 @@ class GlobScanner {
for (String include : includes)
includePatterns.add(new Pattern(include, ignoreCase));
List<Pattern> allExcludePatterns = new ArrayList(excludes.size());
List<Pattern> allExcludePatterns = new ArrayList<Pattern>(excludes.size());
for (String exclude : excludes)
allExcludePatterns.add(new Pattern(exclude, ignoreCase));

52
src/main/java/com/airfrance/diqmqs/logparser/Log.java Normal file
View File

@@ -0,0 +1,52 @@
package com.airfrance.diqmqs.logparser;
import java.text.SimpleDateFormat;
import java.util.Date;
public class Log {
public static final int NONE = -1;
public static final int INFO = 0;
public static final int DEBUG = 1;
private String className = this.getClass().getName();
private static int level = -1;
SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
Date now;
private Log(String className) {
this.className = className;
}
public static Log getLogger(String s)
{
return new Log(s);
}
static void setLevel(int level)
{
Log.level = level;
}
public void info(String message)
{
if ( Log.level >= 0 ) {
writeLog("[INFO]:" + message);
}
}
public void debug(String message)
{
if ( Log.level >= 1 ) {
writeLog("[DEBUG]:" + message);
}
}
private void writeLog(String message)
{
now = new Date();
System.out.println(sdfDate.format(now) + " [" + className + "] " + message );
}
}

78
src/main/java/com/airfrance/diqmqs/logparser/Parser.java
View File

@@ -1,8 +1,7 @@
package com.airfrance.diqmqs.logparser;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.input.Tailer;
import org.apache.commons.io.input.TailerListenerAdapter;
@@ -21,25 +20,23 @@ public class Parser extends TailerListenerAdapter {
private String hostname = "";
private String application = "";
private ParserEngine engine = null;
private String metricLine = "";
String tagName = null;
String tagValue = null;
String value = null;
private Log log = Log.getLogger(Parser.class.getName());
private Random randomGenerator = new Random();
public Parser(String application, String regexName, ParserEngine engine) {
try {
if (engine.isDebug()) {
System.out.println("Create Grok with regex name : " + regexName);
}
log.info("Create Grok with regex name : " + regexName);
grok = Grok.create(engine.getPatternPath());
grok.compile("%{" + regexName + "}");
} catch (GrokException e) {
e.printStackTrace();
}
try {
hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0];
} catch (UnknownHostException e) {
e.printStackTrace();
}
this.hostname = engine.getHostname();
this.application = application;
this.engine = engine;
}
@@ -49,9 +46,8 @@ public class Parser extends TailerListenerAdapter {
super.init(tailer);
fileName = tailer.getFile().getName();
pathName = tailer.getFile().getParent();
if (engine.isDebug()) {
System.out.println("Init tailer on file : " + pathName + "/" + fileName );
}
log.info("Init tailer on file : " + pathName + "/" + fileName );
metricLine = "log,host=" + hostname + ",application=" + application + ",file=" + sanitizeString(fileName) + ",path=" + sanitizeString(pathName);
}
public void handle(String line) {
@@ -60,22 +56,12 @@ public class Parser extends TailerListenerAdapter {
gm.captures();
Map<String,Object> result = gm.toMap();
if ( result.size() > 0 )
{
if (engine.isDebug()) {
System.out.println("Match OK ");
}
{
StringBuffer sb = new StringBuffer();
sb.append("log");
//Tag
sb.append(",host=" + hostname);
sb.append(",application=" + application);
sb.append(",file=" + sanitizeString(fileName) );
sb.append(",path=" + sanitizeString(pathName) );
String tagName = null;
String tagValue = null;
String value = null;
sb.append(metricLine);
tagName = null;
tagValue = null;
value = null;
// iterate on all match pattern
for (Map.Entry<String,Object> entry : result.entrySet() )
@@ -86,37 +72,41 @@ public class Parser extends TailerListenerAdapter {
{
tagValue = entry.getValue().toString();
// if key name is 'value', it's not a tag, but a field
if ( ! tagName.equalsIgnoreCase("value") ) {
// check if tag value is not empty
if (!tagValue.equalsIgnoreCase("") ) {
sb.append("," + tagName + "=" + sanitizeString( tagValue ) );
}
if ( tagName.equalsIgnoreCase("value") ) {
value = tagValue;
}
else
{
value = tagValue;
// check if tag value is not empty
if (!tagValue.equalsIgnoreCase("") ) {
sb.append(",")
.append( sanitizeString(tagName))
.append("=")
.append( sanitizeString(tagValue));
}
}
}
}
sb.append(" ");
//Field
if ( StringUtils.isNumeric(value) ) {
sb.append("value=" + value);
sb.append("value=")
.append(value);
}
else
{
sb.append("value=1");
}
// Time
sb.append(" " + System.currentTimeMillis() + "000000");
// TimeStamp
sb.append(" ")
.append(System.currentTimeMillis())
.append(String.format("%06d",randomGenerator.nextInt(999999)));
log.debug("Line => " + line + " : Match add it to Queue ");
// Add to the queue
engine.addToQueue(sb.toString());
if (engine.isDebug()) {
System.out.println(gm.toJson());
}
log.debug(gm.toJson());
}
}

89
src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java
View File

@@ -4,7 +4,9 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.Executors;
@@ -19,52 +21,49 @@ import org.apache.commons.io.input.TailerListener;
public class ParserEngine implements Runnable{
final private String database = "qualif";
final long delay = 100;
final long delay = 200;
private ArrayList<Thread> threadsParser = null;
static BlockingDeque<String> queue = new LinkedBlockingDeque<String>(2000);
private boolean debug = false;
private String patternPath = "";
ArrayList<String> res = new ArrayList<String>(2000);
Log log = Log.getLogger(ParserEngine.class.getName());
private ScheduledExecutorService scheduler;
private String hostname;
@SuppressWarnings("unused")
private ScheduledFuture<?> timerHandle;
public ParserEngine(String patternPath, boolean debug) {
public ParserEngine(String patternPath) {
threadsParser = new ArrayList<Thread>();
this.debug = debug;
this.patternPath = patternPath;
scheduler = Executors.newScheduledThreadPool(1);
// Don't change this as metrics are per second
this.timerHandle = scheduler.scheduleWithFixedDelay(this, 1, 1, TimeUnit.SECONDS);
if (debug) {
System.out.println("Create ParserEngine with pattern file : " + patternPath);
System.out.println("Schedule sender is set to " + 1 + " sec " );
try {
hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0];
} catch (UnknownHostException e) {
e.printStackTrace();
}
this.timerHandle = scheduler.scheduleWithFixedDelay(this, 1, 1, TimeUnit.SECONDS);
log.info("Create ParserEngine with pattern file : " + patternPath);
log.info("Schedule sender is set to " + 1 + " secs" );
}
void addNewParser(File f , String regexName, String application ) {
TailerListener listener = new Parser(application, regexName, this);
Tailer tailer = new Tailer(f, listener, delay, true);
Tailer tailer = new Tailer(f, listener, delay, true, true, 8192);
Thread thread = new Thread(tailer);
thread.setDaemon(true);
thread.setName("Parser - " + threadsParser.size() );
thread.start();
threadsParser.add(thread);
if (debug) {
System.out.println("Thread Parser - " + threadsParser.size() + " started - file : " + f.getName());
}
log.info("Thread Parser - " + threadsParser.size() + " started on file : " + f.getName());
}
void stopAllParser() {
for (Thread t : threadsParser) {
t.interrupt();
if (debug) {
System.out.println("Stop Thread " + t.getName() );
}
log.info("Stop Thread " + t.getName() );
}
threadsParser.clear();
}
@@ -73,10 +72,7 @@ public class ParserEngine implements Runnable{
public void run() {
StringBuilder postData = new StringBuilder();
if (debug) {
System.out.println("Actual queue size " + queue.size() );
}
ArrayList<String> res = new ArrayList<String>(2000);
log.info("Actual queue size " + queue.size() );
int nbElement = queue.drainTo(res, 2000);
if (nbElement > 0) {
for ( String s : res) {
@@ -85,7 +81,13 @@ public class ParserEngine implements Runnable{
}
try {
if (postData.length() > 0) {
sendMetricToInfluxdb(postData.toString());
if (sendMetricToInfluxdb(postData.toString()))
{
res.clear();
}
else{
}
}
} catch (IOException e) {
e.printStackTrace();
@@ -94,7 +96,8 @@ public class ParserEngine implements Runnable{
public boolean sendMetricToInfluxdb(String postData) throws IOException {
URL url = new URL("http://diqmqs.airfrance.fr/influxdb_query/write?rp=one_week&db=" + database);
URL url = new URL("http://diqmqs.airfrance.fr/influxdb_query/write?rp=one_week&db=" + database);
long start = System.currentTimeMillis();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true);
connection.setRequestMethod("POST");
@@ -104,21 +107,19 @@ public class ParserEngine implements Runnable{
// Write data
OutputStream os = connection.getOutputStream();
if (this.isDebug()) {
System.out.println("Send : " + postData );
}
log.info("Message to send : \n" + postData );
os.write(postData.getBytes());
int HttpResult = connection.getResponseCode();
long end = System.currentTimeMillis();
long elapseTime = end - start;
if (HttpResult == 204) {
log.debug("Message sended in " + elapseTime + " ms");
return true;
} else {
if (this.isDebug()) {
System.err.println(connection.getResponseCode() + " " + connection.getResponseMessage());
}
log.debug(connection.getResponseCode() + " " + connection.getResponseMessage());
return false;
}
}
}
public void addToQueue(String s) {
@@ -130,16 +131,6 @@ public class ParserEngine implements Runnable{
}
public boolean isDebug() {
return debug;
}
public void setDebug(boolean debug) {
this.debug = debug;
}
public String getPatternPath() {
return patternPath;
}
@@ -148,5 +139,13 @@ public class ParserEngine implements Runnable{
public void setPatternPath(String patternPath) {
this.patternPath = patternPath;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
}