fred/logparser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Read from start option + extract date from pattern to put in influxdb

Browse Source
This commit is contained in:
Maxime Chassagneux
2016-09-22 17:21:48 +02:00
parent 70285d5ff9
commit 3900bde616
4 changed files with 84 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/com/airfrance/diqmqs/logparser/Cli.java
View File

@@ -22,6 +22,7 @@ public class Cli {
public static Option patternFilePath = new Option( "pattern", true , "Input pattern path file" );
public static Option logFile = new Option( "logfile", true , "Input log path files" );
public static Option regex = new Option( "regex", true , "Name of the regex to apply" );
public static Option fromStart = new Option( "fromStart", "Read the file from start" );
public static Option fileParam = new Option( "paramfile", true , "Input a param file" );
public static Option debugOption = new Option( "debug", "Active debug output message" );
public static Option infoOption = new Option( "info", "Active info output message" );
@@ -47,8 +48,10 @@ public class Cli {
options.addOption(debugOption);
options.addOption(infoOption);
options.addOption(fileParam);
options.addOption(fromStart);
HelpFormatter formatter = new HelpFormatter();
boolean readFromStart = false;
CommandLineParser parser = new DefaultParser();
CommandLine cmd = null;
try {
@@ -141,13 +144,17 @@ public class Cli {
System.err.println("regex application missing");
System.exit(0);
}
if(cmd.hasOption("fromStart")) {
readFromStart = true;
}
if(cmd.hasOption("logfile")) {
ParserEngine engine = new ParserEngine(patternPath);
String[] filesName = cmd.getOptionValues("logfile");
for (int i = 0; i < filesName.length ; i++) {
File f = new File (filesName[i]);
engine.addNewParser(f, regexName[i], applicationName);
engine.addNewParser(f, regexName[i], applicationName, readFromStart);
}
}
}
@@ -158,16 +165,20 @@ public class Cli {
static void startParserFromFile(File file, ParserEngine engine) throws FileNotFoundException {
Scanner scanner = new Scanner(file);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
boolean readFromStart = false;
if (!line.startsWith("#"))
{
String[] arguments = line.split("\\t");
String applicationName = arguments[0].toLowerCase();
Paths paths = new Paths("/", arguments[1]);
String regexName = arguments[2];
if (arguments[3] != null)
readFromStart = true;
for (File f : paths.getFiles()) {
engine.addNewParser(f, regexName, applicationName);
engine.addNewParser(f, regexName, applicationName, readFromStart);
}
}

21
src/main/java/com/airfrance/diqmqs/logparser/Log.java
View File

@@ -6,11 +6,12 @@ import java.util.Date;
public class Log {
public static final int NONE = -1;
public static final int INFO = 0;
public static final int DEBUG = 1;
public static final int ERROR = 0;
public static final int INFO = 1;
public static final int DEBUG = 2;
private String className = this.getClass().getName();
private static int level = -1;
private static int level = 1;
SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
Date now;
@@ -28,6 +29,13 @@ public class Log {
Log.level = level;
}
public void error(String message)
{
if ( Log.level >= ERROR ) {
writeLogError("[ERROR]:" + message);
}
}
public void info(String message)
{
if ( Log.level >= INFO ) {
@@ -48,5 +56,12 @@ public class Log {
System.out.println(sdfDate.format(now) + " [" + className + "] " + message );
}
private void writeLogError(String message)
{
now = new Date();
System.out.println(sdfDate.format(now) + " [" + className + "] " + message );
System.err.println(sdfDate.format(now) + " [" + className + "] " + message );
}
}

54
src/main/java/com/airfrance/diqmqs/logparser/Parser.java
View File

@@ -1,7 +1,9 @@
package com.airfrance.diqmqs.logparser;
import java.util.Calendar;
import java.util.Map;
import java.util.Random;
import java.util.TimeZone;
import org.apache.commons.io.input.Tailer;
import org.apache.commons.io.input.TailerListenerAdapter;
@@ -53,6 +55,7 @@ public class Parser extends TailerListenerAdapter {
public void handle(String line) {
Match gm = grok.match(line);
TimeZone UTC = TimeZone.getTimeZone("UTC");
gm.captures();
Map<String,Object> result = gm.toMap();
if ( result.size() > 0 )
@@ -64,6 +67,8 @@ public class Parser extends TailerListenerAdapter {
value = null;
// iterate on all match pattern
boolean hasDate = false;
Calendar cal = Calendar.getInstance(UTC);
for (Map.Entry<String,Object> entry : result.entrySet() )
{
tagName = entry.getKey();
@@ -86,6 +91,43 @@ public class Parser extends TailerListenerAdapter {
}
}
}
try {
if ( tagName.equalsIgnoreCase("year") ) {
hasDate = true;
cal.set(Calendar.YEAR, Integer.parseInt(String.valueOf(entry.getValue()) ) );
}
if ( tagName.equalsIgnoreCase("mounth") ) {
hasDate = true;
cal.set(Calendar.MONTH, Integer.parseInt(String.valueOf(entry.getValue()) ) );
}
if ( tagName.equalsIgnoreCase("day") ) {
hasDate = true;
cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(String.valueOf(entry.getValue()) ) );
}
if ( tagName.equalsIgnoreCase("hour") ) {
hasDate = true;
cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(String.valueOf(entry.getValue()) ) );
}
if ( tagName.equalsIgnoreCase("minute") ) {
hasDate = true;
cal.set(Calendar.MINUTE, Integer.parseInt(String.valueOf(entry.getValue()) ) );
}
if ( tagName.equalsIgnoreCase("second") ) {
hasDate = true;
cal.set(Calendar.SECOND, Integer.parseInt(String.valueOf(entry.getValue()) ) );
}
if ( tagName.equalsIgnoreCase("msecond") ) {
hasDate = true;
cal.set(Calendar.MILLISECOND, Integer.parseInt(String.valueOf(entry.getValue()) ) );
}
}catch(Exception e)
{
log.error("Impossible de parser la date " + e.getMessage());
}
}
sb.append(" ");
//Field
@@ -98,9 +140,15 @@ public class Parser extends TailerListenerAdapter {
sb.append("value=1");
}
// TimeStamp
sb.append(" ")
.append(System.currentTimeMillis())
.append(String.format("%06d",randomGenerator.nextInt(999999)));
sb.append(" ");
if (hasDate){
sb.append(cal.getTimeInMillis());
}
else {
sb.append(System.currentTimeMillis());
}
// Time is in nano second from epoch
sb.append(String.format("%06d",randomGenerator.nextInt(999999)));
log.debug("Line => " + line + " : Match add it to Queue ");
// Add to the queue

4
src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java
View File

@@ -50,9 +50,9 @@ public class ParserEngine implements Runnable{
log.info("Schedule sender is set to " + 1 + " secs" );
}
void addNewParser(File f , String regexName, String application ) {
void addNewParser(File f , String regexName, String application , boolean readFromStart) {
TailerListener listener = new Parser(application, regexName, this);
Tailer tailer = new Tailer(f, listener, delay, true, true, 8192);
Tailer tailer = new Tailer(f, listener, delay, !readFromStart, true, 8192);
Thread thread = new Thread(tailer);
thread.setDaemon(true);
thread.setName("Parser - " + threadsParser.size() );