fred/logparser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Share Log Parser project

Browse Source
This commit is contained in:
Maxime Chassagneux
2016-03-23 15:43:44 +01:00
commit e2b46c3e8c
12 changed files with 1367 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
.classpath Normal file
View File

@@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/target/
/.settings/

23
.project Normal file
View File

@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>logParser</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

56
pom.xml Normal file
View File

@@ -0,0 +1,56 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.airfrance.diqmqs</groupId>
<artifactId>logParser</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>logParser</name>
<description>logParser - send data to influxdb</description>
<dependencies>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>io.thekraken</groupId>
<artifactId>grok</artifactId>
<version>0.1.3</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>com.airfrance.diqmqs.logparser.Cli</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

178
src/main/java/com/airfrance/diqmqs/logparser/Cli.java Normal file
View File

@@ -0,0 +1,178 @@
package com.airfrance.diqmqs.logparser;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.Scanner;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
public class Cli {
public final static String VER = "1.0";
public static Options options = new Options();
public static Option help = new Option( "help", "Print this message" );
public static Option version = new Option( "version", "Print the version information and exit" );
public static Option application = new Option( "application", true , "Input application name" );
public static Option patternFilePath = new Option( "pattern", true , "Input pattern path file" );
public static Option logFile = new Option( "logfile", true , "Input log path files" );
public static Option regex = new Option( "regex", true , "Name of the regex to apply" );
public static Option fileParam = new Option( "paramfile", true , "Input a param file" );
public static Option debugOption = new Option( "debug", "Active debug output message" );
private static long lastModifiedTime = 0L;
public Cli() {
// TODO Auto-generated constructor stub
}
public static void main(String[] args) {
application.setArgName("application name");
patternFilePath.setArgName("path to the pattern file");
logFile.setArgName("path to the logs file");
fileParam.setArgName("path to the param file");
options.addOption(help);
options.addOption(version);
options.addOption(application);
options.addOption(patternFilePath);
options.addOption(logFile);
options.addOption(regex);
options.addOption(debugOption);
options.addOption(fileParam);
HelpFormatter formatter = new HelpFormatter();
CommandLineParser parser = new DefaultParser();
CommandLine cmd = null;
try {
cmd = parser.parse( options, args);
}
catch (ParseException e) {
System.err.println("Command parse error : " + e.getMessage() );
System.exit(0);
}
if(cmd.hasOption("help")) {
formatter.printHelp( "logParser" , options );
System.exit(0);
}
if(cmd.hasOption("version")) {
System.out.println("logParser " + Cli.VER );
System.exit(0);
}
boolean debug = false;
if(cmd.hasOption("debug")) {
debug = true;
}
String patternPath = "";
if(cmd.hasOption("pattern")) {
patternPath = cmd.getOptionValue("pattern");
}
else
{
System.err.println("patternFilePath application missing");
System.exit(0);
}
if(cmd.hasOption("paramfile")) {
String fileParamName = cmd.getOptionValue("paramfile");
File param = new File(fileParamName);
ParserEngine engine = new ParserEngine(patternPath, debug);
try {
startParserFromFile(param, engine );
lastModifiedTime = param.lastModified();
if (debug) {
System.out.println("All parsers started");
}
while (true) {
try {
Thread.sleep(5000);
if (param.exists())
{
if ( param.lastModified() != lastModifiedTime )
{
engine.stopAllParser();
startParserFromFile(param, engine );
lastModifiedTime = param.lastModified();
}
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} catch (FileNotFoundException e) {
System.err.println("Execption : " + e.getMessage());
System.exit(0);
}
}
else
{
String applicationName = "";
if(cmd.hasOption("application")) {
applicationName = cmd.getOptionValue("application");
}
else
{
System.err.println("Option application missing");
System.exit(0);
}
String[] regexName = null;
if(cmd.hasOption("regex")) {
regexName = cmd.getOptionValues("regex");
}
else
{
System.err.println("regex application missing");
System.exit(0);
}
if(cmd.hasOption("logfile")) {
ParserEngine engine = new ParserEngine(patternPath, debug);
String[] filesName = cmd.getOptionValues("logfile");
for (int i = 0; i < filesName.length ; i++) {
File f = new File (filesName[i]);
engine.addNewParser(f, regexName[i], applicationName);
}
}
}
}
static void startParserFromFile(File file, ParserEngine engine) throws FileNotFoundException {
Scanner scanner = new Scanner(file);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (!line.startsWith("#"))
{
String[] arguments = line.split("\\t");
String applicationName = arguments[0].toLowerCase();
Paths paths = new Paths("/", arguments[1]);
String regexName = arguments[2];
for (File f : paths.getFiles()) {
engine.addNewParser(f, regexName, applicationName);
}
}
}
scanner.close();
}
}

271
src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java Normal file
View File

@@ -0,0 +1,271 @@
package com.airfrance.diqmqs.logparser;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.airfrance.diqmqs.logparser.GlobScanner.Pattern;
class GlobScanner {
private final File rootDir;
private final List<String> matches = new ArrayList<String>(128);
public GlobScanner (File rootDir, List<String> includes, List<String> excludes, boolean ignoreCase) {
if (rootDir == null) throw new IllegalArgumentException("rootDir cannot be null.");
if (!rootDir.exists()) throw new IllegalArgumentException("Directory does not exist: " + rootDir);
if (!rootDir.isDirectory()) throw new IllegalArgumentException("File must be a directory: " + rootDir);
try {
rootDir = rootDir.getCanonicalFile();
} catch (IOException ex) {
throw new RuntimeException("OS error determining canonical path: " + rootDir, ex);
}
this.rootDir = rootDir;
if (includes == null) throw new IllegalArgumentException("includes cannot be null.");
if (excludes == null) throw new IllegalArgumentException("excludes cannot be null.");
if (includes.isEmpty()) includes.add("**");
List<Pattern> includePatterns = new ArrayList<Pattern>(includes.size());
for (String include : includes)
includePatterns.add(new Pattern(include, ignoreCase));
List<Pattern> allExcludePatterns = new ArrayList(excludes.size());
for (String exclude : excludes)
allExcludePatterns.add(new Pattern(exclude, ignoreCase));
scanDir(rootDir, includePatterns);
if (!allExcludePatterns.isEmpty()) {
// For each file, see if any exclude patterns match.
outerLoop:
//
for (Iterator matchIter = matches.iterator(); matchIter.hasNext();) {
String filePath = (String)matchIter.next();
List<Pattern> excludePatterns = new ArrayList(allExcludePatterns);
try {
// Shortcut for excludes that are "**/XXX", just check file name.
for (Iterator excludeIter = excludePatterns.iterator(); excludeIter.hasNext();) {
Pattern exclude = (Pattern)excludeIter.next();
if (exclude.values.length == 2 && exclude.values[0].equals("**")) {
exclude.incr();
String fileName = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1);
if (exclude.matches(fileName)) {
matchIter.remove();
continue outerLoop;
}
excludeIter.remove();
}
}
// Get the file names after the root dir.
String[] fileNames = filePath.split("\\" + File.separator);
for (String fileName : fileNames) {
for (Iterator excludeIter = excludePatterns.iterator(); excludeIter.hasNext();) {
Pattern exclude = (Pattern)excludeIter.next();
if (!exclude.matches(fileName)) {
excludeIter.remove();
continue;
}
exclude.incr(fileName);
if (exclude.wasFinalMatch()) {
// Exclude pattern matched.
matchIter.remove();
continue outerLoop;
}
}
// Stop processing the file if none of the exclude patterns matched.
if (excludePatterns.isEmpty()) continue outerLoop;
}
} finally {
for (Pattern exclude : allExcludePatterns)
exclude.reset();
}
}
}
}
private void scanDir (File dir, List<Pattern> includes) {
if (!dir.canRead()) return;
// See if patterns are specific enough to avoid scanning every file in the directory.
boolean scanAll = false;
for (Pattern include : includes) {
if (include.value.indexOf('*') != -1 || include.value.indexOf('?') != -1) {
scanAll = true;
break;
}
}
if (!scanAll) {
// If not scanning all the files, we know exactly which ones to include.
List matchingIncludes = new ArrayList(1);
for (Pattern include : includes) {
if (matchingIncludes.isEmpty())
matchingIncludes.add(include);
else
matchingIncludes.set(0, include);
process(dir, include.value, matchingIncludes);
}
} else {
// Scan every file.
for (String fileName : dir.list()) {
// Get all include patterns that match.
List<Pattern> matchingIncludes = new ArrayList(includes.size());
for (Pattern include : includes)
if (include.matches(fileName)) matchingIncludes.add(include);
if (matchingIncludes.isEmpty()) continue;
process(dir, fileName, matchingIncludes);
}
}
}
private void process (File dir, String fileName, List<Pattern> matchingIncludes) {
// Increment patterns that need to move to the next token.
boolean isFinalMatch = false;
List<Pattern> incrementedPatterns = new ArrayList();
for (Iterator iter = matchingIncludes.iterator(); iter.hasNext();) {
Pattern include = (Pattern)iter.next();
if (include.incr(fileName)) {
incrementedPatterns.add(include);
if (include.isExhausted()) iter.remove();
}
if (include.wasFinalMatch()) isFinalMatch = true;
}
File file = new File(dir, fileName);
if (isFinalMatch) {
int length = rootDir.getPath().length();
if (!rootDir.getPath().endsWith(File.separator)) length++; // Lose starting slash.
matches.add(file.getPath().substring(length));
}
if (!matchingIncludes.isEmpty() && file.isDirectory()) scanDir(file, matchingIncludes);
// Decrement patterns.
for (Pattern include : incrementedPatterns)
include.decr();
}
public List<String> matches () {
return matches;
}
public File rootDir () {
return rootDir;
}
static class Pattern {
String value;
boolean ignoreCase;
final String[] values;
private int index;
Pattern (String pattern, boolean ignoreCase) {
this.ignoreCase = ignoreCase;
pattern = pattern.replace('\\', '/');
pattern = pattern.replaceAll("\\*\\*[^/]", "**/*");
pattern = pattern.replaceAll("[^/]\\*\\*", "*/**");
if (ignoreCase) pattern = pattern.toLowerCase();
values = pattern.split("/");
value = values[0];
}
boolean matches (String fileName) {
if (value.equals("**")) return true;
if (ignoreCase) fileName = fileName.toLowerCase();
// Shortcut if no wildcards.
if (value.indexOf('*') == -1 && value.indexOf('?') == -1) return fileName.equals(value);
int i = 0, j = 0;
while (i < fileName.length() && j < value.length() && value.charAt(j) != '*') {
if (value.charAt(j) != fileName.charAt(i) && value.charAt(j) != '?') return false;
i++;
j++;
}
// If reached end of pattern without finding a * wildcard, the match has to fail if not same length.
if (j == value.length()) return fileName.length() == value.length();
int cp = 0;
int mp = 0;
while (i < fileName.length()) {
if (j < value.length() && value.charAt(j) == '*') {
if (j++ >= value.length()) return true;
mp = j;
cp = i + 1;
} else if (j < value.length() && (value.charAt(j) == fileName.charAt(i) || value.charAt(j) == '?')) {
j++;
i++;
} else {
j = mp;
i = cp++;
}
}
// Handle trailing asterisks.
while (j < value.length() && value.charAt(j) == '*')
j++;
return j >= value.length();
}
String nextValue () {
if (index + 1 == values.length) return null;
return values[index + 1];
}
boolean incr (String fileName) {
if (value.equals("**")) {
if (index == values.length - 1) return false;
incr();
if (matches(fileName))
incr();
else {
decr();
return false;
}
} else
incr();
return true;
}
void incr () {
index++;
if (index >= values.length)
value = null;
else
value = values[index];
}
void decr () {
index--;
if (index > 0 && values[index - 1].equals("**")) index--;
value = values[index];
}
void reset () {
index = 0;
value = values[0];
}
boolean isExhausted () {
return index >= values.length;
}
boolean isLast () {
return index >= values.length - 1;
}
boolean wasFinalMatch () {
return isExhausted() || (isLast() && value.equals("**"));
}
}
}

127
src/main/java/com/airfrance/diqmqs/logparser/Parser.java Normal file
View File

@@ -0,0 +1,127 @@
package com.airfrance.diqmqs.logparser;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Map;
import org.apache.commons.io.input.Tailer;
import org.apache.commons.io.input.TailerListenerAdapter;
import org.apache.commons.lang3.StringUtils;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import oi.thekraken.grok.api.exception.GrokException;
public class Parser extends TailerListenerAdapter {
private Grok grok = null;
private String fileName = "";
private String pathName = "";
private String hostname = "";
private String application = "";
private ParserEngine engine = null;
public Parser(String application, String regexName, ParserEngine engine) {
try {
if (engine.isDebug()) {
System.out.println("Create Grok with regex name : " + regexName);
}
grok = Grok.create(engine.getPatternPath());
grok.compile("%{" + regexName + "}");
} catch (GrokException e) {
e.printStackTrace();
}
try {
hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0];
} catch (UnknownHostException e) {
e.printStackTrace();
}
this.application = application;
this.engine = engine;
}
@Override
public void init(Tailer tailer) {
super.init(tailer);
fileName = tailer.getFile().getName();
pathName = tailer.getFile().getParent();
if (engine.isDebug()) {
System.out.println("Init tailer on file : " + pathName + "/" + fileName );
}
}
public void handle(String line) {
Match gm = grok.match(line);
gm.captures();
Map<String,Object> result = gm.toMap();
if ( result.size() > 0 )
{
if (engine.isDebug()) {
System.out.println("Match OK ");
}
StringBuffer sb = new StringBuffer();
sb.append("log");
//Tag
sb.append(",host=" + hostname);
sb.append(",application=" + application);
sb.append(",file=" + sanitizeString(fileName) );
sb.append(",path=" + sanitizeString(pathName) );
String tagName = null;
String tagValue = null;
String value = null;
// iterate on all match pattern
for (Map.Entry<String,Object> entry : result.entrySet() )
{
tagName = entry.getKey();
// Keep only key with lowercase
if ( StringUtils.isAllLowerCase( tagName ) )
{
tagValue = entry.getValue().toString();
// if key name is 'value', it's not a tag, but a field
if ( ! tagName.equalsIgnoreCase("value") ) {
// check if tag value is not empty
if (!tagValue.equalsIgnoreCase("") ) {
sb.append("," + tagName + "=" + sanitizeString( tagValue ) );
}
}
else
{
value = tagValue;
}
}
}
sb.append(" ");
//Field
if ( StringUtils.isNumeric(value) ) {
sb.append("value=" + value);
}
else
{
sb.append("value=1");
}
sb.append(" " + System.currentTimeMillis() + "000000");
// Add to the queue
engine.addToQueue(sb.toString());
if (engine.isDebug()) {
System.out.println(gm.toJson());
}
}
}
String sanitizeString(String s)
{
return s.trim().replaceAll(" ", "\\\\ ").replaceAll(",", "\\\\,").replaceAll("=", "\\\\=");
}
}

155
src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java Normal file
View File

@@ -0,0 +1,155 @@
package com.airfrance.diqmqs.logparser;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.input.Tailer;
import org.apache.commons.io.input.TailerListener;
public class ParserEngine implements Runnable{
private static final long ONE_SECOND = 1L;
private static final int BATCH_SIZE_MAX = 10;
final private String database = "qualif";
final long delay = 500;
private ArrayList<Thread> threadsParser = null;
static BlockingDeque<String> queue = new LinkedBlockingDeque<String>(1000);
private boolean debug = false;
private String patternPath = "";
private ScheduledExecutorService scheduler;
@SuppressWarnings("unused")
private ScheduledFuture<?> timerHandle;
public ParserEngine(String patternPath, boolean debug) {
threadsParser = new ArrayList<Thread>();
this.debug = debug;
this.patternPath = patternPath;
scheduler = Executors.newScheduledThreadPool(1);
// Don't change this as metrics are per second
this.timerHandle = scheduler.scheduleAtFixedRate( this, ONE_SECOND, ONE_SECOND, TimeUnit.SECONDS);
if (debug) {
System.out.println("Create ParserEngine with pattern file : " + patternPath);
System.out.println("Schedule sender is set to " + ONE_SECOND + " second(s)");
}
}
void addNewParser(File f , String regexName, String application ) {
TailerListener listener = new Parser(application, regexName, this);
Tailer tailer = new Tailer(f, listener, delay, true);
Thread thread = new Thread(tailer);
thread.setDaemon(true);
thread.setName("Parser - " + threadsParser.size() );
thread.start();
threadsParser.add(thread);
if (debug) {
System.out.println("Thread Parser - " + threadsParser.size() + " started - file : " + f.getName());
}
}
void stopAllParser() {
for (Thread t : threadsParser) {
t.interrupt();
if (debug) {
System.out.println("Stop Thread " + t.getName() );
}
}
threadsParser.clear();
}
// Read data from the Queue and send it to influxdb
public void run() {
StringBuilder postData = new StringBuilder();
for (int i = 0 ; i < BATCH_SIZE_MAX; i++)
{
try {
String message = queue.poll( 50, TimeUnit.MILLISECONDS);
if (message != null) {
postData.append( message + "\n" );
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
try {
if (postData.length() > 0) {
sendMetricToInfluxdb(postData.toString());
}
} catch (IOException e) {
e.printStackTrace();
}
}
public boolean sendMetricToInfluxdb(String postData) throws IOException {
URL url = new URL("http://diqmqs.airfrance.fr/influxdb_query/write?rp=one_week&db=" + database);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true);
connection.setRequestMethod("POST");
connection.setRequestProperty("User-Agent", "LogParser/1.0");
connection.setRequestProperty("Authorization", "Basic cXVhbGlmOmRpcW0wMQ==");
connection.setRequestProperty("Content-Length", String.valueOf(postData.length()));
// Write data
OutputStream os = connection.getOutputStream();
if (this.isDebug()) {
System.out.println("Send : " + postData );
}
os.write(postData.getBytes());
int HttpResult = connection.getResponseCode();
if (HttpResult == 204) {
return true;
} else {
if (this.isDebug()) {
System.err.println(connection.getResponseCode() + " " + connection.getResponseMessage());
}
return false;
}
}
public void addToQueue(String s) {
try {
queue.put(s);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public boolean isDebug() {
return debug;
}
public void setDebug(boolean debug) {
this.debug = debug;
}
public String getPatternPath() {
return patternPath;
}
public void setPatternPath(String patternPath) {
this.patternPath = patternPath;
}
}

304
src/main/java/com/airfrance/diqmqs/logparser/Paths.java Normal file
View File

@@ -0,0 +1,304 @@
package com.airfrance.diqmqs.logparser;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
/** Collects filesystem paths using wildcards, preserving the directory structure. Copies, deletes, and zips paths. */
public class Paths implements Iterable<String> {
static private List<String> defaultGlobExcludes;
final HashSet<Path> paths = new HashSet<Path>(32);
/** Creates an empty Paths object. */
public Paths () {
}
/** Creates a Paths object and calls {@link #glob(String, String[])} with the specified arguments. */
public Paths (String dir, String... patterns) {
glob(dir, patterns);
}
/** Creates a Paths object and calls {@link #glob(String, List)} with the specified arguments. */
public Paths (String dir, List<String> patterns) {
glob(dir, patterns);
}
private Paths glob (String dir, boolean ignoreCase, String... patterns) {
if (dir == null) dir = ".";
if (patterns != null && patterns.length == 0) {
String[] split = dir.split("\\|");
if (split.length > 1) {
dir = split[0];
patterns = new String[split.length - 1];
for (int i = 1, n = split.length; i < n; i++)
patterns[i - 1] = split[i];
}
}
File dirFile = new File(dir);
if (!dirFile.exists()) return this;
List<String> includes = new ArrayList<String>();
List<String> excludes = new ArrayList<String>();
if (patterns != null) {
for (String pattern : patterns) {
if (pattern.charAt(0) == '!')
excludes.add(pattern.substring(1));
else
includes.add(pattern);
}
}
if (includes.isEmpty()) includes.add("**");
if (defaultGlobExcludes != null) excludes.addAll(defaultGlobExcludes);
GlobScanner scanner = new GlobScanner(dirFile, includes, excludes, ignoreCase);
String rootDir = scanner.rootDir().getPath().replace('\\', '/');
if (!rootDir.endsWith("/")) rootDir += '/';
for (String filePath : scanner.matches())
paths.add(new Path(rootDir, filePath));
return this;
}
/** Collects all files and directories in the specified directory matching the wildcard patterns.
* @param dir The directory containing the paths to collect. If it does not exist, no paths are collected. If null, "." is
* assumed.
* @param patterns The wildcard patterns of the paths to collect or exclude. Patterns may optionally contain wildcards
* represented by asterisks and question marks. If empty or omitted then the dir parameter is split on the "|"
* character, the first element is used as the directory and remaining are used as the patterns. If null, ** is
* assumed (collects all paths).<br>
* <br>
* A single question mark (?) matches any single character. Eg, something? collects any path that is named
* "something" plus any character.<br>
* <br>
* A single asterisk (*) matches any characters up to the next slash (/). Eg, *\*\something* collects any path that
* has two directories of any name, then a file or directory that starts with the name "something".<br>
* <br>
* A double asterisk (**) matches any characters. Eg, **\something\** collects any path that contains a directory
* named "something".<br>
* <br>
* A pattern starting with an exclamation point (!) causes paths matched by the pattern to be excluded, even if other
* patterns would select the paths. */
public Paths glob (String dir, String... patterns) {
return glob(dir, false, patterns);
}
/** Case insensitive glob.
* @see #glob(String, String...) */
public Paths globIgnoreCase (String dir, String... patterns) {
return glob(dir, true, patterns);
}
/** Case sensitive glob.
* @see #glob(String, String...) */
public Paths glob (String dir, List<String> patterns) {
if (patterns == null) throw new IllegalArgumentException("patterns cannot be null.");
glob(dir, false, patterns.toArray(new String[patterns.size()]));
return this;
}
/** Case insensitive glob.
* @see #glob(String, String...) */
public Paths globIgnoreCase (String dir, List<String> patterns) {
if (patterns == null) throw new IllegalArgumentException("patterns cannot be null.");
glob(dir, true, patterns.toArray(new String[patterns.size()]));
return this;
}
public int count () {
return paths.size();
}
public boolean isEmpty () {
return paths.isEmpty();
}
/** Returns the absolute paths delimited by the specified character. */
public String toString (String delimiter) {
StringBuffer buffer = new StringBuffer(256);
for (String path : getPaths()) {
if (buffer.length() > 0) buffer.append(delimiter);
buffer.append(path);
}
return buffer.toString();
}
/** Returns the absolute paths delimited by commas. */
public String toString () {
return toString(", ");
}
/** Returns a Paths object containing the paths that are files, as if each file were selected from its parent directory. */
public Paths flatten () {
Paths newPaths = new Paths();
for (Path path : paths) {
File file = path.file();
if (file.isFile()) newPaths.paths.add(new Path(file.getParent(), file.getName()));
}
return newPaths;
}
/** Returns a Paths object containing the paths that are files. */
public Paths filesOnly () {
Paths newPaths = new Paths();
for (Path path : paths) {
if (path.file().isFile()) newPaths.paths.add(path);
}
return newPaths;
}
/** Returns a Paths object containing the paths that are directories. */
public Paths dirsOnly () {
Paths newPaths = new Paths();
for (Path path : paths) {
if (path.file().isDirectory()) newPaths.paths.add(path);
}
return newPaths;
}
/** Returns the paths as File objects. */
public List<File> getFiles () {
return getFiles(new ArrayList<Path>(paths));
}
private ArrayList<File> getFiles (List<Path> paths) {
ArrayList<File> files = new ArrayList<File>(paths.size());
for (Path path : paths)
files.add(path.file());
return files;
}
/** Returns the portion of the path after the root directory where the path was collected. */
public List<String> getRelativePaths () {
ArrayList<String> stringPaths = new ArrayList<String>(paths.size());
for (Path path : paths)
stringPaths.add(path.name);
return stringPaths;
}
/** Returns the full paths. */
public List<String> getPaths () {
ArrayList<String> stringPaths = new ArrayList<String>(paths.size());
for (File file : getFiles())
stringPaths.add(file.getPath());
return stringPaths;
}
/** Returns the paths' filenames. */
public List<String> getNames () {
ArrayList<String> stringPaths = new ArrayList<String>(paths.size());
for (File file : getFiles())
stringPaths.add(file.getName());
return stringPaths;
}
/** Adds a single path to this Paths object. */
public Paths addFile (String fullPath) {
File file = new File(fullPath);
String parent = file.getParent();
paths.add(new Path(parent == null ? "" : parent, file.getName()));
return this;
}
/** Adds a single path to this Paths object. */
public Paths add (String dir, String name) {
paths.add(new Path(dir, name));
return this;
}
/** Adds all paths from the specified Paths object to this Paths object. */
public void add (Paths paths) {
this.paths.addAll(paths.paths);
}
/** Iterates over the absolute paths. The iterator supports the remove method. */
public Iterator<String> iterator () {
return new Iterator<String>() {
private Iterator<Path> iter = paths.iterator();
public void remove () {
iter.remove();
}
public String next () {
return iter.next().absolute();
}
public boolean hasNext () {
return iter.hasNext();
}
};
}
/** Iterates over the paths as File objects. The iterator supports the remove method. */
public Iterator<File> fileIterator () {
return new Iterator<File>() {
private Iterator<Path> iter = paths.iterator();
public void remove () {
iter.remove();
}
public File next () {
return iter.next().file();
}
public boolean hasNext () {
return iter.hasNext();
}
};
}
static private final class Path {
public final String dir;
public final String name;
public Path (String dir, String name) {
if (dir.length() > 0 && !dir.endsWith("/")) dir += "/";
this.dir = dir;
this.name = name;
}
public String absolute () {
return dir + name;
}
public File file () {
return new File(dir, name);
}
public int hashCode () {
final int prime = 31;
int result = 1;
result = prime * result + ((dir == null) ? 0 : dir.hashCode());
result = prime * result + ((name == null) ? 0 : name.hashCode());
return result;
}
public boolean equals (Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
Path other = (Path)obj;
if (dir == null) {
if (other.dir != null) return false;
} else if (!dir.equals(other.dir)) return false;
if (name == null) {
if (other.name != null) return false;
} else if (!name.equals(other.name)) return false;
return true;
}
}
/** Sets the exclude patterns that will be used in addition to the excludes specified for all glob searches. */
static public void setDefaultGlobExcludes (String... defaultGlobExcludes) {
Paths.defaultGlobExcludes = Arrays.asList(defaultGlobExcludes);
}
}

8
src/main/java/com/airfrance/diqmqs/logparser/package-info.java Normal file
View File

@@ -0,0 +1,8 @@
/**
*
*/
/**
* @author 4163013
*
*/
package com.airfrance.diqmqs.logparser;

107
src/main/java/com/airfrance/diqmqs/logparser/patterns Normal file
View File

@@ -0,0 +1,107 @@
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME:UNWANTED}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM:UNWANTED})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# Networking
MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
HOST %{HOSTNAME:UNWANTED}
IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED})
HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT})
# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+
#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
# Months: January, Feb, 3, 03, 12, December
MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
# Years?
YEAR (?>\d\d){1,2}
# Time: HH:MM:SS
#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
# I'm still on the fence about using grok to perform the time match,
# since it's probably slower.
# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[PMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG (?:[\w._/%-]+)
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
# Shortcuts
QS %{QUOTEDSTRING:UNWANTED}
# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
MESSAGESLOG %{SYSLOGBASE} %{DATA}
COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
COMMONAPACHELOG_DATATYPED %{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth} \[%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}\] "(?:%{WORD:verb;string} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})" %{NUMBER:response;int} (?:%{NUMBER:bytes;long}|-)
# Log Levels
LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

111
src/main/resources/patterns Normal file
View File

@@ -0,0 +1,111 @@
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME:UNWANTED}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM:UNWANTED})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# Networking
MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
HOST %{HOSTNAME:UNWANTED}
IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED})
HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT})
# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+
#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
# Months: January, Feb, 3, 03, 12, December
MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
# Years?
YEAR (?>\d\d){1,2}
# Time: HH:MM:SS
#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
# I'm still on the fence about using grok to perform the time match,
# since it's probably slower.
# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
DATE_JMETER %{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[PMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG (?:[\w._/%-]+)
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
# Shortcuts
QS %{QUOTEDSTRING:UNWANTED}
# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
MESSAGESLOG %{SYSLOGBASE} %{DATA}
TYPE [E|e]rr?(?:or)?|ERR?(?:OR)?
MESSAGE .*
COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
COMMONAPACHELOG_DATATYPED %{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth} \[%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}\] "(?:%{WORD:verb;string} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})" %{NUMBER:response;int} (?:%{NUMBER:bytes;long}|-)
JMETERLOG %{DATE_JMETER} %{TIME} %{TYPE} [- ] %{MESSAGE}
# Log Levels
LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)