From e2b46c3e8c86fee3d55ec354d120cefc2616a587 Mon Sep 17 00:00:00 2001 From: Maxime Chassagneux <4163013@airfrance.fr> Date: Wed, 23 Mar 2016 15:43:44 +0100 Subject: [PATCH] Share Log Parser project --- .classpath | 25 ++ .gitignore | 2 + .project | 23 ++ pom.xml | 56 ++++ .../com/airfrance/diqmqs/logparser/Cli.java | 178 ++++++++++ .../diqmqs/logparser/GlobScanner.java | 271 ++++++++++++++++ .../airfrance/diqmqs/logparser/Parser.java | 127 ++++++++ .../diqmqs/logparser/ParserEngine.java | 155 +++++++++ .../com/airfrance/diqmqs/logparser/Paths.java | 304 ++++++++++++++++++ .../diqmqs/logparser/package-info.java | 8 + .../com/airfrance/diqmqs/logparser/patterns | 107 ++++++ src/main/resources/patterns | 111 +++++++ 12 files changed, 1367 insertions(+) create mode 100644 .classpath create mode 100644 .gitignore create mode 100644 .project create mode 100644 pom.xml create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/Cli.java create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/Parser.java create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/Paths.java create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/package-info.java create mode 100644 src/main/java/com/airfrance/diqmqs/logparser/patterns create mode 100644 src/main/resources/patterns diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..c9ef8f9 --- /dev/null +++ b/.classpath @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..731eb43 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target/ +/.settings/ diff --git a/.project b/.project new file mode 100644 index 0000000..b9b397c --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + logParser + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..94d41eb --- /dev/null +++ b/pom.xml @@ -0,0 +1,56 @@ + + 4.0.0 + com.airfrance.diqmqs + logParser + 0.0.1-SNAPSHOT + logParser + logParser - send data to influxdb + + + + commons-cli + commons-cli + 1.3.1 + + + io.thekraken + grok + 0.1.3 + + + commons-io + commons-io + 2.4 + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + + com.airfrance.diqmqs.logparser.Cli + + + + + + make-assembly + package + + single + + + + + + + + diff --git a/src/main/java/com/airfrance/diqmqs/logparser/Cli.java b/src/main/java/com/airfrance/diqmqs/logparser/Cli.java new file mode 100644 index 0000000..c67bf9d --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/Cli.java @@ -0,0 +1,178 @@ +package com.airfrance.diqmqs.logparser; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.Scanner; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +public class Cli { + + public final static String VER = "1.0"; + public static Options options = new Options(); + public static Option help = new Option( "help", "Print this message" ); + public static Option version = new Option( "version", "Print the version information and exit" ); + public static Option application = new Option( "application", true , "Input application name" ); + public static Option patternFilePath = new Option( "pattern", true , "Input pattern path file" ); + public static Option logFile = new Option( "logfile", true , "Input log path files" ); + public static Option regex = new Option( "regex", true , "Name of the regex to apply" ); + public static Option fileParam = new Option( "paramfile", true , "Input a param file" ); + public static Option debugOption = new Option( "debug", "Active debug output message" ); + private static long lastModifiedTime = 0L; + public Cli() { + // TODO Auto-generated constructor stub + } + + public static void main(String[] args) { + + + application.setArgName("application name"); + patternFilePath.setArgName("path to the pattern file"); + logFile.setArgName("path to the logs file"); + fileParam.setArgName("path to the param file"); + + options.addOption(help); + options.addOption(version); + options.addOption(application); + options.addOption(patternFilePath); + options.addOption(logFile); + options.addOption(regex); + options.addOption(debugOption); + options.addOption(fileParam); + + HelpFormatter formatter = new HelpFormatter(); + CommandLineParser parser = new DefaultParser(); + CommandLine cmd = null; + try { + cmd = parser.parse( options, args); + } + catch (ParseException e) { + System.err.println("Command parse error : " + e.getMessage() ); + System.exit(0); + } + + + if(cmd.hasOption("help")) { + formatter.printHelp( "logParser" , options ); + System.exit(0); + } + + if(cmd.hasOption("version")) { + System.out.println("logParser " + Cli.VER ); + System.exit(0); + } + + boolean debug = false; + if(cmd.hasOption("debug")) { + debug = true; + } + + String patternPath = ""; + if(cmd.hasOption("pattern")) { + patternPath = cmd.getOptionValue("pattern"); + } + else + { + System.err.println("patternFilePath application missing"); + System.exit(0); + } + + if(cmd.hasOption("paramfile")) { + String fileParamName = cmd.getOptionValue("paramfile"); + File param = new File(fileParamName); + ParserEngine engine = new ParserEngine(patternPath, debug); + try { + startParserFromFile(param, engine ); + lastModifiedTime = param.lastModified(); + if (debug) { + System.out.println("All parsers started"); + } + while (true) { + try { + Thread.sleep(5000); + if (param.exists()) + { + if ( param.lastModified() != lastModifiedTime ) + { + engine.stopAllParser(); + startParserFromFile(param, engine ); + lastModifiedTime = param.lastModified(); + } + } + + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + } catch (FileNotFoundException e) { + System.err.println("Execption : " + e.getMessage()); + System.exit(0); + } + } + else + { + String applicationName = ""; + if(cmd.hasOption("application")) { + applicationName = cmd.getOptionValue("application"); + } + else + { + System.err.println("Option application missing"); + System.exit(0); + } + + String[] regexName = null; + if(cmd.hasOption("regex")) { + regexName = cmd.getOptionValues("regex"); + } + else + { + System.err.println("regex application missing"); + System.exit(0); + } + + if(cmd.hasOption("logfile")) { + + ParserEngine engine = new ParserEngine(patternPath, debug); + String[] filesName = cmd.getOptionValues("logfile"); + for (int i = 0; i < filesName.length ; i++) { + File f = new File (filesName[i]); + engine.addNewParser(f, regexName[i], applicationName); + } + + + } + } + + } + + + static void startParserFromFile(File file, ParserEngine engine) throws FileNotFoundException { + + Scanner scanner = new Scanner(file); + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + if (!line.startsWith("#")) + { + String[] arguments = line.split("\\t"); + String applicationName = arguments[0].toLowerCase(); + Paths paths = new Paths("/", arguments[1]); + String regexName = arguments[2]; + for (File f : paths.getFiles()) { + engine.addNewParser(f, regexName, applicationName); + } + + } + } + scanner.close(); + + } + +} diff --git a/src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java b/src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java new file mode 100644 index 0000000..13c0e2f --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/GlobScanner.java @@ -0,0 +1,271 @@ +package com.airfrance.diqmqs.logparser; + + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import com.airfrance.diqmqs.logparser.GlobScanner.Pattern; + +class GlobScanner { + private final File rootDir; + private final List matches = new ArrayList(128); + + public GlobScanner (File rootDir, List includes, List excludes, boolean ignoreCase) { + if (rootDir == null) throw new IllegalArgumentException("rootDir cannot be null."); + if (!rootDir.exists()) throw new IllegalArgumentException("Directory does not exist: " + rootDir); + if (!rootDir.isDirectory()) throw new IllegalArgumentException("File must be a directory: " + rootDir); + try { + rootDir = rootDir.getCanonicalFile(); + } catch (IOException ex) { + throw new RuntimeException("OS error determining canonical path: " + rootDir, ex); + } + this.rootDir = rootDir; + + if (includes == null) throw new IllegalArgumentException("includes cannot be null."); + if (excludes == null) throw new IllegalArgumentException("excludes cannot be null."); + + if (includes.isEmpty()) includes.add("**"); + List includePatterns = new ArrayList(includes.size()); + for (String include : includes) + includePatterns.add(new Pattern(include, ignoreCase)); + + List allExcludePatterns = new ArrayList(excludes.size()); + for (String exclude : excludes) + allExcludePatterns.add(new Pattern(exclude, ignoreCase)); + + scanDir(rootDir, includePatterns); + + if (!allExcludePatterns.isEmpty()) { + // For each file, see if any exclude patterns match. + outerLoop: + // + for (Iterator matchIter = matches.iterator(); matchIter.hasNext();) { + String filePath = (String)matchIter.next(); + List excludePatterns = new ArrayList(allExcludePatterns); + try { + // Shortcut for excludes that are "**/XXX", just check file name. + for (Iterator excludeIter = excludePatterns.iterator(); excludeIter.hasNext();) { + Pattern exclude = (Pattern)excludeIter.next(); + if (exclude.values.length == 2 && exclude.values[0].equals("**")) { + exclude.incr(); + String fileName = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1); + if (exclude.matches(fileName)) { + matchIter.remove(); + continue outerLoop; + } + excludeIter.remove(); + } + } + // Get the file names after the root dir. + String[] fileNames = filePath.split("\\" + File.separator); + for (String fileName : fileNames) { + for (Iterator excludeIter = excludePatterns.iterator(); excludeIter.hasNext();) { + Pattern exclude = (Pattern)excludeIter.next(); + if (!exclude.matches(fileName)) { + excludeIter.remove(); + continue; + } + exclude.incr(fileName); + if (exclude.wasFinalMatch()) { + // Exclude pattern matched. + matchIter.remove(); + continue outerLoop; + } + } + // Stop processing the file if none of the exclude patterns matched. + if (excludePatterns.isEmpty()) continue outerLoop; + } + } finally { + for (Pattern exclude : allExcludePatterns) + exclude.reset(); + } + } + } + } + + private void scanDir (File dir, List includes) { + if (!dir.canRead()) return; + + // See if patterns are specific enough to avoid scanning every file in the directory. + boolean scanAll = false; + for (Pattern include : includes) { + if (include.value.indexOf('*') != -1 || include.value.indexOf('?') != -1) { + scanAll = true; + break; + } + } + + if (!scanAll) { + // If not scanning all the files, we know exactly which ones to include. + List matchingIncludes = new ArrayList(1); + for (Pattern include : includes) { + if (matchingIncludes.isEmpty()) + matchingIncludes.add(include); + else + matchingIncludes.set(0, include); + process(dir, include.value, matchingIncludes); + } + } else { + // Scan every file. + for (String fileName : dir.list()) { + // Get all include patterns that match. + List matchingIncludes = new ArrayList(includes.size()); + for (Pattern include : includes) + if (include.matches(fileName)) matchingIncludes.add(include); + if (matchingIncludes.isEmpty()) continue; + process(dir, fileName, matchingIncludes); + } + } + } + + private void process (File dir, String fileName, List matchingIncludes) { + // Increment patterns that need to move to the next token. + boolean isFinalMatch = false; + List incrementedPatterns = new ArrayList(); + for (Iterator iter = matchingIncludes.iterator(); iter.hasNext();) { + Pattern include = (Pattern)iter.next(); + if (include.incr(fileName)) { + incrementedPatterns.add(include); + if (include.isExhausted()) iter.remove(); + } + if (include.wasFinalMatch()) isFinalMatch = true; + } + + File file = new File(dir, fileName); + if (isFinalMatch) { + int length = rootDir.getPath().length(); + if (!rootDir.getPath().endsWith(File.separator)) length++; // Lose starting slash. + matches.add(file.getPath().substring(length)); + } + if (!matchingIncludes.isEmpty() && file.isDirectory()) scanDir(file, matchingIncludes); + + // Decrement patterns. + for (Pattern include : incrementedPatterns) + include.decr(); + } + + public List matches () { + return matches; + } + + public File rootDir () { + return rootDir; + } + + static class Pattern { + String value; + boolean ignoreCase; + final String[] values; + + private int index; + + Pattern (String pattern, boolean ignoreCase) { + this.ignoreCase = ignoreCase; + + pattern = pattern.replace('\\', '/'); + pattern = pattern.replaceAll("\\*\\*[^/]", "**/*"); + pattern = pattern.replaceAll("[^/]\\*\\*", "*/**"); + if (ignoreCase) pattern = pattern.toLowerCase(); + + values = pattern.split("/"); + value = values[0]; + } + + boolean matches (String fileName) { + if (value.equals("**")) return true; + + if (ignoreCase) fileName = fileName.toLowerCase(); + + // Shortcut if no wildcards. + if (value.indexOf('*') == -1 && value.indexOf('?') == -1) return fileName.equals(value); + + int i = 0, j = 0; + while (i < fileName.length() && j < value.length() && value.charAt(j) != '*') { + if (value.charAt(j) != fileName.charAt(i) && value.charAt(j) != '?') return false; + i++; + j++; + } + + // If reached end of pattern without finding a * wildcard, the match has to fail if not same length. + if (j == value.length()) return fileName.length() == value.length(); + + int cp = 0; + int mp = 0; + while (i < fileName.length()) { + if (j < value.length() && value.charAt(j) == '*') { + if (j++ >= value.length()) return true; + mp = j; + cp = i + 1; + } else if (j < value.length() && (value.charAt(j) == fileName.charAt(i) || value.charAt(j) == '?')) { + j++; + i++; + } else { + j = mp; + i = cp++; + } + } + + // Handle trailing asterisks. + while (j < value.length() && value.charAt(j) == '*') + j++; + + return j >= value.length(); + } + + String nextValue () { + if (index + 1 == values.length) return null; + return values[index + 1]; + } + + boolean incr (String fileName) { + if (value.equals("**")) { + if (index == values.length - 1) return false; + incr(); + if (matches(fileName)) + incr(); + else { + decr(); + return false; + } + } else + incr(); + return true; + } + + void incr () { + index++; + if (index >= values.length) + value = null; + else + value = values[index]; + } + + void decr () { + index--; + if (index > 0 && values[index - 1].equals("**")) index--; + value = values[index]; + } + + void reset () { + index = 0; + value = values[0]; + } + + boolean isExhausted () { + return index >= values.length; + } + + boolean isLast () { + return index >= values.length - 1; + } + + boolean wasFinalMatch () { + return isExhausted() || (isLast() && value.equals("**")); + } + } + + +} diff --git a/src/main/java/com/airfrance/diqmqs/logparser/Parser.java b/src/main/java/com/airfrance/diqmqs/logparser/Parser.java new file mode 100644 index 0000000..596af83 --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/Parser.java @@ -0,0 +1,127 @@ +package com.airfrance.diqmqs.logparser; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; + +import org.apache.commons.io.input.Tailer; +import org.apache.commons.io.input.TailerListenerAdapter; +import org.apache.commons.lang3.StringUtils; + +import oi.thekraken.grok.api.Grok; +import oi.thekraken.grok.api.Match; +import oi.thekraken.grok.api.exception.GrokException; + +public class Parser extends TailerListenerAdapter { + + private Grok grok = null; + + private String fileName = ""; + private String pathName = ""; + private String hostname = ""; + private String application = ""; + private ParserEngine engine = null; + + + public Parser(String application, String regexName, ParserEngine engine) { + + try { + if (engine.isDebug()) { + System.out.println("Create Grok with regex name : " + regexName); + } + grok = Grok.create(engine.getPatternPath()); + grok.compile("%{" + regexName + "}"); + } catch (GrokException e) { + e.printStackTrace(); + } + + try { + hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0]; + } catch (UnknownHostException e) { + e.printStackTrace(); + } + this.application = application; + this.engine = engine; + } + + @Override + public void init(Tailer tailer) { + super.init(tailer); + fileName = tailer.getFile().getName(); + pathName = tailer.getFile().getParent(); + if (engine.isDebug()) { + System.out.println("Init tailer on file : " + pathName + "/" + fileName ); + } + } + + public void handle(String line) { + + Match gm = grok.match(line); + gm.captures(); + Map result = gm.toMap(); + if ( result.size() > 0 ) + { + if (engine.isDebug()) { + System.out.println("Match OK "); + } + + StringBuffer sb = new StringBuffer(); + sb.append("log"); + //Tag + sb.append(",host=" + hostname); + sb.append(",application=" + application); + sb.append(",file=" + sanitizeString(fileName) ); + sb.append(",path=" + sanitizeString(pathName) ); + + String tagName = null; + String tagValue = null; + String value = null; + + // iterate on all match pattern + for (Map.Entry entry : result.entrySet() ) + { + tagName = entry.getKey(); + // Keep only key with lowercase + if ( StringUtils.isAllLowerCase( tagName ) ) + { + tagValue = entry.getValue().toString(); + // if key name is 'value', it's not a tag, but a field + if ( ! tagName.equalsIgnoreCase("value") ) { + // check if tag value is not empty + if (!tagValue.equalsIgnoreCase("") ) { + sb.append("," + tagName + "=" + sanitizeString( tagValue ) ); + } + } + else + { + value = tagValue; + } + } + + } + sb.append(" "); + //Field + if ( StringUtils.isNumeric(value) ) { + sb.append("value=" + value); + } + else + { + sb.append("value=1"); + } + + sb.append(" " + System.currentTimeMillis() + "000000"); + + // Add to the queue + engine.addToQueue(sb.toString()); + + if (engine.isDebug()) { + System.out.println(gm.toJson()); + } + } + } + + String sanitizeString(String s) + { + return s.trim().replaceAll(" ", "\\\\ ").replaceAll(",", "\\\\,").replaceAll("=", "\\\\="); + } +} diff --git a/src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java b/src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java new file mode 100644 index 0000000..1acc091 --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/ParserEngine.java @@ -0,0 +1,155 @@ +package com.airfrance.diqmqs.logparser; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.input.Tailer; +import org.apache.commons.io.input.TailerListener; + +public class ParserEngine implements Runnable{ + + private static final long ONE_SECOND = 1L; + private static final int BATCH_SIZE_MAX = 10; + final private String database = "qualif"; + final long delay = 500; + private ArrayList threadsParser = null; + static BlockingDeque queue = new LinkedBlockingDeque(1000); + private boolean debug = false; + private String patternPath = ""; + private ScheduledExecutorService scheduler; + @SuppressWarnings("unused") + private ScheduledFuture timerHandle; + + public ParserEngine(String patternPath, boolean debug) { + threadsParser = new ArrayList(); + this.debug = debug; + this.patternPath = patternPath; + scheduler = Executors.newScheduledThreadPool(1); + // Don't change this as metrics are per second + this.timerHandle = scheduler.scheduleAtFixedRate( this, ONE_SECOND, ONE_SECOND, TimeUnit.SECONDS); + + if (debug) { + System.out.println("Create ParserEngine with pattern file : " + patternPath); + System.out.println("Schedule sender is set to " + ONE_SECOND + " second(s)"); + } + } + + void addNewParser(File f , String regexName, String application ) { + + TailerListener listener = new Parser(application, regexName, this); + Tailer tailer = new Tailer(f, listener, delay, true); + Thread thread = new Thread(tailer); + thread.setDaemon(true); + thread.setName("Parser - " + threadsParser.size() ); + thread.start(); + threadsParser.add(thread); + if (debug) { + System.out.println("Thread Parser - " + threadsParser.size() + " started - file : " + f.getName()); + } + + } + + void stopAllParser() { + for (Thread t : threadsParser) { + t.interrupt(); + if (debug) { + System.out.println("Stop Thread " + t.getName() ); + } + } + threadsParser.clear(); + } + + // Read data from the Queue and send it to influxdb + public void run() { + + StringBuilder postData = new StringBuilder(); + for (int i = 0 ; i < BATCH_SIZE_MAX; i++) + { + try { + String message = queue.poll( 50, TimeUnit.MILLISECONDS); + if (message != null) { + postData.append( message + "\n" ); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + try { + if (postData.length() > 0) { + sendMetricToInfluxdb(postData.toString()); + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + public boolean sendMetricToInfluxdb(String postData) throws IOException { + + URL url = new URL("http://diqmqs.airfrance.fr/influxdb_query/write?rp=one_week&db=" + database); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setDoOutput(true); + connection.setRequestMethod("POST"); + connection.setRequestProperty("User-Agent", "LogParser/1.0"); + connection.setRequestProperty("Authorization", "Basic cXVhbGlmOmRpcW0wMQ=="); + connection.setRequestProperty("Content-Length", String.valueOf(postData.length())); + + // Write data + OutputStream os = connection.getOutputStream(); + if (this.isDebug()) { + System.out.println("Send : " + postData ); + } + os.write(postData.getBytes()); + + int HttpResult = connection.getResponseCode(); + if (HttpResult == 204) { + return true; + } else { + if (this.isDebug()) { + System.err.println(connection.getResponseCode() + " " + connection.getResponseMessage()); + } + return false; + } + } + + public void addToQueue(String s) { + + try { + queue.put(s); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + + public boolean isDebug() { + return debug; + } + + + public void setDebug(boolean debug) { + this.debug = debug; + } + + + public String getPatternPath() { + return patternPath; + } + + + public void setPatternPath(String patternPath) { + this.patternPath = patternPath; + } + +} diff --git a/src/main/java/com/airfrance/diqmqs/logparser/Paths.java b/src/main/java/com/airfrance/diqmqs/logparser/Paths.java new file mode 100644 index 0000000..1e151ce --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/Paths.java @@ -0,0 +1,304 @@ +package com.airfrance.diqmqs.logparser; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; + +/** Collects filesystem paths using wildcards, preserving the directory structure. Copies, deletes, and zips paths. */ +public class Paths implements Iterable { + + + static private List defaultGlobExcludes; + + final HashSet paths = new HashSet(32); + + /** Creates an empty Paths object. */ + public Paths () { + } + + /** Creates a Paths object and calls {@link #glob(String, String[])} with the specified arguments. */ + public Paths (String dir, String... patterns) { + glob(dir, patterns); + } + + /** Creates a Paths object and calls {@link #glob(String, List)} with the specified arguments. */ + public Paths (String dir, List patterns) { + glob(dir, patterns); + } + + private Paths glob (String dir, boolean ignoreCase, String... patterns) { + if (dir == null) dir = "."; + if (patterns != null && patterns.length == 0) { + String[] split = dir.split("\\|"); + if (split.length > 1) { + dir = split[0]; + patterns = new String[split.length - 1]; + for (int i = 1, n = split.length; i < n; i++) + patterns[i - 1] = split[i]; + } + } + File dirFile = new File(dir); + if (!dirFile.exists()) return this; + + List includes = new ArrayList(); + List excludes = new ArrayList(); + if (patterns != null) { + for (String pattern : patterns) { + if (pattern.charAt(0) == '!') + excludes.add(pattern.substring(1)); + else + includes.add(pattern); + } + } + if (includes.isEmpty()) includes.add("**"); + + if (defaultGlobExcludes != null) excludes.addAll(defaultGlobExcludes); + + GlobScanner scanner = new GlobScanner(dirFile, includes, excludes, ignoreCase); + String rootDir = scanner.rootDir().getPath().replace('\\', '/'); + if (!rootDir.endsWith("/")) rootDir += '/'; + for (String filePath : scanner.matches()) + paths.add(new Path(rootDir, filePath)); + return this; + } + + /** Collects all files and directories in the specified directory matching the wildcard patterns. + * @param dir The directory containing the paths to collect. If it does not exist, no paths are collected. If null, "." is + * assumed. + * @param patterns The wildcard patterns of the paths to collect or exclude. Patterns may optionally contain wildcards + * represented by asterisks and question marks. If empty or omitted then the dir parameter is split on the "|" + * character, the first element is used as the directory and remaining are used as the patterns. If null, ** is + * assumed (collects all paths).
+ *
+ * A single question mark (?) matches any single character. Eg, something? collects any path that is named + * "something" plus any character.
+ *
+ * A single asterisk (*) matches any characters up to the next slash (/). Eg, *\*\something* collects any path that + * has two directories of any name, then a file or directory that starts with the name "something".
+ *
+ * A double asterisk (**) matches any characters. Eg, **\something\** collects any path that contains a directory + * named "something".
+ *
+ * A pattern starting with an exclamation point (!) causes paths matched by the pattern to be excluded, even if other + * patterns would select the paths. */ + public Paths glob (String dir, String... patterns) { + return glob(dir, false, patterns); + } + + /** Case insensitive glob. + * @see #glob(String, String...) */ + public Paths globIgnoreCase (String dir, String... patterns) { + return glob(dir, true, patterns); + } + + /** Case sensitive glob. + * @see #glob(String, String...) */ + public Paths glob (String dir, List patterns) { + if (patterns == null) throw new IllegalArgumentException("patterns cannot be null."); + glob(dir, false, patterns.toArray(new String[patterns.size()])); + return this; + } + + /** Case insensitive glob. + * @see #glob(String, String...) */ + public Paths globIgnoreCase (String dir, List patterns) { + if (patterns == null) throw new IllegalArgumentException("patterns cannot be null."); + glob(dir, true, patterns.toArray(new String[patterns.size()])); + return this; + } + + public int count () { + return paths.size(); + } + + public boolean isEmpty () { + return paths.isEmpty(); + } + + /** Returns the absolute paths delimited by the specified character. */ + public String toString (String delimiter) { + StringBuffer buffer = new StringBuffer(256); + for (String path : getPaths()) { + if (buffer.length() > 0) buffer.append(delimiter); + buffer.append(path); + } + return buffer.toString(); + } + + /** Returns the absolute paths delimited by commas. */ + public String toString () { + return toString(", "); + } + + /** Returns a Paths object containing the paths that are files, as if each file were selected from its parent directory. */ + public Paths flatten () { + Paths newPaths = new Paths(); + for (Path path : paths) { + File file = path.file(); + if (file.isFile()) newPaths.paths.add(new Path(file.getParent(), file.getName())); + } + return newPaths; + } + + /** Returns a Paths object containing the paths that are files. */ + public Paths filesOnly () { + Paths newPaths = new Paths(); + for (Path path : paths) { + if (path.file().isFile()) newPaths.paths.add(path); + } + return newPaths; + } + + /** Returns a Paths object containing the paths that are directories. */ + public Paths dirsOnly () { + Paths newPaths = new Paths(); + for (Path path : paths) { + if (path.file().isDirectory()) newPaths.paths.add(path); + } + return newPaths; + } + + /** Returns the paths as File objects. */ + public List getFiles () { + return getFiles(new ArrayList(paths)); + } + + private ArrayList getFiles (List paths) { + ArrayList files = new ArrayList(paths.size()); + for (Path path : paths) + files.add(path.file()); + return files; + } + + /** Returns the portion of the path after the root directory where the path was collected. */ + public List getRelativePaths () { + ArrayList stringPaths = new ArrayList(paths.size()); + for (Path path : paths) + stringPaths.add(path.name); + return stringPaths; + } + + /** Returns the full paths. */ + public List getPaths () { + ArrayList stringPaths = new ArrayList(paths.size()); + for (File file : getFiles()) + stringPaths.add(file.getPath()); + return stringPaths; + } + + /** Returns the paths' filenames. */ + public List getNames () { + ArrayList stringPaths = new ArrayList(paths.size()); + for (File file : getFiles()) + stringPaths.add(file.getName()); + return stringPaths; + } + + /** Adds a single path to this Paths object. */ + public Paths addFile (String fullPath) { + File file = new File(fullPath); + String parent = file.getParent(); + paths.add(new Path(parent == null ? "" : parent, file.getName())); + return this; + } + + /** Adds a single path to this Paths object. */ + public Paths add (String dir, String name) { + paths.add(new Path(dir, name)); + return this; + } + + /** Adds all paths from the specified Paths object to this Paths object. */ + public void add (Paths paths) { + this.paths.addAll(paths.paths); + } + + /** Iterates over the absolute paths. The iterator supports the remove method. */ + public Iterator iterator () { + return new Iterator() { + private Iterator iter = paths.iterator(); + + public void remove () { + iter.remove(); + } + + public String next () { + return iter.next().absolute(); + } + + public boolean hasNext () { + return iter.hasNext(); + } + }; + } + + /** Iterates over the paths as File objects. The iterator supports the remove method. */ + public Iterator fileIterator () { + return new Iterator() { + private Iterator iter = paths.iterator(); + + public void remove () { + iter.remove(); + } + + public File next () { + return iter.next().file(); + } + + public boolean hasNext () { + return iter.hasNext(); + } + }; + } + + static private final class Path { + public final String dir; + public final String name; + + public Path (String dir, String name) { + if (dir.length() > 0 && !dir.endsWith("/")) dir += "/"; + this.dir = dir; + this.name = name; + } + + public String absolute () { + return dir + name; + } + + public File file () { + return new File(dir, name); + } + + public int hashCode () { + final int prime = 31; + int result = 1; + result = prime * result + ((dir == null) ? 0 : dir.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + + public boolean equals (Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + Path other = (Path)obj; + if (dir == null) { + if (other.dir != null) return false; + } else if (!dir.equals(other.dir)) return false; + if (name == null) { + if (other.name != null) return false; + } else if (!name.equals(other.name)) return false; + return true; + } + } + + /** Sets the exclude patterns that will be used in addition to the excludes specified for all glob searches. */ + static public void setDefaultGlobExcludes (String... defaultGlobExcludes) { + Paths.defaultGlobExcludes = Arrays.asList(defaultGlobExcludes); + } + + +} \ No newline at end of file diff --git a/src/main/java/com/airfrance/diqmqs/logparser/package-info.java b/src/main/java/com/airfrance/diqmqs/logparser/package-info.java new file mode 100644 index 0000000..559d77d --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/package-info.java @@ -0,0 +1,8 @@ +/** + * + */ +/** + * @author 4163013 + * + */ +package com.airfrance.diqmqs.logparser; \ No newline at end of file diff --git a/src/main/java/com/airfrance/diqmqs/logparser/patterns b/src/main/java/com/airfrance/diqmqs/logparser/patterns new file mode 100644 index 0000000..8c7dffe --- /dev/null +++ b/src/main/java/com/airfrance/diqmqs/logparser/patterns @@ -0,0 +1,107 @@ + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME:UNWANTED} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM:UNWANTED}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?/(?>[\w_%!$@:.,~-]+|\\.)*)+ +#UNIXPATH (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +# Time: HH:MM:SS +#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? +# I'm still on the fence about using grok to perform the time match, +# since it's probably slower. +# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG (?:[\w._/%-]+) +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING:UNWANTED} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: + +MESSAGESLOG %{SYSLOGBASE} %{DATA} + +COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} +COMMONAPACHELOG_DATATYPED %{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth} \[%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}\] "(?:%{WORD:verb;string} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})" %{NUMBER:response;int} (?:%{NUMBER:bytes;long}|-) + + +# Log Levels +LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) \ No newline at end of file diff --git a/src/main/resources/patterns b/src/main/resources/patterns new file mode 100644 index 0000000..6d03514 --- /dev/null +++ b/src/main/resources/patterns @@ -0,0 +1,111 @@ + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME:UNWANTED} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM:UNWANTED}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?/(?>[\w_%!$@:.,~-]+|\\.)*)+ +#UNIXPATH (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +# Time: HH:MM:SS +#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? +# I'm still on the fence about using grok to perform the time match, +# since it's probably slower. +# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +DATE_JMETER %{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG (?:[\w._/%-]+) +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING:UNWANTED} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: + +MESSAGESLOG %{SYSLOGBASE} %{DATA} + +TYPE [E|e]rr?(?:or)?|ERR?(?:OR)? +MESSAGE .* + +COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} +COMMONAPACHELOG_DATATYPED %{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth} \[%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}\] "(?:%{WORD:verb;string} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})" %{NUMBER:response;int} (?:%{NUMBER:bytes;long}|-) +JMETERLOG %{DATE_JMETER} %{TIME} %{TYPE} [- ] %{MESSAGE} + +# Log Levels +LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) \ No newline at end of file