Linux.Sys.LogHunter

Allows grep-like searching of Linux, MacOS and Windows logs.

Parameters include SearchRegex and WhitelistRegex as regex terms.

Also included is a Path exclusion regex (ExcludePathRegex) to refine results and automatically exclude hits in commonly unwanted locations such as /proc.

NOTE: The nosymlink feature of glob is set so unexpected results may occur if your targets includes symlink files.

name: Linux.Sys.LogHunter
author: "Matt Green - @mgreen27"
description: |
  Allows grep-like searching of Linux, MacOS and Windows logs.

  Parameters include `SearchRegex` and `WhitelistRegex` as regex terms.

  Also included is a Path exclusion regex (`ExcludePathRegex`) to refine results
  and automatically exclude hits in commonly unwanted locations such as `/proc`.

  NOTE: The `nosymlink` feature of glob is set so unexpected results may occur
  if your targets includes symlink files.

parameters:
  - name: TargetFiles
    default: '/var/log/**'
  - name: SearchRegex
    description: "Regex pattern to match in log lines."
    default: ' POST '
    type: regex
  - name: FilterRegex
    description: "Regex pattern to exclude matched lines from output."
    default:
    type: regex
  - name: ExcludeDirectoryRegex
    type: regex
    description: "Do not descend into directories that match this regex pattern."
    default: "^/(shared|proc|snap)"
  - name: ExcludePathRegex
    description: "Regex pattern describing paths to exclude from scanning."
    default: '\.journal$'
    type: regex

sources:
  - query: |
      LET RecursionCB <= if(condition= ExcludeDirectoryRegex,
         then="x => NOT x.OSPath =~ ExcludeDirectoryRegex",
         else="x => NOT x.OSPath =~ '^/proc' ")

      LET files = SELECT OSPath
        FROM glob(globs=TargetFiles,
            nosymlink=TRUE,
            recursion_callback=RecursionCB)
        WHERE NOT IsDir AND NOT OSPath =~ ExcludePathRegex
          AND log(message="Scanning %v", args=OSPath)

      LET hits = SELECT * FROM foreach(row=files,
          query={
              SELECT OSPath, Line FROM parse_lines(filename=OSPath)
              WHERE Line =~ SearchRegex
          })

      SELECT * FROM if(condition=FilterRegex,
        then={
           SELECT * FROM hits
           WHERE NOT Line =~ FilterRegex
        },
        else={
           SELECT * FROM hits
        })