Skip to content

Quickstart

This quickstart shows how to parse a log dataset (we use data from the AIT Log Data Set V2.0) and then run a detector to check whether the dataset contains anomalies. We use the MatcherParser and the RandomDetector.

1. Parse the logs

from pathlib import Path
from detectmatelibrary.parsers.template_matcher import MatcherParser
from detectmatelibrary.helper.from_to import From, To

ROOT = Path(__file__).resolve().parents[3]  # repository root; adjust if needed
templates_path = str(ROOT / "tests" / "test_data" / "audit_templates.txt")
log_path = str(ROOT / "tests" / "test_data" / "audit.log")
log_json = str(ROOT / "local" / "audit_raw.json")
parsed_path = str(ROOT / "local" / "audit_parsed.json")
(ROOT / "local").mkdir(exist_ok=True)

config_dict = {
    "parsers": {
        "MatcherParser": {
            "auto_config": True,
            "method_type": "matcher_parser",
            "path_templates": templates_path,
            "log_format": r"type=<Type> msg=audit\(<Time>:<Serial>\): <Content>",
        }
    }
}
parser = MatcherParser(name="MatcherParser", config=config_dict)

raw_logs = list(From.log(parser, log_path, do_process=False))
parsed_logs = [parser.process(log) for log in raw_logs]

To.json(raw_logs, log_json)
To.json(parsed_logs, parsed_path)

2. Run the detector

import numpy as np
import yaml
from detectmatelibrary import schemas
from detectmatelibrary.detectors.random_detector import RandomDetector

with open("docs/examples/detectors/random_detector.yaml") as f:
    config = yaml.safe_load(f)
detector = RandomDetector(name="RandomDetector", config=config)

login = schemas.ParserSchema({"EventID": 0, "variables": ["alice", "10.0.0.1", "22"]})

np.random.seed(0)  # only to make this example reproducible

# no training needed: with threshold 0.9, roughly 1 in 10 logs raises an alert
alerts = [detector.process(login) for _ in range(100)]
print(sum(alert is not None for alert in alerts))  # around 10

Common pitfalls

  • When re-running the parser code, delete audit_raw.json and audit_parsed.json first if you want to execute it again --> otherwise output is appended to stale files.

Go back Index