#!/usr/bin/env python3
# Copyright 2026 SigmaSRC, Inc. All Rights Reserved.

"""Poliwall ``gather'' program.

This is ``gather'' between quotes, because it doesn't quite use the
full gathering framework.  Here's the deal:

- The poliwall writes data files to $AVDATA/pwspool/.  These have
  names of the form <datatype>.<interface>.<timestamp>.  The data
  format is whitespace-separated values, without any quoting
  convention.  The last field may be a free-form field containing
  whitespace (but not newlines).  The number of fields is fixed per
  <datatype>.  The first of the line gives the field names (column
  headings).

- Every hour (for example), this program is run by the scheduler.
  It looks through the pwspool directory, and for each valid file it
  finds, it converts the data to standard gather format and spools it.

"""

USAGE = "%prog [-h/--help]"

import os
import re
import sys
import time
import errno
import pickle
import logging
import optparse

if __name__ == "__main__":
    from sigma.util.utf8 import ensure_utf8_mode
    ensure_utf8_mode()
    from sigma.util.winquiet import suppress_child_windows
    suppress_child_windows()

import poliwall
from sigma.util import avalog, config, xmltime, fslock
from sigma.util.errors import usage
from sigma.spool import spooling
from sigma.gather import records

SKIPS = ["netblog", "netilog", "netstatrule"]

options = config.Options()
pwcompfile = options.datapath("pwcompliance.pickle")
packetlogfile = options.datapath("packetlog.txt")
pwspooldir = options.datapath("pwspool")
lockdir = options.datapath("pwspool.lock")
tempdir = options.datapath("temp")

logger = logging.getLogger("pwgather")
parser = optparse.OptionParser(usage=USAGE)

def main(args=None):
    opts, args = parser.parse_args(args)
    if args:
        return usage("This program takes no arguments")

    avalog.setupLogging()

    if not os.path.exists(pwspooldir):
        logger.info("%s doesn't exist; nothing to do", pwspooldir)
        return 0

    lock = getlock()
    if lock is None:
        return 1

    try:
        clean_flushed()

        if poliwall.main(["-w"]) == 0:
            wait_flushed()

        doit(pwspooldir)
    finally:
        unlock(lock)

def wait_flushed():
    logger.info("Waiting for flushed.* file...")
    limit = 30.0
    t = 0
    dt = 0.2
    while t < limit:
        if clean_flushed():
            return
        time.sleep(dt)
        t += dt
    logger.error("No flushed.* file appeared in %g seconds; giving up", limit)

def clean_flushed():
    names = get_flushed()
    for name in names:
        try:
            os.remove(os.path.join(pwspooldir, name))
        except os.error:
            logger.warning("Can't remove %s", name)
        else:
            logger.debug("Removed %s", name)
    return len(names)

def get_flushed():
    return [x for x in list_pwspooldir()
              if x.lower().startswith("flushed.")]

def list_pwspooldir():
    try:
        return os.listdir(pwspooldir)
    except os.error as err:
        logger.warning("Can't list %s: %s", pwspooldir, err)
        return []

def doit(pwspooldir):
    # The pwcompliance dict is a mapping from directive IDs to dicts;
    # the inner dicts map rule IDs to lists of strings, where [] means
    # "in compliance" and anything else means "out of compliance".
    # When a rule ID is seen in the netstatrule table, we must set the
    # corresponding entry to [...] for each directive, but only for rules
    # that are already there (i.e. we never add new entries).  Note:
    # the rule ID and message ID keys are stored as strings.
    pwcompliance = readpwcompliance()
    packetlog = readpacketlog()

    names = list_pwspooldir()
    if not names:
        logger.debug("No files in %s", pwspooldir)
        return 0

    logger.info("Temp dir: %s", tempdir)
    if not os.path.exists(tempdir):
        try:
            os.mkdir(tempdir)
        except OSError:
            pass

    # first spool all the files except netname because we also want to 
    # aggregate all the netstatip files
    _doit(pwspooldir, "other", names, pwcompliance, packetlog)

    # now spool netname files
    return _doit(pwspooldir, "netname", names, pwcompliance, packetlog)



def _doit(pwspooldir, type, names, pwcompliance, packetlog):
    cutoff = time.time() - 12*3600
    regex = re.compile(r"\A(\w+)\.([^.]+)\.(\d+\.\d\d\d(?:-\d+\.\d\d\d)?)\Z")
    fout = None
    netstatiptext = ""
    if type == "netname":
        # read the aggregated netstatip file
        try:
            file = os.path.join(tempdir, "tempspool.txt")
            f = open(file, "r")
        except IOError as err:
            logger.error("Can't open the temp file %r for reading: %s", file, err)
        else:
            try:
                netstatiptext = f.read()
            except IOError as err:
                logger.error("Can't read the temp file %r: %s", file, err)
            try:
                f.close()
                try:
                    os.remove(file)
                except os.error as err:
                    logger.error("Can't remove temp file %r: %s", file, err)

            except IOError as err:
                logger.error("Can't close temp file %r: %s", file, err)
    else:
        # open the aggregate netstatip file for writing
        try:
            file = os.path.join(tempdir, "tempspool.txt")
            fout = open(file, "w")
        except IOError as err:
            logger.error("Can't open the temp file %r for writing: %s", file, err)

    for name in names:
        if type == "netname" and not name.startswith("netname"):
            continue
        if type != "netname" and name.startswith("netname"):
            continue
        file = os.path.join(pwspooldir, name)
        # A typical filename is netbridge.eth0.1081286000.000-1081287727.037
        # (Backwards compatible filenames are netbridge.eth0.1081287727.037)
        m = regex.match(name)
        if m is None:
            # Not one of the files we're looking for; discard the old ones
            try:
                mtime = os.path.getmtime(file)
            except os.error as err:
                # File disappeared; no big deal (probably renamed to
                # proper name by the poliwall process)
                logger.debug("Can't stat %r: %s", file, err)
            else:
                if mtime < cutoff:
                    try:
                        os.remove(file)
                    except os.error as err:
                        if err.errno == errno.EACCES:
                            logger.debug("Can't remove %r (no access)", file)
                        else:
                            logger.error("Can't remove %r: %s", file, err)
                    else:
                        logger.info("Removed orphan file %r", file)
                else:
                    logger.debug("Leaving young orphan file %r", file)
            continue
        datatype, interface, timestamp = m.groups()

        dospool = (datatype not in SKIPS) # Files not to be uploaded
        dostats = (datatype == "netblog") # Non-compliance counters
        donotify = (datatype == "netilog") # Record in packet log

        if not (dospool or dostats or donotify):
            logger.info("Skipping and removing %r",
                        os.path.basename(file))
            try:
                os.remove(file)
            except os.error as err:
                logger.error("Can't remove %r: %s", file, err)
            else:
                logger.debug("Spooled and removed %r", file)
            continue

        try:
            f = open(file)
        except IOError as err:
            logger.error("Can't open %r: %s", file, err)
            continue
        
        # write to the aggregate netstatip file
        try:
            if name.startswith("netstatip"): 
                text = f.read()
                fout.write(text)
        except IOError as err:
            logger.error("Can't add %r to temp file: %s", file, err)

        # position to the beginning of netstatip file
        f.seek(0, 0)

        try:
            logger.info("%s %r",
                        dospool and "Spooling" or "Scanning",
                        os.path.basename(file))
            lineno = 0
            headerline = f.readline()
            lineno += 1
            headers = headerline.split()
            nfields = len(headers)
            headers = headers + ["interface"]
            logger.debug("Headers: %s", " ".join(headers))
            its = None
            if donotify or dostats:
                for required_field in "flags", "time_stamp", "detail":
                    if required_field not in headers:
                        logger.warning("No %r header in %s file?!",
                                    required_field, datatype)
                        donotify = dostats = False
                        break
                its = headers.index("time_stamp")
            if donotify:
                logger.info("Collecting packet block logs...")
            if dostats:
                logger.info("Collecting compliance stats...")
            qitem = qf = None
            if dospool:
                qitem = spooling.new()
                qf = open(qitem.filename, "wb")
            try:
                if dospool:
                    qf.write(records.encodeRecord(headers))
                for line in f:
                    lineno += 1
                    if line.endswith("\n"):
                        line = line[:-1]
                    fields = line.split(None, nfields-1)
                    if len(fields) != nfields:
                        logger.error("Bad data in %r line %d: %r",
                                     file, lineno, line)
                        continue
                    # Strip trailing spaces from last field
                    fields[-1] = fields[-1].rstrip()
                    if dospool:
                        fields = fields + [interface]
                        
                        # If the netip does not exist in aggregated 
                        # netstatip, do not spool it. May be traffic for this
                        # ip is suppressed.
                        if name.startswith("netname"):
                            if netstatiptext.find(" " + fields[2] + " ") < 0:
                                if netstatiptext.find("\n" + fields[2] + " ") < 0:
                                    #logger.info("Dropping IP: %r", fields[2])
                                    continue
                        qf.write(records.encodeRecord(fields))
                    if dostats:
                        ruleid = int(fields[0])
                        msg = None
                        # Flag this rule as non-compliant in each directive
                        for key, d in pwcompliance.items():
                            if ruleid in d:
                                if msg is None:
                                    msg = format_compliance(fields, headers)
                                msgs = d.get(ruleid) or [] # List of strings
                                if msg not in msgs:
                                    msgs.append(msg)
                                d[ruleid] = msgs
                                # XXX get rid of these log messages later
                                logger.debug("Set d[%r] = %r", ruleid, msgs)
                            else:
                                logger.debug("Ruleid %r not found", ruleid)
                    if donotify:
                        ruleid = fields[0]
                        tt = time.localtime(int(fields[its]))
                        datef = time.strftime("%Y-%m-%d", tt)
                        timef = time.strftime("%H:%M:%S", tt)
                        msg = format_compliance(fields, headers, agentui=True)
                        item = datef, timef, ruleid, msg
                        if item not in packetlog: # XXX This is quadratic
                            packetlog.append(item)
            finally:
                if dospool:
                    qf.close()
            if dospool:
                if "-" in timestamp:
                    starttime, endtime = timestamp.split("-", 1)
                    starttime = xmltime.formattimestamp(float(starttime))
                else:
                    starttime = None
                    endtime = timestamp
                endtime = xmltime.formattimestamp(float(endtime))
                metadata = {"type": datatype + ".full", "endtime": endtime}
                if starttime is not None:
                    metadata["starttime"] = starttime
                qitem.spool(**metadata)
        finally:
            f.close()

        try:
            os.remove(file)
        except os.error as err:
            logger.error("Can't remove %r: %s", file, err)
        else:
            logger.debug("Spooled and removed %r", file)

    if type != "netname":
        try:
            fout.close()
        except IOError as err:
            logger.error("Can't close temp file %r after writing: %s", file, err)

    writepwcompliance(pwcompliance)
    writepacketlog(packetlog)

    return 0


def format_compliance(fields, headers, agentui=False):
    ruleid = fields[0]
    detail = fields[headers.index("detail")]
    flags = fields[headers.index("flags")]
    proto, dst, src, ipproto, port = detail.split()
    # Note: the string "blocked" is used by engine.py to detect enforcement
    s = "%s/%s %s %s traffic %s from %s to %s" % (
        ipproto, proto, ipproto in ("tcp","udp") and "port" or "type",
        port, "D" in flags and "blocked" or "seen", src, dst)
    if not agentui:
        when = fields[headers.index("time_stamp")]
        s += " (%s)" % time.ctime(int(when))
    return s

# This lock protects the pwcompliance.pickle file as well as access to
# the pwspool directory; it must be held while using poliwall.py -w as
# well (since that causes a *.flushed file to be created in the
# pwspool directory).

def getlock():
    logger.info("Locking %s", lockdir)
    lock = fslock.FSLock(lockdir)
    if lock.lock(30):
        return lock
    else:
        logging.error("Giving up on locking %s", lockdir)
        return None

def unlock(lock):
    logger.info("Releasing lock %s", lockdir)
    lock.unlock()

def readpwcompliance():
    pwcompliance = {}
    try:
        f = open(pwcompfile, "rb")
    except IOError:
        logger.info("No pwcompliance.pickle file yet")
    else:
        try:
            try:
                pwcompliance = pickle.load(f)
            except Exception as err:
                logger.exception("Failed to read pickle from %s: %s",
                                 pwcompfile, err)
            else:
                logger.info("Read pickle from %s", pwcompfile)
                logger.debug("Read value = %r", pwcompliance)
        finally:
            f.close()
    # Backwards compatibility -- get rid of None key
    if None in pwcompliance:
        del pwcompliance[None]
    # Backwards compatibility -- convert True to [msgs], False to []
    for key, d in pwcompliance.items():
        for subkey, v in d.items():
            if isinstance(v, int): # Found an old one
                d[subkey] = ["Packet filter detected rule violation"] * v
    return pwcompliance

def writepwcompliance(pwcompliance):
    logger.debug("Writing value = %r", pwcompliance)
    # Backwards compatibility -- check that nobody is adding None as a
    # key, nor putting Booleans where they should write lists
    for key, d in pwcompliance.items():
        assert key is not None, "pwcompliance should not have key None"
        for subkey, v in d.items():
            assert isinstance(v, list), ("pwcompliance[%r][%r] is %r;"
                                         " should be list of strings" %
                                         (key, subkey, v))
    try:
        f = open(pwcompfile, "wb")
    except IOError as err:
        logger.error("Can't write %s: %s", pwcompfile, err)
    else:
        try:
            try:
                pickle.dump(pwcompliance, f)
            except Exception as err:
                logger.exception("Failed to write pickle to %s: %s",
                                 pwcompfile, err)
            else:
                logger.info("Wrote pickle to %s", pwcompfile)
        finally:
            f.close()

packetlogregex = r"\s*(\d\d\d\d+-\d\d-\d\d)\s+(\d\d:\d\d:\d\d)\s+(\d+)\s+(.*)"

def readpacketlog():
    fn = packetlogfile
    if not os.path.exists(fn):
        logger.debug("No packet log file: %s", fn)
        return []
    try:
        f = open(fn, "r")
    except IOError as err:
        logger.error("Can't read %s: %s", fn, err)
        return []
    try:
        regex = re.compile(packetlogregex)
        packetlog = []
        for lineno, line in enumerate(f):
            m = regex.match(line)
            if m:
                packetlog.append(tuple(m.groups()))
            else:
                logger.warning("Bad packet log line:%d:%s",
                            lineno+1, line.rstrip("\n"))
        return packetlog
    finally:
        f.close()

def writepacketlog(packetlog):
    packetlog.sort() # Conveniently, date and time are the first two fields
    del packetlog[:-100] # Only preserve 100 most recent items
    fn = packetlogfile
    try:
        f = open(fn, "w")
    except IOError as err:
        logger.error("Can't create %s: %s", fn, err)
    else:
        try:
            for item in packetlog:
                f.write("%s %s %s %s\n" % item)
        finally:
            f.close()
        logger.info("Wrote packetlog to %s", fn)

if __name__ == "__main__":
    sys.exit(main())
