# Copyright 2026 SigmaSRC, Inc. All Rights Reserved.

"""Gathering tools.

This allows various user interfaces for gathering to share code.

Classes in this module:

Gatherer  -- the toplevel object, which controls gathering operations
Collector -- the object that is passed to the gather script
Table     -- the object that collects one tabular data set
MD5Differ -- a helper for computing diffs between MD5 gatherings

Also:

test()    -- a function that runs a gatherer with output directed to stdout
TestCollector -- helper class for test()
TestTable -- helper class for test()

"""

import os
import re
import sys
import glob
import time
import shutil
import socket
from urllib.parse import quote, unquote
import difflib
import logging
import tempfile
import subprocess

from sigma.util import avalog
from sigma.util import config
from sigma.util import xmltime
from sigma.util import timeskew
from sigma.spool import spooling
from sigma.gather import records

global_debug = False # This variable may be set to True by gather.py

options = config.Options()


def read_text_file(path):
    """Open a host text file for reading as UTF-8, replacing undecodable bytes.

    Host files (/etc/passwd GECOS fields, mount tables, ...) can hold non-ASCII
    or even mixed-encoding bytes. Decoding as UTF-8 with errors="replace" yields
    valid Unicode (U+FFFD for bad bytes) that serializes cleanly into the result
    XML -- instead of crashing (strict) or mojibake (locale default). Prefer this
    over a bare open() for any file containing host-supplied text.
    """
    return open(path, "r", encoding="utf-8", errors="replace")


def run_command(cmd):
    """Run a shell command, returning stdout as UTF-8 text (bad bytes replaced).

    Drop-in for os.popen(cmd, "r").read(): decodes with UTF-8 + errors="replace"
    so accented command output (mount points, package names) is preserved and
    invalid bytes never raise. Iterate lines with .splitlines().
    """
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
                         stderr=subprocess.DEVNULL,
                         encoding="utf-8", errors="replace")
    out, _ = p.communicate()
    return out or ""

def get_gather_dir(options):
    """Return the name of the gather directory."""
    return options.datapath("gather")

def get_thirdparty_exe(name):
    """Return the path to an executable that's normally found in thirdparty."""
    result = os.path.join(options.agentactive,
                          "thirdparty", "bin", name)
    if not os.path.exists(result):
        result = "/usr/sbin/" + name # Backup for developers
    return result 

ACTIVE = "active"
NOT_ACTIVE = "not active"

def boolean_to_active(bool):
    """Converts a boolean to active/not active"""
    if bool:
        return ACTIVE
    else:
        return NOT_ACTIVE

def get_interfaces_path():
    """Returns the path to the interfaces file"""
    # The interfaces file is created when we do poliwall gathers.  All
    # active interfaces are listed in it.
    return options.datapath("interfaces.txt")

_ifname_prefix = "\\Device\\NPF_" # Stuff to strip off the name

def get_active_interfaces():
    """Returns a list of the active interfaces.

    Usage: 'eth0' in get_active_interfaces()

    Returns an empty list if the file isn't found or has no lines.

    On Windows the names are of the form {...} where '...' stands for
    a series of hex digits and hyphens.  The \\Device\\NPF_ prefix that
    the Poliwall adds is stripped though.

    """
    result = []
    path = get_interfaces_path()
    try:
        f = open(path, "r")
    except IOError as err:
        pass
    else:
        try:
            for line in f:
                words = line.split()
                name = words[-1]  # The name is the last word in the line
                if name.startswith(_ifname_prefix):
                    # Strip stuff added by poliwall to GUID on Windows systems
                    name = name[len(_ifname_prefix):]
                result.append(name)
        finally:
            f.close()
    # Return an empty list if the file isn't found or the file has no lines.
    return result

# Run a command and return the entire output as a string.
def cmdoutput(cmd):
    pipe = os.popen(cmd)
    try:
        return pipe.read()
    finally:
        pipe.close()

class Gatherer(object):

    """Top-level gathering class.

    Instantiate this with the central server host argument;
    then call full_gather(cls) or incr_gather(cls) to gather data.

    """

    def __init__(self, options=None):
        if options is None:
            options = config.Options()
        self.options = options
        self.logger = logging.getLogger("gather.gathering")
        self.dirname = os.path.dirname(__file__)
        self.hostname = socket.gethostname()

    def full_gather(self, cls):
        c = Collector(self.options)
        try:
            if self.gather_to_collector(cls, c):
                c.flush()
                c.send_full()
                c.shuffle_save_files()
        finally:
            c.cleanup()

    def incr_gather(self, cls):
        c = Collector(self.options)
        try:
            if self.gather_to_collector(cls, c):
                c.flush()
                c.send_diffs()
                c.shuffle_save_files()
        finally:
            c.cleanup()

    def gather_to_collector(self, cls, c):
        gather = self.get_gatherer(cls)
        t0 = time.time()
        self.logger.info("Start gathering %r", cls)
        try:
            gather(c)
        except:
            typ, val = sys.exc_info()[:2]
            self.logger.exception("Exception in gatherer: %s: %s", typ, val)
            return False
        t1 = time.time()
        self.logger.info("Done gathering %r (total time %.3f seconds)",
                         cls, t1 - t0)
        return True

    def check_gatherer(self, cls):
        filename = os.path.join(self.dirname, "gather_%s.py" % cls)
        return os.path.exists(filename)

    def get_gatherer(self, cls):
        modname = "gather_%s" % cls
        modpath = "sigma.gather." + modname
        sigma_mod = __import__(modpath)
        gather = sigma_mod.gather
        module = getattr(gather, modname)
        return module.Gather

    def full_clear(self):
        """Completely clean out the gather directory.

        This removes all files and directories from the gather
        directory, regardless of age.  Use with discretion!

        For a gentler cleanup, see periodic_cleanup().

        """
        dirname = get_gather_dir(self.options)
        if not os.path.exists(dirname):
            self.logger.info("Gather directory %s doesn't exist", dirname)
            return

        for name in os.listdir(dirname):
            filename = os.path.join(dirname, name)
            if os.path.isdir(filename):
                self.logger.info("Removing directory tree %r", filename)
                shutil.rmtree(filename, False, self.onerror)
            else:
                self.remove(filename, "full_clear")

    def onerror(self, func, pathname, exc_tuple):
        self.logger.warning("Can't %s %s: %s",
                            func.__name__, pathname, exc_tuple[1])

    def remove(self, filename, reason):
        try:
            os.remove(filename)
        except os.error as msg:
            self.logger.warning("Can't remove %r (%s): %s",
                                filename, reason, msg)
        else:
            self.logger.info("Removed %r (%s)", filename, reason)

    def periodic_cleanup(self):
        """Do periodic cleanup.

        Delete files from the gather directory that are left behind by
        crashed processes, or that simply don't belong there.

        We delete:

        - if multiple gather files exist for the same gather class,
          all except the latest one

        - other files older than a day

        Subdirectories are *left alone* -- these shouldn't exist anyway.

        """
        dirname = get_gather_dir(self.options)
        if not os.path.exists(dirname):
            self.logger.info("Gather directory %s doesn't exist", dirname)
            return
        byclass = {}
        names = os.listdir(dirname)
        names = sorted(map(os.path.normcase, names))
        tmpsuffix = os.path.normcase(".tmp")
        cutoff = time.time() - 24*3600
        for name in names:
            filename = os.path.join(dirname, name)
            if os.path.isdir(filename):
                continue
            try:
                mtime = os.path.getmtime(filename)
            except os.error:
                # File must've disappeared; this can happen if another
                # process happens to be gathering at the same time
                continue
            if name.count(".") == 1:
                i = name.index(".")
                cls = name[:i]
                if cls not in byclass:
                    byclass[cls] = []
                byclass[cls].append((mtime, filename))
            else:
                if mtime < cutoff:
                    self.remove(filename, "old tmp or bogus file")
        for cls, lst in byclass.items():
            if len(lst) == 1:
                continue
            lst.sort()
            for mtime, filename in lst[:-1]:
                self.remove(filename, "redundant %r data" % cls)

class Collector(object):

    """A Collector instance is passed to the gather script.

    The collector holds a set of Tables.  It has two APIs: one
    (minimal) API for the gather script, another for the Gatherer
    class above.

    Methods for the gather script:

    table(name, columns) -> Table -- Create a new table

    Methods for the Gatherer class:

    flush()                    -- Close all tables
    send_full()                -- send full data to spool queue
    send_diffs()               -- send diffs for data to spool queue
    shuffle_save_files()       -- move temporary file to real file
    cleanup()                  -- Get rid of all tables

    """

    def __init__(self, options):
        self._options = options
        self._tables = {}

    def table(self, name, columns):
        if name in self._tables:
            raise RuntimeError("duplicate table name")
        self._tables[name] = t = Table(name, columns, self._options)
        return t

    def flush(self):
        for t in self._tables.values():
            t.close()

    def send_full(self):
        for t in self._tables.values():
            t.send_full()

    def send_diffs(self):
        for t in self._tables.values():
            t.send_diffs()

    def shuffle_save_files(self):
        for t in self._tables.values():
            t.shuffle_save_files()

    def cleanup(self):
        for key, t in list(self._tables.items()):
            del self._tables[key]
            t.cleanup()

class Table(object):

    """A Table is used to write data to a file.

    API for gather scripts:

    add(record) -- add a data record

    The rest of the API is for the Collector class only.

    """

    _output = None
    _tfn = None
    _isname = re.compile(r"\A[a-zA-Z][a-zA-Z0-9_]*\Z").match

    def __init__(self, name, columns, options):
        assert isinstance(name, str), repr(name)
        assert self._isname(name)
        assert isinstance(columns, list), repr(columns)
        if __debug__:
            for col in columns:
                assert isinstance(col, str), repr(col)
                assert self._isname(col), repr(col)
        assert isinstance(options, config.Options)
        self._name = name
        self._columns = columns
        self._options = options
        self._skew = timeskew.get_skew(options)
        self._ncolumns = len(columns)
        self._logger = logging.getLogger("gather.gathering.Table." + name)
        self._queue = spooling.Queue(None, options)
        self._make_filenames(name)
        # Binary mode: the records.py format is byte-oriented (UTF-8 fields with
        # a byte-length prefix, separator b"\0\n"), so it must be written as bytes
        # with no newline translation -- otherwise Windows would turn a field's
        # '\n' into '\r\n', shift every following byte, and the server's
        # length-prefixed reader rejects the upload (500 / "Invalid digit").
        self._output = open(self._tfn, "wb")
        self._output.write(records.encodeRecord(columns))
        if global_debug:
            print(name, columns)
        self._t0 = time.time() + self._skew

    def _remove(self, filename, reason):
        try:
            os.remove(filename)
        except os.error as msg:
            self._logger.warning("Can't remove %r (%s): %s",
                                 filename, reason, msg)
        else:
            self._logger.debug("Removed %r (%s)", filename, reason)


    def _make_filenames(self, name):
        """Set file-related attributes: _fn, _tfn, _ofn, _version, _oversion.

        These are three filenames in the same directory, and a version
        string.

        - _fn is the filename where our data will eventually end up
        - _tfn is the filename where we write our data
        - _ofn is the filename representing the previous version
          it may be None if not found
        - _version is the unique random string identifying this version
        - _oversion is the version of _ofn, or None if _ofn is None

        """
        dirname = get_gather_dir(self._options)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        while True:
            fd, fn = tempfile.mkstemp("", name + ".", dirname)
            os.close(fd)
            if len(glob.glob(fn + "*")) > 1:
                self._remove(fn, "collision")
                # A rare collision indeed.  Loop around to try again.
            else:
                break
        tfn = fn + ".tmp"
        os.rename(fn, tfn)
        self._fn = fn
        self._tfn = tfn
        fntail = os.path.normcase(os.path.basename(fn))
        i = fntail.index(".")
        self._version = fntail[i+1:]
        oldfiles = glob.glob(os.path.join(dirname, name + ".*"))
        older_fn = None
        older_mtime = None
        for ofn in oldfiles:
            tail = os.path.basename(ofn)
            if tail.count(".") > 1:
                continue # Temporary file
            try:
                mtime = os.path.getmtime(ofn)
            except os.error:
                # File must've been removed by another process doing
                # periodic cleanup
                continue
            if older_mtime is None or mtime > older_mtime:
                older_mtime = mtime
                older_fn = ofn
        self._ofn = older_fn
        self._oversion = None
        if self._ofn is not None:
            tail = os.path.normcase(os.path.basename(older_fn))
            i = tail.index(".")
            self._oversion = tail[i+1:]

    def add(self, record):
        assert self._output
        assert len(record) == self._ncolumns
        if global_debug:
            print(record)
        self._output.write(records.encodeRecord(record))

    def close(self):
        output = self._output
        if output:
            del self._output
            output.close()
            self._t1 = time.time() + self._skew

    def send_full(self):
        self.close()
        qitem = self._queue.new()
        shutil.copyfile(self._tfn, qitem.filename)
        self.spool_it(qitem)

    def send_diffs(self):
        if not self.compare_headers():
            return self.send_full()

        t0 = time.time()

        qitem = self._queue.new()

        if self._name == "md5":
            differ = MD5Differ(self._ofn, self._tfn, qitem.filename)
            try:
                differ.run()
            finally:
                differ.close()
        else:
            # XXX This is calculating the diff in memory.
            # Let's hope the input aren't too big.
            f = open(self._ofn, "rb")
            oldlines = []
            record = records.readRecord(f)
            while record:
                oldlines.append(record)
                record = records.readRecord(f)
            f.close()
            f = open(self._tfn, "rb")
            newlines = []
            record = records.readRecord(f)
            while record:
                newlines.append(record)
                record = records.readRecord(f)
            f.close()
            self._logger.info("Gathered %d lines of data", len(newlines))
            f = open(qitem.filename, "wb")
            f.write(records.recodeRecord("action", oldlines[0]))
            del oldlines[0], newlines[0]
            s = difflib.SequenceMatcher(None, oldlines, newlines)
            for tag, i1, i2, j1, j2 in s.get_opcodes():
                if tag != "equal":
                    for i in range(i1, i2):
                        f.write(records.recodeRecord("d", oldlines[i]))
                    for j in range(j1, j2):
                        f.write(records.recodeRecord("i", newlines[j]))
            f.close()

        t1 = time.time()
        self._logger.info("Diffed in %.3f seconds", t1-t0)
        self.spool_it(qitem, incr=True)

    def spool_it(self, qitem, incr=False):
        if incr:
            suffix = ".incr"
            oversion = self._oversion
        else:
            suffix = ".full"
            oversion = ""
        self._logger.debug("Spooling %d bytes (%s)",
                           os.path.getsize(qitem.filename),
                           self._name + suffix)
        # Keyword args to spool() become spool item metadata
        qitem.spool(type=self._name + suffix,
                    starttime=xmltime.formattimestamp(self._t0),
                    endtime=xmltime.formattimestamp(self._t1),
                    version1=oversion,
                    version2=self._version)

    def compare_headers(self):
        if self._ofn is None:
            return False
        f = open(self._ofn, "rb")
        oldline = records.readRecord(f)
        f.close()
        f = open(self._tfn, "rb")
        newline = records.readRecord(f)
        f.close()
        if oldline == newline:
            return True
        else:
            self._logger.warning(
                "*** Old and new data have different field names!!!"
                " (old: %r, new: %r)", oldline, newline)
            return False

    def shuffle_save_files(self):
        self.close()
        # On Unix, rename() is atomic, but on Windows, it fails if the
        # target already exists; so let's just remove it first
        if os.path.exists(self._fn):
            self._remove(self._fn, "temporary filename")
        try:
            os.rename(self._tfn, self._fn)
        except os.error as msg:
            self._logger.error("Can't rename %r to %r: %s",
                               self._tfn, self._fn, msg)
        else:
            self._tfn = self._fn = None
        if self._ofn is not None:
            self._remove(self._ofn, "old filename")

    def cleanup(self):
        self.close()
        tfn = self._tfn
        if tfn and os.path.exists(tfn):
            del self._tfn
            self._remove(tfn, "cleanup")

    __del__ = cleanup

class MD5Differ(object):
    """A diff algorithm specific to the md5 gatherer.

    This is needed to deal with the fact that the md5 gatherer, in
    classic Heisenberg fashion, affects the atimes it observes.

    """

    def __init__(self, oldfilename, newfilename, outputfilename):
        self.oldfilename = oldfilename
        self.newfilename = newfilename
        self.outf = open(outputfilename, "wb")
        self.logger = logging.getLogger("gather.gathering.MD5Differ")
        self.oldmtime = os.path.getmtime(oldfilename)
        self.oldf = open(self.oldfilename, "rb")
        self.newf = open(self.newfilename, "rb")
        self.newcount = 0
        self.getold()
        self.getnew()
        assert self.oldline
        assert self.oldfields == self.newfields
        self.headers = self.oldfields
        self.outf.write(records.recodeRecord("action", self.oldline))
        self.iatime = self.headers.index("st_atime")
        self.ifilename = self.headers.index("filename")

    def close(self):
        self.outf.close()
        self.oldf.close()
        self.newf.close()

    def run(self):
        self.getold()
        self.getnew()
        while self.oldline and self.newline:
            oldfile = self.oldfields[self.ifilename].strip()
            newfile = self.newfields[self.ifilename].strip()
            if oldfile == newfile:
                try:
                    atime = int(self.newfields[self.iatime])
                except ValueError:
                    pass
                else:
                    if atime <= self.oldmtime:
                        self.newfields[self.iatime] = \
                            self.oldfields[self.iatime]
                if self.oldfields != self.newfields:
                    self.newline = records.encodeRecord(self.newfields)
                    self.outf.write(records.recodeRecord("d", self.oldline))
                    self.outf.write(records.encodeRecord(["i"] +
                                                         self.newfields))
                self.getold()
                self.getnew()
            elif oldfile < newfile:
                self.outf.write(records.recodeRecord("d", self.oldline))
                self.getold()
            else:
                assert oldfile > newfile, "oldfile doesn't compare to newfile!"
                self.outf.write(records.recodeRecord("i", self.newline))
                self.getnew()
        while self.oldline:
            self.outf.write(records.recodeRecord("d", self.oldline))
            self.getold()
        while self.newline:
            self.outf.write(records.recodeRecord("i", self.newline))
            self.getnew()
        self.logger.info("Gathered %d lines of data", self.newcount)

    def getold(self):
        self.oldline = records.readRecord(self.oldf)
        if not self.oldline:
            self.oldfields = []
        else:
            self.oldfields = records.decodeRecord(self.oldline)

    def getnew(self):
        self.newline = records.readRecord(self.newf)
        if not self.newline:
            self.newfields = []
        else:
            self.newcount += 1
            self.newfields = records.decodeRecord(self.newline)

# Test support for gather scripts.
# In the gather script, call test(Gather) if __name__ == "__main__".

def test(Gather, output_name=False):
    avalog.setupLogging()
    c = TestCollector(output_name=output_name)
    Gather(c)

class TestCollector(object):

    def __init__(self, output=None, output_name=False):
        self._output_name = output_name
        # encodeRecord yields bytes -> write to a binary stream (stdout.buffer).
        self._output = output or sys.stdout.buffer

    def table(self, name, columns):
        # Have the option of printing the name of the table so I can
        # read the output from gather_netservice.py.
        if self._output_name:
            self._output.write((name + ":\n").encode("utf-8"))
        self._output.write(records.encodeRecord(columns))
        return TestTable(name, len(columns), self._output)

class TestTable(object):

    def __init__(self, name, ncolumns, output):
        self._name = name
        self._ncolumns = ncolumns
        self._output = output

    def add(self, record):
        assert len(record) == self._ncolumns, record
        self._output.write(records.encodeRecord(record))
