#!/usr/bin/env python3
# Copyright 2026 SigmaSRC, Inc. All Rights Reserved.

"""Create and run a number of synthetic agents, for server stress testing.

    *** THIS IS THE ECS 2.0 VERSION (USING THE REST PROTOCOL) ***

Usage: %prog -s SERVER [options]

Note: This uses several configuration values from $AVCONF/config, in
particular timeout-per-msg, [timeout-per-kb XXX NOT], rfa-interval,
backoff-factor, backoff-limit, as well as the logging parameters (by
default logging to stderr and to $AVDATA/log/log-YYYY-MM-DD.txt).

Also note that $AVROOT defaults to $HOME/.sigma on Unix when not
run as root, and the defaults for $AVCONF and $AVDATA are derived from
$AVROOT by adding conf and data, respectively.

Every 5 seconds two INFO messages are shown like this:

  <date> <time>:root:INFO:P=0.7, F=5, A=15, E=1, S=9 (0, 0.6, 9)
  <date> <time>:root:INFO:agebins: [4, 1]

where:

  * P = the average number of pings in the last second
  * F = the number of open file descriptors > 2
  * A = the number of initialized agents
  * E = the backlog in the event queue
  * S = the total upload spool queue size for all agents
  * (x, y, z) = min, average and max spool queue size per agent
  * The agebins line gives the distribution of the oldest spool item per
  agent, per minute; the example [4, 1] shows 4 agents whose oldest
  spool item was less than a minute old, and 1 agent whose oldest
  spool item was less than 2 minutes old.  (XXX This statistic is
  subject to experimentation.)  The agebins line is suppressed when
  all agents' spool queues are empty.

If all is healthy:

  * P = A/RFA where RFA is the rfa-interval, default 60
  * F = 0 or 1 or at least small
  * A = the value of the -a parameter
  * E = 0 or at least small
  * S = approximately A/2 (but can temporarily be in the hundreds)
  * (x, y, z) = (0, 0.5, 30) approximately
  * agebins shows no spool queue items older than 2 minutes

"""

"""XXX TO DO

Functionality
-------------

- Save schedules across invocations.

Performance
-----------

- The directives are parsed too often.  Downloading, sure, but no need
  to parse'em and store a parsed copy in each Agent.  Could drop some
  of the parsing in the WeakPingPong class too -- all we need to do is
  simulate the network traffic!

- Caching can do wonders (e.g. for find_best_implementation()).

- The PingPongThread should hold on to its queue manager rather than
  creating a new one each time.  Then the Agent can share it rather
  than have its own.  (The constructor does a stat and then some.)

- Other stuff is also probably repeated too much.  Need to profile!

Added realism
-------------

- Replace the actual host IP address with the synthetic host IP
  address in non-poliwall gathers too

- Make some random modifications to the gathers?

Bugs
----

- Still leaks memory, about 1 KB per agent per minute. :-(

"""

# Python imports
import os
import re
import sys
sys.coinit_flags = 0
import glob
import math
import time
import heapq
import base64
import random
import shutil
import socket
import http.client as httplib
import logging
import optparse
import tempfile
import threading
from queue import Queue
try:
    import pythoncom
except ImportError:
    pythoncom = None

# Sigma imports
from sigma.util import avalog, cert, config, xmlparse, xmltime, ostypes
from sigma.util.errors import error, usage, plural
from sigma.protocol import rest, xmlproto
from sigma.sched import pingpong, scheduling, tasks
from sigma.gather import gathering, records, gather_command
from sigma.spool import spooling, qmgr
from sigma.runtime import xmlout, matrix

# Global objects
ALLERRORS = pingpong.ALLERRORS
options = config.Options()
parser = optparse.OptionParser(usage=__doc__.strip())
parser.add_option("-p", "--prefix", action="store", dest="prefix",
                  default="A", help="agent name prefix (default A)")
parser.add_option("-a", "--agents", action="store", dest="agents",
                  type="int", default=1, help="number of agents (default 1)")
parser.add_option("-f", "--first", action="store", dest="first",
                  type="int", default=0, help="first agent id (default 0)")
parser.add_option("-t", "--threads", action="store", dest="threads",
                  type="int", default=1, help="number of threads (default 1)")
parser.add_option("-s", "--server", action="store", dest="server",
                  help="central server hostname or IP address (no default)")
parser.add_option("-d", "--delay", action="store", dest="delay", type="int",
                  default=10,
                  help="seconds delay between registrations (default 10)")
parser.add_option("-x", "--extra", action="store", dest="extra", type="int",
                  default=10,
                  help="number of fake outside hosts per agent (default 10)")
parser.add_option("-c", "--compliance", action="store", dest="compliance",
                  type="float", default=60,
                  help="rule compliance percentage (default 60)")
parser.add_option("-n", "--nmap", action="store", dest="nmap",
                  help="simulate nmap; -n X/Y probes X random TCP ports "
                       "every Y minutes, from 1.1.1.1, to all agents "
                       "(default off)")
parser.add_option("-r", "--restart", action="store", dest="restart",
                  type="float",
                  help="restart after RESTART hours (default off); "
                       "this really just exits at that time, you have "
                       "to restart it in a shell loop")


def main(args=None):
    """Main program.

    Parses the command line and starts the worker threads, then waits
    for them to exit, which they shouldn't do but might in case of
    buggy code (always a possibility).  Also catches ^C and then it
    tells all agents to stop.

    """
    # Parse the command line
    opts, args = parser.parse_args(args)

    # Parse nmap option
    if not opts.nmap:
        opts.nmap_ports = opts.nmap_seconds = 0
    else:
        try:
            if opts.nmap.count("/") != 1:
                raise ValueError("correct format is '-n ports/minutes'")
            ports, minutes = map(int, opts.nmap.split("/"))
            if ports or minutes:
                if ports <= 0:
                    raise ValueError("ports must be >= 0")
                if minutes <= 0:
                    raise ValueError("minutes must be > 0")
        except Exception as err:
            return usage("Invalid -n/--nmap option %r (%s)", opts.nmap, err)
        opts.nmap_ports = ports
        opts.nmap_seconds = 60 * minutes # Minutes for user, seconds for code

    # Additional command line constraints
    if args:
        return usage("No positional arguments please!")
    if not opts.server:
        return usage("Missing -s/--server option.")

    # Create the data directory if necessary
    if not os.path.exists(options.agentdata):
        logging.info("Creating %s directory", options.agentdata)
        try:
            os.makedirs(options.agentdata)
        except os.error as err:
            logging.error("Can't create %s directory: %s",
                          options.agentdata, err)
            return 1

    # Miscellaneous one-time setup
    avalog.setupLogging()

    # Log essentials
    logging.info("synth.py starting...")
    logging.info("python: %s", sys.executable)
    logging.info("argv: %s", " ".join(sys.argv))

    # Create the manager and run
    mgr = Manager(opts)
    return mgr.run()

initial_gathers_unix = [
    "command",
    "cpu",
    "df",
    "dhcpserver",
    "hardware",
    "hosts_equiv",
    "known_hosts",
    "mount",
    "nameserver",
    "netstat",
    "osinfo",
    "ram",
    "rhosts",
    "route",
    "upgrade",
    "wireless",
    ]

initial_gathers_windows = [
    "command",
    "cpu",
    "df",
    "dhcpserver",
    "nameserver",
    "netstat",
    "osinfo",
    "ram",
    "upgrade",
    "winantivirus",
    "windhcpservice",
    "winhw",
    "winsp",
    "winupdate",
    "winwirelessnic",
    ]

if os.name == "nt":
    initial_gathers = initial_gathers_windows
else:
    initial_gathers = initial_gathers_unix

class Manager(object):

    """The manager creates the event queue, worker threads, and agents."""

    def __init__(self, opts):
        """Constructor.  Creates the worker threads but doesn't start them."""
        self.opts = opts
        try:
            testname = self.opts.prefix % 0
        except (TypeError, ValueError):
            self.template = self.opts.prefix + "%04d"
        else:
            self.template = self.opts.prefix
        self.agents = []
        self.threads = []
        self.counter = Counter(self)
        self.eq = EventQueue(self.opts.threads, self.counter)
        self.counter.daemon = True
        self.counter.start()
        host_port = "%s:%s" % (central_server(self.opts.server, False))
        logging.info("getting root cert")
        logging.info("creating %d worker thread%s",
                     self.opts.threads, plural(self.opts.threads))
        for i in range(self.opts.threads):
            t = Worker(i, self.eq)
            t.daemon = True
            self.threads.append(t)

    def run(self):
        """Starts threads, fires off first event, runs main loop."""
        if self.opts.restart:
            self.eq.add(self.opts.restart*3600, None)
        logging.info("creating first agent")
        self.create_agent_event()
##        logging.info("doing initial gathers")
##        if self.agents:
##            a = self.agents[0]
##            for g in initial_gathers:
##                a.gather(g)
        logging.info("starting %d worker thread%s",
                     len(self.threads), plural(len(self.threads)))
        for t in self.threads:
            t.start()
        sts = self.eq.run()
        logging.info("Waiting...")
        for t in self.threads:
            t.join()
        logging.info("Done.")
        return sts

    def create_agent_event(self):
        """Creates an agent and schedules another agent creation."""
        name = self.template % (len(self.agents) + self.opts.first)
        a = Agent(name, self.eq, self.counter, self.opts)
        a.setup()
        delay = self.opts.delay
        if a.is_registered():
            delay = 0
        a.initialize()
        self.agents.append(a)
        if len(self.agents) < self.opts.agents:
            self.eq.add(delay, self.create_agent_event)

    def log_queue_sizes(self, avgpings=0):
        """Logs the queue sizes and misc info."""
        count, total, lowest, highest = 0, 0, sys.maxsize, 0
        agebins = []
        now = int(time.time())
        for a in self.agents:
            x, youngest, average, oldest = a.qm.age_stats()
            count += 1
            total += x
            if x < lowest:
                lowest = x
            if x > highest:
                highest = x
            if x > 0:
                # Age of oldest spool queue item in minutes, truncated:
                age = max(0, (now - oldest + 59) // 60)
                while len(agebins) <= age:
                    agebins.append(0)
                agebins[age] += 1
        eqsize = self.eq.queue.qsize()
        fdn = len(self.eq.get_fds())
        if not count:
            logging.info("P=%.1f, F=%d, A=0, E=%d, S=0", avgpings, fdn, eqsize)
        else:
            logging.info("P=%.1f, F=%d, A=%d, E=%d, S=%d (%d, %.1f, %d)",
                         avgpings, fdn, count, eqsize, total,
                         lowest, float(total) / count, highest)
            if agebins:
                logging.info("agebins: %s", agebins)

class EventQueue(object):

    """Real-time multi-threaded event management.

    To schedule an event: eq.add(when, callable, *args).
    To get the next ready event: callable, *args = eq.get(); this may block.
    Some thead (maybe main) needs to run the main loop: eq.run().

    There's no way to cancel an event once scheduled.

    """

    max_matrix_id = 0 # Highest Matrix ID seen; see FakePingPong.repeatedTasks

    def __init__(self, nthreads, counter):
        """Constructor."""
        self.nthreads = nthreads
        self.counter = counter
        self.queue = Queue() # Ready events
        self.events = [] # Future events
        self.cond = threading.Condition()

    def run(self):
        """Main loop.  Some thread (maybe main) should run this."""
        sts = 0
        self.cond.acquire()
        try:
            while True:
                if not self.events:
                    dt = 1
                else:
                    dt = self.events[0][0] - time.time()
                if dt <= 0:
                    t = heapq.heappop(self.events)
                    # t = when, callable, *args
                    if t[1] is None:
                        logging.warn("Exit to honor --restart option...")
                        break
                    self.queue.put(t[1:])
                else:
                    self.cond.wait(min(dt, 1))
        except Exception as err:
            logging.warn("Caught %s: %s...", err.__class__.__name__, err)
            sts = 1
        self.show_fds()
        # Empty the queue (we're racing with the workers here)
        while not self.queue.empty():
            self.queue.get(False)
        # Push a None for each worker
        for i in range(self.nthreads):
            self.queue.put(None) # Stop all the worker threads
        self.cond.release()
        return sts

    def show_fds(self):
        logging.info("fds: %s", " ".join(map(str, self.get_fds())))

    def get_fds(self):
        fds = []
        # Skip 0, 1, 2; these are always open (stdin, stdout, stderr)
        # Stop at 820; there be valgrind daemons
        for i in range(3, 820):
            try:
                st = os.fstat(i)
            except os.error:
                pass
            else:
                fds.append(i)
        return fds

    def add(self, when, callable, *args):
        """Adds an event to the queue.

        Arguments:

        when -- int or float giving absolute or relative time, in seconds
        callable -- function to be called when the time is ready

        All times less than 1e9 are considered relative (that's
        September 8, 2001, so you should be fine unless you have
        borrowed my time machine :-).  The current time is added to
        relative times before storing them.

        """
        if when < 1e9:
            when += time.time()
        self.cond.acquire()
        try:
            heapq.heappush(self.events, (when, callable) + args)
            self.cond.notify()
        finally:
            self.cond.release()

    def get(self, block=True, timeout=None):
        """Gets the next event from the queue.

        Optional arguments block and timeout are passed straight to
        Queue.get().

        This returns a callable object.  It blocks until the event is
        ready.

        It may also return None, which mean the worker should stop.

        """
        return self.queue.get(block, timeout)

class Worker(threading.Thread):

    """Worker thread."""

    def __init__(self, id, eq):
        """Constructor.

        Arguments:

        id -- string giving this worker's id
        eq -- EventQueue instance

        """
        self.id = id
        self.eq = eq
        if pythoncom != None:
            pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED)
        threading.Thread.__init__(self, name="Worker-%s" % id)

    def __del__(self):
        if pythoncom != None:
            pythoncom.CoUninitialize()

    def run(self):
        """Runs the main loop, started by start() method."""
        self.mainloop()
##        import profile
##        p = profile.Profile()
##        p.runcall(self.mainloop)
##        p.print_stats()

    def mainloop(self):
        """The real main loop."""
        while True:
            t = self.eq.get()
            if t is None:
                break
            try:
                t[0](*t[1:])
            except Exception:
                logging.exception("Exception in EventQueue loop")

class Counter(threading.Thread):

    """A thread that logs a running average of 'clicks' per second."""

    def __init__(self, manager, size=60):
        """Constructor.

        Arguments:

        manager -- Manager instance (for log_queue_sizes())
        size -- optional int giving number of buckets, default 60

        """
        self.manager = manager
        self.size = size
        threading.Thread.__init__(self, name="Counter")
        self.buckets = []
        self.count = 0
        self.time = 0
        self.last_report_time = 0
        self.log_queue_counter = 0
        self.lock = threading.Lock()
        self.stopped = False

    def stop(self):
        self.stopped = True

    def run(self):
        """Main loop.  Report the counters every second on the second."""
        while not self.stopped:
            when = self.time + 1
            now = time.time()
            dt = when - now
            if dt > 0:
                time.sleep(dt)
                now = time.time()
            if now >= when:
                try:
                    self.report_counters()
                except Exception as err:
                    logging.exception("Exception in Counter loop")
            else:
                time.sleep(1)

    def report_counters(self):
        """Reports statistics: last bucket and average of all buckets.

        Returns True iff anything was logged.

        """
        self.lock.acquire()
        try:
            self._check_buckets()
            if not self.buckets:
                return # No counts yet
            last = self.buckets[-1]
            avg = float(sum(self.buckets)) / len(self.buckets)
            if self.last_report_time:
                seconds_covered = self.time - self.last_report_time
            else:
                seconds_covered = len(self.buckets)
            self.last_report_time = self.time
        finally:
            self.lock.release()
        if seconds_covered > 1:
            logging.debug("ping: missed %d: %s", seconds_covered - 1,
                          self.buckets[-seconds_covered:-1])
        logging.debug("ping: %d in last sec, avg %.1f/sec in last %d",
                      last, avg, len(self.buckets))
        self.log_queue_counter -= seconds_covered
        if self.log_queue_counter <= 0:
            self.log_queue_counter = 5
            self.manager.log_queue_sizes(avg)

    def schedule_next_report(self):
        """Schedulers the next report_counters_event call."""
        t = time.time()
        when = math.ceil(t)
        if when == t:
            when += 1
        self.eq.add(when, self.report_counters_event)

    def click(self):
        """Records one 'click' in the correct bucket."""
        self.lock.acquire()
        try:
            self._check_buckets()
            self.count += 1
        finally:
            self.lock.release()

    def _check_buckets(self):
        """Internal helper to correctly update the buckets."""
        if not self.buckets and not self.count:
            return
        t = int(time.time())
        dt = t - self.time
        if dt:
            if not self.buckets:
                dt = 1
            self.buckets.extend([0]*dt)
            if len(self.buckets) > self.size:
                del self.buckets[:len(self.buckets) - self.size]
            self.buckets[-1] = self.count
            self.count = 0
            self.time = t

class MockScheduler(object):

    """Just support list_tasks(), add_list() and save()."""

    def __init__(self, agent):
        self.agent = agent
        self.tasks = {}

    def list_tasks(self):
        return self.tasks

    def add_list(self, tasklist):
        for t in tasklist:
            assert t.id.startswith("engine.")
            key = t.id[7:] # Strip off the "engine." part
            if t.command is None: # delete
                if t.id in self.tasks:
                    del self.tasks[t.id]
                self.agent.remove_directive_task(key)
            elif t.id in self.tasks: # replace
                self.tasks[t.id] = t
            else: # new
                self.tasks[t.id] = t
                self.agent.add_directive_task(key)

    def load(self):
        options = self.agent.options
        for name in xmlproto.list_policydir(options):
            key = xmlproto.get_policy_key(name, options)
            if key and key != "pw.1":
                dir = xmlproto.load_directive(key, options)
                when = freq = None
                if dir.task is not None:
                    freq = dir.task.freq
                tid = "engine." + key
                self.tasks[tid] = tasks.newtask(tid, "engine.py " + key,
                                                when, freq)
                when = (freq or 0) * random.random()
                self.agent.logger.debug("Start %s in %.3f secs", key, when)
                self.agent.add_directive_task(key, when)

    def save(self):
        pass

class LoggerDelegate(object):

    """Helper class to reduce the logging volume from pingpong.py.

    Pragmatically, we suppress the most common INFO-level messages.

    """

    default_skips = [
        "Roundtrip time",
        "Agent home (un)?changed",
        "Matrix unchanged",
        "Configuration is up to date",
        "Uploading",
##        "All queue items uploaded:",
        "Added configuration text:",
        "    ",
        "Configuration updated; restarting",
        "Scheduling engine.pw.1 task to recompute pwconf.txt",
        "Wrote",
        ##"Active directives:",
        ]

    def __init__(self, logger, skips=None):
        self.logger = logger
        if skips is None:
            skips = self.default_skips
        self.skips = skips
        self.skips_pattern = "^(%s)" % "|".join(skips)
        self.skips_prog = re.compile(self.skips_pattern)

    def debug(self, msg, *args, **kwds):
        if True:
            return
        self.logger.debug(msg, *args, **kwds)

    def info(self, msg, *args, **kwds):
        if self.skips_prog.match(msg):
            return
        self.logger.info(msg, *args, **kwds)

    def warn(self, msg, *args, **kwds):
        if msg != "Network is too slow":
            self.logger.warn(msg, *args, **kwds)

    def error(self, msg, *args, **kwds):
        self.logger.error(msg, *args, **kwds)

    def critical(self, msg, *args, **kwds):
        self.logger.critical(msg, *args, **kwds)

    def log(self, level, msg, *args, **kwds):
        self.logger.log(level, msg, *args, **kwds)

class MatrixLoggerDelegate(LoggerDelegate):

    default_skips = [
        "Loaded matrix",
        "Got matrix delta",
        "Stored matrix",
        "Got full matrix",
        ]

matrix.logger = MatrixLoggerDelegate(matrix.logger)

class SchedulingLoggerDelegate(LoggerDelegate):

    default_skips = [
        "Updated",
        ]

scheduling.logger = SchedulingLoggerDelegate(scheduling.logger)

class CommandGatherLoggerDelegate(LoggerDelegate):

    default_skips = [
        "command = ",
        ]

gather_command.logger = CommandGatherLoggerDelegate(gather_command.logger)

class GatheringLoggerDelegate(LoggerDelegate):

    default_skips = [
        "Start gathering",
        "Done gathering",
        ]

class WeakPingPong(pingpong.PingPongThread):

    """Override various methods to use no server and a mock scheduler.

    This pragmatically and ruthlessly overrides various methods.

    This is not usable as a thread!  (XXX A better way would be to
    have the thread-ness mixed in by pingpong.py rather than take it
    out here.)

    """

    def __init__(self, agent):
        self.agent = agent
        sched = MockScheduler(agent)
        pingpong.PingPongThread.__init__(self, sched, None, agent.options)

    def start(self):
        raise RuntimeError("No!")

    def run(self):
        raise RuntimeError("Nee!")

    def mainloop(self):
        raise RuntimeError("Non!")

    def sleep(self, *args):
        raise RuntimeError("Nein!")

    def customize(self):
        self.logger = LoggerDelegate(self.agent.logger)
        self.sched.load()

    def standardGathers(self):
        for name in self.standard_gather_names:
            self.agent.gather(name)

    def repeatedTasks(self):
        try:
            return pingpong.PingPongThread.repeatedTasks(self)
        finally:
            self.agent.counter.click()
            if self.matrix_id > self.agent.eq.max_matrix_id:
                self.agent.eq.max_matrix_id = self.matrix_id
                self.logger.info("Matrix id %s", self.matrix_id)

    def probeMatrix(self, conn):
        try:
            update, matrix_slmtime = conn.getMatrix(False, self.matrix_slmtime)
            if matrix_slmtime:
                self.matrix_slmtime = matrix_slmtime
            if update is None:
                return True
            if self.matrix_id is None: # do not download full matrix at startup
                self.matrix_id = update.id
                return True
            if update.id == self.matrix_id:
                return True
            # download previous deltas or full matrix
            initial_id = self.matrix_id
            while self.matrix_id < update.base:
                try:
                    update, matrix_slmtime = conn.getMatrix(
                        False, None, self.matrix_id, self.matrix_id + 1)
                except RuntimeError as err: # 404 HTTP status code
                    if err.args[1] == 404:
                        # previous delta does not exist, so download full matrix
                        update, matrix_slmtime = conn.getMatrix(True, None)
                        break
                    else: 
                        if initial_id != self.matrix_id:
                            # we got some deltas
                            return True
                        else:
                            # we did not get any deltas
                            self.logError("matrix", err)
                            return False
                if matrix_slmtime:
                    self.matrix_slmtime = matrix_slmtime
                self.matrix_id = update.id
        except ALLERRORS as err:
            self.logError("matrix", err)
            return False
        return True

    def checkServerExit(self):
        return False

    def shutDownServer(self):
        pass

    def restartServer(self):
        pass

    def getConnection(self, ssl):
        return rest.Connection(self.options.central_server,
                               ssl=ssl, certstore=self.certstore,
                               useTimeout=False)

class Agent(object):

    """A synthetic agent."""

    # Directory where all agents create their home directory
    rootdir = os.path.join(tempfile.gettempdir(), "Sigma")

    def __init__(self, name, eq, counter, opts):
        """Constructor.  Does not create files or interact with the server.

        Arguments:

        name -- string giving the synthetic agent's host name
        eq -- EventQueue instance
        counter -- Counter instance
        opts -- command line options returned by optparse

        """
        self.name = name
        self.eq = eq
        self.counter = counter
        self.opts = opts
        self.homedir = os.path.join(self.rootdir, self.name)
        self.confdir = os.path.join(self.homedir, "conf")
        self.conffile = os.path.join(self.confdir, "config")
        self.datadir = os.path.join(self.homedir, "data")
        self.certdir = os.path.join(self.datadir, "cert")
        self.options = config.Options()
        self.options.configfile = self.conffile
        self.options.agentdata = self.datadir
        self.options.agentcert = self.certdir
        self.options.agentconf = self.confdir
        self.options.central_server = "%s:%s" % central_server(opts.server)
        self.gatherdir = gathering.get_gather_dir(self.options)
        self.logger = logging.getLogger(self.name)
        self.delay = self.options.rfa_interval
        self.ostype = ostypes.guess_ostype()
        self.na_msg = "Skipped: not applicable on %s" % self.ostype
        self.directives = {}
        self.pingpong = WeakPingPong(self)

    def setup(self):
        """Creates the agent home directory and necessary substructure."""
        self.logger.info("Agent initialization")
        if not os.path.isdir(self.homedir):
            os.makedirs(self.homedir)
        if not os.path.isdir(self.certdir):
            os.makedirs(self.certdir)
        if not os.path.isdir(self.confdir):
            os.makedirs(self.confdir)
        self.gatherer = gathering.Gatherer(self.options)
        self.gatherer.logger = GatheringLoggerDelegate(self.gatherer.logger)
        self.qm = qmgr.QueueManager(options=self.options)

    def initialize(self):
        """Initialization.  This schedules the first pingpong event."""
        self.pwtime = time.time() # See gather_poliwall()
        when = random.random() * self.delay
        self.eq.add(when, self.pingpong_event)

    def is_registered(self):
        """Check whether we are already registered, and if so, load the

        Returns whether the agent is registered.

        """
        fn = os.path.join(self.certdir, "id.txt")
        return os.path.exists(fn)

    def pingpong_event(self):
        """Executes one pingpong event."""
        DELAY = self.options.rfa_interval
        if self.pingpong.repeatedTasks():
            self.delay = delay = DELAY
        else:
            FACTOR = self.options.backoff_factor
            LIMIT = self.options.backoff_limit
            self.delay = min(self.delay * FACTOR, LIMIT)
            delay = self.pingpong.randomizeDelay(DELAY, self.delay, LIMIT)
        self.eq.add(delay, self.pingpong_event)

    def add_directive_task(self, key, when=0):
        if key == "pw.1":
            return
        assert key not in self.directives, (key, self.directives)
        dir = xmlproto.load_directive(key, self.options)
        self.directives[key] = dir
        self.eq.add(when, self.directive_event, key)

    def remove_directive_task(self, key):
        if key in self.directives:
            del self.directives[key]

##    def handle_directives(self, directives, initial=False):
##        """Stores a new list of directives, and schedule them.

##        Arguments:

##        directives -- list of directives
##        initial -- optional bool, True for the initial loading of directives

##        """
##        nrules = 0
##        newdirectives = {}
##        newkeys = []
##        for directive in directives:
##            nrules += len(directive.rules)
##            secs = directive.task.freq.secs
##            key = (directive.id, directive.version, secs)
##            newdirectives[key] = directive
##            if key in self.directives:
##                token = self.directives[key].token
##            else:
##                token = self.last_token = self.last_token + 1
##                newkeys.append((key, token))
##            directive.token = token
##        self.directives = newdirectives
##        for key, token in newkeys:
##            id, version, secs = key
##            when = 0
##            if initial:
##                rand = random.random()
##                when = secs * rand + time.time()
##                if id == "5002": # The gather directive
##                    # Pretend the poliwall was running since last time
##                    self.pwtime = when - secs
##            self.eq.add(when, self.directive_event, key, token)
##        self.logger.info("%s %d directive%s (%d new), %d rule%s",
##                         initial and "loaded" or "received",
##                         len(directives), plural(len(directives)),
##                         len(newkeys),
##                         nrules, plural(nrules))

    def directive_event(self, key):
        """Simulates the execution of a directive.

        Argument:

        key -- a key in the self.directives dict

        If the key is not found in self.directives, that means this
        event was cancelled, and we exit immediately.

        """
        directive = self.directives.get(key)
        if directive is None:
            return # Event was canceled
        ngathers = 0
        self.logger.debug("directive=%s", directive)
        results = xmlout.Results(directive_id=directive.id,
                                 directive_version=directive.version)
        for rule in directive.rules:
            res = xmlout.Result(rule_id=rule.id)
            module, procedure = find_best_implementation(rule, self.ostype)
            if module is None:
                res.leave = xmlout.Leave(text=self.na_msg)
            elif rule.category == "gather":
                assert procedure.startswith("gather_")
                if not (procedure.startswith("gather_command_") or procedure.startswith("gather_current_user_windows")):
                    name = procedure[7:]
                    self.gather(name)
                    if name != "poliwall":
                        ngathers += 1
            else:
                r = random.random() * 100
                if r >= self.opts.compliance:
                    msg = "Synthetic violation (id=%s, r=%g)" % (rule.id, r)
                    res.violations.append(xmlout.Violation(text=msg))
            results.results.append(res)

        if ngathers:
            self.logger.info("Spooled %d faux gathers", ngathers)
        qitem = self.qm.new(".xml")
        f = open(qitem.filename, "w")
        results.__render__(f)
        f.close()
        t = xmltime.formatnow()
        deployment_id = "%s.%s" % (directive.id, directive.version)
        qitem.spool(type="compliance",
                    deployment_id=deployment_id,
                    starttime=t,
                    endtime=t)
        self.logger.info("Spooled results for directive %s.%s",
                         directive.id, directive.version)
        self.eq.add(directive.task.freq, self.directive_event, key)

    def gather(self, name):
        """Does a gather.  Uses various tricks to save work.

        Arguments:

        name -- string giving the gather class

        """
        if name == "hostname":
            self.gather_hostname()
        elif name == "interfaces":
            self.gather_interfaces()
        elif name == "poliwall":
            self.gather_poliwall()
        elif name == "netstat":
            self.gather_netstat()
        else:
            self.cached_gather(name)

    # Variables used by cached_gather()
    gcachelock = threading.Lock()
    gather_cache = {}

    def cached_gather(self, name):
        """Does a cached gather.

        Arguments:

        name -- string giving the gather class

        """
        # Subtle: the argument gives the gather script name; this is
        # usually but not always the same as the table name produced
        # by the script.  The cache is about the tables.
        tname = name # table name
        if name == "wireless":
            tname = "wirelessnic"
        self.gcachelock.acquire()
        try:
            filename = self.gather_cache.get(tname)
            if not filename:
                self.gatherer.full_gather(name)
                pattern = self.options.datapath("gather/%s.*" % tname)
                matches = glob.glob(pattern)
                if matches:
                    self.gather_cache[tname] = matches[0]
                else:
                    # We don't want to waste time running unusual
                    # gathers each time without picking up any
                    # results, so we mark this table name as
                    # "to be skipped" by setting the filename to "-"
                    self.logger.info("Henceforth skipping %s gather", name)
                    self.gather_cache[tname] = "-" # Skip this one
                return
        finally:
            self.gcachelock.release()

        # Only get here if it was cached; this doesn't need the lock
        if filename == "-":
            self.logger.debug("Skipping %s gather", name)
        else:
            self.logger.debug("Spooling cached %s gather", tname)
            qitem = self.qm.new()
            shutil.copyfile(filename, qitem.filename)
            t = xmltime.formatnow()
            qitem.spool(type=tname+".full", starttime=t, endtime=t)

    def gather_hostname(self):
        """Does a pretend hostname gather, returning the agent name."""
        c = gathering.Collector(self.options)
        t = c.table("hostname", ["hostname"])
        t.add([self.name])
        c.flush()
        c.send_full()
        c.shuffle_save_files()

    def gather_netstat(self):
        """Does a pretend netstat gather.

        The real one is unrealistic because under certain
        circumstances it sees the connections belonging to all the
        simulated agents, which considerably skews the output size (by
        an order of magnitude).

        """
        c = gathering.Collector(self.options)
        t = c.table("netstat", ['proto', 'localip', 'localport',
                                'foreignip', 'foreignport', 'state'])
        for record in fake_netstat_data:
            t.add(record)
        c.flush()
        c.send_full()
        c.shuffle_save_files()

    def gather_interfaces(self):
        """Does a pretend interfaces gather."""
        c = gathering.Collector(self.options)
        cols = ['interface', 'ip', 'broadcast', 'netmask', 'gateway', 'mac',
                'dhcpserver', 'dnshostname']
        t = c.table("interfaces", cols)
        interface = "eth0"
        ip, broadcast, netmask, gateway, mac, dnshostname = self.fake_ip_bc()
        dhcpserver = gateway # why not
        t.add(["eth0", ip, broadcast, netmask, gateway, mac,
               dhcpserver, dnshostname])
        c.flush()
        c.send_full()
        c.shuffle_save_files()

    fakeip = None

    def fake_ip_bc(self):
        """Generates fake IP, broadcast and MAC from the name's hash.

        As a side effect, sets self.fakeip to the fake IP address.

        """
        name = self.name
        x = hash(name) % 0xFFFFFF
        if x & 255 in (0, 255):
            x = x+128
        a = 11
        b = (x>>16) & 255
        c = (x>>8) & 255
        d = x & 255
        ip = "%d.%d.%d.%d" % (a, b, c, d)
        broadcast ="%d.%d.%d.%d" % (a, b, c, 255)
        netmask = "255.0.0.0" # One big happy LAN
        gateway = "%d.0.0.1" % a
        mac = "00402B%06X" % x
        fqdn = name + ".moleflow.net"
        self.fakeip = ip
        self.fakemac = mac
        self.fakefqdn = fqdn
        return ip, broadcast, netmask, gateway, mac, fqdn

    def gather_poliwall(self):
        """Fakes the uploads that a typical pwgather run would produce."""
        if self.fakeip is None:
            self.fake_ip_bc()
            assert self.fakeip is not None
        then = self.pwtime
        now = time.time()
        self.pwtime = now
        interval = now - then
        count = 0
        while then < now:
            count += self.fake_poliwall_gather(then, min(then+1800, now))
            then += 1800
        self.logger.info("Spooled %d faux poliwall gathers", count)

    def fake_poliwall_gather(self, beginning, ending):
        """Sends fake poliwall data for a given period.

        If the period is 30 minutes or larger, send all available fake
        data.  Otherwise, send a proportionally smaller amount of data.

        (The sampled data represents half an hour of data; for longer
        periods, call this method multiple times.)

        Returns the number of files spooled.

        """
        fraction = (ending - beginning) / 1800.
        self.logger.debug("fake_poliwall_gather(%s, %s); fraction=%s",
                          beginning, ending, fraction)
        netstatip_faked = netosfp_faked = False
        count = 0
        for key in fake_poliwall_data:
            blocks = fake_poliwall_data[key]
            clip = int(round(fraction * len(blocks)))
            self.logger.debug("doing %d/%d %s blocks", clip, len(blocks), key)
            blocks = blocks[:clip]
            t0 = beginning
            for block in blocks:
                count += 1
                filename = block[0]
                m = re.match(r"\A(\w+).(\w+).([\d.]+)-([\d.]+)\Z", filename)
                cls, interface, starttime, endtime = m.groups()
                starttime = float(starttime)
                endtime = float(endtime)
                # Normalize the interval to be in the specified interval
                offset = t0 - starttime
                starttime = t0
                endtime += offset
                t0 = endtime
                qitem = self.qm.new()
                f = open(qitem.filename, "wb")
                nfields = 0
                its = -1
                for line in block[1:]:
                    fields = line.split(None, nfields-1)
                    if not nfields:
                        # First line is header
                        nfields = len(fields)
                        assert nfields >= 1
                        try:
                            its = fields.index("time_stamp")
                        except ValueError:
                            its = -1
                        fields.append("interface")
                    else:
                        assert len(fields) == nfields
                        if its >= 0:
                            # Extract time_stamp field
                            t1 = int(fields[its])
                            # Correct time stamp
                            t1 += offset
                            # Clip to ending time just in case
                            t1 = min(t1, ending)
                            # Store it back
                            fields[its] = str(int(t1))
                        for i, field in enumerate(fields):
                            # Patch fields containing "our" IP address
                            if field == THISIP:
                                fields[i] = self.fakeip
                            elif field.lower() == THISMAC:
                                fields[i] = self.fakemac
                            elif field.lower() == THISFQDN:
                                fields[i] = self.fakefqdn
                        fields.append(interface)
                    f.write(records.encodeRecord(fields))
                if cls == "netstatip":
                    if self.opts.nmap:
                        self.add_nmap(f, interface, beginning, ending)
                    if not netstatip_faked:
                        self.add_fake_netstatip(f, interface, fraction)
                        netstatip_faked = True
                elif cls == "netosfp":
                    if not netosfp_faked:
                        self.add_fake_netosfp(f, interface, fraction)
                        netosfp_faked = True
                f.close()
                qitem.spool(type=cls+".full",
                            starttime=xmltime.formattimestamp(starttime),
                            endtime=xmltime.formattimestamp(endtime))
        return count

    def add_nmap(self, f, interface, beginning, ending):
        "Adds fake netstatip records simulating nmap from 1.1.1.1."
        ports = self.opts.nmap_ports
        assert ports > 0, ports
        seconds = self.opts.nmap_seconds
        assert seconds > 0, seconds
        # Compute how many nmaps were in [beginning, ending)
        b = beginning - beginning % seconds
        if b < beginning:
            b += seconds
        count = 0
        while b < ending:
            count += 1
            b += seconds
        if not count:
            return # Nothing to do
        # XXX What if there are ports also mentioned in pre-sampled data?
        for p in range(ports):
            port = 1 + p*65535//ports
            fields = [
                # ipda ipsa dport prot
                self.fakeip, "1.1.1.1", port, "tcp",
                # connections packetsIn bytesIn packetsOut bytesOut
                0, count, 60*count, count, 60*count,
                # interface
                interface,
                ]
            f.write(records.encodeRecord(fields))
        self.logger.info("Added nmap with %d ports, %d count.", ports, count)

    def add_fake_netstatip(self, f, interface, fraction):
        "Adds some fake netstatip records for outgoing traffic."
        n = int(round(self.opts.extra * fraction))
        if n <= 0:
            return
        g = fake_ip_sequence(self.name)
        count = 0
        for i in range(n):
            ipaddr = next(g)
            if random.random() < 0.5:
                continue
            fields = [
                # ipda ipsa dport prot
                ipaddr, self.fakeip, 80, "tcp",
                # connections packetsIn bytesIn packetsOut bytesOut
                1, 20, 20000, 2, 2000,
                # interface
                interface,
                ]
            f.write(records.encodeRecord(fields))
            count += 1
        self.logger.info("Added %d fake netstatip records.", count)

    def add_fake_netosfp(self, f, interface, fraction):
        "Adds some fake netosfp records."
        n = int(round(self.opts.extra * fraction))
        if n <= 0:
            return
        g = fake_ip_sequence(self.name)
        now = int(time.time())
        count = 0
        for i in range(n):
            ipaddr = next(g)
            if random.random() < 0.5:
                continue
            fields = [
                # ip time_stamp ttl distanceByTTL
                ipaddr, now-i%1800, 128, 0,
                # osByStack osByLine osByBanner
                "Microsoft:Windows:NT", ":Unknown:", ":Unknown:",
                # interface
                interface,
                ]
            f.write(records.encodeRecord(fields))
            count += 1
        self.logger.info("Added %d fake netosfp records.", count)

def find_best_implementation(rule, ostype):
    if hasattr(rule, "implementations"):
        for ost in ostypes.allancestors(ostype):
            for impl in rule.implementations:
                ostypes_list = impl.ostypes.split(",")
                if ost in ostypes_list:
                    return impl.module, impl.procedure
    return None, None

def fake_ip_sequence(name):
    """A generator yielding fake IP addresses."""
    r = random.Random(hash(name))
    a = 12
    while True:
        bcd = r.randrange(1<<24)
        bc, d = divmod(bcd, 256)
        if d == 0 or d == 255:
            continue # Skip addresses ending in .0 or .255
        b, c = divmod(bc, 256)
        ipaddr = "%d.%d.%d.%d" % (a, b, c, d)
        yield ipaddr

def central_server(server, ssl=True):
    """Returns the central server's host and port.

    Arguments:

    server -- string giving the central server name
    ssl -- optional bool indicating whether the SSL port should be
           used (default True)

    """
    if ":" in server:
        host, port = server.split(":", 1)
        port = int(port)
    else:
        host = server
        port = config.ES_PORT
    if not ssl:
        port += 2
    return host, port

# This is actual poliwall data observed on my box.  Rather a large
# data set to have in line, but I really want this all in one file.

THISIP = "10.0.5.34"       # The IP address to be substituted
THISMAC = "00402b4dfac8"   # The corresponding MAC address (all lowercase)
THISFQDN = "nitrogen.esi.elementalsecurity.com." # The corresponding FQDN

fake_netarp_data = [
    ['netarp.eth0.1101758767.006-1101759211.004',
     'ip mac occ time_stamp',
     '10.0.5.34 00402b4dfac8 1 1101759201',
     '10.0.5.1 004010186457 2 1101758904',
     '10.0.5.254 000cdb69b3e0 9 1101758777',
    ],
    ['netarp.eth0.1101759211.000-1101759400.037',
     'ip mac occ time_stamp',
     '10.0.5.254 000cdb69b3e0 3 1101759221',
    ],
    ['netarp.eth0.1101759400.002-1101759778.003',
     'ip mac occ time_stamp',
     '10.0.5.254 000cdb69b3e0 8 1101759401',
    ],
    ['netarp.eth0.1101759778.000-1101759967.010',
     'ip mac occ time_stamp',
     '10.0.5.34 00402b4dfac8 1 1101759803',
     '10.0.5.254 000cdb69b3e0 4 1101759791',
    ],
    ['netarp.eth0.1101759967.005-1101760156.010',
     'ip mac occ time_stamp',
     '10.0.5.254 000cdb69b3e0 4 1101760031',
    ],
    ['netarp.eth0.1101760156.005-1101760471.005',
     'ip mac occ time_stamp',
     '10.0.5.34 00402b4dfac8 1 1101760405',
     '10.0.5.1 004010186457 1 1101760223',
     '10.0.5.254 000cdb69b3e0 9 1101760208',
    ],
]

fake_netname_data = [
    ['netname.eth0.1101758767.006-1101759211.004',
     'reqip respip nameip occ port time_stamp fqdn',
     '10.0.5.34 10.69.2.10 10.0.5.34 5362 53 1101758767 nitrogen.esi.elementalsecurity.com.',
     '10.0.5.34 10.69.2.10 207.142.131.205 1 53 1101759047 en.wikipedia.org.',
     '10.0.5.34 10.69.2.10 10.0.5.240 30 53 1101758786 dogfood.esi.elementalsecurity.com.',
     '10.0.5.34 10.69.2.10 192.5.16.65 1 53 1101759160 cesium.clock.org.',
     '10.0.5.34 10.69.2.10 207.142.131.248 1 53 1101759050 cache.wikimedia.org.',
    ],
    ['netname.eth0.1101759211.000-1101759400.037',
     'reqip respip nameip occ port time_stamp fqdn',
     '10.0.5.34 10.69.2.10 10.0.5.34 2225 53 1101759211 nitrogen.esi.elementalsecurity.com.',
     '10.0.5.34 10.69.2.10 10.0.5.240 14 53 1101759211 dogfood.esi.elementalsecurity.com.',
    ],
    ['netname.eth0.1101759400.002-1101759778.003',
     'reqip respip nameip occ port time_stamp fqdn',
     '10.0.5.34 10.69.2.10 10.0.5.34 9275 53 1101759400 nitrogen.esi.elementalsecurity.com.',
     '10.0.5.34 10.69.2.10 216.239.57.99 1 53 1101759623 www.google.akadns.net.',
     '10.0.5.34 10.69.2.10 158.64.4.126 1 53 1101759498 www.crpht.lu.',
     '10.0.5.34 10.69.2.10 216.239.57.98 1 53 1101759622 gmail.google.akadns.net.',
     '10.0.5.34 10.69.2.10 10.0.5.240 25 53 1101759412 dogfood.esi.elementalsecurity.com.',
    ],
    ['netname.eth0.1101759778.000-1101759967.010',
     'reqip respip nameip occ port time_stamp fqdn',
     '10.0.5.34 10.69.2.10 10.0.5.34 2303 53 1101759789 nitrogen.esi.elementalsecurity.com.',
     '10.0.5.34 10.69.2.10 63.208.170.222 1 53 1101759793 a1849.gc.akamai.net.',
     '10.0.5.34 10.69.2.10 159.33.3.85 1 53 1101759798 sdc.cbc.ca.',
     '10.0.5.34 10.69.2.10 10.0.5.240 13 53 1101759788 dogfood.esi.elementalsecurity.com.',
    ],
    ['netname.eth0.1101759967.005-1101760156.010',
     'reqip respip nameip occ port time_stamp fqdn',
     '10.0.5.34 10.69.2.10 10.0.5.34 4009 53 1101759967 nitrogen.esi.elementalsecurity.com.',
     '10.0.5.34 10.69.2.10 10.0.5.240 13 53 1101759982 dogfood.esi.elementalsecurity.com.',
    ],
    ['netname.eth0.1101760156.005-1101760471.005',
     'reqip respip nameip occ port time_stamp fqdn',
     '10.0.5.34 10.69.2.10 10.0.5.34 6510 53 1101760177 nitrogen.esi.elementalsecurity.com.',
     '10.0.5.34 10.69.2.10 10.0.5.240 21 53 1101760169 dogfood.esi.elementalsecurity.com.',
    ],
    ['netname.eth0.1101760471.000-1101760474.012',
     'reqip respip nameip occ port time_stamp fqdn',
     '10.0.5.34 10.69.2.10 10.0.5.34 142 53 1101760471 nitrogen.esi.elementalsecurity.com.',
    ],
]

fake_netosfp_data = [
    ['netosfp.eth0.1101758767.006-1101759211.004',
     'ip time_stamp ttl distanceByTTL osByStack osByLine osByBanner',
     '10.0.5.240 1101758767 64 0 :OpenBSD:3.4 :Unknown: :Unknown:',
     '10.0.5.50 1101759005 128 0 Microsoft:Windows:2000 :Unknown: :Unknown:',
     '10.69.2.10 1101758767 127 1 Microsoft:Windows:NT_4 :Unknown: :Unknown:',
     '10.0.5.73 1101758774 64 0 :NetBSD:1.6 :Unknown: :Unknown:',
     '207.142.131.205 1101759047 56 8 :FreeBSD:5.1 :Unknown: :Unknown:',
     '10.0.5.96 1101759100 128 0 Microsoft:Windows:98 :Unknown: :Unknown:',
     '10.0.5.60 1101758980 128 0 Microsoft:Windows:95 :Unknown: :Unknown:',
     '10.0.5.42 1101758809 128 0 Microsoft:Windows:2003_Server :Unknown: :Unknown:',
     '10.0.5.83 1101759139 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.65 1101759196 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '216.239.57.103 1101758899 248 7 Sun_Microsystems:Solaris:2.9 :Unknown: :Unknown:',
     '10.0.5.11 1101758767 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.52 1101758820 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.34 1101758767 64 0 :Linux:2.4 :Unknown: :Unknown:',
     '10.0.5.93 1101758905 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.75 1101759089 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '207.142.131.248 1101759050 56 8 :Linux:2.4/2.6 :Unknown: :Unknown:',
     '10.0.5.98 1101758799 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.3 1101758797 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.44 1101758770 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.49 1101758779 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.31 1101758909 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.13 1101759045 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.95 1101758829 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '128.9.160.27 1101759194 53 11 :Linux:2.6.8 :Unknown: :Unknown:',
     '10.0.5.195 1101758767 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.41 1101758874 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.46 1101758941 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.28 1101758922 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.10 1101758797 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.223 1101758783 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.51 1101759068 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.15 1101758839 1 31 Microsoft:Windows:9x :Unknown: :Unknown:',
     '10.69.2.11 1101758786 127 1 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.210 1101759068 64 0 Hewlett_Packard:HP-UX:11.00 :Unknown: :Unknown:',
     '10.0.5.97 1101759042 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.43 1101759140 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '192.5.16.65 1101759160 113 15 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.48 1101758795 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.40 1101758805 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.22 1101758768 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.4 1101758767 64 0 IBM:AIX:5.2 :Unknown: :Unknown:',
    ],
    ['netosfp.eth0.1101759211.000-1101759400.037',
     'ip time_stamp ttl distanceByTTL osByStack osByLine osByBanner',
     '10.0.5.45 1101759263 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.240 1101759211 64 0 :OpenBSD:3.4 :Unknown: :Unknown:',
     '10.69.2.10 1101759211 127 1 Microsoft:Windows:NT_4 :Unknown: :Unknown:',
     '10.0.5.73 1101759374 64 0 :NetBSD:1.6 :Unknown: :Unknown:',
     '10.0.5.42 1101759335 128 0 Microsoft:Windows:2003_Server :Unknown: :Unknown:',
     '216.239.57.103 1101759323 248 7 Sun_Microsystems:Solaris:2.9 :Unknown: :Unknown:',
     '10.0.5.47 1101759340 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.11 1101759396 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.34 1101759211 64 0 :Linux:2.4 :Unknown: :Unknown:',
     '10.0.5.98 1101759229 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.3 1101759238 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.44 1101759211 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '128.9.160.27 1101759224 53 11 :Linux:2.6.8 :Unknown: :Unknown:',
     '10.0.5.195 1101759211 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.28 1101759261 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.10 1101759238 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.69.2.11 1101759211 127 1 Microsoft:Windows:XP :Unknown: :Unknown:',
     '192.5.16.65 1101759316 113 15 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.22 1101759211 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.4 1101759211 64 0 IBM:AIX:5.2 :Unknown: :Unknown:',
    ],
    ['netosfp.eth0.1101759400.002-1101759778.003',
     'ip time_stamp ttl distanceByTTL osByStack osByLine osByBanner',
     '0.0.0.0 1101759492 20 12 Microsoft:Windows:9x :Unknown: :Unknown:',
     '10.0.5.240 1101759412 64 0 :OpenBSD:3.4 :Unknown: :Unknown:',
     '10.0.5.50 1101759725 128 0 Microsoft:Windows:2000 :Unknown: :Unknown:',
     '10.69.2.10 1101759400 127 1 Microsoft:Windows:NT_4 :Unknown: :Unknown:',
     '10.0.5.73 1101759527 64 0 :NetBSD:1.6 :Unknown: :Unknown:',
     '10.0.5.60 1101759697 128 0 Microsoft:Windows:95 :Unknown: :Unknown:',
     '216.239.57.98 1101759622 247 8 Sun_Microsystems:Solaris:2.8 :Unknown: :Unknown:',
     '10.0.5.42 1101759467 128 0 Microsoft:Windows:2003_Server :Unknown: :Unknown:',
     '10.0.5.65 1101759593 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '216.239.57.103 1101759669 248 7 Sun_Microsystems:Solaris:2.9 :Unknown: :Unknown:',
     '10.0.5.11 1101759487 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.52 1101759543 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.34 1101759400 64 0 :Linux:2.4 :Unknown: :Unknown:',
     '10.0.5.93 1101759626 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.75 1101759741 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.57 1101759433 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.98 1101759456 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.44 1101759402 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.49 1101759501 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.31 1101759628 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.13 1101759406 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.95 1101759548 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '128.9.160.27 1101759513 53 11 :Linux:2.6.8 :Unknown: :Unknown:',
     '10.0.5.195 1101759402 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.41 1101759592 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.254 1101759492 64 0 :*NIX: :Unknown: :Unknown:',
     '10.0.5.46 1101759659 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.10 1101759574 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.223 1101759505 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.51 1101759532 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.15 1101759443 1 31 Microsoft:Windows:9x :Unknown: :Unknown:',
     '10.69.2.11 1101759412 127 1 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.97 1101759761 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '216.239.57.99 1101759623 247 8 Sun_Microsystems:Solaris:2.9 :Unknown: :Unknown:',
     '192.5.16.65 1101759532 113 15 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.48 1101759516 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '158.64.4.126 1101759498 110 18 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.53 1101759414 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.40 1101759527 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.22 1101759400 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.4 1101759400 64 0 IBM:AIX:5.2 :Unknown: :Unknown:',
    ],
    ['netosfp.eth0.1101759778.000-1101759967.010',
     'ip time_stamp ttl distanceByTTL osByStack osByLine osByBanner',
     '10.0.5.240 1101759788 64 0 :OpenBSD:3.4 :Unknown: :Unknown:',
     '10.69.2.10 1101759778 127 1 Microsoft:Windows:NT_4 :Unknown: :Unknown:',
     '10.0.5.96 1101759822 128 0 Microsoft:Windows:98 :Unknown: :Unknown:',
     '216.239.57.98 1101759778 56 8 Sun_Microsystems:Solaris:2.8 :Unknown: :Unknown:',
     '10.0.5.42 1101759861 128 0 Microsoft:Windows:2003_Server :Unknown: :Unknown:',
     '10.0.5.83 1101759809 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.65 1101759919 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '216.239.57.103 1101759850 248 7 Sun_Microsystems:Solaris:2.9 :Unknown: :Unknown:',
     '10.0.5.11 1101759784 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.34 1101759778 64 0 :Linux:2.4 :Unknown: :Unknown:',
     '10.0.5.75 1101759790 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.98 1101759853 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.3 1101759823 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.44 1101759778 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.31 1101759960 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.195 1101759781 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.28 1101759822 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.10 1101759823 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.51 1101759790 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.69.2.11 1101759788 127 1 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.210 1101759790 64 0 Hewlett_Packard:HP-UX:11.00 :Unknown: :Unknown:',
     '159.33.3.85 1101759798 111 17 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.43 1101759859 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.22 1101759778 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '63.208.170.222 1101759793 57 7 :*NIX: :Unknown: :Unknown:',
     '10.0.5.4 1101759778 64 0 IBM:AIX:5.2 :Unknown: :Unknown:',
    ],
    ['netosfp.eth0.1101759967.005-1101760156.010',
     'ip time_stamp ttl distanceByTTL osByStack osByLine osByBanner',
     '0.0.0.0 1101760052 20 12 Microsoft:Windows:9x :Unknown: :Unknown:',
     '10.0.5.45 1101759981 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.240 1101759967 64 0 :OpenBSD:3.4 :Unknown: :Unknown:',
     '10.69.2.10 1101759967 127 1 Microsoft:Windows:NT_4 :Unknown: :Unknown:',
     '10.0.5.73 1101759974 64 0 :NetBSD:1.6 :Unknown: :Unknown:',
     '216.239.57.98 1101759970 56 8 Sun_Microsystems:Solaris:2.8 :Unknown: :Unknown:',
     '10.0.5.42 1101759993 128 0 Microsoft:Windows:2003_Server :Unknown: :Unknown:',
     '10.0.5.47 1101760063 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.11 1101760098 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.34 1101759967 64 0 :Linux:2.4 :Unknown: :Unknown:',
     '10.0.5.75 1101760085 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.57 1101760154 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.98 1101759984 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.3 1101760129 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.44 1101759967 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.13 1101760125 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.195 1101759968 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.254 1101760052 64 0 :*NIX: :Unknown: :Unknown:',
     '10.0.5.28 1101759981 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.10 1101760129 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.15 1101760045 1 31 Microsoft:Windows:9x :Unknown: :Unknown:',
     '10.69.2.11 1101759982 127 1 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.53 1101760134 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.22 1101759968 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '63.208.170.222 1101760105 57 7 :*NIX: :Unknown: :Unknown:',
     '10.0.5.4 1101759967 64 0 :*NIX: :Unknown: :Unknown:',
    ],
    ['netosfp.eth0.1101760156.005-1101760471.005',
     'ip time_stamp ttl distanceByTTL osByStack osByLine osByBanner',
     '10.0.5.240 1101760169 64 0 :OpenBSD:3.4 :Unknown: :Unknown:',
     '10.0.5.50 1101760447 128 0 Microsoft:Windows:2000 :Unknown: :Unknown:',
     '10.69.2.10 1101760156 127 1 Microsoft:Windows:NT_4 :Unknown: :Unknown:',
     '10.0.5.73 1101760246 64 0 :NetBSD:1.6 :Unknown: :Unknown:',
     '10.0.5.60 1101760416 128 0 Microsoft:Windows:95 :Unknown: :Unknown:',
     '216.239.57.98 1101760218 247 8 Sun_Microsystems:Solaris:2.8 :Unknown: :Unknown:',
     '10.0.5.42 1101760255 128 0 Microsoft:Windows:2003_Server :Unknown: :Unknown:',
     '10.0.5.11 1101760169 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.52 1101760265 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.34 1101760156 64 0 :Linux:2.4 :Unknown: :Unknown:',
     '10.0.5.93 1101760343 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.98 1101760247 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.44 1101760156 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.49 1101760221 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.31 1101760346 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.13 1101760430 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.95 1101760269 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.195 1101760157 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.41 1101760310 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.100 1101760208 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.46 1101760379 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.10 1101760294 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.223 1101760227 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.51 1101760432 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.15 1101760294 1 31 Microsoft:Windows:9x :Unknown: :Unknown:',
     '10.69.2.11 1101760169 127 1 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.48 1101760235 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.40 1101760245 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.22 1101760157 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.4 1101760156 64 0 :*NIX: :Unknown: :Unknown:',
    ],
    ['netosfp.eth0.1101760471.000-1101760474.012',
     'ip time_stamp ttl distanceByTTL osByStack osByLine osByBanner',
     '10.69.2.10 1101760471 127 1 Microsoft:Windows:NT_4 :Unknown: :Unknown:',
     '10.0.5.34 1101760471 64 0 :Linux:2.4 :Unknown: :Unknown:',
     '10.0.5.44 1101760472 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.195 1101760471 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.22 1101760471 128 0 Microsoft:Windows:XP :Unknown: :Unknown:',
     '10.0.5.4 1101760471 64 0 :*NIX: :Unknown: :Unknown:',
    ],
]

fake_netstatip_data = [
    ['netstatip.eth0.1101758767.006-1101759211.004',
     'ipda ipsa dport prot connections packetsIn bytesIn packetsOut bytesOut',
     '10.0.5.240 10.0.5.34 17749 tcp 27 163 18813 177 17036',
     '10.0.5.255 10.0.5.210 138 udp 0 0 0 2 509',
     '10.0.5.255 10.0.5.42 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.52 137 udp 0 0 0 3 276',
     '10.0.5.255 10.0.5.3 138 udp 0 0 0 2 486',
     '10.0.5.255 10.0.5.96 138 udp 0 0 0 1 243',
     '10.0.5.44 10.0.5.34 4715 tcp 0 4 417 5 486',
     '10.0.5.255 10.0.5.28 138 udp 0 0 0 1 253',
     '255.255.255.255 10.0.5.22 5000 udp 0 0 0 274 16440',
     '10.0.5.255 10.0.5.10 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.4 17749 tcp 2720 16353 1870978 16798 1645503',
     '10.0.5.255 10.0.5.60 137 udp 0 0 0 1 92',
     '10.0.5.4 10.0.5.34 1521 tcp 1342 124236 29552989 131826 21717400',
     '10.0.5.240 10.0.5.34 17747 tcp 2 90 13512 126 133916',
     '10.0.5.255 10.0.5.11 138 udp 0 0 0 1 243',
     '192.5.16.65 10.0.5.34 80 tcp 1 8 6090 9 1150',
     '255.255.255.255 10.0.5.195 5000 udp 0 0 0 276 16560',
     '10.0.5.255 10.0.5.65 138 udp 0 0 0 1 243',
     '10.0.5.44 10.0.5.34 4713 tcp 0 2 120 3 379',
     '10.0.5.255 10.0.5.40 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.98 164 udp 0 0 0 4 472',
     '10.0.5.255 10.0.5.51 138 udp 0 0 0 1 285',
     '10.0.5.34 10.0.5.4 17747 tcp 48 1165 256817 1753 1808153',
     '10.0.5.255 10.0.5.223 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.15 137 udp 0 0 0 2 184',
     '10.0.5.255 10.0.5.73 138 udp 0 0 0 1 251',
     '10.0.5.44 10.0.5.34 4139 tcp 0 1 60 1 54',
     '10.0.5.255 10.0.5.83 137 udp 0 0 0 9 828',
     '10.69.2.11 10.0.5.34 53 udp 54 54 3618 54 3618',
     '10.0.5.44 10.0.5.34 4711 tcp 0 2 120 3 379',
     '10.0.5.255 10.0.5.95 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.42 164 udp 0 0 0 4 476',
     '10.0.5.255 10.0.5.52 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 17749 tcp 680 5270 2130724 4840 412097',
     '10.0.5.255 10.0.5.31 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.49 138 udp 0 0 0 1 243',
     '207.142.131.205 10.0.5.34 80 tcp 8 83 70208 89 11216',
     '10.0.5.255 10.0.5.13 137 udp 0 0 0 1 92',
     '10.0.5.255 10.0.5.10 138 udp 0 0 0 2 486',
     '10.0.5.255 10.0.5.60 138 udp 0 0 0 1 243',
     '207.142.131.248 10.0.5.34 80 tcp 1 15 14831 17 1987',
     '10.0.5.255 10.0.5.46 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.43 138 udp 0 0 0 1 243',
     '10.0.5.44 10.0.5.34 4714 tcp 0 8 2211 12 3979',
     '10.0.5.34 10.0.5.44 17747 tcp 379 5820 2003360 4677 1291436',
     '10.0.5.255 10.0.5.93 138 udp 0 0 0 1 260',
     '10.0.5.255 10.0.5.50 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.97 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.28 137 udp 0 0 0 9 828',
     '216.239.57.103 10.0.5.34 80 tcp 1 13 6472 13 2169',
     '10.0.5.255 10.0.5.15 138 udp 0 0 0 1 278',
     '10.0.5.255 10.0.5.11 137 udp 0 0 0 3 276',
     '10.0.5.255 10.0.5.22 138 udp 0 0 0 1 243',
     '10.0.5.44 10.0.5.34 4712 tcp 0 2 120 3 378',
     '10.0.5.255 10.0.5.44 138 udp 0 0 0 1 265',
     '10.0.5.4 10.0.5.34 22 tcp 0 984 179240 984 64944',
     '10.0.5.255 10.0.5.41 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.48 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.73 137 udp 0 0 0 4 368',
     '128.9.160.27 10.0.5.34 80 tcp 1 17 20196 18 1692',
     '10.69.2.10 10.0.5.34 53 udp 11998 11998 1449286 11996 1047825',
     '255.255.255.255 10.0.5.75 137 udp 0 0 0 4 440',
     '10.0.5.255 10.0.5.13 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 8080 tcp 7 88 36111 67 8934',
    ],
    ['netstatip.eth0.1101759211.000-1101759400.037',
     'ipda ipsa dport prot connections packetsIn bytesIn packetsOut bytesOut',
     '10.0.5.240 10.0.5.34 17749 tcp 13 78 8284 82 7990',
     '10.0.5.255 10.0.5.28 138 udp 0 0 0 1 275',
     '255.255.255.255 10.0.5.22 5000 udp 0 0 0 116 6960',
     '10.0.5.255 10.0.5.10 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.4 17749 tcp 1073 6521 850633 6808 661133',
     '10.0.5.4 10.0.5.34 1521 tcp 557 59806 9664655 65507 10667969',
     '10.0.5.255 10.0.5.11 138 udp 0 0 0 1 247',
     '192.5.16.65 10.0.5.34 80 tcp 1 8 6227 10 1172',
     '10.0.5.255 10.0.5.47 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.195 5000 udp 0 0 0 118 7080',
     '255.255.255.255 10.0.5.98 164 udp 0 0 0 1 118',
     '10.0.5.34 10.0.5.4 17747 tcp 58 1404 294681 2097 2190912',
     '10.69.2.11 10.0.5.34 53 udp 26 26 1742 26 1742',
     '10.0.5.255 10.0.5.45 138 udp 0 0 0 1 243',
     '10.0.5.44 10.0.5.34 3383 tcp 0 1 60 1 54',
     '255.255.255.255 10.0.5.42 164 udp 0 0 0 1 119',
     '10.0.5.34 10.0.5.44 17749 tcp 687 4515 996924 4404 387307',
     '10.0.5.255 10.0.5.3 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.44 17747 tcp 194 2866 846106 2353 707395',
     '216.239.57.103 10.0.5.34 80 tcp 0 2 120 2 108',
     '10.0.5.255 10.0.5.44 138 udp 0 0 0 1 248',
     '10.0.5.4 10.0.5.34 22 tcp 0 392 69536 392 25872',
     '10.0.5.255 10.0.5.98 138 udp 0 0 0 1 251',
     '10.0.5.255 10.0.5.195 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.73 137 udp 0 0 0 2 184',
     '128.9.160.27 10.0.5.34 80 tcp 1 36 42586 38 3508',
     '10.69.2.10 10.0.5.34 53 udp 5409 5409 657006 5409 470160',
     '10.0.5.34 10.0.5.44 8080 tcp 2 30 16129 23 2784',
     '10.0.5.44 10.0.5.34 3382 tcp 0 1 60 1 54',
    ],
    ['netstatip.eth0.1101759400.002-1101759778.003',
     'ipda ipsa dport prot connections packetsIn bytesIn packetsOut bytesOut',
     '10.0.5.240 10.0.5.34 17749 tcp 24 144 15038 156 15061',
     '10.0.5.255 10.0.5.42 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.52 137 udp 0 0 0 3 276',
     '10.0.5.44 10.0.5.34 4162 tcp 0 4 418 5 487',
     '10.0.5.255 10.0.5.53 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.22 5000 udp 0 0 0 234 14040',
     '10.0.5.255 10.0.5.10 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.4 17749 tcp 2179 13636 2451341 14110 1360944',
     '10.0.5.4 10.0.5.34 1521 tcp 2317 159584 23956923 175178 28406741',
     '192.5.16.65 10.0.5.34 80 tcp 3 72 88240 66 7428',
     '255.255.255.255 10.0.5.195 5000 udp 0 0 0 236 14160',
     '216.239.57.99 10.0.5.34 443 tcp 2 21 5459 22 4657',
     '10.0.5.44 10.0.5.34 4160 tcp 0 1 60 1 54',
     '10.0.5.255 10.0.5.40 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.98 164 udp 0 0 0 3 354',
     '10.0.5.255 10.0.5.51 138 udp 0 0 0 1 251',
     '10.0.5.34 10.0.5.4 17747 tcp 79 1996 408952 2999 3198188',
     '10.0.5.255 10.0.5.15 137 udp 0 0 0 2 184',
     '10.0.5.255 10.0.5.223 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.73 138 udp 0 0 0 1 251',
     '10.69.2.11 10.0.5.34 53 udp 48 48 3216 48 3216',
     '10.0.5.255 10.0.5.95 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.195 137 udp 0 0 0 5 460',
     '255.255.255.255 10.0.5.42 164 udp 0 0 0 3 357',
     '10.0.5.255 10.0.5.52 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 17749 tcp 1055 6728 1394057 6685 589969',
     '10.0.5.255 10.0.5.31 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.49 138 udp 0 0 0 1 243',
     '158.64.4.126 10.0.5.34 80 tcp 1 5 734 7 1008',
     '10.0.5.255 10.0.5.13 137 udp 0 0 0 5 460',
     '216.239.57.98 10.0.5.34 80 tcp 2 131 113067 127 37954',
     '10.0.5.255 10.0.5.60 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.46 138 udp 0 0 0 1 243',
     '216.239.57.98 10.0.5.34 443 tcp 2 29 11345 29 5878',
     '10.0.5.255 10.0.5.57 138 udp 0 0 0 1 243',
     '10.0.5.254 255.255.255.255 67 udp 3 3 1062 0 0',
     '10.0.5.255 10.0.5.100 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 17747 tcp 230 4020 1443368 3474 1585953',
     '10.0.5.255 10.0.5.93 138 udp 0 0 0 1 260',
     '255.255.255.255 0.0.0.0 67 udp 0 0 0 6 2568',
     '10.0.5.255 10.0.5.50 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.97 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.75 137 udp 0 0 0 31 3410',
     '216.239.57.103 10.0.5.34 80 tcp 1 2 239 4 931',
     '10.0.5.255 10.0.5.15 138 udp 0 0 0 1 278',
     '10.0.5.255 10.0.5.11 137 udp 0 0 0 4 368',
     '10.0.5.255 10.0.5.22 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.65 137 udp 0 0 0 1 92',
     '10.0.5.255 10.0.5.44 138 udp 0 0 0 1 265',
     '10.0.5.255 10.0.5.40 137 udp 0 0 0 4 368',
     '10.0.5.4 10.0.5.34 22 tcp 0 771 139274 771 50886',
     '10.0.5.255 10.0.5.41 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.48 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.73 137 udp 0 0 0 2 184',
     '128.9.160.27 10.0.5.34 80 tcp 1 18 18784 19 1758',
     '10.69.2.10 10.0.5.34 53 udp 17577 17577 2096663 17578 1552537',
     '10.0.5.255 10.0.5.13 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 8080 tcp 6 79 37154 60 7812',
    ],
    ['netstatip.eth0.1101759778.000-1101759967.010',
     'ipda ipsa dport prot connections packetsIn bytesIn packetsOut bytesOut',
     '10.0.5.240 10.0.5.34 17749 tcp 12 71 7615 76 7402',
     '10.0.5.255 10.0.5.210 138 udp 0 0 0 2 509',
     '10.0.5.255 10.0.5.3 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.96 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.31 137 udp 0 0 0 1 92',
     '10.0.5.255 10.0.5.28 138 udp 0 0 0 1 253',
     '10.0.5.255 10.0.5.75 138 udp 0 0 0 1 216',
     '255.255.255.255 10.0.5.22 5000 udp 0 0 0 118 7080',
     '10.0.5.34 10.0.5.4 17749 tcp 1020 6150 810206 6379 621676',
     '10.0.5.4 10.0.5.34 1521 tcp 577 63427 9242813 69147 10201952',
     '10.0.5.255 10.0.5.11 138 udp 0 0 0 1 243',
     '63.208.170.222 10.0.5.34 80 tcp 2 94 88050 100 29145',
     '159.33.3.85 10.0.5.34 80 tcp 1 5 826 6 1003',
     '255.255.255.255 10.0.5.195 5000 udp 0 0 0 116 6960',
     '10.0.5.255 10.0.5.65 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.98 164 udp 0 0 0 1 118',
     '10.0.5.255 10.0.5.51 138 udp 0 0 0 1 285',
     '10.0.5.34 10.0.5.4 17747 tcp 38 834 189233 1215 1214557',
     '10.0.5.255 10.0.5.83 137 udp 0 0 0 18 1656',
     '10.69.2.11 10.0.5.34 53 udp 24 24 1608 24 1608',
     '255.255.255.255 10.0.5.42 164 udp 0 0 0 1 119',
     '10.0.5.34 10.0.5.44 17749 tcp 578 3483 446670 3507 313661',
     '216.239.57.98 10.0.5.34 80 tcp 0 117 49632 121 45498',
     '10.0.5.255 10.0.5.10 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.75 67 udp 0 0 0 2 684',
     '10.0.5.255 10.0.5.43 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 17747 tcp 108 1995 491589 1903 1296974',
     '10.0.5.44 10.0.5.34 3679 tcp 0 8 1883 12 4083',
     '10.0.5.255 10.0.5.28 137 udp 0 0 0 9 828',
     '10.0.5.255 10.0.5.75 137 udp 0 0 0 7 734',
     '216.239.57.103 10.0.5.34 80 tcp 0 2 120 1 54',
     '10.0.5.255 10.0.5.11 137 udp 0 0 0 1 92',
     '10.0.5.255 10.0.5.83 138 udp 0 0 0 3 648',
     '10.0.5.4 10.0.5.34 22 tcp 0 466 83212 466 30756',
     '10.0.5.255 10.0.5.98 138 udp 0 0 0 1 251',
     '10.69.2.10 10.0.5.34 53 udp 5263 5263 636603 5264 459194',
     '10.0.5.34 10.0.5.44 8080 tcp 3 39 16702 30 3906',
    ],
    ['netstatip.eth0.1101759967.005-1101760156.010',
     'ipda ipsa dport prot connections packetsIn bytesIn packetsOut bytesOut',
     '10.0.5.240 10.0.5.34 17749 tcp 12 73 7686 78 7528',
     '10.0.5.255 10.0.5.3 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.53 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.28 138 udp 0 0 0 1 275',
     '10.0.5.255 10.0.5.75 138 udp 0 0 0 3 706',
     '255.255.255.255 10.0.5.22 5000 udp 0 0 0 118 7080',
     '10.0.5.255 10.0.5.10 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.4 17749 tcp 992 5958 718673 6193 604405',
     '10.0.5.4 10.0.5.34 1521 tcp 1003 74721 15299338 81856 13478203',
     '10.0.5.255 10.0.5.11 138 udp 0 0 0 1 243',
     '63.208.170.222 10.0.5.34 80 tcp 0 2 132 4 264',
     '10.0.5.255 10.0.5.47 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.195 5000 udp 0 0 0 118 7080',
     '255.255.255.255 10.0.5.98 164 udp 0 0 0 2 236',
     '10.0.5.34 10.0.5.4 17747 tcp 37 1060 205551 1573 1702655',
     '10.0.5.255 10.0.5.15 137 udp 0 0 0 1 92',
     '10.69.2.11 10.0.5.34 53 udp 24 24 1608 24 1608',
     '10.0.5.255 10.0.5.45 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.42 164 udp 0 0 0 2 238',
     '10.0.5.34 10.0.5.44 17749 tcp 501 3049 358396 3049 272672',
     '216.239.57.98 10.0.5.34 80 tcp 0 43 20496 46 14263',
     '10.0.5.255 10.0.5.10 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.57 138 udp 0 0 0 1 243',
     '10.0.5.254 255.255.255.255 67 udp 3 3 1062 0 0',
     '10.0.5.255 10.0.5.3 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.44 17747 tcp 77 1600 358601 1550 1136785',
     '255.255.255.255 0.0.0.0 67 udp 0 0 0 3 1551',
     '10.0.5.255 10.0.5.75 137 udp 0 0 0 30 3192',
     '10.0.5.4 10.0.5.34 22 tcp 0 407 70466 407 26862',
     '10.0.5.255 10.0.5.195 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.73 137 udp 0 0 0 2 184',
     '10.69.2.10 10.0.5.34 53 udp 7675 7675 916063 7675 677389',
     '10.0.5.255 10.0.5.13 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 8080 tcp 3 37 15405 28 3786',
    ],
    ['netstatip.eth0.1101760156.005-1101760471.005',
     'ipda ipsa dport prot connections packetsIn bytesIn packetsOut bytesOut',
     '10.0.5.240 10.0.5.34 17749 tcp 20 120 12847 127 12354',
     '10.0.5.255 10.0.5.42 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.22 5000 udp 0 0 0 192 11520',
     '10.0.5.255 10.0.5.10 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.4 17749 tcp 1539 9304 1184028 9754 947664',
     '10.0.5.4 10.0.5.34 1521 tcp 1622 131065 21279648 143003 21500313',
     '10.0.5.255 10.0.5.11 138 udp 0 0 0 1 247',
     '10.0.5.255 10.0.5.100 137 udp 0 0 0 1 92',
     '255.255.255.255 10.0.5.195 5000 udp 0 0 0 194 11640',
     '10.0.5.255 10.0.5.40 138 udp 0 0 0 1 243',
     '255.255.255.255 10.0.5.98 164 udp 0 0 0 2 236',
     '10.0.5.255 10.0.5.51 138 udp 0 0 0 1 251',
     '10.0.5.34 10.0.5.4 17747 tcp 102 2472 518342 3497 3549302',
     '10.0.5.255 10.0.5.223 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.15 137 udp 0 0 0 1 92',
     '10.0.5.255 10.0.5.73 138 udp 0 0 0 1 251',
     '10.69.2.11 10.0.5.34 53 udp 40 40 2680 40 2680',
     '10.0.5.255 10.0.5.95 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.195 137 udp 0 0 0 2 184',
     '255.255.255.255 10.0.5.42 164 udp 0 0 0 2 238',
     '10.0.5.255 10.0.5.52 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 17749 tcp 869 5275 648063 5284 472571',
     '10.0.5.255 10.0.5.31 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.49 138 udp 0 0 0 1 243',
     '10.0.5.44 10.0.5.34 3525 tcp 0 28 36363 21 1180',
     '216.239.57.98 10.0.5.34 80 tcp 0 7 1384 8 2365',
     '10.0.5.255 10.0.5.60 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.46 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.100 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 17747 tcp 165 3197 749361 3078 2256855',
     '10.0.5.255 10.0.5.93 138 udp 0 0 0 1 260',
     '10.0.5.255 10.0.5.50 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.15 138 udp 0 0 0 1 278',
     '10.0.5.255 10.0.5.11 137 udp 0 0 0 2 184',
     '10.0.5.255 10.0.5.22 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.44 138 udp 0 0 0 2 513',
     '10.0.5.4 10.0.5.34 22 tcp 0 822 145188 819 54054',
     '10.0.5.255 10.0.5.41 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.223 137 udp 0 0 0 1 92',
     '10.0.5.255 10.0.5.48 138 udp 0 0 0 1 243',
     '10.0.5.255 10.0.5.73 137 udp 0 0 0 2 184',
     '10.69.2.10 10.0.5.34 53 udp 12542 12542 1497848 12545 1106693',
     '10.0.5.255 10.0.5.13 138 udp 0 0 0 1 243',
     '10.0.5.34 10.0.5.44 8080 tcp 3 48 28302 36 4793',
    ],
    ['netstatip.eth0.1101760471.000-1101760474.012',
     'ipda ipsa dport prot connections packetsIn bytesIn packetsOut bytesOut',
     '255.255.255.255 10.0.5.22 5000 udp 0 0 0 2 120',
     '10.0.5.34 10.0.5.4 17749 tcp 9 140 97438 114 9296',
     '10.0.5.4 10.0.5.34 1521 tcp 36 1338 320406 1530 231400',
     '255.255.255.255 10.0.5.195 5000 udp 0 0 0 2 120',
     '10.0.5.34 10.0.5.4 17747 tcp 0 1 66 0 0',
     '10.0.5.255 10.0.5.195 137 udp 0 0 0 1 92',
     '10.0.5.34 10.0.5.44 17749 tcp 25 153 14395 152 13604',
     '10.0.5.34 10.0.5.44 17747 tcp 0 4 554 3 180',
     '10.0.5.4 10.0.5.34 22 tcp 0 18 4116 18 1188',
     '10.69.2.10 10.0.5.34 53 udp 249 249 29530 249 22122',
    ],
]

fake_netstatproto_data = [
    ['netstatproto.eth0.1101758767.006-1101759211.004',
     'mproto nproto packets bytes',
     'ip udp 24728 2549461',
     'ip tcp 315745 63306677',
     'arp arpreq 585 34902',
     'arp arpres 12 702',
     'ip igmp 6 318',
    ],
    ['netstatproto.eth0.1101759211.000-1101759400.037',
     'mproto nproto packets bytes',
     'ip udp 11117 1147045',
     'ip icmp 2 140',
     'ip tcp 157378 27452259',
     'arp arpreq 218 13026',
     'arp arpres 3 180',
     'ip igmp 4 212',
    ],
    ['netstatproto.eth0.1101759400.002-1101759778.003',
     'mproto nproto packets bytes',
     'ip udp 35817 3699746',
     'ip tcp 390954 65360162',
     'arp arpreq 525 31356',
     'arp arpres 8 480',
     'ip igmp 6 318',
    ],
    ['netstatproto.eth0.1101759778.000-1101759967.010',
     'mproto nproto packets bytes',
     'ip udp 10864 1120996',
     'ip icmp 2 140',
     'ip tcp 159654 25199218',
     'arp arpreq 248 14808',
     'arp arpres 5 282',
     'ip igmp 2 106',
    ],
    ['netstatproto.eth0.1101759967.005-1101760156.010',
     'mproto nproto packets bytes',
     'ip udp 15692 1620735',
     'ip tcp 181734 34302167',
     'arp arpreq 254 15168',
     'arp arpres 4 240',
     'ip igmp 4 212',
    ],
    ['netstatproto.eth0.1101760156.005-1101760471.005',
     'mproto nproto packets bytes',
     'ip udp 25589 2639900',
     'ip icmp 2 140',
     'ip tcp 317965 53404977',
     'arp arpreq 397 23640',
     'arp arpres 11 642',
     'ip igmp 4 212',
    ],
    ['netstatproto.eth0.1101760471.000-1101760474.012',
     'mproto nproto packets bytes',
     'ip udp 503 51984',
     'ip tcp 3471 692643',
     'arp arpreq 4 240',
    ],
]

fake_poliwall_data = {
    "netarp": fake_netarp_data,
    "netname": fake_netname_data,
    "netosfp": fake_netosfp_data,
    "netstatip": fake_netstatip_data,
    "netstatproto": fake_netstatproto_data,
    }

# Fake netstat data, see gather_netstat()
# This is a sample from host 'guido', a RHE 3 desktop, around 10am on 1/25/05

fake_netstat_data = [
    ('tcp', '0.0.0.0', '22497', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '127.0.0.1', '32774', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '0.0.0.0', '111', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '0.0.0.0', '6000', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '127.0.0.1', '17748', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '0.0.0.0', '22', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '127.0.0.1', '631', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '127.0.0.1', '25', '0.0.0.0', '*', 'LISTEN'),
    ('tcp', '127.0.0.1', '631', '127.0.0.1', '44236', 'ESTABLISHED'),
    ('tcp', '127.0.0.1', '44236', '127.0.0.1', '631', 'ESTABLISHED'),
    ('tcp', '10.0.5.62', '47263', '216.239.57.107', '80', 'ESTABLISHED'),
    ('udp', '0.0.0.0', '68', '0.0.0.0', '*', ''),
    ('udp', '0.0.0.0', '111', '0.0.0.0', '*', ''),
    ('udp', '0.0.0.0', '631', '0.0.0.0', '*', ''),
    ('udp', '10.0.5.62', '123', '0.0.0.0', '*', ''),
    ('udp', '127.0.0.1', '123', '0.0.0.0', '*', ''),
    ('udp', '0.0.0.0', '123', '0.0.0.0', '*', ''),
]

if __name__ == "__main__":
    sys.exit(main())
