# Copyright 2026 SigmaSRC, Inc. All Rights Reserved.

"""API for writing data records to a file and reading them back.

This is used by gathering and gather-like activities like uploading
Poliwall output.

The purpose of this API is to be able to change *one* place (this
module) when we decide to change the way records are encoded.

Definition of a record: a list or tuple of values.  On output, all
values except None are converted to strings using str(); None is
written as if it were the empty string.  On input, all fields will be
returned as strings.

Present file format (necessitated by the desire to use
Oracle's SQL*Loader):

- record separator to be '\0\n'

- fields to be preceded by 4 decimal digits encoding the length, with
  a maximum of 4000; any string can appear in the field other than
  '\0\n', and (unfortunately) there is no way to quote '\0\n'.

- this requires us to find a different solution for larger data
  fields or data fields containing null bytes; only gather_files.py is
  affected by this restriction

Old file format:

- each record is a line (ending in '\n')

- fields are separated by vertical bars ('|')

- horizontal whitespace surrounding fields is not significant

- certain characters in fields are encoded using %xx ('%' followed by
  two hex digits); this includes the field and record separators ('|'
  and '\n'), the escape character itself ('%'), leading and trailing
  whitespace, null bytes, and optionally (at the implementation's
  discretion) control characters and non-ASCII characters

"""

from urllib.parse import quote as _urllib_quote, unquote as _urllib_unquote

# The "new" record format is BYTE-oriented: each field is its UTF-8 encoding,
# preceded by 4 decimal digits giving the field's BYTE length, records separated
# by b"\0\n". The reader (the server's length-prefixed ByteBufferedReader, and
# decodeRecord below) consumes exactly that many bytes -- so non-ASCII fields
# (accented French/German usernames, filenames, command output) round-trip
# correctly. encode/decode therefore work in `bytes`, and the gather files that
# hold them are opened in binary mode.
#
# Under Python 2 `str` was bytes, so the historical `len(str)` prefix WAS the
# byte count; in Python 3 `len(str)` is the character count, which undercounts
# multi-byte fields and misaligns the server's byte reader ("Invalid digit").
# Encoding to UTF-8 bytes and prefixing the byte length restores the Py2 wire
# semantics the protocol has always required.

_MAXLEN = 9999  # the 4-digit length prefix tops out here

def _truncate_utf8(b):
    """Trim bytes to at most _MAXLEN without splitting a multibyte character."""
    if len(b) <= _MAXLEN:
        return b
    b = b[:_MAXLEN]
    while b:
        try:
            b.decode("utf-8")
            break
        except UnicodeDecodeError:
            b = b[:-1]
    return b

def encodeRecord(values):
    """Encode a record (sequence of values) -> bytes, byte-length prefixed."""
    out = []
    for value in values:
        if value is None:
            b = b""
        else:
            b = str(value).encode("utf-8")
        b = _truncate_utf8(b)
        out.append(b"%04d" % len(b))
        out.append(b)
    out.append(b"\0\n")
    return b"".join(out)

def decodeRecord(record):
    """Decode an encoded record (bytes) -> list of str."""
    i = 0
    fields = []
    while i < len(record):
        if record[i:i+1] == b"\0":
            assert record[i:] == b"\0\n"
            break
        s = record[i:i+4]
        assert len(s) == 4
        assert s.isdigit()
        n = int(s)
        i += 4
        field = record[i:i+n]
        assert len(field) == n
        fields.append(field.decode("utf-8", "replace"))
        i += n
    return fields

def readRecord(stream):
    """Read one record (up to b"\\0\\n") from a BINARY stream. Returns bytes."""
    lines = []
    for line in stream:
        lines.append(line)
        if line.endswith(b"\0\n"):
            break
    else:
        # No break out of loop, no record separator found
        assert lines == []
        return b""
    return b"".join(lines)

def recodeRecord(value, record):
    """Prepend a value (byte-length-prefixed UTF-8) to an encoded record (bytes).

    Equivalent to encodeRecord([value] + decodeRecord(record))[:-2] + record but
    cheaper -- it never decodes/re-encodes the existing record bytes.
    """
    if value is None:
        b = b""
    else:
        b = _truncate_utf8(str(value).encode("utf-8"))
    return b"%04d" % len(b) + b + record

# Historical stuff just in case we decide to retreat someday
def oldEncodeRecord(values,
                    # Remaining arguments are for optimization; don't use!
                    uq = _urllib_quote,
                    s = str):
    """Encode a record.

    Input: a sequence of values.

    Output: a string.

    """
    fields = [value is not None and uq(s(value)) or "" for value in values]
    return "|".join(fields) + "\n"

def oldDecodeRecord(record,
                    # Remaining argument is for optimization; don't use!
                    uu = _urllib_unquote):
    """Decode a record.

    Input: a string ending with the record separator.

    Output: a list of strings.

    """
    assert record.endswith("\n")
    return [uu(field.strip()) for field in record.split("|")]

# Note that decodeRecord(encodeRecord(V)) == V only if V contains only
# strings; encodeRecord(decodeRecord(R)) is even more fickle.  But
# there is a normalized form for records and values, and if the input
# is in normalized form, the equalities hold.  The normalized form is
# defined by these equalities.

def oldRecodeRecord(value, record):
    """Insert a value in front of an encoded record.

    This is equivalent to (but faster than):
    
        values = decodeRecord(record)
        values.insert(0, value)
        return encodeRecord(values)

    """
    if value is None:
        value = ""
    else:
        value = _urllib_quote(str(value))
    return value + "|" + record
    

def oldReadRecord(stream):
    """Read an (unencoded) record from a stream.

    Return an empty string if EOF is reached.

    Using the old format, this is simply stream.readline().

    With the new format, this will be trickier, since it has to read
    up to '\0\n'.

    """
    return stream.readline()
