# Copyright 2026 SigmaSRC, Inc. All Rights Reserved.

"""Syntax tree node definitions.

There is a class or function corresponding to each terminal and
nonterminal symbol.

The grammar is pieced together from the docstrings of the
corresponding classes and functions: text between dollar signs is
assumed to be the right-hand side of the grammar rule corresponding to
that class or function.

We use __slots__ to make the parse tree nodes as small as possible.

NOTE: EVERY CLASS MUST HAVE A __slots__ DEFINITION, EVEN IF EMPTY!
(If not, the instances will get a __dict__, defeating the purpose of
__slots__ in our case.)

"""

# Python imports
import token
import logging

from sigma.fuelc import grammar

logger = logging.getLogger("fuelc.astnode")

class Node(object):
    """Abstract base class for all nodes.

    This has no attributes except a context slot which holds the lone
    number (or more detailed context info).  In the future this might
    change this to several slots (e.g. filename, lineno, column, or
    even filename, start_lineno, start_column, end_lineno,
    end_column).  The context is only referenced by two places: the
    part of the code that sticks it it, and the part of the code that
    reports errors.

    In order to reduce the amount of boilerplate code, the context is
    argument is handled by __new__ rather than __init__.  There are
    also a few subclasses that override __new__ to sometimes avoid
    constructing an instance.

    """

    __slots__ = ["context"]

    def __new__(cls, context, *rest):
        assert cls not in (Node, Nonterminal, Terminal, Constant)
        obj = object.__new__(cls)
        obj.context = context
        return obj

    _stretch = False # Set to true to stretch the repr() vertically

    def __repr__(self, repr=repr):
        stretch = self._stretch
        r = [self.__class__.__name__]
        if stretch:
            r.append("(\n    ")
        else:
            r.append("(") # ")" -- crutch for Emacs python-mode :-(
        cls = self.__class__
        # Get nearest non-empty slots __slots__.  This assumes
        # *something* has non-empty __slots__ before we reach object
        # (which has no __slots__).  The class hierarchy guarantees
        # this.
        slots = cls.__slots__
        while not slots:
            cls = cls.__base__
            slots = cls.__slots__
        first = True
        for name in slots:
            if name == "context":
                continue # Skip this
            if first:
                first = False
            else:
                if stretch:
                    r.append(",\n    ")
                else:
                    r.append(", ")
            try:
                value = getattr(self, name)
            except AttributeError:
                continue
            if stretch and isinstance(value, list):
                rr = map(repr, value)
                rv = "[" + ",\n ".join(rr) + "]"
            else:
                rv = repr(value)
            if stretch:
                rv = rv.replace("\n", "\n    ")
            r.append(rv)
        r.append(")")
        return "".join(r)

    def __str__(self):
        return self.__repr__(repr=str)

class Nonterminal(Node):
    """Abstract base class for nonterminal symbols.

    Nothing beyond Node.

    """

    __slots__ = []

    _stretch = True

class Terminal(Node):
    """Abstract base class for terminal symbols.

    Nothing beyond Node.

    """

    __slots__ = []

class Series(Nonterminal):
    """Abstract base class for nonterminals like stmts: stmt+."""

    __slots__ = []

    def __new__(cls, context, *nodes):
        assert cls is not Series
        if len(nodes) == 0:
            return None
        elif len(nodes) == 1:
            return nodes[0]
        else:
            obj = Nonterminal.__new__(cls, context)
            obj.initseries(nodes)
            return obj

class Constant(Terminal):
    """Abstract base class for constants (e.g. number or string).

    Attributes:

    repr -- a string giving the token, exactly as read from source

    """

    __slots__ = ["repr"]

    def __init__(self, context, repr):
        self.repr = repr

    def __str__(self):
        return self.repr

# Node classes for terminal symbols

class Name(Terminal):
    """Name (e.g. a variable name or an attribute name).

    Attributes:

    name -- a string giving the name.

    """

    __slots__ = ["name"]

    def __init__(self, context, name):
        self.name = name

    def __str__(self):
        return self.name

class Number(Constant):
    """Numeric constant.

    Nothing beyond Constant.

    """

    __slots__ = []

class String(Constant):
    """String constant.

    Nothing beyond Constant.

    """

    __slots__ = []

# Nodes and factory functions for nonterminal symbols

def Program(context, stmts):
    """Program is a nonterminal with only one non-trivial child.

    Grammar: $ OuterStatements ENDMARKER $

    """
    return stmts

class OuterStatements(Nonterminal):
    "Grammar: $ [DocString] OuterStatement+ $"
    __slots__ = ["doc", "stmts"]
    def __init__(self, context, *stuff):
        self.doc = None
        if stuff and isinstance(stuff[0], DocString):
            self.doc = stuff[0]
            stuff = stuff[1:]
        self.stmts = stuff

def OuterStatement(context, child):
    """Statement is a nonterminal with only one child.

    Grammar: $ Import | TypeDef | ConstDef
             | InterfaceDef | StructDef | ProcDef | FuncDef $

    """
    return child

class Import(Nonterminal):
    """Grammar: $ ['from' FromName]
                  'import' ImportName (',' ImportName)* NEWLINE $ """
    __slots__ = ["fromname", "importnames"]
    def __init__(self, context, *stuff):
        if stuff[0] == "from":
            self.fromname = stuff[1]
            stuff = stuff[2:]
        else:
            self.fromname = None
        self.importnames = stuff[1:]

class FromName(Nonterminal):
    "Grammar: $ QualName | RelativeFromName $"
    __slots__ = ["isrel", "qname"]
    def __init__(self, context, node):
        if isinstance(node, RelativeFromName):
            self.isrel = True
            self.qname = node.qname
        else:
            self.isrel = False
            self.qname = node

class RelativeFromName(Nonterminal):
    "Grammar: $ '.' [QualName] $"
    __slots__ = ["qname"]
    def __init__(self, context, qname=None):
        self.qname = qname

class ImportName(Nonterminal):
    "Grammar: $ NAME ['as' NAME] $"
    __slots__ = ["importname", "asname"]
    def __init__(self, context, *stuff):
        if len(stuff) == 1:
            self.importname = self.asname = stuff[0]
        else:
            assert len(stuff) == 3
            assert stuff[1] == "as"
            self.importname = stuff[0]
            self.asname = stuff[2]

class TypeDef(Nonterminal):
    "Grammar: $ 'type' NAME '=' TypeExpression NEWLINE $"
    __slots__ = ["name", "type"]
    def __init__(self, context, name, type):
        self.name = name
        self.type = type

class ConstDef(Nonterminal):
    "Grammar: $ 'const' NAME [':' TypeExpression] '=' Expression NEWLINE $"
    __slots__ = ["name", "type", "expr"]
    def __init__(self, context, name, *stuff):
        self.name = name
        if len(stuff) == 1:
            self.type = None
            self.expr = stuff[0]
        else:
            assert len(stuff) == 2
            self.type, self.expr = stuff

class InterfaceDef(Nonterminal):
    """Grammar: $ 'interface' NAME ':' NEWLINE
                  INDENT
                      [DocString]
                      (InterfaceAttrDefs | 'pass' NEWLINE)
                  DEDENT $ """
    __slots__ = ["name", "doc", "body"]
    def __init__(self, context, name, *stuff):
        self.name = name
        self.doc = self.body = None
        if stuff and isinstance(stuff[0], DocString):
            self.doc = stuff[0]
            stuff = stuff[1:]
        if stuff:
            assert len(stuff) == 1, repr(stuff)
            self.body = stuff[0]

class InterfaceAttrDefs(Series):
    "Grammar: $ InterfaceAttrDef+ $"
    __slots__ = ["defs"]
    def initseries(self, defs):
        assert len(defs) > 1
        self.defs = defs

class InterfaceAttrDef(Nonterminal):
    "Grammar: $ NAME [FormalArgs] ':' TypeExpression NEWLINE $"
    __slots__ = ["name", "formals", "type"]
    def __init__(self, context, name, *stuff):
        self.name = name
        if len(stuff) == 1:
            self.formals = None
            self.type = stuff[0]
        else:
            assert len(stuff) == 2
            self.formals, self.type = stuff

class StructDef(Nonterminal):
    """Grammar: $ 'struct' NAME ':' NEWLINE
                  INDENT
                      [DocString]
                      (StructAttrDefs | 'pass' NEWLINE)
                  DEDENT $ """
    __slots__ = ["name", "doc", "body"]
    def __init__(self, context, name, *stuff):
        self.name = name
        self.doc = self.body = None
        if stuff and isinstance(stuff[0], DocString):
            self.doc = stuff[0]
            stuff = stuff[1:]
        if stuff:
            assert len(stuff) == 1
            self.body = stuff[0]

class StructAttrDefs(Series):
    "Grammar: $ StructAttrDef+ $"
    __slots__ = ["defs"]
    def initseries(self, defs):
        assert len(defs) > 1
        self.defs = defs

class StructAttrDef(Nonterminal):
    "Grammar: $ NAME ':' TypeExpression ['=' Expression] NEWLINE $"
    __slots__ = ["name", "type", "expr"]
    def __init__(self, context, name, type, expr=None):
        self.name = name
        self.type = type
        self.expr = expr

class ProcDef(Nonterminal):
    """Grammar: $ 'proc' NAME FormalArgs ':' NEWLINE
                  INDENT [DocString] InnerStatements DEDENT $
    """
    __slots__ = ["name", "formals", "doc", "body"]
    def __init__(self, context, name, formals, *stuff):
        self.name = name
        self.formals = formals
        self.doc = self.body = None
        if stuff and isinstance(stuff[0], DocString):
            self.doc = stuff[0]
            stuff = stuff[1:]
        if stuff:
            assert len(stuff) == 1
            self.body = stuff[0]

class FuncDef(Nonterminal):
    """Grammar: $ 'func' NAME FormalArgs ':' TypeExpression ':' NEWLINE
                  INDENT [DocString] InnerStatements DEDENT $
    """
    __slots__ = ["name", "formals", "type", "doc", "body"]
    def __init__(self, context, name, formals, type, *stuff):
        self.name = name
        self.formals = formals
        self.type = type
        self.doc = self.body = None
        if stuff and isinstance(stuff[0], DocString):
            self.doc = stuff[0]
            stuff = stuff[1:]
        assert len(stuff) == 1
        self.body = stuff[0]

class FormalArgs(Nonterminal):
    "Grammar: $ '(' [FormalArg (',' FormalArg)*] ')' $"
    __slots__ = ["formals"]
    def __init__(self, context, *formals):
        self.formals = formals

class FormalArg(Nonterminal):
    "Grammar: $ NAME ':' TypeExpression ['=' Expression] $"
    __slots__ = ["name", "type", "expr"]
    def __init__(self, context, name, type, expr=None):
        self.name = name
        self.type = type
        self.expr = expr

class ReturnStatement(Nonterminal):
    "Grammar: $ 'return' [Expression] NEWLINE $"
    __slots__ = ["expr"]
    def __init__(self, context, expr=None):
        self.expr = expr

class DocString(Nonterminal):
    "Grammar: $ STRING NEWLINE $"
    __slots__ = ["str"]
    def __init__(self, context, s):
        self.str = s

class InnerStatements(Nonterminal):
    "Grammar: $ ReturnStatement | InnerStatement+ [ReturnStatement] | Pass $"
    __slots__ = ["stmts"]
    def __init__(self, context, *stmts):
        self.stmts = stmts

def InnerStatement(context, child=None):
    """Statement is a nonterminal with only one child.

    Grammar: $ ForBlock | IfBlock | Action |  LetDef $

    """
    return child

class ForBlock(Nonterminal):
    """Grammar: $ 'for' NAME 'in' Expression ':' NEWLINE
                  INDENT InnerStatements DEDENT $ """

    __slots__ = ["name", "expr", "stmts"]
    def __init__(self, context, name, in_keyword, expr, stmts=None):
        self.name = name
        self.expr = expr
        self.stmts = stmts

class IfBlock(Nonterminal):
    """Grammar: $ 'if' Expression ':' NEWLINE
                      INDENT InnerStatements DEDENT
                ( 'elif' Expression ':' NEWLINE
                      INDENT InnerStatements DEDENT )*
                [ 'else' ':' NEWLINE
                      INDENT InnerStatements DEDENT ]
                $ """
    __slots__ = ["choices"]
    def __init__(self, context, *stuff):
        choices = []
        while len(stuff) >= 2:
            choices.append(stuff[:2])
            stuff = stuff[2:]
        if stuff:
            assert len(stuff) == 1
            choices.append((None, stuff[0]))
        self.choices = choices

def Action(context, *args):
    # Top level actions have to start with a NAME and end with
    # parenthesized arguments.  Suffix is the same as
    # ('.' NAME | CallArgs).  However, we do regular-expression like
    # comprehension only within each production, so we can deal with the
    # required lookahead only when it's mashed into one production
    # like this instead of something like "NAME Suffix* CallArgs NEWLINE".
    "Grammar: $ NAME ('.' NAME | CallArgs)* CallArgs NEWLINE $"
    return FunOrField(context, *args)

class LetDef(Nonterminal):
    "Grammar: $ 'let' NAME [':' TypeExpression] '=' Expression NEWLINE $"
    __slots__ = ["name", "type", "expr"]
    def __init__(self, context, name, *stuff):
        self.name = name
        if len(stuff) == 1:
            self.type = None
            self.expr = stuff[0]
        else:
            assert len(stuff) == 2
            self.type, self.expr = stuff

def Pass(context):
    "Grammar: $ 'pass' NEWLINE $"
    return None

class Expressions(Series):
    "Grammar: $ Expression (',' Expression)* $"

    __slots__ = ["exprs"]

    def initseries(self, exprs):
        assert len(exprs) > 1
        self.exprs = exprs

    def __str__(self):
        return ", ".join(map(str, self.exprs))

class Expression(Series):
    "Grammar: $ BinaryExpression ['?' BinaryExpression ':' BinaryExpression] $"

    __slots__ = ["test", "left", "right"]

    def initseries(self, nodes):
        self.test, self.left, self.right = nodes

    def __str__(self):
        return "%s ? %s : %s" % (self.test, self.left, self.right)

class BinaryExpression(Series):
    "Grammar: $ Operand (Operator Operand)* $"

    __slots__ = ["left", "op", "right"]

    def initseries(self, stuff):
        # Stuff is a list with alternating operands and operators
        if len(stuff) == 3:
            self.left, self.op, self.right = stuff
            return
        assert len(stuff) > 1 and len(stuff) % 2 == 1
        # Find the rightmost lowest-priority operator
        lowest_i = 1
        lowest_op = stuff[lowest_i]
        lowest_pri = lowest_op.priority()
        for i in range(3, len(stuff), 2):
            op = stuff[i]
            pri = op.priority()
            if pri <= lowest_pri:
                lowest_i = i
                lowest_op = op
                lowest_pri = pri
        self.left = self.__class__(self.context, *stuff[:lowest_i])
        self.op = lowest_op
        self.right = self.__class__(self.context, *stuff[lowest_i+1:])

    def __str__(self):
        return "(%s %s %s)" % (self.left, self.op, self.right)

def Operand(context, arg):
    """Operand is a nonterminal with one child.

    Grammar: $ FunOrField | UnaryExpression $

    """
    return arg

class UnaryExpression(Nonterminal):
    "Grammar: $ UnaryOperator Operand $"

    __slots__ = ["op", "arg"]

    def __init__(self, context, op, arg):
        self.op = op
        self.arg = arg

    def __str__(self):
        return "%s%s" % (self.op, self.arg)

def UnaryOperator(context, op):
    "Grammar: $ 'not' | '-' $"
    return op

class Operator(Nonterminal):
    """Operator.

    This has a repr slot and a priority() method.

    The __new__ method implements a sort-of singleton pattern: there's
    only one instance per repr value.  (Yes, this means the context is
    not useful.  Therefore we set it to None.)

    Grammar: $ '**' | '*' | '/' | '+' | '-' | '&' | '^' | '|' |
          ['not'] 'in' | '==' | '<' | '>' | '!=' | '<=' | '>=' |
          'and' | 'or' $

    """

    __slots__ = ["repr"]

    _stretch = False

    _cache = {}

    def __new__(cls, context, *args):
        repr = " ".join(args)
        # For "not in", the argument should be the string "not in"
        obj = cls._cache.get(repr)
        if obj is None:
            obj = Terminal.__new__(cls, None)
            obj.repr = repr
        return obj

    def __str__(self):
        return self.repr

    def priority(self):
        return self._priorities[self.repr]

    _priorities = {
        "or":  0,
        "and": 1,
        "in":  2,
        "not in": 2,
        "<":  2,
        ">":  2,
        "==": 2,
        "<=": 2,
        ">=": 2,
        "!=": 2,
        "|":  3,
        "^":  4,
        "&":  5,
        "+":  6,
        "-":  6,
        "*":  7,
        "/":  7,
        "**": 8,
    }

def Atom(context, arg):
    """ Grammar: $ NAME | STRING | NUMBER | Set | StructConstructor 
                        | ParenthesizedExpression $
    """
    return arg

def Suffix(context, arg):
    """ Grammar: $ '.' NAME | CallArgs $
    """
    return arg

# Return a FunCall or a FieldRef, as appropriate. 
def fieldPair(context, arg1, arg2):
    if isinstance(arg2, CallArgs):
        return FunCall(context, arg1, arg2)
    elif isinstance(arg2, Name):
        return FieldRef(context, arg1, arg2.name)
    else:
        assert false

def FunOrField(context, *args):
    "Grammar: $ Atom Suffix* $"
    # Generate a bunch of nested FunCall and FieldRef's, depending on
    # the suffixes.
    # Example: If we're parsing 'h.report("hello world")', then args is
    #    (Name('h'), Name('report'), CallArgs(String('"hello world"')))
    # and we need to return
    #    FunCall(FieldRef(Name('h'), Name('report')),
    #            CallArgs(String('"hello world"')))
    if 1 == len(args):
        return args[0]
    elif 2 == len(args):
        # In the example, we get here when args is (Name('h'), Name('report')).
        # Return FieldRef(Name('h'), Name('report')).
        result = fieldPair(context, args[0], args[1])
    else:
        # In the example, we get here when args is
        # (Name('h'), Name('report'), CallArgs(String('"hello world"'))).
        # return FunCall(FieldRef(Name('h'), Name('report')),
        #                CallArgs(String('"hello world"')))
        result = fieldPair(context, FunOrField(context, *args[:-1]), args[-1])
    return result

class FunCall(Nonterminal):
    __slots__ = ["qname", "args"]

    def __new__(cls, context, atom, args=None):
        # XXX What type of sequence do we get?
        if args is None:
            return atom
        else:
            return Nonterminal.__new__(cls, context)

    def __init__(self, context, atom, args):
        self.qname = atom
        self.args = args

    def __str__(self):
        return "%s%s" % (self.qname, self.args)

# This is introduced in fieldPair.  The parser never generates it directly.
class FieldRef(Nonterminal):
    __slots__ = ["obj", "field"]

    def __init__(self, context, arg1, arg2):
        self.obj = arg1
        self.field = arg2

    def __str__(self):
        return "%s.%s" % (self.obj, self.field)

    # If an expression is all fieldrefs, then we want to include it in
    # the dictionary for __require__.  See translate.extractnames_FieldRef.
    def is_all_fieldrefs(self):
        if isinstance (self.obj, FieldRef):
            return self.obj.is_allfieldrefs()
        elif isinstance(self.obj, Name):
            return True
        else:
            return False

# This is now only used when parsing import statements.
class QualName(Series):
    "Grammar: $ NAME ('.' NAME)* $"

    __slots__ = ["names"]

    def initseries(self, names):
        assert len(names) > 1
        self.names = names

    def __str__(self):
        return ".".join(map(str, self.names))

class CallArgs(Nonterminal):
    "Grammar: $ '(' [Expressions] ')' $"

    __slots__ = ["args"]

    def __init__(self, context, args=None):
        self.args = args

    def __str__(self):
        if self.args is None:
            return "()"
        else:
            return "(%s)" % self.args

class Set(Nonterminal):
    "Grammar: $ '{' [SetBody] '}' $"

    __slots__ = ["body"]

    def __init__(self, context, body=None):
        self.body = body

    def __str__(self):
        if self.body is None:
            return "{}"
        else:
            return "{%s}" % self.body

class SetBody(Nonterminal):
    """Grammar: $ Expression ForClause [IfClause]
                | Expression (',' Expression)* $"""
    # This takes the raw list of subnodes

    __slots__ = ["expr", "for_clause", "if_clause", "exprs"]

    def __init__(self, context, *stuff):
        self.expr = stuff[0]
        self.for_clause = self.if_clause = self.exprs = None
        if len(stuff) > 1:
            if isinstance(stuff[1], ForClause):
                assert len(stuff) <= 3
                self.for_clause = stuff[1]
                if len(stuff) > 2:
                    self.if_clause = stuff[2]
            else:
                self.exprs = stuff[1:]

    def __str__(self):
        if self.for_clause is None:
            if self.exprs is None:
                return str(self.expr)
            else:
                return ", ".join(map(str, (self.expr,) + self.exprs))
        else:
            assert self.exprs is None
            if self.if_clause is None:
                return "%s %s" % (self.expr, self.for_clause)
            else:
                return "%s %s %s" % (self.expr, self.for_clause,
                                                self.if_clause)

class ForClause(Nonterminal):
    "Grammar: $ 'for' NAME 'in' Expression $"

    __slots__ = ["name", "expr"]

    def __init__(self, context, name, in_keyword, expr):
        self.name = name
        self.expr = expr

    def __str__(self):
        return "for %s in %s" % (self.name, self.expr)

class IfClause(Nonterminal):
    "Grammar: $ 'if' Expression $"
    # This is *not* redundant

    __slots__ = ["expr"]

    def __init__(self, context, expr):
        self.expr = expr

    def __str__(self):
        return "if %s" % self.expr

class StructConstructor(Nonterminal):
    "Grammar: $ 'new' QualName '(' [KeyValuePairs] ')' $"

    __slots__ = ["qname", "args"]

    def __init__(self, context, qname, args=None):
        self.qname = qname
        self.args = args

    def __str__(self):
        return "new %s(%s)" % (self.qname, self.args)

class KeyValuePairs(Nonterminal):
    "Grammar: $ KeyValuePair (',' KeyValuePair)* $"

    __slots__ = ["kvps"]

    _stretch = False

    def __init__(self, context, *kvps):
        assert len(kvps) >= 1
        self.kvps = kvps

    def __str__(self):
        return ", ".join(map(str, self.kvps))

class KeyValuePair(Nonterminal):
    "Grammar: $ NAME '=' Expression $"

    __slots__ = ["name", "expr"]

    def __init__(self, context, name, expr):
        self.name = name
        self.expr = expr

    def __str__(self):
        return "%s=%s" % (self.name, self.expr)

def ParenthesizedExpression(context, expr):
    "Grammar: $ '(' Expression ')' $"
    return expr

def TypeExpression(context, node):
    "Grammar: $ QualName | SetType $"
    return node

class SetType(Nonterminal):
    "Grammar: $ 'set' 'of' TypeExpression $"
    __slots__ = ["etype"]
    def __init__(self, context, set_keyword, of_keyword, etype):
        self.etype = etype


# Conversion from concrete to abstract syntax trees

def vanish(context, value):
    return None

token_mapping = {
    # Tokens that become parse tree nodes
    # (token.NAME is special-cased in the code)
    token.NUMBER: Number,
    token.STRING: String,

    # Tokens that vanish
    token.DOT: vanish,
    token.LPAR: vanish,
    token.RPAR: vanish,
    token.COLON: vanish,
    token.COMMA: vanish,
    token.EQUAL: vanish,
    token.DEDENT: vanish,
    token.INDENT: vanish,
    token.LBRACE: vanish,
    token.RBRACE: vanish,
    token.NEWLINE: vanish,
    token.ENDMARKER: vanish,
    grammar.QUESTIONMARK: vanish,

    # All other tokens return the token's string value (e.g. "+")
    }

vanishing_keywords = {
    "interface": True,
    "const": True,
    "def": True,
    "for": True,
    "func": True,
    "if": True,
    "elif": True,
    "else": True,
    "let": True,
    "new": True,
    "pass": True,
    "proc": True,
    "return": True,
    "struct": True,
    "type": True,
    }

def convert(grammar, node):
    type, value, context, children = node
    # Is it a non-terminal symbol?
    if type in grammar.number2symbol:
        symbol = grammar.number2symbol[type]
        factory = globals().get(symbol)
        if factory is None:
            raise RuntimeError("can't find factory for %s (line %s)" %
                               (symbol, context))
        # Debug variation:
        try:
            return factory(context, *children)
        except:
            logger.debug("%s %s", factory.__name__, "(")
            for child in children:
                logger.debug("%s %s", "==>", child)
            logger.debug(")")
            logger.debug("# Did you remember to declare a 'context' arg?")
            raise
        return factory(context, *children)

    # Must be a terminal symbol.
    if type == token.NAME:
        # Name or keyword.  Special-case the snot out of this.
        if value in grammar.keywords:
            # Keywords become strings, like operators
            if value in vanishing_keywords:
                return None
            else:
                return value
        else:
            return Name(context, value)

    # Look for a handler in the token_mapping table.
    assert type in token.tok_name
    factory = token_mapping.get(type)
    if factory:
        return factory(context, value)
    else:
        return value


# Support code

def generate_grammar(stream):
    """Extract the grammar rules from this module's doc strings."""
    import re
    from types import ModuleType
    lines = []
    startline = None
    for name, obj in globals().items():
        if hasattr(obj, "__doc__") and not isinstance(obj, ModuleType):
            m = re.search(r"Grammar:\s*\$([^$]+)\$", obj.__doc__ or "")
            if m:
                rule = obj.__name__, " ".join(m.group(1).split())
                if rule[0] == "Program":
                    assert not startline
                    startline = rule
                else:
                    lines.append(rule)
    lines.sort()
    # The start symbol *must* be the first rule
    lines.insert(0, startline)
    for name, rhs in lines:
        stream.write("%s: %s\n" % (name, rhs))

if __name__ == '__main__':
    import sys
    generate_grammar(sys.stdout)
