Lexer = ref object
  tokenNodes*: seq[TokenNode]
  fileName*: string
  metContChar*: bool         ## whether is after the line
                             ## whose last token was a line continuation character '\'
  parenLevel*: int
  tripleStr*: tuple[within: bool, val: string, quote: char, tokenKind: Token,
                    raw: bool, escape: LexerEscaper,
                    start: tuple[lineNo, colNo: int]] ## is handling triple string (multiline string)

LexerEscaper = proc (s: string): string {....raises: [SyntaxError].}

Mode {.pure.} = enum
  Single = "single", File = "exec", Eval = "eval"

string_kind_t {.pure.} = enum
  FSTRING, TSTRING

tokenizer_mode = object
  kind*: tokenizer_mode_kind_t
  quote*: char
  quote_size*: int
  raw*: bool
  curly_bracket_depth*, ## depth of f-string expression start, -1 if not in f-string expression
                         ## only affected by the `{`, `}` that starts or ends f-string expressions
                         ## 
                         ## So `>= 0` if in f-string expression
                         ## 
  ## number of unmatched {, unconditional,
  ## e.g. including those in nested f-strings like f"{ {1} }" -> '{1}'
  ## 
  curly_bracket_expr_start_depth*: int
  start*, multi_line_start*: int
  first_line*: int
  start_offset*, multi_line_start_offset*: int
  last_expr_size*, last_expr_end*: int
  when not defined(release):
    in_debug*: bool
  string_kind*: string_kind_t

tokenizer_mode_kind_t = enum
  TOK_REGULAR_MODE, TOK_FSTRING_MODE