Types
Token {.pure.} = enum NULLTOKEN, Endmarker, Name, Number, Bytes, String, Newline, Indent, Dedent, Lpar, Rpar, Lsqb, Rsqb, Colon, Comma, Semi, Plus, Minus, Star, Slash, Vbar, Amper, Less, Greater, Equal, Dot, Percent, Lbrace, Rbrace, Eqequal, Notequal, Lessequal, Greaterequal, Tilde, Circumflex, Leftshift, Rightshift, Doublestar, Plusequal, Minequal, Starequal, Slashequal, Percentequal, Amperequal, Vbarequal, Circumflexequal, Leftshiftequal, Rightshiftequal, Doublestarequal, Doubleslash, Doubleslashequal, At, Atequal, Rarrow, Ellipsis, PEP401, is, False, else, lambda, and, del, from, except, def, assert, with, import, continue, elif, not, True, class, in, or, return, if, as, break, pass, raise, None, nonlocal, global, for, async, try, while, await, yield, finally, boundary, single_input, file_input, eval_input, decorator, decorators, decorated, async_funcdef, funcdef, parameters, typedargslist, tfpdef, varargslist, vfpdef, stmt, simple_stmt, small_stmt, expr_stmt, annassign, testlist_star_expr, augassign, del_stmt, pass_stmt, flow_stmt, break_stmt, continue_stmt, return_stmt, yield_stmt, raise_stmt, import_stmt, import_name, import_from, import_as_name, dotted_as_name, import_as_names, dotted_as_names, dotted_name, global_stmt, nonlocal_stmt, assert_stmt, compound_stmt, async_stmt, if_stmt, while_stmt, for_stmt, try_stmt, with_stmt, with_item, except_clause, suite, test, test_nocond, lambdef, lambdef_nocond, or_test, and_test, not_test, comparison, comp_op, star_expr, expr, xor_expr, and_expr, shift_expr, arith_expr, term, factor, power, atom_expr, atom, testlist_comp, trailer, subscriptlist, subscript, sliceop, exprlist, testlist, dictorsetmaker, classdef, arglist, argument, comp_iter, sync_comp_for, comp_for, comp_if, encoding_decl, yield_expr, yield_arg
- Source Edit
Lets
strTokenMap = newTable([("ENDMARKER", Endmarker), ("NAME", Name), ("NUMBER", Number), ("BYTES", Bytes), ("STRING", String), ("NEWLINE", Newline), ("INDENT", Indent), ("DEDENT", Dedent), ("(", Lpar), (")", Rpar), ("[", Lsqb), ("]", Rsqb), (":", Colon), (",", Comma), (";", Semi), ("+", Plus), ("-", Minus), ("*", Star), ("/", Slash), ("|", Vbar), ("&", Amper), ("<", Less), (">", Greater), ("=", Equal), (".", Dot), ("%", Percent), ("{", Lbrace), ("}", Rbrace), ("==", Eqequal), ("!=", Notequal), ("<=", Lessequal), (">=", Greaterequal), ("~", Tilde), ("^", Circumflex), ("<<", Leftshift), (">>", Rightshift), ("**", Doublestar), ("+=", Plusequal), ("-=", Minequal), ("*=", Starequal), ("/=", Slashequal), ("%=", Percentequal), ("&=", Amperequal), ("|=", Vbarequal), ("^=", Circumflexequal), ("<<=", Leftshiftequal), (">>=", Rightshiftequal), ("**=", Doublestarequal), ("//", Doubleslash), ("//=", Doubleslashequal), ("@", At), ("@=", Atequal), ("->", Rarrow), ("...", Ellipsis), ("<>", PEP401), ("single_input", single_input), ("file_input", file_input), ("eval_input", eval_input), ("decorator", decorator), ("decorators", decorators), ("decorated", decorated), ("async_funcdef", async_funcdef), ("funcdef", funcdef), ("parameters", parameters), ("typedargslist", typedargslist), ("tfpdef", tfpdef), ("varargslist", varargslist), ("vfpdef", vfpdef), ("stmt", stmt), ("simple_stmt", simple_stmt), ("small_stmt", small_stmt), ("expr_stmt", expr_stmt), ("annassign", annassign), ("testlist_star_expr", testlist_star_expr), ("augassign", augassign), ("del_stmt", del_stmt), ("pass_stmt", pass_stmt), ("flow_stmt", flow_stmt), ("break_stmt", break_stmt), ("continue_stmt", continue_stmt), ("return_stmt", return_stmt), ("yield_stmt", yield_stmt), ("raise_stmt", raise_stmt), ("import_stmt", import_stmt), ("import_name", import_name), ("import_from", import_from), ("import_as_name", import_as_name), ("dotted_as_name", dotted_as_name), ("import_as_names", import_as_names), ("dotted_as_names", dotted_as_names), ("dotted_name", dotted_name), ("global_stmt", global_stmt), ("nonlocal_stmt", nonlocal_stmt), ("assert_stmt", assert_stmt), ("compound_stmt", compound_stmt), ("async_stmt", async_stmt), ("if_stmt", if_stmt), ("while_stmt", while_stmt), ("for_stmt", for_stmt), ("try_stmt", try_stmt), ("with_stmt", with_stmt), ("with_item", with_item), ("except_clause", except_clause), ("suite", suite), ("test", test), ("test_nocond", test_nocond), ("lambdef", lambdef), ("lambdef_nocond", lambdef_nocond), ("or_test", or_test), ("and_test", and_test), ("not_test", not_test), ("comparison", comparison), ("comp_op", comp_op), ("star_expr", star_expr), ("expr", expr), ("xor_expr", xor_expr), ("and_expr", and_expr), ("shift_expr", shift_expr), ("arith_expr", arith_expr), ("term", term), ("factor", factor), ("power", power), ("atom_expr", atom_expr), ("atom", atom), ("testlist_comp", testlist_comp), ("trailer", trailer), ("subscriptlist", subscriptlist), ("subscript", subscript), ("sliceop", sliceop), ("exprlist", exprlist), ("testlist", testlist), ("dictorsetmaker", dictorsetmaker), ("classdef", classdef), ("arglist", arglist), ("argument", argument), ("comp_iter", comp_iter), ("sync_comp_for", sync_comp_for), ("comp_for", comp_for), ("comp_if", comp_if), ("encoding_decl", encoding_decl), ("yield_expr", yield_expr), ("yield_arg", yield_arg), ("is", is), ("False", False), ("else", else), ("lambda", lambda), ("and", and), ("del", del), ("from", from), ("except", except), ("def", def), ("assert", assert), ("with", with), ("import", import), ("continue", continue), ("elif", elif), ("not", not), ("True", True), ("class", class), ("in", in), ("or", or), ("return", return), ("if", if), ("as", as), ("break", break), ("pass", pass), ("raise", raise), ("None", None), ("nonlocal", nonlocal), ("global", global), ("for", for), ("async", async), ("try", try), ("while", while), ("await", await), ("yield", yield), ("finally", finally)])
- Source Edit
Consts
contentTokenSet = {Token.Name, Token.Number, Token.String, Token.Bytes}
- Source Edit
grammarTokenList = ["single_input", "file_input", "eval_input", "decorator", "decorators", "decorated", "async_funcdef", "funcdef", "parameters", "typedargslist", "tfpdef", "varargslist", "vfpdef", "stmt", "simple_stmt", "small_stmt", "expr_stmt", "annassign", "testlist_star_expr", "augassign", "del_stmt", "pass_stmt", "flow_stmt", "break_stmt", "continue_stmt", "return_stmt", "yield_stmt", "raise_stmt", "import_stmt", "import_name", "import_from", "import_as_name", "dotted_as_name", "import_as_names", "dotted_as_names", "dotted_name", "global_stmt", "nonlocal_stmt", "assert_stmt", "compound_stmt", "async_stmt", "if_stmt", "while_stmt", "for_stmt", "try_stmt", "with_stmt", "with_item", "except_clause", "suite", "test", "test_nocond", "lambdef", "lambdef_nocond", "or_test", "and_test", "not_test", "comparison", "comp_op", "star_expr", "expr", "xor_expr", "and_expr", "shift_expr", "arith_expr", "term", "factor", "power", "atom_expr", "atom", "testlist_comp", "trailer", "subscriptlist", "subscript", "sliceop", "exprlist", "testlist", "dictorsetmaker", "classdef", "arglist", "argument", "comp_iter", "sync_comp_for", "comp_for", "comp_if", "encoding_decl", "yield_expr", "yield_arg"]
- Source Edit
reserveNameSet = (data: [(3043494581802008320, "is"), (2584949074918387455, "False"), (4167480082662538754, "else"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (-7239118227665889012, "lambda"), (6929542774839622797, "and"), (-4572591901098389106, "del"), (-3075234612834068466, "from"), (0, ""), (0, ""), (-2181833479421681262, "except"), (2360477792506196755, "def"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (-5887635237809093095, "assert"), (3182763688416902937, "with"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (-7868617827981323104, "import"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (2210116261907819816, "continue"), (-5021870210732393176, "elif"), (7500349390622997928, "not"), (7628988648237865897, "True"), (0, ""), (7772006167705430957, "class"), (-2202290457511280594, "in"), (-4840959370558533201, "or"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (-6019295843603950917, "return"), (-7285794198674356036, "if"), (0, ""), (0, ""), (6260224989214919871, "as"), (0, ""), (0, ""), (8380221545607033154, "break"), (0, ""), (3804819118857629636, "pass"), (6137881024046402116, "raise"), (-5242446400123616444, "None"), (0, ""), (0, ""), (-6714109632935678775, "nonlocal"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (5696226971518331620, "global"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (4064080761860438252, "for"), (0, ""), (-6651261360205773074, "async"), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (0, ""), (5649038586232780023, "try"), (-5246625912586894728, "while"), (3282030798056260088, "await"), (-2308960313628641289, "yield"), (0, ""), (0, ""), (0, ""), (0, ""), (-363981861430527361, "finally")], counter: 35)
- Source Edit
Procs
proc isNonTerminator(node: Token): bool {....raises: [], tags: [], forbids: [].}
- Source Edit
proc isTerminator(node: Token): bool {....raises: [], tags: [], forbids: [].}
- Source Edit