module Text.Highlighter.Lexers.Python3 (lexer) where

import Text.Regex.PCRE.Light
import Text.Highlighter.Types

lexer :: Lexer
lexer = Lexer
    { lName = "Python 3"
    , lAliases = ["python3", "py3"]
    , lExtensions = []
    , lMimetypes = ["text/x-python3", "application/x-python3"]
    , lStart = root'
    , lFlags = [multiline, utf8]
    }

tsqs' :: TokenMatcher
tsqs' =
    [ tokNext "'''" (Arbitrary "Literal" :. Arbitrary "String") Pop
    , anyOf strings'
    , anyOf nl'
    ]

sqs' :: TokenMatcher
sqs' =
    [ tokNext "'" (Arbitrary "Literal" :. Arbitrary "String") Pop
    , tok "\\\\\\\\|\\\\'|\\\\\\n" (Arbitrary "Literal" :. Arbitrary "String" :. Arbitrary "Escape")
    , anyOf strings'
    ]

backtick' :: TokenMatcher
backtick' =
    [ 
    ]

classname' :: TokenMatcher
classname' =
    [ tokNext "[A-Z_a-z\170\181\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1488-\1514\1520-\1522\1569-\1594\1600\1601-\1610\1646-\1647\1649-\1747\1749\1765-\1766\1774-\1775\1786-\1788\1791\1808\1810-\1839\1869-\1901\1920-\1957\1969\2308-\2361\2365\2384\2392-\2401\2429\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2493\2510\2524-\2525\2527-\2529\2544-\2545\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2649-\2652\2654\2674-\2676\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2749\2768\2784-\2785\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2877\2908-\2909\2911-\2913\2929\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3168-\3169\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3261\3294\3296-\3297\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3424-\3425\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3585-\3632\3634\3648-\3653\3654\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3762\3773\3776-\3780\3782\3804-\3805\3840\3904-\3911\3913-\3946\3976-\3979\4096-\4129\4131-\4135\4137-\4138\4176-\4181\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5920-\5937\5952-\5969\5984-\5996\5998-\6000\6016-\6067\6103\6108\6176-\6210\6211\6212-\6263\6272-\6312\6400-\6428\6480-\6509\6512-\6516\6528-\6569\6593-\6599\6656-\6678\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8305\8319\8336-\8340\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12337-\12341\12344-\12346\12347\12348\12353-\12438\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43011-\43013\43015-\43018\43020-\43042\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65137\65139\65143\65145\65147\65149\65151-\65276\65313-\65338\65345-\65370\65382-\65391\65392\65393-\65437\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500][0-9A-Z_a-z\170\181\183\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\768-\879\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1155-\1158\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1425-\1465\1467-\1469\1471\1473-\1474\1476-\1477\1479\1488-\1514\1520-\1522\1552-\1557\1569-\1594\1600\1601-\1610\1611-\1630\1632-\1641\1646-\1647\1648\1649-\1747\1749\1750-\1756\1759-\1764\1765-\1766\1767-\1768\1770-\1773\1774-\1775\1776-\1785\1786-\1788\1791\1808\1809\1810-\1839\1840-\1866\1869-\1901\1920-\1957\1958-\1968\1969\2305-\2306\2307\2308-\2361\2364\2365\2366-\2368\2369-\2376\2377-\2380\2381\2384\2385-\2388\2392-\2401\2402-\2403\2406-\2415\2429\2433\2434-\2435\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2492\2493\2494-\2496\2497-\2500\2503-\2504\2507-\2508\2509\2510\2519\2524-\2525\2527-\2529\2530-\2531\2534-\2543\2544-\2545\2561-\2562\2563\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2620\2622-\2624\2625-\2626\2631-\2632\2635-\2637\2649-\2652\2654\2662-\2671\2672-\2673\2674-\2676\2689-\2690\2691\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2748\2749\2750-\2752\2753-\2757\2759-\2760\2761\2763-\2764\2765\2768\2784-\2785\2786-\2787\2790-\2799\2817\2818-\2819\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2876\2877\2878\2879\2880\2881-\2883\2887-\2888\2891-\2892\2893\2902\2903\2908-\2909\2911-\2913\2918-\2927\2929\2946\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3006-\3007\3008\3009-\3010\3014-\3016\3018-\3020\3021\3031\3046-\3055\3073-\3075\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3134-\3136\3137-\3140\3142-\3144\3146-\3149\3157-\3158\3168-\3169\3174-\3183\3202-\3203\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3260\3261\3262\3263\3264-\3268\3270\3271-\3272\3274-\3275\3276-\3277\3285-\3286\3294\3296-\3297\3302-\3311\3330-\3331\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3390-\3392\3393-\3395\3398-\3400\3402-\3404\3405\3415\3424-\3425\3430-\3439\3458-\3459\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3530\3535-\3537\3538-\3540\3542\3544-\3551\3570-\3571\3585-\3632\3633\3634-\3635\3636-\3642\3648-\3653\3654\3655-\3662\3664-\3673\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3761\3762-\3763\3764-\3769\3771-\3772\3773\3776-\3780\3782\3784-\3789\3792-\3801\3804-\3805\3840\3864-\3865\3872-\3881\3893\3895\3897\3902-\3903\3904-\3911\3913-\3946\3953-\3966\3967\3968-\3972\3974-\3975\3976-\3979\3984-\3991\3993-\4028\4038\4096-\4129\4131-\4135\4137-\4138\4140\4141-\4144\4145\4146\4150-\4151\4152\4153\4160-\4169\4176-\4181\4182-\4183\4184-\4185\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4959\4969-\4977\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5906-\5908\5920-\5937\5938-\5940\5952-\5969\5970-\5971\5984-\5996\5998-\6000\6002-\6003\6016-\6067\6070\6071-\6077\6078-\6085\6086\6087-\6088\6089-\6099\6103\6108\6109\6112-\6121\6155-\6157\6160-\6169\6176-\6210\6211\6212-\6263\6272-\6312\6313\6400-\6428\6432-\6434\6435-\6438\6439-\6440\6441-\6443\6448-\6449\6450\6451-\6456\6457-\6459\6470-\6479\6480-\6509\6512-\6516\6528-\6569\6576-\6592\6593-\6599\6600-\6601\6608-\6617\6656-\6678\6679-\6680\6681-\6683\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7616-\7619\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8255-\8256\8276\8305\8319\8336-\8340\8400-\8412\8417\8421-\8427\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12330-\12335\12337-\12341\12344-\12346\12347\12348\12353-\12438\12441-\12442\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43010\43011-\43013\43014\43015-\43018\43019\43020-\43042\43043-\43044\43045-\43046\43047\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64286\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65024-\65039\65056-\65059\65075-\65076\65101-\65103\65137\65139\65143\65145\65147\65149\65151-\65276\65296-\65305\65313-\65338\65343\65345-\65370\65382-\65391\65392\65393-\65437\65438-\65439\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500]*" (Arbitrary "Name" :. Arbitrary "Class") Pop
    ]

stringescape' :: TokenMatcher
stringescape' =
    [ tok "\\\\([\\\\abfnrtv\"\\']|\\n|N{.*?}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})" (Arbitrary "Literal" :. Arbitrary "String" :. Arbitrary "Escape")
    ]

numbers' :: TokenMatcher
numbers' =
    [ tok "(\\d+\\.\\d*|\\d*\\.\\d+)([eE][+-]?[0-9]+)?" (Arbitrary "Literal" :. Arbitrary "Number" :. Arbitrary "Float")
    , tok "0[oO][0-7]+" (Arbitrary "Literal" :. Arbitrary "Number" :. Arbitrary "Oct")
    , tok "0[bB][01]+" (Arbitrary "Literal" :. Arbitrary "Number" :. Arbitrary "Bin")
    , tok "0[xX][a-fA-F0-9]+" (Arbitrary "Literal" :. Arbitrary "Number" :. Arbitrary "Hex")
    , tok "\\d+" (Arbitrary "Literal" :. Arbitrary "Number" :. Arbitrary "Integer")
    ]

keywords' :: TokenMatcher
keywords' =
    [ tok "(assert|break|continue|del|elif|else|except|finally|for|global|if|lambda|pass|raise|return|try|while|yield|as|with|True|False|None)\\b" (Arbitrary "Keyword")
    ]

nl' :: TokenMatcher
nl' =
    [ tok "\\n" (Arbitrary "Literal" :. Arbitrary "String")
    ]

builtins' :: TokenMatcher
builtins' =
    [ tok "(?<!\\.)(__import__|abs|all|any|bin|bool|bytearray|bytes|chr|classmethod|cmp|compile|complex|delattr|dict|dir|divmod|enumerate|eval|filter|float|format|frozenset|getattr|globals|hasattr|hash|hex|id|input|int|isinstance|issubclass|iter|len|list|locals|map|max|memoryview|min|next|object|oct|open|ord|pow|print|property|range|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|vars|zip)\\b" (Arbitrary "Name" :. Arbitrary "Builtin")
    , tok "(?<!\\.)(self|Ellipsis|NotImplemented)\\b" (Arbitrary "Name" :. Arbitrary "Builtin" :. Arbitrary "Pseudo")
    , tok "(?<!\\.)(ArithmeticError|AssertionError|AttributeError|BaseException|BufferError|BytesWarning|DeprecationWarning|EOFError|EnvironmentError|Exception|FloatingPointError|FutureWarning|GeneratorExit|IOError|ImportError|ImportWarning|IndentationError|IndexError|KeyError|KeyboardInterrupt|LookupError|MemoryError|NameError|NotImplementedError|OSError|OverflowError|PendingDeprecationWarning|ReferenceError|RuntimeError|RuntimeWarning|StopIteration|SyntaxError|SyntaxWarning|SystemError|SystemExit|TabError|TypeError|UnboundLocalError|UnicodeDecodeError|UnicodeEncodeError|UnicodeError|UnicodeTranslateError|UnicodeWarning|UserWarning|ValueError|VMSError|Warning|WindowsError|ZeroDivisionError)\\b" (Arbitrary "Name" :. Arbitrary "Exception")
    ]

name' :: TokenMatcher
name' =
    [ tok "@[a-zA-Z0-9_]+" (Arbitrary "Name" :. Arbitrary "Decorator")
    , tok "[A-Z_a-z\170\181\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1488-\1514\1520-\1522\1569-\1594\1600\1601-\1610\1646-\1647\1649-\1747\1749\1765-\1766\1774-\1775\1786-\1788\1791\1808\1810-\1839\1869-\1901\1920-\1957\1969\2308-\2361\2365\2384\2392-\2401\2429\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2493\2510\2524-\2525\2527-\2529\2544-\2545\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2649-\2652\2654\2674-\2676\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2749\2768\2784-\2785\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2877\2908-\2909\2911-\2913\2929\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3168-\3169\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3261\3294\3296-\3297\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3424-\3425\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3585-\3632\3634\3648-\3653\3654\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3762\3773\3776-\3780\3782\3804-\3805\3840\3904-\3911\3913-\3946\3976-\3979\4096-\4129\4131-\4135\4137-\4138\4176-\4181\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5920-\5937\5952-\5969\5984-\5996\5998-\6000\6016-\6067\6103\6108\6176-\6210\6211\6212-\6263\6272-\6312\6400-\6428\6480-\6509\6512-\6516\6528-\6569\6593-\6599\6656-\6678\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8305\8319\8336-\8340\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12337-\12341\12344-\12346\12347\12348\12353-\12438\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43011-\43013\43015-\43018\43020-\43042\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65137\65139\65143\65145\65147\65149\65151-\65276\65313-\65338\65345-\65370\65382-\65391\65392\65393-\65437\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500][0-9A-Z_a-z\170\181\183\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\768-\879\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1155-\1158\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1425-\1465\1467-\1469\1471\1473-\1474\1476-\1477\1479\1488-\1514\1520-\1522\1552-\1557\1569-\1594\1600\1601-\1610\1611-\1630\1632-\1641\1646-\1647\1648\1649-\1747\1749\1750-\1756\1759-\1764\1765-\1766\1767-\1768\1770-\1773\1774-\1775\1776-\1785\1786-\1788\1791\1808\1809\1810-\1839\1840-\1866\1869-\1901\1920-\1957\1958-\1968\1969\2305-\2306\2307\2308-\2361\2364\2365\2366-\2368\2369-\2376\2377-\2380\2381\2384\2385-\2388\2392-\2401\2402-\2403\2406-\2415\2429\2433\2434-\2435\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2492\2493\2494-\2496\2497-\2500\2503-\2504\2507-\2508\2509\2510\2519\2524-\2525\2527-\2529\2530-\2531\2534-\2543\2544-\2545\2561-\2562\2563\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2620\2622-\2624\2625-\2626\2631-\2632\2635-\2637\2649-\2652\2654\2662-\2671\2672-\2673\2674-\2676\2689-\2690\2691\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2748\2749\2750-\2752\2753-\2757\2759-\2760\2761\2763-\2764\2765\2768\2784-\2785\2786-\2787\2790-\2799\2817\2818-\2819\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2876\2877\2878\2879\2880\2881-\2883\2887-\2888\2891-\2892\2893\2902\2903\2908-\2909\2911-\2913\2918-\2927\2929\2946\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3006-\3007\3008\3009-\3010\3014-\3016\3018-\3020\3021\3031\3046-\3055\3073-\3075\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3134-\3136\3137-\3140\3142-\3144\3146-\3149\3157-\3158\3168-\3169\3174-\3183\3202-\3203\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3260\3261\3262\3263\3264-\3268\3270\3271-\3272\3274-\3275\3276-\3277\3285-\3286\3294\3296-\3297\3302-\3311\3330-\3331\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3390-\3392\3393-\3395\3398-\3400\3402-\3404\3405\3415\3424-\3425\3430-\3439\3458-\3459\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3530\3535-\3537\3538-\3540\3542\3544-\3551\3570-\3571\3585-\3632\3633\3634-\3635\3636-\3642\3648-\3653\3654\3655-\3662\3664-\3673\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3761\3762-\3763\3764-\3769\3771-\3772\3773\3776-\3780\3782\3784-\3789\3792-\3801\3804-\3805\3840\3864-\3865\3872-\3881\3893\3895\3897\3902-\3903\3904-\3911\3913-\3946\3953-\3966\3967\3968-\3972\3974-\3975\3976-\3979\3984-\3991\3993-\4028\4038\4096-\4129\4131-\4135\4137-\4138\4140\4141-\4144\4145\4146\4150-\4151\4152\4153\4160-\4169\4176-\4181\4182-\4183\4184-\4185\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4959\4969-\4977\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5906-\5908\5920-\5937\5938-\5940\5952-\5969\5970-\5971\5984-\5996\5998-\6000\6002-\6003\6016-\6067\6070\6071-\6077\6078-\6085\6086\6087-\6088\6089-\6099\6103\6108\6109\6112-\6121\6155-\6157\6160-\6169\6176-\6210\6211\6212-\6263\6272-\6312\6313\6400-\6428\6432-\6434\6435-\6438\6439-\6440\6441-\6443\6448-\6449\6450\6451-\6456\6457-\6459\6470-\6479\6480-\6509\6512-\6516\6528-\6569\6576-\6592\6593-\6599\6600-\6601\6608-\6617\6656-\6678\6679-\6680\6681-\6683\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7616-\7619\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8255-\8256\8276\8305\8319\8336-\8340\8400-\8412\8417\8421-\8427\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12330-\12335\12337-\12341\12344-\12346\12347\12348\12353-\12438\12441-\12442\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43010\43011-\43013\43014\43015-\43018\43019\43020-\43042\43043-\43044\43045-\43046\43047\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64286\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65024-\65039\65056-\65059\65075-\65076\65101-\65103\65137\65139\65143\65145\65147\65149\65151-\65276\65296-\65305\65313-\65338\65343\65345-\65370\65382-\65391\65392\65393-\65437\65438-\65439\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500]*" (Arbitrary "Name")
    ]

dqs' :: TokenMatcher
dqs' =
    [ tokNext "\"" (Arbitrary "Literal" :. Arbitrary "String") Pop
    , tok "\\\\\\\\|\\\\\"|\\\\\\n" (Arbitrary "Literal" :. Arbitrary "String" :. Arbitrary "Escape")
    , anyOf strings'
    ]

root' :: TokenMatcher
root' =
    [ tok "\\n" (Arbitrary "Text")
    , tok "^(\\s*)([rRuU]{,2}\"\"\"(?:.|\\n)*?\"\"\")" (ByGroups [(Arbitrary "Text"), (Arbitrary "Literal" :. Arbitrary "String" :. Arbitrary "Doc")])
    , tok "^(\\s*)([rRuU]{,2}'''(?:.|\\n)*?''')" (ByGroups [(Arbitrary "Text"), (Arbitrary "Literal" :. Arbitrary "String" :. Arbitrary "Doc")])
    , tok "[^\\S\\n]+" (Arbitrary "Text")
    , tok "#.*$" (Arbitrary "Comment")
    , tok "[]{}:(),;[]" (Arbitrary "Punctuation")
    , tok "\\\\\\n" (Arbitrary "Text")
    , tok "\\\\" (Arbitrary "Text")
    , tok "(in|is|and|or|not)\\b" (Arbitrary "Operator" :. Arbitrary "Word")
    , tok "!=|==|<<|>>|[-\126+/*%=<>&^|.]" (Arbitrary "Operator")
    , anyOf keywords'
    , tokNext "(def)((?:\\s|\\\\\\s)+)" (ByGroups [(Arbitrary "Keyword"), (Arbitrary "Text")]) (GoTo funcname')
    , tokNext "(class)((?:\\s|\\\\\\s)+)" (ByGroups [(Arbitrary "Keyword"), (Arbitrary "Text")]) (GoTo classname')
    , tokNext "(from)((?:\\s|\\\\\\s)+)" (ByGroups [(Arbitrary "Keyword" :. Arbitrary "Namespace"), (Arbitrary "Text")]) (GoTo fromimport')
    , tokNext "(import)((?:\\s|\\\\\\s)+)" (ByGroups [(Arbitrary "Keyword" :. Arbitrary "Namespace"), (Arbitrary "Text")]) (GoTo import')
    , anyOf builtins'
    , anyOf backtick'
    , tokNext "(?:[rR]|[uU][rR]|[rR][uU])\"\"\"" (Arbitrary "Literal" :. Arbitrary "String") (GoTo tdqs')
    , tokNext "(?:[rR]|[uU][rR]|[rR][uU])'''" (Arbitrary "Literal" :. Arbitrary "String") (GoTo tsqs')
    , tokNext "(?:[rR]|[uU][rR]|[rR][uU])\"" (Arbitrary "Literal" :. Arbitrary "String") (GoTo dqs')
    , tokNext "(?:[rR]|[uU][rR]|[rR][uU])'" (Arbitrary "Literal" :. Arbitrary "String") (GoTo sqs')
    , tokNext "[uU]?\"\"\"" (Arbitrary "Literal" :. Arbitrary "String") (Combined [stringescape', tdqs'])
    , tokNext "[uU]?'''" (Arbitrary "Literal" :. Arbitrary "String") (Combined [stringescape', tsqs'])
    , tokNext "[uU]?\"" (Arbitrary "Literal" :. Arbitrary "String") (Combined [stringescape', dqs'])
    , tokNext "[uU]?'" (Arbitrary "Literal" :. Arbitrary "String") (Combined [stringescape', sqs'])
    , anyOf name'
    , anyOf numbers'
    ]

funcname' :: TokenMatcher
funcname' =
    [ tokNext "[A-Z_a-z\170\181\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1488-\1514\1520-\1522\1569-\1594\1600\1601-\1610\1646-\1647\1649-\1747\1749\1765-\1766\1774-\1775\1786-\1788\1791\1808\1810-\1839\1869-\1901\1920-\1957\1969\2308-\2361\2365\2384\2392-\2401\2429\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2493\2510\2524-\2525\2527-\2529\2544-\2545\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2649-\2652\2654\2674-\2676\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2749\2768\2784-\2785\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2877\2908-\2909\2911-\2913\2929\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3168-\3169\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3261\3294\3296-\3297\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3424-\3425\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3585-\3632\3634\3648-\3653\3654\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3762\3773\3776-\3780\3782\3804-\3805\3840\3904-\3911\3913-\3946\3976-\3979\4096-\4129\4131-\4135\4137-\4138\4176-\4181\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5920-\5937\5952-\5969\5984-\5996\5998-\6000\6016-\6067\6103\6108\6176-\6210\6211\6212-\6263\6272-\6312\6400-\6428\6480-\6509\6512-\6516\6528-\6569\6593-\6599\6656-\6678\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8305\8319\8336-\8340\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12337-\12341\12344-\12346\12347\12348\12353-\12438\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43011-\43013\43015-\43018\43020-\43042\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65137\65139\65143\65145\65147\65149\65151-\65276\65313-\65338\65345-\65370\65382-\65391\65392\65393-\65437\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500][0-9A-Z_a-z\170\181\183\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\768-\879\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1155-\1158\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1425-\1465\1467-\1469\1471\1473-\1474\1476-\1477\1479\1488-\1514\1520-\1522\1552-\1557\1569-\1594\1600\1601-\1610\1611-\1630\1632-\1641\1646-\1647\1648\1649-\1747\1749\1750-\1756\1759-\1764\1765-\1766\1767-\1768\1770-\1773\1774-\1775\1776-\1785\1786-\1788\1791\1808\1809\1810-\1839\1840-\1866\1869-\1901\1920-\1957\1958-\1968\1969\2305-\2306\2307\2308-\2361\2364\2365\2366-\2368\2369-\2376\2377-\2380\2381\2384\2385-\2388\2392-\2401\2402-\2403\2406-\2415\2429\2433\2434-\2435\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2492\2493\2494-\2496\2497-\2500\2503-\2504\2507-\2508\2509\2510\2519\2524-\2525\2527-\2529\2530-\2531\2534-\2543\2544-\2545\2561-\2562\2563\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2620\2622-\2624\2625-\2626\2631-\2632\2635-\2637\2649-\2652\2654\2662-\2671\2672-\2673\2674-\2676\2689-\2690\2691\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2748\2749\2750-\2752\2753-\2757\2759-\2760\2761\2763-\2764\2765\2768\2784-\2785\2786-\2787\2790-\2799\2817\2818-\2819\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2876\2877\2878\2879\2880\2881-\2883\2887-\2888\2891-\2892\2893\2902\2903\2908-\2909\2911-\2913\2918-\2927\2929\2946\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3006-\3007\3008\3009-\3010\3014-\3016\3018-\3020\3021\3031\3046-\3055\3073-\3075\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3134-\3136\3137-\3140\3142-\3144\3146-\3149\3157-\3158\3168-\3169\3174-\3183\3202-\3203\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3260\3261\3262\3263\3264-\3268\3270\3271-\3272\3274-\3275\3276-\3277\3285-\3286\3294\3296-\3297\3302-\3311\3330-\3331\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3390-\3392\3393-\3395\3398-\3400\3402-\3404\3405\3415\3424-\3425\3430-\3439\3458-\3459\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3530\3535-\3537\3538-\3540\3542\3544-\3551\3570-\3571\3585-\3632\3633\3634-\3635\3636-\3642\3648-\3653\3654\3655-\3662\3664-\3673\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3761\3762-\3763\3764-\3769\3771-\3772\3773\3776-\3780\3782\3784-\3789\3792-\3801\3804-\3805\3840\3864-\3865\3872-\3881\3893\3895\3897\3902-\3903\3904-\3911\3913-\3946\3953-\3966\3967\3968-\3972\3974-\3975\3976-\3979\3984-\3991\3993-\4028\4038\4096-\4129\4131-\4135\4137-\4138\4140\4141-\4144\4145\4146\4150-\4151\4152\4153\4160-\4169\4176-\4181\4182-\4183\4184-\4185\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4959\4969-\4977\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5906-\5908\5920-\5937\5938-\5940\5952-\5969\5970-\5971\5984-\5996\5998-\6000\6002-\6003\6016-\6067\6070\6071-\6077\6078-\6085\6086\6087-\6088\6089-\6099\6103\6108\6109\6112-\6121\6155-\6157\6160-\6169\6176-\6210\6211\6212-\6263\6272-\6312\6313\6400-\6428\6432-\6434\6435-\6438\6439-\6440\6441-\6443\6448-\6449\6450\6451-\6456\6457-\6459\6470-\6479\6480-\6509\6512-\6516\6528-\6569\6576-\6592\6593-\6599\6600-\6601\6608-\6617\6656-\6678\6679-\6680\6681-\6683\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7616-\7619\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8255-\8256\8276\8305\8319\8336-\8340\8400-\8412\8417\8421-\8427\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12330-\12335\12337-\12341\12344-\12346\12347\12348\12353-\12438\12441-\12442\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43010\43011-\43013\43014\43015-\43018\43019\43020-\43042\43043-\43044\43045-\43046\43047\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64286\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65024-\65039\65056-\65059\65075-\65076\65101-\65103\65137\65139\65143\65145\65147\65149\65151-\65276\65296-\65305\65313-\65338\65343\65345-\65370\65382-\65391\65392\65393-\65437\65438-\65439\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500]*" (Arbitrary "Name" :. Arbitrary "Function") Pop
    ]

import' :: TokenMatcher
import' =
    [ tok "(\\s+)(as)(\\s+)" (ByGroups [(Arbitrary "Text"), (Arbitrary "Keyword"), (Arbitrary "Text")])
    , tok "\\." (Arbitrary "Name" :. Arbitrary "Namespace")
    , tok "[A-Z_a-z\170\181\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1488-\1514\1520-\1522\1569-\1594\1600\1601-\1610\1646-\1647\1649-\1747\1749\1765-\1766\1774-\1775\1786-\1788\1791\1808\1810-\1839\1869-\1901\1920-\1957\1969\2308-\2361\2365\2384\2392-\2401\2429\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2493\2510\2524-\2525\2527-\2529\2544-\2545\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2649-\2652\2654\2674-\2676\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2749\2768\2784-\2785\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2877\2908-\2909\2911-\2913\2929\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3168-\3169\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3261\3294\3296-\3297\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3424-\3425\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3585-\3632\3634\3648-\3653\3654\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3762\3773\3776-\3780\3782\3804-\3805\3840\3904-\3911\3913-\3946\3976-\3979\4096-\4129\4131-\4135\4137-\4138\4176-\4181\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5920-\5937\5952-\5969\5984-\5996\5998-\6000\6016-\6067\6103\6108\6176-\6210\6211\6212-\6263\6272-\6312\6400-\6428\6480-\6509\6512-\6516\6528-\6569\6593-\6599\6656-\6678\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8305\8319\8336-\8340\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12337-\12341\12344-\12346\12347\12348\12353-\12438\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43011-\43013\43015-\43018\43020-\43042\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65137\65139\65143\65145\65147\65149\65151-\65276\65313-\65338\65345-\65370\65382-\65391\65392\65393-\65437\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500][0-9A-Z_a-z\170\181\183\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\768-\879\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1155-\1158\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1425-\1465\1467-\1469\1471\1473-\1474\1476-\1477\1479\1488-\1514\1520-\1522\1552-\1557\1569-\1594\1600\1601-\1610\1611-\1630\1632-\1641\1646-\1647\1648\1649-\1747\1749\1750-\1756\1759-\1764\1765-\1766\1767-\1768\1770-\1773\1774-\1775\1776-\1785\1786-\1788\1791\1808\1809\1810-\1839\1840-\1866\1869-\1901\1920-\1957\1958-\1968\1969\2305-\2306\2307\2308-\2361\2364\2365\2366-\2368\2369-\2376\2377-\2380\2381\2384\2385-\2388\2392-\2401\2402-\2403\2406-\2415\2429\2433\2434-\2435\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2492\2493\2494-\2496\2497-\2500\2503-\2504\2507-\2508\2509\2510\2519\2524-\2525\2527-\2529\2530-\2531\2534-\2543\2544-\2545\2561-\2562\2563\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2620\2622-\2624\2625-\2626\2631-\2632\2635-\2637\2649-\2652\2654\2662-\2671\2672-\2673\2674-\2676\2689-\2690\2691\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2748\2749\2750-\2752\2753-\2757\2759-\2760\2761\2763-\2764\2765\2768\2784-\2785\2786-\2787\2790-\2799\2817\2818-\2819\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2876\2877\2878\2879\2880\2881-\2883\2887-\2888\2891-\2892\2893\2902\2903\2908-\2909\2911-\2913\2918-\2927\2929\2946\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3006-\3007\3008\3009-\3010\3014-\3016\3018-\3020\3021\3031\3046-\3055\3073-\3075\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3134-\3136\3137-\3140\3142-\3144\3146-\3149\3157-\3158\3168-\3169\3174-\3183\3202-\3203\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3260\3261\3262\3263\3264-\3268\3270\3271-\3272\3274-\3275\3276-\3277\3285-\3286\3294\3296-\3297\3302-\3311\3330-\3331\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3390-\3392\3393-\3395\3398-\3400\3402-\3404\3405\3415\3424-\3425\3430-\3439\3458-\3459\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3530\3535-\3537\3538-\3540\3542\3544-\3551\3570-\3571\3585-\3632\3633\3634-\3635\3636-\3642\3648-\3653\3654\3655-\3662\3664-\3673\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3761\3762-\3763\3764-\3769\3771-\3772\3773\3776-\3780\3782\3784-\3789\3792-\3801\3804-\3805\3840\3864-\3865\3872-\3881\3893\3895\3897\3902-\3903\3904-\3911\3913-\3946\3953-\3966\3967\3968-\3972\3974-\3975\3976-\3979\3984-\3991\3993-\4028\4038\4096-\4129\4131-\4135\4137-\4138\4140\4141-\4144\4145\4146\4150-\4151\4152\4153\4160-\4169\4176-\4181\4182-\4183\4184-\4185\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4959\4969-\4977\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5906-\5908\5920-\5937\5938-\5940\5952-\5969\5970-\5971\5984-\5996\5998-\6000\6002-\6003\6016-\6067\6070\6071-\6077\6078-\6085\6086\6087-\6088\6089-\6099\6103\6108\6109\6112-\6121\6155-\6157\6160-\6169\6176-\6210\6211\6212-\6263\6272-\6312\6313\6400-\6428\6432-\6434\6435-\6438\6439-\6440\6441-\6443\6448-\6449\6450\6451-\6456\6457-\6459\6470-\6479\6480-\6509\6512-\6516\6528-\6569\6576-\6592\6593-\6599\6600-\6601\6608-\6617\6656-\6678\6679-\6680\6681-\6683\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7616-\7619\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8255-\8256\8276\8305\8319\8336-\8340\8400-\8412\8417\8421-\8427\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12330-\12335\12337-\12341\12344-\12346\12347\12348\12353-\12438\12441-\12442\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43010\43011-\43013\43014\43015-\43018\43019\43020-\43042\43043-\43044\43045-\43046\43047\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64286\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65024-\65039\65056-\65059\65075-\65076\65101-\65103\65137\65139\65143\65145\65147\65149\65151-\65276\65296-\65305\65313-\65338\65343\65345-\65370\65382-\65391\65392\65393-\65437\65438-\65439\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500]*" (Arbitrary "Name" :. Arbitrary "Namespace")
    , tok "(\\s*)(,)(\\s*)" (ByGroups [(Arbitrary "Text"), (Arbitrary "Operator"), (Arbitrary "Text")])
    , tokNext "" (Arbitrary "Text") Pop
    ]

tdqs' :: TokenMatcher
tdqs' =
    [ tokNext "\"\"\"" (Arbitrary "Literal" :. Arbitrary "String") Pop
    , anyOf strings'
    , anyOf nl'
    ]

strings' :: TokenMatcher
strings' =
    [ tok "[^\\\\\\'\"%\\n]+" (Arbitrary "Literal" :. Arbitrary "String")
    , tok "[\\'\"\\\\]" (Arbitrary "Literal" :. Arbitrary "String")
    , tok "%" (Arbitrary "Literal" :. Arbitrary "String")
    ]

fromimport' :: TokenMatcher
fromimport' =
    [ tokNext "(\\s+)(import)\\b" (ByGroups [(Arbitrary "Text"), (Arbitrary "Keyword")]) Pop
    , tok "\\." (Arbitrary "Name" :. Arbitrary "Namespace")
    , tok "[A-Z_a-z\170\181\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1488-\1514\1520-\1522\1569-\1594\1600\1601-\1610\1646-\1647\1649-\1747\1749\1765-\1766\1774-\1775\1786-\1788\1791\1808\1810-\1839\1869-\1901\1920-\1957\1969\2308-\2361\2365\2384\2392-\2401\2429\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2493\2510\2524-\2525\2527-\2529\2544-\2545\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2649-\2652\2654\2674-\2676\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2749\2768\2784-\2785\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2877\2908-\2909\2911-\2913\2929\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3168-\3169\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3261\3294\3296-\3297\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3424-\3425\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3585-\3632\3634\3648-\3653\3654\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3762\3773\3776-\3780\3782\3804-\3805\3840\3904-\3911\3913-\3946\3976-\3979\4096-\4129\4131-\4135\4137-\4138\4176-\4181\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5920-\5937\5952-\5969\5984-\5996\5998-\6000\6016-\6067\6103\6108\6176-\6210\6211\6212-\6263\6272-\6312\6400-\6428\6480-\6509\6512-\6516\6528-\6569\6593-\6599\6656-\6678\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8305\8319\8336-\8340\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12337-\12341\12344-\12346\12347\12348\12353-\12438\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43011-\43013\43015-\43018\43020-\43042\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65137\65139\65143\65145\65147\65149\65151-\65276\65313-\65338\65345-\65370\65382-\65391\65392\65393-\65437\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500][0-9A-Z_a-z\170\181\183\186\192-\214\216-\246\248-\442\443\444-\447\448-\451\452-\577\592-\687\688-\705\710-\721\736-\740\750\768-\879\902\904-\906\908\910-\929\931-\974\976-\1013\1015-\1153\1155-\1158\1162-\1230\1232-\1273\1280-\1295\1329-\1366\1369\1377-\1415\1425-\1465\1467-\1469\1471\1473-\1474\1476-\1477\1479\1488-\1514\1520-\1522\1552-\1557\1569-\1594\1600\1601-\1610\1611-\1630\1632-\1641\1646-\1647\1648\1649-\1747\1749\1750-\1756\1759-\1764\1765-\1766\1767-\1768\1770-\1773\1774-\1775\1776-\1785\1786-\1788\1791\1808\1809\1810-\1839\1840-\1866\1869-\1901\1920-\1957\1958-\1968\1969\2305-\2306\2307\2308-\2361\2364\2365\2366-\2368\2369-\2376\2377-\2380\2381\2384\2385-\2388\2392-\2401\2402-\2403\2406-\2415\2429\2433\2434-\2435\2437-\2444\2447-\2448\2451-\2472\2474-\2480\2482\2486-\2489\2492\2493\2494-\2496\2497-\2500\2503-\2504\2507-\2508\2509\2510\2519\2524-\2525\2527-\2529\2530-\2531\2534-\2543\2544-\2545\2561-\2562\2563\2565-\2570\2575-\2576\2579-\2600\2602-\2608\2610-\2611\2613-\2614\2616-\2617\2620\2622-\2624\2625-\2626\2631-\2632\2635-\2637\2649-\2652\2654\2662-\2671\2672-\2673\2674-\2676\2689-\2690\2691\2693-\2701\2703-\2705\2707-\2728\2730-\2736\2738-\2739\2741-\2745\2748\2749\2750-\2752\2753-\2757\2759-\2760\2761\2763-\2764\2765\2768\2784-\2785\2786-\2787\2790-\2799\2817\2818-\2819\2821-\2828\2831-\2832\2835-\2856\2858-\2864\2866-\2867\2869-\2873\2876\2877\2878\2879\2880\2881-\2883\2887-\2888\2891-\2892\2893\2902\2903\2908-\2909\2911-\2913\2918-\2927\2929\2946\2947\2949-\2954\2958-\2960\2962-\2965\2969-\2970\2972\2974-\2975\2979-\2980\2984-\2986\2990-\3001\3006-\3007\3008\3009-\3010\3014-\3016\3018-\3020\3021\3031\3046-\3055\3073-\3075\3077-\3084\3086-\3088\3090-\3112\3114-\3123\3125-\3129\3134-\3136\3137-\3140\3142-\3144\3146-\3149\3157-\3158\3168-\3169\3174-\3183\3202-\3203\3205-\3212\3214-\3216\3218-\3240\3242-\3251\3253-\3257\3260\3261\3262\3263\3264-\3268\3270\3271-\3272\3274-\3275\3276-\3277\3285-\3286\3294\3296-\3297\3302-\3311\3330-\3331\3333-\3340\3342-\3344\3346-\3368\3370-\3385\3390-\3392\3393-\3395\3398-\3400\3402-\3404\3405\3415\3424-\3425\3430-\3439\3458-\3459\3461-\3478\3482-\3505\3507-\3515\3517\3520-\3526\3530\3535-\3537\3538-\3540\3542\3544-\3551\3570-\3571\3585-\3632\3633\3634-\3635\3636-\3642\3648-\3653\3654\3655-\3662\3664-\3673\3713-\3714\3716\3719-\3720\3722\3725\3732-\3735\3737-\3743\3745-\3747\3749\3751\3754-\3755\3757-\3760\3761\3762-\3763\3764-\3769\3771-\3772\3773\3776-\3780\3782\3784-\3789\3792-\3801\3804-\3805\3840\3864-\3865\3872-\3881\3893\3895\3897\3902-\3903\3904-\3911\3913-\3946\3953-\3966\3967\3968-\3972\3974-\3975\3976-\3979\3984-\3991\3993-\4028\4038\4096-\4129\4131-\4135\4137-\4138\4140\4141-\4144\4145\4146\4150-\4151\4152\4153\4160-\4169\4176-\4181\4182-\4183\4184-\4185\4256-\4293\4304-\4346\4348\4352-\4441\4447-\4514\4520-\4601\4608-\4680\4682-\4685\4688-\4694\4696\4698-\4701\4704-\4744\4746-\4749\4752-\4784\4786-\4789\4792-\4798\4800\4802-\4805\4808-\4822\4824-\4880\4882-\4885\4888-\4954\4959\4969-\4977\4992-\5007\5024-\5108\5121-\5740\5743-\5750\5761-\5786\5792-\5866\5870-\5872\5888-\5900\5902-\5905\5906-\5908\5920-\5937\5938-\5940\5952-\5969\5970-\5971\5984-\5996\5998-\6000\6002-\6003\6016-\6067\6070\6071-\6077\6078-\6085\6086\6087-\6088\6089-\6099\6103\6108\6109\6112-\6121\6155-\6157\6160-\6169\6176-\6210\6211\6212-\6263\6272-\6312\6313\6400-\6428\6432-\6434\6435-\6438\6439-\6440\6441-\6443\6448-\6449\6450\6451-\6456\6457-\6459\6470-\6479\6480-\6509\6512-\6516\6528-\6569\6576-\6592\6593-\6599\6600-\6601\6608-\6617\6656-\6678\6679-\6680\6681-\6683\7424-\7467\7468-\7521\7522-\7543\7544\7545-\7578\7579-\7615\7616-\7619\7680-\7835\7840-\7929\7936-\7957\7960-\7965\7968-\8005\8008-\8013\8016-\8023\8025\8027\8029\8031-\8061\8064-\8116\8118-\8124\8126\8130-\8132\8134-\8140\8144-\8147\8150-\8155\8160-\8172\8178-\8180\8182-\8188\8255-\8256\8276\8305\8319\8336-\8340\8400-\8412\8417\8421-\8427\8450\8455\8458-\8467\8469\8472\8473-\8477\8484\8486\8488\8490-\8493\8494\8495-\8497\8499-\8500\8501-\8504\8505\8508-\8511\8517-\8521\8544-\8579\11264-\11310\11312-\11358\11392-\11492\11520-\11557\11568-\11621\11631\11648-\11670\11680-\11686\11688-\11694\11696-\11702\11704-\11710\11712-\11718\11720-\11726\11728-\11734\11736-\11742\12293\12294\12295\12321-\12329\12330-\12335\12337-\12341\12344-\12346\12347\12348\12353-\12438\12441-\12442\12445-\12446\12447\12449-\12538\12540-\12542\12543\12549-\12588\12593-\12686\12704-\12727\12784-\12799\13312-\19893\19968-\40891\40960-\40980\40981\40982-\42124\43008-\43009\43010\43011-\43013\43014\43015-\43018\43019\43020-\43042\43043-\43044\43045-\43046\43047\44032-\55203\63744-\64045\64048-\64106\64112-\64217\64256-\64262\64275-\64279\64285\64286\64287-\64296\64298-\64310\64312-\64316\64318\64320-\64321\64323-\64324\64326-\64433\64467-\64605\64612-\64829\64848-\64911\64914-\64967\65008-\65017\65024-\65039\65056-\65059\65075-\65076\65101-\65103\65137\65139\65143\65145\65147\65149\65151-\65276\65296-\65305\65313-\65338\65343\65345-\65370\65382-\65391\65392\65393-\65437\65438-\65439\65440-\65470\65474-\65479\65482-\65487\65490-\65495\65498-\65500]*" (Arbitrary "Name" :. Arbitrary "Namespace")
    ]