{"id":2515,"date":"2019-08-26T15:10:09","date_gmt":"2019-08-26T14:10:09","guid":{"rendered":"https:\/\/solidt.eu\/site\/?p=2515"},"modified":"2026-09-29T10:10:58","modified_gmt":"2026-09-29T09:10:58","slug":"simple-javascript-parser","status":"publish","type":"post","link":"https:\/\/solidt.eu\/site\/simple-javascript-parser\/","title":{"rendered":"Typescript simple parser"},"content":{"rendered":"\n<pre class=\"EnlighterJSRAW\" data-enlighter-language=\"typescript\" data-enlighter-theme=\"\" data-enlighter-highlight=\"\" data-enlighter-linenumbers=\"\" data-enlighter-lineoffset=\"\" data-enlighter-title=\"\" data-enlighter-group=\"\">interface ITokenConfig {\n    name?: string;\n    startsWith?: string;\n    endsWith?: string;\n    skip?: string;\n    filter?: boolean;\n    t?: string | string[];\n    e?: string;\n    b?: string;\n    startScope?: string;\n    endScope?: string;\n}\n\ninterface ITokenMatcher extends ITokenConfig {\n    name: string;\n    scopeName: string;\n    regex?: RegExp;\n    blockRegex?: RegExp;\n}\n\ntype IToken = {\n    text: string;\n    name: string;\n    scope?: string;\n    offset: number;\n    line: number;\n    column: number;\n};\n\ntype ITokenArray = Array&lt;ITokenArray | IToken> &amp; {\n    parent?: ITokenArray;\n    scope?: string;\n};\n\ninterface IGrammar {\n    name?: string;\n    scopes: Record&lt;string, Record&lt;string, ITokenConfig>>;\n}\n\nconst getTokens = (grammar: IGrammar): ITokenMatcher[] => {\n    const tokens: ITokenMatcher[] = [];\n\n    for (const [scopeName, scope] of Object.entries(grammar.scopes)) {\n        for (const [name, token] of Object.entries(scope)) {\n            tokens.push({\n                ...token,\n                name,\n                scopeName,\n                regex: token.e\n                    ? new RegExp(`^(?:${token.e})`)\n                    : undefined,\n                blockRegex: token.b\n                    ? new RegExp(token.b, 'm')\n                    : undefined\n            });\n        }\n    }\n\n    return tokens;\n};\n\nconst findEnd = (\n    input: string,\n    start: number,\n    token: ITokenMatcher\n): number => {\n    const sw = token.startsWith!;\n    const ew = token.endsWith!;\n    let end = input.indexOf(ew, start + sw.length);\n\n    if (token.skip) {\n        while (end >= 0) {\n            let p = end - 1;\n\n            while (p >= start + sw.length &amp;&amp; input[p] === '\\\\') {\n                p--;\n            }\n\n            if ((end - p) % 2 !== 0) break;\n\n            end = input.indexOf(ew, end + ew.length);\n        }\n    }\n\n    return end &lt; 0 ? input.length : end + ew.length;\n};\n\nconst matchToken = (\n    input: string,\n    pos: number,\n    token: ITokenMatcher\n): number => {\n    if (token.t) {\n        const values = Array.isArray(token.t) ? token.t : [token.t];\n        let length = 0;\n\n        for (const value of values) {\n            if (input.startsWith(value, pos)) {\n                length = Math.max(length, value.length);\n            }\n        }\n\n        return length;\n    }\n\n    if (token.startsWith &amp;&amp; token.endsWith) {\n        if (!input.startsWith(token.startsWith, pos)) return 0;\n        return findEnd(input, pos, token) - pos;\n    }\n\n    if (token.regex) {\n        const match = input.slice(pos).match(token.regex);\n\n        if (!match) return 0;\n        if (token.blockRegex?.test(match[0])) return 0;\n\n        return match[0].length;\n    }\n\n    return 0;\n};\n\nconst newScope = (grammar: IGrammar) => {\n    const allTokens = getTokens(grammar);\n    let scopeTokens: ITokenMatcher[] = [];\n\n    const root: ITokenArray = [];\n    let current = root;\n\n    const update = () => {\n        scopeTokens = allTokens.filter(\n            token => token.scopeName === current.scope\n        );\n    };\n\n    const push = (scope: string) => {\n        const child: ITokenArray = [];\n        child.parent = current;\n        child.scope = scope;\n        current.push(child);\n        current = child;\n        update();\n    };\n\n    const pop = () => {\n        if (current.parent) current = current.parent;\n        update();\n    };\n\n    push('default');\n\n    return {\n        getTree: () => root,\n        getScopeTokens: () => scopeTokens,\n\n        addMatch: (\n            token: ITokenMatcher | null,\n            text: string,\n            offset: number,\n            line: number,\n            column: number\n        ) => {\n            if (!token) {\n                current.push({\n                    text,\n                    name: 'UNKNOWN',\n                    offset,\n                    line,\n                    column\n                });\n                return;\n            }\n\n            if (!token.filter) {\n                current.push({\n                    text,\n                    name: token.name,\n                    offset,\n                    line,\n                    column\n                });\n            }\n\n            if (token.startScope) {\n                push(token.startScope);\n            }\n\n            if (token.endScope &amp;&amp; current.scope === token.endScope) {\n                pop();\n            }\n        }\n    };\n};\n\nconst lex = (input: string, grammar: IGrammar): ITokenArray => {\n    const scope = newScope(grammar);\n\n    let pos = 0;\n    let line = 1;\n    let column = 1;\n\n    while (pos &lt; input.length) {\n        let best: ITokenMatcher | null = null;\n        let bestLength = 0;\n\n        for (const token of scope.getScopeTokens()) {\n            const length = matchToken(input, pos, token);\n\n            if (length > bestLength) {\n                best = token;\n                bestLength = length;\n            }\n        }\n\n        if (!best || bestLength === 0) {\n            const text = input[pos]!;\n\n            scope.addMatch(\n                null,\n                text,\n                pos,\n                line,\n                column\n            );\n\n            if (text === '\\n') {\n                line++;\n                column = 1;\n            } else {\n                column++;\n            }\n\n            pos++;\n            continue;\n        }\n\n        const text = input.slice(pos, pos + bestLength);\n\n        scope.addMatch(\n            best,\n            text,\n            pos,\n            line,\n            column\n        );\n\n        for (let i = 0; i &lt; text.length; i++) {\n            if (text[i] === '\\n') {\n                line++;\n                column = 1;\n            } else {\n                column++;\n            }\n        }\n\n        pos += bestLength;\n    }\n\n    return scope.getTree();\n};\n\nconst grammar: IGrammar = {\n    name: 'custom1',\n\n    scopes: {\n        default: {\n            'comment-line': {\n                startsWith: '\/\/',\n                endsWith: '\\n',\n                filter: true\n            },\n\n            'comment-block': {\n                startsWith: '\/*',\n                endsWith: '*\/',\n                filter: true\n            },\n\n            'brace-start': {\n                t: '(',\n                startScope: 'default'\n            },\n\n            'brace-end': {\n                t: ')',\n                endScope: 'default'\n            },\n\n            assignment: {\n                t: '='\n            },\n\n            'bool-operator': {\n                t: ['==', '!=']\n            },\n\n            'incr-operator': {\n                t: ['+=', '-=']\n            },\n\n            'math-operator': {\n                t: ['+', '-', '*', '\/']\n            },\n\n            number: {\n                e: '[0-9]+\\\\.?[0-9]*'\n            },\n\n            separator: {\n                t: ';'\n            },\n\n            string: {\n                startsWith: '\"',\n                skip: '\\\\\"',\n                endsWith: '\"'\n            },\n\n            function: {\n                t: 'function',\n                startScope: 'function'\n            },\n\n            name: {\n                e: '[_a-zA-Z][_a-zA-Z0-9]*'\n            },\n\n            whitespace: {\n                e: '\\\\s+',\n                filter: true\n            }\n        },\n\n        function: {\n            'comment-line': {\n                startsWith: '\/\/',\n                endsWith: '\\n',\n                filter: true\n            },\n\n            'comment-block': {\n                startsWith: '\/*',\n                endsWith: '*\/',\n                filter: true\n            },\n\n            number: {\n                e: '[0-9]+\\\\.?[0-9]*'\n            },\n\n            whitespace: {\n                e: '\\\\s+',\n                filter: true\n            },\n\n            end: {\n                t: 'end',\n                endScope: 'function'\n            }\n        }\n    }\n};\n\nconst expr = `\n    var a = (1+(2+3));\n    \/\/ single line comment\n    \"Hello \\\\\"{0}\\\\\" this is nice\";\n    \/* multiline\n       comment *\/\n    work a function 0.5 4 3.22 end 5 6.2 8\n`;\n\nconst tokens = lex(expr, grammar);\n\nfunction printTokens(\n    tokens: ITokenArray,\n    depth = 0\n): string[] {\n    const result: string[] = [];\n\n    for (const token of tokens) {\n        if (Array.isArray(token)) {\n            result.push(\n                ...printTokens(token, depth + 1)\n            );\n        } else {\n            result.push(\n                `${'  '.repeat(depth)}${token.text.trim()} ` +\n                `&lt;-(${token.name}) ` +\n                `[${token.line}:${token.column}]`\n            );\n        }\n    }\n\n    return result;\n}\n\nconsole.log(printTokens(tokens).join('\\n'));\n<\/pre>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<pre class=\"EnlighterJSRAW\" data-enlighter-language=\"raw\" data-enlighter-theme=\"\" data-enlighter-highlight=\"\" data-enlighter-linenumbers=\"\" data-enlighter-lineoffset=\"\" data-enlighter-title=\"\" data-enlighter-group=\"\">  var &lt;-(name) [2:5]\n  a &lt;-(name) [2:9]\n  = &lt;-(assignment) [2:11]\n  ( &lt;-(brace-start) [2:13]\n    1 &lt;-(number) [2:14]\n    + &lt;-(math-operator) [2:15]\n    ( &lt;-(brace-start) [2:16]\n      2 &lt;-(number) [2:17]\n      + &lt;-(math-operator) [2:18]\n      3 &lt;-(number) [2:19]\n      ) &lt;-(brace-end) [2:20]\n    ) &lt;-(brace-end) [2:21]\n  ; &lt;-(separator) [2:22]\n  \"Hello \\\"{0}\\\" this is nice\" &lt;-(string) [4:5]\n  ; &lt;-(separator) [4:33]\n  work &lt;-(name) [7:5]\n  a &lt;-(name) [7:10]\n  function &lt;-(function) [7:12]\n    0.5 &lt;-(number) [7:21]\n    4 &lt;-(number) [7:25]\n    3.22 &lt;-(number) [7:27]\n    end &lt;-(end) [7:32]\n  5 &lt;-(number) [7:36]\n  6.2 &lt;-(number) [7:38]\n  8 &lt;-(number) [7:42]<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n","protected":false},"excerpt":{"rendered":"","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"inline_featured_image":false,"footnotes":""},"categories":[5,4,1],"tags":[],"class_list":["post-2515","post","type-post","status-publish","format-standard","hentry","category-javascript","category-programming","category-uncategorized"],"_links":{"self":[{"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/posts\/2515","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/comments?post=2515"}],"version-history":[{"count":8,"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/posts\/2515\/revisions"}],"predecessor-version":[{"id":10324,"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/posts\/2515\/revisions\/10324"}],"wp:attachment":[{"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/media?parent=2515"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/categories?post=2515"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/solidt.eu\/site\/wp-json\/wp\/v2\/tags?post=2515"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}