@anyhowstep/tokenizer v1.1.1

Installation

npm install @anyhowstep/tokenizer

Sample Usage

See ./src/test/test.ts Or,

See ./src/test/test2.ts Or,

See ./src/test/test3.ts Or,

See ./src/test/test4.ts Or,

import * as tokenizer from "@anyhowstep/tokenizer";

const src = `text goes here`;
const result = tokenizer.tokenize(src, (args) => {
    {
        const match = /^(\s+)/.exec(args.subStr);
        if (match != null) {
            return {
                token : new tokenizer.Token({
                    value  : match[1],
                    type   : "whitespace",
                    row    : args.row,
                    column : args.column,
                })
            };
        }
    }
    {
        const match = /^(\S+)/.exec(args.subStr);
        if (match != null) {
            return {
                token : new tokenizer.Token({
                    value  : match[1],
                    type   : "non-whitespace",
                    row    : args.row,
                    column : args.column,
                })
            };
        }
    }
    return {
        err : new Error(`Unknown character ${args.subStr[0]}`),
        consumeCount : 1,
    };
});

console.log(result);

TODO

• Make errors more customizable (like adding ErrorT)