abstract-scanner v2.0.2

abstract-scanner

Universal scanner for lexical analysis.

The whole project are written with typescript, but it's ok to use it as a common nodejs module. Of course, you can find the umd version of this module in lib/*.

Install

npm install abstract-scanner --save

Exports

• Token: scan result wrapper.
• AbstractScanner: abstract scanner needed to be extended.
• utils: namespace of some helpful functions .
• version: project version.

Interface

class Token {
    type: string;
    loc: SourceLocation;
    source: string;
    value?: any;
    message?: string | string[];
    constructor(type: string, source: string, start: Position, end: Position);
}

interface Position {
    index: number;
    line: number;
    column: number;
}

interface SourceLocation {
    readonly start: Position;
    readonly end: Position;
}

interface Config {
    line?: number;
    column?: number;
    start?: number;
    end?: number;
}

abstract class AbstractScanner {
    readonly source: string;
    readonly length: number;
    marker: Position;
    private scanStartingMarker;
    private scanEndingMarker;
    constructor(source: string, config?: Config);
    saveState(): Position;
    restoreState(state: Position): void;
    startScan(): void;
    endScan(): void;
    clear(): void;
    getScanLength(): number;
    getScanResult(): string;
    constructToken(type: string): Token;
    constructIllegalToken(message?: string | string[]): Token;
    eof(): boolean;
    peek(length?: number): string;
    move(offset?: number): void;
    moveWhen(judge: (cc: number, ch?: string) => boolean): void;
    fromCodePoint(cp: number): string;
    fromCharCode(cc: number): string;
    getCodePoint(offset?: number): number;
    getCharCode(offset?: number): number;
    scanLineTerminator(): string;
    scanBlankSpace(): string;
    skipSpace(): void;
}


namespace utils {
    function isWhiteSpace(cp: number): boolean;
    function isLineTerminator(cp: number): boolean;
    function isDecimalDigit(cp: number): boolean;
    function isHexDigit(cp: number): boolean;
    function isOctalDigit(cp: number): boolean;
    function isBinaryDigit(cp: number): boolean;
    namespace JS {
        isIdentifierStart(cp: number): boolean;
        isIdentifierPart(cp: number): boolean;
        isKeyword(id: string): boolean;
        isFutureReservedWord(id: string): boolean;
        isStrictModeReservedWord(id: string): boolean;
    };
}

Example

A full version scanner of tokenzing javascript is in directory examples/.

For an illustrative purpose, here's an example of how to scan the NumericLiteral of javascript.

import { Token, AbstractScanner, utils } from 'abstract-scanner';

class Scanner extends AbstractScanner {
    scanHexLiteral(): Token | null {
        const str = this.peek(2);
        if (str[0] === '0' && (str[1] === 'x' || str[1] === 'X')) {
            this.startScan();
            this.move(2);
            this.moveWhen(cp => utils.isHexDigit(cp));
            this.endScan();
            if (this.getScanLength() === 2) {
                return this.constructIllegalToken();
            } else {
                const token = this.constructToken('NumericLiteral');
                token.value = parseInt(token.source, 16);
                return token;
            }
        }
        return null;
    }

    scanBinaryLiteral(): Token | null {...}

    scanOctalLiteral(): Token | null {...}

    scanDecimalLiteral(): Token | null {...}

    nextToken(): Token | null {
        return this.scanHexLiteral() || this.scanOctalLiteral()
        || this.scanBinaryLiteral() || this.scanDecimalLiteral();
    }
}