46 lines
1.2 KiB
TypeScript
46 lines
1.2 KiB
TypeScript
import { Stack } from "./stack";
|
|
export declare class CachedToken {
|
|
start: number;
|
|
value: number;
|
|
end: number;
|
|
extended: number;
|
|
lookAhead: number;
|
|
mask: number;
|
|
context: number;
|
|
}
|
|
export declare class InputStream {
|
|
private chunk2;
|
|
private chunk2Pos;
|
|
next: number;
|
|
pos: number;
|
|
private rangeIndex;
|
|
private range;
|
|
peek(offset: number): any;
|
|
acceptToken(token: number, endOffset?: number): void;
|
|
private getChunk;
|
|
private readNext;
|
|
advance(n?: number): number;
|
|
private setDone;
|
|
}
|
|
export interface Tokenizer {
|
|
}
|
|
export declare class LocalTokenGroup implements Tokenizer {
|
|
readonly precTable: number;
|
|
readonly elseToken?: number;
|
|
contextual: boolean;
|
|
fallback: boolean;
|
|
extend: boolean;
|
|
readonly data: Readonly<Uint16Array>;
|
|
constructor(data: Readonly<Uint16Array> | string, precTable: number, elseToken?: number);
|
|
token(input: InputStream, stack: Stack): void;
|
|
}
|
|
interface ExternalOptions {
|
|
contextual?: boolean;
|
|
fallback?: boolean;
|
|
extend?: boolean;
|
|
}
|
|
export declare class ExternalTokenizer {
|
|
constructor(token: (input: InputStream, stack: Stack) => void, options?: ExternalOptions);
|
|
}
|
|
export {};
|