nwex.de/html/tokenizer/token.ts

import { VERIFY, VERIFY_NOT_REACHED } from "../../util/assertions.js";

export const enum Type {
    DOCTYPE = 'DOCTYPE',
    StartTag = 'start tag',
    EndTag = 'end tag',
    Comment = 'comment',
    Character = 'character',
    EndOfFile = 'end-of-file'
}

export const REPLACEMENT_CHARACTER = '\uFFFD';

export type Range = {
    start: Position,
    end: Position
}

export type Position = {
    line: number,
    column: number,
    index: number
}

export class Attribute {
    public name: string;
    public value: string;

    public constructor(name: string, value: string) {
        this.name = name;
        this.value = value;
    }

    public appendToName(characters: string): void {
        this.name += characters;
    }

    public appendReplacementCharacterToName(): void {
        this.appendToName(REPLACEMENT_CHARACTER);
    }

    public appendToValue(characters: string): void {
        this.value += characters;
    }

    public appendReplacementCharacterToValue(): void {
        this.appendToValue(REPLACEMENT_CHARACTER);
    }

    public static createWithEmptyNameAndValue(): Attribute {
        return new Attribute('', '');
    }

    public static createWithEmptyValue(name: string): Attribute {
        return new Attribute(name, '');
    }
}

export class AttributeList {
    private attributes: Array<Attribute>;

    public constructor() {
        this.attributes = new Array<Attribute>();
    }

    public get current(): Attribute {
        return this.attributes[this.attributes.length - 1];
    }

    public get list(): Array<Attribute> {
        return this.attributes;
    }

    public nonEmpty(): boolean {
        return this.list.length !== 0;
    }

    public append(attribute: Attribute): void {
        this.attributes.push(attribute);
    }
}

export abstract class Token {
    #type: Type;
    #range!: Range;

    protected constructor(type: Type) {
        this.#type = type;

        // @ts-expect-error
        this.#range = {};
    }

    public startingAt(position: Position): this {
        this.#range.start = { line: position.line, column: position.column, index: position.index };

        return this;
    }

    public endingAt(position: Position): this {
        this.#range.end = { line: position.line, column: position.column, index: position.index };

        return this;
    }

    public at(position: Position): this {
        this.#range.start = { line: position.line, column: position.column, index: position.index };
        this.#range.end = { line: position.line, column: position.column, index: position.index };

        return this;
    }

    public get range(): Range {
        return this.#range;
    }

    public get type(): Type {
        return this.#type;
    }
}

export class CharacterToken extends Token {
    public readonly data: NonNullable<string>;

    public constructor(data: NonNullable<string>) {
        super(Type.Character);

        this.data = data;
    }

    public static createWith(data: NonNullable<string>): CharacterToken {
        return new CharacterToken(data);
    }

    public static createReplacementCharacter(): CharacterToken {
        return new CharacterToken(REPLACEMENT_CHARACTER);
    }
}

export class CommentToken extends Token {
    public data: NonNullable<string>;

    public constructor(data: NonNullable<string>) {
        super(Type.Comment);

        this.data = data;
    }

    public append(characters: string): void {
        this.data += characters;
    }

    public appendReplacementCharacter(): void {
        this.append(REPLACEMENT_CHARACTER);
    }

    public static createEmpty(): CommentToken {
        return new CommentToken('');
    }

    public static createWith(data: string): CommentToken {
        return new CommentToken(data);
    }
}

export class EndOfFileToken extends Token {
    public constructor() {
        super(Type.EndOfFile);
    }

    public static create(): EndOfFileToken {
        return new EndOfFileToken();
    }
}

export class StartTagToken extends Token {
    public name: NonNullable<string>;
    public readonly attributes: AttributeList;

    public constructor(name: NonNullable<string>, attributes: AttributeList) {
        super(Type.StartTag);

        this.name = name;
        this.attributes = attributes;
    }

    public appendToName(characters: string): void {
        this.name += characters;
    }

    public appendReplacementCharacterToName(): void {
        this.appendToName(REPLACEMENT_CHARACTER);
    }

    public static createEmpty(): StartTagToken {
        return new StartTagToken('', new AttributeList());
    }
}

export class EndTagToken extends Token {
    public name: NonNullable<string>;
    public readonly attributes: AttributeList;

    public constructor(name: NonNullable<string>, attributes: AttributeList) {
        super(Type.EndTag);

        this.name = name;
        this.attributes = attributes;
    }

    public appendToName(characters: string): void {
        this.name += characters;
    }

    public appendReplacementCharacterToName(): void {
        this.appendToName(REPLACEMENT_CHARACTER);
    }

    public static createEmpty(): EndTagToken {
        return new EndTagToken('', new AttributeList());
    }
}

export class DOCTYPEToken extends Token {
    public name?: string;
    public publicIdentifier?: string;
    public systemIdentifier?: string;
    public forceQuirks?: true;

    public constructor(name?: string, publicIdentifier?: string, systemIdentifier?: string, forceQuirks?: true) {
        super(Type.DOCTYPE);

        this.name = name;
        this.publicIdentifier = publicIdentifier;
        this.systemIdentifier = systemIdentifier;
        this.forceQuirks = forceQuirks;
    }

    public appendToName(characters: string): void {
        VERIFY(this.name !== undefined);

        this.name += characters;
    }

    public appendReplacementCharacterToName(): void {
        this.appendToName(REPLACEMENT_CHARACTER);
    }

    public static createWithForcedQuirks(): DOCTYPEToken {
        return new DOCTYPEToken(undefined, undefined, undefined, true);
    }

    public static createWithName(name: string): DOCTYPEToken {
        return new DOCTYPEToken(name, undefined, undefined, undefined);
    }
}

export function stringify(token: Token): string {
    if (token instanceof CharacterToken) return token.data;
    if (token instanceof CommentToken) return `<!--${token.data}-->`;
    if (token instanceof DOCTYPEToken) return `<!DOCTYPE ${token.name}>`;
    if (token instanceof EndOfFileToken) return 'EOF';
    if (token instanceof EndTagToken) return `</${token.name}>`;
    if (token instanceof StartTagToken) {
        let string = `<${token.name}`;

        for (const attribute of token.attributes.list)
            string += ` ${attribute.name}="${attribute.value}"`;

        // TODO: Implemement selfClosing
        // if (token.selfClosing) return `${string} />`;

        return `${string}>`;
    }

    VERIFY_NOT_REACHED(token.constructor.name);

    return '';
}