35 lines
968 B
TypeScript
35 lines
968 B
TypeScript
|
import { Highlighter } from "./html/highlighter.js";
|
||
|
|
import { Node } from "./html/highlighter/node.js";
|
||
|
|
import { Tokenizer } from "./html/tokenizer.js";
|
||
|
|
import { Token, Type } from "./html/tokenizer/token.js";
|
||
|
|
|
||
|
|
export function normalizeNewlines(input: string): string {
|
||
|
|
return input.replaceAll('\u000D\u000A', '\u000A').replaceAll('\u000D', '\u000A');
|
||
|
|
}
|
||
|
|
|
||
|
|
export function tokenize(input: string): Array<Token> {
|
||
|
|
console.time('html tokenizer');
|
||
|
|
|
||
|
|
const tokenizer = new Tokenizer(input);
|
||
|
|
|
||
|
|
while (tokenizer.tokens[tokenizer.tokens.length - 1]?.type !== Type.EndOfFile)
|
||
|
|
tokenizer.spin();
|
||
|
|
|
||
|
|
console.timeEnd('html tokenizer');
|
||
|
|
|
||
|
|
return tokenizer.tokens;
|
||
|
|
}
|
||
|
|
|
||
|
|
export function highlight(tokens: Array<Token>): Array<Node> {
|
||
|
|
const highlighter = new Highlighter(tokens);
|
||
|
|
|
||
|
|
console.time('html highlighter');
|
||
|
|
|
||
|
|
while (!highlighter.finished)
|
||
|
|
highlighter.spin();
|
||
|
|
|
||
|
|
console.timeEnd('html highlighter');
|
||
|
|
|
||
|
|
return highlighter.nodes;
|
||
|
|
}
|