From fbaae88b652dc2173b4f7382e87fa2a3770e119e Mon Sep 17 00:00:00 2001
From: networkException <git@nwex.de>
Date: Thu, 6 Jan 2022 01:58:13 +0100
Subject: [PATCH] HTML: Add fixme to tokenizer about tree construction stage
 state changes

---
 src/html.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/html.ts b/src/html.ts
index 53c8032..6814828 100644
--- a/src/html.ts
+++ b/src/html.ts
@@ -12,6 +12,9 @@ export function tokenize(input: string): Array<Token> {
 
     const tokenizer = new Tokenizer(input);
 
+    // FIXME: This completely ignores any state changes set by the tree construction
+    //        stage - as well, there is no tree construction stage.
+    // See https://html.spec.whatwg.org/multipage/parsing.html#tokenization
     while (tokenizer.tokens[tokenizer.tokens.length - 1]?.type !== Type.EndOfFile)
         tokenizer.spin();