Initial release of Web Tools 0.0.5

tests/syntax.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'vitest'
+import {
+  MAX_HIGHLIGHTED_SYNTAX_LENGTH,
+  MAX_HIGHLIGHTED_SYNTAX_TOKENS,
+  tokenizeJson,
+  tokenizeSyntaxForDisplay,
+  tokenizeXml,
+} from '../src/lib/syntax'
+
+describe('syntax tokenizers', () => {
+  it('classifies JSON tokens without changing content', () => {
+    const source = '{\n  "name": "Web Tools",\n  "count": 3,\n  "local": true,\n  "value": null\n}'
+    const tokens = tokenizeJson(source)
+    expect(tokens.map((token) => token.text).join('')).toBe(source)
+    expect(tokens.some((token) => token.type === 'key' && token.text === '"name"')).toBe(true)
+    expect(tokens.some((token) => token.type === 'string' && token.text === '"Web Tools"')).toBe(true)
+    expect(tokens.some((token) => token.type === 'number' && token.text === '3')).toBe(true)
+    expect(tokens.some((token) => token.type === 'literal' && token.text === 'true')).toBe(true)
+  })
+
+  it('classifies XML tags, attributes, values and comments without changing content', () => {
+    const source = '<?xml version="1.0"?>\n<!-- tools -->\n<tools><tool id="hex">ASCII</tool></tools>'
+    const tokens = tokenizeXml(source)
+    expect(tokens.map((token) => token.text).join('')).toBe(source)
+    expect(tokens.some((token) => token.type === 'tag' && token.text === 'tool')).toBe(true)
+    expect(tokens.some((token) => token.type === 'attribute' && token.text === 'id')).toBe(true)
+    expect(tokens.some((token) => token.type === 'value' && token.text === '"hex"')).toBe(true)
+    expect(tokens.some((token) => token.type === 'comment')).toBe(true)
+    expect(tokens.some((token) => token.type === 'meta')).toBe(true)
+  })
+
+  it('uses plain text for results that would create excessive DOM nodes', () => {
+    expect(tokenizeSyntaxForDisplay('{"ok":true}', 'json')).toBeDefined()
+    expect(tokenizeSyntaxForDisplay('x'.repeat(MAX_HIGHLIGHTED_SYNTAX_LENGTH + 1), 'json')).toBeUndefined()
+    const tokenHeavy = Array.from({ length: Math.floor(MAX_HIGHLIGHTED_SYNTAX_TOKENS / 2) + 1 }, () => '<a/>').join('')
+    expect(tokenHeavy.length).toBeLessThan(MAX_HIGHLIGHTED_SYNTAX_LENGTH)
+    expect(tokenizeSyntaxForDisplay(tokenHeavy, 'xml')).toBeUndefined()
+  })
+})