feat: config.tokenizer

Introduces a way to extend/customize the tokenizer and its provided methods (setup, tokenize).

- Abstract prism tokenizer
- Adds package export `syntax-highlight-element/tokenizer/prism`
- Export  `setupTokenHighlights` method
- Flat merge config with defaultConfig

Author: andreruffert

index.html

@@ -32,12 +32,17 @@
     <script>
       window.she = window.she || {};
       window.she.config = {
-        languages: ['html', 'css', 'js', 'jsx', 'md'], // Default
+        languages: ['html', 'css', 'js', 'jsx', 'md'],
         // Optional: language specific token type overwrites
         languageTokens: {
           css: ['important'],
           md: ['title', 'list'],
-        }
+        },
+        // Optional: extend/customize tokenizer
+        // tokenizer: {
+        //   async setup() {},
+        //   tokenize: () => [],
+        // }
       }
     </script>
   </head>
@@ -87,15 +92,55 @@
 </syntax-highlight>
 
     <script type="module">
-      // Manually loading prism dependencies
-      // import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-core.min.js'
+      // /**
+      //  * Option 1. autoload - default
+      //  */
+      // import '/src/index.js'
+      // import SyntaxHighlightElement from '/src/index.js'
+
+      /**
+       * Option 2. Opt out of prism autoload by manually importing prism core
+       */
+      import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-core.min.js'
+      import SyntaxHighlightElement from '/src/index.js?define=false'
+
+      /**
+       * Option 2.a. Load lang deps manually (without dependency resolution)
+       */
+      // // - CDN
       // import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-markup.min.js'
       // import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-css.min.js'
       // import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-clike.min.js'
       // import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-javascript.min.js'
+      // import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-jsx.min.js'
       // import 'https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-markdown.min.js'
+      // // - npm
+      // import 'prismjs/components/prism-markup'
+      // import 'prismjs/components/prism-css'
+      // import 'prismjs/components/prism-clike'
+      // import 'prismjs/components/prism-javascript'
+      // import 'prismjs/components/prism-jsx'
+      // import 'prismjs/components/prism-markdown'
 
-      import '/src/index.js'
+      /**
+       * Option 2.b. resolve language dependencies and load from URL/CDN
+       */
+      // - CDN
+      import { loadPrismLanguage } from '/src/tokenizer/prism.js'
+      await loadPrismLanguage({
+        baseUrl: 'https://unpkg.com/prismjs@1.30.0',
+        language: window.she.config.languages
+      });
+      // // - npm
+      // import { resolveLanguageDependencies } from '/src/tokenizer/prism.js'
+      // const langDeps = resolveLanguageDependencies(window.she.config.languages);
+      // for await (const lang of langDeps) {
+      //   const importPath = `prismjs/components/prism-${lang}`
+      //   await import(importPath);
+      //   console.log({importPath})
+      // }
+      // // - Manually define SyntaxHighlightElement with ?define=false once all dependecies are loaded
+      window.SyntaxHighlightElement = SyntaxHighlightElement.define();
     </script>
     <script>
       document.addEventListener('DOMContentLoaded', () => {

package.json

@@ -14,15 +14,16 @@
     ".": {
       "import": "./dist/syntax-highlight-element.js"
     },
-    "./themes/*": "./dist/themes/*"
+    "./themes/*": "./dist/themes/*",
+    "./tokenizer/*": "./dist/tokenizer/*"
   },
   "files": [
     "dist"
   ],
   "scripts": {
     "dev": "vite",
     "dev:docs": "vite --config vite.docs.config.js",
-    "build": "vite build && vite build --config vite.themes.config.js",
+    "build": "vite build && vite build --config vite.themes.config.js && vite build --config vite.tokenizer.config.js",
     "preview": "vite preview",
     "format": "biome check --write",
     "lint": "biome ci",

src/constants.js

@@ -1,11 +1,21 @@
+import { tokenizer as defaultTokenizer } from './tokenizer/prism';
+
 export const NAMESPACE = 'she';
 
 /**
  * @typedef Config
  * @type {object}
- * @property {string[]} languages - Prism languages.
+ * @property {string[]} languages - Languages.
  * @property {{ [key: string]: string[] }} languageTokens - Language specific token types.
+ * @property {object} tokenizer - Tokenizer.
  */
 
 /** @type {Config} */
-export const CONFIG = window[NAMESPACE]?.config || {};
+export const CONFIG = Object.assign(
+  {
+    languages: ['markup', 'css', 'javascript'],
+    languageTokens: {},
+    tokenizer: Object.assign(defaultTokenizer, window[NAMESPACE]?.config?.tokenizer || {}),
+  },
+  window[NAMESPACE]?.config || {},
+);

src/index.js

@@ -1,7 +1,8 @@
 import { NAMESPACE } from './constants';
 import { SyntaxHighlightElement } from './syntax-highlight-element';
+import { setupTokenHighlights } from './utils';
 
-export { SyntaxHighlightElement };
+export { SyntaxHighlightElement, setupTokenHighlights };
 export default SyntaxHighlightElement;
 
 window[NAMESPACE] = window[NAMESPACE] || {};

src/syntax-highlight-element.js

@@ -1,18 +1,14 @@
 import { CONFIG } from './constants';
-import { setupTokenHighlights, setupTokenizer, tokenize } from './utils';
-
-const DEFAULT_TAG_NAME = 'syntax-highlight';
 
 export class SyntaxHighlightElement extends HTMLElement {
-  static async define(tagName = DEFAULT_TAG_NAME, registry = customElements) {
+  static async define(tagName = 'syntax-highlight', registry = customElements) {
     if (!CSS.highlights) {
       console.info('The CSS Custom Highlight API is not supported in this browser.');
       return;
     }
 
     if (!registry.get(tagName)) {
-      await setupTokenizer();
-      setupTokenHighlights(CONFIG?.languageTokens || {});
+      CONFIG.tokenizer?.setup && (await CONFIG.tokenizer.setup());
       registry.define(tagName, SyntaxHighlightElement);
       return SyntaxHighlightElement;
     }
@@ -58,7 +54,7 @@ export class SyntaxHighlightElement extends HTMLElement {
    */
   paintTokenHighlights() {
     // Tokenize the text
-    const tokens = tokenize(this.contentElement.innerText, this.language);
+    const tokens = CONFIG.tokenizer?.tokenize(this.contentElement.innerText, this.language) || [];
     const languageTokenTypes = CONFIG.languageTokens?.[this.language] || [];
 
     // Paint highlights