Generate llms.txt and llms-full.txt files per section (#2501)

* Generate llms.txt and llms-full.txt files per section

* remove script and add global page

* update dependencies

* add path transformation config

* edit llms.txt and clean up comments

Author: alexandratran

docusaurus.config.js

@@ -317,6 +317,164 @@ const config = {
         containerId: 'GTM-5FGPLC2Q',
       },
     ],
+    [
+      'docusaurus-plugin-llms',
+      {
+        // Set docsDir to site root to collect from all directories
+        docsDir: '.',
+        // Disable default files since we're generating section-specific files
+        generateLLMsTxt: false,
+        generateLLMsFullTxt: false,
+        // Ignore common non-doc directories
+        // Note: src/pages/** is not ignored so tutorials can be collected
+        // Each customLLMFiles entry filters by includePatterns, so only matching files are included
+        ignoreFiles: [
+          'node_modules/**',
+          'build/**',
+          '.docusaurus/**',
+          'static/**',
+          'src/components/**',
+          'src/theme/**',
+          'src/lib/**',
+          'src/config/**',
+          'src/hooks/**',
+          'src/utils/**',
+          'src/plugins/**',
+          'src/specs/**',
+          'src/client/**',
+          'src/scss/**',
+          'i18n/**',
+          '*.config.js',
+          '*.json',
+          '*.lock',
+          'README.md',
+          'CONTRIBUTING.md',
+          'gator_versioned_docs/**',
+        ],
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        // Path transformation to fix URL construction
+        // Since docsDir is '.', we need to remove 'docs/' prefix and handle src/pages paths
+        pathTransformation: {
+          ignorePaths: ['docs', 'src/pages'],
+        },
+        // Generate separate files for each section
+        customLLMFiles: [
+          {
+            filename: 'llms-embedded-wallets.txt',
+            includePatterns: ['embedded-wallets/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'MetaMask Embedded Wallets documentation',
+            description: 'Documentation links for MetaMask Embedded Wallets',
+          },
+          {
+            filename: 'llms-embedded-wallets-full.txt',
+            includePatterns: ['embedded-wallets/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'MetaMask Embedded Wallets documentation',
+            description: 'Complete documentation for MetaMask Embedded Wallets',
+          },
+          {
+            filename: 'llms-sdk.txt',
+            includePatterns: ['sdk/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'MetaMask SDK documentation',
+            description: 'Documentation links for MetaMask SDK',
+          },
+          {
+            filename: 'llms-sdk-full.txt',
+            includePatterns: ['sdk/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'MetaMask SDK documentation',
+            description: 'Complete documentation for MetaMask SDK',
+          },
+          {
+            filename: 'llms-smart-accounts-kit.txt',
+            includePatterns: ['smart-accounts-kit/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'MetaMask Smart Accounts Kit documentation',
+            description: 'Documentation links for MetaMask Smart Accounts Kit',
+          },
+          {
+            filename: 'llms-smart-accounts-kit-full.txt',
+            includePatterns: ['smart-accounts-kit/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'MetaMask Smart Accounts Kit documentation',
+            description: 'Complete documentation for MetaMask Smart Accounts Kit',
+          },
+          {
+            filename: 'llms-snaps.txt',
+            includePatterns: ['snaps/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'Snaps documentation',
+            description: 'Documentation links for Snaps',
+          },
+          {
+            filename: 'llms-snaps-full.txt',
+            includePatterns: ['snaps/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'Snaps documentation',
+            description: 'Complete documentation for Snaps',
+          },
+          {
+            filename: 'llms-wallet.txt',
+            includePatterns: ['wallet/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'Wallet API documentation',
+            description: 'Documentation links for Wallet API',
+          },
+          {
+            filename: 'llms-wallet-full.txt',
+            includePatterns: ['wallet/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'Wallet API documentation',
+            description: 'Complete documentation for Wallet API',
+          },
+          {
+            filename: 'llms-tutorials.txt',
+            includePatterns: ['src/pages/tutorials/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'Tutorials',
+            description: 'Documentation links for MetaMask tutorials',
+          },
+          {
+            filename: 'llms-tutorials-full.txt',
+            includePatterns: ['src/pages/tutorials/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'Tutorials',
+            description: 'Complete documentation for MetaMask tutorials',
+          },
+          {
+            filename: 'llms-dashboard.txt',
+            includePatterns: ['developer-tools/dashboard/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'Developer dashboard documentation',
+            description: 'Documentation links for MetaMask Developer dashboard',
+          },
+          {
+            filename: 'llms-dashboard-full.txt',
+            includePatterns: ['developer-tools/dashboard/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'Developer dashboard documentation',
+            description: 'Complete documentation for MetaMask Developer dashboard',
+          },
+          {
+            filename: 'llms-services.txt',
+            includePatterns: ['services/**/*.{md,mdx}'],
+            fullContent: false,
+            title: 'Services documentation',
+            description: 'Documentation links for MetaMask services',
+          },
+          {
+            filename: 'llms-services-full.txt',
+            includePatterns: ['services/**/*.{md,mdx}'],
+            fullContent: true,
+            title: 'Services documentation',
+            description: 'Complete documentation for MetaMask services',
+          },
+        ],
+      },
+    ],
   ],
   clientModules: [require.resolve('./src/client/scroll-fix.js')],
   themeConfig:

package-lock.json

@@ -38,10 +38,10 @@
     "@docusaurus/remark-plugin-npm2yarn": "3.9.2",
     "@docusaurus/theme-common": "3.9.2",
     "@docusaurus/theme-mermaid": "3.9.2",
-    "@mermaid-js/layout-elk": "^0.1.9",
     "@intercom/messenger-js-sdk": "^0.0.14",
     "@lottiefiles/react-lottie-player": "^3.6.0",
     "@mdx-js/react": "^3.1.0",
+    "@mermaid-js/layout-elk": "^0.1.9",
     "@metamask/design-tokens": "^7.1.0",
     "@metamask/eth-sig-util": "^7.0.3",
     "@metamask/profile-sync-controller": "^16.0.0",
@@ -82,7 +82,8 @@
     "remark-codesandbox": "^0.10.1",
     "remark-docusaurus-tabs": "^0.2.0",
     "remark-math": "^6.0.0",
-    "sass": "^1.93.0"
+    "sass": "^1.93.0",
+    "docusaurus-plugin-llms": "^0.2.2"
   },
   "devDependencies": {
     "@docusaurus/eslint-plugin": "3.9.2",