docs: add linting and code snippet validation (#1983)

* Add linting and code snippet validation for docs

* Remove unused link checkers

* Add link checking to CONTRIBUTING.md and remove package-lock.json

* update pnpm lockfile to sync with package.json

* feedback from Benjie

* build docs locally to test broken links

* fix failing build

* fix another snippets ignore

---------

Co-authored-by: Saihajpreet Singh <saihajpreet.singh@gmail.com>

Author: sarahxsanders

.eslintrc.cjs

@@ -1,18 +1,14 @@
 /* eslint-env node */
 
 const CODE_EXT = "js,jsx,cjs,mjs,ts,tsx,cts,mts"
-
 const MARKDOWN_EXT = "md,mdx"
 
 module.exports = {
   root: true,
+  plugins: ["@graphql-eslint", "mdx", "@typescript-eslint", "tailwindcss"],
   overrides: [
     {
       files: [`**/*.{${CODE_EXT}}`],
-      // TODO: extract graphql documents from code files
-      // to lint graphql documents marked with /* GraphQL */ comments inside js/ts codeblocks in markdown
-      // processor: '@graphql-eslint/graphql',
-      // plugins: ['@graphql-eslint'],
       extends: [
         "eslint:recommended",
         "plugin:@typescript-eslint/recommended",
@@ -34,7 +30,6 @@ module.exports = {
           },
         ],
         "prefer-const": ["error", { destructuring: "all" }],
-        // TODO: fix below
         "prefer-rest-params": "off",
         "@typescript-eslint/no-explicit-any": "off",
         "@typescript-eslint/no-unused-vars": "off",
@@ -51,27 +46,45 @@ module.exports = {
     {
       files: [`**/*.{${MARKDOWN_EXT}}`],
       parser: "eslint-mdx",
+      extends: ["plugin:mdx/recommended"],
       processor: "mdx/remark",
-      plugins: ["mdx"],
       parserOptions: {
         ecmaVersion: 13,
         sourceType: "module",
       },
       settings: {
         "mdx/code-blocks": true,
+        "mdx/language-mapper": {
+          js: "espree",
+          graphql: "@graphql-eslint/parser",
+          ts: "@typescript-eslint/parser",
+          typescript: "@typescript-eslint/parser",
+        },
       },
       rules: {
         "mdx/remark": "error",
       },
     },
     {
-      files: [`**/*.{${MARKDOWN_EXT}}/*.{${CODE_EXT}}`],
+      files: ["**/*.graphql"],
+      parser: "@graphql-eslint/parser",
       rules: {
-        "no-unused-labels": "off",
-        "no-undef": "off",
-        "no-redeclare": "off",
-        "no-import-assign": "off",
-        "no-prototype-builtins": "off",
+        "@graphql-eslint/no-syntax-errors": "error",
+        "@graphql-eslint/unique-operation-name": "error",
+        "@graphql-eslint/unique-fragment-name": "error",
+        "@graphql-eslint/no-anonymous-operations": "warn",
+        "@graphql-eslint/lone-anonymous-operation": "error",
+        "@graphql-eslint/no-duplicate-fields": "error",
+        "@graphql-eslint/no-unused-fragments": "warn",
+        "@graphql-eslint/no-duplicate-fragment-names": "error",
+        "@graphql-eslint/no-undefined-variables": "error",
+        "@graphql-eslint/unique-variable-names": "error",
+      },
+    },
+    {
+      files: [`**/*.{${CODE_EXT}}`, `**/*.{${MARKDOWN_EXT}}`],
+      parserOptions: {
+        plugins: ["graphql"],
       },
     },
     {
@@ -84,9 +97,5 @@ module.exports = {
         "mdx/remark": "off",
       },
     },
-    {
-      files: ["**/*.graphql"],
-      parser: "@graphql-eslint/eslint-plugin",
-    },
   ],
 }

.github/workflows/docs-validation.yml

@@ -0,0 +1,64 @@
+# Docs validation workflow runs on each PR on main w/ broken link checker
+# and code snippet validation
+
+name: Docs validation
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+link-check:
+  name: Broken link checker
+  runs-on: ubuntu-latest
+
+  steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: "20"
+        cache: "pnpm"
+
+    - name: Install dependencies
+      run: pnpm install --frozen-lockfile
+
+    - name: Build static site
+      run: pnpm build
+
+    - name: Set up Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+
+    - name: Install lychee
+      run: cargo install lychee
+
+    - name: Check links
+      run: lychee --verbose --no-progress './out/**/*.html'
+
+code-validate:
+  name: Code snippet and GraphQL validation
+  runs-on: ubuntu-latest
+  steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-node@v4
+      with:
+        node-version: "20"
+        cache: "pnpm"
+
+    - name: Install dependencies
+      run: pnpm install --frozen-lockfile
+
+    - name: Run validation w/ annotations
+      run: pnpm lint:docs:ci
+
+    - name: Validate code snippets
+      run: pnpm validate:snippets

CONTRIBUTING.md

@@ -10,6 +10,7 @@ There are many ways to get involved. Follow this guide and feel free to [reach o
 
 - [Development guide](#development-guide)
   - [Running the site locally](#running-the-site-locally)
+  - [Checking for broken links](#checking-for-broken-links)
   - [Branching](#branching)
   - [Project structure](#project-structure)
   - [Publishing the updated site](#publishing-the-updated-site)
@@ -53,6 +54,21 @@ Finally, open http://localhost:3000 to view it in the browser.
 
 The GraphQL website is built with [Nextra](https://nextra.site). This means that a hot-reloading development environment will be accessible by default.
 
+### Checking for broken links
+
+We use [Lychee](https://github.com/lycheeverse/lychee), a Rust-based CLI tool, to check for broken links in our documentation.
+
+To install Lychee locally:
+
+1. Install Rust: https://www.rust-lang.org/tools/install
+2. After installing Rust, run:
+
+```bash
+cargo install lychee
+```
+
+With Rust and Lychee installed, run the link checker: `pnpm run check:links`.
+
 ### Branching
 
 Active development for graphql.org happens on the `source` branch. Be sure to create any new branches or direct any pull requests back to `source`.

package.json

@@ -6,14 +6,18 @@
   "packageManager": "pnpm@9.15.2",
   "scripts": {
     "build": "next build && next-image-export-optimizer",
+    "check:links": "lychee --verbose --no-progress './src/pages/**/*.mdx' --base https://graphql.org",
     "dev": "next",
     "format": "pnpm format:check --write",
     "format:check": "prettier --cache --check .",
     "lint": "eslint --ignore-path .gitignore .",
+    "lint:docs": "eslint --ignore-path .gitignore src/pages/learn --format stylish",
+    "lint:docs:ci": "eslint --ignore-path .gitignore src/pages/learn --format eslint-formatter-github",
     "postbuild": "next-sitemap",
     "prebuild": "tsx src/get-github-info.ts",
     "start": "next start",
-    "test": "echo \"no tests\" && exit 1"
+    "test": "echo \"no tests\" && exit 1",
+    "validate:snippets": "node scripts/validate-snippets.js"
   },
   "dependencies": {
     "@graphql-tools/schema": "10.0.15",
@@ -23,7 +27,7 @@
     "@tailwindcss/typography": "^0.5.10",
     "autoprefixer": "^10.4.17",
     "clsx": "^2.1.0",
-    "codemirror": "5.65.1",
+    "codemirror": "^5.65.19",
     "codemirror-graphql": "1.3.2",
     "date-fns": "^2.30.0",
     "fast-glob": "^3.3.2",

pnpm-lock.yaml

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+
+import fs from "node:fs"
+import path from "node:path"
+import glob from "glob"
+import { parse } from "graphql"
+import chalk from "chalk"
+import { fileURLToPath } from "node:url"
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+const projectRoot = path.resolve(__dirname, "../")
+
+const MDX_GLOB = "./src/pages/learn/**/*.mdx"
+const CODE_BLOCK_REGEX = /^(`{3,})(\w+)\s*\n([\s\S]*?)\r?\n\1$/gm
+const IGNORE_COMMENT = "snippet-ignore"
+
+let totalFiles = 0
+let totalSnippets = 0
+let totalErrors = 0
+
+// TODO: Add JS linting after JS code snippet modernization
+// async function lintJavaScript(code, filePath) {
+//   const eslint = new ESLint({
+//     useEslintrc: true,
+//     baseConfig: {
+//       parserOptions: {
+//         ecmaVersion: "latest",
+//         sourceType: "module",
+//       },
+//     },
+//   })
+
+//   let preparedCode = code.trim()
+
+//   if (preparedCode.startsWith("function")) {
+//     preparedCode = "/* eslint-disable no-unused-vars */\n" + preparedCode
+//   }
+
+//   const results = await eslint.lintText(preparedCode, { filePath })
+//   return results.flatMap(result => result.messages)
+// }
+
+function validateGraphQL(code) {
+  try {
+    parse(code)
+    return []
+  } catch (error) {
+    return [{ message: error.message }]
+  }
+}
+
+function extractSnippets(content, filePath) {
+  const snippets = []
+  let match
+
+  while ((match = CODE_BLOCK_REGEX.exec(content)) !== null) {
+    const [fullMatch, openingBackticks, lang, code] = match
+    const beforeBlock = content.slice(0, match.index)
+    const lineNumber = beforeBlock.split(/\r?\n/).length
+
+    if (beforeBlock.includes(IGNORE_COMMENT)) {
+      continue
+    }
+
+    snippets.push({ lang, code, lineNumber, filePath })
+  }
+
+  return snippets
+}
+
+async function validateSnippet(snippet) {
+  const { lang, code, lineNumber, filePath } = snippet
+
+  if (!code.trim()) return []
+
+  // TODO: Add section after JS code snippet modernization
+  // if (["js", "javascript", "ts", "typescript"].includes(lang)) {
+  //   const messages = await lintJavaScript(code, filePath)
+  //   return messages.map(msg => ({
+  //     type: "JS/TS",
+  //     file: filePath,
+  //     line: lineNumber + (msg.line || 1),
+  //     message: msg.message,
+  //   }))
+  // }
+
+  if (lang === "graphql") {
+    const messages = validateGraphQL(code)
+    return messages.map(msg => ({
+      type: "GraphQL",
+      file: filePath,
+      line: lineNumber + (msg.line || 1),
+      message: msg.message,
+    }))
+  }
+
+  return []
+}
+
+async function main() {
+  console.log(`Validating code snippets in: ${projectRoot}/${MDX_GLOB}`)
+
+  const files = glob.sync(MDX_GLOB, { cwd: projectRoot })
+  totalFiles = files.length
+
+  if (totalFiles === 0) {
+    console.log(chalk.green("No MDX files found to validate."))
+    return
+  }
+
+  const errors = []
+
+  for (const file of files) {
+    const content = fs.readFileSync(file, "utf8")
+    const snippets = extractSnippets(content, file)
+    totalSnippets += snippets.length
+
+    for (const snippet of snippets) {
+      const snippetErrors = await validateSnippet(snippet)
+      errors.push(...snippetErrors)
+    }
+  }
+
+  totalErrors = errors.length
+
+  if (totalErrors > 0) {
+    errors.forEach(err => {
+      const errorMessage = `${err.type} Error in ${err.file} at line ${err.line}: ${err.message}`
+      console.error(chalk.red(errorMessage))
+
+      if (process.env.GITHUB_ACTIONS) {
+        console.log(`::error file=${err.file},line=${err.line}::${err.message}`)
+      }
+    })
+
+    console.error(
+      chalk.red("\nCode snippet validation failed. Check error logs."),
+    )
+    console.error(`Files checked: ${totalFiles}`)
+    console.error(`Snippets checked: ${totalSnippets}`)
+    console.error(`Errors found: ${totalErrors}`)
+    process.exit(1)
+  } else {
+    console.log(
+      chalk.green(
+        "\nCode snippet validation passed. All code snippets are valid.",
+      ),
+    )
+    console.log(`Files checked: ${totalFiles}`)
+    console.log(`Snippets checked: ${totalSnippets}`)
+  }
+}
+
+main().catch(err => {
+  console.error(err)
+  process.exit(1)
+})

scripts/validate-snippets.js

@@ -41,6 +41,7 @@ In addition to the `data` key, the GraphQL specification outlines how [errors](h
 
 Request errors typically occur because the client made a mistake. For example, there may be a _syntax error_ in the document, such as a missing bracket or the use of an unknown root operation type keyword:
 
+{/* <!-- snippet-ignore --> */}
 ```graphql
 # { "graphiql": true }
 operation { 

src/pages/learn/response.mdx

@@ -210,6 +210,7 @@ type Character {
 
 As we see above, the Non-Null and List modifiers can be combined. For example, you can have a List of Non-Null `String` types:
 
+{/* <!-- snippet-ignore --> */}
 ```graphql
 myField: [String!]
 ```
@@ -225,6 +226,7 @@ myField: ["a", null, "b"] // error
 
 Now, let's say we defined a Non-Null List of `String` types:
 
+{/* <!-- snippet-ignore --> */}
 ```graphql
 myField: [String]!
 ```
@@ -240,6 +242,7 @@ myField: ["a", null, "b"] // valid
 
 Lastly, you can also have a Non-Null List of Non-Null `String` types:
 
+{/* <!-- snippet-ignore --> */}
 ```graphql
 myField: [String!]!
 ```