feat: add workflow and script to check edit links on docs

.github/workflows/check-edit-links.yml

@@ -0,0 +1,49 @@
+name: Weekly Link Checker
+
+on:
+  schedule:
+    - cron: '0 0 * * 0' # Runs every week at midnight on Sunday
+  workflow_dispatch:
+
+jobs:
+  check-links:
+    name: Run Link Checker and Notify Slack
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run link checker
+        id: linkcheck
+        run: |
+          npm run test:editlinks | tee output.log
+
+      - name: Extract 404 URLs from output
+        id: extract-404
+        run: |
+          ERRORS=$(sed -n '/URLs returning 404:/,$p' output.log)
+          echo "errors<<EOF" >> $GITHUB_OUTPUT
+          echo "$ERRORS" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Notify Slack
+        if: ${{ steps.extract-404.outputs.errors != '' }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.WEBSITE_SLACK_WEBHOOK }}
+          SLACK_TITLE: 'Edit Links Checker Errors Report'
+          SLACK_MESSAGE: |
+            🚨 The following URLs returned 404 during the link check:
+            ```
+            ${{ steps.extract-404.outputs.errors }}
+            ```
+          MSG_MINIMAL: true

components/layout/DocsLayout.tsx

@@ -33,6 +33,10 @@ interface IDocsLayoutProps {
  */
 function generateEditLink(post: IPost) {
   let last = post.id.substring(post.id.lastIndexOf('/') + 1);
+
+  if (last.endsWith('.mdx')) {
+    last = last.replace('.mdx', '.md');
+  }
   const target = editOptions.find((edit) => {
     return post.slug.includes(edit.value);
   });

config/edit-page-config.json

@@ -1,7 +1,7 @@
 [
   {
     "value": "/tools/generator",
-    "href": "https://github.com/asyncapi/generator/tree/master/docs"
+    "href": "https://github.com/asyncapi/generator/tree/master/apps/generator/docs"
   },
   {
     "value": "reference/specification/",
@@ -19,4 +19,4 @@
     "value": "reference/extensions/",
     "href": "https://github.com/asyncapi/extensions-catalog/tree/master/extensions"
   }
-]
+]

jest.config.js

@@ -4,7 +4,12 @@ module.exports = {
   coverageReporters: ['text', 'lcov', 'json-summary'],
   coverageDirectory: 'coverage',
   collectCoverageFrom: ['scripts/**/*.js'],
-  coveragePathIgnorePatterns: ['scripts/compose.js', 'scripts/tools/categorylist.js', 'scripts/tools/tags-color.js'],
+  coveragePathIgnorePatterns: [
+    'scripts/compose.js',
+    'scripts/tools/categorylist.js',
+    'scripts/tools/tags-color.js',
+    'scripts/markdown/check-editlinks.js'
+  ],
   // To disallow netlify edge function tests from running
-  testMatch: ['**/tests/**/*.test.*', '!**/netlify/**/*.test.*'],
-};
+  testMatch: ['**/tests/**/*.test.*', '!**/netlify/**/*.test.*']
+};

package.json

@@ -24,6 +24,7 @@
     "generate:tools": "node scripts/build-tools.js",
     "test:netlify": "deno test --allow-env --trace-ops netlify/**/*.test.ts",
     "test:md": "node scripts/markdown/check-markdown.js",
+    "test:editlinks": "node scripts/markdown/check-editlinks.js",
     "dev:storybook": "storybook dev -p 6006",
     "build:storybook": "storybook build"
   },

scripts/markdown/check-editlinks.js

@@ -0,0 +1,170 @@
+const fs = require('fs').promises;
+const path = require('path');
+const fetch = require('node-fetch-2');
+const editUrls = require('../../config/edit-page-config.json');
+
+const ignoreFiles = [
+  'reference/specification/v2.x.md',
+  'reference/specification/v3.0.0-explorer.md',
+  'reference/specification/v3.0.0.md'
+];
+
+/**
+ * Introduces a delay in the execution flow
+ * @param {number} ms - The number of milliseconds to pause
+ */
+async function pause(ms) {
+  return new Promise((res) => {
+    setTimeout(res, ms);
+  });
+}
+
+/**
+ * Process a batch of URLs to check for 404s
+ * @param {object[]} batch - Array of path objects to check
+ * @returns {Promise<string[]>} Array of URLs that returned 404
+ */
+async function processBatch(batch) {
+  return Promise.all(
+    batch.map(async ({ filePath, urlPath, editLink }) => {
+      try {
+        if (!editLink || ignoreFiles.some((ignorePath) => filePath.endsWith(ignorePath))) return null;
+
+        const response = await fetch(editLink, { method: 'HEAD' });
+        if (response.status === 404) {
+          return { filePath, urlPath, editLink };
+        }
+        return null;
+      } catch (error) {
+        console.error(`Error checking ${editLink}:`, error.message);
+        return editLink;
+      }
+    })
+  );
+}
+
+/**
+ * Check all URLs in batches
+ * @param {object[]} paths - Array of all path objects to check
+ * @returns {Promise<string[]>} Array of URLs that returned 404
+ */
+async function checkUrls(paths) {
+  const result = [];
+  const batchSize = 5;
+
+  for (let i = 0; i < paths.length; i += batchSize) {
+    console.log(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(paths.length / batchSize)}`);
+    const batch = paths.slice(i, i + batchSize);
+    const batchResults = await processBatch(batch);
+    await pause(1000);
+
+    // Filter out null results and add valid URLs to results
+    result.push(...batchResults.filter((url) => url !== null));
+  }
+
+  return result;
+}
+
+/**
+ * Determines the appropriate edit link based on the URL path and file path
+ * @param {string} urlPath - The URL path to generate an edit link for
+ * @param {string} filePath - The actual file path
+ * @param {object[]} editOptions - Array of edit link options
+ * @returns {string|null} The generated edit link or null if no match
+ */
+function determineEditLink(urlPath, filePath, editOptions) {
+  // Remove leading 'docs/' if present for matching
+  const pathForMatching = urlPath.startsWith('docs/') ? urlPath.slice(5) : urlPath;
+
+  const target =
+    editOptions.find((edit) => pathForMatching.includes(edit.value)) || editOptions.find((edit) => edit.value === '');
+
+  if (!target) return null;
+
+  // Handle the empty value case (fallback)
+  if (target.value === '') {
+    return `${target.href}/docs/${urlPath}.md`;
+  }
+
+  // For other cases with specific targets
+  return `${target.href}/${path.basename(filePath)}`;
+}
+
+/**
+ * Recursively processes markdown files in a directory to generate paths and edit links
+ * @param {string} folderPath - The path to the folder to process
+ * @param {object[]} editOptions - Array of edit link options
+ * @param {string} [relativePath=''] - The relative path for URL generation
+ * @param {object[]} [result=[]] - Accumulator for results
+ * @returns {Promise<object[]>} Array of objects containing file paths and edit links
+ */
+async function generatePaths(folderPath, editOptions, relativePath = '', result = []) {
+  try {
+    const files = await fs.readdir(folderPath);
+
+    await Promise.all(
+      files.map(async (file) => {
+        const filePath = path.join(folderPath, file);
+        const relativeFilePath = path.join(relativePath, file);
+
+        // Skip _section.md files
+        if (file === '_section.md') {
+          return;
+        }
+
+        const stats = await fs.stat(filePath);
+
+        if (stats.isDirectory()) {
+          // Process directory
+          await generatePaths(filePath, editOptions, relativeFilePath, result);
+        } else if (stats.isFile() && file.endsWith('.md')) {
+          // Process all markdown files (including index.md)
+          const urlPath = relativeFilePath.split(path.sep).join('/').replace('.md', '');
+          result.push({
+            filePath,
+            urlPath,
+            editLink: determineEditLink(urlPath, filePath, editOptions)
+          });
+        }
+      })
+    );
+
+    return result;
+  } catch (err) {
+    console.error(`Error processing directory ${folderPath}:`, err);
+    throw err;
+  }
+}
+
+async function main() {
+  const editOptions = editUrls;
+
+  try {
+    const docsFolderPath = path.resolve(__dirname, '../../markdown/docs');
+    const paths = await generatePaths(docsFolderPath, editOptions);
+    console.log('Starting URL checks...');
+    const invalidUrls = await checkUrls(paths);
+
+    if (invalidUrls.length === 0) {
+      console.log('All URLs are valid.');
+      process.exit(0);
+    }
+
+    console.log('\nURLs returning 404:\n');
+    invalidUrls.forEach((url) => console.log(`- ${url.editLink} generated from ${url.filePath}\n`));
+    console.log(`\nTotal invalid URLs found: ${invalidUrls.length}`);
+
+    if (invalidUrls.length > 0) {
+      process.exit(1);
+    }
+  } catch (error) {
+    console.error('Failed to check edit links:', error);
+    process.exit(1);
+  }
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = { generatePaths, determineEditLink, main };