Merge pull request #6953 from libgit2/ethomson/docs-search

Add search functionality to our docs generator

.github/workflows/documentation.yml

@@ -6,6 +6,11 @@ on:
     branches: [ main, maint/* ]
   release:
   workflow_dispatch:
+    inputs:
+      force:
+        description: 'Force rebuild'
+        type: boolean
+        required: true
 
 concurrency:
   group: documentation
@@ -42,8 +47,14 @@ jobs:
       working-directory: source
     - name: Generate documentation
       run: |
+        args=""
+
+        if [ "${{ inputs.force }}" = "true" ]; then
+          args="--force"
+        fi
+
         npm install
-        ./generate ../.. ../../../www/docs
+        ./generate $args ../.. ../../../www/docs
       working-directory: source/script/api-docs
     - name: Examine changes
       run: |

script/api-docs/docs-generator.js

@@ -96,6 +96,7 @@ function produceHeader(version, api, type) {
     content += `          <h2 class="apiName ${type}Name">${api.name}</h2>\n`;
 
     content += produceAttributes(version, api, type);
+    content += produceSearchArea(version, type);
 
     content += produceVersionPicker(version,
         `apiHeaderVersionSelect ${type}HeaderVersionSelect`,
@@ -553,6 +554,23 @@ async function layout(data) {
     return layout.toString().replaceAll(/{{([a-z]+)}}/g, (match, p1) => data[p1] || "");
 }
 
+function produceSearchArea(version, type) {
+    let content = "";
+
+    content += `\n`;
+    content += `           <script src="/js/minisearch.js"></script>\n`;
+    content += `           <script src="/js/search.js"></script>\n`;
+
+    content += `           <div class="headerSearchArea ${type}HeaderSearchArea" id="headersearcharea">\n`;
+    content += `             <input class="headerSearchBox ${type}HeaderSearchBox" id="headersearchbox" placeholder="Search..." onInput="handleSearchSuggest({ version: '${version}' })" onFocusIn="handleSearchSuggest({ version: '${version}' })" onKeyUp="if (event.code === 'Enter') { submitSearch({ version: '${version}' }); }"/>\n`;
+    content += `             <div class="headerSearchResults ${type}HeaderSearchResults" id="headersearchresults">\n`;
+    content += `             </div>\n`;
+    content += `           </div>\n`;
+    content += `\n`;
+
+    return content;
+}
+
 async function produceDocumentationForApi(version, api, type) {
     let content = "";
 
@@ -834,6 +852,7 @@ async function produceIndexForGroup(version, group, versionApis) {
     content += `      <div class="groupHeader">\n`;
     content += `        <h2 class="groupName">${groupName}</h2>\n`;
 
+    content += produceSearchArea(version, 'group');
     content += produceVersionPicker(version, "groupHeaderVersionSelect", (v) => {
         if (apiData[v]['groups'][group]) {
             return `${linkPrefix}/${v}/${groupName}/index.html`;
@@ -963,6 +982,7 @@ function versionIndexContent(version, apiData) {
     content += `        <div class="versionHeader">\n`;
     content += `          <h2 class="versionName">${projectTitle} ${version}</h2>\n`;
 
+    content += produceSearchArea(version, 'version');
     content += produceVersionPicker(version, "versionHeaderVersionSelect",
         (v) => `${linkPrefix}/${v}/index.html`);
 
@@ -1187,6 +1207,62 @@ function calculateVersionDeltas(apiData) {
     }
 }
 
+async function produceSearch(versions) {
+    if (options.verbose) {
+        console.log(`Producing search page...`);
+    }
+
+    let content = "";
+
+    content += `<script src="/js/minisearch.js"></script>\n`;
+    content += `<script src="/js/search.js"></script>\n`;
+    content += `<script src="/js/markdown-it.js"></script>\n`;
+    content += `<script>\n`;
+    content += `  const markdown = window.markdownit();\n`;
+    content += `</script>\n`;
+
+    content += `\n`;
+
+    content += `      <div class="search">\n`;
+    content += `        <div class="searchHeader">\n`;
+    content += `          <h2 class="searchName">libgit2 search</h2>\n`;
+    content += `          <div class="searchHeaderVersionSelect">\n`;
+    content += `            <span>Version:</span>\n`;
+    content += `            <select id="searchversion" onChange="setSearchVersion(this.value)">\n`;
+
+    for (const version of versions) {
+        content += `              <option value="${version}">${version}</option>\n`;
+    }
+
+    content += `            </select>\n`;
+    content += `          </div>\n`;
+    content += `        </div>\n`;
+
+    content += `\n`;
+
+    content += `        <div class="searchSearchBox">\n`;
+    content += `          <input type="text" id="bodysearchbox" placeholder="Search..." onKeyDown="if (event.key === 'Enter') { resetSearch(); }" />\n`;
+    content += `          <button onClick="resetSearch()">Search</button>\n`;
+    content += `        </div>\n`;
+
+    content += `\n`;
+
+    content += `        <div class="searchResultsArea" id="bodyresultsarea" style="visibility: hidden;">\n`;
+    content += `          <h3>Results</h3>\n`;
+    content += `          <div class="searchResults" id="bodysearchresults">\n`;
+    content += `          </div>\n`;
+    content += `        </div>\n`;
+    content += `      </div>\n`;
+
+    const filename = `${outputPath}/search.html`;
+
+    await fs.mkdir(outputPath, { recursive: true });
+    await fs.writeFile(filename, await layout({
+        title: `API search (${projectTitle})`,
+        content: content
+    }));
+}
+
 async function produceMainIndex(versions) {
     const versionList = versions.sort(versionSort);
     const versionDefault = versionList[versionList.length - 1];
@@ -1273,6 +1349,7 @@ function versionSort(a, b) {
 program.option('--output <filename>')
        .option('--layout <filename>')
        .option('--jekyll-layout <name>')
+       .option('--version <version...>')
        .option('--verbose')
        .option('--force')
        .option('--strict');
@@ -1290,13 +1367,12 @@ const outputPath = program.args[1];
 
 (async () => {
     try {
-        for (const version of (await fs.readdir(docsPath))
-                             .filter(a => a.endsWith('.json'))
-                             .map(a => a.replace(/\.json$/, ''))
-                             .sort(versionSort)
-                             .reverse()) {
-            versions.push(version);
-        }
+        const v = options.version ? options.version :
+            (await fs.readdir(docsPath))
+                     .filter(a => a.endsWith('.json'))
+                     .map(a => a.replace(/\.json$/, ''));
+
+        versions.push(...v.sort(versionSort).reverse());
 
         for (const version of versions) {
             if (options.verbose) {
@@ -1318,6 +1394,7 @@ const outputPath = program.args[1];
             await produceDocumentationForVersion(version, apiData[version]);
         }
 
+        await produceSearch(versions);
         await produceMainIndex(versions);
     } catch (e) {
         console.error(e);

script/api-docs/generate

@@ -9,17 +9,29 @@
 set -eo pipefail
 
 source_path=$(mktemp -d)
-verbose=true
+verbose=
 force=
 
-if [ "$1" = "" ]; then
-    echo "usage: $0 repo_path output_path" 1>&2
+for var in "$@"; do
+    if [ "${var}" == "--verbose" ]; then
+        verbose=true
+    elif [ "${var}" == "--force" ]; then
+        force=true
+    elif [ "${repo_path}" == "" ]; then
+        repo_path="${var}"
+    elif [ "${output_path}" == "" ]; then
+        output_path="${var}"
+    else
+        repo_path=""
+        output_path=""
+    fi
+done
+
+if [ "${repo_path}" = "" -o "${output_path}" = "" ]; then
+    echo "usage: $0 [--verbose] [--force] repo_path output_path" 1>&2
     exit 1
 fi
 
-repo_path=$1
-output_path=$2
-
 function do_checkout {
     if [ "$1" = "" ]; then
         echo "usage: $0 source_path" 1>&2
@@ -78,14 +90,9 @@ for version in ${source_path}/*; do
         fi
     fi
 
-    options=""
-    if [ "${force}" ]; then
-        options="${options} --force"
-    fi
-
     echo "Generating raw API documentation for ${version}..."
     mkdir -p "${output_path}/api"
-    node ./api-generator.js $options "${source_path}/${version}" > "${output_path}/api/${version}.json"
+    node ./api-generator.js "${source_path}/${version}" > "${output_path}/api/${version}.json"
 done
 
 if [ "${verbose}" ]; then
@@ -94,12 +101,15 @@ if [ "${verbose}" ]; then
     echo ""
 fi
 
-options=""
+search_options=""
+docs_options=""
 if [ "${verbose}" ]; then
-    options="${options} --verbose"
+    search_options="${search_options} --verbose"
+    docs_options="${docs_options} --verbose"
 fi
 if [ "${force}" ]; then
-    options="${options} --force"
+    docs_options="${docs_options} --force"
 fi
 
-node ./docs-generator.js --verbose --jekyll-layout default "${output_path}/api" "${output_path}/reference"
+node ./search-generator.js ${search_options} "${output_path}/api" "${output_path}/search-index"
+node ./docs-generator.js ${docs_options} --jekyll-layout default "${output_path}/api" "${output_path}/reference"

script/api-docs/package-lock.json

@@ -6,7 +6,8 @@
     "": {
       "dependencies": {
         "commander": "^12.1.0",
-        "markdown-it": "^14.1.0"
+        "markdown-it": "^14.1.0",
+        "minisearch": "^7.1.1"
       }
     },
     "node_modules/argparse": {
@@ -62,6 +63,11 @@
       "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
       "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w=="
     },
+    "node_modules/minisearch": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.1.1.tgz",
+      "integrity": "sha512-b3YZEYCEH4EdCAtYP7OlDyx7FdPwNzuNwLQ34SfJpM9dlbBZzeXndGavTrC+VCiRWomL21SWfMc6SCKO/U2ZNw=="
+    },
     "node_modules/punycode.js": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",

script/api-docs/package.json

@@ -1,6 +1,7 @@
 {
   "dependencies": {
     "commander": "^12.1.0",
-    "markdown-it": "^14.1.0"
+    "markdown-it": "^14.1.0",
+    "minisearch": "^7.1.1"
   }
 }

script/api-docs/search-generator.js

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+
+const markdownit = require('markdown-it');
+const { program } = require('commander');
+const minisearch = require('minisearch');
+
+const path = require('node:path');
+const fs = require('node:fs/promises');
+
+const linkPrefix = '/docs/reference';
+
+const defaultBranch = 'main';
+
+function uniqueifyId(api, nodes) {
+    let suffix = "", i = 1;
+
+    while (true) {
+        const possibleId = `${api.kind}-${api.name}${suffix}`;
+	let collision = false;
+
+        for (const item of nodes) {
+	    if (item.id === possibleId) {
+	        collision = true;
+	        break;
+	    }
+        }
+
+	if (!collision) {
+	    return possibleId;
+	}
+
+	suffix = `-${++i}`;
+    }
+}
+
+async function produceSearchIndex(version, apiData) {
+    const nodes = [ ];
+
+    for (const group in apiData['groups']) {
+        for (const name in apiData['groups'][group]['apis']) {
+            const api = apiData['groups'][group]['apis'][name];
+
+	    let displayName = name;
+
+	    if (api.kind === 'macro') {
+	      displayName = displayName.replace(/\(.*/, '');
+	    }
+
+            const apiSearchData = {
+	        id: uniqueifyId(api, nodes),
+                name: displayName,
+		group: group,
+                kind: api.kind
+            };
+
+            apiSearchData.description = Array.isArray(api.comment) ?
+	        api.comment[0] : api.comment;
+
+            let detail = "";
+
+            if (api.kind === 'macro') {
+                detail = api.value;
+            }
+            else if (api.kind === 'alias') {
+                detail = api.type;
+            }
+            else {
+                let details = undefined;
+
+                if (api.kind === 'struct' || api.kind === 'enum') {
+                    details = api.members;
+                }
+                else if (api.kind === 'function' || api.kind === 'callback') {
+                    details = api.params;
+                }
+                else {
+                    throw new Error(`unknown api type '${api.kind}'`);
+                }
+
+                for (const item of details || [ ]) {
+                    if (detail.length > 0) {
+                        detail += ' ';
+                    }
+
+                    detail += item.name;
+
+                    if (item.comment) {
+                        detail += ' ';
+                        detail += item.comment;
+                    }
+                }
+
+                if (api.kind === 'function' || api.kind === 'callback') {
+                    if (detail.length > 0 && api.returns?.type) {
+                        detail += ' ' + api.returns.type;
+                    }
+
+                    if (detail.length > 0 && api.returns?.comment) {
+                        detail += ' ' + api.returns.comment;
+                    }
+                }
+            }
+
+            detail = detail.replaceAll(/\s+/g, ' ')
+                           .replaceAll(/[\"\'\`]/g, '');
+
+            apiSearchData.detail = detail;
+
+            nodes.push(apiSearchData);
+        }
+    }
+
+    const index = new minisearch({
+	fields: [ 'name', 'description', 'detail' ],
+	storeFields: [ 'name', 'group', 'kind', 'description' ],
+	searchOptions: { boost: { name: 5, description: 2 } }
+    });
+
+    index.addAll(nodes);
+
+    const filename = `${outputPath}/${version}.json`;
+    await fs.mkdir(outputPath, { recursive: true });
+    await fs.writeFile(filename, JSON.stringify(index, null, 2));
+}
+
+function versionSort(a, b) {
+    if (a === b) {
+        return 0;
+    }
+
+    const aVersion = a.match(/^v(\d+)(?:\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?(?:-(.*))?$/);
+    const bVersion = b.match(/^v(\d+)(?:\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?(?:-(.*))?$/);
+
+    if (!aVersion && !bVersion) {
+        return a.localeCompare(b);
+    }
+    else if (aVersion && !bVersion) {
+        return -1;
+    }
+    else if (!aVersion && bVersion) {
+        return 1;
+    }
+
+    for (let i = 1; i < 5; i++) {
+        if (!aVersion[i] && !bVersion[i]) {
+            break;
+        }
+        else if (aVersion[i] && !bVersion[i]) {
+            return 1;
+        }
+        else if (!aVersion[i] && bVersion[i]) {
+            return -1;
+        }
+        else if (aVersion[i] !== bVersion[i]) {
+            return aVersion[i] - bVersion[i];
+        }
+    }
+
+    if (aVersion[5] && !bVersion[5]) {
+        return -1;
+    }
+    else if (!aVersion[5] && bVersion[5]) {
+        return 1;
+    }
+    else if (aVersion[5] && bVersion[5]) {
+        return aVersion[5].localeCompare(bVersion[5]);
+    }
+
+    return 0;
+}
+
+program.option('--verbose')
+       .option('--version <version...>');
+program.parse();
+
+const options = program.opts();
+
+if (program.args.length != 2) {
+    console.error(`usage: ${path.basename(process.argv[1])} raw_api_dir output_dir`);
+    process.exit(1);
+}
+
+const docsPath = program.args[0];
+const outputPath = program.args[1];
+
+(async () => {
+    try {
+        const v = options.version ? options.version :
+            (await fs.readdir(docsPath))
+                     .filter(a => a.endsWith('.json'))
+                     .map(a => a.replace(/\.json$/, ''));
+
+        const versions = v.sort(versionSort).reverse();
+
+	for (const version of versions) {
+            if (options.verbose) {
+                console.log(`Reading documentation data for ${version}...`);
+            }
+
+            const apiData = JSON.parse(await fs.readFile(`${docsPath}/${version}.json`));
+
+            if (options.verbose) {
+                console.log(`Creating minisearch index for ${version}...`);
+            }
+
+            await produceSearchIndex(version, apiData);
+        }
+    } catch (e) {
+        console.error(e);
+        process.exit(1);
+    }
+})();