tools: build all.json by combining generated JSON

Notes: 1) Removed a number of root properties that did not seem relevant: source, desc, and introduced_in. There no longer is a source, and the other two are from the first include and do not reflect the entire API. 2) As with https://github.com/nodejs/node/issues/20100, the current "desc" properties sometimes contained in-page links, other times referenced another page, and often did not match the links in the original HTML or JSON file. I chose to standardize on external links as "desc" values are isolated snippets as opposed to all.html which can be viewed as a standalone and self contained document. 3) Eliminated preprocessing for @include entirely, including the test case for this function. 4) _toc.md was renamed to index.md. 5) index comments no longer appear in embedded TOCs (left hand side column in the generated documentation. PR-URL: https://github.com/nodejs/node/pull/21637 Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com> Reviewed-By: Rich Trott <rtrott@gmail.com>
2026-01-15 12:15:26 +00:00 · 2018-07-03 12:46:56 -04:00
parent 1529ef4dc5
commit 0c743b5f77
12 changed files with 165 additions and 216 deletions
--- a/8
+++ b/8
@@ -615,7 +615,7 @@ doc-only: $(apidoc_dirs) $(apiassets)  ## Builds the docs with the local or the
 	if [ ! -d doc/api/assets ]; then \
 		$(MAKE) tools/doc/node_modules/js-yaml/package.json; \
 	fi;
-	@$(MAKE) $(apidocs_html) $(apidocs_json)
+	@$(MAKE) out/doc/api/all.html out/doc/api/all.json

 .PHONY: doc
 doc: $(NODE_EXE) doc-only
@@ -665,10 +665,12 @@ out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js
 out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js
 	$(call available-node, $(gen-html))

-out/doc/api/all.html: $(filter-out out/doc/api/all.html, $(apidocs_html)) \
-	tools/doc/allhtml.js
+out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
 	$(call available-node, tools/doc/allhtml.js)

+out/doc/api/all.json: $(apidocs_json) tools/doc/alljson.js
+	$(call available-node, tools/doc/alljson.js)
+
 .PHONY: docopen
 docopen: $(apidocs_html)
 	@$(PYTHON) -mwebbrowser file://$(PWD)/out/doc/api/all.html
--- a/doc/api/_toc.md
+++ b/doc/api/_toc.md
@@ -1,62 +0,0 @@
-@// NB(chrisdickinson): if you move this file, be sure to update
-@// tools/doc/html.js to point at the new location.
-
-<!--introduced_in=v0.10.0-->
-
-* [About these Docs](documentation.html)
-* [Usage & Example](synopsis.html)
-
-<div class="line"></div>
-
-* [Assertion Testing](assert.html)
-* [Async Hooks](async_hooks.html)
-* [Buffer](buffer.html)
-* [C++ Addons](addons.html)
-* [C/C++ Addons - N-API](n-api.html)
-* [Child Processes](child_process.html)
-* [Cluster](cluster.html)
-* [Command Line Options](cli.html)
-* [Console](console.html)
-* [Crypto](crypto.html)
-* [Debugger](debugger.html)
-* [Deprecated APIs](deprecations.html)
-* [DNS](dns.html)
-* [Domain](domain.html)
-* [ECMAScript Modules](esm.html)
-* [Errors](errors.html)
-* [Events](events.html)
-* [File System](fs.html)
-* [Globals](globals.html)
-* [HTTP](http.html)
-* [HTTP/2](http2.html)
-* [HTTPS](https.html)
-* [Inspector](inspector.html)
-* [Internationalization](intl.html)
-* [Modules](modules.html)
-* [Net](net.html)
-* [OS](os.html)
-* [Path](path.html)
-* [Performance Hooks](perf_hooks.html)
-* [Process](process.html)
-* [Punycode](punycode.html)
-* [Query Strings](querystring.html)
-* [Readline](readline.html)
-* [REPL](repl.html)
-* [Stream](stream.html)
-* [String Decoder](string_decoder.html)
-* [Timers](timers.html)
-* [TLS/SSL](tls.html)
-* [Trace Events](tracing.html)
-* [TTY](tty.html)
-* [UDP/Datagram](dgram.html)
-* [URL](url.html)
-* [Utilities](util.html)
-* [V8](v8.html)
-* [VM](vm.html)
-* [Worker Threads](worker_threads.html)
-* [ZLIB](zlib.html)
-
-<div class="line"></div>
-
-* [GitHub Repo & Issue Tracker](https://github.com/nodejs/node)
-* [Mailing List](https://groups.google.com/group/nodejs)
--- a/doc/api/all.md
+++ b/doc/api/all.md
@@ -1,50 +0,0 @@
-<!--lint disable prohibited-strings-->
-@include documentation
-@include synopsis
-@include assert
-@include async_hooks
-@include buffer
-@include addons
-@include n-api
-@include child_process
-@include cluster
-@include cli
-@include console
-@include crypto
-@include debugger
-@include deprecations
-@include dns
-@include domain
-@include esm
-@include errors
-@include events
-@include fs
-@include globals
-@include http
-@include http2
-@include https
-@include inspector
-@include intl
-@include modules
-@include net
-@include os
-@include path
-@include perf_hooks
-@include process
-@include punycode
-@include querystring
-@include readline
-@include repl
-@include stream
-@include string_decoder
-@include timers
-@include tls
-@include tracing
-@include tty
-@include dgram
-@include url
-@include util
-@include v8
-@include vm
-@include worker_threads
-@include zlib
--- a/doc/api/index.md
+++ b/doc/api/index.md
@@ -1 +1,64 @@
-@include _toc.md
+<!--
+  NB(chrisdickinson): if you move this file, be sure to update
+  tools/doc/html.js to point at the new location.
+-->
+
+<!--introduced_in=v0.10.0-->
+
+* [About these Docs](documentation.html)
+* [Usage & Example](synopsis.html)
+
+<div class="line"></div>
+
+* [Assertion Testing](assert.html)
+* [Async Hooks](async_hooks.html)
+* [Buffer](buffer.html)
+* [C++ Addons](addons.html)
+* [C/C++ Addons - N-API](n-api.html)
+* [Child Processes](child_process.html)
+* [Cluster](cluster.html)
+* [Command Line Options](cli.html)
+* [Console](console.html)
+* [Crypto](crypto.html)
+* [Debugger](debugger.html)
+* [Deprecated APIs](deprecations.html)
+* [DNS](dns.html)
+* [Domain](domain.html)
+* [ECMAScript Modules](esm.html)
+* [Errors](errors.html)
+* [Events](events.html)
+* [File System](fs.html)
+* [Globals](globals.html)
+* [HTTP](http.html)
+* [HTTP/2](http2.html)
+* [HTTPS](https.html)
+* [Inspector](inspector.html)
+* [Internationalization](intl.html)
+* [Modules](modules.html)
+* [Net](net.html)
+* [OS](os.html)
+* [Path](path.html)
+* [Performance Hooks](perf_hooks.html)
+* [Process](process.html)
+* [Punycode](punycode.html)
+* [Query Strings](querystring.html)
+* [Readline](readline.html)
+* [REPL](repl.html)
+* [Stream](stream.html)
+* [String Decoder](string_decoder.html)
+* [Timers](timers.html)
+* [TLS/SSL](tls.html)
+* [Trace Events](tracing.html)
+* [TTY](tty.html)
+* [UDP/Datagram](dgram.html)
+* [URL](url.html)
+* [Utilities](util.html)
+* [V8](v8.html)
+* [VM](vm.html)
+* [Worker Threads](worker_threads.html)
+* [ZLIB](zlib.html)
+
+<div class="line"></div>
+
+* [GitHub Repo & Issue Tracker](https://github.com/nodejs/node)
+* [Mailing List](https://groups.google.com/group/nodejs)
--- a/test/doctool/test-doctool-html.js
+++ b/test/doctool/test-doctool-html.js
@@ -11,7 +11,6 @@ try {
 const assert = require('assert');
 const { readFile } = require('fs');
 const fixtures = require('../common/fixtures');
-const processIncludes = require('../../tools/doc/preprocess.js');
 const toHTML = require('../../tools/doc/html.js');

 // Test data is a list of objects with two properties.
@@ -64,15 +63,6 @@ const testData = [
      '<!-- This is not a metadata comment --> ' +
      '<p>Describe <code>Something</code> in more detail here. </p>'
  },
-  {
-    file: fixtures.path('doc_with_includes.md'),
-    html: '<!-- [start-include:doc_inc_1.md] -->' +
-    '<p>Look <a href="doc_inc_2.html#doc_inc_2_foobar">here</a>!</p>' +
-    '<!-- [end-include:doc_inc_1.md] --><!-- [start-include:doc_inc_2.md] -->' +
-    '<h1>foobar<span><a class="mark" href="#doc_inc_2_foobar" ' +
-    'id="doc_inc_2_foobar">#</a></span></h1>' +
-    '<p>I exist and am being linked to.</p><!-- [end-include:doc_inc_2.md] -->'
-  },
  {
    file: fixtures.path('sample_document.md'),
    html: '<ol><li>fish</li><li><p>fish</p></li><li><p>Redfish</p></li>' +
@@ -90,36 +80,34 @@ testData.forEach(({ file, html, analyticsId }) => {

  readFile(file, 'utf8', common.mustCall((err, input) => {
    assert.ifError(err);
-    processIncludes(file, input, common.mustCall((err, preprocessed) => {
-      assert.ifError(err);

-      toHTML(
-        {
-          input: preprocessed,
-          filename: 'foo',
-          nodeVersion: process.version,
-          analytics: analyticsId,
-        },
-        common.mustCall((err, output) => {
-          assert.ifError(err);
+    toHTML(
+      {
+        input: input,
+        filename: 'foo',
+        nodeVersion: process.version,
+        analytics: analyticsId,
+      },
+      common.mustCall((err, output) => {
+        assert.ifError(err);

-          const actual = output.replace(spaces, '');
-          // Assert that the input stripped of all whitespace contains the
-          // expected markup.
-          assert(actual.includes(expected));
+        const actual = output.replace(spaces, '');
+        // Assert that the input stripped of all whitespace contains the
+        // expected markup.
+        assert(actual.includes(expected));

-          // Testing the insertion of Google Analytics script when
-          // an analytics id is provided. Should not be present by default.
-          const scriptDomain = 'google-analytics.com';
-          if (includeAnalytics) {
-            assert(actual.includes(scriptDomain),
-                   `Google Analytics script was not present in "${actual}"`);
-          } else {
-            assert.strictEqual(actual.includes(scriptDomain), false,
-                               'Google Analytics script was present in ' +
-                               `"${actual}"`);
-          }
-        }));
-    }));
+        // Testing the insertion of Google Analytics script when
+        // an analytics id is provided. Should not be present by default.
+        const scriptDomain = 'google-analytics.com';
+        if (includeAnalytics) {
+          assert(actual.includes(scriptDomain),
+                 `Google Analytics script was not present in "${actual}"`);
+        } else {
+          assert.strictEqual(actual.includes(scriptDomain), false,
+                             'Google Analytics script was present in ' +
+                             `"${actual}"`);
+        }
+      })
+    );
  }));
 });
--- a/test/doctool/test-make-doc.js
+++ b/test/doctool/test-make-doc.js
@@ -13,17 +13,16 @@ const path = require('path');

 const apiPath = path.resolve(__dirname, '..', '..', 'out', 'doc', 'api');
 const allDocs = fs.readdirSync(apiPath);
-assert.ok(allDocs.includes('_toc.html'));
+assert.ok(allDocs.includes('index.html'));

 const actualDocs = allDocs.filter(
  (name) => {
    const extension = path.extname(name);
-    return (extension === '.html' || extension === '.json') &&
-           name !== '_toc.html';
+    return extension === '.html' || extension === '.json';
  }
 );

-const toc = fs.readFileSync(path.resolve(apiPath, '_toc.html'), 'utf8');
+const toc = fs.readFileSync(path.resolve(apiPath, 'index.html'), 'utf8');
 const re = /href="([^/]+\.html)"/;
 const globalRe = new RegExp(re, 'g');
 const links = toc.match(globalRe);
@@ -32,8 +31,7 @@ assert.notStrictEqual(links, null);
 // Filter out duplicate links, leave just filenames, add expected JSON files.
 const linkedHtmls = [...new Set(links)].map((link) => link.match(re)[1]);
 const expectedJsons = linkedHtmls
-                       .map((name) => name.replace('.html', '.json'))
-                       .concat('_toc.json');
+                       .map((name) => name.replace('.html', '.json'));
 const expectedDocs = linkedHtmls.concat(expectedJsons);

 // Test that all the relative links in the TOC match to the actual documents.
--- a/test/fixtures/doc_with_includes.md
+++ b/test/fixtures/doc_with_includes.md
@@ -1,2 +0,0 @@
-@include doc_inc_1
-@include doc_inc_2.md
--- a/tools/doc/allhtml.js
+++ b/tools/doc/allhtml.js
@@ -12,7 +12,7 @@ const htmlFiles = fs.readdirSync(source, 'utf8')
  .filter((name) => name.includes('.html') && name !== 'all.html');

 // Read the table of contents.
-const toc = fs.readFileSync(source + '/_toc.html', 'utf8');
+const toc = fs.readFileSync(source + '/index.html', 'utf8');

 // Extract (and concatenate) the toc and apicontent from each document.
 let contents = '';
@@ -47,11 +47,12 @@ for (const link of toc.match(/<a.*?>/g)) {
  seen[href] = true;
 }

-// Replace various mentions of _toc with all.
-let all = toc.replace(/_toc\.html/g, 'all.html')
-  .replace('_toc.json', 'all.json')
-  .replace('api-section-_toc', 'api-section-all')
-  .replace('data-id="_toc"', 'data-id="all"');
+// Replace various mentions of index with all.
+let all = toc.replace(/index\.html/g, 'all.html')
+  .replace('<a href="all.html" name="toc">', '<a href="index.html" name="toc">')
+  .replace('index.json', 'all.json')
+  .replace('api-section-index', 'api-section-all')
+  .replace('data-id="index"', 'data-id="all"');

 // Clean up the title.
 all = all.replace(/<title>.*?\| /, '<title>');
--- a/tools/doc/alljson.js
+++ b/tools/doc/alljson.js
@@ -0,0 +1,56 @@
+'use strict';
+
+// Build all.json by combining the miscs, modules, classes, globals, and methods
+// from the generated json files.
+
+const fs = require('fs');
+
+const source = `${__dirname}/../../out/doc/api`;
+
+// Get a list of generated API documents.
+const jsonFiles = fs.readdirSync(source, 'utf8')
+  .filter((name) => name.includes('.json') && name !== 'all.json');
+
+// Read the table of contents.
+const toc = fs.readFileSync(source + '/index.html', 'utf8');
+
+// Initialize results. Only these four data values will be collected.
+const results = {
+  miscs: [],
+  modules: [],
+  classes: [],
+  globals: [],
+  methods: []
+};
+
+// Identify files that should be skipped. As files are processed, they
+// are added to this list to prevent dupes.
+const seen = {
+  'all.json': true,
+  'index.json': true
+};
+
+// Extract (and concatenate) the selected data from each document.
+// Expand hrefs found in json to include source HTML file.
+for (const link of toc.match(/<a.*?>/g)) {
+  const href = /href="(.*?)"/.exec(link)[1];
+  const json = href.replace('.html', '.json');
+  if (!jsonFiles.includes(json) || seen[json]) continue;
+  const data = JSON.parse(
+    fs.readFileSync(source + '/' + json, 'utf8')
+      .replace(/<a href=\\"#/g, `<a href=\\"${href}#`)
+  );
+
+  for (const property in data) {
+    if (results.hasOwnProperty(property)) {
+      results[property].push(...data[property]);
+    }
+  }
+
+  // Mark source as seen.
+  seen[json] = true;
+}
+
+// Write results.
+fs.writeFileSync(source + '/all.json',
+                 `${JSON.stringify(results, null, 2)}\n`, 'utf8');
--- a/tools/doc/generate.js
+++ b/tools/doc/generate.js
@@ -21,7 +21,6 @@

 'use strict';

-const processIncludes = require('./preprocess.js');
 const fs = require('fs');

 // Parse the args.
@@ -52,12 +51,6 @@ if (!filename) {
 }

 fs.readFile(filename, 'utf8', (er, input) => {
-  if (er) throw er;
-  // Process the input for @include lines.
-  processIncludes(filename, input, next);
-});
-
-function next(er, input) {
  if (er) throw er;
  switch (format) {
    case 'json':
@@ -78,4 +71,4 @@ function next(er, input) {
    default:
      throw new Error(`Invalid format: ${format}`);
  }
-}
+});
--- a/tools/doc/html.js
+++ b/tools/doc/html.js
@@ -36,8 +36,8 @@ marked.setOptions({ renderer });

 const docPath = path.resolve(__dirname, '..', '..', 'doc');

-const gtocPath = path.join(docPath, 'api', '_toc.md');
-const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^@\/\/.*$/gm, '');
+const gtocPath = path.join(docPath, 'api', 'index.md');
+const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^<!--.*?-->/gms, '');
 const gtocHTML = marked(gtocMD).replace(
  /<a href="(.*?)"/g,
  (all, href) => `<a class="nav-${href.replace('.html', '')
--- a/tools/doc/preprocess.js
+++ b/tools/doc/preprocess.js
@@ -1,38 +0,0 @@
-'use strict';
-
-module.exports = processIncludes;
-
-const path = require('path');
-const fs = require('fs');
-
-const includeExpr = /^@include\s+([\w-]+)(?:\.md)?$/gmi;
-const commentExpr = /^@\/\/.*$/gm;
-
-function processIncludes(inputFile, input, cb) {
-  const includes = input.match(includeExpr);
-  if (includes === null)
-    return cb(null, input.replace(commentExpr, ''));
-
-  let errState = null;
-  let incCount = includes.length;
-
-  includes.forEach((include) => {
-    const fname = include.replace(includeExpr, '$1.md');
-    const fullFname = path.resolve(path.dirname(inputFile), fname);
-
-    fs.readFile(fullFname, 'utf8', function(er, inc) {
-      if (errState) return;
-      if (er) return cb(errState = er);
-      incCount--;
-
-      // Add comments to let the HTML generator know
-      // how the anchors for headings should look like.
-      inc = `<!-- [start-include:${fname}] -->\n` +
-            `${inc}\n<!-- [end-include:${fname}] -->\n`;
-      input = input.split(`${include}\n`).join(`${inc}\n`);
-
-      if (incCount === 0)
-        return cb(null, input.replace(commentExpr, ''));
-    });
-  });
-}