feat: add NGINX variable documentation (#203)

* feat: parse XML for module-defined variables

* feat: new funcs to see variables

New function and new data file from running the converter.

* chore: update changelog

Author: ryepup

CHANGELOG.md

@@ -1,9 +1,9 @@
 # Changelog
 
-## 1.0.0 (Month Date, Year)
+## 1.0.14 (June 6, 2024)
 
-Initial release of the NGINX template repository.
+Changed source and version from the Mercurial repository to the GitHub one.
 
-## 1.0.14 (June 6, 2024)
+## 1.1 (June, 11 2024)
 
-Changed source and version from the Mercurial repository to the GitHub one.
+Added NGINX variables (e.g. `$binary_remote_addr`) to the dataset.

examples/autocomplete/index.ts

@@ -1,4 +1,9 @@
-import { getDirectives, Format, Directive } from '@nginx/reference-lib'
+import {
+  getDirectives,
+  Format,
+  Directive,
+  getVariables,
+} from '@nginx/reference-lib'
 type autocomplete = {
   /** name of the NGINX module */
   m: string
@@ -10,10 +15,10 @@ type autocomplete = {
    * nginx config */
   v?: string
   /** markdown CSV for valid contexts */
-  c: string
+  c?: string
   /** markdown-formatted syntax specifications, including directive name.
    * Multiple syntaxes are seperated by newlines */
-  s: string
+  s?: string
 }
 
 function toAutocomplete(d: Directive): autocomplete {
@@ -32,5 +37,10 @@ function toAutocomplete(d: Directive): autocomplete {
   return ret
 }
 
-const formatted = getDirectives(Format.Markdown).map(toAutocomplete)
-console.log(JSON.stringify(formatted, undefined, 4))
+const directives = getDirectives(Format.Markdown).map(toAutocomplete)
+const variables = getVariables(Format.Markdown).map((v) => ({
+  m: v.module,
+  n: v.name,
+  d: v.description,
+}))
+console.log(JSON.stringify(directives.concat(variables), undefined, 4))

reference-converter/internal/output/output.go

@@ -21,10 +21,17 @@ type Directive struct {
 	DescriptionHtml string   `json:"description_html"`
 }
 
+type Variable struct {
+	Name            string `json:"name"`
+	DescriptionMd   string `json:"description_md"`
+	DescriptionHtml string `json:"description_html"`
+}
+
 type Module struct {
 	Id         string      `json:"id"`
 	Name       string      `json:"name"`
 	Directives []Directive `json:"directives"`
+	Variables  []Variable  `json:"variables,omitempty"`
 }
 
 func toModule(m *parse.Module) Module {
@@ -45,6 +52,13 @@ func toModule(m *parse.Module) Module {
 				DescriptionHtml: directive.Prose.ToHTML(),
 			})
 		}
+		for _, variable := range section.Variables {
+			module.Variables = append(module.Variables, Variable{
+				Name:            variable.Name,
+				DescriptionMd:   variable.Prose.ToMarkdown(),
+				DescriptionHtml: variable.Prose.ToHTML(),
+			})
+		}
 	}
 	return module
 }

reference-converter/internal/parse/elements.go

@@ -3,11 +3,12 @@ package parse
 import (
 	"encoding/xml"
 	"fmt"
+	"regexp"
+	"strings"
+
 	"github.com/gomarkdown/markdown"
 	"github.com/gomarkdown/markdown/html"
 	"github.com/gomarkdown/markdown/parser"
-	"regexp"
-	"strings"
 )
 
 // Syntax contain the markdown formatted syntax for the directive, very close to
@@ -153,10 +154,100 @@ type Directive struct {
 	Prose    Prose    `xml:"para"`
 }
 
+// Variable represents an NGINX variable defined by a module, e.g $binary_remote_addr.
+type Variable struct {
+	Name  string
+	Prose Prose
+}
+
+// unmarshalVariablesCML extracts NGINX variables from the common pattern:
+//
+//	<section id="variables">
+//	<list type="tag">
+//	<tag-name><var>$VARIABLE_NAME</var></tag-name>
+//	<tag-desc>$DOCUMENTATION</tag-desc>
+//	<tag-name><var>$VARIABLE_NAME</var><value>$DYNAMIC_SUFFIX</value></tag-name>
+//	<tag-desc>$DOCUMENTATION</tag-desc>
+//	</list>
+//	</section>
+func unmarshalVariablesCML(d *xml.Decoder, start xml.StartElement) ([]Variable, error) {
+	var v struct {
+		ID         string `xml:"id,attr"`
+		Paragraphs []struct {
+			List struct {
+				TagNames []struct {
+					Name   string `xml:"var"`
+					Suffix string `xml:"value"`
+				} `xml:"tag-name"`
+				TagDesc []Prose `xml:"tag-desc"`
+			} `xml:"list"`
+		} `xml:"para"`
+	}
+	if err := d.DecodeElement(&v, &start); err != nil {
+		return nil, fmt.Errorf("failed to parse variables: %w", err)
+	}
+	var vs []Variable
+	for _, para := range v.Paragraphs {
+		if len(para.List.TagDesc) != len(para.List.TagNames) {
+			return nil, fmt.Errorf(
+				"invalid variables section, need to have the same number of names (%d) and descriptions (%d)",
+				len(para.List.TagNames), len(para.List.TagDesc),
+			)
+		}
+		for idx, tn := range para.List.TagNames {
+			name := tn.Name
+			if tn.Suffix != "" {
+				name += strings.ToUpper(tn.Suffix)
+			}
+			vs = append(vs, Variable{
+				Name:  name,
+				Prose: para.List.TagDesc[idx],
+			})
+		}
+	}
+
+	return vs, nil
+}
+
 type Section struct {
-	ID         string      `xml:"id,attr"`
-	Directives []Directive `xml:"directive"`
-	Prose      Prose       `xml:"para"`
+	ID         string
+	Directives []Directive
+	Prose      Prose
+	Variables  []Variable
+}
+
+// UnmarshalXML handles parsing sections with directives vs sections with variables.
+func (s *Section) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	attrs := newAttrs(start.Attr)
+	if attrs["id"] == "variables" {
+		// parse as a list of variables
+		vs, err := unmarshalVariablesCML(d, start)
+		if err != nil {
+			return fmt.Errorf("failed to unmarshall variables: %w", err)
+		}
+		*s = Section{
+			ID:        "variables",
+			Variables: vs,
+		}
+		return nil
+	}
+
+	// parse as a normal section
+	var sec struct {
+		ID         string      `xml:"id,attr"`
+		Directives []Directive `xml:"directive"`
+		Prose      Prose       `xml:"para"`
+	}
+	if err := d.DecodeElement(&sec, &start); err != nil {
+		return err
+	}
+
+	*s = Section{
+		ID:         sec.ID,
+		Directives: sec.Directives,
+		Prose:      sec.Prose,
+	}
+	return nil
 }
 
 type Module struct {

reference-converter/internal/parse/parse_test.go

@@ -41,6 +41,23 @@ func TestParse(t *testing.T) {
 							},
 						},
 					},
+					{
+						ID: "variables",
+						Variables: []parse.Variable{
+							{
+								Name: "$wildcard_var_NAME",
+								Prose: parse.Prose{
+									{Content: "\nI support a dynamic suffix *`name`*\n"},
+								},
+							},
+							{
+								Name: "$variable",
+								Prose: parse.Prose{
+									{Content: "\nI am a variable with `formatting` in my desc\n"},
+								},
+							},
+						},
+					},
 				},
 			},
 		},

reference-converter/internal/parse/testdata/module.xml

@@ -27,4 +27,21 @@ Can have more than one, with some&nbsp;html&mdash;ish entities and <literal>verb
 
 </directive>
 </section>
+<section id="variables">
+<para>We add these variables:</para>
+
+<para>
+<list type="tag">
+<tag-name><var>$wildcard_var_</var><value>name</value></tag-name>
+<tag-desc>
+I support a dynamic suffix <value>name</value>
+</tag-desc>
+
+<tag-name><var>$variable</var></tag-name>
+<tag-desc>
+I am a variable with <literal>formatting</literal> in my desc
+</tag-desc>
+</list>
+</para>
+</section>
 </module>

reference-lib/index.ts

@@ -1,13 +1,19 @@
 import reference from './src/reference.json'
 
 export interface Directive {
-    name: string
-    module: string
-    description: string
-    syntax: string[]
-    contexts: string[]
-    isBlock: boolean
-    default: string
+  name: string
+  module: string
+  description: string
+  syntax: string[]
+  contexts: string[]
+  isBlock: boolean
+  default: string
+}
+
+export interface Variable {
+  name: string
+  module: string
+  description: string
 }
 
 export enum Format {
@@ -41,37 +47,63 @@ for (const modules of reference.modules) {
  * Returns all the nginx directives
  *
  *  @param: format: format of the return type HTML or markdown
- *
  *  @return: an array of Directives
  */
-export function getDirectives(format=Format.HTML): Directive[] {
-    const directives = reference.modules.flatMap((m) =>
-      m.directives.map((d) => ({...d, module: m.name})))
-    .map ((d) => ({
-        name: d.name,
-        module: d.module,
-        description: format === Format.HTML ? d.description_html : d.description_md,
-        syntax: format === Format.HTML ? d.syntax_html : d.syntax_md,
-        contexts: d.contexts,
-        isBlock: d.isBlock,
-        default: d.default
-    } as Directive))
-    return directives
+export function getDirectives(format = Format.HTML): Directive[] {
+  const directives = reference.modules
+    .flatMap((m) => m.directives.map((d) => ({ ...d, module: m.name })))
+    .map(
+      (d) =>
+        ({
+          name: d.name,
+          module: d.module,
+          description:
+            format === Format.HTML ? d.description_html : d.description_md,
+          syntax: format === Format.HTML ? d.syntax_html : d.syntax_md,
+          contexts: d.contexts,
+          isBlock: d.isBlock,
+          default: d.default,
+        } as Directive)
+    )
+  return directives
+}
+
+/**
+ * Returns all variables defined by any moduled
+ *
+ *  @param: format: format of the return type HTML or markdown
+ *  @return: an array of Variables
+ */
+export function getVariables(format = Format.HTML): Variable[] {
+  return reference.modules.flatMap(
+    (m) =>
+      m.variables?.map((v) => ({
+        name: v.name,
+        description:
+          format === Format.HTML ? v.description_html : v.description_md,
+        module: m.name,
+      })) ?? []
+  )
 }
 
 /**
- * Returns the description corresponding to the directive name
+ * Returns the description corresponding to the directive or variable name
  *
  *  @param: directive: directive name to find
  *  @param: module: optional name of module
  *  @param: format: format of the return type HTML or markdown
  *
  *  @return: a string containing the description of the directive in xml or markdown format
  */
-export function find(directive: string, module: string | undefined, format=Format.HTML): string | undefined {
-  const data =
-    module
-      ? refDirectives.get(directive)?.find((d) => d.module.toUpperCase() === module.toUpperCase())
-      : refDirectives.get(directive)?.at(0)
-  return (format === Format.HTML ? data?.description_html : data?.description_md)
+export function find(
+  directive: string,
+  module: string | undefined,
+  format = Format.HTML
+): string | undefined {
+  const data = module
+    ? refDirectives
+        .get(directive)
+        ?.find((d) => d.module.toUpperCase() === module.toUpperCase())
+    : refDirectives.get(directive)?.at(0)
+  return format === Format.HTML ? data?.description_html : data?.description_md
 }

reference-lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nginx/reference-lib",
-  "version": "1.0.14",
+  "version": "1.1.0",
   "description": "",
   "main": "dist/index.js",
   "type": "module",