Merge pull request #8454 from dragonchaser/envar-delta-reflections

Envar delta reflections

docs/helpers/configenvextractor.go

@@ -12,9 +12,10 @@ import (
 )
 
 var targets = map[string]string{
-	"adoc-generator.go.tmpl":                      "output/adoc/adoc-generator.go",
-	"example-config-generator.go.tmpl":            "output/exampleconfig/example-config-generator.go",
-	"environment-variable-docs-generator.go.tmpl": "output/env/environment-variable-docs-generator.go",
+	"templates/adoc-generator.go.tmpl":                      "output/adoc/adoc-generator.go",
+	"templates/example-config-generator.go.tmpl":            "output/exampleconfig/example-config-generator.go",
+	"templates/environment-variable-docs-generator.go.tmpl": "output/env/environment-variable-docs-generator.go",
+	"templates/envar-delta-table.go.tmpl":                   "output/env/envvar-delta-table.go",
 }
 
 // RenderTemplates does something with templates
@@ -37,7 +38,10 @@ func RenderTemplates() {
 		runIntermediateCode(output)
 	}
 	fmt.Println("Cleaning up")
-	os.RemoveAll("output")
+	err = os.RemoveAll("output")
+	if err != nil {
+		fmt.Println(err)
+	}
 }
 
 func generateIntermediateCode(templatePath string, intermediateCodePath string, paths []string) {

docs/helpers/env-var-delta.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"fmt"
+	"github.com/rogpeppe/go-internal/semver"
+	"gopkg.in/yaml.v2"
+	"log"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+const envVarYamlSource = "env_vars.yaml"
+
+var envVarOutPutTemplates = map[string]string{
+	"added":      "templates/env-vars-added.md.tmpl",
+	"removed":    "templates/env-vars-removed.md.tmpl",
+	"deprecated": "templates/env-vars-deprecated.md.tmpl",
+}
+
+// ConfigField represents the env-var annotation in the code
+type ConfigField struct {
+	Name                string `yaml:"name"`
+	DefaultValue        string `yaml:"defaultValue"`
+	Type                string `yaml:"type"`
+	Description         string `yaml:"description"`
+	IntroductionVersion string `yaml:"introductionVersion"`
+	DeprecationVersion  string `yaml:"deprecationVersion"`
+	RemovalVersion      string `yaml:"removalVersion"`
+	DeprecationInfo     string `yaml:"deprecationInfo"`
+}
+
+type TemplateData struct {
+	StartVersion string
+	EndVersion   string
+	DeltaFields  []*ConfigField
+}
+
+// RenderEnvVarDeltaTable generates tables for env-var deltas
+func RenderEnvVarDeltaTable(osArgs []string) {
+	if !semver.IsValid(osArgs[2]) {
+		log.Fatalf("Start version invalid semver: %s", osArgs[2])
+	}
+	if !semver.IsValid(osArgs[3]) {
+		log.Fatalf("Target version invalid semver: %s", osArgs[3])
+	}
+	if semver.Compare(osArgs[2], osArgs[3]) >= 0 {
+		log.Fatalf("Start version %s is not smaller than target version %s", osArgs[2], osArgs[3])
+	}
+	if semver.Compare(osArgs[2], "v5.0.0") < 0 {
+		log.Fatalf("This tool does not support versions prior v5.0.0, (given %s)", osArgs[2])
+	}
+	startVersion := osArgs[2]
+	endVersion := osArgs[3]
+	fmt.Printf("Generating tables for env-var deltas between version %s and %s...\n", startVersion, endVersion)
+	curdir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fullYamlPath := filepath.Join(curdir, envVarYamlSource)
+	configFields := make(map[string]*ConfigField)
+	variableList := map[string][]*ConfigField{
+		"added":      {},
+		"removed":    {},
+		"deprecated": {},
+	}
+	fmt.Printf("Reading existing variable definitions from %s\n", fullYamlPath)
+	yfile, err := os.ReadFile(fullYamlPath)
+	if err == nil {
+		err := yaml.Unmarshal(yfile, configFields)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	fmt.Printf("Success, found %d entries\n", len(configFields))
+	for _, field := range configFields {
+		if field.IntroductionVersion != "" &&
+			field.IntroductionVersion != "pre5.0" &&
+			!semver.IsValid(field.IntroductionVersion) &&
+			field.IntroductionVersion[0] != 'v' {
+			field.IntroductionVersion = "v" + field.IntroductionVersion
+		}
+		if field.IntroductionVersion != "pre5.0" && !semver.IsValid(field.IntroductionVersion) {
+			fmt.Printf("Invalid semver for field %s: %s\n", field.Name, field.IntroductionVersion)
+			os.Exit(1)
+		}
+		//fmt.Printf("Processing field %s dv: %s, iv: %s\n", field.Name, field.DeprecationVersion, field.IntroductionVersion)
+		if semver.IsValid(field.RemovalVersion) && semver.Compare(startVersion, field.RemovalVersion) < 0 && semver.Compare(endVersion, field.RemovalVersion) >= 0 {
+			variableList["removed"] = append(variableList["removed"], field)
+		}
+		if semver.IsValid(field.DeprecationVersion) && semver.Compare(startVersion, field.DeprecationVersion) <= 0 && semver.Compare(endVersion, field.DeprecationVersion) > 0 {
+			variableList["deprecated"] = append(variableList["deprecated"], field)
+		}
+		if semver.IsValid(field.IntroductionVersion) && semver.Compare(startVersion, field.IntroductionVersion) <= 0 && semver.Compare(endVersion, field.IntroductionVersion) >= 0 {
+			fmt.Printf("Adding field %s iv: %s\n", field.Name, field.IntroductionVersion)
+			variableList["added"] = append(variableList["added"], field)
+		}
+	}
+	for templateName, templatePath := range envVarOutPutTemplates {
+		content, err := os.ReadFile(templatePath)
+		if err != nil {
+			log.Fatal(err)
+		}
+		tpl := template.Must(template.New(templateName).Parse(string(content)))
+		err = os.MkdirAll("output/env-deltas", 0700)
+		if err != nil {
+			log.Fatal(err)
+		}
+		targetFile, err := os.Create(filepath.Join("output/env-deltas", fmt.Sprintf("%s-%s-%s.md", startVersion, endVersion, templateName)))
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = tpl.Execute(targetFile, TemplateData{
+			StartVersion: startVersion,
+			EndVersion:   endVersion,
+			DeltaFields:  variableList[templateName],
+		})
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}

docs/helpers/main.go

@@ -1,8 +1,45 @@
 package main
 
+import (
+	"fmt"
+	"os"
+)
+
 func main() {
-	RenderTemplates()
-	GetRogueEnvs()
-	RenderGlobalVarsTemplate()
-	GenerateServiceIndexMarkdowns()
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "templates":
+			RenderTemplates()
+		case "rogue":
+			GetRogueEnvs()
+		case "globals":
+			RenderGlobalVarsTemplate()
+		case "service-index":
+			GenerateServiceIndexMarkdowns()
+		case "env-var-delta-table":
+			// This step is not covered by the all or default case, because it needs explicit arguments
+			if len(os.Args) != 4 {
+				fmt.Println("Needs two arguments: env-var-delta-table <first-version> <second-version>")
+				fmt.Println("Example: env-var-delta-table v5.0.0 v6.0.0")
+				fmt.Println("Will not generate usable results for versions Prior to v5.0.0")
+			} else {
+				RenderEnvVarDeltaTable(os.Args)
+			}
+		case "all":
+			RenderTemplates()
+			GetRogueEnvs()
+			RenderGlobalVarsTemplate()
+			GenerateServiceIndexMarkdowns()
+		case "help":
+			fallthrough
+		default:
+			fmt.Printf("Usage: %s [templates|rogue|globals|service-index|env-var-delta-table|all|help]\n", os.Args[0])
+		}
+	} else {
+		// Left here, even though present in the switch case, for backwards compatibility
+		RenderTemplates()
+		GetRogueEnvs()
+		RenderGlobalVarsTemplate()
+		GenerateServiceIndexMarkdowns()
+	}
 }

docs/helpers/markdowncreation.go

@@ -53,7 +53,7 @@ func generateMarkdown(filepath string, servicename string) error {
 		Content: fmt.Sprintf(_configMarkdown, servicename, servicename),
 	})
 
-	tpl := template.Must(template.ParseFiles("index.tmpl"))
+	tpl := template.Must(template.ParseFiles("templates/index.tmpl"))
 	b := bytes.NewBuffer(nil)
 	if err := tpl.Execute(b, map[string]interface{}{
 		"ServiceName":  head.Header,

docs/helpers/templates/env-vars-added.md.tmpl

@@ -0,0 +1,7 @@
+Added between Version {{ .StartVersion }} and {{ .EndVersion }}.
+
+| Variable | Description |
+| --- | --- |
+{{- range $key, $value := .DeltaFields}}
+| {{$value.Name}} | {{$value.Description}} |
+{{- end}}

docs/helpers/templates/env-vars-deprecated.md.tmpl

@@ -0,0 +1,7 @@
+Deprecated between Version {{ .StartVersion }} and {{ .EndVersion }}.
+
+| Variable | Description | Deprecation Info |
+| --- | --- |
+{{- range $key, $value := .DeltaFields}}
+| {{$value.Name}} | {{$value.Description}} | {{$value.DeprecationInfo}} |
+{{- end}}

docs/helpers/templates/env-vars-removed.md.tmpl

@@ -0,0 +1,7 @@
+Removed between Version {{ .StartVersion }} and {{ .EndVersion }}.
+
+| Variable | Description | Deprecation Info |
+| --- | --- |
+{{- range $key, $value := .DeltaFields}}
+| {{$value.Name}} | {{$value.Description}} | {{$value.DeprecationInfo}} |
+{{- end}}

docs/helpers/templates/envar-delta-table.go.tmpl

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"html"
+	"reflect"
+	"strings"
+	"log"
+	"os"
+	"path/filepath"
+
+	{{- range $key, $value :=.}}
+	pkg{{$key}} "{{$value}}"
+	{{- end}}
+)
+
+const yamlSource = "env_vars.yaml"
+
+type ConfigField struct {
+	Name                string `yaml:"name"`
+	DefaultValue        string `yaml:"defaultValue"`
+	Type                string `yaml:"type"`
+	Description         string `yaml:"description"`
+	IntroductionVersion string `yaml:"introductionVersion"`
+	DeprecationVersion  string `yaml:"deprecationVersion"`
+	RemovalVersion      string `yaml:"removalVersion"`
+	DeprecationInfo     string `yaml:"deprecationInfo"`
+}
+
+func main() {
+	fmt.Println("Generating tables for env-var deltas...")
+	curdir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fullYamlPath := filepath.Join(curdir, yamlSource)
+	var fields []ConfigField
+	configFields := make(map[string]*ConfigField)
+	fmt.Printf("Reading existing variable definitions from %s\n", fullYamlPath)
+	yfile, err := os.ReadFile(fullYamlPath)
+	if err == nil {
+		err := yaml.Unmarshal(yfile, configFields)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	m := map[string]interface{}{
+		{{- range $key, $value := .}}
+			"{{$value}}": *pkg{{$key}}.FullDefaultConfig(),
+		{{- end }}
+	}
+	for _, conf := range m {
+		fields = GetAnnotatedVariables(conf)
+		for _, field := range fields {
+			variants := strings.Split(field.Name, ";")
+			for _, variant := range variants {
+				if (configFields[variant] != nil && configFields[variant].Name == "") || configFields[variant] == nil {
+					configFields[variant] = &field
+				} else {
+					fmt.Printf("%v, duplicate key, merging\n", variant)
+					if strings.TrimSpace(configFields[variant].DefaultValue) != "" && configFields[variant].DefaultValue != field.DefaultValue {
+						configFields[variant].DefaultValue = field.DefaultValue
+					}
+					if strings.TrimSpace(configFields[variant].Description) != "" && configFields[variant].Description != field.Description {
+						configFields[variant].Description = field.Description
+					}
+					if strings.TrimSpace(configFields[variant].Type) != "" && configFields[variant].Type != field.Type {
+						configFields[variant].Type = field.Type
+					}
+					if strings.TrimSpace(configFields[variant].IntroductionVersion) != "" && configFields[variant].IntroductionVersion != field.IntroductionVersion {
+						configFields[variant].IntroductionVersion = field.IntroductionVersion
+					}
+					if strings.TrimSpace(configFields[variant].DeprecationVersion) != "" && configFields[variant].DeprecationVersion != field.DeprecationVersion {
+						configFields[variant].DeprecationVersion = field.DeprecationVersion
+					}
+					if strings.TrimSpace(configFields[variant].RemovalVersion) != "" && configFields[variant].RemovalVersion != field.RemovalVersion {
+						configFields[variant].RemovalVersion = field.RemovalVersion
+					}
+					if strings.TrimSpace(configFields[variant].Name) != "" && configFields[variant].Name != field.Name {
+						configFields[variant].Name = field.Name
+					}
+					if strings.TrimSpace(configFields[variant].DeprecationInfo) != "" && configFields[variant].DeprecationInfo != field.DeprecationInfo {
+						// there might be multiple superseeding DeprecationInformations, we might want to keep track of those, that's why we are not overwriting the field
+						configFields[variant].DeprecationInfo = configFields[variant].DeprecationInfo + " | " + field.DeprecationInfo
+					}
+				}
+			}
+		}
+	}
+
+	output, err := yaml.Marshal(configFields)
+	if err != nil {
+		log.Fatalf("Could not marshall variables: %v", err)
+	}
+	err = os.WriteFile(fullYamlPath, output, 0666)
+	if err != nil {
+		log.Fatalf("could not write %s", fullYamlPath)
+	}
+	if err := os.Chdir(curdir); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func GetAnnotatedVariables(s interface{}) []ConfigField {
+	t := reflect.TypeOf(s)
+	v := reflect.ValueOf(s)
+
+	var fields []ConfigField
+	for i := 0; i < t.NumField(); i++ {
+		field := t.Field(i)
+		value := v.Field(i)
+
+		switch value.Kind() {
+		default:
+			desc := field.Tag.Get("desc")
+			env, ok := field.Tag.Lookup("env")
+			if !ok {
+				continue
+			}
+			introductionVersion := field.Tag.Get("introductionVersion")
+			deprecationVersion := field.Tag.Get("deprecationVersion")
+			removalVersion := field.Tag.Get("removalVersion")
+			deprecationInfo := field.Tag.Get("deprecationInfo")
+			v := fmt.Sprintf("%v", value.Interface())
+			typeName := value.Type().Name()
+			if typeName == "" {
+				typeName = value.Type().String()
+			}
+			//fields = append(fields, ConfigField{Name: strings.ReplaceAll(env, ";", "<br/>"), DefaultValue: html.EscapeString(strings.Replace(v, "|", "\\|", -1)), Description: desc, Type: typeName})
+			fields = append(fields, ConfigField{
+				Name: env,
+				DefaultValue: html.EscapeString(strings.Replace(v, "|", "\\|", -1)),
+				Description: desc,
+				Type: typeName,
+				IntroductionVersion: introductionVersion,
+				DeprecationVersion: deprecationVersion,
+				RemovalVersion: removalVersion,
+				DeprecationInfo: deprecationInfo,
+			})
+		case reflect.Ptr:
+			// PolicySelectors in the Proxy are being skipped atm
+			// they are not configurable via env vars, if that changes
+			// they are probably added to the Sanitize() function
+			// and this should not be an issue then
+			if !value.IsZero() && value.Elem().CanInterface() {
+				fields = append(fields, GetAnnotatedVariables(value.Elem().Interface())...)
+			}
+		case reflect.Struct:
+			fields = append(fields, GetAnnotatedVariables(value.Interface())...)
+		}
+	}
+	return fields
+}

vendor/github.com/rogpeppe/go-internal/semver/forward.go

@@ -0,0 +1,39 @@
+// Package semver is a thin forwarding layer on top of
+// [golang.org/x/mod/semver]. See that package for documentation.
+//
+// Deprecated: use [golang.org/x/mod/semver] instead.
+package semver
+
+import "golang.org/x/mod/semver"
+
+func IsValid(v string) bool {
+	return semver.IsValid(v)
+}
+
+func Canonical(v string) string {
+	return semver.Canonical(v)
+}
+
+func Major(v string) string {
+	return semver.Major(v)
+}
+
+func MajorMinor(v string) string {
+	return semver.MajorMinor(v)
+}
+
+func Prerelease(v string) string {
+	return semver.Prerelease(v)
+}
+
+func Build(v string) string {
+	return semver.Build(v)
+}
+
+func Compare(v, w string) int {
+	return semver.Compare(v, w)
+}
+
+func Max(v, w string) string {
+	return semver.Max(v, w)
+}

vendor/modules.txt

@@ -1672,6 +1672,7 @@ github.com/rogpeppe/go-internal/internal/syscall/windows
 github.com/rogpeppe/go-internal/internal/syscall/windows/sysdll
 github.com/rogpeppe/go-internal/lockedfile
 github.com/rogpeppe/go-internal/lockedfile/internal/filelock
+github.com/rogpeppe/go-internal/semver
 # github.com/rs/cors v1.10.1
 ## explicit; go 1.13
 github.com/rs/cors