Merge pull request #5164 from dragonchaser/extract-rogue-env-vars

Extract rogue env vars

Makefile

@@ -146,6 +146,7 @@ docs-generate:
 	@for mod in $(OCIS_MODULES); do \
         $(MAKE) --no-print-directory -C $$mod docs-generate || exit 1; \
     done
+	$(MAKE) --no-print-directory -C docs docs-generate || exit 1
 
 .PHONY: ci-go-generate
 ci-go-generate:

changelog/unreleased/add-global-vars-extrator.md

@@ -0,0 +1,7 @@
+Enhancement: add global env variable extractor
+
+We have added a little tool that will extract global env vars, that are loaded
+only through os.Getenv for documentation purposes
+
+https://github.com/owncloud/ocis/issues/4916
+https://github.com/owncloud/ocis/pull/5164

docs/Makefile

@@ -8,8 +8,7 @@ help:
 
 .PHONY: docs-generate
 docs-generate: ## run docs-generate for all oCIS services
-	@pushd helpers && go run configenvextractor.go; popd
-	@$(MAKE) --no-print-directory -C ../ docs-generate
+	@pushd helpers && go run .; popd
 
 .PHONY: docs-init
 docs-init:

docs/helpers/README.md

@@ -0,0 +1,77 @@
+# Docs Helpers
+
+`docs/helpers` contains a small go program creating docs by extracting information from the code. It is manually started with `make docs-generate` or via the CI and has three main responsibilities:
+
+- Generate docs for envvars in config structs
+- Extract envvars that are not mentioned in config structs (aka "Rogue" envvars)
+- Generate docs for rogue envvars
+
+Output:
+
+- The generated yaml files can be found at: `docs/services/_includes` when running locally respectively in the docs branch after the CI has finished.
+- The generated adoc files can be found at: `docs/services/_includes/adoc` when running locally respectively in the docs branch after the CI has finished.
+- The file name for rouge envvars is named: `global_configvars.adoc`.
+
+Admin doc process:
+
+Whenever a build from the admin documentation is triggered, the files generated here are included into the build process and added in a proper manner defined by the admin documentation.
+
+Genreal info:
+
+"Rouge" envvars are variables that need to be present *before* the core or services are starting up as they depend on the info provided like path for config files etc. Therefore they are _not_ bound to services like other envvars do. 
+
+It can happen that rouge envvars are found but do not need to be published as they are for internal use only. Those rouge envvars can be defined to be ignored for further processing.
+
+IMPORTANT:
+
+- Once a rouge envvar has been identified, it is added to the `global_vars.yaml` file but never changed or touched by the process. There is one exception with respect to single/double quote usage. While you manually can (and will) define a text like: `"'/var/lib/ocis'"`, quotes are transformed by the process in the .yaml file to: `'''/var/lib/ocis'''`. There is no need to change this back, as the final step transforms this correctly to the adoc table.
+
+- Because rouge envvars do not have the same structural setup as "normal" envvars like type, description or defaults, these infos need to be provided manually one time - even if found multiple times. Any change on this info will be used on the next CI run and published on the next admin docs build.
+
+- Do not change the sort order of rouge envvar blocks as they are automatically reordered alphabetically.
+
+## Generate Envvar docs for config structs
+
+Generates docs from a template file, mainly extracting `"env"` and `"desc"` tags from the config structs.
+
+Templates can be found in `docs/helpers` folder. (Same as this `README`.) Check `.tmpl` files
+
+## Extract Rogue Envvars
+
+It `grep`s over the code, looking for `os.Getenv` and parses these contents to a yaml file along with the following information:
+```golang
+// Variable contains all information about one rogue envvar
+type Variable struct {
+	// These field structs are automatically filled:
+	// RawName can be the name of the envvar or the name of its var
+	RawName string `yaml:"rawname"`
+	// Path to the envvar with linenumber
+	Path string `yaml:"path"`
+	// FoundInCode indicates if the variable is still found in the codebase.
+	FoundInCode bool `yaml:"foundincode"`
+	// Name is equal to RawName but will not be overwritten in consecutive runs
+	Name string `yaml:"name"`
+
+	// These field structs need manual filling:
+	// Type of the envvar
+	Type string `yaml:"type"`
+	// DefaultValue of the envvar
+	DefaultValue string `yaml:"default_value"`
+	// Description of what this envvar does
+	Description string `yaml:"description"`
+	// Do not export this envvar into the generated adoc table
+	Ignore bool `yaml:"do_ignore"`
+
+	// For simplicity ignored for now:
+	// DependendServices []Service `yaml:"dependend_services"`
+}
+```
+This yaml file can later be manually edited to add descriptions, default values, etc.
+
+IMPORTANT: `RawName`, `Path` and `FoundInCode` are automatically filled by the program. DO NOT EDIT THESE VALUES MANUALLY.
+
+## Generate Rogue Envvar docs
+
+It picks up the `yaml` file generated in `Extract Rogue Envvars` step and renders it to a adoc file (table) using a go template.
+
+The adoc template file for this step can be found at `docs/templates/ADOC_global.tmpl`.

docs/helpers/configenvextractor.go

@@ -17,7 +17,8 @@ var targets = map[string]string{
 	"environment-variable-docs-generator.go.tmpl": "output/env/environment-variable-docs-generator.go",
 }
 
-func main() {
+// RenderTemplates does something with templates
+func RenderTemplates() {
 	fmt.Println("Getting relevant packages")
 	paths, err := filepath.Glob("../../services/*/pkg/config/defaults/defaultconfig.go")
 	if err != nil {
@@ -32,14 +33,14 @@ func main() {
 	}
 
 	for template, output := range targets {
-		GenerateIntermediateCode(template, output, paths)
-		RunIntermediateCode(output)
+		generateIntermediateCode(template, output, paths)
+		runIntermediateCode(output)
 	}
 	fmt.Println("Cleaning up")
 	os.RemoveAll("output")
 }
 
-func GenerateIntermediateCode(templatePath string, intermediateCodePath string, paths []string) {
+func generateIntermediateCode(templatePath string, intermediateCodePath string, paths []string) {
 	content, err := os.ReadFile(templatePath)
 	if err != nil {
 		log.Fatal(err)
@@ -60,7 +61,7 @@ func GenerateIntermediateCode(templatePath string, intermediateCodePath string,
 	}
 }
 
-func RunIntermediateCode(intermediateCodePath string) {
+func runIntermediateCode(intermediateCodePath string) {
 	fmt.Println("Running intermediate go code for " + intermediateCodePath)
 	defaultPath := "~/.ocis"
 	os.Setenv("OCIS_BASE_DATA_PATH", defaultPath)

docs/helpers/global_vars.yaml

@@ -0,0 +1,73 @@
+variables:
+- rawname: CS3_GATEWAY
+  path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:76
+  foundincode: true
+  name: "CS3_GATEWAY"
+  type: ""
+  default_value: ""
+  description: ""
+  do_ignore: true
+- rawname: CS3_MACHINE_AUTH_API_KEY
+  path: services/idp/pkg/backends/cs3/bootstrap/cs3.go:77
+  foundincode: true
+  name: "CS3_MACHINE_AUTH_API_KEY"
+  type: ""
+  default_value: ""
+  description: ""
+  do_ignore: true
+- rawname: MICRO_LOG_LEVEL
+  path: ocis-pkg/log/log.go:34
+  foundincode: true
+  name: "MICRO_LOG_LEVEL"
+  type: "string"
+  default_value: "Error"
+  description: "Set the log level for the internal go micro framework. Only change on supervision of ownCloud Support."
+  do_ignore: false
+- rawname: MICRO_LOG_LEVEL
+  path: ocis-pkg/log/log.go:30
+  foundincode: true
+  name: "MICRO_LOG_LEVEL"
+  type: ""
+  default_value: ""
+  description: ""
+  do_ignore: true
+- rawname: registryEnv
+  path: ocis-pkg/registry/registry.go:44
+  foundincode: true
+  name: "MICRO_REGISTRY"
+  type: "string"
+  default_value: ""
+  description: "Go micro registry type to use. Supported types are: 'nats', 'kubernetes', 'etcd', 'consul' and 'memory'. Will be selected automatically. Only change on supervision of ownCloud Support."
+  do_ignore: false
+- rawname: registryAddressEnv
+  path: ocis-pkg/registry/registry.go:42
+  foundincode: true
+  name: "MICRO_REGISTRY_ADDRESS"
+  type: "string"
+  default_value: ""
+  description: "The bind address of the internal go micro framework. Only change on supervision of ownCloud Support."
+  do_ignore: false
+- rawname: OCIS_BASE_DATA_PATH
+  path: ocis-pkg/config/defaults/paths.go:23
+  foundincode: true
+  name: "OCIS_BASE_DATA_PATH"
+  type: "string"
+  default_value: "'/var/lib/ocis' or '$HOME/.ocis/'"
+  description: "The base directory location used by several services and for user data. Predefined to '/var/lib/ocis' for container images (inside the container) or '$HOME/.ocis/' for binary releases. Can be adapted for services individually."
+  do_ignore: false
+- rawname: OCIS_CONFIG_DIR
+  path: ocis-pkg/config/defaults/paths.go:56
+  foundincode: true
+  name: "OCIS_CONFIG_DIR"
+  type: "string"
+  default_value: "'/etc/ocis' or '$HOME/.ocis/config'"
+  description: "The default directory location for config files. Predefined to '/etc/ocis' for container images (inside the container) or '$HOME/.ocis/config' for binary releases."
+  do_ignore: false
+- rawname: parts[0]
+  path: ocis-pkg/config/envdecode/envdecode.go:382
+  foundincode: true
+  name: parts[0]
+  type: ""
+  default_value: ""
+  description: false positive - code that extract envvars for config structs
+  do_ignore: true

docs/helpers/main.go

@@ -0,0 +1,7 @@
+package main
+
+func main() {
+	RenderTemplates()
+	GetRogueEnvs()
+	RenderGlobalVarsTemplate()
+}

docs/helpers/rogueEnv.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"text/template"
+
+	"gopkg.in/yaml.v2"
+)
+
+const yamlSource = "global_vars.yaml"
+
+// ConfigVars is the main yaml source
+type ConfigVars struct {
+	Variables []Variable `yaml:"variables"`
+}
+
+// Variable contains all information about one rogue envvar
+type Variable struct {
+	// These field structs are automatically filled:
+	// RawName can be the name of the envvar or the name of its var
+	RawName string `yaml:"rawname"`
+	// Path to the envvar with linenumber
+	Path string `yaml:"path"`
+	// FoundInCode indicates if the variable is still found in the codebase. TODO: delete immediately?
+	FoundInCode bool `yaml:"foundincode"`
+	// Name is equal to RawName but will not be overwritten in consecutive runs
+	Name string `yaml:"name"`
+
+	// These field structs need manual filling:
+	// Type of the envvar
+	Type string `yaml:"type"`
+	// DefaultValue of the envvar
+	DefaultValue string `yaml:"default_value"`
+	// Description of what this envvar does
+	Description string `yaml:"description"`
+	// Ignore this envvar when creating docs?
+	Ignore bool `yaml:"do_ignore"`
+
+	// For simplicity ignored for now:
+	// DependendServices []Service `yaml:"dependend_services"`
+}
+
+// GetRogueEnvs extracts the rogue envs from the code
+func GetRogueEnvs() {
+	curdir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fullYamlPath := filepath.Join(curdir, yamlSource)
+	re := regexp.MustCompile(`os.Getenv\(([^\)]+)\)`)
+	vars := &ConfigVars{}
+	fmt.Printf("Reading existing variable definitions from %s\n", fullYamlPath)
+	yfile, err := ioutil.ReadFile(fullYamlPath)
+	if err == nil {
+		err := yaml.Unmarshal(yfile, &vars)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if err := os.Chdir("../../"); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("Gathering variable definitions from source")
+	out, err := exec.Command("bash", "-c", "grep -RHn os.Getenv | grep -v rogueEnv.go |grep \\.go").Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	lines := strings.Split(string(out), "\n")
+
+	// find current vars
+	currentVars := make(map[string]Variable)
+	for _, l := range lines {
+		fmt.Printf("Parsing %s\n", l)
+		r := strings.SplitN(l, ":\t", 2)
+		if len(r) != 2 || r[0] == "" || r[1] == "" {
+			continue
+		}
+
+		res := re.FindAllSubmatch([]byte(r[1]), -1)
+		if len(res) != 1 || len(res[0]) < 2 {
+			fmt.Printf("Error envvar not matching pattern: %s", r[1])
+			continue
+		}
+
+		path := r[0]
+		name := strings.Trim(string(res[0][1]), "\"")
+		currentVars[path+name] = Variable{
+			RawName:     name,
+			Path:        path,
+			FoundInCode: true,
+			Name:        name,
+		}
+	}
+
+	// adjust existing vars
+	for i, v := range vars.Variables {
+		_, ok := currentVars[v.Path+v.RawName]
+		if !ok {
+			vars.Variables[i].FoundInCode = false
+			continue
+		}
+
+		vars.Variables[i].FoundInCode = true
+		delete(currentVars, v.Path+v.RawName)
+	}
+
+	// add new envvars
+	for _, v := range currentVars {
+		vars.Variables = append(vars.Variables, v)
+	}
+
+	less := func(i, j int) bool {
+		return vars.Variables[i].Name < vars.Variables[j].Name
+	}
+
+	sort.Slice(vars.Variables, less)
+
+	output, err := yaml.Marshal(vars)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("Writing new variable definitions to %s\n", fullYamlPath)
+	err = ioutil.WriteFile(fullYamlPath, output, 0666)
+	if err != nil {
+		log.Fatalf("could not write %s", fullYamlPath)
+	}
+	if err := os.Chdir(curdir); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// RenderGlobalVarsTemplate renders the global vars template
+func RenderGlobalVarsTemplate() {
+	curdir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fullYamlPath := filepath.Join(curdir, yamlSource)
+
+	content, err := ioutil.ReadFile("../../docs/templates/ADOC_global.tmpl")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	targetFolder := "../../docs/services/_includes/adoc/"
+
+	vars := &ConfigVars{}
+	fmt.Printf("Reading existing variable definitions from %s\n", fullYamlPath)
+	yfile, err := ioutil.ReadFile(fullYamlPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = yaml.Unmarshal(yfile, &vars)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	targetFile, err := os.Create(filepath.Join(targetFolder, "global_configvars.adoc"))
+	if err != nil {
+		log.Fatalf("Failed to create target file: %s", err)
+	}
+	defer targetFile.Close()
+
+	tpl := template.Must(template.New("").Parse(string(content)))
+	if err = tpl.Execute(targetFile, *vars); err != nil {
+		log.Fatalf("Failed to execute template: %s", err)
+	}
+}

docs/templates/ADOC_global.tmpl

@@ -0,0 +1,27 @@
+// collected through docs/helpers/rougeEnv.go
+
+[caption=]
+.Environment variables with global scope not included in a service
+[width="100%",cols="~,~,~,~",options="header"]
+|===
+| Name
+| Type
+| Default Value
+| Description
+
+{{- range .Variables}}
+
+{{- if .Ignore }}
+    {{ continue }}
+{{- end }}
+
+a| `{{- .Name }}` +
+a| [subs=-attributes]
+++{{ .Type }} ++
+a| [subs=-attributes]
+++{{.DefaultValue}} ++
+a| [subs=-attributes]
+++{{.Description}} ++
+
+{{- end }}
+|===