automate creation of _index.md files

Signed-off-by: jkoberg <jkoberg@owncloud.com>

changelog/unreleased/automate-md-creation.md

@@ -0,0 +1,5 @@
+Enhancement: Automate md creation
+
+Automatically create `_index.md` files from the services `README.md`
+
+https://github.com/owncloud/ocis/pull/5901

docs/helpers/index.tmpl

@@ -0,0 +1,17 @@
+---
+title: {{ .ServiceName }}
+date: {{ .CreationTime }}
+weight: 20
+geekdocRepo: https://github.com/owncloud/ocis
+geekdocEditPath: edit/master/docs/services/{{ .service }}
+geekdocFilePath: _index.md
+geekdocCollapseSection: true
+---
+
+## Abstract
+
+{{ .Abstract }}
+## Table of Contents
+
+{{ .TocTree }}
+{{ .Content }}

docs/helpers/main.go

@@ -4,4 +4,5 @@ func main() {
 	RenderTemplates()
 	GetRogueEnvs()
 	RenderGlobalVarsTemplate()
+	GenerateMarkdowns()
 }

docs/helpers/markdowncreation.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"text/template"
+	"time"
+
+	"github.com/owncloud/ocis/v2/ocis-pkg/markdown"
+)
+
+var _configMarkdown = `{{< include file="services/_includes/%s-config-example.yaml"  language="yaml" >}}
+
+{{< include file="services/_includes/%s_configvars.md" >}}
+`
+
+// GenerateMarkdowns generates the _index.md files for the dev docu
+func GenerateMarkdowns() {
+	paths, err := filepath.Glob("../../services/*/README.md")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, p := range paths {
+		service := filepath.Base(filepath.Dir(p))
+		if err := generateMarkdown(p, service); err != nil {
+			fmt.Printf("error generating markdown for %s: %s\n", service, err)
+		}
+	}
+}
+
+func generateMarkdown(filepath string, servicename string) error {
+	f, err := os.ReadFile(filepath)
+	if err != nil {
+		return err
+	}
+
+	md := markdown.NewMD(f)
+	if len(md.Headings) == 0 || md.Headings[0].Level != 1 {
+		return errors.New("readme has invalid format")
+	}
+
+	// we don't need the main title, we add in our template
+	head := md.Headings[0]
+	md.Headings = md.Headings[1:]
+	md.Headings = append(md.Headings, markdown.Heading{
+		Level:   2,
+		Header:  "Example Yaml Config",
+		Content: fmt.Sprintf(_configMarkdown, servicename, servicename),
+	})
+
+	tpl := template.Must(template.ParseFiles("index.tmpl"))
+	b := bytes.NewBuffer(nil)
+	if err := tpl.Execute(b, map[string]interface{}{
+		"ServiceName":  head.Header,
+		"CreationTime": time.Now().Format(time.RFC3339Nano),
+		"service":      servicename,
+		"Abstract":     head.Content,
+		"TocTree":      string(md.Toc()),
+		"Content":      string(md.Bytes()),
+	}); err != nil {
+		return err
+	}
+
+	targetFile := fmt.Sprintf("../../docs/services/%s/_index.md", servicename)
+	return os.WriteFile(targetFile, b.Bytes(), os.ModePerm)
+}

ocis-pkg/markdown/markdown.go

@@ -0,0 +1,115 @@
+// Package markdown allows reading and editing Markdown files
+package markdown
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"os"
+	"strings"
+	"time"
+)
+
+// Heading represents a markdown Heading
+type Heading struct {
+	Content string
+	Level   int
+	Header  string
+}
+
+// MD represents a markdown file
+type MD struct {
+	Headings []Heading
+}
+
+// Bytes returns the markdown as []bytes to be written to a file
+func (md MD) Bytes() []byte {
+	b, num := bytes.NewBuffer(nil), len(md.Headings)
+	for i, h := range md.Headings {
+		b.Write([]byte(strings.Repeat("#", h.Level) + " " + h.Header + "\n"))
+		b.Write([]byte("\n"))
+		if len(h.Content) > 0 {
+			b.Write([]byte(h.Content))
+			if i < num-1 {
+				b.Write([]byte("\n"))
+			}
+		}
+	}
+	return b.Bytes()
+}
+
+// Toc returns the table of contents as []byte
+func (md MD) Toc() []byte {
+	b := bytes.NewBuffer(nil)
+	for _, h := range md.Headings {
+		if h.Level == 1 {
+			// main title not in toc
+			continue
+		}
+		link := fmt.Sprintf("#%s", strings.ToLower(strings.Replace(h.Header, " ", "-", -1)))
+		s := fmt.Sprintf("%s* [%s](%s)\n", strings.Repeat("  ", h.Level-2), h.Header, link)
+		b.Write([]byte(s))
+	}
+	return b.Bytes()
+}
+
+// NewMD parses a new Markdown
+func NewMD(b []byte) MD {
+	md := MD{}
+	var heading Heading
+	parts := strings.Split(string(b), "\n")
+	for _, p := range parts {
+		if p == "" {
+			continue
+		}
+		if p[:1] == "#" { // this is a header
+			if heading.Header != "" {
+				md.Headings = append(md.Headings, heading)
+			}
+			heading = Heading{}
+			i := strings.LastIndex(p, "#")
+			levs, con := p[:i+1], p[i+1:]
+			heading.Header = strings.TrimPrefix(con, " ")
+			heading.Level = len(levs)
+		} else {
+			heading.Content += p + "\n"
+		}
+	}
+	if heading.Header != "" {
+		md.Headings = append(md.Headings, heading)
+	}
+
+	return md
+}
+
+func main() {
+	f, err := os.ReadFile("/home/jkoberg/ocis/services/antivirus/README.md")
+	if err != nil {
+		fmt.Println("ERROR", err)
+		return
+	}
+
+	md := NewMD(f)
+	head := md.Headings[0]
+	md.Headings = md.Headings[1:]
+
+	tpl := template.Must(template.ParseFiles("index.tmpl"))
+	b := bytes.NewBuffer(nil)
+	err = tpl.Execute(b, map[string]interface{}{
+		"ServiceName":  head.Header,
+		"CreationTime": time.Now().Format(time.RFC3339Nano),
+		"service":      "unknown",
+		"Abstract":     head.Content,
+		"TocTree":      string(md.Toc()),
+		"Content":      string(md.Bytes()),
+	})
+	if err != nil {
+		fmt.Println("ERROR", err)
+		return
+	}
+	err = os.WriteFile("test.md", b.Bytes(), os.ModePerm)
+	if err != nil {
+		fmt.Println("ERROR", err)
+		return
+	}
+}

ocis-pkg/markdown/markdown_suite_test.go

@@ -0,0 +1,13 @@
+package markdown
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestSearch(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Markdown Suite")
+}

ocis-pkg/markdown/markdown_test.go

@@ -0,0 +1,48 @@
+package markdown
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	SmallMarkdown = `# Title
+
+some abstract description
+
+## SubTitle 1
+
+subtitle one description
+
+## SubTitle 2
+
+subtitle two description
+
+### Subpoint to SubTitle 2
+
+description to subpoint
+`
+	SmallMD = MD{
+		Headings: []Heading{
+			{Level: 1, Header: "Title", Content: "some abstract description\n"},
+			{Level: 2, Header: "SubTitle 1", Content: "subtitle one description\n"},
+			{Level: 2, Header: "SubTitle 2", Content: "subtitle two description\n"},
+			{Level: 3, Header: "Subpoint to SubTitle 2", Content: "description to subpoint\n"},
+		},
+	}
+)
+
+var _ = Describe("TestMarkdown", func() {
+	DescribeTable("Conversion works both ways",
+		func(mdfile string, expectedMD MD) {
+			md := NewMD([]byte(mdfile))
+
+			Expect(len(md.Headings)).To(Equal(len(expectedMD.Headings)))
+			for i, h := range md.Headings {
+				Expect(h).To(Equal(expectedMD.Headings[i]))
+			}
+			Expect(string(md.Bytes())).To(Equal(mdfile))
+		},
+		Entry("converts a small markdown", SmallMarkdown, SmallMD),
+	)
+})

ocis-pkg/markdown/test.md

@@ -0,0 +1,56 @@
+---
+title: Antivirus Service
+date: 2023-03-22T14:42:54.504370253+01:00
+weight: 20
+geekdocRepo: https://github.com/owncloud/ocis
+geekdocEditPath: edit/master/docs/services/unknown
+geekdocFilePath: _index.md
+geekdocCollapseSection: true
+---
+
+## Abstract
+
+The `antivirus` service is responsible for scanning files for viruses.
+
+## Table of Contents
+
+* [Configuration](#configuration)
+  * [Antivirus Scanner Type](#antivirus-scanner-type)
+  * [Maximum Scan size](#maximum-scan-size)
+  * [Infected File Handling](#infected-file-handling)
+  * [Scanner Inaccessibility](#scanner-inaccessibility)
+* [Operation Modes](#operation-modes)
+  * [Postprocessing](#postprocessing)
+
+## Configuration
+
+### Antivirus Scanner Type
+
+The antivirus service currently supports [ICAP](https://tools.ietf.org/html/rfc3507) and [ClamAV](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details.
+  -   For `icap`, only scanners using the `X-Infection-Found` header are currently supported.
+  -   For `clamav` only local sockets can currently be configured.
+
+### Maximum Scan size
+
+Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously, it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary.
+
+### Infected File Handling
+
+The antivirus service allows three different ways of handling infected files. Those can be set via the `ANTIVIRUS_INFECTED_FILE_HANDLING` environment variable:
+  -   `delete`: (default): Infected files will be deleted immediately, further postprocessing is cancelled.
+  -   `abort`:  (advanced option): Infected files will be kept, further postprocessing is cancelled. Files can be manually retrieved and inspected by an admin. To identify the file for further investigation, the antivirus service logs the abort/infected state including the file ID. The file is located in the `storage/users/uploads` folder of the ocis data directory and persists until it is manually deleted by the admin via the [Manage Unfinished Uploads](https://doc.owncloud.com/ocis/next/deployment/services/s-list/storage-users.html#manage-unfinished-uploads) command.
+  -   `continue`:  (obviously not recommended): Infected files will be marked via metadata as infected but postprocessing continues normally. Note: Infected Files are moved to their final destination and therefore not prevented from download which includes the risk of spreading viruses.
+In all cases, a log entry is added declaring the infection and handling method and a notification via the `userlog` service sent.
+
+### Scanner Inaccessibility
+
+In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessibility of the scanner used.
+
+## Operation Modes
+
+The antivirus service can scan files during `postprocessing`. `on demand` scanning is currently not available and might be added in a future release.
+
+### Postprocessing
+
+The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `virusscan`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details.
+