From cf8120a70d4b7f305b8356b9d65a1e7f445d57d2 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 22 Mar 2023 15:25:48 +0100 Subject: [PATCH] automate creation of _index.md files Signed-off-by: jkoberg --- changelog/unreleased/automate-md-creation.md | 5 + docs/helpers/index.tmpl | 17 +++ docs/helpers/main.go | 1 + docs/helpers/markdowncreation.go | 71 ++++++++++++ ocis-pkg/markdown/markdown.go | 115 +++++++++++++++++++ ocis-pkg/markdown/markdown_suite_test.go | 13 +++ ocis-pkg/markdown/markdown_test.go | 48 ++++++++ ocis-pkg/markdown/test.md | 56 +++++++++ 8 files changed, 326 insertions(+) create mode 100644 changelog/unreleased/automate-md-creation.md create mode 100644 docs/helpers/index.tmpl create mode 100644 docs/helpers/markdowncreation.go create mode 100644 ocis-pkg/markdown/markdown.go create mode 100644 ocis-pkg/markdown/markdown_suite_test.go create mode 100644 ocis-pkg/markdown/markdown_test.go create mode 100755 ocis-pkg/markdown/test.md diff --git a/changelog/unreleased/automate-md-creation.md b/changelog/unreleased/automate-md-creation.md new file mode 100644 index 0000000000..9631a2d0cf --- /dev/null +++ b/changelog/unreleased/automate-md-creation.md @@ -0,0 +1,5 @@ +Enhancement: Automate md creation + +Automatically create `_index.md` files from the services `README.md` + +https://github.com/owncloud/ocis/pull/5901 diff --git a/docs/helpers/index.tmpl b/docs/helpers/index.tmpl new file mode 100644 index 0000000000..0ea33427be --- /dev/null +++ b/docs/helpers/index.tmpl @@ -0,0 +1,17 @@ +--- +title: {{ .ServiceName }} +date: {{ .CreationTime }} +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/services/{{ .service }} +geekdocFilePath: _index.md +geekdocCollapseSection: true +--- + +## Abstract + +{{ .Abstract }} +## Table of Contents + +{{ .TocTree }} +{{ .Content }} diff --git a/docs/helpers/main.go b/docs/helpers/main.go index 86076042c0..c8d57311b6 100644 --- a/docs/helpers/main.go +++ b/docs/helpers/main.go @@ -4,4 +4,5 @@ func main() { RenderTemplates() GetRogueEnvs() RenderGlobalVarsTemplate() + GenerateMarkdowns() } diff --git a/docs/helpers/markdowncreation.go b/docs/helpers/markdowncreation.go new file mode 100644 index 0000000000..2c2d0ec1ca --- /dev/null +++ b/docs/helpers/markdowncreation.go @@ -0,0 +1,71 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "text/template" + "time" + + "github.com/owncloud/ocis/v2/ocis-pkg/markdown" +) + +var _configMarkdown = `{{< include file="services/_includes/%s-config-example.yaml" language="yaml" >}} + +{{< include file="services/_includes/%s_configvars.md" >}} +` + +// GenerateMarkdowns generates the _index.md files for the dev docu +func GenerateMarkdowns() { + paths, err := filepath.Glob("../../services/*/README.md") + if err != nil { + log.Fatal(err) + } + + for _, p := range paths { + service := filepath.Base(filepath.Dir(p)) + if err := generateMarkdown(p, service); err != nil { + fmt.Printf("error generating markdown for %s: %s\n", service, err) + } + } +} + +func generateMarkdown(filepath string, servicename string) error { + f, err := os.ReadFile(filepath) + if err != nil { + return err + } + + md := markdown.NewMD(f) + if len(md.Headings) == 0 || md.Headings[0].Level != 1 { + return errors.New("readme has invalid format") + } + + // we don't need the main title, we add in our template + head := md.Headings[0] + md.Headings = md.Headings[1:] + md.Headings = append(md.Headings, markdown.Heading{ + Level: 2, + Header: "Example Yaml Config", + Content: fmt.Sprintf(_configMarkdown, servicename, servicename), + }) + + tpl := template.Must(template.ParseFiles("index.tmpl")) + b := bytes.NewBuffer(nil) + if err := tpl.Execute(b, map[string]interface{}{ + "ServiceName": head.Header, + "CreationTime": time.Now().Format(time.RFC3339Nano), + "service": servicename, + "Abstract": head.Content, + "TocTree": string(md.Toc()), + "Content": string(md.Bytes()), + }); err != nil { + return err + } + + targetFile := fmt.Sprintf("../../docs/services/%s/_index.md", servicename) + return os.WriteFile(targetFile, b.Bytes(), os.ModePerm) +} diff --git a/ocis-pkg/markdown/markdown.go b/ocis-pkg/markdown/markdown.go new file mode 100644 index 0000000000..062170479e --- /dev/null +++ b/ocis-pkg/markdown/markdown.go @@ -0,0 +1,115 @@ +// Package markdown allows reading and editing Markdown files +package markdown + +import ( + "bytes" + "fmt" + "html/template" + "os" + "strings" + "time" +) + +// Heading represents a markdown Heading +type Heading struct { + Content string + Level int + Header string +} + +// MD represents a markdown file +type MD struct { + Headings []Heading +} + +// Bytes returns the markdown as []bytes to be written to a file +func (md MD) Bytes() []byte { + b, num := bytes.NewBuffer(nil), len(md.Headings) + for i, h := range md.Headings { + b.Write([]byte(strings.Repeat("#", h.Level) + " " + h.Header + "\n")) + b.Write([]byte("\n")) + if len(h.Content) > 0 { + b.Write([]byte(h.Content)) + if i < num-1 { + b.Write([]byte("\n")) + } + } + } + return b.Bytes() +} + +// Toc returns the table of contents as []byte +func (md MD) Toc() []byte { + b := bytes.NewBuffer(nil) + for _, h := range md.Headings { + if h.Level == 1 { + // main title not in toc + continue + } + link := fmt.Sprintf("#%s", strings.ToLower(strings.Replace(h.Header, " ", "-", -1))) + s := fmt.Sprintf("%s* [%s](%s)\n", strings.Repeat(" ", h.Level-2), h.Header, link) + b.Write([]byte(s)) + } + return b.Bytes() +} + +// NewMD parses a new Markdown +func NewMD(b []byte) MD { + md := MD{} + var heading Heading + parts := strings.Split(string(b), "\n") + for _, p := range parts { + if p == "" { + continue + } + if p[:1] == "#" { // this is a header + if heading.Header != "" { + md.Headings = append(md.Headings, heading) + } + heading = Heading{} + i := strings.LastIndex(p, "#") + levs, con := p[:i+1], p[i+1:] + heading.Header = strings.TrimPrefix(con, " ") + heading.Level = len(levs) + } else { + heading.Content += p + "\n" + } + } + if heading.Header != "" { + md.Headings = append(md.Headings, heading) + } + + return md +} + +func main() { + f, err := os.ReadFile("/home/jkoberg/ocis/services/antivirus/README.md") + if err != nil { + fmt.Println("ERROR", err) + return + } + + md := NewMD(f) + head := md.Headings[0] + md.Headings = md.Headings[1:] + + tpl := template.Must(template.ParseFiles("index.tmpl")) + b := bytes.NewBuffer(nil) + err = tpl.Execute(b, map[string]interface{}{ + "ServiceName": head.Header, + "CreationTime": time.Now().Format(time.RFC3339Nano), + "service": "unknown", + "Abstract": head.Content, + "TocTree": string(md.Toc()), + "Content": string(md.Bytes()), + }) + if err != nil { + fmt.Println("ERROR", err) + return + } + err = os.WriteFile("test.md", b.Bytes(), os.ModePerm) + if err != nil { + fmt.Println("ERROR", err) + return + } +} diff --git a/ocis-pkg/markdown/markdown_suite_test.go b/ocis-pkg/markdown/markdown_suite_test.go new file mode 100644 index 0000000000..2c925074a9 --- /dev/null +++ b/ocis-pkg/markdown/markdown_suite_test.go @@ -0,0 +1,13 @@ +package markdown + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSearch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Markdown Suite") +} diff --git a/ocis-pkg/markdown/markdown_test.go b/ocis-pkg/markdown/markdown_test.go new file mode 100644 index 0000000000..aa2bc86ae3 --- /dev/null +++ b/ocis-pkg/markdown/markdown_test.go @@ -0,0 +1,48 @@ +package markdown + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var ( + SmallMarkdown = `# Title + +some abstract description + +## SubTitle 1 + +subtitle one description + +## SubTitle 2 + +subtitle two description + +### Subpoint to SubTitle 2 + +description to subpoint +` + SmallMD = MD{ + Headings: []Heading{ + {Level: 1, Header: "Title", Content: "some abstract description\n"}, + {Level: 2, Header: "SubTitle 1", Content: "subtitle one description\n"}, + {Level: 2, Header: "SubTitle 2", Content: "subtitle two description\n"}, + {Level: 3, Header: "Subpoint to SubTitle 2", Content: "description to subpoint\n"}, + }, + } +) + +var _ = Describe("TestMarkdown", func() { + DescribeTable("Conversion works both ways", + func(mdfile string, expectedMD MD) { + md := NewMD([]byte(mdfile)) + + Expect(len(md.Headings)).To(Equal(len(expectedMD.Headings))) + for i, h := range md.Headings { + Expect(h).To(Equal(expectedMD.Headings[i])) + } + Expect(string(md.Bytes())).To(Equal(mdfile)) + }, + Entry("converts a small markdown", SmallMarkdown, SmallMD), + ) +}) diff --git a/ocis-pkg/markdown/test.md b/ocis-pkg/markdown/test.md new file mode 100755 index 0000000000..0060f03c2b --- /dev/null +++ b/ocis-pkg/markdown/test.md @@ -0,0 +1,56 @@ +--- +title: Antivirus Service +date: 2023-03-22T14:42:54.504370253+01:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/services/unknown +geekdocFilePath: _index.md +geekdocCollapseSection: true +--- + +## Abstract + +The `antivirus` service is responsible for scanning files for viruses. + +## Table of Contents + +* [Configuration](#configuration) + * [Antivirus Scanner Type](#antivirus-scanner-type) + * [Maximum Scan size](#maximum-scan-size) + * [Infected File Handling](#infected-file-handling) + * [Scanner Inaccessibility](#scanner-inaccessibility) +* [Operation Modes](#operation-modes) + * [Postprocessing](#postprocessing) + +## Configuration + +### Antivirus Scanner Type + +The antivirus service currently supports [ICAP](https://tools.ietf.org/html/rfc3507) and [ClamAV](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details. + - For `icap`, only scanners using the `X-Infection-Found` header are currently supported. + - For `clamav` only local sockets can currently be configured. + +### Maximum Scan size + +Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously, it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary. + +### Infected File Handling + +The antivirus service allows three different ways of handling infected files. Those can be set via the `ANTIVIRUS_INFECTED_FILE_HANDLING` environment variable: + - `delete`: (default): Infected files will be deleted immediately, further postprocessing is cancelled. + - `abort`: (advanced option): Infected files will be kept, further postprocessing is cancelled. Files can be manually retrieved and inspected by an admin. To identify the file for further investigation, the antivirus service logs the abort/infected state including the file ID. The file is located in the `storage/users/uploads` folder of the ocis data directory and persists until it is manually deleted by the admin via the [Manage Unfinished Uploads](https://doc.owncloud.com/ocis/next/deployment/services/s-list/storage-users.html#manage-unfinished-uploads) command. + - `continue`: (obviously not recommended): Infected files will be marked via metadata as infected but postprocessing continues normally. Note: Infected Files are moved to their final destination and therefore not prevented from download which includes the risk of spreading viruses. +In all cases, a log entry is added declaring the infection and handling method and a notification via the `userlog` service sent. + +### Scanner Inaccessibility + +In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessibility of the scanner used. + +## Operation Modes + +The antivirus service can scan files during `postprocessing`. `on demand` scanning is currently not available and might be added in a future release. + +### Postprocessing + +The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `virusscan`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details. +