mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-05-13 06:49:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

simplify markdown pkg

Browse Source 
Signed-off-by: jkoberg <jkoberg@owncloud.com>
This commit is contained in:
jkoberg
2023-03-22 17:24:04 +01:00
parent cf8120a70d
commit 8df0d08fac
4 changed files with 92 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/helpers/markdowncreation.go
+2 -2
View File
@@ -60,8 +60,8 @@ func generateMarkdown(filepath string, servicename string) error {
"CreationTime": time.Now().Format(time.RFC3339Nano),
"service": servicename,
"Abstract": head.Content,
"TocTree": string(md.Toc()),
"Content": string(md.Bytes()),
"TocTree": md.TocString(),
"Content": md.String(),
}); err != nil {
return err
}
ocis-pkg/markdown/markdown.go
+89 -67
View File
@@ -4,17 +4,15 @@ package markdown
import (
"bytes"
"fmt"
"html/template"
"os"
"io"
"strings"
"time"
)
// Heading represents a markdown Heading
type Heading struct {
Content string
Level int
Header string
Level int // the level of the heading. 1 means it's the H1
Content string // the text of the heading
Header string // the heading itself
}
// MD represents a markdown file
@@ -22,25 +20,66 @@ type MD struct {
Headings []Heading
}
// Bytes returns the markdown as []bytes to be written to a file
// Bytes returns the markdown as []bytes, ignoring errors
func (md MD) Bytes() []byte {
b, num := bytes.NewBuffer(nil), len(md.Headings)
for i, h := range md.Headings {
b.Write([]byte(strings.Repeat("#", h.Level) + " " + h.Header + "\n"))
b.Write([]byte("\n"))
if len(h.Content) > 0 {
b.Write([]byte(h.Content))
if i < num-1 {
b.Write([]byte("\n"))
}
}
}
var b bytes.Buffer
_, _ = md.WriteContent(&b)
return b.Bytes()
}
// Toc returns the table of contents as []byte
func (md MD) Toc() []byte {
b := bytes.NewBuffer(nil)
// String returns the markdown as string, ignoring errors
func (md MD) String() string {
var b strings.Builder
_, _ = md.WriteContent(&b)
return b.String()
}
// TocBytes returns the table of contents as []byte, ignoring errors
func (md MD) TocBytes() []byte {
var b bytes.Buffer
_, _ = md.WriteToc(&b)
return b.Bytes()
}
// TocString returns the table of contents as string, ignoring errors
func (md MD) TocString() string {
var b strings.Builder
_, _ = md.WriteToc(&b)
return b.String()
}
// WriteContent writes the MDs content to the given writer
func (md MD) WriteContent(w io.Writer) (int64, error) {
max, written := len(md.Headings), int64(0)
write := func(s string) error {
n, err := w.Write([]byte(s))
written += int64(n)
return err
}
for i, h := range md.Headings {
if err := write(strings.Repeat("#", h.Level) + " " + h.Header + "\n"); err != nil {
return written, err
}
if err := write("\n"); err != nil {
return written, err
}
if len(h.Content) > 0 {
if err := write(h.Content); err != nil {
return written, err
}
if i < max-1 {
if err := write("\n"); err != nil {
return written, err
}
}
}
}
return written, nil
}
// WriteToc writes the table of contents to the given writer
func (md MD) WriteToc(w io.Writer) (int64, error) {
var written int64
for _, h := range md.Headings {
if h.Level == 1 {
// main title not in toc
@@ -48,68 +87,51 @@ func (md MD) Toc() []byte {
}
link := fmt.Sprintf("#%s", strings.ToLower(strings.Replace(h.Header, " ", "-", -1)))
s := fmt.Sprintf("%s* [%s](%s)\n", strings.Repeat(" ", h.Level-2), h.Header, link)
b.Write([]byte(s))
n, err := w.Write([]byte(s))
if err != nil {
return written, err
}
written += int64(n)
}
return b.Bytes()
return written, nil
}
// NewMD parses a new Markdown
func NewMD(b []byte) MD {
md := MD{}
var heading Heading
var (
md MD
heading Heading
content strings.Builder
)
sendHeading := func() {
if heading.Header != "" {
heading.Content = content.String()
md.Headings = append(md.Headings, heading)
content = strings.Builder{}
}
}
parts := strings.Split(string(b), "\n")
for _, p := range parts {
if p == "" {
continue
}
if p[:1] == "#" { // this is a header
if heading.Header != "" {
md.Headings = append(md.Headings, heading)
}
heading = Heading{}
i := strings.LastIndex(p, "#")
levs, con := p[:i+1], p[i+1:]
heading.Header = strings.TrimPrefix(con, " ")
heading.Level = len(levs)
sendHeading()
heading = headingFromString(p)
} else {
heading.Content += p + "\n"
// readd lost "\n"
_, _ = content.WriteString(p + "\n")
}
}
if heading.Header != "" {
md.Headings = append(md.Headings, heading)
}
sendHeading()
return md
}
func main() {
f, err := os.ReadFile("/home/jkoberg/ocis/services/antivirus/README.md")
if err != nil {
fmt.Println("ERROR", err)
return
}
md := NewMD(f)
head := md.Headings[0]
md.Headings = md.Headings[1:]
tpl := template.Must(template.ParseFiles("index.tmpl"))
b := bytes.NewBuffer(nil)
err = tpl.Execute(b, map[string]interface{}{
"ServiceName": head.Header,
"CreationTime": time.Now().Format(time.RFC3339Nano),
"service": "unknown",
"Abstract": head.Content,
"TocTree": string(md.Toc()),
"Content": string(md.Bytes()),
})
if err != nil {
fmt.Println("ERROR", err)
return
}
err = os.WriteFile("test.md", b.Bytes(), os.ModePerm)
if err != nil {
fmt.Println("ERROR", err)
return
func headingFromString(s string) Heading {
i := strings.LastIndex(s, "#")
levs, con := s[:i+1], s[i+1:]
return Heading{
Level: len(levs),
Header: strings.TrimPrefix(con, " "),
}
}
ocis-pkg/markdown/markdown_test.go
+1 -1
View File
@@ -41,7 +41,7 @@ var _ = Describe("TestMarkdown", func() {
for i, h := range md.Headings {
Expect(h).To(Equal(expectedMD.Headings[i]))
}
Expect(string(md.Bytes())).To(Equal(mdfile))
Expect(md.String()).To(Equal(mdfile))
},
Entry("converts a small markdown", SmallMarkdown, SmallMD),
)
ocis-pkg/markdown/test.md
-56
View File
@@ -1,56 +0,0 @@
---
title: Antivirus Service
date: 2023-03-22T14:42:54.504370253&#43;01:00
weight: 20
geekdocRepo: https://github.com/owncloud/ocis
geekdocEditPath: edit/master/docs/services/unknown
geekdocFilePath: _index.md
geekdocCollapseSection: true
---
## Abstract
The `antivirus` service is responsible for scanning files for viruses.
## Table of Contents
* [Configuration](#configuration)
* [Antivirus Scanner Type](#antivirus-scanner-type)
* [Maximum Scan size](#maximum-scan-size)
* [Infected File Handling](#infected-file-handling)
* [Scanner Inaccessibility](#scanner-inaccessibility)
* [Operation Modes](#operation-modes)
* [Postprocessing](#postprocessing)
## Configuration
### Antivirus Scanner Type
The antivirus service currently supports [ICAP](https://tools.ietf.org/html/rfc3507) and [ClamAV](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details.
- For `icap`, only scanners using the `X-Infection-Found` header are currently supported.
- For `clamav` only local sockets can currently be configured.
### Maximum Scan size
Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously, it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary.
### Infected File Handling
The antivirus service allows three different ways of handling infected files. Those can be set via the `ANTIVIRUS_INFECTED_FILE_HANDLING` environment variable:
- `delete`: (default): Infected files will be deleted immediately, further postprocessing is cancelled.
- `abort`: (advanced option): Infected files will be kept, further postprocessing is cancelled. Files can be manually retrieved and inspected by an admin. To identify the file for further investigation, the antivirus service logs the abort/infected state including the file ID. The file is located in the `storage/users/uploads` folder of the ocis data directory and persists until it is manually deleted by the admin via the [Manage Unfinished Uploads](https://doc.owncloud.com/ocis/next/deployment/services/s-list/storage-users.html#manage-unfinished-uploads) command.
- `continue`: (obviously not recommended): Infected files will be marked via metadata as infected but postprocessing continues normally. Note: Infected Files are moved to their final destination and therefore not prevented from download which includes the risk of spreading viruses.
In all cases, a log entry is added declaring the infection and handling method and a notification via the `userlog` service sent.
### Scanner Inaccessibility
In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessibility of the scanner used.
## Operation Modes
The antivirus service can scan files during `postprocessing`. `on demand` scanning is currently not available and might be added in a future release.
### Postprocessing
The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `virusscan`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details.