From 5b2253402e27fac4fb9b3241c070967845dcefa8 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Fri, 21 Jul 2023 15:45:41 +0200 Subject: [PATCH 1/6] use magicmime --- go.mod | 3 ++- go.sum | 2 ++ services/policies/pkg/engine/opa/rf_mimetype.go | 17 +++++++++++++---- vendor/modules.txt | 3 +++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 90a3d83b8..9a7e002c0 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 - github.com/gabriel-vasile/mimetype v1.4.2 github.com/ggwhite/go-masker v1.0.9 github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/cors v1.2.1 @@ -68,6 +67,7 @@ require ( github.com/pkg/xattr v0.4.9 github.com/prometheus/client_golang v1.16.0 github.com/r3labs/sse/v2 v2.10.0 + github.com/rakyll/magicmime v0.1.0 github.com/riandyrn/otelchi v0.5.1 github.com/rogpeppe/go-internal v1.10.0 github.com/rs/zerolog v1.29.1 @@ -176,6 +176,7 @@ require ( github.com/fatih/color v1.14.1 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gdexlab/go-render v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-acme/lego/v4 v4.4.0 // indirect diff --git a/go.sum b/go.sum index 2a921a5a0..2e55700c2 100644 --- a/go.sum +++ b/go.sum @@ -1477,6 +1477,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= +github.com/rakyll/magicmime v0.1.0 h1:aFIp1DqgzjcB3FI7rQk6uZl73i1VPpWswab1YKU4CL4= +github.com/rakyll/magicmime v0.1.0/go.mod h1:OKs4S+1GpIAB1PCebhwp3rxhyipe7TiImiIeVyFlQt8= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/services/policies/pkg/engine/opa/rf_mimetype.go b/services/policies/pkg/engine/opa/rf_mimetype.go index 033d664b6..8e317bb2c 100644 --- a/services/policies/pkg/engine/opa/rf_mimetype.go +++ b/services/policies/pkg/engine/opa/rf_mimetype.go @@ -1,13 +1,14 @@ package opa import ( + "log" "mime" "strings" - "github.com/gabriel-vasile/mimetype" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/types" + "github.com/rakyll/magicmime" ) var RFMimetypeExtensions = rego.Function1( @@ -23,7 +24,9 @@ var RFMimetypeExtensions = rego.Function1( if err := ast.As(a.Value, &mt); err != nil { return nil, err } - + if err := mime.AddExtensionType(".oform", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); err != nil { + return nil, err + } detectedExtensions, err := mime.ExtensionsByType(mt) if err != nil { return nil, err @@ -52,8 +55,14 @@ var RFMimetypeDetect = rego.Function1( return nil, err } - mimetype := mimetype.Detect(body).String() - + if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil { + log.Fatal(err) + } + defer magicmime.Close() + mimetype, err := magicmime.TypeByBuffer(body) + if err != nil { + return nil, err + } return ast.StringTerm(strings.Split(mimetype, ";")[0]), nil }, ) diff --git a/vendor/modules.txt b/vendor/modules.txt index d2d0f2675..a3bf4e648 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1565,6 +1565,9 @@ github.com/prometheus/statsd_exporter/pkg/mapper/fsm # github.com/r3labs/sse/v2 v2.10.0 ## explicit; go 1.13 github.com/r3labs/sse/v2 +# github.com/rakyll/magicmime v0.1.0 +## explicit +github.com/rakyll/magicmime # github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 ## explicit github.com/rcrowley/go-metrics From 4898a07239b86636c3cb3bfe927831aeec0ae14b Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Tue, 1 Aug 2023 19:36:01 +0200 Subject: [PATCH 2/6] enhancement: add config option to load an RFC 4288 mimes file --- ...dable-policy-mimetype-extension-mapping.md | 8 ++ go.mod | 3 +- go.sum | 2 - services/policies/README.md | 36 +++++-- services/policies/pkg/config/config.go | 2 + services/policies/pkg/engine/opa/engine.go | 37 ++++--- .../policies/pkg/engine/opa/rf_mimetype.go | 97 ++++++++++++------- .../pkg/engine/opa/rf_mimetype_test.go | 40 ++++++-- .../policies/pkg/engine/opa/rf_resource.go | 4 + vendor/modules.txt | 3 - 10 files changed, 166 insertions(+), 66 deletions(-) create mode 100644 changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md diff --git a/changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md b/changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md new file mode 100644 index 000000000..d33c4adeb --- /dev/null +++ b/changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md @@ -0,0 +1,8 @@ +Enhancement: Extendable policy mimetype extension mapping + +The extension mimetype mappings known from rego can now be extended. +To do this, ocis must be informed where the mimetype file (apache mime.types file format) is located. + +`export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types` + +https://github.com/owncloud/ocis/pull/6869 diff --git a/go.mod b/go.mod index 9a7e002c0..90a3d83b8 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 + github.com/gabriel-vasile/mimetype v1.4.2 github.com/ggwhite/go-masker v1.0.9 github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/cors v1.2.1 @@ -67,7 +68,6 @@ require ( github.com/pkg/xattr v0.4.9 github.com/prometheus/client_golang v1.16.0 github.com/r3labs/sse/v2 v2.10.0 - github.com/rakyll/magicmime v0.1.0 github.com/riandyrn/otelchi v0.5.1 github.com/rogpeppe/go-internal v1.10.0 github.com/rs/zerolog v1.29.1 @@ -176,7 +176,6 @@ require ( github.com/fatih/color v1.14.1 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gdexlab/go-render v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-acme/lego/v4 v4.4.0 // indirect diff --git a/go.sum b/go.sum index 2e55700c2..2a921a5a0 100644 --- a/go.sum +++ b/go.sum @@ -1477,8 +1477,6 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= -github.com/rakyll/magicmime v0.1.0 h1:aFIp1DqgzjcB3FI7rQk6uZl73i1VPpWswab1YKU4CL4= -github.com/rakyll/magicmime v0.1.0/go.mod h1:OKs4S+1GpIAB1PCebhwp3rxhyipe7TiImiIeVyFlQt8= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/services/policies/README.md b/services/policies/README.md index 4dccbf52d..5e43041f7 100644 --- a/services/policies/README.md +++ b/services/policies/README.md @@ -20,7 +20,7 @@ To configure the policies service, three environment variables need to be define Note that each query setting defines the [Complete Rules](https://www.openpolicyagent.org/docs/latest/#complete-rules) variable defined in the rego rule set the corresponding step uses for the evaluation. If the variable is mistyped or not found, the evaluation defaults to deny. Individual query definitions can be defined for each module. -To activate a the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny. +To activate the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny. When using async post-processing which is done via the postprocessing service, the value `policies` must be added to the `POSTPROCESSING_STEPS` configuration in postprocessing service in the order where the evaluation should take place. @@ -88,8 +88,8 @@ proxy: The same can be achieved by setting the following environment variable: -```yaml -PROXY_POLICIES_QUERY=data.proxy.granted +```shell +export PROXY_POLICIES_QUERY=data.proxy.granted ``` ### Postprocessing @@ -102,14 +102,14 @@ policies: The same can be achieved by setting the following environment variable: -```yaml -POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted +```shell +export POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted ``` As soon as that query is configured, the postprocessing service must be informed to use the policies step by setting the environment variable: -```yaml -POSTPROCESSING_STEPS=policies +```shell +export POSTPROCESSING_STEPS=policies ``` Note that additional steps can be configured and their position in the list defines the order of processing. For details see the postprocessing service documentation. @@ -118,6 +118,28 @@ Note that additional steps can be configured and their position in the list defi To identify available keys for OPA, you need to look at [engine.go](https://github.com/owncloud/ocis/blob/master/services/policies/pkg/engine/engine.go) and the [policies.swagger.json](https://github.com/owncloud/ocis/blob/master/protogen/gen/ocis/services/policies/v0/policies.swagger.json) file. Note that which keys are available depends on from which module it is used. +### Extend mimetype file extension mapping + +In rego it is possible to get a list of associated file extensions based on a mimetype, e.g. 'ocis.mimetype.extensions("application/pdf")'. + +The list of mappings is restricted by default and is provided by the host system. + +In order to extend this list, oCis must be provided with the path to a mime.types file. + +This can be done via yaml configuration or an environment variable. + +```shell +export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types +``` + +```yaml +policies: + engine: + mimes: OCIS_HOME/mime.types +``` + +A good example of how such a file should be formatted can be found in the [apache svn repository](https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types). + ## Example Policies The policies service contains a set of preconfigured example policies. See the [deployment examples](https://github.com/owncloud/ocis/tree/master/deployments/examples) directory for details. The contained policies disallow Infinite Scale to create certain file types, both via the proxy middleware and the events service via postprocessing. diff --git a/services/policies/pkg/config/config.go b/services/policies/pkg/config/config.go index 3eb9b05f0..9fb5df70f 100644 --- a/services/policies/pkg/config/config.go +++ b/services/policies/pkg/config/config.go @@ -45,6 +45,8 @@ type TokenManager struct { type Engine struct { Timeout time.Duration `yaml:"timeout" env:"POLICIES_ENGINE_TIMEOUT" desc:"Sets the timeout the rego expression evaluation can take. The timeout can be set as number followed by a unit identifier like ms, s, etc. Rules default to deny if the timeout was reached."` Policies []string `yaml:"policies"` + // Mimes file path, RFC 4288 + Mimes string `yaml:"mimes" env:"POLICIES_ENGINE_MIMES" desc:"Sets the mimes file path which maps media types to unique file extension(s)."` } // Postprocessing defines the config options for the postprocessing policy handling. diff --git a/services/policies/pkg/engine/opa/engine.go b/services/policies/pkg/engine/opa/engine.go index 45bd211bf..f152a6403 100644 --- a/services/policies/pkg/engine/opa/engine.go +++ b/services/policies/pkg/engine/opa/engine.go @@ -2,6 +2,8 @@ package opa import ( "context" + "io" + "os" "time" "github.com/open-policy-agent/opa/rego" @@ -17,16 +19,35 @@ type OPA struct { printHook print.Hook policies []string timeout time.Duration + options []func(r *rego.Rego) } // NewOPA returns a ready to use opa engine. func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) { + var mtReader io.Reader = nil + if conf.Mimes != "" { + mtReader, err := os.Open(conf.Mimes) + if err != nil { + return OPA{}, err + } + defer mtReader.Close() + } + + rfMimetypeExtensions, err := RFMimetypeExtensions(mtReader) + if err != nil { + return OPA{}, err + } + return OPA{ - policies: conf.Policies, - timeout: timeout, - printHook: logPrinter{logger: logger}, + policies: conf.Policies, + timeout: timeout, + printHook: logPrinter{logger: logger}, + options: []func(r *rego.Rego){ + RFMimetypeDetect, + RFResourceDownload, + rfMimetypeExtensions, }, - nil + }, nil } // Evaluate evaluates the opa policies and returns the result. @@ -34,19 +55,13 @@ func (o OPA) Evaluate(ctx context.Context, qs string, env engine.Environment) (b ctx, cancel := context.WithTimeout(ctx, o.timeout) defer cancel() - customFns := []func(r *rego.Rego){ - RFResourceDownload, - RFMimetypeDetect, - RFMimetypeExtensions, - } - q, err := rego.New( append([]func(r *rego.Rego){ rego.Query(qs), rego.Load(o.policies, nil), rego.EnablePrintStatements(true), rego.PrintHook(o.printHook), - }, customFns...)..., + }, o.options...)..., ).PrepareForEval(ctx) if err != nil { return false, err diff --git a/services/policies/pkg/engine/opa/rf_mimetype.go b/services/policies/pkg/engine/opa/rf_mimetype.go index 8e317bb2c..4fb5b02ad 100644 --- a/services/policies/pkg/engine/opa/rf_mimetype.go +++ b/services/policies/pkg/engine/opa/rf_mimetype.go @@ -1,46 +1,81 @@ package opa import ( - "log" + "bufio" + "io" "mime" "strings" + "github.com/gabriel-vasile/mimetype" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/types" - "github.com/rakyll/magicmime" ) -var RFMimetypeExtensions = rego.Function1( - ®o.Function{ - Name: "ocis.mimetype.extensions", - Decl: types.NewFunction(types.Args(types.S), types.A), - Memoize: true, - Nondeterministic: true, - }, - func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) { - var mt string +// RFMimetypeExtensions extends the rego dictionary with the possibility of mapping mimetypes to file extensions. +// Be careful calling this multiple times with individual readers, the mime store is global, +// which results in one global store which holds all known mimetype mappings at once. +// +// Rego: `ocis.mimetype.extensions("application/pdf")` +// Result `[.pdf]` +func RFMimetypeExtensions(f io.Reader) (func(*rego.Rego), error) { + if f != nil { + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) <= 1 || fields[0][0] == '#' { + continue + } + mimeType := fields[0] + for _, ext := range fields[1:] { + if ext[0] == '#' { + break + } + if err := mime.AddExtensionType("."+ext, mimeType); err != nil { + return nil, err + } - if err := ast.As(a.Value, &mt); err != nil { - return nil, err + } } - if err := mime.AddExtensionType(".oform", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); err != nil { - return nil, err - } - detectedExtensions, err := mime.ExtensionsByType(mt) - if err != nil { - return nil, err + if err := scanner.Err(); err != nil { + panic(err) } + } - var mimeTerms []*ast.Term - for _, extension := range detectedExtensions { - mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension))) - } + return rego.Function1( + ®o.Function{ + Name: "ocis.mimetype.extensions", + Decl: types.NewFunction(types.Args(types.S), types.A), + Memoize: true, + Nondeterministic: true, + }, + func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) { + var mt string - return ast.ArrayTerm(mimeTerms...), nil - }, -) + if err := ast.As(a.Value, &mt); err != nil { + return nil, err + } + detectedExtensions, err := mime.ExtensionsByType(mt) + if err != nil { + return nil, err + } + + var mimeTerms []*ast.Term + for _, extension := range detectedExtensions { + mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension))) + } + + return ast.ArrayTerm(mimeTerms...), nil + }, + ), nil +} + +// RFMimetypeDetect extends the rego dictionary with the possibility to detect mimetypes. +// Be careful, the list of known mimetypes is limited. +// +// Rego: `ocis.mimetype.extensions(".txt")` +// Result `text/plain` var RFMimetypeDetect = rego.Function1( ®o.Function{ Name: "ocis.mimetype.detect", @@ -55,14 +90,8 @@ var RFMimetypeDetect = rego.Function1( return nil, err } - if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil { - log.Fatal(err) - } - defer magicmime.Close() - mimetype, err := magicmime.TypeByBuffer(body) - if err != nil { - return nil, err - } + mimetype := mimetype.Detect(body).String() + return ast.StringTerm(strings.Split(mimetype, ";")[0]), nil }, ) diff --git a/services/policies/pkg/engine/opa/rf_mimetype_test.go b/services/policies/pkg/engine/opa/rf_mimetype_test.go index 404dc2b45..d2f692deb 100644 --- a/services/policies/pkg/engine/opa/rf_mimetype_test.go +++ b/services/policies/pkg/engine/opa/rf_mimetype_test.go @@ -2,6 +2,8 @@ package opa_test import ( "context" + "io" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -19,13 +21,37 @@ var _ = Describe("opa ocis mimetype functions", func() { Expect(rs[0].Expressions[0].String()).To(Equal("text/plain")) }) }) + Describe("ocis.mimetype.extensions", func() { + DescribeTable("resolves extensions by mimetype", + func(mimetype string, expectations []string, f io.Reader) { + rfMimetypeExtensions, err := opa.RFMimetypeExtensions(f) + Expect(err).ToNot(HaveOccurred()) - Describe("ocis.mimetype.extension_for_mimetype", func() { - It("provides matching extensions", func() { - r := rego.New(rego.Query(`ocis.mimetype.extensions("application/pdf")`), opa.RFMimetypeExtensions) - rs, err := r.Eval(context.Background()) - Expect(err).ToNot(HaveOccurred()) - Expect(rs[0].Expressions[0].String()).To(Equal("[.pdf]")) - }) + r := rego.New(rego.Query(`ocis.mimetype.extensions("`+mimetype+`")`), rfMimetypeExtensions) + rs, err := r.Eval(context.Background()) + Expect(err).ToNot(HaveOccurred()) + + got := rs[0].Expressions[0].String() + + if len(expectations) == 0 { + Expect(got).To(Equal("[]")) + } + + for i, expectation := range expectations { + if i+1 != len(expectations) { + expectation += " " + } + + Expect(string(got[0])).To(Equal("[")) + Expect(strings.Contains(got, expectation)).To(BeTrue()) + Expect(string(got[len(got)-1])).To(Equal("]")) + } + }, + Entry("With default mimetype", "application/pdf", []string{".pdf"}, nil), + Entry("With unknown mimetype", "ocis/with.custom.mt", []string{}, nil), + Entry("With custom mimetype", "ocis/with.custom.mt", []string{".with.custom.mt"}, strings.NewReader("ocis/with.custom.mt with.custom.mt")), + Entry("With multiple custom mimetypes", "ocis/with.multiple.custom.mt", []string{".with.multiple.custom.1.mt", ".with.multiple.custom.2.mt"}, strings.NewReader("ocis/with.multiple.custom.mt with.multiple.custom.1.mt with.multiple.custom.2.mt")), + Entry("With custom ignored mimetype", "ocis/with.multiple.custom.ignored.mt", []string{}, strings.NewReader("#ocis/with.multiple.custom.ignored.mt with.multiple.custom.ignored.mt")), + ) }) }) diff --git a/services/policies/pkg/engine/opa/rf_resource.go b/services/policies/pkg/engine/opa/rf_resource.go index 60543c0b8..9ff1d0ba1 100644 --- a/services/policies/pkg/engine/opa/rf_resource.go +++ b/services/policies/pkg/engine/opa/rf_resource.go @@ -11,6 +11,10 @@ import ( "github.com/open-policy-agent/opa/types" ) +// RFResourceDownload extends the rego dictionary with the possibility to download oCis resources. +// +// Rego: `ocis.resource.download("ocis/path/0034892347349827")` +// Result: bytes var RFResourceDownload = rego.Function1( ®o.Function{ Name: "ocis.resource.download", diff --git a/vendor/modules.txt b/vendor/modules.txt index a3bf4e648..d2d0f2675 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1565,9 +1565,6 @@ github.com/prometheus/statsd_exporter/pkg/mapper/fsm # github.com/r3labs/sse/v2 v2.10.0 ## explicit; go 1.13 github.com/r3labs/sse/v2 -# github.com/rakyll/magicmime v0.1.0 -## explicit -github.com/rakyll/magicmime # github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 ## explicit github.com/rcrowley/go-metrics From 9638dd524c1bd74bbce0948ef1a2949d3ffbad37 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Thu, 3 Aug 2023 13:13:38 +0200 Subject: [PATCH 3/6] fix: review feedback --- services/policies/pkg/engine/opa/engine.go | 3 ++- services/policies/pkg/engine/opa/rf_mimetype.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/policies/pkg/engine/opa/engine.go b/services/policies/pkg/engine/opa/engine.go index f152a6403..b4107ba18 100644 --- a/services/policies/pkg/engine/opa/engine.go +++ b/services/policies/pkg/engine/opa/engine.go @@ -24,7 +24,8 @@ type OPA struct { // NewOPA returns a ready to use opa engine. func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) { - var mtReader io.Reader = nil + var mtReader io.Reader + if conf.Mimes != "" { mtReader, err := os.Open(conf.Mimes) if err != nil { diff --git a/services/policies/pkg/engine/opa/rf_mimetype.go b/services/policies/pkg/engine/opa/rf_mimetype.go index 4fb5b02ad..f3b7b300f 100644 --- a/services/policies/pkg/engine/opa/rf_mimetype.go +++ b/services/policies/pkg/engine/opa/rf_mimetype.go @@ -38,7 +38,7 @@ func RFMimetypeExtensions(f io.Reader) (func(*rego.Rego), error) { } } if err := scanner.Err(); err != nil { - panic(err) + return nil, err } } From 06613d3c6bbe634bfaf0258bcdf7e99c3ae6cc5b Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Thu, 3 Aug 2023 14:13:17 +0200 Subject: [PATCH 4/6] fix: variable shadowing --- services/policies/pkg/engine/opa/engine.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/policies/pkg/engine/opa/engine.go b/services/policies/pkg/engine/opa/engine.go index b4107ba18..21c07cdb4 100644 --- a/services/policies/pkg/engine/opa/engine.go +++ b/services/policies/pkg/engine/opa/engine.go @@ -24,13 +24,15 @@ type OPA struct { // NewOPA returns a ready to use opa engine. func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) { - var mtReader io.Reader + var mtReader io.ReadCloser if conf.Mimes != "" { - mtReader, err := os.Open(conf.Mimes) + var err error + mtReader, err = os.Open(conf.Mimes) if err != nil { return OPA{}, err } + defer mtReader.Close() } From ce5d6cd603e910c58b5dd2b096c437950192d414 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Thu, 3 Aug 2023 14:23:09 +0200 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: Martin --- services/policies/README.md | 18 ++++++++++-------- services/policies/pkg/config/config.go | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/services/policies/README.md b/services/policies/README.md index 5e43041f7..f5ef63162 100644 --- a/services/policies/README.md +++ b/services/policies/README.md @@ -118,27 +118,29 @@ Note that additional steps can be configured and their position in the list defi To identify available keys for OPA, you need to look at [engine.go](https://github.com/owncloud/ocis/blob/master/services/policies/pkg/engine/engine.go) and the [policies.swagger.json](https://github.com/owncloud/ocis/blob/master/protogen/gen/ocis/services/policies/v0/policies.swagger.json) file. Note that which keys are available depends on from which module it is used. -### Extend mimetype file extension mapping +## Extend Mimetype File Extension Mapping -In rego it is possible to get a list of associated file extensions based on a mimetype, e.g. 'ocis.mimetype.extensions("application/pdf")'. +In the extended set of the rego query language, it is possible to get a list of associated file extensions based on a mimetype, for example `ocis.mimetype.extensions("application/pdf")`. -The list of mappings is restricted by default and is provided by the host system. +The list of mappings is restricted by default and is provided by the host system ocis is installed on. -In order to extend this list, oCis must be provided with the path to a mime.types file. +In order to extend this list, ocis must be provided with the path to a custom `mime.types` file that maps mimetypes to extensions. +The location for the file must be accessible by all instances of the policy service. As a rule of thumb, use the directory where the ocis configuration files are stored. +Note that existing mappings from the host are extended by the definitions from the mime types file, but not replaced. -This can be done via yaml configuration or an environment variable. +The path to that file can be provided via a yaml configuration or an environment variable. Note to replace the `OCIS_CONFIG_DIR` string by an existing path. ```shell -export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types +export OCIS_MACHINE_AUTH_API_KEY=OCIS_CONFIG_DIR/mime.types ``` ```yaml policies: engine: - mimes: OCIS_HOME/mime.types + mimes: OCIS_CONFIG_DIR/mime.types ``` -A good example of how such a file should be formatted can be found in the [apache svn repository](https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types). +A good example of how such a file should be formatted can be found in the [Apache svn repository](https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types). ## Example Policies diff --git a/services/policies/pkg/config/config.go b/services/policies/pkg/config/config.go index 9fb5df70f..11011376c 100644 --- a/services/policies/pkg/config/config.go +++ b/services/policies/pkg/config/config.go @@ -46,7 +46,7 @@ type Engine struct { Timeout time.Duration `yaml:"timeout" env:"POLICIES_ENGINE_TIMEOUT" desc:"Sets the timeout the rego expression evaluation can take. The timeout can be set as number followed by a unit identifier like ms, s, etc. Rules default to deny if the timeout was reached."` Policies []string `yaml:"policies"` // Mimes file path, RFC 4288 - Mimes string `yaml:"mimes" env:"POLICIES_ENGINE_MIMES" desc:"Sets the mimes file path which maps media types to unique file extension(s)."` + Mimes string `yaml:"mimes" env:"POLICIES_ENGINE_MIMES" desc:"Sets the mimes file path which maps mimetypes to associated file extensions. See the text description for details."` } // Postprocessing defines the config options for the postprocessing policy handling. From cc529d99da409367a74b5558cbda71d5f406c5cc Mon Sep 17 00:00:00 2001 From: kobergj Date: Thu, 3 Aug 2023 15:30:02 +0200 Subject: [PATCH 6/6] Update services/policies/README.md --- services/policies/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/policies/README.md b/services/policies/README.md index f5ef63162..067c3312a 100644 --- a/services/policies/README.md +++ b/services/policies/README.md @@ -20,7 +20,7 @@ To configure the policies service, three environment variables need to be define Note that each query setting defines the [Complete Rules](https://www.openpolicyagent.org/docs/latest/#complete-rules) variable defined in the rego rule set the corresponding step uses for the evaluation. If the variable is mistyped or not found, the evaluation defaults to deny. Individual query definitions can be defined for each module. -To activate the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny. +To activate the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file is not present or accessible, the evaluation defaults to deny. When using async post-processing which is done via the postprocessing service, the value `policies` must be added to the `POSTPROCESSING_STEPS` configuration in postprocessing service in the order where the evaluation should take place.