Merge pull request #6869 from owncloud/magic-mime

use magicmime

changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md

@@ -0,0 +1,8 @@
+Enhancement: Extendable policy mimetype extension mapping
+
+The extension mimetype mappings known from rego can now be extended.
+To do this, ocis must be informed where the mimetype file (apache mime.types file format) is located.
+
+`export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types`
+
+https://github.com/owncloud/ocis/pull/6869

services/policies/README.md

@@ -20,7 +20,7 @@ To configure the policies service, three environment variables need to be define
 
 Note that each query setting defines the [Complete Rules](https://www.openpolicyagent.org/docs/latest/#complete-rules) variable defined in the rego rule set the corresponding step uses for the evaluation. If the variable is mistyped or not found, the evaluation defaults to deny. Individual query definitions can be defined for each module.
 
-To activate a the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny.
+To activate the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file is not present or accessible, the evaluation defaults to deny.
 
 When using async post-processing which is done via the postprocessing service, the value `policies` must be added to the `POSTPROCESSING_STEPS` configuration in postprocessing service in the order where the evaluation should take place.
 
@@ -88,8 +88,8 @@ proxy:
 
 The same can be achieved by setting the following environment variable:
 
-```yaml
-PROXY_POLICIES_QUERY=data.proxy.granted
+```shell
+export PROXY_POLICIES_QUERY=data.proxy.granted
 ```
 
 ### Postprocessing
@@ -102,14 +102,14 @@ policies:
 
 The same can be achieved by setting the following environment variable:
 
-```yaml
-POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted
+```shell
+export POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted
 ```
 
 As soon as that query is configured, the postprocessing service must be informed to use the policies step by setting the environment variable:
 
-```yaml
-POSTPROCESSING_STEPS=policies
+```shell
+export POSTPROCESSING_STEPS=policies
 ```
 
 Note that additional steps can be configured and their position in the list defines the order of processing. For details see the postprocessing service documentation.
@@ -118,6 +118,30 @@ Note that additional steps can be configured and their position in the list defi
 
 To identify available keys for OPA, you need to look at [engine.go](https://github.com/owncloud/ocis/blob/master/services/policies/pkg/engine/engine.go) and the [policies.swagger.json](https://github.com/owncloud/ocis/blob/master/protogen/gen/ocis/services/policies/v0/policies.swagger.json) file. Note that which keys are available depends on from which module it is used.
 
+## Extend Mimetype File Extension Mapping
+
+In the extended set of the rego query language, it is possible to get a list of associated file extensions based on a mimetype, for example `ocis.mimetype.extensions("application/pdf")`.
+
+The list of mappings is restricted by default and is provided by the host system ocis is installed on.
+
+In order to extend this list, ocis must be provided with the path to a custom `mime.types` file that maps mimetypes to extensions.
+The location for the file must be accessible by all instances of the policy service. As a rule of thumb, use the directory where the ocis configuration files are stored.
+Note that existing mappings from the host are extended by the definitions from the mime types file, but not replaced.
+
+The path to that file can be provided via a yaml configuration or an environment variable. Note to replace the `OCIS_CONFIG_DIR` string by an existing path.
+
+```shell
+export OCIS_MACHINE_AUTH_API_KEY=OCIS_CONFIG_DIR/mime.types
+```
+
+```yaml
+policies:
+  engine:
+    mimes: OCIS_CONFIG_DIR/mime.types
+```
+
+A good example of how such a file should be formatted can be found in the [Apache svn repository](https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types).
+
 ## Example Policies
 
 The policies service contains a set of preconfigured example policies. See the [deployment examples](https://github.com/owncloud/ocis/tree/master/deployments/examples) directory for details. The contained policies disallow Infinite Scale to create certain file types, both via the proxy middleware and the events service via postprocessing.

services/policies/pkg/config/config.go

@@ -45,6 +45,8 @@ type TokenManager struct {
 type Engine struct {
 	Timeout  time.Duration `yaml:"timeout" env:"POLICIES_ENGINE_TIMEOUT" desc:"Sets the timeout the rego expression evaluation can take. The timeout can be set as number followed by a unit identifier like ms, s, etc. Rules default to deny if the timeout was reached."`
 	Policies []string      `yaml:"policies"`
+	// Mimes file path, RFC 4288
+	Mimes string `yaml:"mimes" env:"POLICIES_ENGINE_MIMES" desc:"Sets the mimes file path which maps mimetypes to associated file extensions. See the text description for details."`
 }
 
 // Postprocessing defines the config options for the postprocessing policy handling.

services/policies/pkg/engine/opa/engine.go

@@ -2,6 +2,8 @@ package opa
 
 import (
 	"context"
+	"io"
+	"os"
 	"time"
 
 	"github.com/open-policy-agent/opa/rego"
@@ -17,16 +19,38 @@ type OPA struct {
 	printHook print.Hook
 	policies  []string
 	timeout   time.Duration
+	options   []func(r *rego.Rego)
 }
 
 // NewOPA returns a ready to use opa engine.
 func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) {
+	var mtReader io.ReadCloser
+
+	if conf.Mimes != "" {
+		var err error
+		mtReader, err = os.Open(conf.Mimes)
+		if err != nil {
+			return OPA{}, err
+		}
+
+		defer mtReader.Close()
+	}
+
+	rfMimetypeExtensions, err := RFMimetypeExtensions(mtReader)
+	if err != nil {
+		return OPA{}, err
+	}
+
 	return OPA{
-			policies:  conf.Policies,
-			timeout:   timeout,
-			printHook: logPrinter{logger: logger},
+		policies:  conf.Policies,
+		timeout:   timeout,
+		printHook: logPrinter{logger: logger},
+		options: []func(r *rego.Rego){
+			RFMimetypeDetect,
+			RFResourceDownload,
+			rfMimetypeExtensions,
 		},
-		nil
+	}, nil
 }
 
 // Evaluate evaluates the opa policies and returns the result.
@@ -34,19 +58,13 @@ func (o OPA) Evaluate(ctx context.Context, qs string, env engine.Environment) (b
 	ctx, cancel := context.WithTimeout(ctx, o.timeout)
 	defer cancel()
 
-	customFns := []func(r *rego.Rego){
-		RFResourceDownload,
-		RFMimetypeDetect,
-		RFMimetypeExtensions,
-	}
-
 	q, err := rego.New(
 		append([]func(r *rego.Rego){
 			rego.Query(qs),
 			rego.Load(o.policies, nil),
 			rego.EnablePrintStatements(true),
 			rego.PrintHook(o.printHook),
-		}, customFns...)...,
+		}, o.options...)...,
 	).PrepareForEval(ctx)
 	if err != nil {
 		return false, err

services/policies/pkg/engine/opa/rf_mimetype.go

@@ -1,6 +1,8 @@
 package opa
 
 import (
+	"bufio"
+	"io"
 	"mime"
 	"strings"
 
@@ -10,34 +12,70 @@ import (
 	"github.com/open-policy-agent/opa/types"
 )
 
-var RFMimetypeExtensions = rego.Function1(
-	&rego.Function{
-		Name:             "ocis.mimetype.extensions",
-		Decl:             types.NewFunction(types.Args(types.S), types.A),
-		Memoize:          true,
-		Nondeterministic: true,
-	},
-	func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
-		var mt string
+// RFMimetypeExtensions extends the rego dictionary with the possibility of mapping mimetypes to file extensions.
+// Be careful calling this multiple times with individual readers, the mime store is global,
+// which results in one global store which holds all known mimetype mappings at once.
+//
+// Rego: `ocis.mimetype.extensions("application/pdf")`
+// Result `[.pdf]`
+func RFMimetypeExtensions(f io.Reader) (func(*rego.Rego), error) {
+	if f != nil {
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			fields := strings.Fields(scanner.Text())
+			if len(fields) <= 1 || fields[0][0] == '#' {
+				continue
+			}
+			mimeType := fields[0]
+			for _, ext := range fields[1:] {
+				if ext[0] == '#' {
+					break
+				}
+				if err := mime.AddExtensionType("."+ext, mimeType); err != nil {
+					return nil, err
+				}
 
-		if err := ast.As(a.Value, &mt); err != nil {
+			}
+		}
+		if err := scanner.Err(); err != nil {
 			return nil, err
 		}
+	}
 
-		detectedExtensions, err := mime.ExtensionsByType(mt)
-		if err != nil {
-			return nil, err
-		}
+	return rego.Function1(
+		&rego.Function{
+			Name:             "ocis.mimetype.extensions",
+			Decl:             types.NewFunction(types.Args(types.S), types.A),
+			Memoize:          true,
+			Nondeterministic: true,
+		},
+		func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
+			var mt string
 
-		var mimeTerms []*ast.Term
-		for _, extension := range detectedExtensions {
-			mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension)))
-		}
+			if err := ast.As(a.Value, &mt); err != nil {
+				return nil, err
+			}
 
-		return ast.ArrayTerm(mimeTerms...), nil
-	},
-)
+			detectedExtensions, err := mime.ExtensionsByType(mt)
+			if err != nil {
+				return nil, err
+			}
 
+			var mimeTerms []*ast.Term
+			for _, extension := range detectedExtensions {
+				mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension)))
+			}
+
+			return ast.ArrayTerm(mimeTerms...), nil
+		},
+	), nil
+}
+
+// RFMimetypeDetect extends the rego dictionary with the possibility to detect mimetypes.
+// Be careful, the list of known mimetypes is limited.
+//
+// Rego: `ocis.mimetype.extensions(".txt")`
+// Result `text/plain`
 var RFMimetypeDetect = rego.Function1(
 	&rego.Function{
 		Name:             "ocis.mimetype.detect",

services/policies/pkg/engine/opa/rf_mimetype_test.go

@@ -2,6 +2,8 @@ package opa_test
 
 import (
 	"context"
+	"io"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -19,13 +21,37 @@ var _ = Describe("opa ocis mimetype functions", func() {
 			Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
 		})
 	})
+	Describe("ocis.mimetype.extensions", func() {
+		DescribeTable("resolves extensions by mimetype",
+			func(mimetype string, expectations []string, f io.Reader) {
+				rfMimetypeExtensions, err := opa.RFMimetypeExtensions(f)
+				Expect(err).ToNot(HaveOccurred())
 
-	Describe("ocis.mimetype.extension_for_mimetype", func() {
-		It("provides matching extensions", func() {
-			r := rego.New(rego.Query(`ocis.mimetype.extensions("application/pdf")`), opa.RFMimetypeExtensions)
-			rs, err := r.Eval(context.Background())
-			Expect(err).ToNot(HaveOccurred())
-			Expect(rs[0].Expressions[0].String()).To(Equal("[.pdf]"))
-		})
+				r := rego.New(rego.Query(`ocis.mimetype.extensions("`+mimetype+`")`), rfMimetypeExtensions)
+				rs, err := r.Eval(context.Background())
+				Expect(err).ToNot(HaveOccurred())
+
+				got := rs[0].Expressions[0].String()
+
+				if len(expectations) == 0 {
+					Expect(got).To(Equal("[]"))
+				}
+
+				for i, expectation := range expectations {
+					if i+1 != len(expectations) {
+						expectation += " "
+					}
+
+					Expect(string(got[0])).To(Equal("["))
+					Expect(strings.Contains(got, expectation)).To(BeTrue())
+					Expect(string(got[len(got)-1])).To(Equal("]"))
+				}
+			},
+			Entry("With default mimetype", "application/pdf", []string{".pdf"}, nil),
+			Entry("With unknown mimetype", "ocis/with.custom.mt", []string{}, nil),
+			Entry("With custom mimetype", "ocis/with.custom.mt", []string{".with.custom.mt"}, strings.NewReader("ocis/with.custom.mt    with.custom.mt")),
+			Entry("With multiple custom mimetypes", "ocis/with.multiple.custom.mt", []string{".with.multiple.custom.1.mt", ".with.multiple.custom.2.mt"}, strings.NewReader("ocis/with.multiple.custom.mt                                with.multiple.custom.1.mt with.multiple.custom.2.mt")),
+			Entry("With custom ignored mimetype", "ocis/with.multiple.custom.ignored.mt", []string{}, strings.NewReader("#ocis/with.multiple.custom.ignored.mt with.multiple.custom.ignored.mt")),
+		)
 	})
 })

services/policies/pkg/engine/opa/rf_resource.go

@@ -11,6 +11,10 @@ import (
 	"github.com/open-policy-agent/opa/types"
 )
 
+// RFResourceDownload extends the rego dictionary with the possibility to download oCis resources.
+//
+// Rego: `ocis.resource.download("ocis/path/0034892347349827")`
+// Result: bytes
 var RFResourceDownload = rego.Function1(
 	&rego.Function{
 		Name:             "ocis.resource.download",