mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2025-12-30 17:00:57 -06:00
Code Issues Packages Projects Releases Wiki Activity

enhancement: add mimetype to file extension rego function (#6133)

Browse Source 
* enhancement: add mimetype to file extension rego function

add rego function to detect the resource extension by mimetype, at the same time this pr introduces a custom ocis namespace for the rego functions.

* enhancement: add custom logPrinter to opa policies service

* fix: imports and test

TypeByExtension which is used to resolve extension by mimetype relies on MIME-info database which differs at my local env (macos <-> drone). This is fixed by using one of the builtinTypes for testing

---------

Signed-off-by: Christian Richter <crichter@owncloud.com>
Co-authored-by: Christian Richter <crichter@owncloud.com>
This commit is contained in:
Florian Schade
2023-07-10 16:28:23 +02:00
committed by GitHub
parent ecfe2d9a7b
commit c09f82405f
14 changed files with 303 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
changelog/unreleased/enhancement-add-postprocessing-mimetype-to-extension.md Normal file
View File

@@ -0,0 +1,6 @@
Enhancement: Add postprocessing mimetype to extension helper
Add rego helper to resolve extensions from mimetype `ocis.mimetype.extensions(mimetype)`.
Besides that, a rego print helper is included also `print("PRINT MESSAGE EXAMPLE")`
https://github.com/owncloud/ocis/pull/6133

9
deployments/examples/service_policies/policies/postprocessing.rego
View File

@@ -6,5 +6,12 @@ import data.utils
default granted := true
granted = false if {
not utils.collection_contains(utils.ALLOWED_FILE_EXTENSIONS, input.resource.name)
not utils.is_extension_allowed(input.resource.name)
}
granted = false if {
bytes := ocis.resource.download(input.resource.url)
mimetype := ocis.mimetype.detect(bytes)
not utils.is_mimetype_allowed(mimetype)
}

11
deployments/examples/service_policies/policies/proxy.rego
View File

@@ -6,13 +6,14 @@ import data.utils
default granted := true
granted = false if {
utils.is_request_type_put
not input.request.path == "/data"
not utils.collection_contains(utils.ALLOWED_FILE_EXTENSIONS, input.request.path)
print("PRINT MESSAGE EXAMPLE")
input.request.method == "PUT"
not startswith(input.request.path, "/ocs")
not utils.is_extension_allowed(input.request.path)
}
granted = false if {
utils.is_request_type_post
input.request.method == "POST"
startswith(input.request.path, "/remote.php")
not utils.collection_contains(utils.ALLOWED_FILE_EXTENSIONS, input.resource.name)
not utils.is_extension_allowed(input.resource.name)
}

47
deployments/examples/service_policies/policies/utils.rego
View File

@@ -1,8 +1,6 @@
package utils
import future.keywords.if
ALLOWED_FILE_EXTENSIONS := [
ALLOWED_RESOURCE_EXTENSIONS := [
".apk", ".avi", ".bat", ".bmp", ".css", ".csv", ".doc", ".docm", ".docx",
".docxf", ".dotx", ".eml", ".epub", ".htm", ".html", ".ipa", ".jar", ".java",
".jpg", ".js", ".json", ".mp3", ".mp4", ".msg", ".odp", ".ods", ".odt", ".oform",
@@ -11,43 +9,14 @@ ALLOWED_FILE_EXTENSIONS := [
".txt", ".xls", ".xlsm", ".xlsx", ".xltm", ".xltx", ".xml", ".zip", ".md"
]
##
is_stage_http {
input.stage == "http"
is_extension_allowed(identifier) {
extension := ALLOWED_RESOURCE_EXTENSIONS[_]
endswith(identifier, extension)
}
is_stage_pp {
input.stage == "pp"
}
##
is_user_admin {
input.user.username == "admin"
}
##
is_request_type_put {
is_stage_http
input.request.method == "PUT"
}
is_request_type_post {
is_stage_http
input.request.method == "POST"
}
is_request_type_mkcol {
is_stage_http
input.request.method == "MKCOL"
}
##
collection_contains(collection, source) {
current := collection[_]
endswith(source, current)
is_mimetype_allowed(mimetype) {
extensions := ocis.mimetype.extensions(mimetype)
extension := extensions[_]
is_extension_allowed(extension)
}

6
services/policies/pkg/command/server.go
View File

@@ -21,7 +21,7 @@ import (
svcProtogen "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/policies/v0"
"github.com/owncloud/ocis/v2/services/policies/pkg/config"
"github.com/owncloud/ocis/v2/services/policies/pkg/config/parser"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
svcEvent "github.com/owncloud/ocis/v2/services/policies/pkg/service/event"
svcGRPC "github.com/owncloud/ocis/v2/services/policies/pkg/service/grpc"
"github.com/urfave/cli/v2"
@@ -51,11 +51,11 @@ func Server(cfg *config.Config) *cli.Command {
log.Pretty(cfg.Log.Pretty),
log.Color(cfg.Log.Color),
log.File(cfg.Log.File),
)
).SubloggerWithRequestID(ctx)
)
defer cancel()
e, err := engine.NewOPA(cfg.Engine.Timeout, cfg.Engine)
e, err := opa.NewOPA(cfg.Engine.Timeout, logger, cfg.Engine)
if err != nil {
return err
}

124
services/policies/pkg/engine/opa.go
View File

@@ -1,124 +0,0 @@
package engine
import (
"bytes"
"context"
"fmt"
"net/http"
"strings"
"time"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/gabriel-vasile/mimetype"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/types"
"github.com/owncloud/ocis/v2/services/policies/pkg/config"
)
// OPA wraps open policy agent makes it possible to ask if an action is granted.
type OPA struct {
policies []string
timeout time.Duration
}
// NewOPA returns a ready to use opa engine.
func NewOPA(timeout time.Duration, conf config.Engine) (OPA, error) {
return OPA{
policies: conf.Policies,
timeout: timeout,
},
nil
}
// Evaluate evaluates the opa policies and returns the result.
func (o OPA) Evaluate(ctx context.Context, qs string, env Environment) (bool, error) {
ctx, cancel := context.WithTimeout(ctx, o.timeout)
defer cancel()
q, err := rego.New(
rego.Query(qs),
rego.Load(o.policies, nil),
GetMimetype,
GetResource,
).PrepareForEval(ctx)
if err != nil {
return false, err
}
result, err := q.Eval(ctx, rego.EvalInput(env))
if err != nil {
return false, err
}
return result.Allowed(), nil
}
var GetResource = rego.Function1(
&rego.Function{
Name: "ocis_get_resource",
Decl: types.NewFunction(types.Args(types.S), types.A),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var url string
if err := ast.As(a.Value, &url); err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
client := rhttp.GetHTTPClient(rhttp.Insecure(true))
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode)
}
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(res.Body); err != nil {
return nil, err
}
v, err := ast.InterfaceToValue(buf.Bytes())
if err != nil {
return nil, err
}
return ast.NewTerm(v), nil
},
)
var GetMimetype = rego.Function1(
&rego.Function{
Name: "ocis_get_mimetype",
Decl: types.NewFunction(types.Args(types.A), types.S),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var body []byte
if err := ast.As(a.Value, &body); err != nil {
return nil, err
}
mimeInfo := mimetype.Detect(body).String()
detectedMimetype := strings.Split(mimeInfo, ";")[0]
v, err := ast.InterfaceToValue(detectedMimetype)
if err != nil {
return nil, err
}
return ast.NewTerm(v), nil
},
)

61
services/policies/pkg/engine/opa/engine.go Normal file
View File

@@ -0,0 +1,61 @@
package opa
import (
"context"
"time"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/topdown/print"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/policies/pkg/config"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
)
// OPA wraps open policy agent makes it possible to ask if an action is granted.
type OPA struct {
printHook print.Hook
policies []string
timeout time.Duration
}
// NewOPA returns a ready to use opa engine.
func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) {
return OPA{
policies: conf.Policies,
timeout: timeout,
printHook: logPrinter{logger: logger},
},
nil
}
// Evaluate evaluates the opa policies and returns the result.
func (o OPA) Evaluate(ctx context.Context, qs string, env engine.Environment) (bool, error) {
ctx, cancel := context.WithTimeout(ctx, o.timeout)
defer cancel()
customFns := []func(r *rego.Rego){
RFResourceDownload,
RFMimetypeDetect,
RFMimetypeExtensions,
}
q, err := rego.New(
append([]func(r *rego.Rego){
rego.Query(qs),
rego.Load(o.policies, nil),
rego.EnablePrintStatements(true),
rego.PrintHook(o.printHook),
}, customFns...)...,
).PrepareForEval(ctx)
if err != nil {
return false, err
}
result, err := q.Eval(ctx, rego.EvalInput(env))
if err != nil {
return false, err
}
return result.Allowed(), nil
}

16
services/policies/pkg/engine/opa/opa.go Normal file
View File

@@ -0,0 +1,16 @@
package opa
import (
"github.com/open-policy-agent/opa/topdown/print"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
)
type logPrinter struct {
logger log.Logger
}
func (lp logPrinter) Print(_ print.Context, msg string) error {
lp.logger.Info().Msg(msg)
return nil
}

13
services/policies/pkg/engine/opa/opa_suite_test.go Normal file
View File

@@ -0,0 +1,13 @@
package opa_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestOpa(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Opa Suite")
}

59
services/policies/pkg/engine/opa/rf_mimetype.go Normal file
View File

@@ -0,0 +1,59 @@
package opa
import (
"mime"
"strings"
"github.com/gabriel-vasile/mimetype"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/types"
)
var RFMimetypeExtensions = rego.Function1(
&rego.Function{
Name: "ocis.mimetype.extensions",
Decl: types.NewFunction(types.Args(types.S), types.A),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var mt string
if err := ast.As(a.Value, &mt); err != nil {
return nil, err
}
detectedExtensions, err := mime.ExtensionsByType(mt)
if err != nil {
return nil, err
}
var mimeTerms []*ast.Term
for _, extension := range detectedExtensions {
mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension)))
}
return ast.ArrayTerm(mimeTerms...), nil
},
)
var RFMimetypeDetect = rego.Function1(
&rego.Function{
Name: "ocis.mimetype.detect",
Decl: types.NewFunction(types.Args(types.A), types.S),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var body []byte
if err := ast.As(a.Value, &body); err != nil {
return nil, err
}
mimetype := mimetype.Detect(body).String()
return ast.StringTerm(strings.Split(mimetype, ";")[0]), nil
},
)

31
services/policies/pkg/engine/opa/rf_mimetype_test.go Normal file
View File

@@ -0,0 +1,31 @@
package opa_test
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/open-policy-agent/opa/rego"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
)
var _ = Describe("opa ocis mimetype functions", func() {
Describe("ocis.mimetype.detect", func() {
It("detects the mimetype", func() {
r := rego.New(rego.Query(`ocis.mimetype.detect("")`), opa.RFMimetypeDetect)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
})
})
Describe("ocis.mimetype.extension_for_mimetype", func() {
It("provides matching extensions", func() {
r := rego.New(rego.Query(`ocis.mimetype.extensions("application/pdf")`), opa.RFMimetypeExtensions)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(rs[0].Expressions[0].String()).To(Equal("[.pdf]"))
})
})
})

56
services/policies/pkg/engine/opa/rf_resource.go Normal file
View File

@@ -0,0 +1,56 @@
package opa
import (
"bytes"
"fmt"
"net/http"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/types"
)
var RFResourceDownload = rego.Function1(
&rego.Function{
Name: "ocis.resource.download",
Decl: types.NewFunction(types.Args(types.S), types.A),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var url string
if err := ast.As(a.Value, &url); err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
client := rhttp.GetHTTPClient(rhttp.Insecure(true))
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode)
}
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(res.Body); err != nil {
return nil, err
}
v, err := ast.InterfaceToValue(buf.Bytes())
if err != nil {
return nil, err
}
return ast.NewTerm(v), nil
},
)

36
services/policies/pkg/engine/opa/rf_resource_test.go Normal file
View File

@@ -0,0 +1,36 @@
package opa_test
import (
"context"
"encoding/base64"
"net/http"
"net/http/httptest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/open-policy-agent/opa/rego"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
)
var _ = Describe("opa ocis resource functions", func() {
Describe("ocis.resource.download", func() {
It("downloads reva resources", func() {
ts := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(ts)
}))
defer srv.Close()
r := rego.New(rego.Query(`ocis.resource.download("`+srv.URL+`")`), opa.RFResourceDownload)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
data, err := base64.StdEncoding.DecodeString(rs[0].Expressions[0].String())
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(ts))
})
})
})

46
services/policies/pkg/engine/opa_test.go
View File

@@ -1,46 +0,0 @@
package engine_test
import (
"context"
"encoding/base64"
"net/http"
"net/http/httptest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/open-policy-agent/opa/rego"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
)
var _ = Describe("Opa", func() {
Describe("Custom OPA function", func() {
Describe("GetResource", func() {
It("loads reva resources", func() {
ts := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(ts)
}))
defer srv.Close()
r := rego.New(rego.Query(`ocis_get_resource("`+srv.URL+`")`), engine.GetResource)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
data, err := base64.StdEncoding.DecodeString(rs[0].Expressions[0].String())
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(ts))
})
})
Describe("GetMimetype", func() {
It("is defined and returns a mimetype", func() {
r := rego.New(rego.Query(`ocis_get_mimetype("")`), engine.GetMimetype)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
})
})
})
})