feat: add maximum image dimension to be processed by the thumbnailer (#9035)

* feat: add maximum image dimension to be processed by the thumbnailer

* chore: make golangci-lint happy
This commit is contained in:
Thomas Müller
2024-05-03 12:20:27 +02:00
committed by GitHub
parent a1e4da239f
commit 9db3fd028e
31 changed files with 219 additions and 244 deletions

View File

@@ -0,0 +1,5 @@
Change: Define maximum input image dimensions when generating previews
This is a general hardening change to limit processing time and resources of the thumbnailer.
https://github.com/owncloud/ocis/pull/9035

View File

@@ -26,10 +26,10 @@ func Server(cfg *config.Config) *cli.Command {
Name: "server",
Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
Category: "server",
Before: func(c *cli.Context) error {
Before: func(_ *cli.Context) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
Action: func(c *cli.Context) error {
Action: func(_ *cli.Context) error {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
traceProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name)
@@ -49,12 +49,12 @@ func Server(cfg *config.Config) *cli.Command {
}
return context.WithCancel(cfg.Context)
}()
metrics = metrics.New()
m = metrics.New()
)
defer cancel()
metrics.BuildInfo.WithLabelValues(version.GetString()).Set(1)
m.BuildInfo.WithLabelValues(version.GetString()).Set(1)
service := grpc.NewService(
grpc.Logger(logger),
@@ -63,7 +63,7 @@ func Server(cfg *config.Config) *cli.Command {
grpc.Name(cfg.Service.Name),
grpc.Namespace(cfg.GRPC.Namespace),
grpc.Address(cfg.GRPC.Addr),
grpc.Metrics(metrics),
grpc.Metrics(m),
grpc.TraceProvider(traceProvider),
)
@@ -95,7 +95,7 @@ func Server(cfg *config.Config) *cli.Command {
http.Logger(logger),
http.Context(ctx),
http.Config(cfg),
http.Metrics(metrics),
http.Metrics(m),
http.Namespace(cfg.HTTP.Namespace),
http.TraceProvider(traceProvider),
)

View File

@@ -18,7 +18,7 @@ func Version(cfg *config.Config) *cli.Command {
Name: "version",
Usage: "print the version of this binary and the running service instances",
Category: "info",
Action: func(c *cli.Context) error {
Action: func(_ *cli.Context) error {
fmt.Println("Version: " + version.GetString())
fmt.Printf("Compiled: %s\n", version.Compiled())
fmt.Println("")

View File

@@ -44,4 +44,6 @@ type Thumbnail struct {
FontMapFile string `yaml:"font_map_file" env:"THUMBNAILS_TXT_FONTMAP_FILE" desc:"The path to a font file for txt thumbnails." introductionVersion:"pre5.0"`
TransferSecret string `yaml:"transfer_secret" env:"THUMBNAILS_TRANSFER_TOKEN" desc:"The secret to sign JWT to download the actual thumbnail file." introductionVersion:"pre5.0"`
DataEndpoint string `yaml:"data_endpoint" env:"THUMBNAILS_DATA_ENDPOINT" desc:"The HTTP endpoint where the actual thumbnail file can be downloaded." introductionVersion:"pre5.0"`
MaxInputWidth int `yaml:"max_input_width" env:"THUMBNAILS_MAX_INPUT_WIDTH" desc:"The maximum width of an input image which is being processed." introductionVersion:"6.0"`
MaxInputHeight int `yaml:"max_input_height" env:"THUMBNAILS_MAX_INPUT_HEIGHT" desc:"The maximum height of an input image which is being processed." introductionVersion:"6.0"`
}

View File

@@ -48,6 +48,8 @@ func DefaultConfig() *config.Config {
RevaGateway: shared.DefaultRevaConfig().Address,
CS3AllowInsecure: false,
DataEndpoint: "http://127.0.0.1:9186/thumbnails/data",
MaxInputWidth: 7680,
MaxInputHeight: 4320,
},
}
}

View File

@@ -33,6 +33,7 @@ func ParseConfig(cfg *config.Config) error {
return Validate(cfg)
}
func Validate(cfg *config.Config) error {
// Validate can validate the configuration
func Validate(_ *config.Config) error {
return nil
}

View File

@@ -0,0 +1,18 @@
package errors
import "errors"
var (
// ErrImageTooLarge defines an error when an input image is too large
ErrImageTooLarge = errors.New("thumbnails: image is too large")
// ErrInvalidType represents the error when a type can't be encoded.
ErrInvalidType = errors.New("thumbnails: can't encode this type")
// ErrNoEncoderForType represents the error when an encoder couldn't be found for a type.
ErrNoEncoderForType = errors.New("thumbnails: no encoder for this type found")
// ErrNoImageFromAudioFile defines an error when an image cannot be extracted from an audio file
ErrNoImageFromAudioFile = errors.New("thumbnails: could not extract image from audio file")
// ErrNoConverterForExtractedImageFromAudioFile defines an error when the extracted image from an audio file could not be converted
ErrNoConverterForExtractedImageFromAudioFile = errors.New("thumbnails: could not find converter for image extracted from audio file")
// ErrCS3AuthorizationMissing defines an error when the CS3 authorization is missing
ErrCS3AuthorizationMissing = errors.New("thumbnails: cs3source - authorization missing")
)

View File

@@ -25,28 +25,28 @@ type FontMap struct {
DefaultFont string `json:"defaultFont"`
}
// It contains the location of the loaded file (in FLoc) and the FontMap loaded
// FontMapData contains the location of the loaded file (in FLoc) and the FontMap loaded
// from the file
type FontMapData struct {
FMap *FontMap
FLoc string
}
// It contains the location of the font used, and the loaded face (font.Face)
// LoadedFace contains the location of the font used, and the loaded face (font.Face)
// ready to be used
type LoadedFace struct {
FontFile string
Face font.Face
}
// Represents a FontLoader. Use the "NewFontLoader" to get a instance
// FontLoader represents a FontLoader. Use the "NewFontLoader" to get a instance
type FontLoader struct {
faceCache sync.Cache
fontMapData *FontMapData
faceOpts *opentype.FaceOptions
}
// Create a new FontLoader based on the fontMapFile. The FaceOptions will
// NewFontLoader creates a new FontLoader based on the fontMapFile. The FaceOptions will
// be the same for all the font loaded by this instance.
// Note that only the fonts described in the fontMapFile will be used.
//
@@ -92,7 +92,7 @@ func NewFontLoader(fontMapFile string, faceOpts *opentype.FaceOptions) (*FontLoa
}, nil
}
// Load and return the font face to be used for that script according to the
// LoadFaceForScript loads and returns the font face to be used for that script according to the
// FontMap set when the FontLoader was created. If the script doesn't have
// an associated font, a default font will be used. Note that the default font
// might not be able to handle properly the script
@@ -146,14 +146,17 @@ func (fl *FontLoader) LoadFaceForScript(script string) (*LoadedFace, error) {
return loadedFace, nil
}
// GetFaceOptSize returns face opt size
func (fl *FontLoader) GetFaceOptSize() float64 {
return fl.faceOpts.Size
}
// GetFaceOptDPI returns face opt DPI
func (fl *FontLoader) GetFaceOptDPI() float64 {
return fl.faceOpts.DPI
}
// GetScriptList returns script list
func (fl *FontLoader) GetScriptList() []string {
fontMap := fl.fontMapData.FMap.FontMap

View File

@@ -19,6 +19,7 @@ import (
"golang.org/x/image/math/fixed"
"github.com/dhowden/tag"
thumbnailerErrors "github.com/owncloud/ocis/v2/services/thumbnails/pkg/errors"
)
// FileConverter is the interface for the file converter
@@ -98,12 +99,12 @@ func (i AudioDecoder) Convert(r io.Reader) (interface{}, error) {
picture := m.Picture()
if picture == nil {
return nil, errors.New(`could not extract image from audio file`)
return nil, thumbnailerErrors.ErrNoImageFromAudioFile
}
converter := ForType(picture.MIMEType, nil)
if converter == nil {
return nil, errors.New(`could not find converter for image extraced from audio file`)
return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromAudioFile
}
return converter.Convert(bytes.NewReader(picture.Data))
@@ -259,7 +260,7 @@ func ForType(mimeType string, opts map[string]interface{}) FileConverter {
fontLoader, err := NewFontLoader(fontFileMap, fontFaceOpts)
if err != nil {
// if couldn't create the FontLoader with the specified fontFileMap,
// if it couldn't create the FontLoader with the specified fontFileMap,
// try to use the default font
fontLoader, _ = NewFontLoader("", fontFaceOpts)
}

View File

@@ -27,8 +27,8 @@ func Server(opts ...Option) (*http.Server, error) {
}
// health implements the health check.
func health(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
func health(_ *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
@@ -43,8 +43,8 @@ func health(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
}
// ready implements the ready check.
func ready(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
func ready(_ *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)

View File

@@ -8,6 +8,7 @@ import (
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/metrics"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
)
// Option defines a single option function.
@@ -76,7 +77,7 @@ func TraceProvider(traceProvider trace.TracerProvider) Option {
if traceProvider != nil {
o.TraceProvider = traceProvider
} else {
o.TraceProvider = trace.NewNoopTracerProvider()
o.TraceProvider = noop.NewTracerProvider()
}
}
}

View File

@@ -6,7 +6,7 @@ import (
thumbnailssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/thumbnails/v0"
)
// Interface acting as facade, holding all the interfaces that this
// DecoratedService is an interface acting as facade, holding all the interfaces that this
// thumbnails microservice is expecting to implement.
// For now, only the thumbnailssvc.ThumbnailServiceHandler is present,
// but a future configsvc.ConfigServiceHandler is expected to be added here
@@ -17,7 +17,7 @@ type DecoratedService interface {
thumbnailssvc.ThumbnailServiceHandler
}
// Base type to implement the decorators. It will provide a basic implementation
// Decorator is the base type to implement the decorators. It will provide a basic implementation
// by delegating to the decoratedService
//
// Expected implementations will be like:
@@ -43,7 +43,7 @@ type Decorator struct {
next DecoratedService
}
// Base implementation for the GetThumbnail (for the thumbnailssvc).
// GetThumbnail is the base implementation for the thumbnailssvc.GetThumbnail.
// It will just delegate to the underlying decoratedService
//
// Your custom decorator is expected to overwrite this function,

View File

@@ -34,11 +34,11 @@ func (l logging) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbna
Logger()
if err != nil {
merror := merrors.FromError(err)
switch merror.Code {
fromError := merrors.FromError(err)
switch fromError.GetCode() {
case http.StatusNotFound:
logger.Debug().
Str("error_detail", merror.Detail).
Str("error_detail", fromError.GetDetail()).
Msg("no thumbnail found")
default:
logger.Warn().

View File

@@ -35,10 +35,10 @@ func (t tracing) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbna
defer span.End()
span.SetAttributes(
attribute.KeyValue{Key: "filepath", Value: attribute.StringValue(req.Filepath)},
attribute.KeyValue{Key: "thumbnail_type", Value: attribute.StringValue(req.ThumbnailType.String())},
attribute.KeyValue{Key: "width", Value: attribute.IntValue(int(req.Width))},
attribute.KeyValue{Key: "height", Value: attribute.IntValue(int(req.Height))},
attribute.KeyValue{Key: "filepath", Value: attribute.StringValue(req.GetFilepath())},
attribute.KeyValue{Key: "thumbnail_type", Value: attribute.StringValue(req.GetThumbnailType().String())},
attribute.KeyValue{Key: "width", Value: attribute.IntValue(int(req.GetWidth()))},
attribute.KeyValue{Key: "height", Value: attribute.IntValue(int(req.GetHeight()))},
)
}

View File

@@ -50,13 +50,6 @@ func Config(val *config.Config) Option {
}
}
// Middleware provides a function to set the middleware option.
func Middleware(val ...func(http.Handler) http.Handler) Option {
return func(o *Options) {
o.Middleware = val
}
}
// ThumbnailStorage provides a function to set the thumbnail storage option.
func ThumbnailStorage(val storage.Storage) Option {
return func(o *Options) {

View File

@@ -43,6 +43,8 @@ func NewService(opts ...Option) decorators.DecoratedService {
resolutions,
options.ThumbnailStorage,
logger,
options.Config.Thumbnail.MaxInputWidth,
options.Config.Thumbnail.MaxInputHeight,
),
webdavSource: options.ImageSource,
cs3Source: options.CS3Source,
@@ -116,7 +118,7 @@ func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumb
func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
src := req.GetCs3Source()
sRes, err := g.stat(src.Path, src.Authorization)
sRes, err := g.stat(src.GetPath(), src.GetAuthorization())
if err != nil {
return "", err
}
@@ -125,7 +127,7 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh
if tType == "" {
tType = req.GetThumbnailType().String()
}
tr, err := thumbnail.PrepareRequest(int(req.Width), int(req.Height), tType, sRes.GetInfo().GetChecksum().GetSum(), req.Processor)
tr, err := thumbnail.PrepareRequest(int(req.GetWidth()), int(req.GetHeight()), tType, sRes.GetInfo().GetChecksum().GetSum(), req.GetProcessor())
if err != nil {
return "", merrors.BadRequest(g.serviceID, err.Error())
}
@@ -134,12 +136,12 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh
return key, nil
}
ctx = imgsource.ContextSetAuthorization(ctx, src.Authorization)
r, err := g.cs3Source.Get(ctx, src.Path)
ctx = imgsource.ContextSetAuthorization(ctx, src.GetAuthorization())
r, err := g.cs3Source.Get(ctx, src.GetPath())
if err != nil {
return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
}
defer r.Close() // nolint:errcheck
defer r.Close()
ppOpts := map[string]interface{}{
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
}
@@ -158,13 +160,13 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh
func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
src := req.GetWebdavSource()
imgURL, err := url.Parse(src.Url)
imgURL, err := url.Parse(src.GetUrl())
if err != nil {
return "", errors.Wrap(err, "source url is invalid")
}
var auth, statPath string
if src.IsPublicLink {
if src.GetIsPublicLink() {
q := imgURL.Query()
var rsp *gateway.AuthenticateResponse
client, err := g.selector.Next()
@@ -177,13 +179,13 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge
exp := q.Get("expiration")
rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
Type: "publicshares",
ClientId: src.PublicLinkToken,
ClientId: src.GetPublicLinkToken(),
ClientSecret: strings.Join([]string{"signature", sig, exp}, "|"),
})
} else {
rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
Type: "publicshares",
ClientId: src.PublicLinkToken,
ClientId: src.GetPublicLinkToken(),
// We pass an empty password because we expect non pre-signed public links
// to not be password protected
ClientSecret: "password|",
@@ -193,11 +195,11 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge
if err != nil {
return "", merrors.InternalServerError(g.serviceID, "could not authenticate: %s", err.Error())
}
auth = rsp.Token
statPath = path.Join("/public", src.PublicLinkToken, req.Filepath)
auth = rsp.GetToken()
statPath = path.Join("/public", src.GetPublicLinkToken(), req.GetFilepath())
} else {
auth = src.RevaAuthorization
statPath = req.Filepath
auth = src.GetRevaAuthorization()
statPath = req.GetFilepath()
}
sRes, err := g.stat(statPath, auth)
if err != nil {
@@ -208,7 +210,7 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge
if tType == "" {
tType = req.GetThumbnailType().String()
}
tr, err := thumbnail.PrepareRequest(int(req.Width), int(req.Height), tType, sRes.GetInfo().GetChecksum().GetSum(), req.Processor)
tr, err := thumbnail.PrepareRequest(int(req.GetWidth()), int(req.GetHeight()), tType, sRes.GetInfo().GetChecksum().GetSum(), req.GetProcessor())
if err != nil {
return "", merrors.BadRequest(g.serviceID, err.Error())
}
@@ -217,8 +219,8 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge
return key, nil
}
if src.WebdavAuthorization != "" {
ctx = imgsource.ContextSetAuthorization(ctx, src.WebdavAuthorization)
if src.GetWebdavAuthorization() != "" {
ctx = imgsource.ContextSetAuthorization(ctx, src.GetWebdavAuthorization())
}
// add signature and expiration to webdav url
@@ -232,7 +234,7 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge
if err != nil {
return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
}
defer r.Close() // nolint:errcheck
defer r.Close()
ppOpts := map[string]interface{}{
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
}
@@ -272,16 +274,16 @@ func (g Thumbnail) stat(path, auth string) (*provider.StatResponse, error) {
return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", err.Error())
}
if rsp.Status.Code != rpc.Code_CODE_OK {
switch rsp.Status.Code {
if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
switch rsp.GetStatus().GetCode() {
case rpc.Code_CODE_NOT_FOUND:
return nil, merrors.NotFound(g.serviceID, "could not stat file: %s", rsp.Status.Message)
return nil, merrors.NotFound(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
default:
g.logger.Error().Str("status_message", rsp.Status.Message).Str("path", path).Msg("could not stat file")
return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", rsp.Status.Message)
g.logger.Error().Str("status_message", rsp.GetStatus().GetMessage()).Str("path", path).Msg("could not stat file")
return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
}
}
if rsp.Info.Type != provider.ResourceType_RESOURCE_TYPE_FILE {
if rsp.GetInfo().GetType() != provider.ResourceType_RESOURCE_TYPE_FILE {
return nil, merrors.BadRequest(g.serviceID, "Unsupported file type")
}
if utils.ReadPlainFromOpaque(rsp.GetInfo().GetOpaque(), "status") == "processing" {
@@ -292,11 +294,11 @@ func (g Thumbnail) stat(path, auth string) (*provider.StatResponse, error) {
Status: http.StatusText(http.StatusTooEarly),
}
}
if rsp.Info.GetChecksum().GetSum() == "" {
if rsp.GetInfo().GetChecksum().GetSum() == "" {
g.logger.Error().Msg("resource info is missing checksum")
return nil, merrors.NotFound(g.serviceID, "resource info is missing a checksum")
}
if !thumbnail.IsMimeTypeSupported(rsp.Info.MimeType) {
if !thumbnail.IsMimeTypeSupported(rsp.GetInfo().GetMimeType()) {
return nil, merrors.NotFound(g.serviceID, "Unsupported file type")
}
return rsp, nil

View File

@@ -59,14 +59,3 @@ func ThumbnailStorage(storage storage.Storage) Option {
o.ThumbnailStorage = storage
}
}
// TraceProvider provides a function to configure the trace provider
func TraceProvider(traceProvider trace.TracerProvider) Option {
return func(o *Options) {
if traceProvider != nil {
o.TraceProvider = traceProvider
} else {
o.TraceProvider = trace.NewNoopTracerProvider()
}
}
}

View File

@@ -25,8 +25,8 @@ const (
// Service defines the service handlers.
type Service interface {
ServeHTTP(http.ResponseWriter, *http.Request)
GetThumbnail(http.ResponseWriter, *http.Request)
ServeHTTP(w http.ResponseWriter, r *http.Request)
GetThumbnail(w http.ResponseWriter, r *http.Request)
}
// NewService returns a service implementation for Service.
@@ -58,6 +58,8 @@ func NewService(opts ...Option) Service {
resolutions,
options.ThumbnailStorage,
logger,
options.Config.Thumbnail.MaxInputWidth,
options.Config.Thumbnail.MaxInputHeight,
),
}
@@ -66,7 +68,7 @@ func NewService(opts ...Option) Service {
r.Get("/data", svc.GetThumbnail)
})
_ = chi.Walk(m, func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error {
_ = chi.Walk(m, func(method string, route string, _ http.Handler, middlewares ...func(http.Handler) http.Handler) error {
options.Logger.Debug().Str("method", method).Str("route", route).Int("middlewares", len(middlewares)).Msg("serving endpoint")
return nil
})
@@ -92,7 +94,7 @@ func (s Thumbnails) GetThumbnail(w http.ResponseWriter, r *http.Request) {
logger := s.logger.SubloggerWithRequestID(r.Context())
key := r.Context().Value(keyContextKey).(string)
thumbnail, err := s.manager.GetThumbnail(key)
thumbnailBytes, err := s.manager.GetThumbnail(key)
if err != nil {
logger.Debug().
Err(err).
@@ -103,8 +105,8 @@ func (s Thumbnails) GetThumbnail(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Length", strconv.Itoa(len(thumbnail)))
if _, err = w.Write(thumbnail); err != nil {
w.Header().Set("Content-Length", strconv.Itoa(len(thumbnailBytes)))
if _, err = w.Write(thumbnailBytes); err != nil {
logger.Error().
Err(err).
Str("key", key).
@@ -112,6 +114,7 @@ func (s Thumbnails) GetThumbnail(w http.ResponseWriter, r *http.Request) {
}
}
// TransferTokenValidator validates a transfer token
func (s Thumbnails) TransferTokenValidator(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
logger := s.logger.SubloggerWithRequestID(r.Context())

View File

@@ -2,6 +2,7 @@ package jwt
import "github.com/golang-jwt/jwt/v4"
// ThumbnailClaims defines the claims for thumb-nailing
type ThumbnailClaims struct {
jwt.RegisteredClaims
Key string `json:"key"`

View File

@@ -1,13 +1,14 @@
package thumbnail
import (
"errors"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"strings"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/errors"
)
const (
@@ -18,17 +19,10 @@ const (
typeGgs = "ggs"
)
var (
// ErrInvalidType represents the error when a type can't be encoded.
ErrInvalidType = errors.New("can't encode this type")
// ErrNoEncoderForType represents the error when an encoder couldn't be found for a type.
ErrNoEncoderForType = errors.New("no encoder for this type found")
)
// Encoder encodes the thumbnail to a specific format.
type Encoder interface {
// Encode encodes the image to a format.
Encode(io.Writer, interface{}) error
Encode(w io.Writer, img interface{}) error
// Types returns the formats suffixes.
Types() []string
// MimeType returns the mimetype used by the encoder.
@@ -42,7 +36,7 @@ type PngEncoder struct{}
func (e PngEncoder) Encode(w io.Writer, img interface{}) error {
m, ok := img.(image.Image)
if !ok {
return ErrInvalidType
return errors.ErrInvalidType
}
return png.Encode(w, m)
}
@@ -57,14 +51,14 @@ func (e PngEncoder) MimeType() string {
return "image/png"
}
// JpegEncoder encodes to jpg.
// JpegEncoder encodes to jpg
type JpegEncoder struct{}
// Encode encodes to jpg
func (e JpegEncoder) Encode(w io.Writer, img interface{}) error {
m, ok := img.(image.Image)
if !ok {
return ErrInvalidType
return errors.ErrInvalidType
}
return jpeg.Encode(w, m, nil)
}
@@ -79,17 +73,19 @@ func (e JpegEncoder) MimeType() string {
return "image/jpeg"
}
// GifEncoder encodes to gif
type GifEncoder struct{}
// Encode encodes the image to a gif format
func (e GifEncoder) Encode(w io.Writer, img interface{}) error {
g, ok := img.(*gif.GIF)
if !ok {
return ErrInvalidType
return errors.ErrInvalidType
}
return gif.EncodeAll(w, g)
}
// Types returns the supported types of the GifEncoder
func (e GifEncoder) Types() []string {
return []string{typeGif}
}
@@ -110,7 +106,7 @@ func EncoderForType(fileType string) (Encoder, error) {
case typeGif:
return GifEncoder{}, nil
default:
return nil, ErrNoEncoderForType
return nil, errors.ErrNoEncoderForType
}
}

View File

@@ -8,11 +8,12 @@ import (
"strings"
"github.com/kovidgoyal/imaging"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/errors"
)
// Generator generates a web friendly file version.
type Generator interface {
Generate(image.Rectangle, interface{}, Processor) (interface{}, error)
Generate(size image.Rectangle, img interface{}, processor Processor) (interface{}, error)
}
// SimpleGenerator is the default image generator and is used for all image types expect gif.
@@ -22,7 +23,7 @@ type SimpleGenerator struct{}
func (g SimpleGenerator) Generate(size image.Rectangle, img interface{}, processor Processor) (interface{}, error) {
m, ok := img.(image.Image)
if !ok {
return nil, ErrInvalidType
return nil, errors.ErrInvalidType
}
return processor.Process(m, size.Dx(), size.Dy(), imaging.Lanczos), nil
@@ -37,7 +38,7 @@ func (g GifGenerator) Generate(size image.Rectangle, img interface{}, processor
m, ok := img.(*gif.GIF)
if !ok {
return nil, ErrInvalidType
return nil, errors.ErrInvalidType
}
// Create a new RGBA image to hold the incremental frames.
srcX, srcY := m.Config.Width, m.Config.Height
@@ -80,6 +81,6 @@ func GeneratorForType(fileType string) (Generator, error) {
case typeGif:
return GifGenerator{}, nil
default:
return nil, ErrNoEncoderForType
return nil, errors.ErrNoEncoderForType
}
}

View File

@@ -15,13 +15,13 @@ import (
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/config"
"github.com/pkg/errors"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/errors"
"google.golang.org/grpc/metadata"
)
const (
// "github.com/cs3org/reva/v2/internal/http/services/datagateway" is internal so we redeclare it here
// TokenTransportHeader holds the header key for the reva transfer token
// "github.com/cs3org/reva/v2/internal/http/services/datagateway" is internal so we redeclare it here
TokenTransportHeader = "X-Reva-Transfer"
)
@@ -44,7 +44,7 @@ func NewCS3Source(cfg config.Thumbnail, gatewaySelector pool.Selectable[gateway.
func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
auth, ok := ContextGetAuthorization(ctx)
if !ok {
return nil, errors.New("cs3source: authorization missing")
return nil, errors.ErrCS3AuthorizationMissing
}
ref, err := storagespace.ParseReference(path)
if err != nil {
@@ -66,18 +66,18 @@ func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
return nil, err
}
if rsp.Status.Code != rpc.Code_CODE_OK {
return nil, fmt.Errorf("could not load image: %s", rsp.Status.Message)
if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
return nil, fmt.Errorf("could not load image: %s", rsp.GetStatus().GetMessage())
}
var ep, tk string
for _, p := range rsp.Protocols {
if p.Protocol == "spaces" {
ep, tk = p.DownloadEndpoint, p.Token
for _, p := range rsp.GetProtocols() {
if p.GetProtocol() == "spaces" {
ep, tk = p.GetDownloadEndpoint(), p.GetToken()
break
}
}
if (ep == "" || tk == "") && len(rsp.Protocols) > 0 {
ep, tk = rsp.Protocols[0].DownloadEndpoint, rsp.Protocols[0].Token
if (ep == "" || tk == "") && len(rsp.GetProtocols()) > 0 {
ep, tk = rsp.GetProtocols()[0].GetDownloadEndpoint(), rsp.GetProtocols()[0].GetToken()
}
httpReq, err := rhttp.NewRequest(ctx, "GET", ep, nil)
@@ -93,7 +93,7 @@ func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
}
client := &http.Client{}
resp, err := client.Do(httpReq) // nolint:bodyclose
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}

View File

@@ -44,7 +44,7 @@ func (s WebDav) Get(ctx context.Context, url string) (io.ReadCloser, error) {
}
client := &http.Client{}
resp, err := client.Do(req) // nolint:bodyclose
resp, err := client.Do(req)
if err != nil {
return nil, errors.Wrapf(err, `could not get the image "%s"`, url)
}

View File

@@ -31,6 +31,7 @@ type FileSystem struct {
logger log.Logger
}
// Stat returns if a file for the given key exists on the filesystem
func (s FileSystem) Stat(key string) bool {
img := filepath.Join(s.root, filesDir, key)
if _, err := os.Stat(img); err != nil {
@@ -39,6 +40,7 @@ func (s FileSystem) Stat(key string) bool {
return true
}
// Get returns the file content for the given key
func (s FileSystem) Get(key string) ([]byte, error) {
img := filepath.Join(s.root, filesDir, key)
content, err := os.ReadFile(img)
@@ -51,6 +53,7 @@ func (s FileSystem) Get(key string) ([]byte, error) {
return content, nil
}
// Put stores image data in the file system for the given key
func (s FileSystem) Put(key string, img []byte) error {
imgPath := filepath.Join(s.root, filesDir, key)
dir := filepath.Dir(imgPath)

View File

@@ -1,50 +0,0 @@
package storage
import (
"strings"
)
// NewInMemoryStorage creates a new InMemory instance.
func NewInMemoryStorage() InMemory {
return InMemory{
store: make(map[string][]byte),
}
}
// InMemory represents an in memory storage for thumbnails
// Can be used during development
type InMemory struct {
store map[string][]byte
}
func (s InMemory) Stat(key string) bool {
_, exists := s.store[key]
return exists
}
// Get loads the thumbnail from memory.
func (s InMemory) Get(key string) ([]byte, error) {
return s.store[key], nil
}
// Set stores the thumbnail in memory.
func (s InMemory) Put(key string, thumbnail []byte) error {
s.store[key] = thumbnail
return nil
}
// BuildKey generates a unique key to store and retrieve the thumbnail.
func (s InMemory) BuildKey(r Request) string {
parts := []string{
r.Checksum,
r.Resolution.String(),
}
if r.Characteristic != "" {
parts = append(parts, r.Characteristic)
}
parts = append(parts, strings.Join(r.Types, ","))
return strings.Join(parts, "+")
}

View File

@@ -1,65 +0,0 @@
package storage_test
import (
"image"
"testing"
tAssert "github.com/stretchr/testify/assert"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/thumbnail/storage"
)
func TestInMemory_BuildKey(t *testing.T) {
tests := []struct {
r storage.Request
want string
}{
{
r: storage.Request{
Checksum: "cs",
Types: []string{"png", "jpg"},
Resolution: image.Rectangle{
Min: image.Point{
X: 1,
Y: 2,
},
Max: image.Point{
X: 3,
Y: 4,
},
},
Characteristic: "",
},
want: "cs+(1,2)-(3,4)+png,jpg",
},
{
r: storage.Request{
Checksum: "cs",
Types: []string{"png", "jpg"},
Resolution: image.Rectangle{
Min: image.Point{
X: 1,
Y: 2,
},
Max: image.Point{
X: 3,
Y: 4,
},
},
Characteristic: "fill",
},
want: "cs+(1,2)-(3,4)+fill+png,jpg",
},
}
s := storage.InMemory{}
assert := tAssert.New(t)
for _, tt := range tests {
tt := tt
t.Run("", func(t *testing.T) {
assert.Equal(s.BuildKey(tt.r), tt.want)
})
}
}

View File

@@ -16,7 +16,7 @@ type Request struct {
// The resolution of the thumbnail
Resolution image.Rectangle
// Characteristic defines the different image characteristics,
// for example, if its scaled up to fit in the bounding box or not,
// for example, if it's scaled up to fit in the bounding box or not,
// is it a chroma version of the image, and so on...
// the main propose for this is to be able to differentiate between images which have
// the same resolution but different characteristics.
@@ -25,8 +25,8 @@ type Request struct {
// Storage defines the interface for a thumbnail store.
type Storage interface {
Stat(string) bool
Get(string) ([]byte, error)
Put(string, []byte) error
BuildKey(Request) string
Stat(key string) bool
Get(key string) ([]byte, error)
Put(key string, img []byte) error
BuildKey(r Request) string
}

View File

@@ -7,11 +7,12 @@ import (
"mime"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/errors"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/thumbnail/storage"
)
var (
// SupportedMimeTypes contains a all mimetypes which are supported by the thumbnailer.
// SupportedMimeTypes contains an all mimetypes which are supported by the thumbnailer.
SupportedMimeTypes = map[string]struct{}{
"image/png": {},
"image/jpg": {},
@@ -28,7 +29,7 @@ var (
}
)
// Request bundles information needed to generate a thumbnail for afile
// Request bundles information needed to generate a thumbnail for a file
type Request struct {
Resolution image.Rectangle
Encoder Encoder
@@ -41,37 +42,48 @@ type Request struct {
type Manager interface {
// Generate creates a thumbnail and stores it.
// The function returns a key with which the actual file can be retrieved.
Generate(Request, interface{}) (string, error)
Generate(r Request, img interface{}) (string, error)
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
// The function will return a status if the file exists and the key to the file.
CheckThumbnail(Request) (string, bool)
CheckThumbnail(r Request) (string, bool)
// GetThumbnail will load the thumbnail from the storage and return its content.
GetThumbnail(key string) ([]byte, error)
}
// NewSimpleManager creates a new instance of SimpleManager
func NewSimpleManager(resolutions Resolutions, storage storage.Storage, logger log.Logger) SimpleManager {
func NewSimpleManager(resolutions Resolutions, storage storage.Storage, logger log.Logger, maxInputWidth, maxInputHeight int) SimpleManager {
return SimpleManager{
storage: storage,
logger: logger,
resolutions: resolutions,
storage: storage,
logger: logger,
resolutions: resolutions,
maxDimension: image.Point{X: maxInputWidth, Y: maxInputHeight},
}
}
// SimpleManager is a simple implementation of Manager
type SimpleManager struct {
storage storage.Storage
logger log.Logger
resolutions Resolutions
storage storage.Storage
logger log.Logger
resolutions Resolutions
maxDimension image.Point
}
// Generate creates a thumbnail and stores it
func (s SimpleManager) Generate(r Request, img interface{}) (string, error) {
var match image.Rectangle
var inputDimensions image.Rectangle
switch m := img.(type) {
case *gif.GIF:
match = s.resolutions.ClosestMatch(r.Resolution, m.Image[0].Bounds())
inputDimensions = m.Image[0].Bounds()
case image.Image:
match = s.resolutions.ClosestMatch(r.Resolution, m.Bounds())
inputDimensions = m.Bounds()
}
// validate max input image dimensions - 6016x4000
if inputDimensions.Size().X > s.maxDimension.X || inputDimensions.Size().Y > s.maxDimension.Y {
return "", errors.ErrImageTooLarge
}
thumbnail, err := r.Generator.Generate(match, img, r.Processor)
@@ -92,11 +104,13 @@ func (s SimpleManager) Generate(r Request, img interface{}) (string, error) {
return k, nil
}
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
func (s SimpleManager) CheckThumbnail(r Request) (string, bool) {
k := s.storage.BuildKey(mapToStorageRequest(r))
return k, s.storage.Stat(k)
}
// GetThumbnail will load the thumbnail from the storage and return its content.
func (s SimpleManager) GetThumbnail(key string) ([]byte, error) {
return s.storage.Get(key)
}

View File

@@ -1,8 +1,12 @@
package thumbnail
import (
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/errors"
"github.com/owncloud/ocis/v2/services/thumbnails/pkg/preprocessor"
"github.com/stretchr/testify/assert"
"image"
"os"
"path"
"path/filepath"
"testing"
@@ -17,11 +21,11 @@ type NoOpManager struct {
storage.Storage
}
func (m NoOpManager) BuildKey(r storage.Request) string {
func (m NoOpManager) BuildKey(_ storage.Request) string {
return ""
}
func (m NoOpManager) Set(username, key string, thumbnail []byte) error {
func (m NoOpManager) Set(_, _ string, _ []byte) error {
return nil
}
@@ -31,6 +35,8 @@ func BenchmarkGet(b *testing.B) {
Resolutions{},
NoOpManager{},
log.NewLogger(),
6016,
4000,
)
res, _ := ParseResolution("32x32")
@@ -126,10 +132,59 @@ func TestPrepareRequest(t *testing.T) {
return
}
// func's are not reflactable, ignore
// funcs are not reflactable, ignore
if diff := cmp.Diff(tt.want, got, cmpopts.IgnoreFields(Request{}, "Processor")); diff != "" {
t.Errorf("PrepareRequest(): %v", diff)
}
})
}
}
func TestPreviewGenerationTooBigImage(t *testing.T) {
tests := []struct {
name string
fileName string
mimeType string
}{
{name: "png", mimeType: "image/png", fileName: "../../testdata/oc.png"},
{name: "gif", mimeType: "image/gif", fileName: "../../testdata/oc.gif"},
{name: "ggs", mimeType: "application/vnd.geogebra.slides", fileName: "../../testdata/test.ggs"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sut := NewSimpleManager(
Resolutions{},
NoOpManager{},
log.NewLogger(),
1024,
768,
)
res, _ := ParseResolution("32x32")
req := Request{
Resolution: res,
Checksum: "1872ade88f3013edeb33decd74a4f947",
}
cwd, _ := os.Getwd()
p := filepath.Join(cwd, tt.fileName)
f, _ := os.Open(p)
defer f.Close()
preproc := preprocessor.ForType(tt.mimeType, nil)
convert, err := preproc.Convert(f)
if err != nil {
return
}
ext := path.Ext(tt.fileName)
req.Encoder, _ = EncoderForType(ext)
generate, err := sut.Generate(req, convert)
if err != nil {
return
}
assert.ErrorIs(t, err, errors.ErrImageTooLarge)
assert.Equal(t, "", generate)
})
}
}

BIN
services/thumbnails/testdata/oc.gif vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

BIN
services/thumbnails/testdata/test.ggs vendored Normal file

Binary file not shown.