From 7f8fd423a241b34197215fc6510b5d66b5c0cbda Mon Sep 17 00:00:00 2001 From: Abhishek Shroff Date: Thu, 29 Aug 2024 11:31:11 +0530 Subject: [PATCH] [server] Use SHA-256 instead of MD5 for etag --- server/internal/core/filesystem.go | 19 ++++++------ server/internal/core/resource.go | 4 +-- .../db/migrations/data/001_resources.sql | 18 +++++------ server/internal/db/models.go | 18 +++++------ server/internal/db/permissions.sql.go | 30 +++++++++---------- server/internal/db/resources.sql.go | 14 ++++----- server/internal/storage/local_storage.go | 5 ++-- server/internal/storage/minio_storage.go | 6 ++-- server/internal/storage/storage.go | 7 +++-- server/internal/storage/storage_backend.go | 3 +- server/sql/queries/permissions.sql | 14 ++++----- server/sql/queries/resources.sql | 2 +- 12 files changed, 71 insertions(+), 69 deletions(-) diff --git a/server/internal/core/filesystem.go b/server/internal/core/filesystem.go index 2b07ef80..c9194873 100644 --- a/server/internal/core/filesystem.go +++ b/server/internal/core/filesystem.go @@ -2,6 +2,7 @@ package core import ( "context" + "crypto/sha256" "errors" "io" "io/fs" @@ -71,7 +72,7 @@ func OpenFileSystem(dbh *db.DbHandler, ctx context.Context, cs storage.Storage, collection: true, modTime: res.Modified.Time, delTime: nil, - etag: "", + sha256sum: "", } } return filesystem{ @@ -128,7 +129,7 @@ func (f filesystem) ResourceByPath(path string) (Resource, error) { collection: res.Dir, modTime: res.Modified.Time, delTime: nil, - etag: res.Etag.String, + sha256sum: res.Sha256sum.String, }, nil } @@ -155,7 +156,7 @@ func (f filesystem) ResourceByID(id uuid.UUID) (Resource, error) { collection: res.Dir, modTime: res.Modified.Time, delTime: delTime, - etag: res.Etag.String, + sha256sum: res.Sha256sum.String, }, nil } func (f filesystem) OpenRead(r Resource, start, length int64) (io.ReadCloser, error) { @@ -169,11 +170,11 @@ func (f filesystem) OpenWrite(r Resource) (io.WriteCloser, error) { if r.Permission() < PermissionReadWrite { return nil, ErrInsufficientPermissions } - return f.cs.OpenWrite(r.ID(), func(len int, etag string) error { + return f.cs.OpenWrite(r.ID(), sha256.New, func(len int, sum string) error { return f.db.UpdateResourceContents(f.ctx, db.UpdateResourceContentsParams{ - ID: r.ID(), - Size: pgtype.Int8{Int64: int64(len), Valid: true}, - Etag: pgtype.Text{String: etag, Valid: true}, + ID: r.ID(), + Size: pgtype.Int8{Int64: int64(len), Valid: true}, + Sha256sum: pgtype.Text{String: sum, Valid: true}, }) }) } @@ -204,7 +205,7 @@ func (f filesystem) ReadDir(r Resource) ([]Resource, error) { modTime: c.Modified.Time, delTime: nil, collection: c.Dir, - etag: c.Etag.String, + sha256sum: c.Sha256sum.String, } } return result, nil @@ -250,7 +251,7 @@ func (f filesystem) CreateMemberResource(r Resource, id uuid.UUID, name string, modTime: result.Modified.Time, delTime: nil, collection: dir, - etag: "", + sha256sum: "", }, nil } diff --git a/server/internal/core/resource.go b/server/internal/core/resource.go index 04339e98..1bfa5be6 100644 --- a/server/internal/core/resource.go +++ b/server/internal/core/resource.go @@ -30,7 +30,7 @@ type resource struct { collection bool modTime time.Time delTime *time.Time - etag string + sha256sum string } func (r resource) ID() uuid.UUID { return r.id } @@ -50,7 +50,7 @@ func (r resource) DelTime() *time.Time { return r.delTime } func (r resource) IsDir() bool { return r.collection } func (r resource) ETag() string { - return r.etag + return r.sha256sum } func (r resource) ContentType() string { diff --git a/server/internal/db/migrations/data/001_resources.sql b/server/internal/db/migrations/data/001_resources.sql index d117b8e5..59485478 100644 --- a/server/internal/db/migrations/data/001_resources.sql +++ b/server/internal/db/migrations/data/001_resources.sql @@ -1,13 +1,13 @@ CREATE TABLE resources ( - id uuid PRIMARY KEY, - parent uuid REFERENCES resources(id) ON UPDATE CASCADE ON DELETE CASCADE, - name TEXT NOT NULL, - dir BOOLEAN NOT NULL, - created TIMESTAMP NOT NULL, - modified TIMESTAMP NOT NULL, - deleted TIMESTAMP, - size BIGINT, - etag TEXT + id uuid PRIMARY KEY, + parent uuid REFERENCES resources(id) ON UPDATE CASCADE ON DELETE CASCADE, + name TEXT NOT NULL, + dir BOOLEAN NOT NULL, + created TIMESTAMP NOT NULL, + modified TIMESTAMP NOT NULL, + deleted TIMESTAMP, + size BIGINT, + sha256sum TEXT ); CREATE UNIQUE INDEX unique_member_resource_name ON resources(parent, name) WHERE deleted IS NULL; diff --git a/server/internal/db/models.go b/server/internal/db/models.go index a4209729..ddb20bcf 100644 --- a/server/internal/db/models.go +++ b/server/internal/db/models.go @@ -23,15 +23,15 @@ type Permission struct { } type Resource struct { - ID uuid.UUID - Parent *uuid.UUID - Name string - Dir bool - Created pgtype.Timestamp - Modified pgtype.Timestamp - Deleted pgtype.Timestamp - Size pgtype.Int8 - Etag pgtype.Text + ID uuid.UUID + Parent *uuid.UUID + Name string + Dir bool + Created pgtype.Timestamp + Modified pgtype.Timestamp + Deleted pgtype.Timestamp + Size pgtype.Int8 + Sha256sum pgtype.Text } type StorageBackend struct { diff --git a/server/internal/db/permissions.sql.go b/server/internal/db/permissions.sql.go index d7460e20..58b1bb5d 100644 --- a/server/internal/db/permissions.sql.go +++ b/server/internal/db/permissions.sql.go @@ -88,12 +88,12 @@ func (q *Queries) GetLocalPermissionsForResource(ctx context.Context, resourceID } const readDir = `-- name: ReadDir :many -WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, permission) AS ( - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, $2::int +WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, permission) AS ( + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, $2::int FROM resources r WHERE r.id = $3::uuid UNION ALL - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name), + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name), CASE WHEN p.permission > n.permission THEN p.permission ELSE n.permission @@ -106,7 +106,7 @@ WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth AND r.id != $3::uuid AND CASE WHEN $5::boolean THEN true ELSE depth < 1 END ) -SELECT id, parent, name, dir, created, modified, size, etag, depth, path, permission from nodes +SELECT id, parent, name, dir, created, modified, size, sha256sum, depth, path, permission from nodes WHERE CASE WHEN $1::boolean THEN true ELSE depth > 0 END ` @@ -126,7 +126,7 @@ type ReadDirRow struct { Created pgtype.Timestamp Modified pgtype.Timestamp Size pgtype.Int8 - Etag pgtype.Text + Sha256sum pgtype.Text Depth int32 Path string Permission int32 @@ -155,7 +155,7 @@ func (q *Queries) ReadDir(ctx context.Context, arg ReadDirParams) ([]ReadDirRow, &i.Created, &i.Modified, &i.Size, - &i.Etag, + &i.Sha256sum, &i.Depth, &i.Path, &i.Permission, @@ -202,7 +202,7 @@ WITH RECURSIVE nodes(resid, id, parent, found, permission) AS ( LEFT JOIN permissions p ON r.id = p.resource_id AND p.user_id = $3::int ) -SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, etag FROM nodes n +SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, sha256sum FROM nodes n JOIN resources r ON r.id = n.resid WHERE n.parent IS NULL @@ -225,7 +225,7 @@ type ResourceByIDRow struct { Modified pgtype.Timestamp Deleted pgtype.Timestamp Size pgtype.Int8 - Etag pgtype.Text + Sha256sum pgtype.Text } func (q *Queries) ResourceByID(ctx context.Context, arg ResourceByIDParams) (ResourceByIDRow, error) { @@ -242,18 +242,18 @@ func (q *Queries) ResourceByID(ctx context.Context, arg ResourceByIDParams) (Res &i.Modified, &i.Deleted, &i.Size, - &i.Etag, + &i.Sha256sum, ) return i, err } const resourceByPath = `-- name: ResourceByPath :one -WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, search, permission) AS ( - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, $1::text[], $2::int +WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, search, permission) AS ( + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, $1::text[], $2::int FROM resources r WHERE r.id = $3::uuid UNION ALL - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name), n.search, + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name), n.search, CASE WHEN p.permission > n.permission THEN p.permission ELSE n.permission @@ -266,7 +266,7 @@ WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth WHERE deleted IS NULL AND r.name = n.search[n.depth + 1] ) -SELECT id, parent, name, dir, created, modified, size, etag, depth, path, search, permission FROM nodes WHERE cardinality(search) = depth +SELECT id, parent, name, dir, created, modified, size, sha256sum, depth, path, search, permission FROM nodes WHERE cardinality(search) = depth ` type ResourceByPathParams struct { @@ -284,7 +284,7 @@ type ResourceByPathRow struct { Created pgtype.Timestamp Modified pgtype.Timestamp Size pgtype.Int8 - Etag pgtype.Text + Sha256sum pgtype.Text Depth int32 Path string Search []string @@ -307,7 +307,7 @@ func (q *Queries) ResourceByPath(ctx context.Context, arg ResourceByPathParams) &i.Created, &i.Modified, &i.Size, - &i.Etag, + &i.Sha256sum, &i.Depth, &i.Path, &i.Search, diff --git a/server/internal/db/resources.sql.go b/server/internal/db/resources.sql.go index bae181ff..a155af0e 100644 --- a/server/internal/db/resources.sql.go +++ b/server/internal/db/resources.sql.go @@ -17,7 +17,7 @@ INSERT INTO resources( id, parent, name, dir, created, modified ) VALUES ( $1, $2, $3, $4, NOW(), NOW() -) RETURNING id, parent, name, dir, created, modified, deleted, size, etag +) RETURNING id, parent, name, dir, created, modified, deleted, size, sha256sum ` type CreateResourceParams struct { @@ -44,7 +44,7 @@ func (q *Queries) CreateResource(ctx context.Context, arg CreateResourceParams) &i.Modified, &i.Deleted, &i.Size, - &i.Etag, + &i.Sha256sum, ) return i, err } @@ -122,19 +122,19 @@ const updateResourceContents = `-- name: UpdateResourceContents :exec UPDATE resources SET size = $1, - etag = $2, + sha256sum = $2, modified = NOW() WHERE id = $3 ` type UpdateResourceContentsParams struct { - Size pgtype.Int8 - Etag pgtype.Text - ID uuid.UUID + Size pgtype.Int8 + Sha256sum pgtype.Text + ID uuid.UUID } func (q *Queries) UpdateResourceContents(ctx context.Context, arg UpdateResourceContentsParams) error { - _, err := q.db.Exec(ctx, updateResourceContents, arg.Size, arg.Etag, arg.ID) + _, err := q.db.Exec(ctx, updateResourceContents, arg.Size, arg.Sha256sum, arg.ID) return err } diff --git a/server/internal/storage/local_storage.go b/server/internal/storage/local_storage.go index 77527916..02783f1b 100644 --- a/server/internal/storage/local_storage.go +++ b/server/internal/storage/local_storage.go @@ -1,7 +1,6 @@ package storage import ( - "crypto/md5" "encoding/hex" "errors" "hash" @@ -45,12 +44,12 @@ func (l localStorage) OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser return file, nil } -func (l localStorage) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) { +func (l localStorage) OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) { file, err := os.OpenFile(l.path(id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640) if err != nil || callback == nil { return file, err } - return &hasher{dest: file, sum: md5.New(), closeCallback: func(len int, sum hash.Hash, err error) error { + return &hasher{dest: file, sum: h(), closeCallback: func(len int, sum hash.Hash, err error) error { if err != nil { return err } diff --git a/server/internal/storage/minio_storage.go b/server/internal/storage/minio_storage.go index e08e9793..a0594486 100644 --- a/server/internal/storage/minio_storage.go +++ b/server/internal/storage/minio_storage.go @@ -2,7 +2,7 @@ package storage import ( "context" - "crypto/md5" + "crypto/sha256" "encoding/hex" "errors" "fmt" @@ -70,13 +70,13 @@ func (s minioStorage) OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser return s.client.GetObject(context.Background(), s.bucketName, s.prefix+id.String(), minio.GetObjectOptions{}) } -func (s minioStorage) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) { +func (s minioStorage) OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) { path := filepath.Join(s.tmp, id.String()) file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640) if err != nil { return nil, err } - return &hasher{dest: file, sum: md5.New(), closeCallback: func(len int, sum hash.Hash, err error) error { + return &hasher{dest: file, sum: sha256.New(), closeCallback: func(len int, sum hash.Hash, err error) error { if err != nil { return err } diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index 34f5c1be..0bc2c922 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -3,6 +3,7 @@ package storage import ( "context" "errors" + "hash" "io" "github.com/google/uuid" @@ -13,7 +14,7 @@ type Storage interface { CreateBackend(ctx context.Context, name string, driver string, params map[string]string) error ListBackends() map[string]Backend OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, error) - OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) + OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) Delete(id uuid.UUID) error DeleteAll(ids uuid.UUIDs) []error } @@ -43,11 +44,11 @@ func (s storage) OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, err return backend.OpenRead(id, start, length) } } -func (s storage) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) { +func (s storage) OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) { if backend, err := s.findStorageBackend(id); err != nil { return nil, err } else { - return backend.OpenWrite(id, callback) + return backend.OpenWrite(id, h, callback) } } diff --git a/server/internal/storage/storage_backend.go b/server/internal/storage/storage_backend.go index 294864fd..eb8901f9 100644 --- a/server/internal/storage/storage_backend.go +++ b/server/internal/storage/storage_backend.go @@ -1,6 +1,7 @@ package storage import ( + "hash" "io" "github.com/google/uuid" @@ -9,7 +10,7 @@ import ( type Backend interface { Name() string OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, error) - OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) + OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) Delete(id uuid.UUID) error DeleteAll(ids uuid.UUIDs) []error String() string diff --git a/server/sql/queries/permissions.sql b/server/sql/queries/permissions.sql index 05214e91..a355f67a 100644 --- a/server/sql/queries/permissions.sql +++ b/server/sql/queries/permissions.sql @@ -1,11 +1,11 @@ -- name: ReadDir :many -WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, permission) AS ( - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, @permission::int +WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, permission) AS ( + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, @permission::int FROM resources r WHERE r.id = @id::uuid UNION ALL - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name), + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name), CASE WHEN p.permission > n.permission THEN p.permission ELSE n.permission @@ -22,12 +22,12 @@ SELECT * from nodes WHERE CASE WHEN @include_root::boolean THEN true ELSE depth > 0 END; -- name: ResourceByPath :one -WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, search, permission) AS ( - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, @search::text[], @permission::int +WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, search, permission) AS ( + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, @search::text[], @permission::int FROM resources r WHERE r.id = @root::uuid UNION ALL - SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name), n.search, + SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name), n.search, CASE WHEN p.permission > n.permission THEN p.permission ELSE n.permission @@ -74,7 +74,7 @@ WITH RECURSIVE nodes(resid, id, parent, found, permission) AS ( LEFT JOIN permissions p ON r.id = p.resource_id AND p.user_id = @user_id::int ) -SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, etag FROM nodes n +SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, sha256sum FROM nodes n JOIN resources r ON r.id = n.resid WHERE n.parent IS NULL; diff --git a/server/sql/queries/resources.sql b/server/sql/queries/resources.sql index 87eb4126..55419a62 100644 --- a/server/sql/queries/resources.sql +++ b/server/sql/queries/resources.sql @@ -9,7 +9,7 @@ INSERT INTO resources( UPDATE resources SET size = $1, - etag = $2, + sha256sum = $2, modified = NOW() WHERE id = $3;