mirror/phylum
1
0
Fork 0
mirror of https://codeberg.org/shroff/phylum.git synced 2026-05-02 18:20:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

[server] Use SHA-256 instead of MD5 for etag

Browse Source
This commit is contained in:
Abhishek Shroff
2024-08-29 11:31:11 +05:30
parent 9bf11a82f3
commit 7f8fd423a2
12 changed files with 71 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/core/filesystem.go
+10 -9
View File
@@ -2,6 +2,7 @@ package core
import (
"context"
"crypto/sha256"
"errors"
"io"
"io/fs"
@@ -71,7 +72,7 @@ func OpenFileSystem(dbh *db.DbHandler, ctx context.Context, cs storage.Storage,
collection: true,
modTime: res.Modified.Time,
delTime: nil,
etag: "",
sha256sum: "",
}
}
return filesystem{
@@ -128,7 +129,7 @@ func (f filesystem) ResourceByPath(path string) (Resource, error) {
collection: res.Dir,
modTime: res.Modified.Time,
delTime: nil,
etag: res.Etag.String,
sha256sum: res.Sha256sum.String,
}, nil
}
@@ -155,7 +156,7 @@ func (f filesystem) ResourceByID(id uuid.UUID) (Resource, error) {
collection: res.Dir,
modTime: res.Modified.Time,
delTime: delTime,
etag: res.Etag.String,
sha256sum: res.Sha256sum.String,
}, nil
}
func (f filesystem) OpenRead(r Resource, start, length int64) (io.ReadCloser, error) {
@@ -169,11 +170,11 @@ func (f filesystem) OpenWrite(r Resource) (io.WriteCloser, error) {
if r.Permission() < PermissionReadWrite {
return nil, ErrInsufficientPermissions
}
return f.cs.OpenWrite(r.ID(), func(len int, etag string) error {
return f.cs.OpenWrite(r.ID(), sha256.New, func(len int, sum string) error {
return f.db.UpdateResourceContents(f.ctx, db.UpdateResourceContentsParams{
ID: r.ID(),
Size: pgtype.Int8{Int64: int64(len), Valid: true},
Etag: pgtype.Text{String: etag, Valid: true},
ID: r.ID(),
Size: pgtype.Int8{Int64: int64(len), Valid: true},
Sha256sum: pgtype.Text{String: sum, Valid: true},
})
})
}
@@ -204,7 +205,7 @@ func (f filesystem) ReadDir(r Resource) ([]Resource, error) {
modTime: c.Modified.Time,
delTime: nil,
collection: c.Dir,
etag: c.Etag.String,
sha256sum: c.Sha256sum.String,
}
}
return result, nil
@@ -250,7 +251,7 @@ func (f filesystem) CreateMemberResource(r Resource, id uuid.UUID, name string,
modTime: result.Modified.Time,
delTime: nil,
collection: dir,
etag: "",
sha256sum: "",
}, nil
}
server/internal/core/resource.go
+2 -2
View File
@@ -30,7 +30,7 @@ type resource struct {
collection bool
modTime time.Time
delTime *time.Time
etag string
sha256sum string
}
func (r resource) ID() uuid.UUID { return r.id }
@@ -50,7 +50,7 @@ func (r resource) DelTime() *time.Time { return r.delTime }
func (r resource) IsDir() bool { return r.collection }
func (r resource) ETag() string {
return r.etag
return r.sha256sum
}
func (r resource) ContentType() string {
server/internal/db/migrations/data/001_resources.sql
+9 -9
View File
@@ -1,13 +1,13 @@
CREATE TABLE resources (
id uuid PRIMARY KEY,
parent uuid REFERENCES resources(id) ON UPDATE CASCADE ON DELETE CASCADE,
name TEXT NOT NULL,
dir BOOLEAN NOT NULL,
created TIMESTAMP NOT NULL,
modified TIMESTAMP NOT NULL,
deleted TIMESTAMP,
size BIGINT,
etag TEXT
id uuid PRIMARY KEY,
parent uuid REFERENCES resources(id) ON UPDATE CASCADE ON DELETE CASCADE,
name TEXT NOT NULL,
dir BOOLEAN NOT NULL,
created TIMESTAMP NOT NULL,
modified TIMESTAMP NOT NULL,
deleted TIMESTAMP,
size BIGINT,
sha256sum TEXT
);
CREATE UNIQUE INDEX unique_member_resource_name ON resources(parent, name) WHERE deleted IS NULL;
server/internal/db/models.go
+9 -9
View File
@@ -23,15 +23,15 @@ type Permission struct {
}
type Resource struct {
ID uuid.UUID
Parent *uuid.UUID
Name string
Dir bool
Created pgtype.Timestamp
Modified pgtype.Timestamp
Deleted pgtype.Timestamp
Size pgtype.Int8
Etag pgtype.Text
ID uuid.UUID
Parent *uuid.UUID
Name string
Dir bool
Created pgtype.Timestamp
Modified pgtype.Timestamp
Deleted pgtype.Timestamp
Size pgtype.Int8
Sha256sum pgtype.Text
}
type StorageBackend struct {
server/internal/db/permissions.sql.go
+15 -15
View File
@@ -88,12 +88,12 @@ func (q *Queries) GetLocalPermissionsForResource(ctx context.Context, resourceID
}
const readDir = `-- name: ReadDir :many
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, $2::int
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, $2::int
FROM resources r
WHERE r.id = $3::uuid
UNION ALL
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name),
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name),
CASE
WHEN p.permission > n.permission THEN p.permission
ELSE n.permission
@@ -106,7 +106,7 @@ WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth
AND r.id != $3::uuid
AND CASE WHEN $5::boolean THEN true ELSE depth < 1 END
)
SELECT id, parent, name, dir, created, modified, size, etag, depth, path, permission from nodes
SELECT id, parent, name, dir, created, modified, size, sha256sum, depth, path, permission from nodes
WHERE CASE WHEN $1::boolean THEN true ELSE depth > 0 END
`
@@ -126,7 +126,7 @@ type ReadDirRow struct {
Created pgtype.Timestamp
Modified pgtype.Timestamp
Size pgtype.Int8
Etag pgtype.Text
Sha256sum pgtype.Text
Depth int32
Path string
Permission int32
@@ -155,7 +155,7 @@ func (q *Queries) ReadDir(ctx context.Context, arg ReadDirParams) ([]ReadDirRow,
&i.Created,
&i.Modified,
&i.Size,
&i.Etag,
&i.Sha256sum,
&i.Depth,
&i.Path,
&i.Permission,
@@ -202,7 +202,7 @@ WITH RECURSIVE nodes(resid, id, parent, found, permission) AS (
LEFT JOIN permissions p
ON r.id = p.resource_id AND p.user_id = $3::int
)
SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, etag FROM nodes n
SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, sha256sum FROM nodes n
JOIN resources r
ON r.id = n.resid
WHERE n.parent IS NULL
@@ -225,7 +225,7 @@ type ResourceByIDRow struct {
Modified pgtype.Timestamp
Deleted pgtype.Timestamp
Size pgtype.Int8
Etag pgtype.Text
Sha256sum pgtype.Text
}
func (q *Queries) ResourceByID(ctx context.Context, arg ResourceByIDParams) (ResourceByIDRow, error) {
@@ -242,18 +242,18 @@ func (q *Queries) ResourceByID(ctx context.Context, arg ResourceByIDParams) (Res
&i.Modified,
&i.Deleted,
&i.Size,
&i.Etag,
&i.Sha256sum,
)
return i, err
}
const resourceByPath = `-- name: ResourceByPath :one
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, search, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, $1::text[], $2::int
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, search, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, $1::text[], $2::int
FROM resources r
WHERE r.id = $3::uuid
UNION ALL
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name), n.search,
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name), n.search,
CASE
WHEN p.permission > n.permission THEN p.permission
ELSE n.permission
@@ -266,7 +266,7 @@ WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth
WHERE deleted IS NULL
AND r.name = n.search[n.depth + 1]
)
SELECT id, parent, name, dir, created, modified, size, etag, depth, path, search, permission FROM nodes WHERE cardinality(search) = depth
SELECT id, parent, name, dir, created, modified, size, sha256sum, depth, path, search, permission FROM nodes WHERE cardinality(search) = depth
`
type ResourceByPathParams struct {
@@ -284,7 +284,7 @@ type ResourceByPathRow struct {
Created pgtype.Timestamp
Modified pgtype.Timestamp
Size pgtype.Int8
Etag pgtype.Text
Sha256sum pgtype.Text
Depth int32
Path string
Search []string
@@ -307,7 +307,7 @@ func (q *Queries) ResourceByPath(ctx context.Context, arg ResourceByPathParams)
&i.Created,
&i.Modified,
&i.Size,
&i.Etag,
&i.Sha256sum,
&i.Depth,
&i.Path,
&i.Search,
server/internal/db/resources.sql.go
+7 -7
View File
@@ -17,7 +17,7 @@ INSERT INTO resources(
id, parent, name, dir, created, modified
) VALUES (
$1, $2, $3, $4, NOW(), NOW()
) RETURNING id, parent, name, dir, created, modified, deleted, size, etag
) RETURNING id, parent, name, dir, created, modified, deleted, size, sha256sum
`
type CreateResourceParams struct {
@@ -44,7 +44,7 @@ func (q *Queries) CreateResource(ctx context.Context, arg CreateResourceParams)
&i.Modified,
&i.Deleted,
&i.Size,
&i.Etag,
&i.Sha256sum,
)
return i, err
}
@@ -122,19 +122,19 @@ const updateResourceContents = `-- name: UpdateResourceContents :exec
UPDATE resources
SET
size = $1,
etag = $2,
sha256sum = $2,
modified = NOW()
WHERE id = $3
`
type UpdateResourceContentsParams struct {
Size pgtype.Int8
Etag pgtype.Text
ID uuid.UUID
Size pgtype.Int8
Sha256sum pgtype.Text
ID uuid.UUID
}
func (q *Queries) UpdateResourceContents(ctx context.Context, arg UpdateResourceContentsParams) error {
_, err := q.db.Exec(ctx, updateResourceContents, arg.Size, arg.Etag, arg.ID)
_, err := q.db.Exec(ctx, updateResourceContents, arg.Size, arg.Sha256sum, arg.ID)
return err
}
server/internal/storage/local_storage.go
+2 -3
View File
@@ -1,7 +1,6 @@
package storage
import (
"crypto/md5"
"encoding/hex"
"errors"
"hash"
@@ -45,12 +44,12 @@ func (l localStorage) OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser
return file, nil
}
func (l localStorage) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) {
func (l localStorage) OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) {
file, err := os.OpenFile(l.path(id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
if err != nil || callback == nil {
return file, err
}
return &hasher{dest: file, sum: md5.New(), closeCallback: func(len int, sum hash.Hash, err error) error {
return &hasher{dest: file, sum: h(), closeCallback: func(len int, sum hash.Hash, err error) error {
if err != nil {
return err
}
server/internal/storage/minio_storage.go
+3 -3
View File
@@ -2,7 +2,7 @@ package storage
import (
"context"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
@@ -70,13 +70,13 @@ func (s minioStorage) OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser
return s.client.GetObject(context.Background(), s.bucketName, s.prefix+id.String(), minio.GetObjectOptions{})
}
func (s minioStorage) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) {
func (s minioStorage) OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) {
path := filepath.Join(s.tmp, id.String())
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
if err != nil {
return nil, err
}
return &hasher{dest: file, sum: md5.New(), closeCallback: func(len int, sum hash.Hash, err error) error {
return &hasher{dest: file, sum: sha256.New(), closeCallback: func(len int, sum hash.Hash, err error) error {
if err != nil {
return err
}
server/internal/storage/storage.go
+4 -3
View File
@@ -3,6 +3,7 @@ package storage
import (
"context"
"errors"
"hash"
"io"
"github.com/google/uuid"
@@ -13,7 +14,7 @@ type Storage interface {
CreateBackend(ctx context.Context, name string, driver string, params map[string]string) error
ListBackends() map[string]Backend
OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, error)
OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error)
OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error)
Delete(id uuid.UUID) error
DeleteAll(ids uuid.UUIDs) []error
}
@@ -43,11 +44,11 @@ func (s storage) OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, err
return backend.OpenRead(id, start, length)
}
}
func (s storage) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) {
func (s storage) OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error) {
if backend, err := s.findStorageBackend(id); err != nil {
return nil, err
} else {
return backend.OpenWrite(id, callback)
return backend.OpenWrite(id, h, callback)
}
}
server/internal/storage/storage_backend.go
+2 -1
View File
@@ -1,6 +1,7 @@
package storage
import (
"hash"
"io"
"github.com/google/uuid"
@@ -9,7 +10,7 @@ import (
type Backend interface {
Name() string
OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, error)
OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error)
OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string) error) (io.WriteCloser, error)
Delete(id uuid.UUID) error
DeleteAll(ids uuid.UUIDs) []error
String() string
server/sql/queries/permissions.sql
+7 -7
View File
@@ -1,11 +1,11 @@
-- name: ReadDir :many
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, @permission::int
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, @permission::int
FROM resources r
WHERE r.id = @id::uuid
UNION ALL
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name),
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name),
CASE
WHEN p.permission > n.permission THEN p.permission
ELSE n.permission
@@ -22,12 +22,12 @@ SELECT * from nodes
WHERE CASE WHEN @include_root::boolean THEN true ELSE depth > 0 END;
-- name: ResourceByPath :one
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, etag, depth, path, search, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, 0, ''::text, @search::text[], @permission::int
WITH RECURSIVE nodes(id, parent, name, dir, created, modified, size, sha256sum, depth, path, search, permission) AS (
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, 0, ''::text, @search::text[], @permission::int
FROM resources r
WHERE r.id = @root::uuid
UNION ALL
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.etag, n.depth + 1, concat(n.path, '/', r.name), n.search,
SELECT r.id, r.parent, r.name, r.dir, r.created, r.modified, r.size, r.sha256sum, n.depth + 1, concat(n.path, '/', r.name), n.search,
CASE
WHEN p.permission > n.permission THEN p.permission
ELSE n.permission
@@ -74,7 +74,7 @@ WITH RECURSIVE nodes(resid, id, parent, found, permission) AS (
LEFT JOIN permissions p
ON r.id = p.resource_id AND p.user_id = @user_id::int
)
SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, etag FROM nodes n
SELECT resid AS id, found, permission, r.parent, name, dir, created, modified, deleted, size, sha256sum FROM nodes n
JOIN resources r
ON r.id = n.resid
WHERE n.parent IS NULL;
server/sql/queries/resources.sql
+1 -1
View File
@@ -9,7 +9,7 @@ INSERT INTO resources(
UPDATE resources
SET
size = $1,
etag = $2,
sha256sum = $2,
modified = NOW()
WHERE id = $3;