mirror of
https://github.com/dolthub/dolt.git
synced 2026-01-30 10:45:18 -06:00
store/nbs: s3_object_reader: Fix a regression which prevented loading large table files with the AWS NomsBlockStore backend.
Affects 1.50.0, 1.50.1 and 1.50.2.
This commit is contained in:
@@ -235,14 +235,17 @@ func (m *fakeS3) GetObject(ctx context.Context, input *s3.GetObjectInput, opts .
|
||||
if !present {
|
||||
return nil, mockAWSError("NoSuchKey")
|
||||
}
|
||||
var outputRange *string
|
||||
if input.Range != nil {
|
||||
start, end := parseRange(*input.Range, len(obj))
|
||||
outputRange = aws.String(*input.Range + "/" + strconv.Itoa(len(obj)))
|
||||
obj = obj[start:end]
|
||||
}
|
||||
|
||||
return &s3.GetObjectOutput{
|
||||
Body: io.NopCloser(bytes.NewReader(obj)),
|
||||
ContentLength: aws.Int64(int64(len(obj))),
|
||||
ContentRange: outputRange,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -185,10 +185,8 @@ func (s3or *s3ObjectReader) readS3ObjectFromEnd(ctx context.Context, name string
|
||||
}
|
||||
bs := p[start:end]
|
||||
rangeStart := sz - uint64(len(p)) + uint64(start)
|
||||
rangeEnd := sz - uint64(len(p)) + uint64(end) - 1
|
||||
length := rangeEnd - rangeStart
|
||||
eg.Go(func() error {
|
||||
n, _, err := s3or.readRange(egctx, name, bs, httpRangeHeader(int64(rangeStart), int64(length)))
|
||||
n, _, err := s3or.readRange(egctx, name, bs, httpRangeHeader(int64(rangeStart), int64(len(bs))))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
51
go/store/nbs/s3_object_reader_test.go
Normal file
51
go/store/nbs/s3_object_reader_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright 2025 Dolthub, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package nbs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestS3ObjectReader(t *testing.T) {
|
||||
t.Run("LargeReadFromObjectEnd", func(t *testing.T) {
|
||||
ns := "5c2d1e52-e9bc-4779-881a-9087ad9a9f7b"
|
||||
obj := "rhi4dnrr2ov6420h7j419lc4f72u7egf"
|
||||
key := fmt.Sprintf("%s/%s", ns, obj)
|
||||
s3client := makeFakeS3(t)
|
||||
s3or := &s3ObjectReader{
|
||||
s3client,
|
||||
"dolthub-chunks-prod",
|
||||
nil,
|
||||
"5c2d1e52-e9bc-4779-881a-9087ad9a9f7b",
|
||||
}
|
||||
sz := maxS3ReadFromEndReqSize * 2 + maxS3ReadFromEndReqSize / 2
|
||||
data := make([]byte, sz)
|
||||
for i := range data {
|
||||
data[i] = byte((i % 256))
|
||||
}
|
||||
rd := make([]byte, sz-256)
|
||||
s3client.data[key] = data
|
||||
n, rdSz, err := s3or.readS3ObjectFromEnd(context.Background(), obj, rd, &Stats{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(sz), rdSz)
|
||||
assert.Equal(t, n, len(rd))
|
||||
assert.Equal(t, data[256:], rd)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user