store/nbs: s3_object_reader: Fix a regression which prevented loading large table files with the AWS NomsBlockStore backend.

Affects 1.50.0, 1.50.1 and 1.50.2.
This commit is contained in:
Aaron Son
2025-03-07 18:06:00 -08:00
parent 51eb05981d
commit d1a76fb041
3 changed files with 55 additions and 3 deletions

View File

@@ -235,14 +235,17 @@ func (m *fakeS3) GetObject(ctx context.Context, input *s3.GetObjectInput, opts .
if !present {
return nil, mockAWSError("NoSuchKey")
}
var outputRange *string
if input.Range != nil {
start, end := parseRange(*input.Range, len(obj))
outputRange = aws.String(*input.Range + "/" + strconv.Itoa(len(obj)))
obj = obj[start:end]
}
return &s3.GetObjectOutput{
Body: io.NopCloser(bytes.NewReader(obj)),
ContentLength: aws.Int64(int64(len(obj))),
ContentRange: outputRange,
}, nil
}

View File

@@ -185,10 +185,8 @@ func (s3or *s3ObjectReader) readS3ObjectFromEnd(ctx context.Context, name string
}
bs := p[start:end]
rangeStart := sz - uint64(len(p)) + uint64(start)
rangeEnd := sz - uint64(len(p)) + uint64(end) - 1
length := rangeEnd - rangeStart
eg.Go(func() error {
n, _, err := s3or.readRange(egctx, name, bs, httpRangeHeader(int64(rangeStart), int64(length)))
n, _, err := s3or.readRange(egctx, name, bs, httpRangeHeader(int64(rangeStart), int64(len(bs))))
if err != nil {
return err
}

View File

@@ -0,0 +1,51 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nbs
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestS3ObjectReader(t *testing.T) {
t.Run("LargeReadFromObjectEnd", func(t *testing.T) {
ns := "5c2d1e52-e9bc-4779-881a-9087ad9a9f7b"
obj := "rhi4dnrr2ov6420h7j419lc4f72u7egf"
key := fmt.Sprintf("%s/%s", ns, obj)
s3client := makeFakeS3(t)
s3or := &s3ObjectReader{
s3client,
"dolthub-chunks-prod",
nil,
"5c2d1e52-e9bc-4779-881a-9087ad9a9f7b",
}
sz := maxS3ReadFromEndReqSize * 2 + maxS3ReadFromEndReqSize / 2
data := make([]byte, sz)
for i := range data {
data[i] = byte((i % 256))
}
rd := make([]byte, sz-256)
s3client.data[key] = data
n, rdSz, err := s3or.readS3ObjectFromEnd(context.Background(), obj, rd, &Stats{})
require.NoError(t, err)
assert.Equal(t, uint64(sz), rdSz)
assert.Equal(t, n, len(rd))
assert.Equal(t, data[256:], rd)
})
}