mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-08 11:21:17 -05:00
Code Issues Packages Projects Releases 163 Wiki Activity

Merge pull request #8963 from dolthub/aaron/store-nbs-s3objectreader-fix-large-table-files-regression

Browse Source 
store/nbs: s3_object_reader: Fix a regression which prevented loading large table files with the AWS NomsBlockStore backend.
This commit is contained in:
Aaron Son
2025-03-10 11:07:23 -07:00
committed by GitHub
parent 51eb05981d 2812d021b7
commit e541090868
5 changed files with 55 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/go.mod
-2
View File
@@ -63,7 +63,6 @@ require (
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
github.com/dolthub/go-mysql-server v0.19.1-0.20250305230031-14a57e076a0a
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
github.com/dolthub/swiss v0.1.0
github.com/esote/minmaxheap v1.0.0
github.com/goccy/go-json v0.10.2
github.com/google/btree v1.1.2
@@ -132,7 +131,6 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dolthub/go-icu-regex v0.0.0-20250303123116-549b8d7cad00 // indirect
github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect
github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577 // indirect
github.com/go-fonts/liberation v0.2.0 // indirect
github.com/go-kit/kit v0.10.0 // indirect
github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect
go/go.sum
-8
View File
@@ -102,8 +102,6 @@ github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible/
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250=
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
@@ -231,12 +229,8 @@ github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1
github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto=
github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTEtT5tOBsCuCrlYnLRKpbJVJkDbrTRhwQ=
github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI=
github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577 h1:SegEguMxToBn045KRHLIUlF2/jR7Y2qD6fF+3tdOfvI=
github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE=
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
github.com/dolthub/vitess v0.0.0-20250304211657-920ca9ec2b9a h1:HIH9g4z+yXr4DIFyT6L5qOIEGJ1zVtlj6baPyHAG4Yw=
github.com/dolthub/vitess v0.0.0-20250304211657-920ca9ec2b9a/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -712,8 +706,6 @@ github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE=
github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777 h1:5u+6YWU2faS+Sr/x8j9yalMpSDUkatNOZWXV3wMUCGQ=
github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777/go.mod h1:4af3KxEsswy6aTzsTcwa8QZUSh4V+80oHdp1QX9uJHA=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
go/store/nbs/s3_fake_test.go
+3
View File
@@ -235,14 +235,17 @@ func (m *fakeS3) GetObject(ctx context.Context, input *s3.GetObjectInput, opts .
if !present {
return nil, mockAWSError("NoSuchKey")
}
var outputRange *string
if input.Range != nil {
start, end := parseRange(*input.Range, len(obj))
outputRange = aws.String(*input.Range + "/" + strconv.Itoa(len(obj)))
obj = obj[start:end]
}
return &s3.GetObjectOutput{
Body: io.NopCloser(bytes.NewReader(obj)),
ContentLength: aws.Int64(int64(len(obj))),
ContentRange: outputRange,
}, nil
}
go/store/nbs/s3_object_reader.go
+1 -3
View File
@@ -185,10 +185,8 @@ func (s3or *s3ObjectReader) readS3ObjectFromEnd(ctx context.Context, name string
}
bs := p[start:end]
rangeStart := sz - uint64(len(p)) + uint64(start)
rangeEnd := sz - uint64(len(p)) + uint64(end) - 1
length := rangeEnd - rangeStart
eg.Go(func() error {
n, _, err := s3or.readRange(egctx, name, bs, httpRangeHeader(int64(rangeStart), int64(length)))
n, _, err := s3or.readRange(egctx, name, bs, httpRangeHeader(int64(rangeStart), int64(len(bs))))
if err != nil {
return err
}
go/store/nbs/s3_object_reader_test.go
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nbs
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestS3ObjectReader(t *testing.T) {
t.Run("LargeReadFromObjectEnd", func(t *testing.T) {
ns := "5c2d1e52-e9bc-4779-881a-9087ad9a9f7b"
obj := "rhi4dnrr2ov6420h7j419lc4f72u7egf"
key := fmt.Sprintf("%s/%s", ns, obj)
s3client := makeFakeS3(t)
s3or := &s3ObjectReader{
s3client,
"dolthub-chunks-prod",
nil,
"5c2d1e52-e9bc-4779-881a-9087ad9a9f7b",
}
sz := maxS3ReadFromEndReqSize*2 + maxS3ReadFromEndReqSize/2
data := make([]byte, sz)
for i := range data {
data[i] = byte((i % 256))
}
rd := make([]byte, sz-256)
s3client.data[key] = data
n, rdSz, err := s3or.readS3ObjectFromEnd(context.Background(), obj, rd, &Stats{})
require.NoError(t, err)
assert.Equal(t, uint64(sz), rdSz)
assert.Equal(t, n, len(rd))
assert.Equal(t, data[256:], rd)
})
}