From 79e80f756927dad7c59fee98019da15b13b0c76b Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 24 Jan 2023 16:36:27 -0800 Subject: [PATCH 1/3] go/libraries/doltcore/remotesrv: Response with HTTP status code 206 when writing partial responses. Also add Accept-Ranges headers in all HTTP responses for portions of table files and Content-Range headers for partial file responses. This makes the HTTP server more correct and can improve compatibility with various HTTP middleware. --- go/libraries/doltcore/remotesrv/http.go | 39 ++++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/go/libraries/doltcore/remotesrv/http.go b/go/libraries/doltcore/remotesrv/http.go index 41dbb8066c..241e46f07e 100644 --- a/go/libraries/doltcore/remotesrv/http.go +++ b/go/libraries/doltcore/remotesrv/http.go @@ -104,6 +104,7 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) { respWr.WriteHeader(http.StatusInternalServerError) return } + respWr.Header().Add("Accept-Ranges", "bytes") logger, statusCode = readTableFile(logger, abs, respWr, req.Header.Get("Range")) case http.MethodPost, http.MethodPut: @@ -184,7 +185,7 @@ func readTableFile(logger *logrus.Entry, path string, respWr http.ResponseWriter logger = logger.WithField("whole_file", true) r, readSize, fileErr = getFileReader(path) } else { - offset, length, err := offsetAndLenFromRange(rangeStr) + offset, length, headerStr, err := offsetAndLenFromRange(rangeStr) if err != nil { logger.Println(err.Error()) return logger, http.StatusBadRequest @@ -194,7 +195,11 @@ func readTableFile(logger *logrus.Entry, path string, respWr http.ResponseWriter "read_length": length, }) readSize = length - r, fileErr = getFileReaderAt(path, offset, length) + var fSize int64 + r, fSize, fileErr = getFileReaderAt(path, offset, length) + if fileErr == nil { + respWr.Header().Add("Content-Range", headerStr+strconv.Itoa(int(fSize))) + } } } if fileErr != nil { @@ -217,6 +222,12 @@ func readTableFile(logger *logrus.Entry, path string, respWr http.ResponseWriter } }() + if rangeStr == "" { + respWr.WriteHeader(http.StatusPartialContent) + } else { + respWr.WriteHeader(http.StatusOK) + } + n, err := io.Copy(respWr, r) if err != nil { logger = logger.WithField("status", http.StatusInternalServerError) @@ -313,34 +324,34 @@ func writeTableFile(ctx context.Context, logger *logrus.Entry, dbCache DBCache, return logger, http.StatusOK } -func offsetAndLenFromRange(rngStr string) (int64, int64, error) { +func offsetAndLenFromRange(rngStr string) (int64, int64, string, error) { if rngStr == "" { - return -1, -1, nil + return -1, -1, "", nil } if !strings.HasPrefix(rngStr, "bytes=") { - return -1, -1, errors.New("range string does not start with 'bytes=") + return -1, -1, "", errors.New("range string does not start with 'bytes=") } tokens := strings.Split(rngStr[6:], "-") if len(tokens) != 2 { - return -1, -1, errors.New("invalid range format. should be bytes=#-#") + return -1, -1, "", errors.New("invalid range format. should be bytes=#-#") } start, err := strconv.ParseUint(strings.TrimSpace(tokens[0]), 10, 64) if err != nil { - return -1, -1, errors.New("invalid offset is not a number. should be bytes=#-#") + return -1, -1, "", errors.New("invalid offset is not a number. should be bytes=#-#") } end, err := strconv.ParseUint(strings.TrimSpace(tokens[1]), 10, 64) if err != nil { - return -1, -1, errors.New("invalid length is not a number. should be bytes=#-#") + return -1, -1, "", errors.New("invalid length is not a number. should be bytes=#-#") } - return int64(start), int64(end-start) + 1, nil + return int64(start), int64(end-start) + 1, "bytes " + tokens[0] + "-" + tokens[1] + "/", nil } // getFileReader opens a file at the given path and returns an io.ReadCloser, @@ -368,21 +379,21 @@ type closerReaderWrapper struct { io.Closer } -func getFileReaderAt(path string, offset int64, length int64) (io.ReadCloser, error) { +func getFileReaderAt(path string, offset int64, length int64) (io.ReadCloser, int64, error) { f, fSize, err := openFile(path) if err != nil { - return nil, err + return nil, 0, err } if fSize < int64(offset+length) { - return nil, fmt.Errorf("failed to read file %s at offset %d, length %d: %w", path, offset, length, ErrReadOutOfBounds) + return nil, 0, fmt.Errorf("failed to read file %s at offset %d, length %d: %w", path, offset, length, ErrReadOutOfBounds) } _, err = f.Seek(int64(offset), 0) if err != nil { - return nil, fmt.Errorf("failed to seek file at path %s to offset %d: %w", path, offset, err) + return nil, 0, fmt.Errorf("failed to seek file at path %s to offset %d: %w", path, offset, err) } r := closerReaderWrapper{io.LimitReader(f, length), f} - return r, nil + return r, fSize, nil } From c438bcd986c9feb57cb0057f5e9bc1cd4f5e6646 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 24 Jan 2023 16:41:23 -0800 Subject: [PATCH 2/3] format_repo.sh. --- go/libraries/doltcore/sqle/enginetest/dolt_queries.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index bae3f80f74..bb12cf45bf 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -2067,7 +2067,7 @@ var LogTableFunctionScriptTests = []queries.ScriptTest{ Expected: []sql.Row{{3}}, }, { - Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", + Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", Expected: []sql.Row{{2}}, }, }, From c1c93b8bf81a78ff5f683cd62da05e2cc09e67dd Mon Sep 17 00:00:00 2001 From: reltuk Date: Wed, 25 Jan 2023 00:43:35 +0000 Subject: [PATCH 3/3] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/enginetest/dolt_queries.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index bb12cf45bf..bae3f80f74 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -2067,7 +2067,7 @@ var LogTableFunctionScriptTests = []queries.ScriptTest{ Expected: []sql.Row{{3}}, }, { - Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", + Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", Expected: []sql.Row{{2}}, }, },