mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-26 19:46:39 -05:00
Code Issues Packages Projects Releases 158 Wiki Activity

Merge aaron changes

Browse Source
This commit is contained in:
Zach Musgrave
2022-12-19 16:46:01 -08:00
parent 273afd7d34 e9de6a90ff
commit 09ce74d905
106 changed files with 10132 additions and 7842 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/ses-email-action/action.yaml
+8
View File
@@ -21,6 +21,14 @@ inputs:
required: false
description: 'path to email data file'
default: ''
bodyPath:
required: false
description: 'path to email body file'
default: ''
subject:
required: false
description: 'path to email subject'
default: ''
toAddresses:
description: 'json string list of to addresses'
required: true
.github/actions/ses-email-action/bats-windows-failure-template.json
-8
View File
@@ -1,8 +0,0 @@
{
"Template": {
"TemplateName": "BatsWindowsFailureTemplate",
"SubjectPart": "Bats on Windows failed for ref: {{version}}",
"HtmlPart": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n <title>Bats on Windows Failed</title>\n</head><body>Bats on Windows failed for ref: {{version}}\n <a href={{workflowURL}}>{{workflowURL}}</a></body></html>",
"TextPart": "Bats on Windows failed for ref {{version}}\r\n"
}
}
.github/actions/ses-email-action/dist/index.js
+1612 -1605
View File
File diff suppressed because it is too large Load Diff
.github/actions/ses-email-action/import-benchmarking-release-template.json
-8
View File
@@ -1,8 +0,0 @@
{
"Template": {
"TemplateName": "ImportBenchmarkingReleaseTemplate",
"SubjectPart": "Import Benchmarks for {{format}} {{version}}",
"HtmlPart": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n <title>Dolt {{format}} {{version}} Import Results</title>\n <style>\n table {\n border: 1px solid black;\n letter-spacing: 1px;\n font-family: sans-serif;\n font-size: .8rem;\n padding: 5px;\n margin: 5px;\n }\n th {\n border: 1px solid rgb(190, 190, 190);\n padding: 10px;\n }\n td {\n padding: 5px;\n }\n tr:nth-child(even) {background-color: #f2f2f2;}\n </style>\n</head><body>{{results}}</body></html>",
"TextPart": "Dolt {{format}} {{version}} Import Results,\r\n{{results}}"
}
}
.github/actions/ses-email-action/index.js
+7
View File
@@ -11,14 +11,19 @@ const CcAddresses = JSON.parse(core.getInput('ccAddresses'));
const ToAddresses = JSON.parse(core.getInput('toAddresses'));
const ReplyToAddresses = JSON.parse(core.getInput('replyToAddresses'));
const workflowURL = core.getInput('workflowURL');
const subject = core.getInput('subject');
const bodyPath = core.getInput('bodyPath');
const data = dataFilePath ? fs.readFileSync(dataFilePath, { encoding: 'utf-8' }) : "";
const body = bodyPath ? fs.readFileSync(bodyPath, { encoding: 'utf-8' }) : "";
const templated = {
version,
format,
results: data,
workflowURL,
subject,
body,
};
// Set the region
@@ -36,6 +41,8 @@ const params = {
ReplyToAddresses,
};
console.log(params)
// Create the promise and SES service object
// const sendPromise = new aws.SES({apiVersion: '2010-12-01'}).sendEmail(params).promise();
const sendPromise = new aws.SES({apiVersion: '2010-12-01'}).sendTemplatedEmail(params).promise();
.github/actions/ses-email-action/orm-integration-failure-template.json
-8
View File
@@ -1,8 +0,0 @@
{
"Template": {
"TemplateName": "OrmIntegrationFailureTemplate",
"SubjectPart": "ORM Integrations Test failed for ref: {{version}}",
"HtmlPart": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n <title>ORM Integration Test Failed</title>\n</head><body>ORM Integrations Test failed for ref: {{version}}\n <a href={{workflowURL}}>{{workflowURL}}</a></body></html>",
"TextPart": "ORM Integrations Test failed for ref {{version}}\r\n"
}
}
.github/actions/ses-email-action/performance-benchmarking-release-template.json
-8
View File
@@ -1,8 +0,0 @@
{
"Template": {
"TemplateName": "PerformanceBenchmarkingReleaseTemplate",
"SubjectPart": "Performance Benchmarks for {{format}} {{version}}",
"HtmlPart": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n <title>Dolt {{format}} {{version}} Performance Results</title>\n <style>\n table {\n border: 1px solid black;\n letter-spacing: 1px;\n font-family: sans-serif;\n font-size: .8rem;\n padding: 5px;\n margin: 5px;\n }\n th {\n border: 1px solid rgb(190, 190, 190);\n padding: 10px;\n }\n td {\n padding: 5px;\n }\n tr:nth-child(even) {background-color: #f2f2f2;}\n </style>\n</head><body>{{results}}</body></html>",
"TextPart": "Dolt {{format}} {{version}} Performance Results,\r\n{{results}}"
}
}
.github/actions/ses-email-action/sql-correctness-release-template.json
-8
View File
@@ -1,8 +0,0 @@
{
"Template": {
"TemplateName": "SqlCorrectnessReleaseTemplate",
"SubjectPart": "SQL Correctness for {{format}} {{version}}",
"HtmlPart": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n <title>Dolt {{format}} {{version}} Performance Results</title>\n <style>\n table {\n border: 1px solid black;\n letter-spacing: 1px;\n font-family: sans-serif;\n font-size: .8rem;\n padding: 5px;\n margin: 5px;\n }\n th {\n border: 1px solid rgb(190, 190, 190);\n padding: 10px;\n }\n td {\n padding: 5px;\n }\n tr:nth-child(even) {background-color: #f2f2f2;}\n </style>\n</head><body>{{results}}</body></html>",
"TextPart": "Dolt {{format}} {{version}} SQL correctness,\r\n{{results}}"
}
}
.github/workflows/ci-bats-unix.yaml
+7 -1
View File
@@ -30,13 +30,19 @@ jobs:
use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }}
steps:
- name: Conditionally Set ENV VARS for AWS tests
env:
DOLT_FMT: ${{ matrix.dolt_fmt }}
run: |
if [[ $use_credentials == true ]]; then
echo "AWS_SDK_LOAD_CONFIG=1" >> $GITHUB_ENV
echo "AWS_REGION=us-west-2" >> $GITHUB_ENV
echo "DOLT_BATS_AWS_TABLE=dolt-ci-bats-manifests-us-west-2" >> $GITHUB_ENV
echo "DOLT_BATS_AWS_BUCKET=dolt-ci-bats-chunks-us-west-2" >> $GITHUB_ENV
echo "DOLT_BATS_AWS_EXISTING_REPO=aws_remote_bats_tests" >> $GITHUB_ENV
if [[ "$DOLT_FMT" == "__DOLT__" ]]; then
echo "DOLT_BATS_AWS_EXISTING_REPO=aws_remote_bats_tests__dolt__" >> $GITHUB_ENV
else
echo "DOLT_BATS_AWS_EXISTING_REPO=aws_remote_bats_tests" >> $GITHUB_ENV
fi
fi
- name: Configure AWS Credentials
if: ${{ env.use_credentials == 'true' }}
.github/workflows/ci-go-tests.yaml
+6
View File
@@ -23,9 +23,14 @@ jobs:
matrix:
os: [macos-latest, ubuntu-22.04, windows-latest]
dolt_fmt: [ "__DOLT__", "__LD_1__" ]
journal_store: [ "" ]
include:
- os: "ubuntu-22.04"
dolt_fmt: "__DOLT_DEV__"
journal_store: ""
- os: "ubuntu-22.04"
dolt_fmt: "__DOLT__"
journal_store: "true"
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v3
@@ -64,6 +69,7 @@ jobs:
env:
MATRIX_OS: ${{ matrix.os }}
DOLT_DEFAULT_BIN_FORMAT: ${{ matrix.dolt_fmt }}
DOLT_ENABLE_CHUNK_JOURNAL: ${{ matrix.journal_store }}
noracetest:
name: Go tests - no race
defaults:
.github/workflows/ci-sql-server-integration-tests.yaml
+6
View File
@@ -23,11 +23,16 @@ jobs:
matrix:
os: [ ubuntu-22.04, macos-latest ] # [ ubuntu-22.04, macos-latest, windows-latest ]
dolt_fmt: [ "__DOLT__", "__LD_1__" ]
journal_store: [ "" ]
exclude:
- os: "macos-latest"
dolt_fmt: ["__LD_1__" ]
- os: "windows-latest"
dolt_fmt: ["__LD_1__" ]
include:
- os: "ubuntu-22.04"
dolt_fmt: "__DOLT__"
journal_store: "true"
steps:
- name: Setup Go 1.x
uses: actions/setup-go@v3
@@ -45,6 +50,7 @@ jobs:
- name: Test all
env:
DOLT_FMT: ${{ matrix.dolt_fmt }}
DOLT_ENABLE_CHUNK_JOURNAL: ${{ matrix.journal_store }}
run: |
if [ -n "$DOLT_FMT" ]; then export DOLT_DEFAULT_BIN_FORMAT="$DOLT_FMT"; fi
export DOLT_BIN_PATH="$(pwd)/../../.ci_bin/dolt"
.github/workflows/import-benchmarks-pull-report.yaml
+1
View File
@@ -65,6 +65,7 @@ jobs:
{
"version": "${{ steps.comment-branch.outputs.head_sha }}",
"run_file": "ci.yaml",
"summary": "summary.yaml",
"report": "three_way_compare.sql",
"commit_to_branch": "${{ steps.comment-branch.outputs.head_sha }}",
"actor": "${{ github.actor }}",
.github/workflows/import-perf.yaml
+11 -4
View File
@@ -66,6 +66,8 @@ jobs:
gw=$GITHUB_WORKSPACE
in="${{ steps.bench.outputs.result_path }}"
query="$(pwd)/${{ env.BENCH_DIR }}/reporting/${{ github.event.client_payload.report }}"
summaryq="$(pwd)/${{ env.BENCH_DIR }}/reporting/${{ github.event.client_payload.summary }}"
out="$gw/results.csv"
dolt_dir="$gw/import-perf"
@@ -101,6 +103,9 @@ jobs:
cat "$out"
echo "::set-output name=report_path::$out"
avg=$(dolt sql -r csv < "$summaryq" | tail -1)
echo "::set-output name=avg::$avg"
- name: Format HTML
id: html
if: ${{ github.event.client_payload.email_recipient }} != ""
@@ -121,8 +126,10 @@ jobs:
done < "$in"
echo "</table>" >> "$out"
cat "$out"
avg="${{ steps.report.outputs.avg }}"
echo "<table><tr><th>Average</th></tr><tr><td>$avg</tr></td></table>" >> "$out"
cat "$out"
echo "::set-output name=html::$(echo $out)"
- name: Configure AWS Credentials
@@ -139,9 +146,9 @@ jobs:
with:
region: us-west-2
toAddresses: '["${{ github.event.client_payload.email_recipient }}"]'
version: ${{ steps.version.outputs.ref }}
format: '__DOLT__'
dataFile: ${{ steps.html.outputs.html }}
subject: 'Import Performance Benchmarks: ${{ github.event.client_payload.version }}'
bodyPath: ${{ steps.html.outputs.html }}
template: 'SysbenchTemplate'
- name: Read CSV
if: ${{ github.event.client_payload.issue_id }} != ""
.github/workflows/k8s-benchmark-latency.yaml
+1 -1
View File
@@ -10,7 +10,7 @@ jobs:
name: Benchmark Performance
strategy:
matrix:
dolt_fmt: [ "__LD_1__", "__DOLT__" ]
dolt_fmt: [ "__DOLT__" ]
steps:
- name: Checkout
uses: actions/checkout@v3
.github/workflows/nightly-performance-benchmarks-email-report.yaml
+15
View File
@@ -29,6 +29,7 @@ jobs:
"email_recipient": "${{ secrets.PERF_REPORTS_EMAIL_ADDRESS }}",
"version": "${{ github.sha }}",
"run_file": "ci.yaml",
"summary": "summary.sql",
"report": "three_way_compare.sql",
"commit_to_branch": "nightly",
"actor": "${{ github.actor }}"
@@ -37,3 +38,17 @@ jobs:
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: test-orm-integrations
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-systab
client-payload: |
{
"email_recipient": "${{ secrets.PERF_REPORTS_EMAIL_ADDRESS }}",
"version": "${{ github.sha }}",
"run_file": "systab.yaml",
"report": "systab.sql",
"summary": "systab_summary.sql",
"commit_to_branch": "nightly",
"actor": "${{ github.actor }}"
}
.github/workflows/performance-benchmarks-email-report.yaml
+16 -1
View File
@@ -50,10 +50,25 @@ jobs:
event-type: benchmark-import
client-payload: |
{
"email_recipient": "${{ secrets.PERF_REPORTS_EMAIL_ADDRESS }}",
"email_recipient": "${{ needs.set-version-actor.outputs.actor_email }}",
"version": "${{ github.sha }}",
"run_file": "ci.yaml",
"report": "three_way_compare.sql",
"summary": "summary.sql",
"commit_to_branch": "main",
"actor": "${{ github.actor }}"
}
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-systab
client-payload: |
{
"email_recipient": "${{ needs.set-version-actor.outputs.actor_email }}",
"version": "${{ github.sha }}",
"run_file": "systab.yaml",
"report": "systab.sql",
"summary": "systab_summary.sql",
"commit_to_branch": "main",
"actor": "${{ github.actor }}"
}
.github/workflows/sysbench-benchmarks-pull-report.yaml
+73
View File
@@ -0,0 +1,73 @@
name: Run Systab Benchmark on Pull Requests
on:
pull_request:
types: [ opened ]
issue_comment:
types: [ created ]
jobs:
validate-commentor:
runs-on: ubuntu-22.04
outputs:
valid: ${{ steps.set_valid.outputs.valid }}
steps:
- uses: actions/checkout@v3
- name: Validate Commentor
id: set_valid
run: ./.github/scripts/performance-benchmarking/validate-commentor.sh "$ACTOR"
env:
ACTOR: ${{ github.actor }}
check-comments:
runs-on: ubuntu-22.04
needs: validate-commentor
if: ${{ needs.validate-commentor.outputs.valid == 'true' }}
outputs:
benchmark: ${{ steps.set_benchmark.outputs.benchmark }}
comment-body: ${{ steps.set_body.outputs.body }}
steps:
- name: Check for Deploy Trigger
uses: dolthub/pull-request-comment-trigger@master
id: check
with:
trigger: '#systab-benchmark'
reaction: rocket
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set Benchmark
if: ${{ steps.check.outputs.triggered == 'true' }}
id: set_benchmark
run: |
echo "benchmark=true" >> $GITHUB_OUTPUT
performance:
runs-on: ubuntu-22.04
needs: [validate-commentor, check-comments]
if: ${{ needs.check-comments.outputs.benchmark == 'true' }}
name: Trigger Benchmark Systab Workflow
steps:
- uses: dolthub/pull-request-comment-branch@v3
id: comment-branch
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get pull number
uses: actions/github-script@v6
id: get_pull_number
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: core.setOutput("pull_number", JSON.stringify(context.issue.number));
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-systab
client-payload: |
{
"version": "${{ steps.comment-branch.outputs.head_sha }}",
"run_file": "systab.yaml",
"report": "systab.sql",
"summary": "systab_summary.sql",
"commit_to_branch": "${{ steps.comment-branch.outputs.head_sha }}",
"actor": "${{ github.actor }}",
"issue_id": "${{ steps.get_pull_number.outputs.pull_number }}"
}
.github/workflows/sysbench-perf.yaml
+178
View File
@@ -0,0 +1,178 @@
name: Systab Benchmarks
on:
repository_dispatch:
types: [ benchmark-systab ]
env:
BENCH_DIR: 'go/performance/sysbench'
RESULT_TABLE_NAME: 'sysbench_results'
DOLTHUB_DB: 'import-perf/systab-perf'
jobs:
bench:
name: Benchmark
defaults:
run:
shell: bash
strategy:
fail-fast: true
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.x
id: go
uses: actions/setup-go@v3
with:
go-version: ^1.19
- name: Dolt version
id: version
run: |
version=${{ github.event.client_payload.version }}
- uses: actions/checkout@v3
with:
ref: ${{ github.event.client_payload.version }}
- name: install sysbench
run: |
curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | sudo bash
sudo apt -y install sysbench
- name: Install dolt
working-directory: ./go
run: go install ./cmd/dolt
- name: Clone sysbench scripts
run: |
scripts=$GITHUB_WORKSPACE/scripts
git clone https://github.com/dolthub/systab-sysbench-scripts.git "$scripts"
- name: Run bench
id: bench
working-directory: go/
run: |
out="$GITHUB_WORKSPACE/results.sql"
testspec="../${{ env.BENCH_DIR }}/testdata/${{ github.event.client_payload.run_file }}"
config="../${{ env.BENCH_DIR }}/testdata/default-config.yaml"
scripts="$GITHUB_WORKSPACE/scripts"
go run \
"github.com/dolthub/dolt/${{ env.BENCH_DIR }}/cmd" \
-run "$testspec" \
-config "$config" \
-script-dir "$scripts" \
-out "$out"
echo "::set-output name=result_path::$out"
- name: Report
id: report
run: |
gw=$GITHUB_WORKSPACE
in="${{ steps.bench.outputs.result_path }}"
query="$(pwd)/${{ env.BENCH_DIR }}/reporting/${{ github.event.client_payload.report }}"
summaryq="$(pwd)/${{ env.BENCH_DIR }}/reporting/${{ github.event.client_payload.summary }}"
out="$gw/results.csv"
dolt_dir="$gw/systab-perf"
dolt config --global --add user.email "systab-perf@dolthub.com"
dolt config --global --add user.name "systab-perf"
echo '${{ secrets.DOLTHUB_IMPORT_PERF_CREDS_VALUE }}' | dolt creds import
dolt clone ${{ env.DOLTHUB_DB }} "$dolt_dir"
cd "$dolt_dir"
branch="${{ github.event.client_payload.commit_to_branch }}"
# checkout branch
if [ -z $(dolt sql -q "select 1 from dolt_branches where name = '$branch';") ]; then
dolt checkout -b $branch
else
dolt checkout $branch
fi
dolt sql -q "drop table if exists sysbench_results"
# load results
dolt sql < "$in"
# push results to dolthub
dolt add sysbench_results
dolt commit -m "CI commit"
dolt push -f origin $branch
# generate report
dolt sql -r csv < "$query" > "$out"
cat "$out"
echo "::set-output name=report_path::$out"
avg=$(dolt sql -r csv < "$summaryq" | tail -1)
echo "::set-output name=avg::$avg"
- name: Format Results
id: html
if: ${{ github.event.client_payload.email_recipient }} != ""
run: |
gw="$GITHUB_WORKSPACE"
in="${{ steps.report.outputs.report_path }}"
out="$gw/results.html"
echo "<table>" > "$out"
print_header=true
while read line; do
if "$print_header"; then
echo " <tr><th>${line//,/</th><th>}</th></tr>" >> "$out"
print_header=false
continue
fi
echo " <tr><td>${line//,/</td><td>}</td></tr>" >> "$out"
done < "$in"
echo "</table>" >> "$out"
avg="${{ steps.report.outputs.avg }}"
echo "<table><tr><th>Average</th></tr><tr><td>$avg</tr></td></table>" >> "$out"
cat "$out"
echo "::set-output name=html::$(echo $out)"
- name: Configure AWS Credentials
if: ${{ github.event.client_payload.email_recipient }} != ""
uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Send Email
uses: ./.github/actions/ses-email-action
if: ${{ github.event.client_payload.email_recipient }} != ""
with:
region: us-west-2
toAddresses: '["${{ github.event.client_payload.email_recipient }}"]'
subject: 'System Table Performance Benchmarks: ${{ github.event.client_payload.version }}'
bodyPath: ${{ steps.html.outputs.html }}
template: 'SysbenchTemplate'
- name: Read CSV
if: ${{ github.event.client_payload.issue_id }} != ""
id: csv
uses: juliangruber/read-file-action@v1
with:
path: "${{ steps.report.outputs.report_path }}"
- name: Create MD
if: ${{ github.event.client_payload.issue_id }} != ""
uses: petems/csv-to-md-table-action@master
id: md
with:
csvinput: ${{ steps.csv.outputs.content }}
- uses: mshick/add-pr-comment@v2
if: ${{ github.event.client_payload.issue_id }} != ""
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
issue: ${{ github.event.client_payload.issue_id }}
message-failure: systab benchmark failed
message-cancelled: systab benchmark cancelled
allow-repeats: true
message: |
@${{ github.event.client_payload.actor }} __DOLT__
${{ steps.md.outputs.markdown-table }}
go/cmd/dolt/commands/checkout.go
+15 -1
View File
@@ -120,7 +120,12 @@ func (cmd CheckoutCmd) Exec(ctx context.Context, commandStr string, args []strin
if err != nil {
return HandleVErrAndExitCode(errhand.BuildDError(err.Error()).Build(), usagePrt)
}
verr := actions.ResetHard(ctx, dEnv, "HEAD", roots)
headRef := dEnv.RepoStateReader().CWBHeadRef()
ws, err := dEnv.WorkingSet(ctx)
if err != nil {
HandleVErrAndExitCode(errhand.BuildDError(err.Error()).Build(), usagePrt)
}
verr := actions.ResetHard(ctx, dEnv, "HEAD", roots, headRef, ws)
return handleResetError(verr, usagePrt)
}
@@ -292,6 +297,15 @@ func checkoutBranch(ctx context.Context, dEnv *env.DoltEnv, name string, force b
// Being on the same branch shouldn't be an error
cli.Printf("Already on branch '%s'\n", name)
return nil
} else if err == actions.ErrWorkingSetsOnBothBranches {
str := fmt.Sprintf("error: There are uncommitted changes already on branch '%s'.", name) +
"This can happen when someone modifies that branch in a SQL session." +
fmt.Sprintf("You have uncommitted changes on this branch, and they would overwrite the uncommitted changes on branch %s on checkout.", name) +
"To solve this problem, you can " +
"1) commit or reset your changes on this branch, using `dolt commit` or `dolt reset`, before checking out the other branch, " +
"2) use the `-f` flag with `dolt checkout` to force an overwrite, or " +
"3) connect to branch '%s' with the SQL server and revert or commit changes there before proceeding."
return errhand.BuildDError(str).AddCause(err).Build()
} else {
bdr := errhand.BuildDError("fatal: Unexpected error checking out branch '%s'", name)
bdr.AddCause(err)
go/cmd/dolt/commands/filter-branch.go
+49 -6
View File
@@ -43,7 +43,8 @@ import (
)
const (
dbName = "filterDB"
dbName = "filterDB"
branchesFlag = "branches"
)
var filterBranchDocs = cli.CommandDocumentationContent{
@@ -52,7 +53,9 @@ var filterBranchDocs = cli.CommandDocumentationContent{
If a {{.LessThan}}commit-spec{{.GreaterThan}} is provided, the traversal will stop when the commit is reached and rewriting will begin at that commit, or will error if the commit is not found.
If the {{.EmphasisLeft}}--all{{.EmphasisRight}} flag is supplied, the traversal starts with the HEAD commits of all branches.
If the {{.EmphasisLeft}}--branches{{.EmphasisRight}} flag is supplied, filter-branch traverses and rewrites commits for all branches.
If the {{.EmphasisLeft}}--all{{.EmphasisRight}} flag is supplied, filter-branch traverses and rewrites commits for all branches and tags.
`,
Synopsis: []string{
@@ -81,7 +84,9 @@ func (cmd FilterBranchCmd) Docs() *cli.CommandDocumentation {
func (cmd FilterBranchCmd) ArgParser() *argparser.ArgParser {
ap := argparser.NewArgParser()
ap.SupportsFlag(allFlag, "a", "filter all branches")
ap.SupportsFlag(verboseFlag, "v", "logs more information")
ap.SupportsFlag(branchesFlag, "b", "filter all branches")
ap.SupportsFlag(allFlag, "a", "filter all branches and tags")
return ap
}
@@ -107,9 +112,44 @@ func (cmd FilterBranchCmd) Exec(ctx context.Context, commandStr string, args []s
}
query := apr.Arg(0)
verbose := apr.Contains(verboseFlag)
notFound := make(missingTbls)
replay := func(ctx context.Context, commit, _, _ *doltdb.Commit) (*doltdb.RootValue, error) {
return processFilterQuery(ctx, dEnv, commit, query, notFound)
var cmHash, before hash.Hash
if verbose {
var err error
cmHash, err = commit.HashOf()
if err != nil {
return nil, err
}
cli.Printf("processing commit %s\n", cmHash.String())
root, err := commit.GetRootValue(ctx)
if err != nil {
return nil, err
}
before, err = root.HashOf()
if err != nil {
return nil, err
}
}
root, err := processFilterQuery(ctx, dEnv, commit, query, notFound)
if err != nil {
return nil, err
}
if verbose {
after, err := root.HashOf()
if err != nil {
return nil, err
}
if before != after {
cli.Printf("updated commit %s (root: %s -> %s)\n",
cmHash.String(), before.String(), after.String())
}
}
return root, nil
}
nerf, err := getNerf(ctx, dEnv, apr)
@@ -117,9 +157,12 @@ func (cmd FilterBranchCmd) Exec(ctx context.Context, commandStr string, args []s
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
if apr.Contains(allFlag) {
switch {
case apr.Contains(branchesFlag):
err = rebase.AllBranches(ctx, dEnv, replay, nerf)
} else {
case apr.Contains(allFlag):
err = rebase.AllBranchesAndTags(ctx, dEnv, replay, nerf)
default:
err = rebase.CurrentBranch(ctx, dEnv, replay, nerf)
}
if err != nil {
go/cmd/dolt/commands/read_tables.go
+2 -1
View File
@@ -25,6 +25,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
"github.com/dolthub/dolt/go/libraries/utils/earl"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/types"
)
@@ -189,7 +190,7 @@ func pullTableValue(ctx context.Context, dEnv *env.DoltEnv, srcDB *doltdb.DoltDB
cli.Println("Retrieving", tblName)
runProgFunc := buildProgStarter(language)
wg, progChan, pullerEventCh := runProgFunc(newCtx)
err = dEnv.DoltDB.PullChunks(ctx, tmpDir, srcDB, tblHash, progChan, pullerEventCh)
err = dEnv.DoltDB.PullChunks(ctx, tmpDir, srcDB, []hash.Hash{tblHash}, progChan, pullerEventCh)
stopProgFuncs(cancelFunc, wg, progChan, pullerEventCh)
if err != nil {
return nil, errhand.BuildDError("Failed reading chunks for remote table '%s' at '%s'", tblName, commitStr).AddCause(err).Build()
go/cmd/dolt/commands/reset.go
+7 -1
View File
@@ -106,7 +106,13 @@ func (cmd ResetCmd) Exec(ctx context.Context, commandStr string, args []string,
arg = apr.Arg(0)
}
err = actions.ResetHard(ctx, dEnv, arg, roots)
headRef := dEnv.RepoStateReader().CWBHeadRef()
ws, err := dEnv.WorkingSet(ctx)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
err = actions.ResetHard(ctx, dEnv, arg, roots, headRef, ws)
} else {
// Check whether the input argument is a ref.
if apr.NArg() == 1 {
go/cmd/dolt/commands/sql.go
+38 -69
View File
@@ -21,7 +21,6 @@ import (
"os"
"os/signal"
"path/filepath"
"regexp"
"strings"
"syscall"
@@ -111,8 +110,6 @@ const (
# "exit" or "quit" (or Ctrl-D) to exit.`
)
var delimiterRegex = regexp.MustCompile(`(?i)^\s*DELIMITER\s+(\S+)\s*(\s+\S+\s*)?$`)
func init() {
dsqle.AddDoltSystemVariables()
}
@@ -809,47 +806,39 @@ func runMultiStatementMode(ctx *sql.Context, se *engine.SqlEngine, input io.Read
if len(query) == 0 || query == "\n" {
continue
}
shouldProcessQuery := true
if matches := delimiterRegex.FindStringSubmatch(query); len(matches) == 3 {
// If we don't match from anything, then we just pass to the SQL engine and let it complain.
scanner.Delimiter = matches[1]
shouldProcessQuery = false
sqlStatement, err := sqlparser.Parse(query)
if err == sqlparser.ErrEmpty {
continue
} else if err != nil {
handleError(scanner.statementStartLine, query, err)
// If continueOnErr is set keep executing the remaining queries but print the error out anyway.
if !continueOnErr {
return err
}
}
if shouldProcessQuery {
sqlStatement, err := sqlparser.Parse(query)
if err == sqlparser.ErrEmpty {
continue
} else if err != nil {
handleError(scanner.statementStartLine, query, err)
// If continueOnErr is set keep executing the remaining queries but print the error out anyway.
if !continueOnErr {
return err
sqlSch, rowIter, err := processParsedQuery(ctx, query, se, sqlStatement)
if err != nil {
handleError(scanner.statementStartLine, query, err)
// If continueOnErr is set keep executing the remaining queries but print the error out anyway.
if !continueOnErr {
return err
}
}
if rowIter != nil {
switch sqlStatement.(type) {
case *sqlparser.Select, *sqlparser.Insert, *sqlparser.Update, *sqlparser.Delete,
*sqlparser.OtherRead, *sqlparser.Show, *sqlparser.Explain, *sqlparser.Union:
// For any statement that prints out result, print a newline to put the regular output on its own line
if fileReadProg != nil {
fileReadProg.printNewLineIfNeeded()
}
}
sqlSch, rowIter, err := processParsedQuery(ctx, query, se, sqlStatement)
err = engine.PrettyPrintResults(ctx, se.GetResultFormat(), sqlSch, rowIter)
if err != nil {
handleError(scanner.statementStartLine, query, err)
// If continueOnErr is set keep executing the remaining queries but print the error out anyway.
if !continueOnErr {
return err
}
}
if rowIter != nil {
switch sqlStatement.(type) {
case *sqlparser.Select, *sqlparser.Insert, *sqlparser.Update, *sqlparser.Delete,
*sqlparser.OtherRead, *sqlparser.Show, *sqlparser.Explain, *sqlparser.Union:
// For any statement that prints out result, print a newline to put the regular output on its own line
if fileReadProg != nil {
fileReadProg.printNewLineIfNeeded()
}
}
err = engine.PrettyPrintResults(ctx, se.GetResultFormat(), sqlSch, rowIter)
if err != nil {
handleError(scanner.statementStartLine, query, err)
return err
}
return err
}
}
query = ""
@@ -881,22 +870,14 @@ func runBatchMode(ctx *sql.Context, se *engine.SqlEngine, input io.Reader, conti
if len(query) == 0 || query == "\n" {
continue
}
shouldProcessQuery := true
if matches := delimiterRegex.FindStringSubmatch(query); len(matches) == 3 {
// If we don't match from anything, then we just pass to the SQL engine and let it complain.
scanner.Delimiter = matches[1]
shouldProcessQuery = false
}
if shouldProcessQuery {
if err := processBatchQuery(ctx, query, se); err != nil {
// TODO: this line number will not be accurate for errors that occur when flushing a batch of inserts (as opposed
// to processing the query)
verr := formatQueryError(fmt.Sprintf("error on line %d for query %s", scanner.statementStartLine, query), err)
cli.PrintErrln(verr.Verbose())
// If continueOnErr is set keep executing the remaining queries but print the error out anyway.
if !continueOnErr {
return err
}
if err := processBatchQuery(ctx, query, se); err != nil {
// TODO: this line number will not be accurate for errors that occur when flushing a batch of inserts (as opposed
// to processing the query)
verr := formatQueryError(fmt.Sprintf("error on line %d for query %s", scanner.statementStartLine, query), err)
cli.PrintErrln(verr.Verbose())
// If continueOnErr is set keep executing the remaining queries but print the error out anyway.
if !continueOnErr {
return err
}
}
query = ""
@@ -1002,30 +983,18 @@ func runShell(ctx context.Context, se *engine.SqlEngine, mrEnv *env.MultiRepoEnv
query = strings.TrimSuffix(query, shell.LineTerminator())
resultFormat := se.GetResultFormat()
// TODO: it would be better to build this into the statement parser rahter than special case it here
// TODO: it would be better to build this into the statement parser rather than special case it here
for _, terminator := range verticalOutputLineTerminators {
if strings.HasSuffix(query, "\\G") {
if strings.HasSuffix(query, terminator) {
resultFormat = engine.FormatVertical
}
query = strings.TrimSuffix(query, terminator)
}
//TODO: Handle comments and enforce the current line terminator
if matches := delimiterRegex.FindStringSubmatch(query); len(matches) == 3 {
// If we don't match from anything, then we just pass to the SQL engine and let it complain.
shell.SetLineTerminator(matches[1])
return
}
var nextPrompt string
var sqlSch sql.Schema
var rowIter sql.RowIter
// The SQL parser does not understand any other terminator besides semicolon, so we remove it.
if shell.LineTerminator() != ";" && strings.HasSuffix(query, shell.LineTerminator()) {
query = query[:len(query)-len(shell.LineTerminator())]
}
cont := func() bool {
subCtx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
defer stop()
go/cmd/dolt/commands/sql_statement_scanner.go
+4 -2
View File
@@ -54,7 +54,7 @@ const (
backtick = '`'
)
var scannerDelimiterRegex = regexp.MustCompile(`(?i)^\s*DELIMITER\s+(\S+)[ \t]*([\n]+|\S+\s*)?`)
var scannerDelimiterRegex = regexp.MustCompile(`(?i)^\s*DELIMITER\s+(\S+)\s*`)
// ScanStatements is a split function for a Scanner that returns each SQL statement in the input as a token.
func (s *statementScanner) scanStatements(data []byte, atEOF bool) (advance int, token []byte, err error) {
@@ -74,7 +74,9 @@ func (s *statementScanner) scanStatements(data []byte, atEOF bool) (advance int,
s.startLineNum = s.lineNum
if idxs := scannerDelimiterRegex.FindIndex(data); len(idxs) == 2 {
return idxs[1], data[0:idxs[1]], nil
s.Delimiter = scannerDelimiterRegex.FindStringSubmatch(string(data))[1]
// Returning a nil token is interpreted as an error condition, so we return an empty token instead
return idxs[1], []byte{}, nil
}
for i := 0; i < len(data); i++ {
go/cmd/dolt/dolt.go
+1 -1
View File
@@ -56,7 +56,7 @@ import (
)
const (
Version = "0.51.10"
Version = "0.51.13"
)
var dumpDocsCommand = &commands.DumpDocsCmd{}
go/go.mod
+3 -3
View File
@@ -13,9 +13,9 @@ require (
github.com/denisbrodbeck/machineid v1.0.1
github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20201005193433-3ee972b1d078
github.com/dolthub/fslock v0.0.3
github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371
github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
github.com/dolthub/vitess v0.0.0-20221201212538-2fc2edcc1ee6
github.com/dolthub/vitess v0.0.0-20221213234424-b21f99e8027d
github.com/dustin/go-humanize v1.0.0
github.com/fatih/color v1.13.0
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@@ -58,7 +58,7 @@ require (
github.com/cenkalti/backoff/v4 v4.1.3
github.com/cespare/xxhash v1.1.0
github.com/creasty/defaults v1.6.0
github.com/dolthub/go-mysql-server v0.14.1-0.20221207213844-250562a12676
github.com/dolthub/go-mysql-server v0.14.1-0.20221219174548-47116e7701cc
github.com/google/flatbuffers v2.0.6+incompatible
github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
github.com/mitchellh/go-ps v1.0.0
go/go.sum
+6 -7
View File
@@ -59,7 +59,6 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg=
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8=
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530=
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
@@ -162,16 +161,16 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC
github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE=
github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-mysql-server v0.14.1-0.20221207213844-250562a12676 h1:kTzDSFVsd8Cs9yoVfuvynzeV9KGkH4p0IzIsYsJQNnA=
github.com/dolthub/go-mysql-server v0.14.1-0.20221207213844-250562a12676/go.mod h1:Wnl8wxOYGCqvMPmpxzfzEY7hMo+onCoC7n8crNKBl3A=
github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g=
github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms=
github.com/dolthub/go-mysql-server v0.14.1-0.20221219174548-47116e7701cc h1:889/n0VyMQW2vZS4yLq8skR4hvrgXFHsb0TaRhOEZHo=
github.com/dolthub/go-mysql-server v0.14.1-0.20221219174548-47116e7701cc/go.mod h1:t/oSRkxu+4cg7KAJ4mK6q62hdzkyeTmRqKrsSq2DZTU=
github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514=
github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto=
github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=
github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474/go.mod h1:kMz7uXOXq4qRriCEyZ/LUeTqraLJCjf0WVZcUi6TxUY=
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE=
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
github.com/dolthub/vitess v0.0.0-20221201212538-2fc2edcc1ee6 h1:dkS8u6XOdMmpZtq8CzEsQYc4X96EshHBkm0Pbu187rk=
github.com/dolthub/vitess v0.0.0-20221201212538-2fc2edcc1ee6/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs=
github.com/dolthub/vitess v0.0.0-20221213234424-b21f99e8027d h1:SFBb6boDelnyNAA/wefNvqKtxlITLGNBMO1cagUZPVw=
github.com/dolthub/vitess v0.0.0-20221213234424-b21f99e8027d/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
go/libraries/doltcore/dbfactory/file.go
+14 -1
View File
@@ -65,6 +65,13 @@ func CloseAllLocalDatabases() (err error) {
return
}
func DeleteFromSingletonCache(path string) error {
singletonLock.Lock()
defer singletonLock.Unlock()
delete(singletons, path)
return nil
}
// PrepareDB creates the directory for the DB if it doesn't exist, and returns an error if a file or symlink is at the
// path given
func (fact FileFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat, u *url.URL, params map[string]interface{}) error {
@@ -113,8 +120,14 @@ func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat,
if err != nil {
return nil, nil, nil, err
}
var newGenSt *nbs.NomsBlockStore
q := nbs.NewUnlimitedMemQuotaProvider()
newGenSt, err := nbs.NewLocalStore(ctx, nbf.VersionString(), path, defaultMemTableSize, q)
if nbs.ChunkJournalFeatureFlag {
newGenSt, err = nbs.NewLocalJournalingStore(ctx, nbf.VersionString(), path, q)
} else {
newGenSt, err = nbs.NewLocalStore(ctx, nbf.VersionString(), path, defaultMemTableSize, q)
}
if err != nil {
return nil, nil, nil, err
go/libraries/doltcore/doltdb/commit_hooks.go
+1 -1
View File
@@ -60,7 +60,7 @@ func pushDataset(ctx context.Context, destDB, srcDB datas.Database, ds datas.Dat
return err
}
err := pullHash(ctx, destDB, srcDB, addr, tmpDir, nil, nil)
err := pullHash(ctx, destDB, srcDB, []hash.Hash{addr}, tmpDir, nil, nil)
if err != nil {
return err
}
go/libraries/doltcore/doltdb/doltdb.go
+9 -5
View File
@@ -215,6 +215,10 @@ func (ddb *DoltDB) WriteEmptyRepoWithCommitTimeAndDefaultBranch(
return err
}
func (ddb *DoltDB) Close() error {
return ddb.db.Close()
}
func getCommitValForRefStr(ctx context.Context, db datas.Database, vrw types.ValueReadWriter, ref string) (*datas.Commit, error) {
if err := datas.ValidateDatasetId(ref); err != nil {
return nil, fmt.Errorf("invalid ref format: %s", ref)
@@ -1263,17 +1267,17 @@ func (ddb *DoltDB) PullChunks(
ctx context.Context,
tempDir string,
srcDB *DoltDB,
targetHash hash.Hash,
targetHashes [] hash.Hash,
progChan chan pull.PullProgress,
statsCh chan pull.Stats,
) error {
return pullHash(ctx, ddb.db, srcDB.db, targetHash, tempDir, progChan, statsCh)
return pullHash(ctx, ddb.db, srcDB.db, targetHashes, tempDir, progChan, statsCh)
}
func pullHash(
ctx context.Context,
destDB, srcDB datas.Database,
targetHash hash.Hash,
targetHashes [] hash.Hash,
tempDir string,
progChan chan pull.PullProgress,
statsCh chan pull.Stats,
@@ -1283,7 +1287,7 @@ func pullHash(
waf := types.WalkAddrsForNBF(srcDB.Format())
if datas.CanUsePuller(srcDB) && datas.CanUsePuller(destDB) {
puller, err := pull.NewPuller(ctx, tempDir, defaultChunksPerTF, srcCS, destCS, waf, targetHash, statsCh)
puller, err := pull.NewPuller(ctx, tempDir, defaultChunksPerTF, srcCS, destCS, waf, targetHashes, statsCh)
if err == pull.ErrDBUpToDate {
return nil
} else if err != nil {
@@ -1292,7 +1296,7 @@ func pullHash(
return puller.Pull(ctx)
} else {
return pull.Pull(ctx, srcCS, destCS, waf, targetHash, progChan)
return pull.Pull(ctx, srcCS, destCS, waf, targetHashes, progChan)
}
}
go/libraries/doltcore/env/actions/branch.go
Vendored
+81 -3
View File
@@ -20,7 +20,6 @@ import (
"fmt"
"github.com/dolthub/dolt/go/libraries/doltcore/branch_control"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
@@ -31,6 +30,7 @@ import (
var ErrAlreadyExists = errors.New("already exists")
var ErrCOBranchDelete = errors.New("attempted to delete checked out branch")
var ErrUnmergedBranchDelete = errors.New("attempted to delete a branch that is not fully merged into its parent; use `-f` to force")
var ErrWorkingSetsOnBothBranches = errors.New("checkout would overwrite uncommitted changes on target branch")
func RenameBranch(ctx context.Context, dbData env.DbData, config *env.DoltCliConfig, oldBranch, newBranch string, force bool) error {
oldRef := ref.NewBranchRef(oldBranch)
@@ -306,6 +306,7 @@ func checkoutBranchNoDocs(ctx context.Context, roots doltdb.Roots, branchRoot *d
func CheckoutBranch(ctx context.Context, dEnv *env.DoltEnv, brName string, force bool) error {
branchRef := ref.NewBranchRef(brName)
branchHeadRef := dEnv.RepoStateReader().CWBHeadRef()
db := dEnv.DoltDB
hasRef, err := db.HasRef(ctx, branchRef)
@@ -325,15 +326,43 @@ func CheckoutBranch(ctx context.Context, dEnv *env.DoltEnv, brName string, force
return err
}
currentWs, err := dEnv.WorkingSet(ctx)
if err != nil {
// working set does not exist, skip check
return nil
}
if !force {
err = checkWorkingSetCompatibility(ctx, dEnv, branchRef, currentWs)
if err != nil {
return err
}
}
shouldResetWorkingSet := true
roots, err := dEnv.Roots(ctx)
// roots will be empty/nil if the working set is not set (working set is not set if the current branch was deleted)
if errors.Is(err, doltdb.ErrBranchNotFound) || errors.Is(err, doltdb.ErrWorkingSetNotFound) {
roots, err = dEnv.RecoveryRoots(ctx)
roots, _ = dEnv.RecoveryRoots(ctx)
shouldResetWorkingSet = false
} else if err != nil {
return err
}
return checkoutBranchNoDocs(ctx, roots, branchRoot, dEnv.RepoStateWriter(), branchRef, force)
err = checkoutBranchNoDocs(ctx, roots, branchRoot, dEnv.RepoStateWriter(), branchRef, force)
if err != nil {
return err
}
if shouldResetWorkingSet {
// reset the source branch's working set to the branch head, leaving the source branch unchanged
err = ResetHard(ctx, dEnv, "", roots, branchHeadRef, currentWs)
if err != nil {
return err
}
}
return nil
}
// BranchRoot returns the root value at the branch with the name given
@@ -462,6 +491,55 @@ func overwriteRoot(ctx context.Context, head *doltdb.RootValue, tblHashes map[st
return head, nil
}
// checkWorkingSetCompatibility checks that the current working set is "compatible" with the dest working set.
// This means that if both working sets are present (ie there are changes on both source and dest branches),
// we check if the changes are identical before allowing a clobbering checkout.
// Working set errors are ignored by this function, because they are properly handled elsewhere.
func checkWorkingSetCompatibility(ctx context.Context, dEnv *env.DoltEnv, branchRef ref.BranchRef, currentWs *doltdb.WorkingSet) error {
db := dEnv.DoltDB
destWsRef, err := ref.WorkingSetRefForHead(branchRef)
if err != nil {
// dest working set does not exist, skip check
return nil
}
destWs, err := db.ResolveWorkingSet(ctx, destWsRef)
if err != nil {
// dest working set does not resolve, skip check
return nil
}
sourceHasChanges, sourceHash, err := detectWorkingSetChanges(currentWs)
if err != nil {
// error detecting source changes, skip check
return nil
}
destHasChanges, destHash, err := detectWorkingSetChanges(destWs)
if err != nil {
// error detecting dest changes, skip check
return nil
}
areHashesEqual := sourceHash.Equal(destHash)
if sourceHasChanges && destHasChanges && !areHashesEqual {
return ErrWorkingSetsOnBothBranches
}
return nil
}
// detectWorkingSetChanges returns a boolean indicating whether the working set has changes, and a hash of the changes
func detectWorkingSetChanges(ws *doltdb.WorkingSet) (hasChanges bool, wrHash hash.Hash, err error) {
wrHash, err = ws.WorkingRoot().HashOf()
if err != nil {
return false, hash.Hash{}, err
}
srHash, err := ws.StagedRoot().HashOf()
if err != nil {
return false, hash.Hash{}, err
}
hasChanges = !wrHash.Equal(srHash)
return hasChanges, wrHash, nil
}
func IsBranch(ctx context.Context, ddb *doltdb.DoltDB, str string) (bool, error) {
return IsBranchOnDB(ctx, ddb, str)
}
go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go
Vendored
+1 -1
View File
@@ -273,7 +273,7 @@ func mustForkDB(t *testing.T, fromDB *doltdb.DoltDB, bn string, cm *doltdb.Commi
for range p2 {
}
}()
err = forkEnv.DoltDB.PullChunks(context.Background(), "", fromDB, h, p1, p2)
err = forkEnv.DoltDB.PullChunks(context.Background(), "", fromDB, []hash.Hash{h}, p1, p2)
if err == pull.ErrDBUpToDate {
err = nil
}
go/libraries/doltcore/env/actions/remotes.go
Vendored
+6 -5
View File
@@ -29,6 +29,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/earl"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/datas/pull"
"github.com/dolthub/dolt/go/store/hash"
)
var ErrCantFF = errors.New("can't fast forward merge")
@@ -66,7 +67,7 @@ func Push(ctx context.Context, tempTableDir string, mode ref.UpdateMode, destRef
return err
}
err = destDB.PullChunks(ctx, tempTableDir, srcDB, h, progChan, statsCh)
err = destDB.PullChunks(ctx, tempTableDir, srcDB, []hash.Hash{h}, progChan, statsCh)
if err != nil {
return err
@@ -132,7 +133,7 @@ func PushTag(ctx context.Context, tempTableDir string, destRef ref.TagRef, srcDB
return err
}
err = destDB.PullChunks(ctx, tempTableDir, srcDB, addr, progChan, statsCh)
err = destDB.PullChunks(ctx, tempTableDir, srcDB, []hash.Hash{addr}, progChan, statsCh)
if err != nil {
return err
@@ -239,7 +240,7 @@ func FetchCommit(ctx context.Context, tempTablesDir string, srcDB, destDB *doltd
return err
}
return destDB.PullChunks(ctx, tempTablesDir, srcDB, h, progChan, statsCh)
return destDB.PullChunks(ctx, tempTablesDir, srcDB, []hash.Hash{h}, progChan, statsCh)
}
// FetchTag takes a fetches a commit tag and all underlying data from a remote source database to the local destination database.
@@ -249,7 +250,7 @@ func FetchTag(ctx context.Context, tempTableDir string, srcDB, destDB *doltdb.Do
return err
}
return destDB.PullChunks(ctx, tempTableDir, srcDB, addr, progChan, statsCh)
return destDB.PullChunks(ctx, tempTableDir, srcDB, []hash.Hash{addr}, progChan, statsCh)
}
// Clone pulls all data from a remote source database to a local destination database.
@@ -479,7 +480,7 @@ func SyncRoots(ctx context.Context, srcDb, destDb *doltdb.DoltDB, tempTableDir s
}
}()
err = destDb.PullChunks(ctx, tempTableDir, srcDb, srcRoot, progChan, statsCh)
err = destDb.PullChunks(ctx, tempTableDir, srcDb, []hash.Hash{srcRoot}, progChan, statsCh)
if err != nil {
return err
}
go/libraries/doltcore/env/actions/reset.go
Vendored
+14 -9
View File
@@ -18,10 +18,10 @@ import (
"context"
"fmt"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
)
@@ -136,7 +136,17 @@ func ResetHardTables(ctx context.Context, dbData env.DbData, cSpecStr string, ro
return resetHardTables(ctx, dbData, cSpecStr, roots)
}
func ResetHard(ctx context.Context, dEnv *env.DoltEnv, cSpecStr string, roots doltdb.Roots) error {
// ResetHard resets the working, staged, and head to the ones in the provided roots and head ref.
// The reset can be performed on a non-current branch and working set.
// Returns an error if the reset fails.
func ResetHard(
ctx context.Context,
dEnv *env.DoltEnv,
cSpecStr string,
roots doltdb.Roots,
headRef ref.DoltRef,
ws *doltdb.WorkingSet,
) error {
dbData := dEnv.DbData()
newHead, roots, err := resetHardTables(ctx, dbData, cSpecStr, roots)
@@ -144,18 +154,13 @@ func ResetHard(ctx context.Context, dEnv *env.DoltEnv, cSpecStr string, roots do
return err
}
ws, err := dEnv.WorkingSet(ctx)
if err != nil {
return err
}
err = dEnv.UpdateWorkingSet(ctx, ws.WithWorkingRoot(roots.Working).WithStagedRoot(roots.Staged).ClearMerge())
if err != nil {
return err
}
if newHead != nil {
err = dEnv.DoltDB.SetHeadToCommit(ctx, dEnv.RepoStateReader().CWBHeadRef(), newHead)
err = dEnv.DoltDB.SetHeadToCommit(ctx, headRef, newHead)
if err != nil {
return err
}
go/libraries/doltcore/migrate/transform.go
+10 -5
View File
@@ -42,11 +42,6 @@ var (
)
func migrateWorkingSet(ctx context.Context, menv Environment, brRef ref.BranchRef, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB) error {
oldWs, err := old.ResolveWorkingSet(ctx, wsRef)
if err != nil {
return err
}
oldHead, err := old.ResolveCommitRef(ctx, brRef)
if err != nil {
return err
@@ -56,6 +51,16 @@ func migrateWorkingSet(ctx context.Context, menv Environment, brRef ref.BranchRe
return err
}
oldWs, err := old.ResolveWorkingSet(ctx, wsRef)
if err == doltdb.ErrWorkingSetNotFound {
// If a branch was created prior to dolt version 0.26.10, no working set will exist for it.
// In this case, we will pretend it exists with the same root as the head commit.
oldWs = doltdb.EmptyWorkingSet(wsRef)
oldWs = oldWs.WithWorkingRoot(oldHeadRoot).WithStagedRoot(oldHeadRoot)
} else if err != nil {
return err
}
newHead, err := new.ResolveCommitRef(ctx, brRef)
if err != nil {
return err
go/libraries/doltcore/rebase/rebase.go
+26 -28
View File
@@ -87,6 +87,20 @@ func wrapReplayRootFn(fn ReplayRootFn) ReplayCommitFn {
}
}
// AllBranchesAndTags rewrites the history of all branches and tags in the repo using the |replay| function.
func AllBranchesAndTags(ctx context.Context, dEnv *env.DoltEnv, replay ReplayCommitFn, nerf NeedsRebaseFn) error {
branches, err := dEnv.DoltDB.GetBranches(ctx)
if err != nil {
return err
}
tags, err := dEnv.DoltDB.GetTags(ctx)
if err != nil {
return err
}
return rebaseRefs(ctx, dEnv.DbData(), replay, nerf, append(branches, tags...)...)
}
// AllBranches rewrites the history of all branches in the repo using the |replay| function.
func AllBranches(ctx context.Context, dEnv *env.DoltEnv, replay ReplayCommitFn, nerf NeedsRebaseFn) error {
branches, err := dEnv.DoltDB.GetBranches(ctx)
@@ -121,11 +135,6 @@ func CurrentBranchByRoot(ctx context.Context, dEnv *env.DoltEnv, replay ReplayRo
func rebaseRefs(ctx context.Context, dbData env.DbData, replay ReplayCommitFn, nerf NeedsRebaseFn, refs ...ref.DoltRef) error {
ddb := dbData.Ddb
rsr := dbData.Rsr
rsw := dbData.Rsw
cwbRef := rsr.CWBHeadRef()
heads := make([]*doltdb.Commit, len(refs))
for i, dRef := range refs {
var err error
@@ -140,41 +149,30 @@ func rebaseRefs(ctx context.Context, dbData env.DbData, replay ReplayCommitFn, n
return err
}
for i, dRef := range refs {
switch dRef.(type) {
for i, r := range refs {
switch dRef := r.(type) {
case ref.BranchRef:
err = ddb.NewBranchAtCommit(ctx, dRef, newHeads[i])
if err != nil {
case ref.TagRef:
// rewrite tag with new commit
var tag *doltdb.Tag
if tag, err = ddb.ResolveTag(ctx, dRef); err != nil {
return err
}
if err = ddb.DeleteTag(ctx, dRef); err != nil {
return err
}
err = ddb.NewTagAtCommit(ctx, dRef, newHeads[i], tag.Meta)
default:
return fmt.Errorf("cannot rebase ref: %s", ref.String(dRef))
}
if err != nil {
return err
}
}
cm, err := ddb.ResolveCommitRef(ctx, cwbRef)
if err != nil {
return err
}
r, err := cm.GetRootValue(ctx)
if err != nil {
return err
}
// TODO: this should be a single update to repo state, not two
err = rsw.UpdateStagedRoot(ctx, r)
if err != nil {
return err
}
return rsw.UpdateWorkingRoot(ctx, r)
return nil
}
func rebase(ctx context.Context, ddb *doltdb.DoltDB, replay ReplayCommitFn, nerf NeedsRebaseFn, origins ...*doltdb.Commit) ([]*doltdb.Commit, error) {
go/libraries/doltcore/schema/typeinfo/decimal_test.go
+1 -1
View File
@@ -337,7 +337,7 @@ func TestDecimalMarshal(t *testing.T) {
"16976349273982359874209023948672021737840592720387475.271912873754", false},
{65, 12, "99999999999999999999999999999999999999999999999999999.9999999999999", "", true},
{20, 10, []byte{32}, "32", false},
{20, 10, []byte{32}, "", true},
{20, 10, time.Date(2019, 12, 12, 12, 12, 12, 0, time.UTC), "", true},
}
go/libraries/doltcore/sqle/cluster/commithook.go
+1 -1
View File
@@ -220,7 +220,7 @@ func (h *commithook) attemptReplicate(ctx context.Context) {
}
lgr.Tracef("cluster/commithook: pushing chunks for root hash %v to destDB", toPush.String())
err := destDB.PullChunks(ctx, h.tempDir, h.srcDB, toPush, nil, nil)
err := destDB.PullChunks(ctx, h.tempDir, h.srcDB, []hash.Hash{toPush}, nil, nil)
if err == nil {
lgr.Tracef("cluster/commithook: successfully pushed chunks, setting root")
datasDB := doltdb.HackDatasDatabaseFromDoltDB(destDB)
go/libraries/doltcore/sqle/database_provider.go
+14
View File
@@ -592,15 +592,29 @@ func (p DoltDatabaseProvider) DropDatabase(ctx *sql.Context, name string) error
dbKey := formatDbMapKeyName(name)
db := p.databases[dbKey]
ddb := db.(Database).ddb
err = ddb.Close()
if err != nil {
return err
}
// get location of database that's being dropped
dbLoc := p.dbLocations[dbKey]
if dbLoc == nil {
return sql.ErrDatabaseNotFound.New(db.Name())
}
dropDbLoc, err := dbLoc.Abs("")
if err != nil {
return err
}
// If this database is re-created, we don't want to return any cached results.
err = dbfactory.DeleteFromSingletonCache("file://" + dropDbLoc + "/.dolt/noms")
if err != nil {
return err
}
rootDbLoc, err := p.fs.Abs("")
if err != nil {
return err
go/libraries/doltcore/sqle/dfunctions/hashof.go
+12
View File
@@ -32,6 +32,8 @@ type HashOf struct {
expression.UnaryExpression
}
var _ sql.FunctionExpression = (*HashOf)(nil)
// NewHashOf creates a new HashOf expression.
func NewHashOf(e sql.Expression) sql.Expression {
return &HashOf{expression.UnaryExpression{Child: e}}
@@ -104,6 +106,16 @@ func (t *HashOf) String() string {
return fmt.Sprintf("HASHOF(%s)", t.Child.String())
}
// FunctionName implements the FunctionExpression interface
func (t *HashOf) FunctionName() string {
return HashOfFuncName
}
// Description implements the FunctionExpression interface
func (t *HashOf) Description() string {
return "returns the commit hash of a branch or other commit spec"
}
// IsNullable implements the Expression interface.
func (t *HashOf) IsNullable() bool {
return t.Child.IsNullable()
go/libraries/doltcore/sqle/dolt_diff_summary_table_function.go
+4
View File
@@ -193,6 +193,10 @@ func (ds *DiffSummaryTableFunction) WithExpressions(expression ...sql.Expression
if !expr.Resolved() {
return nil, ErrInvalidNonLiteralArgument.New(ds.Name(), expr.String())
}
// prepared statements resolve functions beforehand, so above check fails
if _, ok := expr.(sql.FunctionExpression); ok {
return nil, ErrInvalidNonLiteralArgument.New(ds.Name(), expr.String())
}
}
if strings.Contains(expression[0].String(), "..") {
go/libraries/doltcore/sqle/dolt_diff_table_function.go
+4
View File
@@ -105,6 +105,10 @@ func (dtf *DiffTableFunction) WithExpressions(expression ...sql.Expression) (sql
if !expr.Resolved() {
return nil, ErrInvalidNonLiteralArgument.New(dtf.Name(), expr.String())
}
// prepared statements resolve functions beforehand, so above check fails
if _, ok := expr.(sql.FunctionExpression); ok {
return nil, ErrInvalidNonLiteralArgument.New(dtf.Name(), expr.String())
}
}
if strings.Contains(expression[0].String(), "..") {
go/libraries/doltcore/sqle/dolt_log_table_function.go
+4
View File
@@ -252,6 +252,10 @@ func (ltf *LogTableFunction) WithExpressions(expression ...sql.Expression) (sql.
if !expr.Resolved() {
return nil, ErrInvalidNonLiteralArgument.New(ltf.Name(), expr.String())
}
// prepared statements resolve functions beforehand, so above check fails
if _, ok := expr.(sql.FunctionExpression); ok {
return nil, ErrInvalidNonLiteralArgument.New(ltf.Name(), expr.String())
}
}
if err := ltf.addOptions(expression); err != nil {
go/libraries/doltcore/sqle/dtables/commit_diff_table.go
+1 -1
View File
@@ -264,7 +264,7 @@ func (dt *CommitDiffTable) HandledFilters(filters []sql.Expression) []sql.Expres
// Filters returns the list of filters that are applied to this table.
func (dt *CommitDiffTable) Filters() []sql.Expression {
if dt.toCommitFilter == nil || dt.fromCommitFilter == nil {
if dt.toCommitFilter == nil && dt.fromCommitFilter == nil {
return nil
}
go/libraries/doltcore/sqle/dtables/diff_iter.go
+27 -15
View File
@@ -242,19 +242,31 @@ func newProllyDiffIter(ctx *sql.Context, dp DiffPartition, ddb *doltdb.DoltDB, t
name: dp.toName,
ts: (*time.Time)(dp.toDate),
}
var from, to prolly.Map
// dp.from may be nil
f, fSch, err := tableData(ctx, dp.from, ddb)
if err != nil {
return prollyDiffIter{}, nil
var fsch schema.Schema = schema.EmptySchema
if dp.from != nil {
idx, err := dp.from.GetRowData(ctx)
if err != nil {
return prollyDiffIter{}, err
}
from = durable.ProllyMapFromIndex(idx)
if fsch, err = dp.from.GetSchema(ctx); err != nil {
return prollyDiffIter{}, err
}
}
from := durable.ProllyMapFromIndex(f)
t, tSch, err := tableData(ctx, dp.to, ddb)
if err != nil {
return prollyDiffIter{}, nil
var tsch schema.Schema = schema.EmptySchema
if dp.to != nil {
idx, err := dp.to.GetRowData(ctx)
if err != nil {
return prollyDiffIter{}, err
}
to = durable.ProllyMapFromIndex(idx)
if tsch, err = dp.to.GetSchema(ctx); err != nil {
return prollyDiffIter{}, err
}
}
to := durable.ProllyMapFromIndex(t)
var nodeStore tree.NodeStore
if dp.to != nil {
@@ -263,25 +275,25 @@ func newProllyDiffIter(ctx *sql.Context, dp DiffPartition, ddb *doltdb.DoltDB, t
nodeStore = dp.from.NodeStore()
}
fromConverter, err := NewProllyRowConverter(fSch, targetFromSchema, ctx.Warn, nodeStore)
fromConverter, err := NewProllyRowConverter(fsch, targetFromSchema, ctx.Warn, nodeStore)
if err != nil {
return prollyDiffIter{}, err
}
toConverter, err := NewProllyRowConverter(tSch, targetToSchema, ctx.Warn, nodeStore)
toConverter, err := NewProllyRowConverter(tsch, targetToSchema, ctx.Warn, nodeStore)
if err != nil {
return prollyDiffIter{}, err
}
fromVD := fSch.GetValueDescriptor()
toVD := tSch.GetValueDescriptor()
fromVD := fsch.GetValueDescriptor()
toVD := tsch.GetValueDescriptor()
keyless := schema.IsKeyless(targetFromSchema) && schema.IsKeyless(targetToSchema)
child, cancel := context.WithCancel(ctx)
iter := prollyDiffIter{
from: from,
to: to,
fromSch: fSch,
toSch: tSch,
fromSch: fsch,
toSch: tsch,
targetFromSch: targetFromSchema,
targetToSch: targetToSchema,
fromConverter: fromConverter,
go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go
+9 -4
View File
@@ -46,7 +46,7 @@ var skipPrepared bool
// SkipPreparedsCount is used by the "ci-check-repo CI workflow
// as a reminder to consider prepareds when adding a new
// enginetest suite.
const SkipPreparedsCount = 84
const SkipPreparedsCount = 83
const skipPreparedFlag = "DOLT_SKIP_PREPARED_ENGINETESTS"
@@ -272,7 +272,7 @@ func TestDoltDiffQueryPlans(t *testing.T) {
defer e.Close()
for _, tt := range DoltDiffPlanTests {
enginetest.TestQueryPlan(t, harness, e, tt.Query, tt.ExpectedPlan)
enginetest.TestQueryPlan(t, harness, e, tt.Query, tt.ExpectedPlan, false)
}
}
@@ -690,8 +690,8 @@ func TestRollbackTriggers(t *testing.T) {
func TestStoredProcedures(t *testing.T) {
tests := make([]queries.ScriptTest, 0, len(queries.ProcedureLogicTests))
for _, test := range queries.ProcedureLogicTests {
//TODO: fix REPLACE always returning a successful deletion
if test.Name != "Parameters resolve inside of REPLACE" {
//TODO: this passes locally but SOMETIMES fails tests on GitHub, no clue why
if test.Name != "ITERATE and LEAVE loops" {
tests = append(tests, test)
}
}
@@ -1454,6 +1454,11 @@ func TestPreparedInsert(t *testing.T) {
enginetest.TestPreparedInsert(t, newDoltHarness(t))
}
func TestPreparedStatements(t *testing.T) {
skipPreparedTests(t)
enginetest.TestPreparedStatements(t, newDoltHarness(t))
}
func TestCharsetCollationEngine(t *testing.T) {
skipOldFormat(t)
enginetest.TestCharsetCollationEngine(t, newDoltHarness(t))
go/libraries/doltcore/sqle/enginetest/dolt_queries.go
+8 -5625
View File
File diff suppressed because it is too large Load Diff
go/libraries/doltcore/sqle/enginetest/dolt_queries_diff.go
+2566
View File
File diff suppressed because it is too large Load Diff
go/libraries/doltcore/sqle/enginetest/dolt_queries_merge.go
+3030
View File
File diff suppressed because it is too large Load Diff
go/libraries/doltcore/sqle/enginetest/dolt_query_plans.go
+10 -5
View File
@@ -22,7 +22,8 @@ import (
var DoltDiffPlanTests = []queries.QueryPlanTest{
{
Query: `select * from dolt_diff_one_pk where to_pk=1`,
ExpectedPlan: "Filter(dolt_diff_one_pk.to_pk = 1)\n" +
ExpectedPlan: "Filter\n" +
" ├─ (dolt_diff_one_pk.to_pk = 1)\n" +
" └─ IndexedTableAccess(dolt_diff_one_pk)\n" +
" ├─ index: [dolt_diff_one_pk.to_pk]\n" +
" └─ filters: [{[1, 1]}]\n" +
@@ -30,7 +31,8 @@ var DoltDiffPlanTests = []queries.QueryPlanTest{
},
{
Query: `select * from dolt_diff_one_pk where to_pk>=10 and to_pk<=100`,
ExpectedPlan: "Filter((dolt_diff_one_pk.to_pk >= 10) AND (dolt_diff_one_pk.to_pk <= 100))\n" +
ExpectedPlan: "Filter\n" +
" ├─ ((dolt_diff_one_pk.to_pk >= 10) AND (dolt_diff_one_pk.to_pk <= 100))\n" +
" └─ IndexedTableAccess(dolt_diff_one_pk)\n" +
" ├─ index: [dolt_diff_one_pk.to_pk]\n" +
" └─ filters: [{[10, 100]}]\n" +
@@ -38,7 +40,8 @@ var DoltDiffPlanTests = []queries.QueryPlanTest{
},
{
Query: `select * from dolt_diff_two_pk where to_pk1=1`,
ExpectedPlan: "Filter(dolt_diff_two_pk.to_pk1 = 1)\n" +
ExpectedPlan: "Filter\n" +
" ├─ (dolt_diff_two_pk.to_pk1 = 1)\n" +
" └─ IndexedTableAccess(dolt_diff_two_pk)\n" +
" ├─ index: [dolt_diff_two_pk.to_pk1,dolt_diff_two_pk.to_pk2]\n" +
" └─ filters: [{[1, 1], [NULL, ∞)}]\n" +
@@ -46,7 +49,8 @@ var DoltDiffPlanTests = []queries.QueryPlanTest{
},
{
Query: `select * from dolt_diff_two_pk where to_pk1=1 and to_pk2=2`,
ExpectedPlan: "Filter((dolt_diff_two_pk.to_pk1 = 1) AND (dolt_diff_two_pk.to_pk2 = 2))\n" +
ExpectedPlan: "Filter\n" +
" ├─ ((dolt_diff_two_pk.to_pk1 = 1) AND (dolt_diff_two_pk.to_pk2 = 2))\n" +
" └─ IndexedTableAccess(dolt_diff_two_pk)\n" +
" ├─ index: [dolt_diff_two_pk.to_pk1,dolt_diff_two_pk.to_pk2]\n" +
" └─ filters: [{[1, 1], [2, 2]}]\n" +
@@ -54,7 +58,8 @@ var DoltDiffPlanTests = []queries.QueryPlanTest{
},
{
Query: `select * from dolt_diff_two_pk where to_pk1 < 1 and to_pk2 > 10`,
ExpectedPlan: "Filter((dolt_diff_two_pk.to_pk1 < 1) AND (dolt_diff_two_pk.to_pk2 > 10))\n" +
ExpectedPlan: "Filter\n" +
" ├─ ((dolt_diff_two_pk.to_pk1 < 1) AND (dolt_diff_two_pk.to_pk2 > 10))\n" +
" └─ IndexedTableAccess(dolt_diff_two_pk)\n" +
" ├─ index: [dolt_diff_two_pk.to_pk1,dolt_diff_two_pk.to_pk2]\n" +
" └─ filters: [{(NULL, 1), (10, ∞)}]\n" +
go/libraries/doltcore/sqle/integration_test/stockmarket_test.go
+4 -2
View File
@@ -20282,9 +20282,11 @@ func TestExplain(t *testing.T) {
expectedExplain := "Project\n" +
" ├─ columns: [d.Type, d.Symbol, d.Country, d.TradingDate, d.Open, d.High, d.Low, d.Close, d.Volume, d.OpenInt, t.Symbol, t.Name, t.Sector, t.IPOYear]\n" +
" └─ LookupJoin(d.Symbol = t.Symbol)\n" +
" └─ LookupJoin\n" +
" ├─ (d.Symbol = t.Symbol)\n" +
" ├─ TableAlias(t)\n" +
" │ └─ Table(symbols)\n" +
" │ └─ Table\n" +
" │ ├─ name: symbols\n" +
" │ └─ columns: [symbol name sector ipoyear]\n" +
" └─ TableAlias(d)\n" +
" └─ IndexedTableAccess(daily_summary)\n" +
go/libraries/doltcore/sqle/single_table_info_db.go
+3 -3
View File
@@ -162,12 +162,12 @@ func (db *SingleTableInfoDatabase) UpdateForeignKey(ctx *sql.Context, fkName str
return fmt.Errorf("cannot create foreign keys on a single table information database")
}
// GetForeignKeyUpdater implements sql.ForeignKeyTable.
func (db *SingleTableInfoDatabase) GetForeignKeyUpdater(ctx *sql.Context) sql.ForeignKeyUpdater {
// GetForeignKeyEditor implements sql.ForeignKeyTable.
func (db *SingleTableInfoDatabase) GetForeignKeyEditor(ctx *sql.Context) sql.ForeignKeyEditor {
return nil
}
// WithIndexLookup implements sql.IndexedTable.
// IndexedAccess implements sql.IndexedTable.
func (db *SingleTableInfoDatabase) IndexedAccess(sql.Index) sql.IndexedTable {
return db
}
go/libraries/doltcore/sqle/sqlutil/static_errors.go
+1 -1
View File
@@ -60,7 +60,7 @@ type StaticErrorEditor struct {
err error
}
var _ sql.ForeignKeyUpdater = (*StaticErrorEditor)(nil)
var _ sql.ForeignKeyEditor = (*StaticErrorEditor)(nil)
func NewStaticErrorEditor(err error) *StaticErrorEditor {
return &StaticErrorEditor{err}
go/libraries/doltcore/sqle/tables.go
+23 -4
View File
@@ -342,6 +342,25 @@ func (t *DoltTable) Name() string {
func (t *DoltTable) String() string {
return t.tableName
}
func (t *DoltTable) DebugString() string {
p := sql.NewTreePrinter()
children := []string{fmt.Sprintf("name: %s", t.tableName)}
cols := t.sch.GetAllCols()
if len(t.projectedCols) > 0 {
var projections []string
for _, tag := range t.projectedCols {
projections = append(projections, fmt.Sprintf("%d", cols.TagToIdx[tag]))
}
children = append(children, fmt.Sprintf("projections: %s", projections))
}
_ = p.WriteNode("Table")
p.WriteChildren(children...)
return p.String()
}
// NumRows returns the unfiltered count of rows contained in the table
func (t *DoltTable) numRows(ctx *sql.Context) (uint64, error) {
@@ -1006,8 +1025,8 @@ func (t DoltTable) UpdateForeignKey(ctx *sql.Context, fkName string, fk sql.Fore
return fmt.Errorf("no foreign key operations on a read-only table")
}
// GetForeignKeyUpdater implements sql.ForeignKeyTable
func (t DoltTable) GetForeignKeyUpdater(ctx *sql.Context) sql.ForeignKeyUpdater {
// GetForeignKeyEditor implements sql.ForeignKeyTable
func (t DoltTable) GetForeignKeyEditor(ctx *sql.Context) sql.ForeignKeyEditor {
return nil
}
@@ -2398,8 +2417,8 @@ func (t *AlterableDoltTable) CreateIndexForForeignKey(ctx *sql.Context, idx sql.
return t.updateFromRoot(ctx, newRoot)
}
// GetForeignKeyUpdater implements sql.ForeignKeyTable
func (t *AlterableDoltTable) GetForeignKeyUpdater(ctx *sql.Context) sql.ForeignKeyUpdater {
// GetForeignKeyEditor implements sql.ForeignKeyTable
func (t *AlterableDoltTable) GetForeignKeyEditor(ctx *sql.Context) sql.ForeignKeyEditor {
te, err := t.getTableEditor(ctx)
if err != nil {
return sqlutil.NewStaticErrorEditor(err)
go/libraries/doltcore/sqle/temp_table.go
+1 -1
View File
@@ -348,7 +348,7 @@ func (t *TempTable) DropForeignKey(ctx *sql.Context, fkName string) error {
return sql.ErrTemporaryTablesForeignKeySupport.New()
}
func (t *TempTable) GetForeignKeyUpdater(ctx *sql.Context) sql.ForeignKeyUpdater {
func (t *TempTable) GetForeignKeyEditor(ctx *sql.Context) sql.ForeignKeyEditor {
return nil
}
go/libraries/doltcore/sqle/writer/noms_table_writer.go
+3 -7
View File
@@ -27,13 +27,9 @@ import (
)
type TableWriter interface {
sql.RowReplacer
sql.RowUpdater
sql.RowInserter
sql.RowDeleter
sql.ForeignKeyUpdater
sql.AutoIncrementSetter
GetNextAutoIncrementValue(ctx *sql.Context, insertVal interface{}) (uint64, error)
sql.TableEditor
sql.ForeignKeyEditor
sql.AutoIncrementEditor
}
// SessionRootSetter sets the root value for the session.
go/performance/import_benchmarker/reporting/summary.sql
+38
View File
@@ -0,0 +1,38 @@
select round(avg(sql_mult),2) as avg
from (
select
o.test_name as test_name,
o.detail,
o.row_cnt,
o.sorted as sorted,
o.time as mysql_time,
(
select round((a.time / b.time),2) m
from import_perf_results a
join import_perf_results b
on
a.test_name = b.test_name and
a.detail = b.detail
where
a.server = 'dolt_server' and
b.server = 'mysql' and
a.test_name = o.test_name and
a.detail = o.detail
) as sql_mult,
(
select round((a.time / b.time),2) m
from import_perf_results a
join import_perf_results b
on
a.test_name = b.test_name and
a.detail = b.detail
where
a.server = 'dolt_cli' and
b.server = 'mysql' and
a.test_name = o.test_name and
a.detail = o.detail
) as cli_mult
from import_perf_results as o
where o.server = 'mysql'
order by 1,2
) stats;
go/performance/sysbench/cmd/main.go
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"fmt"
"log"
"os"
"github.com/dolthub/dolt/go/performance/sysbench"
driver "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils/sql_server_driver"
)
var run = flag.String("run", "", "the path to a test file")
var scriptDir = flag.String("script-dir", "", "the path to the script directory")
var config = flag.String("config", "", "the path to a config file")
var out = flag.String("out", "", "result output path")
func main() {
flag.Parse()
defs, err := sysbench.ParseTestsFile(*run)
if err != nil {
log.Fatalln(err)
}
conf, err := sysbench.ParseConfig(*config)
if err != nil {
log.Fatalln(err)
}
conf = conf.WithScriptDir(*scriptDir)
if err := os.Chdir(*scriptDir); err != nil {
log.Fatalf("failed to 'cd %s'", *scriptDir)
}
tmpdir, err := os.MkdirTemp("", "repo-store-")
if err != nil {
log.Fatalln(err)
}
results := new(sysbench.Results)
u, err := driver.NewDoltUser()
for _, test := range defs.Tests {
test.Results = results
test.InitWithTmpDir(tmpdir)
for _, r := range test.Repos {
var err error
switch {
case r.ExternalServer != nil:
err = test.RunExternalServerTests(r.Name, r.ExternalServer, conf)
case r.Server != nil:
err = test.RunSqlServerTests(r, u, conf)
default:
panic("unsupported")
}
if err != nil {
log.Fatalln(err)
}
results.Append(test.Results.Res...)
}
}
if *out != "" {
of, err := os.Create(*out)
if err != nil {
log.Fatalln(err)
}
fmt.Fprintf(of, results.SqlDump())
} else {
fmt.Println(results.SqlDump())
}
os.Exit(0)
}
go/performance/sysbench/reporting/systab.sql
+13
View File
@@ -0,0 +1,13 @@
select name, mean, mean_mult
from (
select
round(first_value(avg) over w, 2) as mean,
trim(TRAILING '.gen.lua' FROM trim(LEADING 'gen/' FROM test_name)) as name,
round(avg / lead(avg) over w, 2) as mean_mult,
round(median / lead(median) over w, 2) as med_mult,
round(sqrt(power(first_value(stdd) over w, 2) + power(last_value(stdd) over w, 2)), 3) as stdd,
row_number() over w as rn
from sysbench_results
having mod(rn,2) = 1
window w as (order by test_name rows between CURRENT ROW and 1 following)
) sq;
go/performance/sysbench/reporting/systab_summary.sql
+13
View File
@@ -0,0 +1,13 @@
select round(avg(mean_mult),2) as avg
from (
select
round(first_value(avg) over w, 2) as mean,
trim(TRAILING '.gen.lua' FROM trim(LEADING 'gen/' FROM test_name)) as name,
round(avg / lead(avg) over w, 2) as mean_mult,
round(median / lead(median) over w, 2) as med_mult,
round(sqrt(power(first_value(stdd) over w, 2) + power(last_value(stdd) over w, 2)), 3) as stdd,
row_number() over w as rn
from sysbench_results
having mod(rn,2) = 1
window w as (order by test_name rows between CURRENT ROW and 1 following)
) sq;
go/performance/sysbench/sysbench_test.go
+143
View File
@@ -0,0 +1,143 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sysbench
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestPopulateHistogram(t *testing.T) {
tests := []struct {
name string
in []byte
exp Result
}{
{
name: "w/ histogram",
in: []byte(`
sysbench 1.0.20 (using system LuaJIT 2.1.0-beta3)
Creating table 'sbtest1'...
Inserting 10000 records into 'sbtest1'
Creating a secondary index on 'sbtest1'...
sysbench 1.0.20 (using system LuaJIT 2.1.0-beta3)
Running the test with following options:
Number of threads: 1
Initializing random number generator from seed (1).
Initializing worker threads...
Threads started!
Latency histogram (values are in milliseconds)
value ------------- distribution ------------- count
1.319 |**** 3
1.343 |******** 6
1.367 |****************** 14
1.392 |********************* 17
1.417 |******************************* 25
1.443 |**************************************** 32
1.469 |************************ 19
1.496 |******************* 15
1.523 |****** 5
1.551 |****** 5
1.579 |******** 6
1.608 |******** 6
1.637 |****** 5
1.667 |*** 2
1.697 |***** 4
1.791 |* 1
1.891 |* 1
2.106 |**** 3
2.184 |*** 2
2.223 |**** 3
2.264 |*** 2
2.305 |**** 3
2.347 |**** 3
2.389 |**** 3
2.433 |****** 5
2.477 |*** 2
2.522 |**** 3
2.568 |*** 2
2.662 |* 1
2.760 |* 1
3.130 |* 1
SQL statistics:
queries performed:
read: 200
write: 0
other: 0
total: 200
transactions: 200 (609.48 per sec.)
queries: 200 (609.48 per sec.)
ignored errors: 0 (0.00 per sec.)
reconnects: 0 (0.00 per sec.)
General statistics:
total time: 0.3272s
total number of events: 200
Latency (ms):
min: 1.32
avg: 1.62
max: 3.14
95th percentile: 2.43
sum: 324.91
Threads fairness:
events (avg/stddev): 200.0000/0.00
execution time (avg/stddev): 0.3249/0.00
sysbench 1.0.20 (using system LuaJIT 2.1.0-beta3)
Dropping table 'sbtest1'...`),
exp: Result{
time: .327,
iters: 200,
avg: 1.62,
median: 1.496,
stddev: .369,
hist: &Hist{
bins: []float64{1.319, 1.343, 1.367, 1.392, 1.417, 1.443, 1.469, 1.496, 1.523, 1.551, 1.579, 1.608, 1.637, 1.667, 1.697, 1.791, 1.891, 2.106, 2.184, 2.223, 2.264, 2.305, 2.347, 2.389, 2.433, 2.477, 2.522, 2.568, 2.662, 2.760, 3.130},
cnts: []int{3, 6, 14, 17, 25, 32, 19, 15, 5, 5, 6, 6, 5, 2, 4, 1, 1, 3, 2, 3, 2, 3, 3, 3, 5, 2, 3, 2, 1, 1, 1},
cnt: 200,
mn: 1.623,
md: 1.496,
v: .136,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := &Result{}
r.populateHistogram(tt.in)
require.Equal(t, r.String(), tt.exp.String())
})
}
}
func TestLogJoin(t *testing.T) {
t.Skip()
scriptDir := "/Users/max-hoffman/go/src/github.com/dolthub/systab-sysbench-scripts"
RunTestsFile(t, "testdata/log-join.yaml", scriptDir)
}
go/performance/sysbench/testdata/default-config.yaml
Vendored
+11
View File
@@ -0,0 +1,11 @@
dbDriver: mysql
host: 127.0.0.1
port: 3309
user: root
password:
seed: 1
tableSize: 10000
randType: uniform
eventCnt: 1500
time: 120
histogram: true
go/performance/sysbench/testdata/log-join.yaml
Vendored
+17
View File
@@ -0,0 +1,17 @@
tests:
- name: "system table"
repos:
- name: dolt
server:
port: 3309
args: [ "--port", "3309", "--password", "password"]
scripts:
- gen/dolt_diff_log_join_on_commit.gen.lua
- name: "dummy system table"
repos:
- name: dolt
server:
port: 3309
args: [ "--port", "3309" ]
scripts:
- gen/dolt_diff_log_join_on_commit_dummy.gen.lua
go/performance/sysbench/testdata/read-write.yaml
Vendored
+47
View File
@@ -0,0 +1,47 @@
tests:
- name: "read"
repos:
- name: dolt
server:
port: 3309
args: [ "--port", "3309", "--password", "password"]
- name: mysql
external-server:
name: test
host: 127.0.0.1
user: root
password:
port: 3309
scripts:
- covering_index_scan.lua
- groupby_scan.lua
- index_join.lua
- index_join_scan.lua
- index_scan.lua
- oltp_point_select
- oltp_read_only
- select_random_points
- select_random_ranges
- table_scan.lua
- types_table_scan.lua
- name: "write"
repos:
- name: dolt
server:
port: 3308
args: [ "--port", "3308" ]
- name: mysql
external-server:
name: test
host: 127.0.0.1
user: root
password:
port: 3309
scripts:
- oltp_delete_insert.lua
- oltp_insert
- oltp_read_write
- oltp_update_index
- oltp_update_non_index
- oltp_write_only
- types_delete_insert.lua
go/performance/sysbench/testdata/systab.yaml
Vendored
+29
View File
@@ -0,0 +1,29 @@
tests:
- name: "system table"
repos:
- name: dolt
server:
port: 3309
args: [ "--port", "3309", "--password", "password"]
scripts:
- gen/dolt_commit_ancestors_commit_filter.gen.lua
- gen/dolt_commits_commit_filter.gen.lua
- gen/dolt_diff_log_join_on_commit.gen.lua
- gen/dolt_diff_table_commit_filter.gen.lua
- gen/dolt_diffs_commit_filter.gen.lua
- gen/dolt_history_commit_filter.gen.lua
- gen/dolt_log_commit_filter.gen.lua
- name: "dummy system table"
repos:
- name: dolt
server:
port: 3309
args: [ "--port", "3309" ]
scripts:
- gen/dolt_commit_ancestors_commit_filter_dummy.gen.lua
- gen/dolt_commits_commit_filter_dummy.gen.lua
- gen/dolt_diff_log_join_on_commit_dummy.gen.lua
- gen/dolt_diff_table_commit_filter_dummy.gen.lua
- gen/dolt_diffs_commit_filter_dummy.gen.lua
- gen/dolt_history_commit_filter_dummy.gen.lua
- gen/dolt_log_commit_filter_dummy.gen.lua
go/performance/sysbench/testdef.go
+593
View File
@@ -0,0 +1,593 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sysbench
import (
"bytes"
"database/sql"
"fmt"
"math"
"os"
"os/exec"
"path"
"regexp"
"strconv"
"strings"
"testing"
"github.com/creasty/defaults"
"github.com/stretchr/testify/require"
yaml "gopkg.in/yaml.v3"
driver "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils/sql_server_driver"
)
// TestDef is the top-level definition of tests to run.
type TestDef struct {
Tests []Script `yaml:"tests"`
}
// Config specifies default sysbench script arguments
type Config struct {
DbDriver string `yaml:"dbDriver"`
Host string `yaml:"host"`
Port string `yaml:"port"`
User string `yaml:"user"`
Password string `yaml:"password"`
RandType string `yaml:"randType"`
EventCnt int `yaml:"eventCnt"`
Time int `yaml:"time"`
Seed int `yaml:"seed"`
TableSize int `yaml:"tableSize"`
Histogram bool `yaml:"histogram"`
ScriptDir string `yaml:"scriptDir"`
}
func (c Config) WithScriptDir(dir string) Config {
c.ScriptDir = dir
return c
}
func (c Config) AsOpts() []string {
var ret []string
if c.DbDriver != "" {
ret = append(ret, fmt.Sprintf("--db-driver=%s", c.DbDriver))
}
if c.Host != "" {
ret = append(ret, fmt.Sprintf("--mysql-host=%s", c.Host))
}
if c.User != "" {
ret = append(ret, fmt.Sprintf("--mysql-user=%s", c.User))
}
if c.Port != "" {
ret = append(ret, fmt.Sprintf("--mysql-port=%s", c.Port))
}
if c.Password != "" {
ret = append(ret, fmt.Sprintf("--mysql-password=%s", c.Password))
}
if c.RandType != "" {
ret = append(ret, fmt.Sprintf("--rand-type=%s", c.RandType))
}
if c.EventCnt > 0 {
ret = append(ret, fmt.Sprintf("--events=%s", strconv.Itoa(c.EventCnt)))
}
if c.Time > 0 {
ret = append(ret, fmt.Sprintf("--time=%s", strconv.Itoa(c.Time)))
}
ret = append(ret,
fmt.Sprintf("--rand-seed=%s", strconv.Itoa(c.Seed)),
fmt.Sprintf("--table-size=%s", strconv.Itoa(c.TableSize)),
fmt.Sprintf("--histogram=%s", strconv.FormatBool(c.Histogram)))
return ret
}
// Script is a single test to run. The Repos and MultiRepos will be created, and
// any Servers defined within them will be started. The interactions and
// assertions defined in Conns will be run.
type Script struct {
Name string `yaml:"name"`
Repos []driver.TestRepo `yaml:"repos"`
Scripts []string `yaml:"scripts"`
// Skip the entire test with this reason.
Skip string `yaml:"skip"`
Results *Results
tmpdir string
scriptDir string
}
func (s *Script) UnmarshalYAML(unmarshal func(interface{}) error) error {
defaults.Set(s)
type plain Script
if err := unmarshal((*plain)(s)); err != nil {
return err
}
return nil
}
func ParseTestsFile(path string) (TestDef, error) {
contents, err := os.ReadFile(path)
if err != nil {
return TestDef{}, err
}
dec := yaml.NewDecoder(bytes.NewReader(contents))
dec.KnownFields(true)
var res TestDef
err = dec.Decode(&res)
return res, err
}
func ParseConfig(path string) (Config, error) {
contents, err := os.ReadFile(path)
if err != nil {
return Config{}, err
}
dec := yaml.NewDecoder(bytes.NewReader(contents))
dec.KnownFields(true)
var res Config
err = dec.Decode(&res)
return res, err
}
func MakeRepo(rs driver.RepoStore, r driver.TestRepo) (driver.Repo, error) {
repo, err := rs.MakeRepo(r.Name)
if err != nil {
return driver.Repo{}, err
}
return repo, nil
}
func MakeServer(dc driver.DoltCmdable, s *driver.Server) (*driver.SqlServer, error) {
if s == nil {
return nil, nil
}
opts := []driver.SqlServerOpt{driver.WithArgs(s.Args...)}
if s.Port != 0 {
opts = append(opts, driver.WithPort(s.Port))
}
server, err := driver.StartSqlServer(dc, opts...)
if err != nil {
return nil, err
}
return server, nil
}
type Hist struct {
bins []float64
cnts []int
sum float64
cnt int
mn float64
md float64
v float64
}
func newHist() *Hist {
return &Hist{bins: make([]float64, 0), cnts: make([]int, 0)}
}
// add assumes bins are passed in increasing order before
// any statistics are evaluated
func (h *Hist) add(bin float64, cnt int) {
if h.mn != 0 || h.v != 0 || h.md != 0 {
panic("tried to edit finished histogram")
}
h.sum += bin * float64(cnt)
h.cnt += cnt
h.bins = append(h.bins, bin)
h.cnts = append(h.cnts, cnt)
}
// mean is a lossy mean based on histogram bucket sizes
func (h *Hist) mean() float64 {
if h.mn == 0 {
h.mn = math.Round(h.sum*1e3/(float64(h.cnt))) / 1e3
}
return h.mn
}
// median is a lossy median based on histogram bucket sizes
func (h *Hist) median() float64 {
if h.md == 0 {
mid := h.cnt / 2
var i int
var cnt int
for cnt < mid {
cnt += h.cnts[i]
i++
}
h.md = h.bins[i]
}
return h.md
}
// variance is a lossy sum of least squares
func (h *Hist) variance() float64 {
if h.v == 0 {
ss := 0.0
for i := range h.cnts {
ss += float64(h.cnts[i]) * math.Pow(h.bins[i]-h.mean(), 2)
}
h.v = math.Round(ss*1e3/(float64(h.cnt-1))) / 1e3
}
return h.v
}
var histRe = regexp.MustCompile(`([0-9]+\.+[0-9]+)\s+.+\s+([1-9][0-9]*)\n`)
func (h *Hist) populate(buf []byte) error {
res := histRe.FindAllSubmatch(buf, -1)
var bin float64
var cnt int
var err error
if len(res) == 0 {
return fmt.Errorf("histogram not found")
}
for _, r := range res {
bin, err = strconv.ParseFloat(string(r[1]), 0)
if err != nil {
return fmt.Errorf("failed to parse bin: %s -> '%s' - '%s'; %s", r[0], r[1], r[2], err)
}
cnt, err = strconv.Atoi(string(r[2]))
if err != nil {
return fmt.Errorf("failed to parse cnt: %s -> '%s' - '%s'; %s", r[0], r[1], r[2], err)
}
h.add(bin, cnt)
}
return nil
}
func (r *Result) populateHistogram(buf []byte) error {
r.hist = newHist()
r.hist.populate(buf)
r.stddev = math.Sqrt(r.hist.variance())
r.median = r.hist.median()
var err error
{
timeRe := regexp.MustCompile(`total time:\s+([0-9][0-9]*\.[0-9]+)s\n`)
res := timeRe.FindSubmatch(buf)
if len(res) == 0 {
return fmt.Errorf("time not found")
}
time, err := strconv.ParseFloat(string(res[1]), 0)
if err != nil {
return fmt.Errorf("failed to parse time: %s -> '%s' ; %s", res[0], res[1], err)
}
r.time = math.Round(time*1e3) / 1e3
}
{
itersRe := regexp.MustCompile(`total number of events:\s+([1-9][0-9]*)\n`)
res := itersRe.FindSubmatch(buf)
if len(res) == 0 {
return fmt.Errorf("time not found")
}
r.iters, err = strconv.Atoi(string(res[1]))
if err != nil {
return fmt.Errorf("failed to parse bin: %s -> '%s'; %s", res[0], res[1], err)
}
}
{
avgRe := regexp.MustCompile(`avg:\s+([0-9][0-9]*\.[0-9]+)\n`)
res := avgRe.FindSubmatch(buf)
if len(res) == 0 {
return fmt.Errorf("avg not found")
}
r.avg, err = strconv.ParseFloat(string(res[1]), 0)
if err != nil {
return fmt.Errorf("failed to parse avg: %s -> '%s'; %s", res[0], res[1], err)
}
}
return nil
}
func (r *Result) populateAvg(buf []byte) error {
panic("TODO")
}
type Result struct {
server string
detail string
test string
time float64
iters int
hist *Hist
avg float64
median float64
stddev float64
}
func newResult(server, test, detail string) *Result {
return &Result{
server: server,
detail: detail,
test: test,
}
}
func (r *Result) String() string {
b := &strings.Builder{}
fmt.Fprintf(b, "result:\n")
b.WriteString("result:\n")
if r.test != "" {
fmt.Fprintf(b, "- test: '%s'\n", r.test)
}
if r.detail != "" {
fmt.Fprintf(b, "- detail: '%s'\n", r.detail)
}
if r.server != "" {
fmt.Fprintf(b, "- server: '%s'\n", r.server)
}
fmt.Fprintf(b, "- time: %.3f\n", r.time)
fmt.Fprintf(b, "- iters: %d\n", r.iters)
fmt.Fprintf(b, "- mean: %.3f\n", r.hist.mean())
fmt.Fprintf(b, "- median: %.3f\n", r.median)
fmt.Fprintf(b, "- stddev: %.3f\n", r.stddev)
return b.String()
}
type Results struct {
Res []*Result
}
func (r *Results) Append(ir ...*Result) {
r.Res = append(r.Res, ir...)
}
func (r *Results) String() string {
b := strings.Builder{}
b.WriteString("Results:\n")
for _, x := range r.Res {
b.WriteString(x.String())
}
return b.String()
}
func (r *Results) SqlDump() string {
b := strings.Builder{}
b.WriteString(`CREATE TABLE IF NOT EXISTS sysbench_results (
test_name varchar(64),
detail varchar(64),
server varchar(64),
time double,
iters int,
avg double,
median double,
stdd double,
primary key (test_name, detail, server)
);
`)
b.WriteString("insert into sysbench_results values\n")
for i, r := range r.Res {
if i > 0 {
b.WriteString(",\n ")
}
b.WriteString(fmt.Sprintf(
"('%s', '%s', '%s', %.3f, %d, %.3f, %.3f, %.3f)",
r.test, r.detail, r.server, r.time, r.iters, r.avg, r.median, r.stddev))
}
b.WriteString(";\n")
return b.String()
}
func (test *Script) InitWithTmpDir(s string) {
test.tmpdir = s
test.Results = new(Results)
}
// Run executes an import configuration. Test parallelism makes
// runtimes resulting from this method unsuitable for reporting.
func (test *Script) Run(t *testing.T) {
if test.Skip != "" {
t.Skip(test.Skip)
}
conf, err := ParseConfig("testdata/default-config.yaml")
if err != nil {
require.NoError(t, err)
}
if _, err = os.Stat(test.scriptDir); err != nil {
require.NoError(t, err)
}
conf = conf.WithScriptDir(test.scriptDir)
tmpdir, err := os.MkdirTemp("", "repo-store-")
if err != nil {
require.NoError(t, err)
}
results := new(Results)
u, err := driver.NewDoltUser()
test.Results = results
test.InitWithTmpDir(tmpdir)
for _, r := range test.Repos {
var err error
switch {
case r.ExternalServer != nil:
panic("unsupported")
case r.Server != nil:
err = test.RunSqlServerTests(r, u, conf)
default:
panic("unsupported")
}
if err != nil {
require.NoError(t, err)
}
results.Append(test.Results.Res...)
}
fmt.Println(test.Results)
}
func modifyServerForImport(db *sql.DB) error {
_, err := db.Exec("CREATE DATABASE sbtest")
if err != nil {
return err
}
return nil
}
// RunExternalServerTests connects to a single externally provided server to run every test
func (test *Script) RunExternalServerTests(repoName string, s *driver.ExternalServer, conf Config) error {
return test.IterSysbenchScripts(conf, test.Scripts, func(script string, prep, run, clean *exec.Cmd) error {
db, err := driver.ConnectDB(s.User, s.Password, s.Name, s.Host, s.Port, nil)
if err != nil {
return err
}
defer db.Close()
if err := prep.Run(); err != nil {
return err
}
buf := new(bytes.Buffer)
run.Stdout = buf
err = run.Run()
if err != nil {
return err
}
// TODO scrape histogram data
r := newResult(repoName, script, test.Name)
if conf.Histogram {
r.populateHistogram(buf.Bytes())
} else {
r.populateAvg(buf.Bytes())
}
test.Results.Append(r)
return clean.Run()
})
}
// RunSqlServerTests creates a new repo and server for every import test.
func (test *Script) RunSqlServerTests(repo driver.TestRepo, user driver.DoltUser, conf Config) error {
return test.IterSysbenchScripts(conf, test.Scripts, func(script string, prep, run, clean *exec.Cmd) error {
//make a new server for every test
server, err := newServer(user, repo)
if err != nil {
return err
}
defer server.GracefulStop()
db, err := server.DB(driver.Connection{User: "root", Pass: ""})
if err != nil {
return err
}
err = modifyServerForImport(db)
if err != nil {
return err
}
if err := prep.Run(); err != nil {
return err
}
buf := new(bytes.Buffer)
run.Stdout = buf
err = run.Run()
if err != nil {
return err
}
// TODO scrape histogram data
r := newResult(repo.Name, script, test.Name)
if conf.Histogram {
r.populateHistogram(buf.Bytes())
} else {
r.populateAvg(buf.Bytes())
}
test.Results.Append(r)
return clean.Run()
})
}
func newServer(u driver.DoltUser, r driver.TestRepo) (*driver.SqlServer, error) {
rs, err := u.MakeRepoStore()
if err != nil {
return nil, err
}
// start dolt server
repo, err := MakeRepo(rs, r)
if err != nil {
return nil, err
}
r.Server.Args = append(r.Server.Args, "")
server, err := MakeServer(repo, r.Server)
if err != nil {
return nil, err
}
if server != nil {
server.DBName = r.Name
}
return server, nil
}
const luaExt = ".lua"
// IterSysbenchScripts returns 3 executable commands for the given script path: prepare, run, cleanup
func (test *Script) IterSysbenchScripts(conf Config, scripts []string, cb func(name string, prep, run, clean *exec.Cmd) error) error {
newCmd := func(command, script string) *exec.Cmd {
cmd := exec.Command("sysbench")
cmd.Args = append(cmd.Args, conf.AsOpts()...)
cmd.Args = append(cmd.Args, script, command)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd
}
for _, script := range scripts {
p := script
if strings.HasSuffix(script, luaExt) {
p = path.Join(conf.ScriptDir, script)
if _, err := os.Stat(p); err != nil {
return fmt.Errorf("failed to run script: '%s'", err)
}
}
prep := newCmd("prepare", p)
run := newCmd("run", p)
clean := newCmd("cleanup", p)
if err := cb(script, prep, run, clean); err != nil {
return err
}
}
return nil
}
func RunTestsFile(t *testing.T, path, scriptDir string) {
def, err := ParseTestsFile(path)
require.NoError(t, err)
for _, test := range def.Tests {
test.scriptDir = scriptDir
t.Run(test.Name, test.Run)
}
}
go/store/cmd/noms/noms_sync.go
+2 -1
View File
@@ -34,6 +34,7 @@ import (
"github.com/dolthub/dolt/go/store/config"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/datas/pull"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/util/profile"
"github.com/dolthub/dolt/go/store/util/status"
@@ -107,7 +108,7 @@ func runSync(ctx context.Context, args []string) int {
f := func() error {
defer profile.MaybeStartProfile().Stop()
addr := sourceRef.TargetHash()
err := pull.Pull(ctx, srcCS, sinkCS, waf, addr, progressCh)
err := pull.Pull(ctx, srcCS, sinkCS, waf, []hash.Hash{addr}, progressCh)
if err != nil {
return err
go/store/datas/pull/pull.go
+13 -14
View File
@@ -58,29 +58,27 @@ func makeProgTrack(progressCh chan PullProgress) func(moreDone, moreKnown, moreA
}
// Pull objects that descend from sourceHash from srcDB to sinkDB.
func Pull(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkAddrs WalkAddrs, sourceHash hash.Hash, progressCh chan PullProgress) error {
return pull(ctx, srcCS, sinkCS, walkAddrs, sourceHash, progressCh, defaultBatchSize)
func Pull(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkAddrs WalkAddrs, hashes []hash.Hash, progressCh chan PullProgress) error {
return pull(ctx, srcCS, sinkCS, walkAddrs, hashes, progressCh, defaultBatchSize)
}
func pull(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkAddrs WalkAddrs, sourceHash hash.Hash, progressCh chan PullProgress, batchSize int) error {
func pull(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkAddrs WalkAddrs, hashes []hash.Hash, progressCh chan PullProgress, batchSize int) error {
// Sanity Check
exists, err := srcCS.Has(ctx, sourceHash)
hs := hash.NewHashSet(hashes...)
missing, err := srcCS.HasMany(ctx, hs)
if err != nil {
return err
}
if !exists {
if missing.Size() != 0 {
return errors.New("not found")
}
exists, err = sinkCS.Has(ctx, sourceHash)
hs = hash.NewHashSet(hashes...)
missing, err = sinkCS.HasMany(ctx, hs)
if err != nil {
return err
}
if exists {
if missing.Size() == 0 {
return nil // already up to date
}
@@ -92,7 +90,8 @@ func pull(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkAddrs WalkAd
updateProgress := makeProgTrack(progressCh)
// TODO: This batches based on limiting the _number_ of chunks processed at the same time. We really want to batch based on the _amount_ of chunk data being processed simultaneously. We also want to consider the chunks in a particular order, however, and the current GetMany() interface doesn't provide any ordering guarantees. Once BUG 3750 is fixed, we should be able to revisit this and do a better job.
absent := hash.HashSlice{sourceHash}
absent := make([]hash.Hash, len(hashes))
copy(absent, hashes)
for absentCount := len(absent); absentCount != 0; absentCount = len(absent) {
updateProgress(0, uint64(absentCount), 0)
@@ -160,9 +159,9 @@ func persistChunks(ctx context.Context, cs chunks.ChunkStore) error {
// PullWithoutBatching effectively removes the batching of chunk retrieval done on each level of the tree. This means
// all chunks from one level of the tree will be retrieved from the underlying chunk store in one call, which pushes the
// optimization problem down to the chunk store which can make smarter decisions.
func PullWithoutBatching(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkAddrs WalkAddrs, sourceHash hash.Hash, progressCh chan PullProgress) error {
func PullWithoutBatching(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkAddrs WalkAddrs, hashes []hash.Hash, progressCh chan PullProgress) error {
// by increasing the batch size to MaxInt32 we effectively remove batching here.
return pull(ctx, srcCS, sinkCS, walkAddrs, sourceHash, progressCh, math.MaxInt32)
return pull(ctx, srcCS, sinkCS, walkAddrs, hashes, progressCh, math.MaxInt32)
}
// concurrently pull all chunks from this batch that the sink is missing out of the source
go/store/datas/pull/pull_test.go
+78 -15
View File
@@ -26,6 +26,7 @@ import (
"context"
"errors"
"io"
"os"
"reflect"
"sync"
"testing"
@@ -61,10 +62,14 @@ func TestRemoteToRemotePulls(t *testing.T) {
suite.Run(t, &RemoteToRemoteSuite{})
}
func TestChunkJournalPulls(t *testing.T) {
suite.Run(t, &ChunkJournalSuite{})
}
type PullSuite struct {
suite.Suite
sinkCS *chunks.TestStoreView
sourceCS *chunks.TestStoreView
sinkCS chunks.ChunkStore
sourceCS chunks.ChunkStore
sinkVRW types.ValueReadWriter
sourceVRW types.ValueReadWriter
sinkDB datas.Database
@@ -72,6 +77,13 @@ type PullSuite struct {
commitReads int // The number of reads triggered by commit differs across chunk store impls
}
type metricsChunkStore interface {
chunks.ChunkStore
Reads() int
Hases() int
Writes() int
}
func makeTestStoreViews() (ts1, ts2 *chunks.TestStoreView) {
st1, st2 := &chunks.TestStorage{}, &chunks.TestStorage{}
return st1.NewView(), st2.NewView()
@@ -128,6 +140,32 @@ func (suite *RemoteToRemoteSuite) SetupTest() {
suite.commitReads = 1
}
type ChunkJournalSuite struct {
PullSuite
}
func (suite *ChunkJournalSuite) SetupTest() {
ctx := context.Background()
q := nbs.NewUnlimitedMemQuotaProvider()
nbf := types.Format_Default.VersionString()
path, err := os.MkdirTemp("", "remote")
suite.NoError(err)
sink, err := nbs.NewLocalJournalingStore(ctx, nbf, path, q)
suite.NoError(err)
path, err = os.MkdirTemp("", "local")
suite.NoError(err)
src, err := nbs.NewLocalJournalingStore(ctx, nbf, path, q)
suite.NoError(err)
suite.sinkCS, suite.sourceCS = sink, src
sinkVRW, sourceVRW := types.NewValueStore(suite.sinkCS), types.NewValueStore(suite.sourceCS)
suite.sinkVRW, suite.sourceVRW = sinkVRW, sourceVRW
suite.sourceDB = datas.NewTypesDatabase(sourceVRW, tree.NewNodeStore(suite.sourceCS))
suite.sinkDB = datas.NewTypesDatabase(sinkVRW, tree.NewNodeStore(suite.sinkCS))
suite.commitReads = 1
}
func (suite *PullSuite) TearDownTest() {
suite.sinkCS.Close()
suite.sourceCS.Close()
@@ -184,7 +222,11 @@ func (pt *progressTracker) Validate(suite *PullSuite) {
//
// Sink: Nada
func (suite *PullSuite) TestPullEverything() {
expectedReads := suite.sinkCS.Reads()
var expectedReads int
mcs, metrics := suite.sinkCS.(metricsChunkStore)
if metrics {
expectedReads = mcs.Reads()
}
l := buildListOfHeight(2, suite.sourceVRW)
sourceAddr := suite.commitToSource(l, nil)
@@ -192,12 +234,14 @@ func (suite *PullSuite) TestPullEverything() {
waf, err := types.WalkAddrsForChunkStore(suite.sourceCS)
suite.NoError(err)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, sourceAddr, pt.Ch)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, []hash.Hash{sourceAddr}, pt.Ch)
suite.NoError(err)
suite.True(expectedReads-suite.sinkCS.Reads() <= suite.commitReads)
if metrics {
suite.True(expectedReads-suite.sinkCS.(metricsChunkStore).Reads() <= suite.commitReads)
}
pt.Validate(suite)
v := mustValue(suite.sinkVRW.ReadValue(context.Background(), sourceAddr)).(types.Struct)
v := mustValue(suite.sinkVRW.ReadValue(context.Background(), sourceAddr))
suite.NotNil(v)
suite.True(l.Equals(mustGetCommittedValue(suite.sinkVRW, v)))
}
@@ -228,7 +272,11 @@ func (suite *PullSuite) TestPullEverything() {
func (suite *PullSuite) TestPullMultiGeneration() {
sinkL := buildListOfHeight(2, suite.sinkVRW)
suite.commitToSink(sinkL, nil)
expectedReads := suite.sinkCS.Reads()
var expectedReads int
mcs, metrics := suite.sinkCS.(metricsChunkStore)
if metrics {
expectedReads = mcs.Reads()
}
srcL := buildListOfHeight(2, suite.sourceVRW)
sourceAddr := suite.commitToSource(srcL, nil)
@@ -241,10 +289,12 @@ func (suite *PullSuite) TestPullMultiGeneration() {
waf, err := types.WalkAddrsForChunkStore(suite.sourceCS)
suite.NoError(err)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, sourceAddr, pt.Ch)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, []hash.Hash{sourceAddr}, pt.Ch)
suite.NoError(err)
suite.True(expectedReads-suite.sinkCS.Reads() <= suite.commitReads)
if metrics {
suite.True(expectedReads-suite.sinkCS.(metricsChunkStore).Reads() <= suite.commitReads)
}
pt.Validate(suite)
v, err := suite.sinkVRW.ReadValue(context.Background(), sourceAddr)
@@ -292,16 +342,22 @@ func (suite *PullSuite) TestPullDivergentHistory() {
srcL, err = srcL.Edit().Set(1, buildListOfHeight(5, suite.sourceVRW)).List(context.Background())
suite.NoError(err)
sourceAddr = suite.commitToSource(srcL, []hash.Hash{sourceAddr})
preReads := suite.sinkCS.Reads()
var preReads int
mcs, metrics := suite.sinkCS.(metricsChunkStore)
if metrics {
preReads = mcs.Reads()
}
pt := startProgressTracker()
waf, err := types.WalkAddrsForChunkStore(suite.sourceCS)
suite.NoError(err)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, sourceAddr, pt.Ch)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, []hash.Hash{sourceAddr}, pt.Ch)
suite.NoError(err)
suite.True(preReads-suite.sinkCS.Reads() <= suite.commitReads)
if metrics {
suite.True(preReads-suite.sinkCS.(metricsChunkStore).Reads() <= suite.commitReads)
}
pt.Validate(suite)
v, err := suite.sinkVRW.ReadValue(context.Background(), sourceAddr)
@@ -334,7 +390,12 @@ func (suite *PullSuite) TestPullDivergentHistory() {
func (suite *PullSuite) TestPullUpdates() {
sinkL := buildListOfHeight(4, suite.sinkVRW)
suite.commitToSink(sinkL, nil)
expectedReads := suite.sinkCS.Reads()
var expectedReads int
mcs, metrics := suite.sinkCS.(metricsChunkStore)
if metrics {
expectedReads = mcs.Reads()
}
srcL := buildListOfHeight(4, suite.sourceVRW)
sourceAddr := suite.commitToSource(srcL, nil)
@@ -355,10 +416,12 @@ func (suite *PullSuite) TestPullUpdates() {
waf, err := types.WalkAddrsForChunkStore(suite.sourceCS)
suite.NoError(err)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, sourceAddr, pt.Ch)
err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, waf, []hash.Hash{sourceAddr}, pt.Ch)
suite.NoError(err)
suite.True(expectedReads-suite.sinkCS.Reads() <= suite.commitReads)
if metrics {
suite.True(expectedReads-suite.sinkCS.(metricsChunkStore).Reads() <= suite.commitReads)
}
pt.Validate(suite)
v, err := suite.sinkVRW.ReadValue(context.Background(), sourceAddr)
go/store/datas/pull/puller.go
+10 -12
View File
@@ -66,7 +66,7 @@ type Puller struct {
srcChunkStore nbs.NBSCompressedChunkStore
sinkDBCS chunks.ChunkStore
rootChunkHash hash.Hash
hashes hash.HashSet
downloaded hash.HashSet
wr *nbs.CmpChunkTableWriter
@@ -88,27 +88,25 @@ func NewPuller(
chunksPerTF int,
srcCS, sinkCS chunks.ChunkStore,
walkAddrs WalkAddrs,
rootChunkHash hash.Hash,
hashes [] hash.Hash,
statsCh chan Stats,
) (*Puller, error) {
// Sanity Check
exists, err := srcCS.Has(ctx, rootChunkHash)
hs := hash.NewHashSet(hashes...)
missing, err := srcCS.HasMany(ctx, hs)
if err != nil {
return nil, err
}
if !exists {
if missing.Size() != 0 {
return nil, errors.New("not found")
}
exists, err = sinkCS.Has(ctx, rootChunkHash)
hs = hash.NewHashSet(hashes...)
missing, err = sinkCS.HasMany(ctx, hs)
if err != nil {
return nil, err
}
if exists {
if missing.Size() == 0 {
return nil, ErrDBUpToDate
}
@@ -141,7 +139,7 @@ func NewPuller(
waf: walkAddrs,
srcChunkStore: srcChunkStore,
sinkDBCS: sinkCS,
rootChunkHash: rootChunkHash,
hashes: hash.NewHashSet(hashes...),
downloaded: hash.HashSet{},
tablefileSema: semaphore.NewWeighted(outstandingTableFiles),
tempDir: tempDir,
@@ -388,7 +386,7 @@ func (p *Puller) Pull(ctx context.Context) error {
leaves := make(hash.HashSet)
absent := make(hash.HashSet)
absent.Insert(p.rootChunkHash)
absent.InsertAll(p.hashes)
eg, ctx := errgroup.WithContext(ctx)
go/store/datas/pull/puller_test.go
+40 -24
View File
@@ -36,6 +36,41 @@ import (
"github.com/dolthub/dolt/go/store/util/clienttest"
)
func TestNbsPuller(t *testing.T) {
testPuller(t, func(ctx context.Context) (types.ValueReadWriter, datas.Database) {
dir := filepath.Join(os.TempDir(), uuid.New().String())
err := os.MkdirAll(dir, os.ModePerm)
require.NoError(t, err)
nbf := types.Format_Default.VersionString()
q := nbs.NewUnlimitedMemQuotaProvider()
st, err := nbs.NewLocalStore(ctx, nbf, dir, clienttest.DefaultMemTableSize, q)
require.NoError(t, err)
ns := tree.NewNodeStore(st)
vs := types.NewValueStore(st)
return vs, datas.NewTypesDatabase(vs, ns)
})
}
func TestChunkJournalPuller(t *testing.T) {
testPuller(t, func(ctx context.Context) (types.ValueReadWriter, datas.Database) {
dir := filepath.Join(os.TempDir(), uuid.New().String())
err := os.MkdirAll(dir, os.ModePerm)
require.NoError(t, err)
nbf := types.Format_Default.VersionString()
q := nbs.NewUnlimitedMemQuotaProvider()
st, err := nbs.NewLocalJournalingStore(ctx, nbf, dir, q)
require.NoError(t, err)
ns := tree.NewNodeStore(st)
vs := types.NewValueStore(st)
return vs, datas.NewTypesDatabase(vs, ns)
})
}
func addTableValues(ctx context.Context, vrw types.ValueReadWriter, m types.Map, tableName string, alternatingKeyVals ...types.Value) (types.Map, error) {
val, ok, err := m.MaybeGet(ctx, types.String(tableName))
@@ -124,29 +159,11 @@ func deleteTableValues(ctx context.Context, vrw types.ValueReadWriter, m types.M
return me.Map(ctx)
}
func tempDirDB(ctx context.Context) (types.ValueReadWriter, datas.Database, error) {
dir := filepath.Join(os.TempDir(), uuid.New().String())
err := os.MkdirAll(dir, os.ModePerm)
type datasFactory func(context.Context) (types.ValueReadWriter, datas.Database)
if err != nil {
return nil, nil, err
}
st, err := nbs.NewLocalStore(ctx, types.Format_Default.VersionString(), dir, clienttest.DefaultMemTableSize, nbs.NewUnlimitedMemQuotaProvider())
if err != nil {
return nil, nil, err
}
ns := tree.NewNodeStore(st)
vs := types.NewValueStore(st)
return vs, datas.NewTypesDatabase(vs, ns), nil
}
func TestPuller(t *testing.T) {
func testPuller(t *testing.T, makeDB datasFactory) {
ctx := context.Background()
vs, db, err := tempDirDB(ctx)
require.NoError(t, err)
vs, db := makeDB(ctx)
deltas := []struct {
name string
@@ -307,15 +324,14 @@ func TestPuller(t *testing.T) {
}
}()
sinkvs, sinkdb, err := tempDirDB(ctx)
require.NoError(t, err)
sinkvs, sinkdb := makeDB(ctx)
tmpDir := filepath.Join(os.TempDir(), uuid.New().String())
err = os.MkdirAll(tmpDir, os.ModePerm)
require.NoError(t, err)
waf, err := types.WalkAddrsForChunkStore(datas.ChunkStoreFromDatabase(db))
require.NoError(t, err)
plr, err := NewPuller(ctx, tmpDir, 128, datas.ChunkStoreFromDatabase(db), datas.ChunkStoreFromDatabase(sinkdb), waf, rootAddr, statsCh)
plr, err := NewPuller(ctx, tmpDir, 128, datas.ChunkStoreFromDatabase(db), datas.ChunkStoreFromDatabase(sinkdb), waf, []hash.Hash{rootAddr}, statsCh)
require.NoError(t, err)
err = plr.Pull(ctx)
go/store/nbs/aws_table_persister.go
+3 -2
View File
@@ -63,6 +63,8 @@ type awsTablePersister struct {
q MemoryQuotaProvider
}
var _ tablePersister = awsTablePersister{}
type awsLimits struct {
partTarget, partMin, partMax uint64
itemMax int
@@ -512,8 +514,7 @@ func dividePlan(ctx context.Context, plan compactionPlan, minPartSize, maxPartSi
var offset int64
for ; i < len(plan.sources.sws); i++ {
sws := plan.sources.sws[i]
rdr, err := sws.source.reader(ctx)
rdr, _, err := sws.source.reader(ctx)
if err != nil {
return nil, nil, nil, err
}
go/store/nbs/bs_persister.go
+2
View File
@@ -29,6 +29,8 @@ type blobstorePersister struct {
q MemoryQuotaProvider
}
var _ tablePersister = &blobstorePersister{}
// Persist makes the contents of mt durable. Chunks already present in
// |haver| may be dropped in the process.
func (bsp *blobstorePersister) Persist(ctx context.Context, mt *memTable, haver chunkReader, stats *Stats) (chunkSource, error) {
go/store/nbs/chunk_journal.go
+157 -79
View File
@@ -16,6 +16,7 @@ package nbs
import (
"context"
"errors"
"fmt"
"io"
"os"
@@ -29,88 +30,138 @@ import (
"github.com/dolthub/dolt/go/store/hash"
)
var chunkJournalFeatureFlag = false
var ChunkJournalFeatureFlag = false
func init() {
if os.Getenv("DOLT_ENABLE_CHUNK_JOURNAL") != "" {
chunkJournalFeatureFlag = true
ChunkJournalFeatureFlag = true
}
}
const (
chunkJournalName = "nbs_chunk_journal"
chunkJournalName = chunkJournalAddr // todo
)
type chunkJournal struct {
journal *journalWriter
source journalChunkSource
path string
source journalChunkSource
contents manifestContents
backing manifest
contents manifestContents
backing manifest
persister *fsTablePersister
}
var _ tablePersister = &chunkJournal{}
var _ tableFilePersister = &chunkJournal{}
var _ manifest = &chunkJournal{}
var _ io.Closer = &chunkJournal{}
type journalChunkSource struct {
address addr
journal io.ReaderAt
// todo: jrecordLookup -> Range
address addr
journal snapshotReader
lookups map[addr]jrecordLookup
compressedSz uint64
}
var _ chunkSource = journalChunkSource{}
type jrecordLookup struct {
offset int64
length uint32
}
func newChunkJournal(ctx context.Context, dir string, m manifest) (*chunkJournal, error) {
func newChunkJournal(ctx context.Context, nbfVers, dir string, m manifest, p *fsTablePersister) (*chunkJournal, error) {
path, err := filepath.Abs(filepath.Join(dir, chunkJournalName))
if err != nil {
return nil, err
}
wr, err := openJournalWriter(ctx, path)
j := &chunkJournal{path: path, backing: m, persister: p}
j.contents.nbfVers = nbfVers
ok, err := journalFileExists(path)
if err != nil {
return nil, err
} else if ok {
// only open a journalWriter if the journal file exists,
// otherwise we wait to open in case we're cloning
if err = j.openJournal(ctx); err != nil {
return nil, err
}
}
return j, nil
}
func (j *chunkJournal) openJournal(ctx context.Context) (err error) {
var ok bool
ok, err = journalFileExists(j.path)
if err != nil {
return err
}
root, source, err := wr.bootstrapJournal(ctx)
if err != nil {
return nil, err
if !ok { // create new journal file
j.journal, err = createJournalWriter(ctx, j.path)
if err != nil {
return err
}
_, j.source, err = j.journal.processJournal(ctx)
if err != nil {
return err
}
var contents manifestContents
ok, contents, err = j.backing.ParseIfExists(ctx, &Stats{}, nil)
if err != nil {
return err
}
if ok {
// write the current root hash to the journal file
if err = j.journal.writeRootHash(contents.root); err != nil {
return
}
j.contents = contents
}
return
}
ok, contents, err := m.ParseIfExists(ctx, &Stats{}, nil)
j.journal, ok, err = openJournalWriter(ctx, j.path)
if err != nil {
return nil, err
return err
} else if !ok {
return errors.New("missing chunk journal " + j.path)
}
// parse existing journal file
var root hash.Hash
root, j.source, err = j.journal.processJournal(ctx)
if err != nil {
return err
}
var contents manifestContents
ok, contents, err = j.backing.ParseIfExists(ctx, &Stats{}, nil)
if err != nil {
return err
}
if ok {
// the journal file is the source of truth for the root hash, true-up persisted manifest
contents.root = root
if contents, err = m.Update(ctx, contents.lock, contents, &Stats{}, nil); err != nil {
return nil, err
contents, err = j.backing.Update(ctx, contents.lock, contents, &Stats{}, nil)
if err != nil {
return err
}
} else if !emptyAddr(addr(root)) {
// journal file contains root hash, but manifest is missing
return nil, fmt.Errorf("missing manifest while initializing chunk journal")
} else {
return fmt.Errorf("manifest not found when opening chunk journal")
}
return &chunkJournal{
journal: wr,
source: source,
contents: contents,
backing: m,
}, nil
j.contents = contents
return
}
// Persist implements tablePersister.
func (j *chunkJournal) Persist(ctx context.Context, mt *memTable, haver chunkReader, stats *Stats) (chunkSource, error) {
if err := j.maybeInit(ctx); err != nil {
return nil, err
}
if haver != nil {
sort.Sort(hasRecordByPrefix(mt.order)) // hasMany() requires addresses to be sorted.
if _, err := haver.hasMany(mt.order); err != nil {
@@ -137,36 +188,48 @@ func (j *chunkJournal) Persist(ctx context.Context, mt *memTable, haver chunkRea
// ConjoinAll implements tablePersister.
func (j *chunkJournal) ConjoinAll(ctx context.Context, sources chunkSources, stats *Stats) (chunkSource, error) {
panic("unimplemented")
return j.persister.ConjoinAll(ctx, sources, stats)
}
// Open implements tablePersister.
func (j *chunkJournal) Open(ctx context.Context, name addr, chunkCount uint32, stats *Stats) (chunkSource, error) {
if name == j.source.address {
if name == journalAddr {
if err := j.maybeInit(ctx); err != nil {
return nil, err
}
return j.source, nil
}
return nil, fmt.Errorf("unknown chunk source %s", name.String())
return j.persister.Open(ctx, name, chunkCount, stats)
}
// Exists implements tablePersister.
func (j *chunkJournal) Exists(ctx context.Context, name addr, chunkCount uint32, stats *Stats) (bool, error) {
panic("unimplemented")
return j.persister.Exists(ctx, name, chunkCount, stats)
}
// PruneTableFiles implements tablePersister.
func (j *chunkJournal) PruneTableFiles(ctx context.Context, contents manifestContents, mtime time.Time) error {
panic("unimplemented")
return j.persister.PruneTableFiles(ctx, contents, mtime)
}
func (j *chunkJournal) Path() string {
return filepath.Dir(j.path)
}
// Name implements manifest.
func (j *chunkJournal) Name() string {
return j.journal.filepath()
return j.path
}
// Update implements manifest.
func (j *chunkJournal) Update(ctx context.Context, lastLock addr, next manifestContents, stats *Stats, writeHook func() error) (manifestContents, error) {
if j.journal == nil {
// pass the update to |j.backing| if the journals is not initialized
return j.backing.Update(ctx, lastLock, next, stats, writeHook)
}
if j.contents.gcGen != next.gcGen {
panic("chunkJournal cannot update GC generation")
return manifestContents{}, errors.New("chunkJournal cannot update GC generation")
} else if j.contents.lock != lastLock {
return j.contents, nil // |next| is stale
}
@@ -177,10 +240,6 @@ func (j *chunkJournal) Update(ctx context.Context, lastLock addr, next manifestC
}
}
if emptyAddr(addr(next.root)) {
panic(next)
}
if err := j.journal.writeRootHash(next.root); err != nil {
return manifestContents{}, err
}
@@ -191,13 +250,9 @@ func (j *chunkJournal) Update(ctx context.Context, lastLock addr, next manifestC
// ParseIfExists implements manifest.
func (j *chunkJournal) ParseIfExists(ctx context.Context, stats *Stats, readHook func() error) (ok bool, mc manifestContents, err error) {
if emptyAddr(j.contents.lock) {
ok, mc, err = j.backing.ParseIfExists(ctx, stats, readHook)
if err != nil || !ok {
return false, manifestContents{}, err
}
j.contents = mc
return
if j.journal == nil {
// parse contents from |j.backing| if the journal is not initialized
return j.backing.ParseIfExists(ctx, stats, readHook)
}
if readHook != nil {
if err = readHook(); err != nil {
@@ -208,29 +263,58 @@ func (j *chunkJournal) ParseIfExists(ctx context.Context, stats *Stats, readHook
return
}
func (j *chunkJournal) flushManifest() error {
ctx, s := context.Background(), &Stats{}
_, last, err := j.backing.ParseIfExists(ctx, s, nil)
if err != nil {
return err
func (j *chunkJournal) maybeInit(ctx context.Context) (err error) {
if j.journal == nil {
err = j.openJournal(ctx)
}
if !emptyAddr(j.contents.lock) {
_, err = j.backing.Update(ctx, last.lock, j.contents, s, nil)
}
return err
return
}
// Close implements io.Closer
func (j *chunkJournal) Close() (err error) {
if cerr := j.flushManifest(); cerr != nil {
ctx := context.Background()
_, last, cerr := j.backing.ParseIfExists(ctx, &Stats{}, nil)
if cerr != nil {
err = cerr
} else if !emptyAddr(j.contents.lock) {
// best effort update to |backing|, this will
// fail if the database has been deleted.
// if we spuriously fail, we'll update |backing|
// next time we open this chunkJournal
_, _ = j.backing.Update(ctx, last.lock, j.contents, &Stats{}, nil)
}
if cerr := j.journal.Close(); cerr != nil {
err = cerr
if j.journal != nil {
err = j.journal.Close()
}
return
}
type journalConjoiner struct {
child conjoinStrategy
}
func (c journalConjoiner) conjoinRequired(ts tableSet) bool {
return c.child.conjoinRequired(ts)
}
func (c journalConjoiner) chooseConjoinees(upstream []tableSpec) (conjoinees, keepers []tableSpec, err error) {
var stash tableSpec // don't conjoin journal
pruned := make([]tableSpec, 0, len(upstream))
for _, ts := range upstream {
if isJournalAddr(ts.name) {
stash = ts
} else {
pruned = append(pruned, ts)
}
}
conjoinees, keepers, err = c.child.chooseConjoinees(pruned)
if err != nil {
return nil, nil, err
}
keepers = append(keepers, stash)
return
}
func (s journalChunkSource) has(h addr) (bool, error) {
_, ok := s.lookups[h]
return ok, nil
@@ -264,7 +348,6 @@ func (s journalChunkSource) getCompressed(_ context.Context, h addr, _ *Stats) (
return CompressedChunk{}, fmt.Errorf("chunk record hash does not match lookup hash (%s != %s)",
h.String(), rec.address.String())
}
return NewCompressedChunk(hash.Hash(h), rec.payload)
}
@@ -316,7 +399,8 @@ func (s journalChunkSource) getManyCompressed(ctx context.Context, _ *errgroup.G
}
func (s journalChunkSource) count() (uint32, error) {
return uint32(len(s.lookups)), nil
cnt := uint32(len(s.lookups))
return cnt, nil
}
func (s journalChunkSource) uncompressedLen() (uint64, error) {
@@ -329,12 +413,9 @@ func (s journalChunkSource) hash() addr {
}
// reader implements chunkSource.
func (s journalChunkSource) reader(context.Context) (io.Reader, error) {
// todo(andy): |reader()| belongs to the chunkSource interface and exists
// due to the duality between chunkSources & table files. chunkJournal
// seeks to create many chunkSources that depend on a single file.
// |reader()| in particular is relevant to conjoin implementations.
panic("unimplemented")
func (s journalChunkSource) reader(context.Context) (io.Reader, uint64, error) {
rdr, sz, err := s.journal.snapshot()
return rdr, uint64(sz), err
}
func (s journalChunkSource) getRecordRanges(requests []getRecord) (map[hash.Hash]Range, error) {
@@ -348,23 +429,20 @@ func (s journalChunkSource) getRecordRanges(requests []getRecord) (map[hash.Hash
continue
}
req.found = true // update |requests|
ranges[hash.Hash(*req.a)] = Range{
Offset: uint64(l.offset),
Length: l.length,
}
ranges[hash.Hash(*req.a)] = rangeFromLookup(l)
}
return ranges, nil
}
// size implements chunkSource.
// size returns the total size of the chunkSource: chunks, index, and footer
func (s journalChunkSource) size() (uint64, error) {
return s.compressedSz, nil // todo(andy)
func (s journalChunkSource) currentSize() uint64 {
return uint64(s.journal.currentSize())
}
// index implements chunkSource.
func (s journalChunkSource) index() (tableIndex, error) {
panic("unimplemented")
return nil, fmt.Errorf("journalChunkSource cannot be conjoined")
}
func (s journalChunkSource) clone() (chunkSource, error) {
go/store/nbs/chunk_journal_test.go
+59 -20
View File
@@ -26,27 +26,32 @@ import (
"github.com/stretchr/testify/suite"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/constants"
"github.com/dolthub/dolt/go/store/d"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/types"
)
func makeTestChunkJournal(t *testing.T) *chunkJournal {
cacheOnce.Do(makeGlobalCaches)
ctx := context.Background()
dir, err := os.MkdirTemp("", "")
require.NoError(t, err)
m, err := getFileManifest(ctx, dir)
require.NoError(t, err)
q := NewUnlimitedMemQuotaProvider()
p := newFSTablePersister(dir, globalFDCache, q)
nbf := types.Format_Default.VersionString()
j, err := newChunkJournal(ctx, nbf, dir, m, p.(*fsTablePersister))
require.NoError(t, err)
return j
}
func TestChunkJournalBlockStoreSuite(t *testing.T) {
cacheOnce.Do(makeGlobalCaches)
fn := func(ctx context.Context, dir string) (*NomsBlockStore, error) {
m, err := getFileManifest(ctx, dir)
if err != nil {
return nil, err
}
j, err := newChunkJournal(ctx, dir, m)
if err != nil {
return nil, err
}
nbf := constants.FormatDefaultString
mm := makeManifestManager(j)
q := NewUnlimitedMemQuotaProvider()
c := inlineConjoiner{defaultMaxTables}
return newNomsBlockStore(ctx, nbf, mm, j, q, c, testMemTableSize)
nbf := types.Format_Default.VersionString()
return NewLocalJournalingStore(ctx, nbf, dir, q)
}
suite.Run(t, &BlockStoreSuite{
factory: fn,
@@ -56,13 +61,7 @@ func TestChunkJournalBlockStoreSuite(t *testing.T) {
func TestChunkJournalPersist(t *testing.T) {
ctx := context.Background()
dir, err := os.MkdirTemp("", "")
require.NoError(t, err)
m, err := getFileManifest(ctx, dir)
require.NoError(t, err)
j, err := newChunkJournal(ctx, dir, m)
require.NoError(t, err)
j := makeTestChunkJournal(t)
const iters = 64
stats := &Stats{}
haver := emptyChunkSource{}
@@ -86,6 +85,46 @@ func TestChunkJournalPersist(t *testing.T) {
}
}
func TestReadRecordRanges(t *testing.T) {
ctx := context.Background()
j := makeTestChunkJournal(t)
var buf []byte
mt, data := randomMemTable(256)
gets := make([]getRecord, 0, len(data))
for h := range data {
gets = append(gets, getRecord{a: &h, prefix: h.Prefix()})
}
jcs, err := j.Persist(ctx, mt, emptyChunkSource{}, &Stats{})
require.NoError(t, err)
rdr, sz, err := jcs.(journalChunkSource).journal.snapshot()
require.NoError(t, err)
buf = make([]byte, sz)
n, err := rdr.Read(buf)
require.NoError(t, err)
assert.Equal(t, int(sz), n)
ranges, err := jcs.getRecordRanges(gets)
require.NoError(t, err)
for h, rng := range ranges {
b, err := jcs.get(ctx, addr(h), &Stats{})
assert.NoError(t, err)
ch1 := chunks.NewChunkWithHash(h, b)
assert.Equal(t, data[addr(h)], ch1)
start, stop := rng.Offset, uint32(rng.Offset)+rng.Length
cc2, err := NewCompressedChunk(h, buf[start:stop])
assert.NoError(t, err)
ch2, err := cc2.ToChunk()
assert.NoError(t, err)
assert.Equal(t, data[addr(h)], ch2)
}
}
func TestRoundTripRecords(t *testing.T) {
t.Run("chunk record", func(t *testing.T) {
for i := 0; i < 64; i++ {
go/store/nbs/conjoiner.go
+2 -2
View File
@@ -187,13 +187,13 @@ func conjoin(ctx context.Context, s conjoinStrategy, upstream manifestContents,
}
func conjoinTables(ctx context.Context, conjoinees []tableSpec, p tablePersister, stats *Stats) (conjoined tableSpec, err error) {
eg, ctx := errgroup.WithContext(ctx)
eg, ectx := errgroup.WithContext(ctx)
toConjoin := make(chunkSources, len(conjoinees))
for idx := range conjoinees {
i, spec := idx, conjoinees[idx]
eg.Go(func() (err error) {
toConjoin[i], err = p.Open(ctx, spec.name, spec.chunkCount, stats)
toConjoin[i], err = p.Open(ectx, spec.name, spec.chunkCount, stats)
return
})
}
go/store/nbs/empty_chunk_source.go
+4 -4
View File
@@ -70,16 +70,16 @@ func (ecs emptyChunkSource) index() (tableIndex, error) {
return onHeapTableIndex{}, nil
}
func (ecs emptyChunkSource) reader(context.Context) (io.Reader, error) {
return &bytes.Buffer{}, nil
func (ecs emptyChunkSource) reader(context.Context) (io.Reader, uint64, error) {
return &bytes.Buffer{}, 0, nil
}
func (ecs emptyChunkSource) getRecordRanges(lookups []getRecord) (map[hash.Hash]Range, error) {
return map[hash.Hash]Range{}, nil
}
func (ecs emptyChunkSource) size() (uint64, error) {
return 0, nil
func (ecs emptyChunkSource) currentSize() uint64 {
return 0
}
func (ecs emptyChunkSource) calcReads(reqs []getRecord, blockSize uint64) (reads int, remaining bool, err error) {
go/store/nbs/file_table_persister.go
+8 -1
View File
@@ -50,6 +50,9 @@ type fsTablePersister struct {
q MemoryQuotaProvider
}
var _ tablePersister = &fsTablePersister{}
var _ tableFilePersister = &fsTablePersister{}
func (ftp *fsTablePersister) Open(ctx context.Context, name addr, chunkCount uint32, stats *Stats) (chunkSource, error) {
return newFileTableReader(ctx, ftp.dir, name, chunkCount, ftp.q, ftp.fc)
}
@@ -71,6 +74,10 @@ func (ftp *fsTablePersister) Persist(ctx context.Context, mt *memTable, haver ch
return ftp.persistTable(ctx, name, data, chunkCount, stats)
}
func (ftp *fsTablePersister) Path() string {
return ftp.dir
}
func (ftp *fsTablePersister) persistTable(ctx context.Context, name addr, data []byte, chunkCount uint32, stats *Stats) (cs chunkSource, err error) {
if chunkCount == 0 {
return emptyChunkSource{}, nil
@@ -150,7 +157,7 @@ func (ftp *fsTablePersister) ConjoinAll(ctx context.Context, sources chunkSource
for _, sws := range plan.sources.sws {
var r io.Reader
r, ferr = sws.source.reader(ctx)
r, _, ferr = sws.source.reader(ctx)
if ferr != nil {
return "", ferr
go/store/nbs/journal_writer.go
+121 -54
View File
@@ -23,7 +23,6 @@ import (
"io"
"os"
"path/filepath"
"sync"
"github.com/dolthub/dolt/go/store/d"
"github.com/dolthub/dolt/go/store/hash"
@@ -40,56 +39,84 @@ const (
)
var (
openJournals = new(sync.Map)
journalAddr = addr(hash.Parse(chunkJournalAddr))
journalAddr = addr(hash.Parse(chunkJournalAddr))
)
func openJournalWriter(ctx context.Context, path string) (wr *journalWriter, err error) {
var f *os.File
func isJournalAddr(a addr) bool {
return a == journalAddr
}
func journalFileExists(path string) (bool, error) {
var err error
if path, err = filepath.Abs(path); err != nil {
return false, err
}
info, err := os.Stat(path)
if errors.Is(err, os.ErrNotExist) {
return false, nil
} else if info.IsDir() {
return true, fmt.Errorf("expected file %s found directory", chunkJournalName)
}
return true, nil
}
func openJournalWriter(ctx context.Context, path string) (wr *journalWriter, exists bool, err error) {
var f *os.File
if path, err = filepath.Abs(path); err != nil {
return nil, false, err
}
info, err := os.Stat(path)
if errors.Is(err, os.ErrNotExist) {
return nil, false, nil
} else if err != nil {
return nil, false, err
} else if info.IsDir() {
return nil, true, fmt.Errorf("expected file %s found directory", chunkJournalName)
}
if f, err = os.OpenFile(path, os.O_RDWR, 0666); err != nil {
return nil, true, err
}
return &journalWriter{
buf: make([]byte, 0, journalWriterBuffSize),
file: f,
path: path,
}, true, nil
}
func createJournalWriter(ctx context.Context, path string) (wr *journalWriter, err error) {
var f *os.File
if path, err = filepath.Abs(path); err != nil {
return nil, err
}
if _, ok := openJournals.Load(path); ok {
return nil, fmt.Errorf("journal (%s) already opened in-process", path)
}
openJournals.Store(path, true)
var create bool
info, err := os.Stat(path)
if errors.Is(err, os.ErrNotExist) {
create = true
} else if err != nil {
_, err = os.Stat(path)
if err == nil {
return nil, fmt.Errorf("journal file %s already exists", chunkJournalName)
} else if !errors.Is(err, os.ErrNotExist) {
return nil, err
} else if info.IsDir() {
return nil, fmt.Errorf("expected file %s found directory", chunkJournalName)
}
if create {
if f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666); err != nil {
return nil, err
}
const batch = 1024 * 1024
b := make([]byte, batch)
for i := 0; i < chunkJournalFileSize; i += batch {
if _, err = f.Write(b); err != nil { // zero fill |f|
return nil, err
}
}
if err = f.Sync(); err != nil {
return nil, err
}
if o, err := f.Seek(0, io.SeekStart); err != nil {
return nil, err
} else if o != 0 {
return nil, fmt.Errorf("expected file offset 0, got %d", o)
}
} else {
if f, err = os.OpenFile(path, os.O_RDWR, 0666); err != nil {
if f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666); err != nil {
return nil, err
}
const batch = 1024 * 1024
b := make([]byte, batch)
for i := 0; i < chunkJournalFileSize; i += batch {
if _, err = f.Write(b); err != nil { // zero fill |f|
return nil, err
}
}
if err = f.Sync(); err != nil {
return nil, err
}
if o, err := f.Seek(0, io.SeekStart); err != nil {
return nil, err
} else if o != 0 {
return nil, fmt.Errorf("expected file offset 0, got %d", o)
}
return &journalWriter{
buf: make([]byte, 0, journalWriterBuffSize),
@@ -98,6 +125,16 @@ func openJournalWriter(ctx context.Context, path string) (wr *journalWriter, err
}, nil
}
type snapshotReader interface {
io.ReaderAt
// Snapshot returns an io.Reader that provides a consistent view
// of the current state of the snapshotReader.
snapshot() (io.Reader, int64, error)
// currentSize returns the current size.
currentSize() int64
}
type journalWriter struct {
buf []byte
file *os.File
@@ -105,8 +142,8 @@ type journalWriter struct {
path string
}
var _ io.ReaderAt = &journalWriter{}
var _ io.WriteCloser = &journalWriter{}
var _ snapshotReader = &journalWriter{}
func (wr *journalWriter) filepath() string {
return wr.path
@@ -135,6 +172,23 @@ func (wr *journalWriter) ReadAt(p []byte, off int64) (n int, err error) {
return
}
func (wr *journalWriter) snapshot() (io.Reader, int64, error) {
if err := wr.flush(); err != nil {
return nil, 0, err
}
// open a new file descriptor with an
// independent lifecycle from |wr.file|
f, err := os.Open(wr.path)
if err != nil {
return nil, 0, err
}
return io.LimitReader(f, wr.off), wr.off, nil
}
func (wr *journalWriter) currentSize() int64 {
return wr.off
}
func (wr *journalWriter) Write(p []byte) (n int, err error) {
if len(p) > len(wr.buf) {
// write directly to |wr.file|
@@ -153,8 +207,8 @@ func (wr *journalWriter) Write(p []byte) (n int, err error) {
return
}
func (wr *journalWriter) bootstrapJournal(ctx context.Context) (last hash.Hash, cs journalChunkSource, err error) {
// bootstrap chunk journal from |wr.file|
func (wr *journalWriter) processJournal(ctx context.Context) (last hash.Hash, cs journalChunkSource, err error) {
// maybeInitJournal chunk journal from |wr.file|
src := journalChunkSource{
journal: wr,
address: journalAddr,
@@ -245,21 +299,9 @@ func (wr *journalWriter) Close() (err error) {
if cerr := wr.file.Close(); cerr != nil {
err = cerr
}
openJournals.Delete(wr.path)
return
}
// todo(andy): extensible record format
type jrecord struct {
length uint32
kind jrecordKind
address addr
payload []byte
checksum uint32
}
type jrecordKind uint8
const (
unknownKind jrecordKind = 0
rootHashKind jrecordKind = 1
@@ -270,9 +312,34 @@ const (
recMinSz = recLenSz + recKindSz + addrSize + checksumSize
recMaxSz = 128 * 1024 // todo(andy): less arbitrary
rootHashRecordSize = recMinSz
chunkRecordHeaderSize = recLenSz + recKindSz + addrSize
rootHashRecordSize = recMinSz
)
type jrecordKind uint8
// todo(andy): extensible record format
type jrecord struct {
length uint32
kind jrecordKind
address addr
payload []byte
checksum uint32
}
type jrecordLookup struct {
offset int64
length uint32
}
func rangeFromLookup(l jrecordLookup) Range {
return Range{
Offset: uint64(l.offset) + chunkRecordHeaderSize,
// jrecords are currently double check-summed
Length: uint32(l.length) - (chunkRecordHeaderSize + checksumSize),
}
}
func chunkRecordSize(c CompressedChunk) uint32 {
return uint32(len(c.FullCompressedChunk)) + recMinSz
}
go/store/nbs/journal_writer_test.go
+11 -7
View File
@@ -157,7 +157,8 @@ func TestJournalWriter(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
j, err := openJournalWriter(ctx, newTestFilePath(t))
j, err := createJournalWriter(ctx, newTestFilePath(t))
require.NotNil(t, j)
require.NoError(t, err)
var off int64
@@ -189,7 +190,8 @@ func TestJournalWriter(t *testing.T) {
func TestJournalWriterWriteChunk(t *testing.T) {
ctx := context.Background()
j, err := openJournalWriter(ctx, newTestFilePath(t))
j, err := createJournalWriter(ctx, newTestFilePath(t))
require.NotNil(t, j)
require.NoError(t, err)
data := randomCompressedChunks()
@@ -210,7 +212,8 @@ func TestJournalWriterWriteChunk(t *testing.T) {
func TestJournalWriterBootstrap(t *testing.T) {
ctx := context.Background()
path := newTestFilePath(t)
j, err := openJournalWriter(ctx, path)
j, err := createJournalWriter(ctx, path)
require.NotNil(t, j)
require.NoError(t, err)
data := randomCompressedChunks()
@@ -222,9 +225,9 @@ func TestJournalWriterBootstrap(t *testing.T) {
}
assert.NoError(t, j.Close())
j, err = openJournalWriter(ctx, path)
j, _, err = openJournalWriter(ctx, path)
require.NoError(t, err)
_, source, err := j.bootstrapJournal(ctx)
_, source, err := j.processJournal(ctx)
require.NoError(t, err)
for a, l := range lookups {
@@ -252,9 +255,10 @@ func validateLookup(t *testing.T, j *journalWriter, l jrecordLookup, cc Compress
func TestJournalWriterSyncClose(t *testing.T) {
ctx := context.Background()
j, err := openJournalWriter(ctx, newTestFilePath(t))
j, err := createJournalWriter(ctx, newTestFilePath(t))
require.NotNil(t, j)
require.NoError(t, err)
_, _, err = j.bootstrapJournal(ctx)
_, _, err = j.processJournal(ctx)
require.NoError(t, err)
// close triggers flush
go/store/nbs/manifest.go
+5
View File
@@ -427,6 +427,11 @@ func (mm manifestManager) UpdateGCGen(ctx context.Context, lastLock addr, newCon
return
}
func (mm manifestManager) Close() error {
mm.cache.Delete(mm.Name())
return nil
}
func (mm manifestManager) Name() string {
return mm.m.Name()
}
go/store/nbs/manifest_cache.go
+14
View File
@@ -114,3 +114,17 @@ func (mc *manifestCache) Put(db string, contents manifestContents, t time.Time)
return nil
}
// Delete removes a key from the cache.
func (mc *manifestCache) Delete(db string) {
mc.mu.Lock()
defer mc.mu.Unlock()
if entry, ok := mc.entry(db); ok {
mc.totalSize -= entry.contents.size()
mc.lru.Remove(entry.lruEntry)
delete(mc.cache, db)
}
return
}
go/store/nbs/manifest_cache_test.go
+11
View File
@@ -128,4 +128,15 @@ func TestSizeCache(t *testing.T) {
assert.False(t, ok)
})
t.Run("Delete", func(t *testing.T) {
c := newManifestCache(1 * defSize)
err := c.Put("db", manifestContents{}, time.Now())
require.NoError(t, err)
_, _, ok := c.Get("db")
assert.True(t, ok)
c.Delete("db")
_, _, ok = c.Get("db")
assert.False(t, ok)
})
}
go/store/nbs/store.go
+49 -32
View File
@@ -476,21 +476,33 @@ func newLocalStore(ctx context.Context, nbfVerStr string, dir string, memTableSi
return nil, err
}
p := newFSTablePersister(dir, globalFDCache, q)
c := conjoinStrategy(inlineConjoiner{maxTables})
if chunkJournalFeatureFlag {
j, err := newChunkJournal(ctx, dir, m)
if err != nil {
return nil, err
}
m, p = j, j
return newNomsBlockStore(ctx, nbfVerStr, makeManifestManager(m), p, q, c, memTableSize)
}
func NewLocalJournalingStore(ctx context.Context, nbfVers, dir string, q MemoryQuotaProvider) (*NomsBlockStore, error) {
cacheOnce.Do(makeGlobalCaches)
if err := checkDir(dir); err != nil {
return nil, err
}
mm := makeManifestManager(m)
nbs, err := newNomsBlockStore(ctx, nbfVerStr, mm, p, q, inlineConjoiner{maxTables}, memTableSize)
m, err := getFileManifest(ctx, dir)
if err != nil {
return nil, err
}
return nbs, nil
p := newFSTablePersister(dir, globalFDCache, q)
journal, err := newChunkJournal(ctx, nbfVers, dir, m, p.(*fsTablePersister))
if err != nil {
return nil, err
}
mm := makeManifestManager(journal)
c := journalConjoiner{child: inlineConjoiner{defaultMaxTables}}
// |journal| serves as the manifest and tablePersister
return newNomsBlockStore(ctx, nbfVers, mm, journal, q, c, defaultMemTableSize)
}
func checkDir(dir string) error {
@@ -1111,6 +1123,9 @@ func (nbs *NomsBlockStore) Close() (err error) {
if cerr := nbs.tables.close(); cerr != nil {
err = cerr
}
if cerr := nbs.mm.Close(); cerr != nil {
err = cerr
}
return
}
@@ -1205,15 +1220,10 @@ func newTableFile(cs chunkSource, info tableSpec) tableFile {
return tableFile{
info: info,
open: func(ctx context.Context) (io.ReadCloser, uint64, error) {
s, err := cs.size()
r, s, err := cs.reader(ctx)
if err != nil {
return nil, 0, err
}
r, err := cs.reader(ctx)
if err != nil {
return nil, 0, err
}
return io.NopCloser(r), s, nil
},
}
@@ -1247,11 +1257,7 @@ func (nbs *NomsBlockStore) Size(ctx context.Context) (uint64, error) {
if !ok {
return uint64(0), errors.New("manifest referenced table file for which there is no chunkSource.")
}
sz, err := cs.size()
if err != nil {
return uint64(0), fmt.Errorf("error getting table file index for chunkSource. %w", err)
}
size += sz
size += cs.currentSize()
}
return size, nil
}
@@ -1269,7 +1275,11 @@ func (nbs *NomsBlockStore) chunkSourcesByAddr() (map[addr]chunkSource, error) {
}
func (nbs *NomsBlockStore) SupportedOperations() TableFileStoreOps {
_, ok := nbs.p.(*fsTablePersister)
var ok bool
switch nbs.p.(type) {
case *fsTablePersister, *chunkJournal:
ok = true
}
return TableFileStoreOps{
CanRead: true,
CanWrite: ok,
@@ -1279,17 +1289,21 @@ func (nbs *NomsBlockStore) SupportedOperations() TableFileStoreOps {
}
func (nbs *NomsBlockStore) Path() (string, bool) {
fsPersister, ok := nbs.p.(*fsTablePersister)
if !ok {
return "", false
if tfp, ok := nbs.p.(tableFilePersister); ok {
return tfp.Path(), true
}
return fsPersister.dir, true
return "", false
}
// WriteTableFile will read a table file from the provided reader and write it to the TableFileStore
func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error {
fsPersister, ok := nbs.p.(*fsTablePersister)
if !ok {
var fsPersister *fsTablePersister
switch t := nbs.p.(type) {
case *fsTablePersister:
fsPersister = t
case *chunkJournal:
fsPersister = t.persister
default:
return errors.New("Not implemented")
}
@@ -1478,6 +1492,12 @@ func (nbs *NomsBlockStore) copyMarkedChunks(ctx context.Context, keepChunks <-ch
return nil, err
}
tfp, ok := dest.p.(tableFilePersister)
if !ok {
return nil, fmt.Errorf("NBS does not support copying garbage collection")
}
path := tfp.Path()
LOOP:
for {
select {
@@ -1506,13 +1526,10 @@ LOOP:
return nil, ctx.Err()
}
}
nomsDir := dest.p.(*fsTablePersister).dir
return gcc.copyTablesToDir(ctx, nomsDir)
return gcc.copyTablesToDir(ctx, path)
}
// todo: what's the optimal table size to copy to?
// todo: what's the optimal table currentSize to copy to?
func (nbs *NomsBlockStore) gcTableSize() (uint64, error) {
total, err := nbs.tables.physicalLen()
go/store/nbs/store_test.go
-4
View File
@@ -41,10 +41,6 @@ import (
)
func makeTestLocalStore(t *testing.T, maxTableFiles int) (st *NomsBlockStore, nomsDir string, q MemoryQuotaProvider) {
if chunkJournalFeatureFlag {
t.Skip()
}
ctx := context.Background()
nomsDir = filepath.Join(tempfiles.MovableTempFileProvider.GetTempDir(), "noms_"+uuid.New().String()[:8])
err := os.MkdirAll(nomsDir, os.ModePerm)
go/store/nbs/table.go
+4 -4
View File
@@ -260,14 +260,11 @@ type chunkSource interface {
hash() addr
// opens a Reader to the first byte of the chunkData segment of this table.
reader(context.Context) (io.Reader, error)
reader(context.Context) (io.Reader, uint64, error)
// getRecordRanges sets getRecord.found to true, and returns a Range for each present getRecord query.
getRecordRanges(requests []getRecord) (map[hash.Hash]Range, error)
// size returns the total size of the chunkSource: chunks, index, and footer
size() (uint64, error)
// index returns the tableIndex of this chunkSource.
index() (tableIndex, error)
@@ -277,6 +274,9 @@ type chunkSource interface {
// retained in two objects with independent life-cycle, it should be
// |Clone|d first.
clone() (chunkSource, error)
// currentSize returns the current total physical size of the chunkSource.
currentSize() uint64
}
type chunkSources []chunkSource
go/store/nbs/table_persister.go
+7
View File
@@ -60,6 +60,13 @@ type tablePersister interface {
io.Closer
}
type tableFilePersister interface {
tablePersister
// Path returns the file system path.
Path() string
}
type chunkSourcesByDescendingDataSize struct {
sws []sourceWithSize
err error
go/store/nbs/table_reader.go
+5 -8
View File
@@ -645,9 +645,10 @@ func (tr tableReader) extract(ctx context.Context, chunks chan<- extractRecord)
return nil
}
func (tr tableReader) reader(ctx context.Context) (io.Reader, error) {
func (tr tableReader) reader(ctx context.Context) (io.Reader, uint64, error) {
i, _ := tr.index()
return io.LimitReader(&readerAdapter{tr.r, 0, ctx}, int64(i.tableFileSize())), nil
sz := i.tableFileSize()
return io.LimitReader(&readerAdapter{tr.r, 0, ctx}, int64(sz)), sz, nil
}
func (tr tableReader) getRecordRanges(requests []getRecord) (map[hash.Hash]Range, error) {
@@ -666,12 +667,8 @@ func (tr tableReader) getRecordRanges(requests []getRecord) (map[hash.Hash]Range
return ranges, nil
}
func (tr tableReader) size() (uint64, error) {
i, err := tr.index()
if err != nil {
return 0, err
}
return i.tableFileSize(), nil
func (tr tableReader) currentSize() uint64 {
return tr.idx.tableFileSize()
}
func (tr tableReader) close() error {
go/store/nbs/table_set.go
+1 -5
View File
@@ -231,11 +231,7 @@ func (ts tableSet) uncompressedLen() (uint64, error) {
func (ts tableSet) physicalLen() (uint64, error) {
f := func(css chunkSourceSet) (data uint64, err error) {
for _, haver := range css {
sz, err := haver.size()
if err != nil {
return 0, err
}
data += sz
data += haver.currentSize()
}
return
}
go/store/prolly/map_diff_test.go
+20
View File
@@ -95,6 +95,15 @@ func TestMapDiff(t *testing.T) {
testUpdateDiffs(t, prollyMap.(Map), tuples, s/2)
}
})
// one-sided diffs
var empty Map
t.Run("empty from map", func(t *testing.T) {
testOneSidedDiff(t, s, empty, prollyMap.(Map), tree.AddedDiff)
})
t.Run("empty to map", func(t *testing.T) {
testOneSidedDiff(t, s, prollyMap.(Map), empty, tree.RemovedDiff)
})
})
}
}
@@ -218,6 +227,17 @@ func testUpdateDiffs(t *testing.T, from Map, tups [][2]val.Tuple, numUpdates int
assert.Equal(t, numUpdates, cnt)
}
func testOneSidedDiff(t *testing.T, sz int, from, to Map, typ tree.DiffType) {
var seen int
err := DiffMaps(context.Background(), from, to, func(ctx context.Context, diff tree.Diff) error {
assert.Equal(t, diff.Type, typ)
seen++
return nil
})
assert.Error(t, err, io.EOF)
assert.Equal(t, sz, seen)
}
func makeMapWithDeletes(t *testing.T, m Map, deletes ...[2]val.Tuple) Map {
ctx := context.Background()
mut := m.Mutate()
go/store/prolly/map_test.go
+31
View File
@@ -238,6 +238,37 @@ func TestMutateMapWithTupleIter(t *testing.T) {
}
}
func TestVisitMapLevelOrder(t *testing.T) {
scales := []int{
20,
200,
2000,
20_000,
}
for _, s := range scales {
t.Run("scale "+strconv.Itoa(s), func(t *testing.T) {
ctx := context.Background()
tm, _ := makeProllyMap(t, s)
set1 := hash.NewHashSet()
err := tm.(Map).WalkAddresses(ctx, func(ctx context.Context, addr hash.Hash) error {
set1.Insert(addr)
return nil
})
require.NoError(t, err)
set2 := hash.NewHashSet()
err = VisitMapLevelOrder(ctx, tm.(Map), func(h hash.Hash) (int64, error) {
set2.Insert(h)
return 0, nil
})
require.NoError(t, err)
assert.Equal(t, set1.Size(), set2.Size())
for h := range set1 {
assert.True(t, set2.Has(h))
}
})
}
}
func TestNewEmptyNode(t *testing.T) {
s := message.NewProllyMapSerializer(val.TupleDesc{}, sharedPool)
msg := s.Serialize(nil, nil, nil, 0)
go/store/prolly/tree/diff.go
+17 -6
View File
@@ -48,14 +48,25 @@ func DifferFromRoots[K ~[]byte, O Ordering[K]](
from, to Node,
order O,
) (Differ[K, O], error) {
fc, err := NewCursorAtStart(ctx, fromNs, from)
if err != nil {
return Differ[K, O]{}, err
var fc, tc *Cursor
var err error
if !from.empty() {
fc, err = NewCursorAtStart(ctx, fromNs, from)
if err != nil {
return Differ[K, O]{}, err
}
} else {
fc = &Cursor{}
}
tc, err := NewCursorAtStart(ctx, toNs, to)
if err != nil {
return Differ[K, O]{}, err
if !to.empty() {
tc, err = NewCursorAtStart(ctx, toNs, to)
if err != nil {
return Differ[K, O]{}, err
}
} else {
tc = &Cursor{}
}
fs, err := NewCursorPastEnd(ctx, fromNs, from)
go/store/prolly/tree/map.go
+36
View File
@@ -149,6 +149,42 @@ func MergeOrderedTrees[K, V ~[]byte, O Ordering[K], S message.Serializer](
}, stats, nil
}
// VisitMapLevelOrder visits each internal node of the tree in level order and calls the provided callback `cb` on each hash
// encountered. This function is used primarily for building appendix table files for databases to help optimize reads.
func VisitMapLevelOrder[K, V ~[]byte, O Ordering[K]](
ctx context.Context,
m StaticMap[K, V, O],
cb func(h hash.Hash) (int64, error),
) error {
// get cursor to leaves
cur, err := NewCursorAtStart(ctx, m.NodeStore, m.Root)
if err != nil {
return err
}
first := cur.CurrentKey()
// start by iterating level 1 nodes,
// then recurse upwards until we're at the root
for cur.parent != nil {
cur = cur.parent
for cur.Valid() {
_, err = cb(cur.CurrentRef())
if err != nil {
return err
}
if err = cur.Advance(ctx); err != nil {
return err
}
}
// return cursor to the start of the map
if err = Seek(ctx, cur, K(first), m.Order); err != nil {
return err
}
}
return err
}
func (t StaticMap[K, V, O]) Count() (int, error) {
return t.Root.TreeCount()
}
go/store/prolly/tree/node.go
-27
View File
@@ -113,33 +113,6 @@ func walkOpaqueNodes(ctx context.Context, nd Node, ns NodeStore, cb NodeCb) erro
})
}
type nodeArena []Node
const nodeArenaSize = 10000
func (a *nodeArena) Get() Node {
if len(*a) == 0 {
*a = make([]Node, nodeArenaSize)
}
n := (*a)[len(*a)-1]
*a = (*a)[:len(*a)-1]
return n
}
func (a *nodeArena) NodeFromBytes(msg []byte) (Node, error) {
keys, values, level, count, err := message.UnpackFields(msg)
if err != nil {
return Node{}, err
}
n := a.Get()
n.keys = keys
n.values = values
n.count = count
n.level = level
n.msg = msg
return n, nil
}
func NodeFromBytes(msg []byte) (Node, error) {
keys, values, level, count, err := message.UnpackFields(msg)
return Node{
go/store/prolly/tuple_map.go
+6
View File
@@ -205,6 +205,12 @@ func MergeMaps(ctx context.Context, left, right, base Map, cb tree.CollisionFn)
}, stats, nil
}
// VisitMapLevelOrder visits each internal node of the tree in level order and calls the provided callback `cb` on each hash
// encountered. This function is used primarily for building appendix table files for databases to help optimize reads.
func VisitMapLevelOrder(ctx context.Context, m Map, cb func(h hash.Hash) (int64, error)) error {
return tree.VisitMapLevelOrder(ctx, m.tuples, cb)
}
// NodeStore returns the map's NodeStore
func (m Map) NodeStore() tree.NodeStore {
return m.tuples.NodeStore
integration-tests/bats/checkout.bats
+296
View File
@@ -285,3 +285,299 @@ SQL
[[ "$commitmeta" =~ "$shaparent1" ]] || false
[[ "$commitmeta" =~ "$shaparent2" ]] || false
}
@test "checkout: dolt_checkout brings in changes from main to feature branch that has no working set" {
# original setup
dolt sql -q "create table users (id int primary key, name varchar(32));"
dolt add .
dolt commit -m "original users table"
# create feature branch
dolt branch -c main feature
# make changes on main and verify
dolt sql -q 'insert into users (id, name) values (1, "main-change");'
run dolt sql -q "select name from users"
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# checkout feature branch and bring over main changes
dolt checkout feature
# verify working set changes are brought in from main
run dolt sql << SQL
call dolt_checkout('feature');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# verify working set changes are not on main
run dolt sql << SQL
call dolt_checkout('main');
select count(*) from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
# revert working set changes on feature branch
dolt reset --hard HEAD
run dolt sql -q "select count(*) from users"
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
# switch to main and verify working set changes are not present
dolt checkout main
run dolt sql -q "select count(*) from users"
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
}
@test "checkout: dolt_checkout switches from clean main to feature branch that has changes" {
# original setup
dolt sql -q "create table users (id int primary key, name varchar(32));"
dolt add .
dolt commit -m "original users table"
# create feature branch
dolt branch -c main feature
# make changes on feature (through SQL)
dolt sql << SQL
call dolt_checkout('feature');
insert into users (id, name) values (1, "feature-change");
SQL
# verify feature branch changes are present
run dolt sql << SQL
call dolt_checkout('feature');
select name from users;
SQL
echo "output = $output"
[ "$status" -eq 0 ]
[[ "$output" =~ "feature-change" ]] || false
# checkout feature branch
dolt checkout feature
# verify feature's working set changes are gone
run dolt sql << SQL
call dolt_checkout('feature');
select count(*) from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
# verify working set changes are not on main
run dolt sql << SQL
call dolt_checkout('main');
select count(*) from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
}
@test "checkout: dolt_checkout brings in changes from main to feature branch that has identical changes" {
# original setup
dolt sql -q "create table users (id int primary key, name varchar(32));"
dolt add .
dolt commit -m "original users table"
# create feature branch
dolt branch -c main feature
# make changes on main and verify
dolt sql -q 'insert into users (id, name) values (1, "main-change");'
run dolt sql -q "select name from users"
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# make identical changes on feature (through SQL)
dolt sql << SQL
call dolt_checkout('feature');
insert into users (id, name) values (1, "main-change");
SQL
# verify feature branch changes are present
run dolt sql << SQL
call dolt_checkout('feature');
select name from users;
SQL
echo "output = $output"
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# checkout feature branch
dolt checkout feature
# verify working set changes are still the same on feature branch
run dolt sql << SQL
call dolt_checkout('feature');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# verify working set changes are not on main
run dolt sql << SQL
call dolt_checkout('main');
select count(*) from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
# revert working set changes on feature branch
dolt reset --hard HEAD
# verify working set changes are not on feature branch
run dolt sql << SQL
call dolt_checkout('feature');
select count(*) from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
# switch to main and verify working set changes are not present
dolt checkout main
run dolt sql << SQL
call dolt_checkout('main');
select count(*) from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
}
@test "checkout: dolt_checkout needs -f to bring in changes from main to feature branch that has different changes" {
# original setup
dolt sql -q "create table users (id int primary key, name varchar(32));"
dolt add .
dolt commit -m "original users table"
# create feature branch from main
dolt branch feature
# make changes on main and verify
dolt sql -q 'insert into users (id, name) values (1, "main-change");'
run dolt sql -q "select name from users"
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# make different changes on feature (through SQL)
dolt sql << SQL
call dolt_checkout('feature');
insert into users (id, name) values (2, "feature-change");
SQL
# verify feature branch changes are present
run dolt sql << SQL
call dolt_checkout('feature');
select name from users;
SQL
echo "output = $output"
[ "$status" -eq 0 ]
[[ "$output" =~ "feature-change" ]] || false
# checkout feature branch: should fail due to working set changes
run dolt checkout feature
echo "output = $output"
[ "$status" -eq 1 ]
[[ "$output" =~ "checkout would overwrite uncommitted changes on target branch" ]] || false
# force checkout feature branch
dolt checkout -f feature
# verify working set changes on feature are from main
run dolt sql << SQL
call dolt_checkout('feature');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# verify working set changes are not on main
run dolt sql << SQL
call dolt_checkout('main');
select count(*) from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
}
@test "checkout: dolt_checkout brings changes from main to multiple feature branches and back to main" {
# original setup
dolt sql -q "create table users (id int primary key, name varchar(32));"
dolt add .
dolt commit -m "original users table"
# make changes on main and verify
dolt sql -q 'insert into users (id, name) values (0, "main-change");'
run dolt sql << SQL
call dolt_checkout('main');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# create feature1 branch and bring changes to the new feature branch
dolt checkout -b feature1
# verify the changes are brought to feature1
run dolt sql << SQL
call dolt_checkout('feature1');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
# make changes on feature1 and verify
dolt sql -q 'insert into users (id, name) values (1, "feature1-change");'
run dolt sql << SQL
call dolt_checkout('feature1');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
[[ "$output" =~ "feature1-change" ]] || false
# create feature2 branch and bring changes to next feature branch
dolt checkout -b feature2
# verify the changes are brought to feature1
run dolt sql << SQL
call dolt_checkout('feature2');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
[[ "$output" =~ "feature1-change" ]] || false
# make changes on feature2 and verify
dolt sql -q 'insert into users (id, name) values (2, "feature2-change");'
run dolt sql << SQL
call dolt_checkout('feature2');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
[[ "$output" =~ "feature1-change" ]] || false
[[ "$output" =~ "feature2-change" ]] || false
# bring changes back to main
dolt checkout main
# verify the changes are brought to main
run dolt sql << SQL
call dolt_checkout('main');
select name from users
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "main-change" ]] || false
[[ "$output" =~ "feature1-change" ]] || false
[[ "$output" =~ "feature2-change" ]] || false
}
integration-tests/bats/config.bats
+4 -4
View File
@@ -198,11 +198,11 @@ teardown() {
dolt sql-client --use-db testdb -u dolt -P $PORT -q "insert into a values (1), (2)"
[ -d "testdb" ]
cd testdb
run dolt log
dolt sql-client --use-db testdb -u dolt -P $PORT -q "select * from dolt_log"
run dolt sql-client --use-db testdb -u dolt -P $PORT -q "select * from dolt_log"
[ "$status" -eq 0 ]
regex='Dolt System Account <doltuser@dolthub.com>'
[[ "$output" =~ "$regex" ]] || false
[[ "$output" =~ "Dolt System Account" ]] || false
[[ "$output" =~ "doltuser@dolthub.com" ]] || false
}
@test "config: SQL COMMIT uses default values when user.name or user.email is unset." {
integration-tests/bats/default-values.bats
+6 -6
View File
@@ -537,8 +537,8 @@ DELIM
dolt sql -q "CREATE TABLE test(pk BIGINT PRIMARY KEY, v1 SMALLINT DEFAULT (GREATEST(pk, 2)))"
run dolt sql -q "SELECT column_name, is_nullable, column_default FROM information_schema.columns WHERE table_name = 'test'"
[ "$status" -eq "0" ]
[[ "$output" =~ "| pk | NO | NULL |" ]] || false
[[ "$output" =~ "| v1 | YES | GREATEST(pk, 2) |" ]] || false
[[ "$output" =~ "| pk | NO | NULL |" ]] || false
[[ "$output" =~ "| v1 | YES | greatest(pk,2) |" ]] || false
}
@test "default-values: Additional test with function defaults" {
@@ -550,9 +550,9 @@ DELIM
run dolt sql -q "SELECT column_name, column_default FROM information_schema.columns WHERE table_name = 'test_table'"
[ "$status" -eq "0" ]
[[ "$output" =~ "| pk | NULL |" ]] || false
[[ "$output" =~ "| col2 | LENGTH('hello') |" ]] || false
[[ "$output" =~ "| col3 | ROUND(-1.58, 0) |" ]] || false
[[ "$output" =~ "| col4 | RAND() |" ]] || false
[[ "$output" =~ "| col2 | length('hello') |" ]] || false
[[ "$output" =~ "| col3 | round(-1.58,0) |" ]] || false
[[ "$output" =~ "| col4 | rand() |" ]] || false
}
@test "default-values: Outputting the string version of a more complex default value works" {
@@ -564,6 +564,6 @@ DELIM
[ "$status" -eq "0" ]
[[ "$output" =~ "COLUMN_NAME,COLUMN_DEFAULT" ]] || false
[[ "$output" =~ "pk," ]] || false
[[ "$output" =~ "col2,(RAND() + RAND())" ]] || false
[[ "$output" =~ "col2,(rand() + rand())" ]] || false
[[ "$output" =~ "col3,CASE pk WHEN 1 THEN false ELSE true END" ]] || false
}
integration-tests/bats/filter-branch.bats
+77
View File
@@ -38,6 +38,17 @@ teardown() {
[[ "$output" =~ "1,1" ]] || false
}
@test "filter-branch: verbose mode" {
dolt sql -q "INSERT INTO test VALUES (7,7),(8,8),(9,9);"
dolt commit -Am "added more rows"
run dolt filter-branch -v "DELETE FROM test WHERE pk = 8;"
[ "$status" -eq 0 ]
[[ "$output" =~ "processing commit" ]] || false
[[ "$output" =~ "updated commit" ]] || false
}
@test "filter-branch: filter multiple branches" {
dolt branch other
@@ -72,6 +83,72 @@ teardown() {
[[ "$output" =~ "4,4" ]] || false
}
@test "filter-branch: filter tags" {
dolt sql <<SQL
create table t (pk int primary key);
insert into t values (1),(2);
call dcommit('-Am', 'msg');
insert into t values (3);
call dcommit('-Am', 'three');
call dtag('myTag');
insert into t values (4);
call dcommit('-Am', 'four');
SQL
run dolt sql -q "select * from t as of 'myTag'" -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "2" ]] || false
[[ "$output" =~ "3" ]] || false
dolt filter-branch --all "delete from t where pk >= 3"
run dolt sql -q "select * from t as of 'myTag'" -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "2" ]] || false
[[ ! "$output" =~ "3" ]] || false
}
@test "filter-branch: filter branches only" {
dolt sql <<SQL
create table t (pk int primary key);
insert into t values (1),(2);
call dcommit('-Am', 'msg');
insert into t values (3);
call dcommit('-Am', 'three');
call dtag('myTag');
insert into t values (4);
call dcommit('-Am', 'four');
SQL
run dolt sql -q "select * from t" -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "2" ]] || false
[[ "$output" =~ "3" ]] || false
[[ "$output" =~ "4" ]] || false
run dolt sql -q "select * from t as of 'myTag'" -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "2" ]] || false
[[ "$output" =~ "3" ]] || false
dolt filter-branch --branches "delete from t where pk >= 3"
run dolt sql -q "select * from t" -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "2" ]] || false
[[ ! "$output" =~ "3" ]] || false
[[ ! "$output" =~ "4" ]] || false
run dolt sql -q "select * from t as of 'myTag'" -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "2" ]] || false
[[ "$output" =~ "3" ]] || false
}
@test "filter-branch: with missing table" {
dolt sql -q "DROP TABLE test;"
dolt add -A && dolt commit -m "dropped test"
integration-tests/bats/remotes-aws.bats
-1
View File
@@ -4,7 +4,6 @@ load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
skip_nbf_dolt "need to add a __DOLT__ repo for AWS tests"
}
teardown() {

Some files were not shown because too many files have changed in this diff Show More