mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-12 18:59:03 -06:00
Code Issues Packages Projects Releases 129 Wiki Activity

Merging latest master into release for release 0.19.0

Browse Source
This commit is contained in:
Oscar Batori
2020-09-01 15:20:51 -07:00
parent 567cb90770 38dd2fea0e
commit 560fde25e9
108 changed files with 3994 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/actions/mysql-client-tests/action.yaml vendored Normal file
View File

@@ -0,0 +1,5 @@
name: 'Dolt MySQL client integration tests'
description: 'Smoke tests for mysql client integrations'
runs:
using: 'docker'
image: '../../../MySQLDockerfile'

13
.github/workflows/ci-mysql-client-tests.yaml vendored Normal file
View File

@@ -0,0 +1,13 @@
name: Test MySQL Client integrations
on: [pull_request]
jobs:
mysql_client_integrations_job:
runs-on: ubuntu-latest
name: Run tests
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Test mysql client integrations
uses: ./.github/actions/mysql-client-tests

68
MySQLDockerfile Normal file
View File

@@ -0,0 +1,68 @@
FROM golang:1.15.0-buster as builder
WORKDIR /root/building/go
COPY ./go/ .
# install dolt from source
ENV GOFLAGS="-mod=readonly"
RUN go build -o dolt ./cmd/dolt
FROM ubuntu:18.04
COPY --from=builder /root/building/go/dolt /usr/local/bin/dolt
COPY ./mysql-client-tests /mysql-client-tests
COPY ./mysql-client-tests-entrypoint.sh /mysql-client-tests/entrypoint.sh
# install python
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update -y
RUN apt install -y software-properties-common
RUN add-apt-repository ppa:deadsnakes/ppa -y
RUN apt install python3.8 -y
RUN python3 -V
# install pip
RUN apt-get -y install python3-pip
# install curl
RUN apt-get -y install curl
# install mysql connector and pymsql
RUN pip3 install mysql-connector-python
RUN pip3 install PyMySQL
# install OpenJDK-8
RUN apt-get update && \
apt-get install -y openjdk-8-jdk && \
apt-get install -y ant && \
apt-get clean;
# Fix certificate issues
RUN apt-get update && \
apt-get install ca-certificates-java && \
apt-get clean && \
update-ca-certificates -f;
# Setup JAVA_HOME -- useful for docker commandline
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
RUN export JAVA_HOME
# install mysql connector java
RUN curl -L -o /mysql-client-tests/java/mysql-connector-java-8.0.21.jar \
https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.21/mysql-connector-java-8.0.21.jar
# install node and npm
RUN apt update
RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
RUN cat /etc/apt/sources.list.d/nodesource.list
RUN apt -y install nodejs
# install bats
RUN apt-get -y update
RUN apt-get -y install bats
# install node deps
WORKDIR /mysql-client-tests/node
RUN npm install
WORKDIR /mysql-client-tests
ENTRYPOINT ["/mysql-client-tests/entrypoint.sh"]

4
README.md
View File

@@ -8,6 +8,8 @@ It is inspired by RDBMS and Git, and attempts to blend concepts about both in a
We also built [DoltHub](https://www.dolthub.com), a cloud based storage solution for hosting Dolt databases, that facilitates collaborative management of databases. We host public data for free!
If you want to discuss Dolt, feel free to join our [Discord](https://discord.com/invite/RFwfYpu). We are happy to discuss use-cases, the roadmap, or anything related to version controlled databases.
## Dolt CLI
```
@@ -304,6 +306,8 @@ dolt also supports directory, aws, and gcs based remotes:
If you have any issues with Dolt, find any bugs, or simply have a question, feel free to file an issue!
If you want to discuss the project in a more open ended manner, join us on [Discord](https://discord.com/invite/RFwfYpu).
# Credits and License
Dolt relies heavily on open source code and ideas from the [Noms](https://github.com/attic-labs/noms) project. We are very thankful to the Noms team for making this code freely available, without which we would not have been able to build Dolt so rapidly.

32
bats/README.md
View File

@@ -12,6 +12,32 @@ bats .
```
This will run all the tests. Specify a particular .bats file to run only those tests.
# Test coverage needed for: #
* large tables
* dolt login
## Here Docs
BATS tests in Dolt make extensive use of [Here Docs](https://en.wikipedia.org/wiki/Here_document).
Common patterns include piping SQL scripts to `dolt sql`:
```sql
dolt sql <<SQL
CREATE TABLE my_table (pk int PRIMARY KEY);
SQL
```
And creating data files for import:
```sql
cat <<DELIM > data.csv
pk,c1,c2
1,1,1
2,2,2
DELIM
dolt table import -c -pk=pk my_table data.csv
```
## Skipped BATS
Various tests are skipped as TODOs and/or as documentation of known bugs. Eg:
```sql
@test "..." {
...
skip "this test is currently failing because..."
}
```
Skipped BATS can still be partially useful for testing as they execute normally up to `skip` statement.

0
bats/create-tables.bats → bats/import-create-tables.bats
View File

0
bats/replace-tables.bats → bats/import-replace-tables.bats
View File

0
bats/update-tables.bats → bats/import-update-tables.bats
View File

1
bats/no-repo.bats
View File

@@ -34,6 +34,7 @@ teardown() {
[[ "$output" =~ "blame - Show what revision and author last modified each row of a table." ]] || false
[[ "$output" =~ "merge - Merge a branch." ]] || false
[[ "$output" =~ "branch - Create, list, edit, delete branches." ]] || false
[[ "$output" =~ "tag - Create, list, delete tags" ]] || false
[[ "$output" =~ "checkout - Checkout a branch or overwrite a table from HEAD." ]] || false
[[ "$output" =~ "remote - Manage set of tracked repositories." ]] || false
[[ "$output" =~ "push - Push to a dolt remote." ]] || false

62
bats/remotes.bats
View File

@@ -138,6 +138,68 @@ teardown() {
[[ "$output" =~ "this text should remain after pull :p" ]] || false
}
@test "push and pull tags to/from remote" {
dolt remote add test-remote http://localhost:50051/test-org/test-repo
dolt sql <<SQL
CREATE TABLE test (pk int PRIMARY KEY);
INSERT INTO test VALUES (1),(2),(3);
SQL
dolt add . && dolt commit -m "added table test"
dolt push test-remote master
cd "dolt-repo-clones"
run dolt clone http://localhost:50051/test-org/test-repo
[ "$status" -eq 0 ]
cd ../
dolt tag v1 head
dolt push test-remote v1
cd dolt-repo-clones/test-repo
run dolt pull
[[ "$output" =~ "Successfully" ]] || false
run dolt tag
[ "$status" -eq 0 ]
[[ "$output" =~ "v1" ]] || false
}
@test "tags are only pulled if their commit is pulled" {
dolt remote add test-remote http://localhost:50051/test-org/test-repo
dolt sql <<SQL
CREATE TABLE test (pk int PRIMARY KEY);
INSERT INTO test VALUES (1),(2),(3);
SQL
dolt add . && dolt commit -m "added table test"
dolt push test-remote master
cd "dolt-repo-clones"
run dolt clone http://localhost:50051/test-org/test-repo
[ "$status" -eq 0 ]
cd ../
dolt tag v1 head -m "tag message"
dolt push test-remote v1
dolt checkout -b other
dolt sql -q "INSERT INTO test VALUES (8),(9),(10)"
dolt add . && dolt commit -m "added values on branch other"
dolt push -u test-remote other
dolt tag other_tag head -m "other message"
dolt push test-remote other_tag
cd dolt-repo-clones/test-repo
run dolt pull
[[ "$output" =~ "Successfully" ]] || false
run dolt tag
[ "$status" -eq 0 ]
[[ "$output" =~ "v1" ]] || false
[[ ! "$output" =~ "other_tag" ]] || false
dolt fetch
run dolt tag -v
[ "$status" -eq 0 ]
[[ "$output" =~ "v1" ]] || false
[[ "$output" =~ "tag message" ]] || false
[[ "$output" =~ "other_tag" ]] || false
[[ "$output" =~ "other message" ]] || false
}
@test "clone a remote" {
dolt remote add test-remote http://localhost:50051/test-org/test-repo
dolt sql <<SQL

115
bats/rename-tables.bats Normal file
View File

@@ -0,0 +1,115 @@
#!/usr/bin/env bats
load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
}
teardown() {
teardown_common
}
@test "rename a table with sql" {
dolt sql -q "CREATE TABLE test (pk int PRIMARY KEY);"
run dolt sql -q "ALTER TABLE test RENAME quiz;"
[ "$status" -eq 0 ]
run dolt ls
[ "$status" -eq 0 ]
[[ "$output" =~ "quiz" ]] || false
[[ ! "$output" =~ "test" ]] || false
run dolt sql -q "CREATE TABLE test (pk int primary key);"
[ "$status" -eq 0 ]
run dolt sql -q "CREATE TABLE quiz (pk int primary key);"
[ "$status" -ne 0 ]
run dolt ls
[ "$status" -eq 0 ]
[[ "$output" =~ "quiz" ]] || false
[[ "$output" =~ "test" ]] || false
}
# see 'cp-and-mv.bats' for renaming a table with `dolt mv`
@test "diff a renamed table" {
dolt sql <<SQL
CREATE TABLE test (pk int PRIMARY KEY);
SQL
dolt add test
dolt commit -m 'added table test'
run dolt sql -q 'alter table test rename to quiz'
[ "$status" -eq 0 ]
run dolt diff
[ "$status" -eq 0 ]
[[ "${lines[0]}" =~ "diff --dolt a/test b/quiz" ]] || false
[[ "${lines[1]}" =~ "--- a/test @" ]] || false
[[ "${lines[2]}" =~ "+++ b/quiz @" ]] || false
}
@test "sql diff a renamed table" {
dolt sql <<SQL
CREATE TABLE test (pk int PRIMARY KEY);
SQL
dolt add test
dolt commit -m 'added table test'
run dolt sql -q 'alter table test rename to quiz'
[ "$status" -eq 0 ]
dolt diff -r sql
run dolt diff -r sql
[ "$status" -eq 0 ]
[[ "$output" =~ "RENAME TABLE \`test\` TO \`quiz\`" ]] || false
}
@test "merge a renamed table" {
dolt sql <<SQL
CREATE TABLE test (pk int PRIMARY KEY);
INSERT INTO test VALUES (0),(1);
SQL
dolt add -A && dolt commit -m "created table test"
dolt checkout -b other
dolt sql -q "INSERT INTO test VALUES (8);"
dolt add -A && dolt commit -m "inserted some values on branch other"
dolt checkout master
dolt sql <<SQL
RENAME TABLE test TO quiz;
INSERT INTO quiz VALUES (9);
SQL
dolt add -A && dolt commit -m "renamed test to quiz, added values"
skip "merge works on matching table names currently, panics on renames"
run dolt merge other
[ "$status" -eq 0 ]
run dolt ls
[ "$status" -eq 0 ]
[[ "$output" =~ "quiz" ]] || false
[[ ! "$output" =~ "test" ]] || false
run dolt sql -q "SELECT * FROM quiz;"
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "8" ]] || false
[[ "$output" =~ "9" ]] || false
}
@test "Reusing a table name should NOT fail" {
dolt sql -q "CREATE TABLE one (pk int PRIMARY KEY);"
dolt sql -q "CREATE TABLE two (pk int PRIMARY KEY);"
dolt add -A && dolt commit -m "create test tables"
dolt sql -q "DROP TABLE one;"
dolt add -A && dolt commit -m "dropped table one"
dolt sql -q "DROP TABLE two;"
dolt sql -q "CREATE TABLE three (pk int PRIMARY KEY);"
dolt sql -q "DROP TABLE three;"
run dolt sql -q "CREATE TABLE one (pk int PRIMARY KEY);"
[ $status -eq 0 ]
run dolt sql -q "CREATE TABLE two (pk int PRIMARY KEY);"
[ $status -eq 0 ]
run dolt sql -q "CREATE TABLE three (pk int PRIMARY KEY);"
[ $status -eq 0 ]
run dolt ls
[[ "$output" =~ "one" ]] || false
[[ "$output" =~ "two" ]] || false
[[ "$output" =~ "three" ]] || false
run dolt add -A
[ $status -eq 0 ]
run dolt commit -m "commit recreated tables"
[ $status -eq 0 ]
}

21
bats/schema-changes.bats
View File

@@ -34,27 +34,6 @@ teardown() {
[[ ! "$ouput" =~ "Failed to merge schemas" ]] || false
}
@test "rename a table" {
dolt add test
dolt commit -m 'added table test'
run dolt sql -q 'alter table test rename to quiz'
[ "$status" -eq 0 ]
run dolt diff
[ "$status" -eq 0 ]
[[ "${lines[0]}" =~ "diff --dolt a/test b/quiz" ]] || false
[[ "${lines[1]}" =~ "--- a/test @" ]] || false
[[ "${lines[2]}" =~ "+++ b/quiz @" ]] || false
run dolt status
[ "$status" -eq 0 ]
skip "table renames currently ignored by status"
[[ "$output" =~ "deleted: test" ]] || false
[[ "$output" =~ "new table: quiz" ]] || false
dolt add .
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "renamed: test -> quiz" ]] || false
}
@test "dolt schema rename column" {
dolt sql -q 'insert into test values (1,1,1,1,1,1)'
run dolt sql -q "alter table test rename column c1 to c0"

6
bats/sql-conflicts.bats
View File

@@ -45,7 +45,7 @@ teardown() {
dolt merge feature_branch
EXPECTED=$( echo -e "table,num_conflicts\none_pk,1\ntwo_pk,1")
run dolt sql -r csv -q "SELECT * FROM dolt_conflicts"
run dolt sql -r csv -q "SELECT * FROM dolt_conflicts ORDER BY \`table\`"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED" ]] || false
@@ -100,7 +100,7 @@ teardown() {
dolt merge feature_branch
EXPECTED=$( echo -e "table,num_conflicts\none_pk,1\ntwo_pk,1")
run dolt sql -r csv -q "SELECT * FROM dolt_conflicts"
run dolt sql -r csv -q "SELECT * FROM dolt_conflicts ORDER BY \`table\`"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED" ]] || false
@@ -154,7 +154,7 @@ teardown() {
dolt merge feature_branch
EXPECTED=$( echo -e "table,num_conflicts\none_pk,1\ntwo_pk,1")
run dolt sql -r csv -q "SELECT * FROM dolt_conflicts"
run dolt sql -r csv -q "SELECT * FROM dolt_conflicts ORDER BY \`table\`"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED" ]] || false

12
bats/sql-server.bats
View File

@@ -182,24 +182,24 @@ teardown() {
INSERT INTO r2_one_pk (pk,c3) VALUES (1,1);
INSERT INTO r2_one_pk (pk,c3,c4) VALUES (2,2,2),(3,3,3)"
server_query 1 "SELECT * FROM repo1.r1_one_pk" "pk,c1,c2\n0,None,None\n1,1,None\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo2.r2_one_pk" "pk,c3,c4\n0,None,None\n1,1,None\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo1.r1_one_pk ORDER BY pk" "pk,c1,c2\n0,None,None\n1,1,None\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo2.r2_one_pk ORDER BY pk" "pk,c3,c4\n0,None,None\n1,1,None\n2,2,2\n3,3,3"
multi_query 1 "
DELETE FROM r1_one_pk where pk=0;
USE repo2;
DELETE FROM r2_one_pk where pk=0"
server_query 1 "SELECT * FROM repo1.r1_one_pk" "pk,c1,c2\n1,1,None\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo2.r2_one_pk" "pk,c3,c4\n1,1,None\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo1.r1_one_pk ORDER BY pk" "pk,c1,c2\n1,1,None\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo2.r2_one_pk ORDER BY pk" "pk,c3,c4\n1,1,None\n2,2,2\n3,3,3"
multi_query 1 "
UPDATE r1_one_pk SET c2=1 WHERE pk=1;
USE repo2;
UPDATE r2_one_pk SET c4=1 where pk=1"
server_query 1 "SELECT * FROM repo1.r1_one_pk" "pk,c1,c2\n1,1,1\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo2.r2_one_pk" "pk,c3,c4\n1,1,1\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo1.r1_one_pk ORDER BY pk" "pk,c1,c2\n1,1,1\n2,2,2\n3,3,3"
server_query 1 "SELECT * FROM repo2.r2_one_pk ORDER BY pk" "pk,c3,c4\n1,1,1\n2,2,2\n3,3,3"
}

144
bats/status.bats Normal file
View File

@@ -0,0 +1,144 @@
#!/usr/bin/env bats
load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
}
teardown() {
teardown_common
}
@test "no changes" {
dolt status
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "On branch master" ]] || false
[[ "$output" =~ "nothing to commit, working tree clean" ]] || false
dolt sql <<SQL
CREATE TABLE test (pk int PRIMARY KEY);
INSERT INTO test VALUES (0),(1),(2);
SQL
dolt add -A && dolt commit -m "new table"
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "On branch master" ]] || false
[[ "$output" =~ "nothing to commit, working tree clean" ]] || false
}
@test "staged, unstaged, untracked tables" {
dolt sql <<SQL
CREATE TABLE t (pk int PRIMARY KEY);
CREATE TABLE u (pk int PRIMARY KEY);
SQL
dolt add -A && dolt commit -m "tables t, u"
dolt sql <<SQL
INSERT INTO t VALUES (1),(2),(3);
INSERT INTO u VALUES (1),(2),(3);
CREATE TABLE v (pk int PRIMARY KEY);
SQL
dolt add t
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "On branch master" ]] || false
[[ "$output" =~ "Changes to be committed:" ]] || false
[[ "$output" =~ " (use \"dolt reset <table>...\" to unstage)" ]] || false
[[ "$output" =~ " modified: t" ]] || false
[[ "$output" =~ "Changes not staged for commit:" ]] || false
[[ "$output" =~ " (use \"dolt add <table>\" to update what will be committed)" ]] || false
[[ "$output" =~ " (use \"dolt checkout <table>\" to discard changes in working directory)" ]] || false
[[ "$output" =~ " modified: u" ]] || false
[[ "$output" =~ "Untracked files:" ]] || false
[[ "$output" =~ " (use \"dolt add <table|doc>\" to include in what will be committed)" ]] || false
[[ "$output" =~ " new table: v" ]] || false
}
@test "deleted table" {
dolt sql <<SQL
CREATE TABLE t (pk int PRIMARY KEY);
CREATE TABLE u (pk int PRIMARY KEY);
SQL
dolt add -A && dolt commit -m "tables t, u"
dolt table rm t u
dolt add t
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "On branch master" ]] || false
[[ "$output" =~ "Changes to be committed:" ]] || false
[[ "$output" =~ " (use \"dolt reset <table>...\" to unstage)" ]] || false
[[ "$output" =~ " deleted: t" ]] || false
[[ "$output" =~ "Changes not staged for commit:" ]] || false
[[ "$output" =~ " (use \"dolt add <table>\" to update what will be committed)" ]] || false
[[ "$output" =~ " (use \"dolt checkout <table>\" to discard changes in working directory)" ]] || false
[[ "$output" =~ " deleted: u" ]] || false
}
@test "tables in conflict" {
dolt sql <<SQL
CREATE TABLE t (pk int PRIMARY KEY, c0 int);
INSERT INTO t VALUES (1,1);
SQL
dolt add -A && dolt commit -m "created table t"
dolt checkout -b other
dolt sql -q "INSERT INTO t VALUES (2,12);"
dolt add -A && dolt commit -m "added values on branch other"
dolt checkout master
dolt sql -q "INSERT INTO t VALUES (2,2);"
dolt add -A && dolt commit -m "added values on branch master"
run dolt merge other
[ "$status" -eq 0 ]
[[ "$output" =~ "CONFLICT (content): Merge conflict in t" ]] || false
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "On branch master" ]] || false
[[ "$output" =~ "You have unmerged tables." ]] || false
[[ "$output" =~ " (fix conflicts and run \"dolt commit\")" ]] || false
[[ "$output" =~ " (use \"dolt merge --abort\" to abort the merge)" ]] || false
[[ "$output" =~ "Unmerged paths:" ]] || false
[[ "$output" =~ " (use \"dolt add <file>...\" to mark resolution)" ]] || false
[[ "$output" =~ " both modified: t" ]] || false
}
@test "renamed table" {
dolt sql <<SQL
CREATE TABLE test (pk int PRIMARY KEY);
SQL
dolt add test
dolt commit -m 'added table test'
run dolt sql -q 'alter table test rename to quiz'
[ "$status" -eq 0 ]
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "deleted: test" ]] || false
[[ "$output" =~ "new table: quiz" ]] || false
dolt add .
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "renamed: test -> quiz" ]] || false
}
@test "unstaged changes after reset" {
dolt sql <<SQL
CREATE TABLE one (pk int PRIMARY KEY);
CREATE TABLE two (pk int PRIMARY KEY);
INSERT INTO one VALUES (0);
INSERT INTO two VALUES (0);
SQL
dolt add -A && dolt commit -m "create tables one, two"
dolt sql <<SQL
INSERT INTO one VALUES (1);
DROP TABLE two;
CREATE TABLE three (pk int PRIMARY KEY);
SQL
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "modified: one" ]] || false
[[ "$output" =~ "deleted: two" ]] || false
[[ "$output" =~ "new table: three" ]] || false
run dolt reset
[ "$status" -eq 0 ]
[[ "$output" =~ "Unstaged changes after reset:" ]] || false
[[ "$output" =~ "M one" ]] || false
[[ "$output" =~ "D two" ]] || false
}

6
bats/system-tables.bats
View File

@@ -171,12 +171,18 @@ teardown() {
dolt add test
dolt commit -m "Added (1,1) row"
run dolt sql -q "select * from dolt_history_test"
echo $output
echo ${#lines[@]}
[ $status -eq 0 ]
[ "${#lines[@]}" -eq 7 ]
run dolt sql -q "select * from dolt_history_test where pk=1"
echo $output
echo ${#lines[@]}
[ $status -eq 0 ]
[ "${#lines[@]}" -eq 5 ]
run dolt sql -q "select * from dolt_history_test where pk=0"
echo $output
echo ${#lines[@]}
[ $status -eq 0 ]
[ "${#lines[@]}" -eq 6 ]
}

31
benchmark/sql_regressions/DoltRegressionsJenkinsfile
View File

@@ -1,7 +1,36 @@
pipeline {
agent {
kubernetes {
label "liquidata-inc-ld-build"
defaultContainer "jnlp"
yaml """
kind: "Pod"
metadata:
annotations:
app: "jenkins-agent"
sidecar.istio.io/inject: "false"
spec:
containers:
- image: "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/jnlp:latest"
imagePullPolicy: "Always"
name: "jnlp"
resources:
limits:
cpu: "1.8"
memory: "3Gi"
requests:
cpu: "1.8"
memory: "3Gi"
securityContext:
privileged: false
tty: true
workingDir: "/home/jenkins/agent"
restartPolicy: "Never"
securityContext:
fsGroup: 1000
runAsGroup: 1000
runAsUser: 1000
serviceAccount: "jenkins-agent-doltci"
"""
}
}
stages {

2
go/cmd/dolt/commands/clone.go
View File

@@ -350,7 +350,7 @@ func cloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch s
}
remoteRef := ref.NewRemoteRef(remoteName, brnch.GetPath())
err = dEnv.DoltDB.SetHead(ctx, remoteRef, cm)
err = dEnv.DoltDB.SetHeadToCommit(ctx, remoteRef, cm)
if err != nil {
return errhand.BuildDError("error: could not create remote ref at " + remoteRef.String()).AddCause(err).Build()
}

2
go/cmd/dolt/commands/commit.go
View File

@@ -213,7 +213,7 @@ func buildInitalCommitMsg(ctx context.Context, dEnv *env.DoltEnv) string {
color.NoColor = true
currBranch := dEnv.RepoState.CWBHeadRef()
stagedTblDiffs, notStagedTblDiffs, _ := diff.GetTableDiffs(ctx, dEnv)
stagedTblDiffs, notStagedTblDiffs, _ := diff.GetStagedUnstagedTableDeltas(ctx, dEnv)
workingTblsInConflict, _, _, err := merge.GetTablesInConflict(ctx, dEnv)
if err != nil {

76
go/cmd/dolt/commands/fetch.go
View File

@@ -179,12 +179,13 @@ func mapRefspecsToRemotes(refSpecs []ref.RemoteRefSpec, dEnv *env.DoltEnv) (map[
}
func fetchRefSpecs(ctx context.Context, mode ref.RefUpdateMode, dEnv *env.DoltEnv, rem env.Remote, refSpecs []ref.RemoteRefSpec) errhand.VerboseError {
for _, rs := range refSpecs {
srcDB, err := rem.GetRemoteDB(ctx, dEnv.DoltDB.ValueReadWriter().Format())
srcDB, err := rem.GetRemoteDB(ctx, dEnv.DoltDB.ValueReadWriter().Format())
if err != nil {
return errhand.BuildDError("error: failed to get remote db").AddCause(err).Build()
}
if err != nil {
return errhand.BuildDError("error: failed to get remote db").AddCause(err).Build()
}
for _, rs := range refSpecs {
branchRefs, err := srcDB.GetRefs(ctx)
@@ -204,7 +205,7 @@ func fetchRefSpecs(ctx context.Context, mode ref.RefUpdateMode, dEnv *env.DoltEn
switch mode {
case ref.ForceUpdate:
err = dEnv.DoltDB.SetHead(ctx, remoteTrackRef, srcDBCommit)
err = dEnv.DoltDB.SetHeadToCommit(ctx, remoteTrackRef, srcDBCommit)
case ref.FastForwardOnly:
ok, err := dEnv.DoltDB.CanFastForward(ctx, remoteTrackRef, srcDBCommit)
if !ok {
@@ -222,6 +223,12 @@ func fetchRefSpecs(ctx context.Context, mode ref.RefUpdateMode, dEnv *env.DoltEn
}
}
verr := fetchFollowTags(ctx, dEnv, srcDB, dEnv.DoltDB)
if verr != nil {
return verr
}
return nil
}
@@ -243,7 +250,7 @@ func fetchRemoteBranch(ctx context.Context, dEnv *env.DoltEnv, rem env.Remote, s
return nil, errhand.BuildDError("error: unable to find '%s' on '%s'", srcRef.GetPath(), rem.Name).Build()
} else {
wg, progChan, pullerEventCh := runProgFuncs()
err = actions.Fetch(ctx, dEnv, destRef, srcDB, destDB, srcDBCommit, progChan, pullerEventCh)
err = actions.FetchCommit(ctx, dEnv, srcDB, destDB, srcDBCommit, progChan, pullerEventCh)
stopProgFuncs(wg, progChan, pullerEventCh)
if err != nil {
@@ -253,3 +260,58 @@ func fetchRemoteBranch(ctx context.Context, dEnv *env.DoltEnv, rem env.Remote, s
return srcDBCommit, nil
}
// fetchFollowTags fetches all tags from the source DB whose commits have already
// been fetched into the destination DB.
// todo: potentially too expensive to iterate over all srcDB tags
func fetchFollowTags(ctx context.Context, dEnv *env.DoltEnv, srcDB, destDB *doltdb.DoltDB) errhand.VerboseError {
err := actions.IterResolvedTags(ctx, srcDB, func(tag *doltdb.Tag) (stop bool, err error) {
stRef, err := tag.GetStRef()
if err != nil {
return true, err
}
tagHash := stRef.TargetHash()
tv, err := destDB.ValueReadWriter().ReadValue(ctx, tagHash)
if err != nil {
return true, err
}
if tv != nil {
// tag is already fetched
return false, nil
}
cmHash, err := tag.Commit.HashOf()
if err != nil {
return true, err
}
cv, err := destDB.ValueReadWriter().ReadValue(ctx, cmHash)
if err != nil {
return true, err
}
if cv == nil {
// neither tag nor commit has been fetched
return false, nil
}
wg, progChan, pullerEventCh := runProgFuncs()
err = actions.FetchTag(ctx, dEnv, srcDB, destDB, tag, progChan, pullerEventCh)
stopProgFuncs(wg, progChan, pullerEventCh)
if err != nil {
return true, err
}
err = destDB.SetHead(ctx, tag.GetDoltRef(), stRef)
return false, err
})
if err != nil {
return errhand.VerboseErrorFromError(err)
}
return nil
}

66
go/cmd/dolt/commands/pull.go
View File

@@ -70,40 +70,60 @@ func (cmd PullCmd) Exec(ctx context.Context, commandStr string, args []string, d
ap := cmd.createArgParser()
help, usage := cli.HelpAndUsagePrinters(cli.GetCommandDocumentation(commandStr, pullDocs, ap))
apr := cli.ParseArgs(ap, args, help)
verr := pullFromRemote(ctx, dEnv, apr)
return HandleVErrAndExitCode(verr, usage)
}
func pullFromRemote(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseResults) errhand.VerboseError {
if apr.NArg() > 1 {
return errhand.BuildDError("dolt pull takes at most one arg").SetPrintUsage().Build()
}
branch := dEnv.RepoState.CWBHeadRef()
var verr errhand.VerboseError
var remoteName string
if apr.NArg() > 1 {
verr = errhand.BuildDError("").SetPrintUsage().Build()
} else {
if apr.NArg() == 1 {
remoteName = apr.Arg(0)
}
if apr.NArg() == 1 {
remoteName = apr.Arg(0)
}
var refSpecs []ref.RemoteRefSpec
refSpecs, verr = dEnv.GetRefSpecs(remoteName)
refSpecs, verr := dEnv.GetRefSpecs(remoteName)
if verr != nil {
return verr
}
if verr == nil {
if len(refSpecs) == 0 {
verr = errhand.BuildDError("error: no refspec for remote").Build()
} else {
remote := dEnv.RepoState.Remotes[refSpecs[0].GetRemote()]
if len(refSpecs) == 0 {
return errhand.BuildDError("error: no refspec for remote").Build()
}
for _, refSpec := range refSpecs {
if remoteTrackRef := refSpec.DestRef(branch); remoteTrackRef != nil {
verr = pullRemoteBranch(ctx, dEnv, remote, branch, remoteTrackRef)
remote := dEnv.RepoState.Remotes[refSpecs[0].GetRemote()]
if verr != nil {
break
}
}
}
for _, refSpec := range refSpecs {
remoteTrackRef := refSpec.DestRef(branch)
if remoteTrackRef != nil {
verr = pullRemoteBranch(ctx, dEnv, remote, branch, remoteTrackRef)
if verr != nil {
return verr
}
}
}
return HandleVErrAndExitCode(verr, usage)
srcDB, err := remote.GetRemoteDB(ctx, dEnv.DoltDB.ValueReadWriter().Format())
if err != nil {
return errhand.BuildDError("error: failed to get remote db").AddCause(err).Build()
}
verr = fetchFollowTags(ctx, dEnv, srcDB, dEnv.DoltDB)
if verr != nil {
return verr
}
return nil
}
func pullRemoteBranch(ctx context.Context, dEnv *env.DoltEnv, r env.Remote, srcRef, destRef ref.DoltRef) errhand.VerboseError {

251
go/cmd/dolt/commands/push.go
View File

@@ -42,6 +42,15 @@ const (
ForcePushFlag = "force"
)
type pushOpts struct {
srcRef ref.DoltRef
destRef ref.DoltRef
remoteRef ref.DoltRef
remote env.Remote
mode ref.RefUpdateMode
setUpstream bool
}
var pushDocs = cli.CommandDocumentationContent{
ShortDesc: "Update remote refs along with associated objects",
LongDesc: `Updates remote refs using local refs, while sending objects necessary to complete the given refs.
@@ -94,16 +103,27 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
help, usage := cli.HelpAndUsagePrinters(cli.GetCommandDocumentation(commandStr, pushDocs, ap))
apr := cli.ParseArgs(ap, args, help)
opts, verr := parsePushArgs(ctx, apr, dEnv)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
verr = doPush(ctx, dEnv, opts)
return HandleVErrAndExitCode(verr, usage)
}
func parsePushArgs(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env.DoltEnv) (*pushOpts, errhand.VerboseError) {
remotes, err := dEnv.GetRemotes()
if err != nil {
cli.PrintErrln("error: failed to read remotes from config.")
return 1
return nil, errhand.BuildDError("error: failed to read remotes from config.").Build()
}
remoteName := "origin"
args = apr.Args()
args := apr.Args()
if len(args) == 1 {
if _, ok := remotes[args[0]]; ok {
remoteName = args[0]
@@ -119,6 +139,12 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
var verr errhand.VerboseError
if remoteOK && len(args) == 1 {
refSpecStr := args[0]
refSpecStr, err = disambiguateRefSpecStr(ctx, dEnv.DoltDB, refSpecStr)
if err != nil {
verr = errhand.VerboseErrorFromError(err)
}
refSpec, err = ref.ParseRefSpec(refSpecStr)
if err != nil {
@@ -127,8 +153,13 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
} else if len(args) == 2 {
remoteName = args[0]
refSpecStr := args[1]
refSpec, err = ref.ParseRefSpec(refSpecStr)
refSpecStr, err = disambiguateRefSpecStr(ctx, dEnv.DoltDB, refSpecStr)
if err != nil {
verr = errhand.VerboseErrorFromError(err)
}
refSpec, err = ref.ParseRefSpec(refSpecStr)
if err != nil {
verr = errhand.BuildDError("error: invalid refspec '%s'", refSpecStr).AddCause(err).Build()
}
@@ -136,21 +167,17 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
verr = errhand.BuildDError("error: --set-upstream requires <remote> and <refspec> params.").SetPrintUsage().Build()
} else if hasUpstream {
if len(args) > 0 {
cli.PrintErrf("fatal: upstream branch set for '%s'. Use 'dolt push' without arguments to push.\n", currentBranch)
return 1
return nil, errhand.BuildDError("fatal: upstream branch set for '%s'. Use 'dolt push' without arguments to push.\n", currentBranch).Build()
}
if currentBranch.GetPath() != upstream.Merge.Ref.GetPath() {
cli.PrintErrln("fatal: The upstream branch of your current branch does not match")
cli.PrintErrln("the name of your current branch. To push to the upstream branch")
cli.PrintErrln("on the remote, use")
cli.PrintErrln()
cli.PrintErrln("\tdolt push origin HEAD:" + currentBranch.GetPath())
cli.PrintErrln()
cli.PrintErrln("To push to the branch of the same name on the remote, use")
cli.PrintErrln()
cli.PrintErrln("\tdolt push origin HEAD")
return 1
return nil, errhand.BuildDError("fatal: The upstream branch of your current branch does not match"+
"the name of your current branch. To push to the upstream branch\n"+
"on the remote, use\n\n"+
"\tdolt push origin HEAD: %s\n\n"+
"To push to the branch of the same name on the remote, use\n\n"+
"\tdolt push origin HEAD",
currentBranch.GetPath()).Build()
}
remoteName = upstream.Remote
@@ -162,11 +189,9 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
remoteName = defRemote.Name
}
cli.PrintErrln("fatal: The current branch " + currentBranch.GetPath() + " has no upstream branch.")
cli.PrintErrln("To push the current branch and set the remote as upstream, use")
cli.PrintErrln()
cli.PrintErrln("\tdolt push --set-upstream " + remoteName + " " + currentBranch.GetPath())
return 1
return nil, errhand.BuildDError("fatal: The current branch " + currentBranch.GetPath() + " has no upstream branch.\n" +
"To push the current branch and set the remote as upstream, use\n" +
"\tdolt push --set-upstream " + remoteName + " " + currentBranch.GetPath()).Build()
}
verr = errhand.BuildDError("").SetPrintUsage().Build()
@@ -175,67 +200,135 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
remote, remoteOK = remotes[remoteName]
if !remoteOK {
cli.PrintErrln("fatal: unknown remote " + remoteName)
return 1
return nil, errhand.BuildDError("fatal: unknown remote " + remoteName).Build()
}
if verr == nil {
hasRef, err := dEnv.DoltDB.HasRef(ctx, currentBranch)
hasRef, err := dEnv.DoltDB.HasRef(ctx, currentBranch)
if err != nil {
verr = errhand.BuildDError("error: failed to read from db").AddCause(err).Build()
} else if !hasRef {
verr = errhand.BuildDError("fatal: unknown branch " + currentBranch.GetPath()).Build()
} else {
src := refSpec.SrcRef(currentBranch)
dest := refSpec.DestRef(src)
if err != nil {
return nil, errhand.BuildDError("error: failed to read from db").AddCause(err).Build()
} else if !hasRef {
return nil, errhand.BuildDError("fatal: unknown branch " + currentBranch.GetPath()).Build()
}
var remoteRef ref.DoltRef
remoteRef, verr = getTrackingRef(dest, remote)
src := refSpec.SrcRef(currentBranch)
dest := refSpec.DestRef(src)
if verr == nil {
destDB, err := remote.GetRemoteDB(ctx, dEnv.DoltDB.ValueReadWriter().Format())
var remoteRef ref.DoltRef
if err != nil {
bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err)
switch src.GetType() {
case ref.BranchRefType:
remoteRef, verr = getTrackingRef(dest, remote)
case ref.TagRefType:
if apr.Contains(SetUpstreamFlag) {
verr = errhand.BuildDError("cannot set upstream for tag").Build()
}
default:
verr = errhand.BuildDError("cannot push ref %s of type %s", src.String(), src.GetType()).Build()
}
if err == remotestorage.ErrInvalidDoltSpecPath {
urlObj, _ := earl.Parse(remote.Url)
bdr.AddDetails("For the remote: %s %s", remote.Name, remote.Url)
if verr != nil {
return nil, verr
}
path := urlObj.Path
if path[0] == '/' {
path = path[1:]
}
opts := &pushOpts{
srcRef: src,
destRef: dest,
remoteRef: remoteRef,
remote: remote,
mode: ref.RefUpdateMode{
Force: apr.Contains(ForcePushFlag),
},
setUpstream: apr.Contains(SetUpstreamFlag),
}
bdr.AddDetails("'%s' should be in the format 'organization/repo'", path)
}
return opts, nil
}
verr = bdr.Build()
} else if src == ref.EmptyBranchRef {
verr = deleteRemoteBranch(ctx, dest, remoteRef, dEnv.DoltDB, destDB, remote)
} else {
updateMode := ref.RefUpdateMode{Force: apr.Contains(ForcePushFlag)}
verr = pushToRemoteBranch(ctx, dEnv, updateMode, src, dest, remoteRef, dEnv.DoltDB, destDB, remote)
}
}
// if possible, convert refs to full spec names. prefer branches over tags.
// eg "master" -> "refs/heads/master", "v1" -> "refs/tags/v1"
func disambiguateRefSpecStr(ctx context.Context, ddb *doltdb.DoltDB, refSpecStr string) (string, error) {
brachRefs, err := ddb.GetBranches(ctx)
if verr == nil && apr.Contains(SetUpstreamFlag) {
dEnv.RepoState.Branches[src.GetPath()] = env.BranchConfig{
Merge: ref.MarshalableRef{Ref: dest},
Remote: remoteName,
}
if err != nil {
return "", err
}
err := dEnv.RepoState.Save(dEnv.FS)
if err != nil {
verr = errhand.BuildDError("error: failed to save repo state").AddCause(err).Build()
}
}
for _, br := range brachRefs {
if br.GetPath() == refSpecStr {
return br.String(), nil
}
}
return HandleVErrAndExitCode(verr, usage)
tagRefs, err := ddb.GetTags(ctx)
if err != nil {
return "", err
}
for _, tr := range tagRefs {
if tr.GetPath() == refSpecStr {
return tr.String(), nil
}
}
return refSpecStr, nil
}
func doPush(ctx context.Context, dEnv *env.DoltEnv, opts *pushOpts) (verr errhand.VerboseError) {
destDB, err := opts.remote.GetRemoteDB(ctx, dEnv.DoltDB.ValueReadWriter().Format())
if err != nil {
bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err)
if err == remotestorage.ErrInvalidDoltSpecPath {
urlObj, _ := earl.Parse(opts.remote.Url)
bdr.AddDetails("For the remote: %s %s", opts.remote.Name, opts.remote.Url)
path := urlObj.Path
if path[0] == '/' {
path = path[1:]
}
bdr.AddDetails("'%s' should be in the format 'organization/repo'", path)
}
return bdr.Build()
}
switch opts.srcRef.GetType() {
case ref.BranchRefType:
if opts.srcRef == ref.EmptyBranchRef {
verr = deleteRemoteBranch(ctx, opts.destRef, opts.remoteRef, dEnv.DoltDB, destDB, opts.remote)
} else {
verr = pushToRemoteBranch(ctx, dEnv, opts.mode, opts.srcRef, opts.destRef, opts.remoteRef, dEnv.DoltDB, destDB, opts.remote)
}
case ref.TagRefType:
verr = pushTagToRemote(ctx, dEnv, opts.srcRef, opts.destRef, dEnv.DoltDB, destDB)
default:
verr = errhand.BuildDError("cannot push ref %s of type %s", opts.srcRef.String(), opts.srcRef.GetType()).Build()
}
if verr != nil {
return verr
}
if opts.setUpstream {
dEnv.RepoState.Branches[opts.srcRef.GetPath()] = env.BranchConfig{
Merge: ref.MarshalableRef{
Ref: opts.destRef,
},
Remote: opts.remote.Name,
}
err := dEnv.RepoState.Save(dEnv.FS)
if err != nil {
verr = errhand.BuildDError("error: failed to save repo state").AddCause(err).Build()
}
}
return verr
}
func getTrackingRef(branchRef ref.DoltRef, remote env.Remote) (ref.DoltRef, errhand.VerboseError) {
@@ -307,6 +400,30 @@ func pushToRemoteBranch(ctx context.Context, dEnv *env.DoltEnv, mode ref.RefUpda
return nil
}
func pushTagToRemote(ctx context.Context, dEnv *env.DoltEnv, srcRef, destRef ref.DoltRef, localDB, remoteDB *doltdb.DoltDB) errhand.VerboseError {
tg, err := localDB.ResolveTag(ctx, srcRef.(ref.TagRef))
if err != nil {
return errhand.VerboseErrorFromError(err)
}
wg, progChan, pullerEventCh := runProgFuncs()
err = actions.PushTag(ctx, dEnv, destRef.(ref.TagRef), localDB, remoteDB, tg, progChan, pullerEventCh)
stopProgFuncs(wg, progChan, pullerEventCh)
if err != nil {
if err == doltdb.ErrUpToDate {
cli.Println("Everything up-to-date")
} else {
return errhand.BuildDError("error: push failed").AddCause(err).Build()
}
}
return nil
}
func pullerProgFunc(pullerEventCh chan datas.PullerEvent) {
var pos int
for evt := range pullerEventCh {

29
go/cmd/dolt/commands/reset.go
View File

@@ -244,7 +244,7 @@ func printNotStaged(ctx context.Context, dEnv *env.DoltEnv, staged *doltdb.RootV
return
}
notStagedTbls, err := diff.NewTableDiffs(ctx, working, staged)
notStagedTbls, err := diff.GetTableDeltas(ctx, staged, working)
if err != nil {
return
}
@@ -254,15 +254,28 @@ func printNotStaged(ctx context.Context, dEnv *env.DoltEnv, staged *doltdb.RootV
return
}
if notStagedTbls.NumRemoved+notStagedTbls.NumModified+notStagedDocs.NumRemoved+notStagedDocs.NumModified > 0 {
removeModified := 0
for _, td := range notStagedTbls {
if !td.IsAdd() {
removeModified++
}
}
if removeModified+notStagedDocs.NumRemoved+notStagedDocs.NumModified > 0 {
cli.Println("Unstaged changes after reset:")
lines := make([]string, 0, notStagedTbls.Len()+notStagedDocs.Len())
for _, tblName := range notStagedTbls.Tables {
tdt := notStagedTbls.TableToType[tblName]
if tdt != diff.AddedTable && !doltdb.IsReadOnlySystemTable(tblName) {
lines = append(lines, fmt.Sprintf("%s\t%s", tblDiffTypeToShortLabel[tdt], tblName))
var lines []string
for _, td := range notStagedTbls {
if td.IsAdd() {
// pre Git, unstaged new tables are untracked
continue
} else if td.IsDrop() {
lines = append(lines, fmt.Sprintf("%s\t%s", tblDiffTypeToShortLabel[diff.RemovedTable], td.CurName()))
} else if td.IsRename() {
// per Git, unstaged renames are shown as drop + add
lines = append(lines, fmt.Sprintf("%s\t%s", tblDiffTypeToShortLabel[diff.RemovedTable], td.FromName))
} else {
lines = append(lines, fmt.Sprintf("%s\t%s", tblDiffTypeToShortLabel[diff.ModifiedTable], td.CurName()))
}
}

3
go/cmd/dolt/commands/sql.go
View File

@@ -20,6 +20,7 @@ import (
"io"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/abiosoft/readline"
@@ -1076,7 +1077,7 @@ func newSqlEngine(sqlCtx *sql.Context, readOnly bool, mrEnv env.MultiRepoEnv, ro
return nil, err
}
engine := sqle.New(c, analyzer.NewDefault(c), &sqle.Config{Auth: au})
engine := sqle.New(c, analyzer.NewBuilder(c).WithParallelism(runtime.NumCPU()).Build(), &sqle.Config{Auth: au})
engine.AddDatabase(sql.NewInformationSchemaDatabase(engine.Catalog))
dsess := dsqle.DSessFromSess(sqlCtx.Session)

6
go/cmd/dolt/commands/sqlserver/server.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/liquidata-inc/go-mysql-server/auth"
"github.com/liquidata-inc/go-mysql-server/server"
"github.com/liquidata-inc/go-mysql-server/sql"
"github.com/liquidata-inc/go-mysql-server/sql/analyzer"
"github.com/liquidata-inc/vitess/go/mysql"
"github.com/sirupsen/logrus"
@@ -79,7 +80,10 @@ func Serve(ctx context.Context, version string, serverConfig ServerConfig, serve
}
userAuth := auth.NewAudit(auth.NewNativeSingle(serverConfig.User(), serverConfig.Password(), permissions), auth.NewAuditLog(logrus.StandardLogger()))
sqlEngine := sqle.NewDefault()
c := sql.NewCatalog()
a := analyzer.NewBuilder(c).WithParallelism(serverConfig.QueryParallelism()).Build()
sqlEngine := sqle.New(c, a, nil)
err := sqlEngine.Catalog.Register(dfunctions.DoltFunctions...)

79
go/cmd/dolt/commands/sqlserver/serverconfig.go
View File

@@ -36,15 +36,16 @@ const (
)
const (
defaultHost = "localhost"
defaultPort = 3306
defaultUser = "root"
defaultPass = ""
defaultTimeout = 30 * 1000
defaultReadOnly = false
defaultLogLevel = LogLevel_Info
defaultAutoCommit = true
defaultMaxConnections = 1
defaultHost = "localhost"
defaultPort = 3306
defaultUser = "root"
defaultPass = ""
defaultTimeout = 30 * 1000
defaultReadOnly = false
defaultLogLevel = LogLevel_Info
defaultAutoCommit = true
defaultMaxConnections = 1
defaultQueryParallelism = 2
)
// String returns the string representation of the log level.
@@ -93,19 +94,22 @@ type ServerConfig interface {
DatabaseNamesAndPaths() []env.EnvNameAndPath
// MaxConnections returns the maximum number of simultaneous connections the server will allow. The default is 1
MaxConnections() uint64
// QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer
QueryParallelism() int
}
type commandLineServerConfig struct {
host string
port int
user string
password string
timeout uint64
readOnly bool
logLevel LogLevel
dbNamesAndPaths []env.EnvNameAndPath
autoCommit bool
maxConnections uint64
host string
port int
user string
password string
timeout uint64
readOnly bool
logLevel LogLevel
dbNamesAndPaths []env.EnvNameAndPath
autoCommit bool
maxConnections uint64
queryParallelism int
}
// Host returns the domain that the server will run on. Accepts an IPv4 or IPv6 address, in addition to localhost.
@@ -158,6 +162,11 @@ func (cfg *commandLineServerConfig) MaxConnections() uint64 {
return cfg.maxConnections
}
// QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer
func (cfg *commandLineServerConfig) QueryParallelism() int {
return cfg.queryParallelism
}
// DatabaseNamesAndPaths returns an array of env.EnvNameAndPathObjects corresponding to the databases to be loaded in
// a multiple db configuration. If nil is returned the server will look for a database in the current directory and
// give it a name automatically.
@@ -207,6 +216,19 @@ func (cfg *commandLineServerConfig) withLogLevel(loglevel LogLevel) *commandLine
return cfg
}
// withMaxConnections updates the maximum number of concurrent connections and returns the called
// `*commandLineServerConfig`, which is useful for chaining calls.
func (cfg *commandLineServerConfig) withMaxConnections(maxConnections uint64) *commandLineServerConfig {
cfg.maxConnections = maxConnections
return cfg
}
// withQueryParallelism updates the query parallelism and returns the called `*commandLineServerConfig`, which is useful for chaining calls.
func (cfg *commandLineServerConfig) withQueryParallelism(queryParallelism int) *commandLineServerConfig {
cfg.queryParallelism = queryParallelism
return cfg
}
func (cfg *commandLineServerConfig) withDBNamesAndPaths(dbNamesAndPaths []env.EnvNameAndPath) *commandLineServerConfig {
cfg.dbNamesAndPaths = dbNamesAndPaths
return cfg
@@ -215,15 +237,16 @@ func (cfg *commandLineServerConfig) withDBNamesAndPaths(dbNamesAndPaths []env.En
// DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values.
func DefaultServerConfig() *commandLineServerConfig {
return &commandLineServerConfig{
host: defaultHost,
port: defaultPort,
user: defaultUser,
password: defaultPass,
timeout: defaultTimeout,
readOnly: defaultReadOnly,
logLevel: defaultLogLevel,
autoCommit: defaultAutoCommit,
maxConnections: defaultMaxConnections,
host: defaultHost,
port: defaultPort,
user: defaultUser,
password: defaultPass,
timeout: defaultTimeout,
readOnly: defaultReadOnly,
logLevel: defaultLogLevel,
autoCommit: defaultAutoCommit,
maxConnections: defaultMaxConnections,
queryParallelism: defaultQueryParallelism,
}
}

30
go/cmd/dolt/commands/sqlserver/sqlserver.go
View File

@@ -32,16 +32,17 @@ import (
)
const (
hostFlag = "host"
portFlag = "port"
userFlag = "user"
passwordFlag = "password"
timeoutFlag = "timeout"
readonlyFlag = "readonly"
logLevelFlag = "loglevel"
multiDBDirFlag = "multi-db-dir"
noAutoCommitFlag = "no-auto-commit"
configFileFlag = "config"
hostFlag = "host"
portFlag = "port"
userFlag = "user"
passwordFlag = "password"
timeoutFlag = "timeout"
readonlyFlag = "readonly"
logLevelFlag = "loglevel"
multiDBDirFlag = "multi-db-dir"
noAutoCommitFlag = "no-auto-commit"
configFileFlag = "config"
queryParallelismFlag = "query-parallelism"
)
var sqlServerDocs = cli.CommandDocumentationContent{
@@ -77,6 +78,8 @@ SUPPORTED CONFIG FILE FIELDS:
{{.EmphasisLeft}}listener.write_timeout_millis{{.EmphasisRight}} - The number of milliseconds that the server will wait for a write operation
{{.EmphasisLeft}}performance.query_parallelism{{.EmphasisRight}} - Amount of go routines spawned to process each query
{{.EmphasisLeft}}databases{{.EmphasisRight}} - a list of dolt data repositories to make available as SQL databases. If databases is missing or empty then the working directory must be a valid dolt data repository which will be made available as a SQL database
{{.EmphasisLeft}}databases[i].path{{.EmphasisRight}} - A path to a dolt data repository
@@ -86,7 +89,7 @@ SUPPORTED CONFIG FILE FIELDS:
If a config file is not provided many of these settings may be configured on the command line.`,
Synopsis: []string{
"--config {{.LessThan}}file{{.GreaterThan}}",
"[-H {{.LessThan}}host{{.GreaterThan}}] [-P {{.LessThan}}port{{.GreaterThan}}] [-u {{.LessThan}}user{{.GreaterThan}}] [-p {{.LessThan}}password{{.GreaterThan}}] [-t {{.LessThan}}timeout{{.GreaterThan}}] [-l {{.LessThan}}loglevel{{.GreaterThan}}] [--multi-db-dir {{.LessThan}}directory{{.GreaterThan}}] [-r]",
"[-H {{.LessThan}}host{{.GreaterThan}}] [-P {{.LessThan}}port{{.GreaterThan}}] [-u {{.LessThan}}user{{.GreaterThan}}] [-p {{.LessThan}}password{{.GreaterThan}}] [-t {{.LessThan}}timeout{{.GreaterThan}}] [-l {{.LessThan}}loglevel{{.GreaterThan}}] [--multi-db-dir {{.LessThan}}directory{{.GreaterThan}}] [--query-parallelism {{.LessThan}}num-go-routines{{.GreaterThan}}] [-r]",
},
}
@@ -124,6 +127,7 @@ func createArgParser() *argparser.ArgParser {
ap.SupportsString(logLevelFlag, "l", "Log level", fmt.Sprintf("Defines the level of logging provided\nOptions are: `trace', `debug`, `info`, `warning`, `error`, `fatal` (default `%v`)", serverConfig.LogLevel()))
ap.SupportsString(multiDBDirFlag, "", "directory", "Defines a directory whose subdirectories should all be dolt data repositories accessible as independent databases.")
ap.SupportsFlag(noAutoCommitFlag, "", "When provided sessions will not automatically commit their changes to the working set. Anything not manually committed will be lost.")
ap.SupportsInt(queryParallelismFlag, "", "num-go-routines", fmt.Sprintf("Set the number of go routines spawned to handle each query (default `%d`)", serverConfig.QueryParallelism()))
return ap
}
@@ -232,6 +236,10 @@ func getCommandLineServerConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResult
}
}
if queryParallelism, ok := apr.GetInt(queryParallelismFlag); ok {
serverConfig.withQueryParallelism(queryParallelism)
}
serverConfig.autoCommit = !apr.Contains(noAutoCommitFlag)
return serverConfig, nil
}

25
go/cmd/dolt/commands/sqlserver/yaml_config.go
View File

@@ -67,13 +67,19 @@ type ListenerYAMLConfig struct {
WriteTimeoutMillis *uint64 `yaml:"write_timeout_millis"`
}
// PerformanceYAMLConfig contains configuration parameters for performance tweaking
type PerformanceYAMLConfig struct {
QueryParallelism *int `yaml:"query_parallelism"`
}
// YAMLConfig is a ServerConfig implementation which is read from a yaml file
type YAMLConfig struct {
LogLevelStr *string `yaml:"log_level"`
BehaviorConfig BehaviorYAMLConfig `yaml:"behavior"`
UserConfig UserYAMLConfig `yaml:"user"`
ListenerConfig ListenerYAMLConfig `yaml:"listener"`
DatabaseConfig []DatabaseYAMLConfig `yaml:"databases"`
LogLevelStr *string `yaml:"log_level"`
BehaviorConfig BehaviorYAMLConfig `yaml:"behavior"`
UserConfig UserYAMLConfig `yaml:"user"`
ListenerConfig ListenerYAMLConfig `yaml:"listener"`
DatabaseConfig []DatabaseYAMLConfig `yaml:"databases"`
PerformanceConfig PerformanceYAMLConfig `yaml:"performance"`
}
func serverConfigAsYAMLConfig(cfg ServerConfig) YAMLConfig {
@@ -225,3 +231,12 @@ func (cfg YAMLConfig) MaxConnections() uint64 {
return *cfg.ListenerConfig.MaxConnections
}
// QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer
func (cfg YAMLConfig) QueryParallelism() int {
if cfg.PerformanceConfig.QueryParallelism == nil {
return defaultQueryParallelism
}
return *cfg.PerformanceConfig.QueryParallelism
}

88
go/cmd/dolt/commands/status.go
View File

@@ -70,7 +70,7 @@ func (cmd StatusCmd) Exec(ctx context.Context, commandStr string, args []string,
help, _ := cli.HelpAndUsagePrinters(cli.GetCommandDocumentation(commandStr, statusDocs, ap))
cli.ParseArgs(ap, args, help)
stagedTblDiffs, notStagedTblDiffs, err := diff.GetTableDiffs(ctx, dEnv)
staged, notStaged, err := diff.GetStagedUnstagedTableDeltas(ctx, dEnv)
if err != nil {
cli.PrintErrln(toStatusVErr(err).Verbose())
@@ -97,12 +97,13 @@ func (cmd StatusCmd) Exec(ctx context.Context, commandStr string, args []string,
return 1
}
printStatus(ctx, dEnv, stagedTblDiffs, notStagedTblDiffs, workingTblsInConflict, workingDocsInConflict, stagedDocDiffs, notStagedDocDiffs)
printStatus(ctx, dEnv, staged, notStaged, workingTblsInConflict, workingDocsInConflict, stagedDocDiffs, notStagedDocDiffs)
return 0
}
var tblDiffTypeToLabel = map[diff.TableDiffType]string{
diff.ModifiedTable: "modified:",
diff.RenamedTable: "renamed:",
diff.RemovedTable: "deleted:",
diff.AddedTable: "new table:",
}
@@ -150,22 +151,31 @@ const (
untrackedHeaderHelp = ` (use "dolt add <table|doc>" to include in what will be committed)`
statusFmt = "\t%-16s%s"
statusRenameFmt = "\t%-16s%s -> %s"
bothModifiedLabel = "both modified:"
)
func printStagedDiffs(wr io.Writer, stagedTbls *diff.TableDiffs, stagedDocs *diff.DocDiffs, printHelp bool) int {
if stagedTbls.Len()+stagedDocs.Len() > 0 {
func printStagedDiffs(wr io.Writer, stagedTbls []diff.TableDelta, stagedDocs *diff.DocDiffs, printHelp bool) int {
if len(stagedTbls)+stagedDocs.Len() > 0 {
iohelp.WriteLine(wr, stagedHeader)
if printHelp {
iohelp.WriteLine(wr, stagedHeaderHelp)
}
lines := make([]string, 0, stagedTbls.Len()+stagedDocs.Len())
for _, tblName := range stagedTbls.Tables {
if !doltdb.IsReadOnlySystemTable(tblName) {
tdt := stagedTbls.TableToType[tblName]
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[tdt], tblName))
lines := make([]string, 0, len(stagedTbls)+stagedDocs.Len())
for _, td := range stagedTbls {
if !doltdb.IsReadOnlySystemTable(td.CurName()) {
if td.IsAdd() {
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.AddedTable], td.CurName()))
} else if td.IsDrop() {
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.RemovedTable], td.CurName()))
} else if td.IsRename() {
lines = append(lines, fmt.Sprintf(statusRenameFmt, tblDiffTypeToLabel[diff.RenamedTable], td.FromName, td.ToName))
} else {
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.ModifiedTable], td.CurName()))
}
}
}
@@ -180,7 +190,7 @@ func printStagedDiffs(wr io.Writer, stagedTbls *diff.TableDiffs, stagedDocs *dif
return 0
}
func printDiffsNotStaged(ctx context.Context, dEnv *env.DoltEnv, wr io.Writer, notStagedTbls *diff.TableDiffs, notStagedDocs *diff.DocDiffs, printHelp bool, linesPrinted int, workingTblsInConflict []string) int {
func printDiffsNotStaged(ctx context.Context, dEnv *env.DoltEnv, wr io.Writer, notStagedTbls []diff.TableDelta, notStagedDocs *diff.DocDiffs, printHelp bool, linesPrinted int, workingTblsInConflict []string) int {
inCnfSet := set.NewStrSet(workingTblsInConflict)
if len(workingTblsInConflict) > 0 {
@@ -194,7 +204,7 @@ func printDiffsNotStaged(ctx context.Context, dEnv *env.DoltEnv, wr io.Writer, n
iohelp.WriteLine(wr, mergedTableHelp)
}
lines := make([]string, 0, notStagedTbls.Len())
lines := make([]string, 0, len(notStagedTbls))
for _, tblName := range workingTblsInConflict {
lines = append(lines, fmt.Sprintf(statusFmt, bothModifiedLabel, tblName))
}
@@ -203,7 +213,20 @@ func printDiffsNotStaged(ctx context.Context, dEnv *env.DoltEnv, wr io.Writer, n
linesPrinted += len(lines)
}
numRemovedOrModified := notStagedTbls.NumRemoved + notStagedTbls.NumModified + notStagedDocs.NumRemoved + notStagedDocs.NumModified
added := 0
removeModified := 0
for _, td := range notStagedTbls {
if td.IsAdd() {
added++
} else if td.IsRename() {
added++
removeModified++
} else {
removeModified++
}
}
numRemovedOrModified := removeModified + notStagedDocs.NumRemoved + notStagedDocs.NumModified
docsInCnf, _ := docCnfsOnWorkingRoot(ctx, dEnv)
if numRemovedOrModified-inCnfSet.Size() > 0 {
@@ -211,7 +234,7 @@ func printDiffsNotStaged(ctx context.Context, dEnv *env.DoltEnv, wr io.Writer, n
cli.Println()
}
printChanges := !(notStagedTbls.NumRemoved+notStagedTbls.NumModified == 1 && docsInCnf)
printChanges := !(removeModified == 1 && docsInCnf)
if printChanges {
iohelp.WriteLine(wr, workingHeader)
@@ -228,12 +251,12 @@ func printDiffsNotStaged(ctx context.Context, dEnv *env.DoltEnv, wr io.Writer, n
}
if notStagedTbls.NumAdded > 0 || notStagedDocs.NumAdded > 0 {
if added > 0 || notStagedDocs.NumAdded > 0 {
if linesPrinted > 0 {
cli.Println()
}
printChanges := !(notStagedTbls.NumAdded == 1 && docsInCnf)
printChanges := !(added == 1 && docsInCnf)
if printChanges {
iohelp.WriteLine(wr, untrackedHeader)
@@ -254,13 +277,19 @@ func printDiffsNotStaged(ctx context.Context, dEnv *env.DoltEnv, wr io.Writer, n
return linesPrinted
}
func getModifiedAndRemovedNotStaged(notStagedTbls *diff.TableDiffs, notStagedDocs *diff.DocDiffs, inCnfSet *set.StrSet) (lines []string) {
lines = make([]string, 0, notStagedTbls.Len()+notStagedDocs.Len())
for _, tblName := range notStagedTbls.Tables {
tdt := notStagedTbls.TableToType[tblName]
if tdt != diff.AddedTable && !inCnfSet.Contains(tblName) && tblName != doltdb.DocTableName {
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[tdt], tblName))
func getModifiedAndRemovedNotStaged(notStagedTbls []diff.TableDelta, notStagedDocs *diff.DocDiffs, inCnfSet *set.StrSet) (lines []string) {
lines = make([]string, 0, len(notStagedTbls)+notStagedDocs.Len())
for _, td := range notStagedTbls {
if td.IsAdd() || inCnfSet.Contains(td.CurName()) || td.CurName() == doltdb.DocTableName {
continue
}
if td.IsDrop() {
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.RemovedTable], td.CurName()))
} else if td.IsRename() {
// per Git, unstaged renames are shown as drop + add
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.RemovedTable], td.FromName))
} else {
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.ModifiedTable], td.CurName()))
}
}
@@ -276,13 +305,12 @@ func getModifiedAndRemovedNotStaged(notStagedTbls *diff.TableDiffs, notStagedDoc
return lines
}
func getAddedNotStaged(notStagedTbls *diff.TableDiffs, notStagedDocs *diff.DocDiffs) (lines []string) {
lines = make([]string, 0, notStagedTbls.Len()+notStagedDocs.Len())
for _, tblName := range notStagedTbls.Tables {
tdt := notStagedTbls.TableToType[tblName]
if tdt == diff.AddedTable {
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[tdt], tblName))
func getAddedNotStaged(notStagedTbls []diff.TableDelta, notStagedDocs *diff.DocDiffs) (lines []string) {
lines = make([]string, 0, len(notStagedTbls)+notStagedDocs.Len())
for _, td := range notStagedTbls {
if td.IsAdd() || td.IsRename() {
// per Git, unstaged renames are shown as drop + add
lines = append(lines, fmt.Sprintf(statusFmt, tblDiffTypeToLabel[diff.AddedTable], td.CurName()))
}
}
@@ -297,7 +325,7 @@ func getAddedNotStaged(notStagedTbls *diff.TableDiffs, notStagedDocs *diff.DocDi
return lines
}
func printStatus(ctx context.Context, dEnv *env.DoltEnv, stagedTbls, notStagedTbls *diff.TableDiffs, workingTblsInConflict []string, workingDocsInConflict *diff.DocDiffs, stagedDocs, notStagedDocs *diff.DocDiffs) {
func printStatus(ctx context.Context, dEnv *env.DoltEnv, stagedTbls, notStagedTbls []diff.TableDelta, workingTblsInConflict []string, workingDocsInConflict *diff.DocDiffs, stagedDocs, notStagedDocs *diff.DocDiffs) {
cli.Printf(branchHeader, dEnv.RepoState.CWBHeadRef().GetPath())
if dEnv.RepoState.Merge != nil {

50
go/cmd/dolt/commands/tag.go
View File

@@ -27,16 +27,22 @@ import (
"github.com/liquidata-inc/dolt/go/libraries/doltcore/doltdb"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/env"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/env/actions"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/ref"
"github.com/liquidata-inc/dolt/go/libraries/utils/argparser"
"github.com/liquidata-inc/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/dolt/go/store/hash"
)
var tagDocs = cli.CommandDocumentationContent{
ShortDesc: `Create, list, delete tags.`,
LongDesc: ``,
Synopsis: []string{},
LongDesc: `If there are no non-option arguments, existing tags are listed.
The command's second form creates a new tag named {{.LessThan}}tagname{{.GreaterThan}} which points to the current {{.EmphasisLeft}}HEAD{{.EmphasisRight}}, or {{.LessThan}}ref{{.GreaterThan}} if given. Optionally, a tag message can be passed using the {{.EmphasisLeft}}-m{{.EmphasisRight}} option.
With a {{.EmphasisLeft}}-d{{.EmphasisRight}}, {{.LessThan}}tagname{{.GreaterThan}} will be deleted.`,
Synopsis: []string{
`[-v]`,
`[-m {{.LessThan}}message{{.GreaterThan}}] {{.LessThan}}tagname{{.GreaterThan}} [{{.LessThan}}ref{{.GreaterThan}}]`,
`-d {{.LessThan}}tagname{{.GreaterThan}}`,
},
}
const (
@@ -55,11 +61,6 @@ func (cmd TagCmd) Description() string {
return "Create, list, delete tags."
}
// Hidden should return true if this command should be hidden from the help text
func (cmd *TagCmd) Hidden() bool {
return true
}
// CreateMarkdown creates a markdown file containing the helptext for the command at the given path
func (cmd TagCmd) CreateMarkdown(fs filesys.Filesys, path, commandStr string) error {
ap := cmd.createArgParser()
@@ -69,9 +70,9 @@ func (cmd TagCmd) CreateMarkdown(fs filesys.Filesys, path, commandStr string) er
func (cmd TagCmd) createArgParser() *argparser.ArgParser {
ap := argparser.NewArgParser()
// todo: docs
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"start-point", "A commit that a new branch should point at."})
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"ref", "A commit ref that the tag should point at."})
ap.SupportsString(tagMessageArg, "m", "msg", "Use the given {{.LessThan}}msg{{.GreaterThan}} as the tag message.")
ap.SupportsFlag(verboseFlag, "v", "")
ap.SupportsFlag(verboseFlag, "v", "list tags along with their metadata.")
ap.SupportsFlag(deleteFlag, "d", "Delete a tag.")
return ap
}
@@ -162,18 +163,13 @@ func getTagProps(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) (props actio
func listTags(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseResults) errhand.VerboseError {
var err error
if apr.Contains(verboseFlag) {
err = actions.IterResolvedTags(ctx, dEnv, func(tag ref.DoltRef, c *doltdb.Commit, meta *doltdb.CommitMeta) (bool, error) {
h, err := c.HashOf()
if err != nil {
return false, nil
}
verboseTagPrint(tag, h, meta)
err = actions.IterResolvedTags(ctx, dEnv.DoltDB, func(tag *doltdb.Tag) (bool, error) {
verboseTagPrint(tag)
return false, nil
})
} else {
err = actions.IterResolvedTags(ctx, dEnv, func(tag ref.DoltRef, _ *doltdb.Commit, _ *doltdb.CommitMeta) (bool, error) {
cli.Println(fmt.Sprintf("\t%s", tag.GetPath()))
err = actions.IterResolvedTags(ctx, dEnv.DoltDB, func(tag *doltdb.Tag) (bool, error) {
cli.Println(fmt.Sprintf("\t%s", tag.Name))
return false, nil
})
}
@@ -185,16 +181,18 @@ func listTags(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseRes
return nil
}
func verboseTagPrint(tag ref.DoltRef, h hash.Hash, meta *doltdb.CommitMeta) {
cli.Println(color.YellowString("%s\t%s", tag.GetPath(), h.String()))
func verboseTagPrint(tag *doltdb.Tag) {
h, _ := tag.Commit.HashOf()
cli.Printf("Tagger: %s <%s>\n", meta.Name, meta.Email)
cli.Println(color.YellowString("%s\t%s", tag.Name, h.String()))
timeStr := meta.FormatTS()
cli.Printf("Tagger: %s <%s>\n", tag.Meta.Name, tag.Meta.Email)
timeStr := tag.Meta.FormatTS()
cli.Println("Date: ", timeStr)
if meta.Description != "" {
formattedDesc := "\n\t" + strings.Replace(meta.Description, "\n", "\n\t", -1)
if tag.Meta.Description != "" {
formattedDesc := "\n\t" + strings.Replace(tag.Meta.Description, "\n", "\n\t", -1)
cli.Println(formattedDesc)
}
cli.Println("")

2
go/cmd/dolt/dolt.go
View File

@@ -59,6 +59,7 @@ var doltCommand = cli.NewSubCommandHandler("dolt", "it's git for data", []cli.Co
commands.BlameCmd{},
commands.MergeCmd{},
commands.BranchCmd{},
commands.TagCmd{},
commands.CheckoutCmd{},
commands.RemoteCmd{},
commands.PushCmd{},
@@ -78,7 +79,6 @@ var doltCommand = cli.NewSubCommandHandler("dolt", "it's git for data", []cli.Co
commands.MigrateCmd{},
indexcmds.Commands,
commands.ReadTablesCmd{},
commands.TagCmd{},
})
func init() {

2
go/go.mod
View File

@@ -45,7 +45,7 @@ require (
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d
github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
github.com/liquidata-inc/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20200730200742-c031ec8cba06
github.com/liquidata-inc/go-mysql-server v0.6.1-0.20200811170127-8f76b9511589
github.com/liquidata-inc/go-mysql-server v0.6.1-0.20200820215800-6e65638249cd
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0
github.com/liquidata-inc/mmap-go v1.0.3
github.com/liquidata-inc/sqllogictest/go v0.0.0-20200320151923-b11801f10e15

5
go/go.sum
View File

@@ -156,6 +156,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -419,8 +420,8 @@ github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/liquidata-inc/go-mysql-server v0.6.1-0.20200811170127-8f76b9511589 h1:wqEDpnG3ew30SXBssKtusox/LDRNffYAAKFkYpbzkpU=
github.com/liquidata-inc/go-mysql-server v0.6.1-0.20200811170127-8f76b9511589/go.mod h1:DOvPP9xyN0JcJl6kE25qQaTFT0bKaIu+PWXAbCf6Cg4=
github.com/liquidata-inc/go-mysql-server v0.6.1-0.20200820215800-6e65638249cd h1:nMET3TWaidBmzpRjD2OtB3HY4vT+7NEZUl1q3rqrWcw=
github.com/liquidata-inc/go-mysql-server v0.6.1-0.20200820215800-6e65638249cd/go.mod h1:DOvPP9xyN0JcJl6kE25qQaTFT0bKaIu+PWXAbCf6Cg4=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0 h1:phMgajKClMUiIr+hF2LGt8KRuUa2Vd2GI1sNgHgSXoU=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0/go.mod h1:YC1rI9k5gx8D02ljlbxDfZe80s/iq8bGvaaQsvR+qxs=
github.com/liquidata-inc/mmap-go v1.0.3 h1:2LndAeAtup9rpvUmu4wZSYCsjCQ0Zpc+NqE+6+PnT7g=

103
go/libraries/doltcore/diff/diffs.go
View File

@@ -30,17 +30,10 @@ type TableDiffType int
const (
AddedTable TableDiffType = iota
ModifiedTable
RenamedTable
RemovedTable
)
type TableDiffs struct {
NumAdded int
NumModified int
NumRemoved int
TableToType map[string]TableDiffType
Tables []string
}
type DocDiffType int
const (
@@ -91,91 +84,6 @@ func (rvu RootValueUnreadable) Error() string {
return "error: Unable to read " + rvu.rootType.String()
}
// NewTableDiffs returns the TableDiffs between two roots.
func NewTableDiffs(ctx context.Context, newer, older *doltdb.RootValue) (*TableDiffs, error) {
deltas, err := GetTableDeltas(ctx, older, newer)
if err != nil {
return nil, err
}
var added []string
var modified []string
var removed []string
for _, d := range deltas {
switch {
case d.IsAdd():
added = append(added, d.ToName)
case d.IsDrop():
removed = append(removed, d.FromName)
default:
modified = append(modified, d.ToName)
}
}
var tbls []string
tbls = append(tbls, added...)
tbls = append(tbls, modified...)
tbls = append(tbls, removed...)
tblToType := make(map[string]TableDiffType)
for _, tbl := range added {
tblToType[tbl] = AddedTable
}
for _, tbl := range modified {
tblToType[tbl] = ModifiedTable
}
for _, tbl := range removed {
tblToType[tbl] = RemovedTable
}
sort.Strings(tbls)
return &TableDiffs{len(added), len(modified), len(removed), tblToType, tbls}, err
}
func (td *TableDiffs) Len() int {
return len(td.Tables)
}
// GetTableDiffs returns the staged and unstaged TableDiffs for the repo.
func GetTableDiffs(ctx context.Context, dEnv *env.DoltEnv) (*TableDiffs, *TableDiffs, error) {
headRoot, err := dEnv.HeadRoot(ctx)
if err != nil {
return nil, nil, RootValueUnreadable{HeadRoot, err}
}
stagedRoot, err := dEnv.StagedRoot(ctx)
if err != nil {
return nil, nil, RootValueUnreadable{StagedRoot, err}
}
workingRoot, err := dEnv.WorkingRoot(ctx)
if err != nil {
return nil, nil, RootValueUnreadable{WorkingRoot, err}
}
stagedDiffs, err := NewTableDiffs(ctx, stagedRoot, headRoot)
if err != nil {
return nil, nil, err
}
notStagedDiffs, err := NewTableDiffs(ctx, workingRoot, stagedRoot)
if err != nil {
return nil, nil, err
}
return stagedDiffs, notStagedDiffs, nil
}
// NewDocDiffs returns DocDiffs for Dolt Docs between two roots.
func NewDocDiffs(ctx context.Context, dEnv *env.DoltEnv, older *doltdb.RootValue, newer *doltdb.RootValue, docDetails []doltdb.DocDetails) (*DocDiffs, error) {
var added []string
@@ -401,6 +309,7 @@ func (td TableDelta) IsDrop() bool {
return td.FromTable != nil && td.ToTable == nil
}
// IsRename return true if the table was renamed between the fromRoot and toRoot.
func (td TableDelta) IsRename() bool {
if td.IsAdd() || td.IsDrop() {
return false
@@ -408,6 +317,14 @@ func (td TableDelta) IsRename() bool {
return td.FromName != td.ToName
}
// CurName returns the most recent name of the table.
func (td TableDelta) CurName() string {
if td.ToName != "" {
return td.ToName
}
return td.FromName
}
func (td TableDelta) HasFKChanges() bool {
return !fkSlicesAreEqual(td.FromFks, td.ToFks)
}

16
go/libraries/doltcore/diff/querydiff/query_differ_test.go
View File

@@ -378,15 +378,15 @@ var engineTestSetup = []testCommand{
"(2, 'second row'), " +
"(3, 'third row');"}},
{commands.SqlCmd{}, []string{"-q", "insert into one_pk values " +
"(0, 0, 0, 0, 0, 0)," +
"(1, 10, 10, 10, 10, 10)," +
"(2, 20, 20, 20, 20, 20)," +
"(3, 30, 30, 30, 30, 30);"}},
"(0, 0, 1, 2, 3, 4)," +
"(1, 10, 11, 12, 13, 14)," +
"(2, 20, 21, 22, 23, 24)," +
"(3, 30, 31, 32, 33, 34);"}},
{commands.SqlCmd{}, []string{"-q", "insert into two_pk values " +
"(0, 0, 0, 0, 0, 0, 0)," +
"(0, 1, 10, 10, 10, 10, 10)," +
"(1, 0, 20, 20, 20, 20, 20)," +
"(1, 1, 30, 30, 30, 30, 30);"}},
"(0, 0, 0, 1, 2, 3, 4)," +
"(0, 1, 10, 11, 12, 13, 14)," +
"(1, 0, 20, 21, 22, 23, 24)," +
"(1, 1, 30, 31, 32, 33, 34);"}},
{commands.SqlCmd{}, []string{"-q", "insert into othertable values " +
"('first', 3)," +
"('second', 2)," +

14
go/libraries/doltcore/doltdb/commit.go
View File

@@ -24,10 +24,10 @@ import (
)
const (
metaField = "meta"
parentsField = "parents"
parentsListField = "parentsList"
rootValueField = "value"
metaField = datas.CommitMetaField
parentsField = datas.ParentsField
parentsListField = datas.ParentsListField
rootValueField = datas.ValueField
)
var errCommitHasNoMeta = errors.New("commit has no metadata")
@@ -130,6 +130,7 @@ func (c *Commit) GetCommitMeta() (*CommitMeta, error) {
return nil, errors.New(h.String() + " is a commit without the required metadata.")
}
// ParentHashes returns the commit hashes for all parent commits.
func (c *Commit) ParentHashes(ctx context.Context) ([]hash.Hash, error) {
hashes := make([]hash.Hash, len(c.parents))
for i, pr := range c.parents {
@@ -178,6 +179,11 @@ func (c *Commit) GetRootValue() (*RootValue, error) {
return nil, errHasNoRootValue
}
// GetStRef returns a Noms Ref for this Commit's Noms commit Struct.
func (c *Commit) GetStRef() (types.Ref, error) {
return types.NewRef(c.commitSt, c.vrw.Format())
}
var ErrNoCommonAncestor = errors.New("no common ancestor")
func GetCommitAncestor(ctx context.Context, cm1, cm2 *Commit) (*Commit, error) {

21
go/libraries/doltcore/doltdb/commit_meta.go
View File

@@ -31,7 +31,8 @@ const (
commitMetaUserTSKey = "user_timestamp"
commitMetaVersionKey = "metaversion"
metaVersion = "1.0"
commitMetaStName = "metadata"
commitMetaVersion = "1.0"
)
var CommitNowFunc = time.Now
@@ -74,20 +75,6 @@ func NewCommitMetaWithUserTS(name, email, desc string, userTS time.Time) (*Commi
return &CommitMeta{n, e, ms, d, userMS}, nil
}
// NewTagMeta returns CommitMeta that can be used to create a tag.
func NewTagMeta(name, email, desc string) *CommitMeta {
n := strings.TrimSpace(name)
e := strings.TrimSpace(email)
d := strings.TrimSpace(desc)
ns := uint64(CommitNowFunc().UnixNano())
ms := ns / uMilliToNano
userMS := int64(ns) / milliToNano
return &CommitMeta{n, e, ms, d, userMS}
}
func getRequiredFromSt(st types.Struct, k string) (types.Value, error) {
if v, ok, err := st.MaybeGet(k); err != nil {
return nil, err
@@ -146,11 +133,11 @@ func (cm *CommitMeta) toNomsStruct(nbf *types.NomsBinFormat) (types.Struct, erro
commitMetaEmailKey: types.String(cm.Email),
commitMetaDescKey: types.String(cm.Description),
commitMetaTimestampKey: types.Uint(cm.Timestamp),
commitMetaVersionKey: types.String(metaVersion),
commitMetaVersionKey: types.String(commitMetaVersion),
commitMetaUserTSKey: types.Int(cm.UserTimestamp),
}
return types.NewStruct(nbf, "metadata", metadata)
return types.NewStruct(nbf, commitMetaStName, metadata)
}
// Time returns the time at which the commit occurred

138
go/libraries/doltcore/doltdb/doltdb.go
View File

@@ -194,11 +194,35 @@ func getCommitStForRefStr(ctx context.Context, db datas.Database, ref string) (t
}
dsHead, hasHead := ds.MaybeHead()
if hasHead {
if !hasHead {
return types.EmptyStruct(db.Format()), ErrBranchNotFound
}
if dsHead.Name() == datas.CommitName {
return dsHead, nil
}
return types.EmptyStruct(db.Format()), ErrBranchNotFound
if dsHead.Name() == datas.TagName {
commitRef, ok, err := dsHead.MaybeGet(datas.TagCommitRefField)
if err != nil {
return types.EmptyStruct(db.Format()), err
}
if !ok {
err = fmt.Errorf("tag struct does not have field %s", datas.TagCommitRefField)
return types.EmptyStruct(db.Format()), err
}
commitSt, err := commitRef.(types.Ref).TargetValue(ctx, db)
if err != nil {
return types.EmptyStruct(db.Format()), err
}
return commitSt.(types.Struct), nil
}
err = fmt.Errorf("dataset head is neither commit nor tag")
return types.EmptyStruct(db.Format()), err
}
func getCommitStForHash(ctx context.Context, db datas.Database, c string) (types.Struct, error) {
@@ -279,8 +303,8 @@ func (ddb *DoltDB) Resolve(ctx context.Context, cs *CommitSpec, cwb ref.DoltRef)
// For a ref in a CommitSpec, we have the following behavior.
// If it starts with `refs/`, we look for an exact match before
// we try any suffix matches. After that, we try a match on the
// user supplied input, with the following three prefixes, in
// order: `refs/`, `refs/heads/`, `refs/remotes/`.
// user supplied input, with the following four prefixes, in
// order: `refs/`, `refs/heads/`, `refs/tags/`, `refs/remotes/`.
candidates := []string{
"refs/" + cs.baseSpec,
"refs/heads/" + cs.baseSpec,
@@ -333,6 +357,27 @@ func (ddb *DoltDB) ResolveRef(ctx context.Context, ref ref.DoltRef) (*Commit, er
return NewCommit(ddb.db, commitSt), nil
}
// ResolveTag takes a TagRef and returns the corresponding Tag object.
func (ddb *DoltDB) ResolveTag(ctx context.Context, tagRef ref.TagRef) (*Tag, error) {
ds, err := ddb.db.GetDataset(ctx, tagRef.String())
if err != nil {
return nil, ErrTagNotFound
}
tagSt, hasHead := ds.MaybeHead()
if !hasHead {
return nil, ErrTagNotFound
}
if tagSt.Name() != datas.TagName {
return nil, fmt.Errorf("tagRef head is not a tag")
}
return NewTag(ctx, tagRef.GetPath(), ddb.db, tagSt)
}
// TODO: convenience method to resolve the head commit of a branch.
// WriteRootValue will write a doltdb.RootValue instance to the database. This value will not be associated with a commit
@@ -417,21 +462,26 @@ func (ddb *DoltDB) CanFastForward(ctx context.Context, branch ref.DoltRef, new *
return current.CanFastForwardTo(ctx, new)
}
// SetHead sets the given ref to point at the given commit. It is used in the course of 'force' updates.
func (ddb *DoltDB) SetHead(ctx context.Context, ref ref.DoltRef, cm *Commit) error {
// SetHeadToCommit sets the given ref to point at the given commit. It is used in the course of 'force' updates.
func (ddb *DoltDB) SetHeadToCommit(ctx context.Context, ref ref.DoltRef, cm *Commit) error {
stRef, err := types.NewRef(cm.commitSt, ddb.db.Format())
if err != nil {
return err
}
return ddb.SetHead(ctx, ref, stRef)
}
func (ddb *DoltDB) SetHead(ctx context.Context, ref ref.DoltRef, stRef types.Ref) error {
ds, err := ddb.db.GetDataset(ctx, ref.String())
if err != nil {
return err
}
r, err := types.NewRef(cm.commitSt, ddb.db.Format())
if err != nil {
return err
}
_, err = ddb.db.SetHead(ctx, ds, r)
_, err = ddb.db.SetHead(ctx, ds, stRef)
return err
}
@@ -450,6 +500,7 @@ func (ddb *DoltDB) CommitWithParentSpecs(ctx context.Context, valHash hash.Hash,
return ddb.CommitWithParentCommits(ctx, valHash, dref, parentCommits, cm)
}
// todo: merge with CommitDanglingWithParentCommits
func (ddb *DoltDB) WriteDanglingCommit(ctx context.Context, valHash hash.Hash, parentCommits []*Commit, cm *CommitMeta) (*Commit, error) {
var commitSt types.Struct
val, err := ddb.db.ReadValue(ctx, valHash)
@@ -767,10 +818,6 @@ func (ddb *DoltDB) NewBranchAtCommit(ctx context.Context, dref ref.DoltRef, comm
panic(fmt.Sprintf("invalid branch name %s, use IsValidUserBranchName check", dref.String()))
}
return ddb.newRefAtCommit(ctx, dref, commit)
}
func (ddb *DoltDB) newRefAtCommit(ctx context.Context, dref ref.DoltRef, commit *Commit) error {
ds, err := ddb.db.GetDataset(ctx, dref.String())
if err != nil {
@@ -809,12 +856,43 @@ func (ddb *DoltDB) deleteRef(ctx context.Context, dref ref.DoltRef) error {
}
// NewTagAtCommit create a new tag at the commit given.
func (ddb *DoltDB) NewTagAtCommit(ctx context.Context, tagRef ref.DoltRef, commit *Commit) error {
func (ddb *DoltDB) NewTagAtCommit(ctx context.Context, tagRef ref.DoltRef, c *Commit, meta *TagMeta) error {
if !IsValidTagRef(tagRef) {
panic(fmt.Sprintf("invalid tag name %s, use IsValidUserTagName check", tagRef.String()))
}
return ddb.newRefAtCommit(ctx, tagRef, commit)
ds, err := ddb.db.GetDataset(ctx, tagRef.String())
if err != nil {
return err
}
_, hasHead, err := ds.MaybeHeadRef()
if err != nil {
return err
}
if hasHead {
return fmt.Errorf("dataset already exists for tag %s", tagRef.String())
}
r, err := types.NewRef(c.commitSt, ddb.Format())
if err != nil {
return err
}
st, err := meta.toNomsStruct(ddb.db.Format())
if err != nil {
return err
}
tag := datas.TagOptions{Meta: st}
ds, err = ddb.db.Tag(ctx, ds, r, tag)
return err
}
func (ddb *DoltDB) DeleteTag(ctx context.Context, tag ref.DoltRef) error {
@@ -827,15 +905,9 @@ func (ddb *DoltDB) DeleteTag(ctx context.Context, tag ref.DoltRef) error {
return err
}
// PushChunks initiates a push into a database from the source database given, at the commit given. Pull progress is
// PushChunks initiates a push into a database from the source database given, at the Value ref given. Pull progress is
// communicated over the provided channel.
func (ddb *DoltDB) PushChunks(ctx context.Context, tempDir string, srcDB *DoltDB, cm *Commit, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
rf, err := types.NewRef(cm.commitSt, ddb.db.Format())
if err != nil {
return err
}
func (ddb *DoltDB) PushChunks(ctx context.Context, tempDir string, srcDB *DoltDB, rf types.Ref, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
if datas.CanUsePuller(srcDB.db) && datas.CanUsePuller(ddb.db) {
puller, err := datas.NewPuller(ctx, tempDir, defaultChunksPerTF, srcDB.db, ddb.db, rf.TargetHash(), pullerEventCh)
@@ -869,15 +941,9 @@ func (ddb *DoltDB) PushChunksForRefHash(ctx context.Context, tempDir string, src
// PullChunks initiates a pull into a database from the source database given, at the commit given. Progress is
// communicated over the provided channel.
func (ddb *DoltDB) PullChunks(ctx context.Context, tempDir string, srcDB *DoltDB, cm *Commit, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
rf, err := types.NewRef(cm.commitSt, ddb.db.Format())
if err != nil {
return err
}
func (ddb *DoltDB) PullChunks(ctx context.Context, tempDir string, srcDB *DoltDB, stRef types.Ref, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
if datas.CanUsePuller(srcDB.db) && datas.CanUsePuller(ddb.db) {
puller, err := datas.NewPuller(ctx, tempDir, 256*1024, srcDB.db, ddb.db, rf.TargetHash(), pullerEventCh)
puller, err := datas.NewPuller(ctx, tempDir, 256*1024, srcDB.db, ddb.db, stRef.TargetHash(), pullerEventCh)
if err == datas.ErrDBUpToDate {
return nil
@@ -887,7 +953,7 @@ func (ddb *DoltDB) PullChunks(ctx context.Context, tempDir string, srcDB *DoltDB
return puller.Pull(ctx)
} else {
return datas.PullWithoutBatching(ctx, srcDB.db, ddb.db, rf, progChan)
return datas.PullWithoutBatching(ctx, srcDB.db, ddb.db, stRef, progChan)
}
}

84
go/libraries/doltcore/doltdb/tag.go Normal file
View File

@@ -0,0 +1,84 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package doltdb
import (
"context"
"fmt"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/ref"
"github.com/liquidata-inc/dolt/go/store/datas"
"github.com/liquidata-inc/dolt/go/store/types"
)
type Tag struct {
Name string
vrw types.ValueReadWriter
tagSt types.Struct
Meta *TagMeta
Commit *Commit
}
// NewTag creates a new Tag object.
func NewTag(ctx context.Context, name string, vrw types.ValueReadWriter, tagSt types.Struct) (*Tag, error) {
metaSt, ok, err := tagSt.MaybeGet(datas.TagMetaField)
if err != nil {
return nil, err
}
if !ok {
return nil, fmt.Errorf("tag struct does not have field %s", datas.TagMetaField)
}
meta, err := tagMetaFromNomsSt(metaSt.(types.Struct))
if err != nil {
return nil, err
}
commitRef, ok, err := tagSt.MaybeGet(datas.TagCommitRefField)
if err != nil {
return nil, err
}
if !ok {
return nil, fmt.Errorf("tag struct does not have field %s", datas.TagCommitRefField)
}
commitSt, err := commitRef.(types.Ref).TargetValue(ctx, vrw)
if err != nil {
return nil, err
}
commit := NewCommit(vrw, commitSt.(types.Struct))
return &Tag{
Name: name,
vrw: vrw,
tagSt: tagSt,
Meta: meta,
Commit: commit,
}, nil
}
// GetStRef returns a Noms Ref for this Tag's Noms tag Struct.
func (t *Tag) GetStRef() (types.Ref, error) {
return types.NewRef(t.tagSt, t.vrw.Format())
}
// GetDoltRef returns a DoltRef for this Tag.
func (t *Tag) GetDoltRef() ref.DoltRef {
return ref.NewTagRef(t.Name)
}

140
go/libraries/doltcore/doltdb/tag_meta.go Normal file
View File

@@ -0,0 +1,140 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package doltdb
import (
"fmt"
"strings"
"time"
"github.com/liquidata-inc/dolt/go/store/types"
)
const (
tagMetaNameKey = "name"
tagMetaEmailKey = "email"
tagMetaDescKey = "desc"
tagMetaTimestampKey = "timestamp"
tagMetaUserTSKey = "user_timestamp"
tagMetaVersionKey = "metaversion"
tagMetaStName = "metadata"
tagMetaVersion = "1.0"
)
var TagNowFunc = CommitNowFunc
var TagLoc = CommitLoc
// TagMeta contains all the metadata that is associated with a tag within a data repo.
type TagMeta struct {
Name string
Email string
Timestamp uint64
Description string
UserTimestamp int64
}
// NewTagMetaWithUserTS returns TagMeta that can be used to create a tag.
// It uses the current time as the user timestamp.
func NewTagMeta(name, email, desc string) *TagMeta {
return NewTagMetaWithUserTS(name, email, desc, TagNowFunc())
}
// NewTagMetaWithUserTS returns TagMeta that can be used to create a tag.
func NewTagMetaWithUserTS(name, email, desc string, userTS time.Time) *TagMeta {
n := strings.TrimSpace(name)
e := strings.TrimSpace(email)
d := strings.TrimSpace(desc)
ns := uint64(TagNowFunc().UnixNano())
ms := ns / uMilliToNano
userMS := userTS.UnixNano() / milliToNano
return &TagMeta{n, e, ms, d, userMS}
}
func tagMetaFromNomsSt(st types.Struct) (*TagMeta, error) {
e, err := getRequiredFromSt(st, tagMetaEmailKey)
if err != nil {
return nil, err
}
n, err := getRequiredFromSt(st, tagMetaNameKey)
if err != nil {
return nil, err
}
d, err := getRequiredFromSt(st, tagMetaDescKey)
if err != nil {
return nil, err
}
ts, err := getRequiredFromSt(st, tagMetaTimestampKey)
if err != nil {
return nil, err
}
userTS, ok, err := st.MaybeGet(tagMetaUserTSKey)
if err != nil {
return nil, err
} else if !ok {
userTS = types.Int(int64(uint64(ts.(types.Uint))))
}
return &TagMeta{
string(n.(types.String)),
string(e.(types.String)),
uint64(ts.(types.Uint)),
string(d.(types.String)),
int64(userTS.(types.Int)),
}, nil
}
func (tm *TagMeta) toNomsStruct(nbf *types.NomsBinFormat) (types.Struct, error) {
metadata := types.StructData{
tagMetaNameKey: types.String(tm.Name),
tagMetaEmailKey: types.String(tm.Email),
tagMetaDescKey: types.String(tm.Description),
tagMetaTimestampKey: types.Uint(tm.Timestamp),
tagMetaVersionKey: types.String(tagMetaVersion),
commitMetaUserTSKey: types.Int(tm.UserTimestamp),
}
return types.NewStruct(nbf, tagMetaStName, metadata)
}
// Time returns the time at which the tag occurred
func (tm *TagMeta) Time() time.Time {
seconds := int64(tm.Timestamp) / secToMilli
nanos := (int64(tm.Timestamp) % secToMilli) * milliToNano
return time.Unix(seconds, nanos)
}
// FormatTS takes the internal timestamp and turns it into a human readable string in the time.RubyDate format
// which looks like: "Mon Jan 02 15:04:05 -0700 2006"
func (tm *TagMeta) FormatTS() string {
return tm.Time().In(TagLoc).Format(time.RubyDate)
}
// String returns the human readable string representation of the tag data
func (tm *TagMeta) String() string {
return fmt.Sprintf("name: %s, email: %s, timestamp: %s, description: %s", tm.Name, tm.Email, tm.FormatTS(), tm.Description)
}

39
go/libraries/doltcore/doltdb/tag_meta_test.go Normal file
View File

@@ -0,0 +1,39 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package doltdb
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"github.com/liquidata-inc/dolt/go/store/types"
)
func TestTagMetaToAndFromNomsStruct(t *testing.T) {
tm := NewTagMeta("Bill Billerson", "bigbillieb@fake.horse", "This is a test commit")
cmSt, err := tm.toNomsStruct(types.Format_7_18)
assert.NoError(t, err)
result, err := tagMetaFromNomsSt(cmSt)
if err != nil {
t.Fatal("Failed to convert from types.Struct to CommitMeta")
} else if !reflect.DeepEqual(tm, result) {
t.Error("CommitMeta was not converted without error.")
}
t.Log(tm.String())
}

6
go/libraries/doltcore/env/actions/commit.go vendored
View File

@@ -64,7 +64,7 @@ func CommitStaged(ctx context.Context, dEnv *env.DoltEnv, props CommitStagedProp
return ErrEmptyCommitMessage
}
staged, _, err := diff.GetStagedUnstagedTableDeltas(ctx, dEnv)
staged, notStaged, err := diff.GetStagedUnstagedTableDeltas(ctx, dEnv)
if err != nil {
return err
}
@@ -79,13 +79,11 @@ func CommitStaged(ctx context.Context, dEnv *env.DoltEnv, props CommitStagedProp
}
if len(staged) == 0 && !dEnv.IsMergeActive() && !props.AllowEmpty {
// todo: use TableDelta for status/errors
_, notStagedTbls, err := diff.GetTableDiffs(ctx, dEnv)
_, notStagedDocs, err := diff.GetDocDiffs(ctx, dEnv)
if err != nil {
return err
}
return NothingStaged{notStagedTbls, notStagedDocs}
return NothingStaged{notStaged, notStagedDocs}
}
name, email, err := GetNameAndEmail(dEnv.Config)

4
go/libraries/doltcore/env/actions/errors.go vendored
View File

@@ -193,7 +193,7 @@ func CheckoutWouldOverwriteTables(err error) []string {
}
type NothingStaged struct {
NotStagedTbls *diff.TableDiffs
NotStagedTbls []diff.TableDelta
NotStagedDocs *diff.DocDiffs
}
@@ -206,7 +206,7 @@ func IsNothingStaged(err error) bool {
return ok
}
func NothingStagedTblDiffs(err error) *diff.TableDiffs {
func NothingStagedTblDiffs(err error) []diff.TableDelta {
ns, ok := err.(NothingStaged)
if !ok {

54
go/libraries/doltcore/env/actions/remotes.go vendored
View File

@@ -43,7 +43,13 @@ func Push(ctx context.Context, dEnv *env.DoltEnv, mode ref.RefUpdateMode, destRe
}
}
err = destDB.PushChunks(ctx, dEnv.TempTableFilesDir(), srcDB, commit, progChan, pullerEventCh)
rf, err := commit.GetStRef()
if err != nil {
return err
}
err = destDB.PushChunks(ctx, dEnv.TempTableFilesDir(), srcDB, rf, progChan, pullerEventCh)
if err != nil {
return err
@@ -51,11 +57,11 @@ func Push(ctx context.Context, dEnv *env.DoltEnv, mode ref.RefUpdateMode, destRe
switch mode {
case ref.ForceUpdate:
err = destDB.SetHead(ctx, destRef, commit)
err = destDB.SetHeadToCommit(ctx, destRef, commit)
if err != nil {
return err
}
err = srcDB.SetHead(ctx, remoteRef, commit)
err = srcDB.SetHeadToCommit(ctx, remoteRef, commit)
case ref.FastForwardOnly:
err = destDB.FastForward(ctx, destRef, commit)
if err != nil {
@@ -67,6 +73,25 @@ func Push(ctx context.Context, dEnv *env.DoltEnv, mode ref.RefUpdateMode, destRe
return err
}
// PushTag pushes a commit tag and all underlying data from a local source database to a remote destination database.
func PushTag(ctx context.Context, dEnv *env.DoltEnv, destRef ref.TagRef, srcDB, destDB *doltdb.DoltDB, tag *doltdb.Tag, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
var err error
rf, err := tag.GetStRef()
if err != nil {
return err
}
err = destDB.PushChunks(ctx, dEnv.TempTableFilesDir(), srcDB, rf, progChan, pullerEventCh)
if err != nil {
return err
}
return destDB.SetHead(ctx, destRef, rf)
}
// DeleteRemoteBranch validates targetRef is a branch on the remote database, and then deletes it, then deletes the
// remote tracking branch from the local database.
func DeleteRemoteBranch(ctx context.Context, targetRef ref.BranchRef, remoteRef ref.RemoteRef, localDB, remoteDB *doltdb.DoltDB) error {
@@ -93,10 +118,29 @@ func DeleteRemoteBranch(ctx context.Context, targetRef ref.BranchRef, remoteRef
return nil
}
func Fetch(ctx context.Context, dEnv *env.DoltEnv, destRef ref.DoltRef, srcDB, destDB *doltdb.DoltDB, srcDBCommit *doltdb.Commit, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
return destDB.PullChunks(ctx, dEnv.TempTableFilesDir(), srcDB, srcDBCommit, progChan, pullerEventCh)
// FetchCommit takes a fetches a commit and all underlying data from a remote source database to the local destination database.
func FetchCommit(ctx context.Context, dEnv *env.DoltEnv, srcDB, destDB *doltdb.DoltDB, srcDBCommit *doltdb.Commit, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
stRef, err := srcDBCommit.GetStRef()
if err != nil {
return err
}
return destDB.PullChunks(ctx, dEnv.TempTableFilesDir(), srcDB, stRef, progChan, pullerEventCh)
}
// FetchCommit takes a fetches a commit tag and all underlying data from a remote source database to the local destination database.
func FetchTag(ctx context.Context, dEnv *env.DoltEnv, srcDB, destDB *doltdb.DoltDB, srcDBTag *doltdb.Tag, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
stRef, err := srcDBTag.GetStRef()
if err != nil {
return err
}
return destDB.PullChunks(ctx, dEnv.TempTableFilesDir(), srcDB, stRef, progChan, pullerEventCh)
}
// Clone pulls all data from a remote source database to a local destination database.
func Clone(ctx context.Context, srcDB, destDB *doltdb.DoltDB, eventCh chan<- datas.TableFileEvent) error {
return srcDB.Clone(ctx, destDB, eventCh)
}

60
go/libraries/doltcore/env/actions/tag.go vendored
View File

@@ -16,6 +16,7 @@ package actions
import (
"context"
"fmt"
"sort"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/doltdb"
@@ -58,28 +59,9 @@ func CreateTag(ctx context.Context, dEnv *env.DoltEnv, tagName, startPoint strin
return err
}
err = dEnv.DoltDB.NewTagAtCommit(ctx, tagRef, cm)
if err != nil {
return err
}
root, err := cm.GetRootValue()
if err != nil {
return err
}
h, err := root.HashOf()
if err != nil {
return err
}
meta := doltdb.NewTagMeta(props.TaggerName, props.TaggerEmail, props.Description)
_, err = dEnv.DoltDB.CommitWithParentCommits(ctx, h, tagRef, nil, meta)
return err
return dEnv.DoltDB.NewTagAtCommit(ctx, tagRef, cm, meta)
}
func DeleteTags(ctx context.Context, dEnv *env.DoltEnv, tagNames ...string) error {
@@ -104,48 +86,36 @@ func DeleteTags(ctx context.Context, dEnv *env.DoltEnv, tagNames ...string) erro
return nil
}
type resolvedTag struct {
tag ref.DoltRef
commit *doltdb.Commit
meta *doltdb.CommitMeta
}
// IterResolvedTags iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb().
func IterResolvedTags(ctx context.Context, dEnv *env.DoltEnv, cb func(tag ref.DoltRef, c *doltdb.Commit, meta *doltdb.CommitMeta) (stop bool, err error)) error {
tagRefs, err := dEnv.DoltDB.GetTags(ctx)
func IterResolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.Tag) (stop bool, err error)) error {
tagRefs, err := ddb.GetTags(ctx)
if err != nil {
return err
}
var resolved []resolvedTag
for _, tag := range tagRefs {
commit, err := dEnv.DoltDB.ResolveRef(ctx, tag)
var resolved []*doltdb.Tag
for _, r := range tagRefs {
tr, ok := r.(ref.TagRef)
if !ok {
return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef")
}
tag, err := ddb.ResolveTag(ctx, tr)
if err != nil {
return err
}
meta, err := commit.GetCommitMeta()
if err != nil {
return err
}
resolved = append(resolved, resolvedTag{
tag: tag,
commit: commit,
meta: meta,
})
resolved = append(resolved, tag)
}
// iterate newest to oldest
sort.Slice(resolved, func(i, j int) bool {
return resolved[i].meta.Timestamp > resolved[j].meta.Timestamp
return resolved[i].Meta.Timestamp > resolved[j].Meta.Timestamp
})
for _, st := range resolved {
stop, err := cb(st.tag, st.commit, st.meta)
for _, tag := range resolved {
stop, err := cb(tag)
if err != nil {
return err

1
go/libraries/doltcore/ref/ref.go
View File

@@ -120,6 +120,7 @@ func MarshalJSON(dr DoltRef) ([]byte, error) {
}
// Parse will parse ref strings and return a DoltRef or an error for refs that can't be parsed.
// refs without a RefType prefix ("refs/heads/", "refs/tags/", etc) are assumed to be branches)
func Parse(str string) (DoltRef, error) {
if !IsRef(str) {
if strings.HasPrefix(str, remotesPrefix) {

30
go/libraries/doltcore/ref/ref_spec.go
View File

@@ -101,6 +101,8 @@ func ParseRefSpecForRemote(remote, refSpecStr string) (RefSpec, error) {
return newLocalToRemoteTrackingRef(remote, fromRef.(BranchRef), toRef.(RemoteRef))
} else if fromRef.GetType() == BranchRefType && toRef.GetType() == BranchRefType {
return NewBranchToBranchRefSpec(fromRef.(BranchRef), toRef.(BranchRef))
} else if fromRef.GetType() == TagRefType && toRef.GetType() == TagRefType {
return NewTagToTagRefSpec(fromRef.(TagRef), toRef.(TagRef))
}
return nil, ErrUnsupportedMapping
@@ -164,6 +166,34 @@ func (rs BranchToBranchRefSpec) DestRef(r DoltRef) DoltRef {
return nil
}
type TagToTagRefSpec struct {
srcRef DoltRef
destRef DoltRef
}
// NewTagToTagRefSpec takes a source and destination TagRef and returns a RefSpec that maps source to dest.
func NewTagToTagRefSpec(srcRef, destRef TagRef) (RefSpec, error) {
return TagToTagRefSpec{
srcRef: srcRef,
destRef: destRef,
}, nil
}
// SrcRef will always determine the DoltRef specified as the source ref regardless to the cwbRef
func (rs TagToTagRefSpec) SrcRef(_ DoltRef) DoltRef {
return rs.srcRef
}
// DestRef verifies the localRef matches the refspecs local pattern, and then maps it to a remote tracking branch, or
// nil if it does not match the local pattern.
func (rs TagToTagRefSpec) DestRef(r DoltRef) DoltRef {
if Equals(r, rs.srcRef) {
return rs.destRef
}
return nil
}
// BranchToTrackingBranchRefSpec maps a branch to the branch that should be tracking it
type BranchToTrackingBranchRefSpec struct {
localPattern pattern

2
go/libraries/doltcore/schema/typeinfo/typeinfo_test.go
View File

@@ -361,7 +361,7 @@ func generateTypeInfoArrays(t *testing.T) ([][]TypeInfo, [][]types.Value) {
types.Decimal(decimal.RequireFromString("-1076416.875")),
types.Decimal(decimal.RequireFromString("198728394234798423466321.27349757"))},
{types.Uint(1), types.Uint(3), types.Uint(5), types.Uint(7), types.Uint(8)}, //Enum
{types.Float(1.0), types.Float(65513.75), types.Float(4293902592), types.Float(4.58E71), types.Float(7.172E285)}, //Float
{types.Float(1.0), types.Float(65513.75), types.Float(4293902592), types.Float(4.58e71), types.Float(7.172e285)}, //Float
{types.InlineBlob{0}, types.InlineBlob{21}, types.InlineBlob{1, 17}, types.InlineBlob{72, 42}, types.InlineBlob{21, 122, 236}}, //InlineBlob
{types.Int(20), types.Int(215), types.Int(237493), types.Int(2035753568), types.Int(2384384576063)}, //Int
{types.Uint(1), types.Uint(5), types.Uint(64), types.Uint(42), types.Uint(192)}, //Set

2
go/libraries/doltcore/sqle/branches_table.go
View File

@@ -72,7 +72,7 @@ func (bt *BranchesTable) Schema() sql.Schema {
// Partitions is a sql.Table interface function that returns a partition of the data. Currently the data is unpartitioned.
func (bt *BranchesTable) Partitions(*sql.Context) (sql.PartitionIter, error) {
return &doltTablePartitionIter{}, nil
return newSinglePartitionIter(), nil
}
// PartitionRows is a sql.Table interface function that gets a row iterator for a partition

2
go/libraries/doltcore/sqle/conflicts_tables.go
View File

@@ -87,7 +87,7 @@ func (ct ConflictsTable) Schema() sql.Schema {
// Partitions returns a PartitionIter which can be used to get all the data partitions
func (ct ConflictsTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
return &doltTablePartitionIter{}, nil
return newSinglePartitionIter(), nil
}
// PartitionRows returns a RowIter for the given partition

4
go/libraries/doltcore/sqle/database.go
View File

@@ -595,7 +595,7 @@ func (db Database) createTable(ctx *sql.Context, tableName string, sch sql.Schem
return err
}
if exists, err := root.TableNameInUse(ctx, tableName); err != nil {
if exists, err := root.HasTable(ctx, tableName); err != nil {
return err
} else if exists {
return sql.ErrTableAlreadyExists.New(tableName)
@@ -773,7 +773,7 @@ func RegisterSchemaFragments(ctx *sql.Context, db Database, root *doltdb.RootVal
if err != nil {
return err
}
iter, err := newRowIterator(&tbl.DoltTable, ctx)
iter, err := newRowIterator(&tbl.DoltTable, ctx, nil)
if err != nil {
return err
}

6
go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go
View File

@@ -17,9 +17,15 @@ package enginetest
import (
"testing"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/sqle"
"github.com/liquidata-inc/go-mysql-server/enginetest"
)
func init() {
sqle.MinRowsPerPartition = 2
}
func TestQueries(t *testing.T) {
enginetest.TestQueries(t, newDoltHarness(t))
}

10
go/libraries/doltcore/sqle/enginetest/dolt_harness.go
View File

@@ -16,6 +16,7 @@ package enginetest
import (
"context"
"runtime"
"strings"
"testing"
@@ -62,7 +63,14 @@ func (d *DoltHarness) SkipQueryTest(query string) bool {
}
func (d *DoltHarness) Parallelism() int {
return 1
// always test with some parallelism
parallelism := runtime.NumCPU()
if parallelism <= 1 {
parallelism = 2
}
return parallelism
}
func (d *DoltHarness) NewContext() *sql.Context {

35
go/libraries/doltcore/sqle/filtered_reader.go
View File

@@ -17,6 +17,7 @@ package sqle
import (
"context"
"strings"
"sync"
"github.com/liquidata-inc/go-mysql-server/sql"
"github.com/liquidata-inc/go-mysql-server/sql/expression"
@@ -26,6 +27,7 @@ import (
"github.com/liquidata-inc/dolt/go/libraries/doltcore/sqle/setalgebra"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/table"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/table/typed/noms"
"github.com/liquidata-inc/dolt/go/store/hash"
"github.com/liquidata-inc/dolt/go/store/types"
)
@@ -208,6 +210,39 @@ func setForInExp(nbf *types.NomsBinFormat, col schema.Column, be expression.Bina
// readers created by calling this function will have the rows they return limited by a setalgebra.Set implementation.
type CreateReaderFunc func(ctx context.Context, m types.Map) (table.TableReadCloser, error)
// ThreadSafeCRFuncCache is a thread safe CreateReaderFunc cache
type ThreadSafeCRFuncCache struct {
funcs map[hash.Hash]CreateReaderFunc
mu *sync.Mutex
}
// NewThreadSafeCRFuncCache creates a new ThreadSafeCRFuncCache
func NewThreadSafeCRFuncCache() *ThreadSafeCRFuncCache {
return &ThreadSafeCRFuncCache{make(map[hash.Hash]CreateReaderFunc), &sync.Mutex{}}
}
// GetOrCreate gets a CreateReaderFunc for the given hash if it exists in the cache. If it doesn't it creates it and
// caches it.
func (cache *ThreadSafeCRFuncCache) GetOrCreate(h hash.Hash, nbf *types.NomsBinFormat, tblSch schema.Schema, filters []sql.Expression) (CreateReaderFunc, error) {
cache.mu.Lock()
defer cache.mu.Unlock()
f, ok := cache.funcs[h]
if ok {
return f, nil
}
f, err := CreateReaderFuncLimitedByExpressions(nbf, tblSch, filters)
if err != nil {
return nil, err
}
cache.funcs[h] = f
return f, nil
}
// CreateReaderFuncLimitedByExpressions takes a table schema and a slice of sql filters and returns a CreateReaderFunc
// which limits the rows read based on the filters supplied.
func CreateReaderFuncLimitedByExpressions(nbf *types.NomsBinFormat, tblSch schema.Schema, filters []sql.Expression) (CreateReaderFunc, error) {

17
go/libraries/doltcore/sqle/history_table.go
View File

@@ -55,7 +55,7 @@ type HistoryTable struct {
commitFilters []sql.Expression
rowFilters []sql.Expression
cmItr doltdb.CommitItr
readerCreateFuncCache map[hash.Hash]CreateReaderFunc
readerCreateFuncCache *ThreadSafeCRFuncCache
}
// NewHistoryTable creates a history table
@@ -115,7 +115,7 @@ func NewHistoryTable(ctx *sql.Context, db Database, tblName string) (sql.Table,
ss: ss,
sqlSch: sqlSch,
cmItr: cmItr,
readerCreateFuncCache: make(map[hash.Hash]CreateReaderFunc),
readerCreateFuncCache: NewThreadSafeCRFuncCache(),
}, nil
}
@@ -325,7 +325,7 @@ func newRowItrForTableAtCommit(
tblName string,
ss *schema.SuperSchema,
filters []sql.Expression,
readerCreateFuncCache map[hash.Hash]CreateReaderFunc) (*rowItrForTableAtCommit, error) {
readerCreateFuncCache *ThreadSafeCRFuncCache) (*rowItrForTableAtCommit, error) {
root, err := cm.GetRootValue()
if err != nil {
@@ -367,15 +367,10 @@ func newRowItrForTableAtCommit(
return nil, err
}
var createReaderFunc CreateReaderFunc
if createReaderFunc, ok = readerCreateFuncCache[schHash]; !ok {
createReaderFunc, err = CreateReaderFuncLimitedByExpressions(tbl.Format(), tblSch, filters)
createReaderFunc, err := readerCreateFuncCache.GetOrCreate(schHash, tbl.Format(), tblSch, filters)
if err != nil {
return nil, err
}
readerCreateFuncCache[schHash] = createReaderFunc
if err != nil {
return nil, err
}
rd, err := createReaderFunc(ctx, m)

2
go/libraries/doltcore/sqle/indexed_dolt_table.go
View File

@@ -48,7 +48,7 @@ func (idt *IndexedDoltTable) Schema() sql.Schema {
}
func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
return idt.table.Partitions(ctx)
return newSinglePartitionIter(), nil
}
func (idt *IndexedDoltTable) PartitionRows(ctx *sql.Context, _ sql.Partition) (sql.RowIter, error) {

4
go/libraries/doltcore/sqle/log_table.go
View File

@@ -67,11 +67,11 @@ func (dt *LogTable) Schema() sql.Schema {
// Partitions is a sql.Table interface function that returns a partition of the data. Currently the data is unpartitioned.
func (dt *LogTable) Partitions(*sql.Context) (sql.PartitionIter, error) {
return &doltTablePartitionIter{}, nil
return newSinglePartitionIter(), nil
}
// PartitionRows is a sql.Table interface function that gets a row iterator for a partition
func (dt *LogTable) PartitionRows(sqlCtx *sql.Context, part sql.Partition) (sql.RowIter, error) {
func (dt *LogTable) PartitionRows(sqlCtx *sql.Context, _ sql.Partition) (sql.RowIter, error) {
return NewLogItr(sqlCtx, dt.dbName, dt.ddb)
}

38
go/libraries/doltcore/sqle/rows.go
View File

@@ -32,23 +32,43 @@ type doltTableRowIter struct {
rowData types.Map
ctx *sql.Context
nomsIter types.MapIterator
end types.Value
nbf *types.NomsBinFormat
}
// Returns a new row iterator for the table given
func newRowIterator(tbl *DoltTable, ctx *sql.Context) (*doltTableRowIter, error) {
func newRowIterator(tbl *DoltTable, ctx *sql.Context, partition *doltTablePartition) (*doltTableRowIter, error) {
rowData, err := tbl.table.GetRowData(ctx)
if err != nil {
return nil, err
}
mapIter, err := rowData.BufferedIterator(ctx)
var mapIter types.MapIterator
var end types.Value = nil
if partition == nil {
mapIter, err = rowData.BufferedIterator(ctx)
} else {
endIter, err := rowData.IteratorAt(ctx, partition.end)
if err != nil && err != io.EOF {
return nil, err
} else if err != io.EOF {
end, _, err = endIter.Next(ctx)
if err != nil && err != io.EOF {
return nil, err
}
}
mapIter, err = rowData.BufferedIteratorAt(ctx, partition.start)
}
if err != nil {
return nil, err
}
return &doltTableRowIter{table: tbl, rowData: rowData, ctx: ctx, nomsIter: mapIter}, nil
return &doltTableRowIter{table: tbl, rowData: rowData, ctx: ctx, nomsIter: mapIter, end: end, nbf: rowData.Format()}, nil
}
// Next returns the next row in this row iterator, or an io.EOF error if there aren't any more.
@@ -63,6 +83,18 @@ func (itr *doltTableRowIter) Next() (sql.Row, error) {
return nil, io.EOF
}
if itr.end != nil {
isLess, err := key.Less(itr.nbf, itr.end)
if err != nil {
return nil, err
}
if !isLess {
return nil, io.EOF
}
}
doltRow, err := row.FromNoms(itr.table.sch, key.(types.Tuple), val.(types.Tuple))
if err != nil {

146
go/libraries/doltcore/sqle/tables.go
View File

@@ -18,7 +18,11 @@ import (
"errors"
"fmt"
"io"
"os"
"runtime"
"strconv"
"strings"
"sync"
"github.com/liquidata-inc/go-mysql-server/sql"
"github.com/liquidata-inc/vitess/go/sqltypes"
@@ -32,6 +36,27 @@ import (
"github.com/liquidata-inc/dolt/go/store/types"
)
const (
partitionMultiplier = 2.0
)
var MinRowsPerPartition uint64 = 1024
func init() {
isTest := false
for _, arg := range os.Args {
lwr := strings.ToLower(arg)
if lwr == "-test.v" || lwr == "-test.run" || strings.HasPrefix(lwr, "-test.testlogfile") {
isTest = true
break
}
}
if isTest {
MinRowsPerPartition = 2
}
}
// DoltTable implements the sql.Table interface and gives access to dolt table rows and schema.
type DoltTable struct {
name string
@@ -143,15 +168,47 @@ func (t *DoltTable) sqlSchema() sql.Schema {
return sqlSch
}
// Returns the partitions for this table. We return a single partition, but could potentially get more performance by
// returning multiple.
func (t *DoltTable) Partitions(*sql.Context) (sql.PartitionIter, error) {
return &doltTablePartitionIter{}, nil
// Returns the partitions for this table.
func (t *DoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
rowData, err := t.table.GetRowData(ctx)
if err != nil {
return nil, err
}
numElements := rowData.Len()
if numElements == 0 {
return newSinglePartitionIter(), nil
}
maxPartitions := uint64(partitionMultiplier * runtime.NumCPU())
numPartitions := (numElements / MinRowsPerPartition) + 1
if numPartitions > maxPartitions {
numPartitions = maxPartitions
}
partitions := make([]doltTablePartition, numPartitions)
itemsPerPartition := numElements / numPartitions
for i := uint64(0); i < numPartitions-1; i++ {
partitions[i] = doltTablePartition{i * itemsPerPartition, (i + 1) * itemsPerPartition}
}
partitions[numPartitions-1] = doltTablePartition{(numPartitions - 1) * itemsPerPartition, numElements}
return newDoltTablePartitionIter(rowData, partitions), nil
}
// Returns the table rows for the partition given (all rows of the table).
func (t *DoltTable) PartitionRows(ctx *sql.Context, _ sql.Partition) (sql.RowIter, error) {
return newRowIterator(t, ctx)
// Returns the table rows for the partition given
func (t *DoltTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) {
switch typedPartition := partition.(type) {
case doltTablePartition:
return newRowIterator(t, ctx, &typedPartition)
case singlePartition:
return newRowIterator(t, ctx, nil)
}
return nil, errors.New("unsupported partition type")
}
// WritableDoltTable allows updating, deleting, and inserting new rows. It implements sql.UpdatableTable and friends.
@@ -222,10 +279,57 @@ func (t *WritableDoltTable) Updater(ctx *sql.Context) sql.RowUpdater {
return te
}
var _ sql.PartitionIter = singlePartitionIter{}
type singlePartitionIter struct {
once *sync.Once
}
func newSinglePartitionIter() singlePartitionIter {
return singlePartitionIter{&sync.Once{}}
}
// Close is required by the sql.PartitionIter interface. Does nothing.
func (itr singlePartitionIter) Close() error {
return nil
}
// Next returns the next partition if there is one, or io.EOF if there isn't.
func (itr singlePartitionIter) Next() (sql.Partition, error) {
first := false
itr.once.Do(func() {
first = true
})
if !first {
return nil, io.EOF
}
return singlePartition{}, nil
}
var _ sql.Partition = singlePartition{}
type singlePartition struct{}
// Key returns the key for this partition, which must uniquely identity the partition. We have only a single partition
// per table, so we use a constant.
func (sp singlePartition) Key() []byte {
return []byte("single")
}
var _ sql.PartitionIter = (*doltTablePartitionIter)(nil)
// doltTablePartitionIter, an object that knows how to return the single partition exactly once.
type doltTablePartitionIter struct {
sql.PartitionIter
i int
i int
mu *sync.Mutex
rowData types.Map
partitions []doltTablePartition
}
func newDoltTablePartitionIter(rowData types.Map, partitions []doltTablePartition) *doltTablePartitionIter {
return &doltTablePartitionIter{0, &sync.Mutex{}, rowData, partitions}
}
// Close is required by the sql.PartitionIter interface. Does nothing.
@@ -235,25 +339,31 @@ func (itr *doltTablePartitionIter) Close() error {
// Next returns the next partition if there is one, or io.EOF if there isn't.
func (itr *doltTablePartitionIter) Next() (sql.Partition, error) {
if itr.i > 0 {
itr.mu.Lock()
defer itr.mu.Unlock()
if itr.i >= len(itr.partitions) {
return nil, io.EOF
}
partition := itr.partitions[itr.i]
itr.i++
return &doltTablePartition{}, nil
return partition, nil
}
// A table partition, currently an unused layer of abstraction but required for the framework.
var _ sql.Partition = (*doltTablePartition)(nil)
type doltTablePartition struct {
sql.Partition
// start is the first index of this partition (inclusive)
start uint64
// all elements in the partition will be less than end (exclusive)
end uint64
}
const partitionName = "single"
// Key returns the key for this partition, which must uniquely identity the partition. We have only a single partition
// per table, so we use a constant.
// Key returns the key for this partition, which must uniquely identity the partition.
func (p doltTablePartition) Key() []byte {
return []byte(partitionName)
return []byte(strconv.FormatUint(p.start, 10) + " >= i < " + strconv.FormatUint(p.end, 10))
}
// AlterableDoltTable allows altering the schema of the table. It implements sql.AlterableTable.

26
go/libraries/doltcore/sqle/tables_test.go Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqle
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMinRowsPerPartitionInTests(t *testing.T) {
// If this fails then the method for determining if we are running in a test doesn't work all the time.
assert.Equal(t, uint64(2), MinRowsPerPartition)
}

76
go/libraries/doltcore/table/read_ahead_table_reader.go Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package table
import (
"context"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/row"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/schema"
"github.com/liquidata-inc/dolt/go/libraries/utils/async"
)
var _ TableReadCloser = (*AsyncReadAheadTableReader)(nil)
// AsyncReadAheadTableReader is a TableReadCloser implementation that spins up a go routine to keep reading data into
// a buffered channel so that it is ready when the caller wants it.
type AsyncReadAheadTableReader struct {
backingReader TableReadCloser
reader *async.AsyncReader
}
// NewAsyncReadAheadTableReader creates a new AsyncReadAheadTableReader
func NewAsyncReadAheadTableReader(tr TableReadCloser, bufferSize int) *AsyncReadAheadTableReader {
read := func(ctx context.Context) (interface{}, error) {
return tr.ReadRow(ctx)
}
reader := async.NewAsyncReader(read, bufferSize)
return &AsyncReadAheadTableReader{tr, reader}
}
// Start the worker routine reading rows to the channel
func (tr *AsyncReadAheadTableReader) Start(ctx context.Context) error {
return tr.reader.Start(ctx)
}
// GetSchema gets the schema of the rows that this reader will return
func (tr *AsyncReadAheadTableReader) GetSchema() schema.Schema {
return tr.backingReader.GetSchema()
}
// ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and calling
// IsBadRow(err) will be return true. This is a potentially non-fatal error and callers can decide if they want to
// continue on a bad row, or fail.
func (tr *AsyncReadAheadTableReader) ReadRow(ctx context.Context) (row.Row, error) {
obj, err := tr.reader.Read()
if err != nil {
return nil, err
}
return obj.(row.Row), err
}
// VerifySchema checks that the incoming schema matches the schema from the existing table
func (tr *AsyncReadAheadTableReader) VerifySchema(outSch schema.Schema) (bool, error) {
return tr.backingReader.VerifySchema(outSch)
}
// Close releases resources being held
func (tr *AsyncReadAheadTableReader) Close(ctx context.Context) error {
_ = tr.reader.Close()
return tr.backingReader.Close(ctx)
}

2
go/libraries/doltcore/table/untyped/csv/reader.go
View File

@@ -206,7 +206,7 @@ func validDelim(s string) bool {
return !(strings.Contains(s, "\"") ||
strings.Contains(s, "\r") ||
strings.Contains(s, "\n") ||
strings.Contains(s, string(0xFFFD))) // Unicode replacement char
strings.Contains(s, string([]byte{0xFF, 0xFD}))) // Unicode replacement char
}
func lengthNL(b []byte) int {

2
go/libraries/doltcore/table/untyped/tabular/tablewriter.go
View File

@@ -85,7 +85,7 @@ func (ttw *TextTableWriter) writeTableHeader(r row.Row) error {
colnames.WriteString(" ")
colNameVal, ok := r.GetColVal(tag)
if !ok {
return false, errors.New("No column name value for tag " + string(tag))
return false, fmt.Errorf("No column name value for tag %d", tag)
}
colName := string(colNameVal.(types.String))

86
go/libraries/utils/async/async_read_test.go Normal file
View File

@@ -0,0 +1,86 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package async
import (
"context"
"fmt"
"io"
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReader(t *testing.T) {
testReadNItems(t, 0, 32)
testReadNItems(t, 1, 32)
testReadNItems(t, 65536, 32)
const maxItems = 4 * 1024
const minItems = 4
const maxTestBufferSize = 128
for i := 0; i < 32; i++ {
testReadNItems(t, rand.Int63n(maxItems-minItems)+minItems, rand.Int63n(maxTestBufferSize-1)+1)
}
}
func testReadNItems(t *testing.T, n int64, bufferSize int64) {
t.Run(fmt.Sprintf("%d_%d", n, bufferSize), func(t *testing.T) {
arr := make([]int64, n)
for i := int64(0); i < n; i++ {
arr[i] = i
}
readFunc := readFuncForArr(arr)
rd := NewAsyncReader(readFunc, 32)
err := rd.Start(context.Background())
require.NoError(t, err)
res := make([]int64, 0)
for {
val, err := rd.Read()
if err == io.EOF {
break
}
assert.NoError(t, err)
res = append(res, val.(int64))
}
err = rd.Close()
require.NoError(t, err)
assert.Equal(t, arr, res)
})
}
func readFuncForArr(arr []int64) ReadFunc {
pos := 0
return func(ctx context.Context) (interface{}, error) {
if pos >= len(arr) {
return nil, io.EOF
}
val := arr[pos]
pos++
return val, nil
}
}

93
go/libraries/utils/async/async_reader.go Normal file
View File

@@ -0,0 +1,93 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package async
import (
"context"
"io"
"sync"
)
// ReadFunc is a function that is called repeatedly in order to retrieve a stream of objects. When all objects have been
// been read from the stream then (nil, io.EOF) should be returned.
type ReadFunc func(ctx context.Context) (interface{}, error)
type objErrTuple struct {
obj interface{}
err error
}
// AsyncReader is a TableReadCloser implementation that spins up a go routine to keep reading data into
// a buffer so that it is ready when the caller wants it.
type AsyncReader struct {
readFunc ReadFunc
stopCh chan struct{}
rowCh chan objErrTuple
wg *sync.WaitGroup
}
// NewAsyncReader creates a new AsyncReader
func NewAsyncReader(rf ReadFunc, bufferSize int) *AsyncReader {
return &AsyncReader{rf, make(chan struct{}), make(chan objErrTuple, bufferSize), &sync.WaitGroup{}}
}
// Start the worker routine reading rows to the channel
func (asRd *AsyncReader) Start(ctx context.Context) error {
asRd.wg.Add(1)
go func() {
defer asRd.wg.Done()
defer close(asRd.rowCh)
asRd.readObjects(ctx)
}()
return nil
}
// ReadObject reads an object
func (asRd *AsyncReader) Read() (interface{}, error) {
objErrTup := <-asRd.rowCh
if objErrTup.obj == nil && objErrTup.err == nil {
return nil, io.EOF
}
return objErrTup.obj, objErrTup.err
}
// Close releases resources being held
func (asRd *AsyncReader) Close() error {
close(asRd.stopCh)
asRd.wg.Wait()
return nil
}
// background read loop running in separate go routine
func (asRd *AsyncReader) readObjects(ctx context.Context) {
for {
select {
case <-asRd.stopCh:
return
default:
}
obj, err := asRd.readFunc(ctx)
asRd.rowCh <- objErrTuple{obj, err}
if err != nil {
break
}
}
}

10
go/store/cmd/noms/noms_commit_test.go
View File

@@ -95,7 +95,7 @@ func (s *nomsCommitTestSuite) TestNomsCommitReadPathFromStdin() {
s.NoError(err)
s.True(h == ref.TargetHash(), "commit.value hash == writevalue hash")
meta, ok, err := commit.MaybeGet(datas.MetaField)
meta, ok, err := commit.MaybeGet(datas.CommitMetaField)
s.NoError(err)
s.True(ok)
d, ok, err := meta.(types.Struct).MaybeGet("date")
@@ -127,7 +127,7 @@ func (s *nomsCommitTestSuite) TestNomsCommitToDatasetWithoutHead() {
s.NoError(err)
s.True(h == ref.TargetHash(), "commit.value hash == writevalue hash")
metaVal, ok, err := commit.MaybeGet(datas.MetaField)
metaVal, ok, err := commit.MaybeGet(datas.CommitMetaField)
s.NoError(err)
s.True(ok)
meta := metaVal.(types.Struct)
@@ -195,7 +195,7 @@ func (s *nomsCommitTestSuite) TestNomsCommitMetadata() {
dsHead, ok := sp.GetDataset(context.Background()).MaybeHead()
s.True(ok)
metaOldVal, ok, err := dsHead.MaybeGet(datas.MetaField)
metaOldVal, ok, err := dsHead.MaybeGet(datas.CommitMetaField)
s.NoError(err)
s.True(ok)
metaOld := metaOldVal.(types.Struct)
@@ -210,7 +210,7 @@ func (s *nomsCommitTestSuite) TestNomsCommitMetadata() {
dsHead, ok = sp.GetDataset(context.Background()).MaybeHead()
s.True(ok)
metaNewVal, ok, err := dsHead.MaybeGet(datas.MetaField)
metaNewVal, ok, err := dsHead.MaybeGet(datas.CommitMetaField)
s.NoError(err)
s.True(ok)
metaNew := metaNewVal.(types.Struct)
@@ -234,7 +234,7 @@ func (s *nomsCommitTestSuite) TestNomsCommitMetadata() {
dsHead, ok = sp.GetDataset(context.Background()).MaybeHead()
s.True(ok)
metaNewVal, ok, err = dsHead.MaybeGet(datas.MetaField)
metaNewVal, ok, err = dsHead.MaybeGet(datas.CommitMetaField)
s.NoError(err)
s.True(ok)
metaNew = metaNewVal.(types.Struct)

4
go/store/cmd/noms/noms_log.go
View File

@@ -172,7 +172,7 @@ func runLog(ctx context.Context, args []string) int {
func printCommit(ctx context.Context, node LogNode, path types.Path, w io.Writer, db datas.Database, tz *time.Location) (err error) {
maxMetaFieldNameLength := func(commit types.Struct) int {
maxLen := 0
if m, ok, err := commit.MaybeGet(datas.MetaField); err != nil {
if m, ok, err := commit.MaybeGet(datas.CommitMetaField); err != nil {
panic(err)
} else if ok {
meta := m.(types.Struct)
@@ -293,7 +293,7 @@ func genGraph(node LogNode, lineno int) string {
}
func writeMetaLines(ctx context.Context, node LogNode, maxLines, lineno, maxLabelLen int, w io.Writer, tz *time.Location) (int, error) {
if m, ok, err := node.commit.MaybeGet(datas.MetaField); err != nil {
if m, ok, err := node.commit.MaybeGet(datas.CommitMetaField); err != nil {
panic(err)
} else if ok {
genPrefix := func(w *writers.PrefixWriter) []byte {

12
go/store/datas/commit.go
View File

@@ -40,16 +40,16 @@ const (
// created with newer versions of still usable by older versions.
ParentsListField = "parents_list"
ValueField = "value"
MetaField = "meta"
commitName = "Commit"
CommitMetaField = "meta"
CommitName = "Commit"
)
var commitTemplate = types.MakeStructTemplate(commitName, []string{MetaField, ParentsField, ParentsListField, ValueField})
var commitTemplate = types.MakeStructTemplate(CommitName, []string{CommitMetaField, ParentsField, ParentsListField, ValueField})
var valueCommitType = nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
parentsList?: List<Ref<Cycle<Commit>>>,
parents_list?: List<Ref<Cycle<Commit>>>,
value: Value,
}`)
@@ -172,9 +172,9 @@ func findCommonRef(a, b types.RefSlice) (types.Ref, bool) {
}
func makeCommitStructType(metaType, parentsType, parentsListType, valueType *types.Type) (*types.Type, error) {
return types.MakeStructType("Commit",
return types.MakeStructType(CommitName,
types.StructField{
Name: MetaField,
Name: CommitMetaField,
Type: metaType,
},
types.StructField{

9
go/store/datas/commit_test.go
View File

@@ -358,3 +358,12 @@ func TestNewCommitRegressionTest(t *testing.T) {
_, err = NewCommit(context.Background(), value, parents, meta)
assert.NoError(t, err)
}
func TestPersistedCommitConsts(t *testing.T) {
// changing constants that are persisted requires a migration strategy
assert.Equal(t, "parents", ParentsField)
assert.Equal(t, "parents_list", ParentsListField)
assert.Equal(t, "value", ValueField)
assert.Equal(t, "meta", CommitMetaField)
assert.Equal(t, "Commit", CommitName)
}

15
go/store/datas/database.go
View File

@@ -37,7 +37,7 @@ import (
// to read data by inspecting the Head of a Dataset and write new data by
// updating the Head of a Dataset via Commit() or similar. Particularly, new
// data is not guaranteed to be persistent until after a Commit (Delete,
// SetHead, or FastForward) operation completes.
// SetHeadToCommit, or FastForward) operation completes.
// The Database API is stateful, meaning that calls to GetDataset() or
// Datasets() occurring after a call to Commit() (et al) will represent the
// result of the Commit().
@@ -95,6 +95,15 @@ type Database interface {
// of a conflict, Commit returns an 'ErrMergeNeeded' error.
CommitValue(ctx context.Context, ds Dataset, v types.Value) (Dataset, error)
// Tag stores an immutable reference to a Value. It takes a Ref and a Dataset
// whose head must be nil (ie a newly created Dataset).
// The new Tag struct is constructed with `ref` and metadata about the tag
// contained in the struct `opts.Meta`.
// The returned Dataset is always the newest snapshot, regardless of
// success or failure, and Datasets() is updated to match backing storage
// upon return as well.
Tag(ctx context.Context, ds Dataset, ref types.Ref, opts TagOptions) (Dataset, error)
// Delete removes the Dataset named ds.ID() from the map at the root of
// the Database. The Dataset data is not necessarily cleaned up at this
// time, but may be garbage collected in the future.
@@ -104,11 +113,11 @@ type Database interface {
// of a conflict, Delete returns an 'ErrMergeNeeded' error.
Delete(ctx context.Context, ds Dataset) (Dataset, error)
// SetHead ignores any lineage constraints (e.g. the current Head being in
// SetHeadToCommit ignores any lineage constraints (e.g. the current Head being in
// commit’s Parent set) and force-sets a mapping from datasetID: commit in
// this database.
// All Values that have been written to this Database are guaranteed to be
// persistent after SetHead(). If the update cannot be performed, e.g.,
// persistent after SetHeadToCommit(). If the update cannot be performed, e.g.,
// because another process moved the current Head out from under you,
// error will be non-nil.
// The newest snapshot of the Dataset is always returned, so the caller an

134
go/store/datas/database_common.go
View File

@@ -158,15 +158,41 @@ func (db *database) SetHead(ctx context.Context, ds Dataset, newHeadRef types.Re
}
func (db *database) doSetHead(ctx context.Context, ds Dataset, newHeadRef types.Ref) error {
if currentHeadRef, ok, err := ds.MaybeHeadRef(); err != nil {
newSt, err := newHeadRef.TargetValue(ctx, db)
if err != nil {
return err
} else if ok {
}
headType := newSt.(types.Struct).Name()
currentHeadRef, ok, err := ds.MaybeHeadRef()
if err != nil {
return err
}
if ok {
if newHeadRef.Equals(currentHeadRef) {
return nil
}
currSt, err := currentHeadRef.TargetValue(ctx, db)
if err != nil {
return err
}
headType = currSt.(types.Struct).Name()
}
commit, err := db.validateRefAsCommit(ctx, newHeadRef)
// the new head value must match the type of the old head value
switch headType {
case CommitName:
_, err = db.validateRefAsCommit(ctx, newHeadRef)
case TagName:
err = db.validateTag(ctx, newSt.(types.Struct))
default:
return fmt.Errorf("Unrecognized dataset value: %s", headType)
}
if err != nil {
return err
@@ -184,13 +210,13 @@ func (db *database) doSetHead(ctx context.Context, ds Dataset, newHeadRef types.
return err
}
commitRef, err := db.WriteValue(ctx, commit) // will be orphaned if the tryCommitChunks() below fails
refSt, err := db.WriteValue(ctx, newSt) // will be orphaned if the tryCommitChunks() below fails
if err != nil {
return err
}
ref, err := types.ToRefOfValue(commitRef, db.Format())
ref, err := types.ToRefOfValue(refSt, db.Format())
if err != nil {
return err
@@ -417,6 +443,78 @@ func (db *database) doCommit(ctx context.Context, datasetID string, commit types
return tryCommitErr
}
func (db *database) Tag(ctx context.Context, ds Dataset, ref types.Ref, opts TagOptions) (Dataset, error) {
return db.doHeadUpdate(
ctx,
ds,
func(ds Dataset) error {
st, err := NewTag(ctx, ref, opts.Meta)
if err != nil {
return err
}
return db.doTag(ctx, ds.ID(), st)
},
)
}
// doTag manages concurrent access the single logical piece of mutable state: the current Root. It uses
// the same optimistic writing algorithm as doCommit (see above).
func (db *database) doTag(ctx context.Context, datasetID string, tag types.Struct) error {
err := db.validateTag(ctx, tag)
if err != nil {
return err
}
// This could loop forever, given enough simultaneous writers. BUG 2565
var tryCommitErr error
for tryCommitErr = ErrOptimisticLockFailed; tryCommitErr == ErrOptimisticLockFailed; {
currentRootHash, err := db.rt.Root(ctx)
if err != nil {
return err
}
currentDatasets, err := db.Datasets(ctx)
if err != nil {
return err
}
tagRef, err := db.WriteValue(ctx, tag) // will be orphaned if the tryCommitChunks() below fails
if err != nil {
return err
}
_, hasHead, err := currentDatasets.MaybeGet(ctx, types.String(datasetID))
if err != nil {
return err
}
if hasHead {
return fmt.Errorf("datasets for tags (refs/tags/*) cannot be altered after creation")
}
ref, err := types.ToRefOfValue(tagRef, db.Format())
if err != nil {
return err
}
currentDatasets, err = currentDatasets.Edit().Set(types.String(datasetID), ref).Map(ctx)
if err != nil {
return err
}
tryCommitErr = db.tryCommitChunks(ctx, currentDatasets, currentRootHash)
}
return tryCommitErr
}
func (db *database) Delete(ctx context.Context, ds Dataset) (Dataset, error) {
return db.doHeadUpdate(ctx, ds, func(ds Dataset) error { return db.doDelete(ctx, ds.ID()) })
}
@@ -519,6 +617,32 @@ func (db *database) validateRefAsCommit(ctx context.Context, r types.Ref) (types
return v.(types.Struct), nil
}
func (db *database) validateTag(ctx context.Context, t types.Struct) error {
is, err := IsTag(t)
if err != nil {
return err
}
if !is {
return fmt.Errorf("Tag struct %s is malformed, IsTag() == false", t.String())
}
r, ok, err := t.MaybeGet(TagCommitRefField)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("tag is missing field %s", TagCommitRefField)
}
_, err = db.validateRefAsCommit(ctx, r.(types.Ref))
if err != nil {
return err
}
return nil
}
func buildNewCommit(ctx context.Context, ds Dataset, v types.Value, opts CommitOptions) (types.Struct, error) {
parents := opts.ParentsList
if parents == types.EmptyList || parents.Len() == 0 {

4
go/store/datas/database_test.go
View File

@@ -564,13 +564,13 @@ func (suite *DatabaseSuite) TestSetHead() {
a := types.String("a")
ds, err = suite.db.CommitValue(context.Background(), ds, a)
suite.NoError(err)
aCommitRef := mustHeadRef(ds) // To use in non-FF SetHead() below.
aCommitRef := mustHeadRef(ds) // To use in non-FF SetHeadToCommit() below.
b := types.String("b")
ds, err = suite.db.CommitValue(context.Background(), ds, b)
suite.NoError(err)
suite.True(mustHeadValue(ds).Equals(b))
bCommitRef := mustHeadRef(ds) // To use in FF SetHead() below.
bCommitRef := mustHeadRef(ds) // To use in FF SetHeadToCommit() below.
ds, err = suite.db.SetHead(context.Background(), ds, aCommitRef)
suite.NoError(err)

19
go/store/datas/dataset.go
View File

@@ -44,10 +44,19 @@ type Dataset struct {
}
func newDataset(db Database, id string, head types.Value) (Dataset, error) {
headNilOrIsCommit := head == nil
if !headNilOrIsCommit {
var err error
headNilOrIsCommit, err = IsCommit(head)
check := head == nil
var err error
if !check {
check, err = IsCommit(head)
if err != nil {
return Dataset{}, err
}
}
if !check {
check, err = IsTag(head)
if err != nil {
return Dataset{}, err
@@ -55,7 +64,7 @@ func newDataset(db Database, id string, head types.Value) (Dataset, error) {
}
// precondition checks
d.PanicIfFalse(headNilOrIsCommit)
d.PanicIfFalse(check)
return Dataset{db, id, head}, nil
}

80
go/store/datas/tag.go Normal file
View File

@@ -0,0 +1,80 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datas
import (
"context"
"github.com/liquidata-inc/dolt/go/store/nomdl"
"github.com/liquidata-inc/dolt/go/store/types"
)
const (
TagMetaField = "meta"
TagCommitRefField = "ref"
TagName = "Tag"
)
var tagTemplate = types.MakeStructTemplate(TagName, []string{TagMetaField, TagCommitRefField})
// ref is a Ref<Commit>, but 'Commit' is not defined in this snippet.
// Tag refs are validated to point at Commits during write.
var valueTagType = nomdl.MustParseType(`Struct Tag {
meta: Struct {},
ref: Ref<Value>,
}`)
// TagOptions is used to pass options into Tag.
type TagOptions struct {
// Meta is a Struct that describes arbitrary metadata about this Tag,
// e.g. a timestamp or descriptive text.
Meta types.Struct
}
// NewTag creates a new tag object.
//
// A tag has the following type:
//
// ```
// struct Tag {
// meta: M,
// commitRef: T,
// }
// ```
// where M is a struct type and R is a ref type.
func NewTag(_ context.Context, commitRef types.Ref, meta types.Struct) (types.Struct, error) {
return tagTemplate.NewStruct(meta.Format(), []types.Value{meta, commitRef})
}
func IsTag(v types.Value) (bool, error) {
if s, ok := v.(types.Struct); !ok {
return false, nil
} else {
return types.IsValueSubtypeOf(s.Format(), v, valueTagType)
}
}
func makeTagStructType(metaType, refType *types.Type) (*types.Type, error) {
return types.MakeStructType(TagName,
types.StructField{
Name: TagMetaField,
Type: metaType,
},
types.StructField{
Name: TagCommitRefField,
Type: refType,
},
)
}

70
go/store/datas/tag_test.go Normal file
View File

@@ -0,0 +1,70 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datas
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/liquidata-inc/dolt/go/store/chunks"
"github.com/liquidata-inc/dolt/go/store/types"
)
func TestNewTag(t *testing.T) {
assert := assert.New(t)
assertTypeEquals := func(e, a *types.Type) {
t.Helper()
assert.True(a.Equals(e), "Actual: %s\nExpected %s", mustString(a.Describe(context.Background())), mustString(e.Describe(context.Background())))
}
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
parents := mustList(types.NewList(context.Background(), db))
commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18))
assert.NoError(err)
cmRef, err := types.NewRef(commit, types.Format_7_18)
assert.NoError(err)
tag, err := NewTag(context.Background(), cmRef, types.EmptyStruct(types.Format_7_18))
ct, err := makeCommitStructType(
types.EmptyStructType,
mustType(types.MakeSetType(mustType(types.MakeUnionType()))),
mustType(types.MakeListType(mustType(types.MakeUnionType()))),
types.PrimitiveTypeMap[types.FloatKind],
)
assert.NoError(err)
et, err := makeTagStructType(
types.EmptyStructType,
mustType(types.MakeRefType(ct)),
)
assert.NoError(err)
at, err := types.TypeOf(tag)
assert.NoError(err)
assertTypeEquals(et, at)
}
func TestPersistedTagConsts(t *testing.T) {
// changing constants that are persisted requires a migration strategy
assert.Equal(t, "meta", TagMetaField)
assert.Equal(t, "ref", TagCommitRefField)
assert.Equal(t, "Tag", TagName)
}

10
go/store/nbs/aws_chunk_source.go
View File

@@ -28,7 +28,9 @@ import (
"time"
)
func newAWSChunkSource(ctx context.Context, ddb *ddbTableStore, s3 *s3ObjectReader, al awsLimits, name addr, chunkCount uint32, indexCache *indexCache, stats *Stats) (cs chunkSource, err error) {
type indexParserF func([]byte) (tableIndex, error)
func newAWSChunkSource(ctx context.Context, ddb *ddbTableStore, s3 *s3ObjectReader, al awsLimits, name addr, chunkCount uint32, indexCache *indexCache, stats *Stats, parseIndex indexParserF) (cs chunkSource, err error) {
if indexCache != nil {
indexCache.lockEntry(name)
defer func() {
@@ -86,14 +88,14 @@ func newAWSChunkSource(ctx context.Context, ddb *ddbTableStore, s3 *s3ObjectRead
stats.IndexBytesPerRead.Sample(uint64(len(indexBytes)))
stats.IndexReadLatency.SampleTimeSince(t1)
index, err := parseTableIndex(indexBytes)
index, err := parseIndex(indexBytes)
if err != nil {
return emptyChunkSource{}, err
}
if indexCache != nil {
indexCache.put(name, index)
if ohi, ok := index.(onHeapTableIndex); indexCache != nil && ok {
indexCache.put(name, ohi)
}
return &chunkSourceAdapter{newTableReader(index, tra, s3BlockSize), name}, nil

3
go/store/nbs/aws_chunk_source_test.go
View File

@@ -53,6 +53,9 @@ func TestAWSChunkSource(t *testing.T) {
uint32(len(chunks)),
ic,
&Stats{},
func(bs []byte) (tableIndex, error) {
return parseTableIndex(bs)
},
)
assert.NoError(t, err)

2
go/store/nbs/aws_table_persister.go
View File

@@ -61,6 +61,7 @@ type awsTablePersister struct {
limits awsLimits
indexCache *indexCache
ns string
parseIndex indexParserF
}
type awsLimits struct {
@@ -90,6 +91,7 @@ func (s3p awsTablePersister) Open(ctx context.Context, name addr, chunkCount uin
chunkCount,
s3p.indexCache,
stats,
s3p.parseIndex,
)
}

34
go/store/nbs/aws_table_persister_test.go
View File

@@ -37,6 +37,10 @@ import (
"github.com/liquidata-inc/dolt/go/store/util/sizecache"
)
var parseIndexF = func(bs []byte) (tableIndex, error) {
return parseTableIndex(bs)
}
func TestAWSTablePersisterPersist(t *testing.T) {
calcPartSize := func(rdr chunkReader, maxPartNum uint64) uint64 {
return maxTableSize(uint64(mustUint32(rdr.count())), mustUint64(rdr.uncompressedLen())) / maxPartNum
@@ -54,7 +58,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
s3svc, ddb := makeFakeS3(t), makeFakeDTS(makeFakeDDB(t), nil)
ic := newIndexCache(1024)
limits := awsLimits{partTarget: calcPartSize(mt, 3)}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, indexCache: ic, ns: ns}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, indexCache: ic, ns: ns, parseIndex: parseIndexF}
src, err := s3p.Persist(context.Background(), mt, nil, &Stats{})
assert.NoError(err)
@@ -71,7 +75,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
s3svc, ddb := makeFakeS3(t), makeFakeDTS(makeFakeDDB(t), nil)
limits := awsLimits{partTarget: calcPartSize(mt, 3)}
tc := &waitOnStoreTableCache{readers: map[addr]io.ReaderAt{}}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, tc: tc, ns: ns}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, tc: tc, ns: ns, parseIndex: parseIndexF}
// Persist and wait until tc.store() has completed
tc.storeWG.Add(1)
@@ -99,7 +103,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
s3svc, ddb := makeFakeS3(t), makeFakeDTS(makeFakeDDB(t), nil)
limits := awsLimits{partTarget: calcPartSize(mt, 1)}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, ns: ns}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, ns: ns, parseIndex: parseIndexF}
src, err := s3p.Persist(context.Background(), mt, nil, &Stats{})
assert.NoError(err)
@@ -123,7 +127,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
s3svc, ddb := makeFakeS3(t), makeFakeDTS(makeFakeDDB(t), nil)
limits := awsLimits{partTarget: 1 << 10}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, ns: ns}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, ns: ns, parseIndex: parseIndexF}
src, err := s3p.Persist(context.Background(), mt, existingTable, &Stats{})
assert.NoError(err)
@@ -139,7 +143,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
s3svc := &failingFakeS3{makeFakeS3(t), sync.Mutex{}, 1}
ddb := makeFakeDTS(makeFakeDDB(t), nil)
limits := awsLimits{partTarget: calcPartSize(mt, 4)}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, ns: ns}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: ddb, limits: limits, ns: ns, parseIndex: parseIndexF}
_, err := s3p.Persist(context.Background(), mt, nil, &Stats{})
assert.Error(err)
@@ -161,7 +165,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
ddb := makeFakeDDB(t)
s3svc, dts := makeFakeS3(t), makeFakeDTS(ddb, nil)
limits := awsLimits{itemMax: maxDynamoItemSize, chunkMax: 2 * mustUint32(mt.count())}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: ""}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: "", parseIndex: parseIndexF}
src, err := s3p.Persist(context.Background(), mt, nil, &Stats{})
assert.NoError(err)
@@ -181,7 +185,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
s3svc, dts := makeFakeS3(t), makeFakeDTS(ddb, tc)
limits := awsLimits{itemMax: maxDynamoItemSize, chunkMax: 2 * mustUint32(mt.count())}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: ""}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: "", parseIndex: parseIndexF}
tableData, name, err := buildTable(testChunks)
assert.NoError(err)
@@ -206,7 +210,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
ddb := makeFakeDDB(t)
s3svc, dts := makeFakeS3(t), makeFakeDTS(ddb, nil)
limits := awsLimits{itemMax: maxDynamoItemSize, chunkMax: 1, partTarget: calcPartSize(mt, 1)}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: ""}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: "", parseIndex: parseIndexF}
src, err := s3p.Persist(context.Background(), mt, nil, &Stats{})
assert.NoError(err)
@@ -226,7 +230,7 @@ func TestAWSTablePersisterPersist(t *testing.T) {
ddb := makeFakeDDB(t)
s3svc, dts := makeFakeS3(t), makeFakeDTS(ddb, nil)
limits := awsLimits{itemMax: 0, chunkMax: 2 * mustUint32(mt.count()), partTarget: calcPartSize(mt, 1)}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: ""}
s3p := awsTablePersister{s3: s3svc, bucket: "bucket", ddb: dts, limits: limits, ns: "", parseIndex: parseIndexF}
src, err := s3p.Persist(context.Background(), mt, nil, &Stats{})
assert.NoError(err)
@@ -356,7 +360,17 @@ func TestAWSTablePersisterConjoinAll(t *testing.T) {
defer close(rl)
newPersister := func(s3svc s3svc, ddb *ddbTableStore) awsTablePersister {
return awsTablePersister{s3svc, "bucket", rl, nil, ddb, awsLimits{targetPartSize, minPartSize, maxPartSize, maxItemSize, maxChunkCount}, ic, ""}
return awsTablePersister{
s3svc,
"bucket",
rl,
nil,
ddb,
awsLimits{targetPartSize, minPartSize, maxPartSize, maxItemSize, maxChunkCount},
ic,
"",
parseIndexF,
}
}
var smallChunks [][]byte

10
go/store/nbs/block_store_test.go
View File

@@ -412,7 +412,7 @@ func TestBlockStoreConjoinOnCommit(t *testing.T) {
assertContainAll := func(t *testing.T, store chunks.ChunkStore, srcs ...chunkSource) {
rdrs := make(chunkReaderGroup, len(srcs))
for i, src := range srcs {
rdrs[i] = src
rdrs[i] = src.Clone()
}
chunkChan := make(chan extractRecord, mustUint32(rdrs.count()))
err := rdrs.extract(context.Background(), chunkChan)
@@ -492,6 +492,10 @@ func TestBlockStoreConjoinOnCommit(t *testing.T) {
assert.NoError(t, err)
assert.True(t, ok)
assertContainAll(t, smallTableStore, srcs...)
for _, src := range srcs {
err := src.Close()
assert.NoError(t, err)
}
})
t.Run("ConjoinRetry", func(t *testing.T) {
@@ -523,6 +527,10 @@ func TestBlockStoreConjoinOnCommit(t *testing.T) {
assert.NoError(t, err)
assert.True(t, ok)
assertContainAll(t, smallTableStore, srcs...)
for _, src := range srcs {
err := src.Close()
assert.NoError(t, err)
}
})
}

12
go/store/nbs/chunk_source_adapter.go
View File

@@ -23,10 +23,6 @@ func (csa chunkSourceAdapter) hash() (addr, error) {
return csa.h, nil
}
func (csa chunkSourceAdapter) index() (tableIndex, error) {
return csa.tableIndex, nil
}
func newReaderFromIndexData(indexCache *indexCache, idxData []byte, name addr, tra tableReaderAt, blockSize uint64) (cs chunkSource, err error) {
index, err := parseTableIndex(idxData)
@@ -48,3 +44,11 @@ func newReaderFromIndexData(indexCache *indexCache, idxData []byte, name addr, t
return &chunkSourceAdapter{newTableReader(index, tra, blockSize), name}, nil
}
func (csa chunkSourceAdapter) Close() error {
return csa.tableReader.Close()
}
func (csa chunkSourceAdapter) Clone() chunkSource {
return &chunkSourceAdapter{csa.tableReader.Clone(), csa.h}
}

4
go/store/nbs/conjoiner_test.go
View File

@@ -63,7 +63,7 @@ func makeTestSrcs(t *testing.T, tableSizes []uint32, p tablePersister) (srcs chu
}
cs, err := p.Persist(context.Background(), mt, nil, &Stats{})
assert.NoError(t, err)
srcs = append(srcs, cs)
srcs = append(srcs, cs.Clone())
}
return
}
@@ -73,6 +73,8 @@ func TestConjoin(t *testing.T) {
makeTestTableSpecs := func(tableSizes []uint32, p tablePersister) (specs []tableSpec) {
for _, src := range makeTestSrcs(t, tableSizes, p) {
specs = append(specs, tableSpec{mustAddr(src.hash()), mustUint32(src.count())})
err := src.Close()
assert.NoError(t, err)
}
return
}

2
go/store/nbs/file_table_persister.go
View File

@@ -182,7 +182,7 @@ func (ftp *fsTablePersister) ConjoinAll(ctx context.Context, sources chunkSource
return "", ferr
}
var index tableIndex
var index onHeapTableIndex
index, ferr = parseTableIndex(plan.mergedIndex)
if ferr != nil {

4
go/store/nbs/mem_table.go
View File

@@ -214,3 +214,7 @@ func (mt *memTable) write(haver chunkReader, stats *Stats) (name addr, data []by
return name, buff[:tableSize], count, nil
}
func (mt *memTable) Close() error {
return nil
}

11
go/store/nbs/mem_table_test.go
View File

@@ -314,3 +314,14 @@ func (crg chunkReaderGroup) extract(ctx context.Context, chunks chan<- extractRe
return nil
}
func (crg chunkReaderGroup) Close() error {
var firstErr error
for _, c := range crg {
err := c.Close()
if err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}

12
go/store/nbs/mmap_table_reader.go
View File

@@ -58,7 +58,7 @@ func init() {
func newMmapTableReader(dir string, h addr, chunkCount uint32, indexCache *indexCache, fc *fdCache) (cs chunkSource, err error) {
path := filepath.Join(dir, h.String())
var index tableIndex
var index onHeapTableIndex
found := false
if indexCache != nil {
indexCache.lockEntry(h)
@@ -73,7 +73,7 @@ func newMmapTableReader(dir string, h addr, chunkCount uint32, indexCache *index
}
if !found {
f := func() (ti tableIndex, err error) {
f := func() (ti onHeapTableIndex, err error) {
var f *os.File
f, err = fc.RefFile(path)
@@ -163,6 +163,14 @@ func (mmtr *mmapTableReader) hash() (addr, error) {
return mmtr.h, nil
}
func (mmtr *mmapTableReader) Close() error {
return mmtr.tableReader.Close()
}
func (mmtr *mmapTableReader) Clone() chunkSource {
return &mmapTableReader{mmtr.tableReader.Clone(), mmtr.fc, mmtr.h}
}
type cacheReaderAt struct {
path string
fc *fdCache

24
go/store/nbs/persisting_chunk_source.go
View File

@@ -94,6 +94,16 @@ func (ccs *persistingChunkSource) getReader() chunkReader {
return ccs.cs
}
func (ccs *persistingChunkSource) Close() error {
// persistingChunkSource does not own |cs| or |mt|. No need to close them.
return nil
}
func (ccs *persistingChunkSource) Clone() chunkSource {
// persistingChunkSource does not own |cs| or |mt|. No need to Clone.
return ccs
}
func (ccs *persistingChunkSource) has(h addr) (bool, error) {
cr := ccs.getReader()
@@ -196,11 +206,11 @@ func (ccs *persistingChunkSource) index() (tableIndex, error) {
err := ccs.wait()
if err != nil {
return tableIndex{}, err
return onHeapTableIndex{}, err
}
if ccs.cs == nil {
return tableIndex{}, ErrNoChunkSource
return onHeapTableIndex{}, ErrNoChunkSource
}
return ccs.cs.index()
@@ -283,7 +293,7 @@ func (ecs emptyChunkSource) hash() (addr, error) {
}
func (ecs emptyChunkSource) index() (tableIndex, error) {
return tableIndex{}, nil
return onHeapTableIndex{}, nil
}
func (ecs emptyChunkSource) reader(context.Context) (io.Reader, error) {
@@ -297,3 +307,11 @@ func (ecs emptyChunkSource) calcReads(reqs []getRecord, blockSize uint64) (reads
func (ecs emptyChunkSource) extract(ctx context.Context, chunks chan<- extractRecord) error {
return nil
}
func (ecs emptyChunkSource) Close() error {
return nil
}
func (ecs emptyChunkSource) Clone() chunkSource {
return ecs
}

71
go/store/nbs/store.go
View File

@@ -115,10 +115,8 @@ func (nbs *NomsBlockStore) GetChunkLocations(hashes hash.HashSet) (map[hash.Hash
}
for _, offsetRec := range offsetRecSlice {
ord := offsetRec.ordinal
length := tr.lengths[ord]
h := hash.Hash(*offsetRec.a)
y[h] = Range{Offset: offsetRec.offset, Length: length}
y[h] = Range{Offset: offsetRec.offset, Length: offsetRec.length}
delete(hashes, h)
}
@@ -144,11 +142,11 @@ func (nbs *NomsBlockStore) GetChunkLocations(hashes hash.HashSet) (map[hash.Hash
var foundHashes []hash.Hash
for h := range hashes {
ord := tableIndex.lookupOrdinal(addr(h))
if ord < tableIndex.chunkCount {
a := addr(h)
e, ok := tableIndex.Lookup(&a)
if ok {
foundHashes = append(foundHashes, h)
y[h] = Range{Offset: tableIndex.offsets[ord], Length: tableIndex.lengths[ord]}
y[h] = Range{Offset: e.Offset(), Length: e.Length()}
}
}
@@ -239,11 +237,40 @@ func (nbs *NomsBlockStore) UpdateManifest(ctx context.Context, updates map[hash.
}
nbs.upstream = updatedContents
oldTables := nbs.tables
nbs.tables = newTables
err = oldTables.Close()
if err != nil {
return manifestContents{}, err
}
return updatedContents, nil
}
func NewAWSStoreWithMMapIndex(ctx context.Context, nbfVerStr string, table, ns, bucket string, s3 s3svc, ddb ddbsvc, memTableSize uint64) (*NomsBlockStore, error) {
cacheOnce.Do(makeGlobalCaches)
readRateLimiter := make(chan struct{}, 32)
p := &awsTablePersister{
s3,
bucket,
readRateLimiter,
nil,
&ddbTableStore{ddb, table, readRateLimiter, nil},
awsLimits{defaultS3PartSize, minS3PartSize, maxS3PartSize, maxDynamoItemSize, maxDynamoChunks},
globalIndexCache,
ns,
func(bs []byte) (tableIndex, error) {
ohi, err := parseTableIndex(bs)
if err != nil {
return nil, err
}
return newMmapTableIndex(ohi, nil)
},
}
mm := makeManifestManager(newDynamoManifest(table, ns, ddb))
return newNomsBlockStore(ctx, nbfVerStr, mm, p, inlineConjoiner{defaultMaxTables}, memTableSize)
}
func NewAWSStore(ctx context.Context, nbfVerStr string, table, ns, bucket string, s3 s3svc, ddb ddbsvc, memTableSize uint64) (*NomsBlockStore, error) {
cacheOnce.Do(makeGlobalCaches)
readRateLimiter := make(chan struct{}, 32)
@@ -256,6 +283,9 @@ func NewAWSStore(ctx context.Context, nbfVerStr string, table, ns, bucket string
awsLimits{defaultS3PartSize, minS3PartSize, maxS3PartSize, maxDynamoItemSize, maxDynamoChunks},
globalIndexCache,
ns,
func(bs []byte) (tableIndex, error) {
return parseTableIndex(bs)
},
}
mm := makeManifestManager(newDynamoManifest(table, ns, ddb))
return newNomsBlockStore(ctx, nbfVerStr, mm, p, inlineConjoiner{defaultMaxTables}, memTableSize)
@@ -335,7 +365,12 @@ func newNomsBlockStore(ctx context.Context, nbfVerStr string, mm manifestManager
}
nbs.upstream = contents
oldTables := nbs.tables
nbs.tables = newTables
err = oldTables.Close()
if err != nil {
return nil, err
}
}
return nbs, nil
@@ -658,7 +693,12 @@ func (nbs *NomsBlockStore) Rebase(ctx context.Context) error {
}
nbs.upstream = contents
oldTables := nbs.tables
nbs.tables = newTables
err = oldTables.Close()
if err != nil {
return err
}
}
return nil
@@ -762,12 +802,18 @@ func (nbs *NomsBlockStore) updateManifest(ctx context.Context, current, last has
}
nbs.upstream = upstream
oldTables := nbs.tables
nbs.tables = newTables
err = oldTables.Close()
if last != upstream.root {
return errOptimisticLockFailedRoot
}
if err != nil {
return err
}
return errOptimisticLockFailedTables
}
@@ -804,7 +850,12 @@ func (nbs *NomsBlockStore) updateManifest(ctx context.Context, current, last has
}
nbs.upstream = newUpstream
oldTables := nbs.tables
nbs.tables = newTables
err = oldTables.Close()
if err != nil {
return err
}
return errOptimisticLockFailedTables
}
@@ -848,8 +899,8 @@ func (nbs *NomsBlockStore) Version() string {
return nbs.upstream.vers
}
func (nbs *NomsBlockStore) Close() (err error) {
return
func (nbs *NomsBlockStore) Close() error {
return nbs.tables.Close()
}
func (nbs *NomsBlockStore) Stats() interface{} {
@@ -965,7 +1016,7 @@ func (nbs *NomsBlockStore) Size(ctx context.Context) (uint64, error) {
if err != nil {
return uint64(0), fmt.Errorf("error getting table file index for chunkSource. %w", err)
}
size += ti.tableFileSize()
size += ti.TableFileSize()
}
return size, nil
}

10
go/store/nbs/table.go
View File

@@ -232,6 +232,9 @@ type chunkReader interface {
extract(ctx context.Context, chunks chan<- extractRecord) error
count() (uint32, error)
uncompressedLen() (uint64, error)
// Close releases resources retained by the |chunkReader|.
Close() error
}
type chunkReadPlanner interface {
@@ -264,6 +267,13 @@ type chunkSource interface {
// opens a Reader to the first byte of the chunkData segment of this table.
reader(context.Context) (io.Reader, error)
index() (tableIndex, error)
// Clone returns a |chunkSource| with the same contents as the
// original, but with independent |Close| behavior. A |chunkSource|
// cannot be |Close|d more than once, so if a |chunkSource| is being
// retained in two objects with independent life-cycle, it should be
// |Clone|d first.
Clone() chunkSource
}
type chunkSources []chunkSource

61
go/store/nbs/table_persister.go
View File

@@ -98,14 +98,14 @@ func (sic *indexCache) unlockEntry(name addr) error {
return nil
}
func (sic *indexCache) get(name addr) (tableIndex, bool) {
func (sic *indexCache) get(name addr) (onHeapTableIndex, bool) {
if idx, found := sic.cache.Get(name); found {
return idx.(tableIndex), true
return idx.(onHeapTableIndex), true
}
return tableIndex{}, false
return onHeapTableIndex{}, false
}
func (sic *indexCache) put(name addr, idx tableIndex) {
func (sic *indexCache) put(name addr, idx onHeapTableIndex) {
indexSize := uint64(idx.chunkCount) * (addrSize + ordinalSize + lengthSize + uint64Size)
sic.cache.Add(name, indexSize, idx)
}
@@ -226,7 +226,7 @@ func planConjoin(sources chunkSources, stats *Stats) (plan compactionPlan, err e
return compactionPlan{}, err
}
plan.chunkCount += index.chunkCount
plan.chunkCount += index.ChunkCount()
// Calculate the amount of chunk data in |src|
chunkDataLen := calcChunkDataLen(index)
@@ -253,9 +253,12 @@ func planConjoin(sources chunkSources, stats *Stats) (plan compactionPlan, err e
return compactionPlan{}, err
}
ordinals := index.Ordinals()
prefixes := index.Prefixes()
// Add all the prefix tuples from this index to the list of all prefixIndexRecs, modifying the ordinals such that all entries from the 1st item in sources come after those in the 0th and so on.
for j, prefix := range index.prefixes {
rec := prefixIndexRec{prefix: prefix, order: ordinalOffset + index.ordinals[j]}
for j, prefix := range prefixes {
rec := prefixIndexRec{prefix: prefix, order: ordinalOffset + ordinals[j]}
prefixIndexRecs = append(prefixIndexRecs, rec)
}
@@ -268,21 +271,35 @@ func planConjoin(sources chunkSources, stats *Stats) (plan compactionPlan, err e
ordinalOffset += cnt
// TODO: copy the lengths and suffixes as a byte-copy from src BUG #3438
// Bring over the lengths block, in order
for _, length := range index.lengths {
binary.BigEndian.PutUint32(plan.mergedIndex[lengthsPos:], length)
lengthsPos += lengthSize
if onHeap, ok := index.(onHeapTableIndex); ok {
// TODO: copy the lengths and suffixes as a byte-copy from src BUG #3438
// Bring over the lengths block, in order
for _, length := range onHeap.lengths {
binary.BigEndian.PutUint32(plan.mergedIndex[lengthsPos:], length)
lengthsPos += lengthSize
}
// Bring over the suffixes block, in order
n := copy(plan.mergedIndex[suffixesPos:], onHeap.suffixes)
if n != len(onHeap.suffixes) {
return compactionPlan{}, errors.New("failed to copy all data")
}
suffixesPos += uint64(n)
} else {
// Build up the index one entry at a time.
var a addr
for i := 0; i < len(ordinals); i++ {
e := index.IndexEntry(uint32(i), &a)
li := lengthsPos + lengthSize*uint64(ordinals[i])
si := suffixesPos + addrSuffixSize*uint64(ordinals[i])
binary.BigEndian.PutUint32(plan.mergedIndex[li:], e.Length())
copy(plan.mergedIndex[si:], a[addrPrefixSize:])
}
lengthsPos += lengthSize * uint64(len(ordinals))
suffixesPos += addrSuffixSize * uint64(len(ordinals))
}
// Bring over the suffixes block, in order
n := copy(plan.mergedIndex[suffixesPos:], index.suffixes)
if n != len(index.suffixes) {
return compactionPlan{}, errors.New("failed to copy all data")
}
suffixesPos += uint64(n)
}
// Sort all prefixTuples by hash and then insert them starting at the beginning of plan.mergedIndex
@@ -312,5 +329,5 @@ func nameFromSuffixes(suffixes []byte) (name addr) {
}
func calcChunkDataLen(index tableIndex) uint64 {
return index.offsets[index.chunkCount-1] + uint64(index.lengths[index.chunkCount-1])
return index.TableFileSize() - indexSize(index.ChunkCount()) - footerSize
}

460
go/store/nbs/table_reader.go
View File

@@ -27,10 +27,13 @@ import (
"encoding/binary"
"errors"
"io"
"os"
"sort"
"sync"
"sync/atomic"
"github.com/golang/snappy"
"github.com/liquidata-inc/mmap-go"
"github.com/liquidata-inc/dolt/go/store/atomicerr"
"github.com/liquidata-inc/dolt/go/store/chunks"
@@ -101,7 +104,7 @@ func init() {
// ErrInvalidTableFile is an error returned when a table file is corrupt or invalid.
var ErrInvalidTableFile = errors.New("invalid or corrupt table file")
type tableIndex struct {
type onHeapTableIndex struct {
chunkCount uint32
totalUncompressedData uint64
prefixes, offsets []uint64
@@ -109,6 +112,199 @@ type tableIndex struct {
suffixes []byte
}
type indexEntry interface {
Offset() uint64
Length() uint32
}
type indexResult struct {
o uint64
l uint32
}
func (ir indexResult) Offset() uint64 {
return ir.o
}
func (ir indexResult) Length() uint32 {
return ir.l
}
// An mmapIndexEntry is an addrSuffix, a BigEndian uint64 for the offset and a
// BigEnding uint32 for the chunk size.
const mmapIndexEntrySize = addrSuffixSize + uint64Size + lengthSize
type mmapOrdinalSlice []mmapOrdinal
func (s mmapOrdinalSlice) Len() int { return len(s) }
func (s mmapOrdinalSlice) Less(i, j int) bool { return s[i].offset < s[j].offset }
func (s mmapOrdinalSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (i mmapTableIndex) Ordinals() []uint32 {
s := mmapOrdinalSlice(make([]mmapOrdinal, i.chunkCount))
for idx := 0; uint32(idx) < i.chunkCount; idx++ {
mi := idx * mmapIndexEntrySize
e := mmapIndexEntry(i.data[mi : mi+mmapIndexEntrySize])
s[idx] = mmapOrdinal{idx, e.Offset()}
}
sort.Sort(s)
res := make([]uint32, i.chunkCount)
for j, r := range s {
res[r.idx] = uint32(j)
}
return res
}
type mmapTableIndex struct {
chunkCount uint32
totalUncompressedData uint64
fileSz uint64
prefixes []uint64
data mmap.MMap
refCnt *int32
}
func (i mmapTableIndex) Prefixes() []uint64 {
return i.prefixes
}
type mmapOrdinal struct {
idx int
offset uint64
}
func (i mmapTableIndex) TableFileSize() uint64 {
return i.fileSz
}
func (i mmapTableIndex) ChunkCount() uint32 {
return i.chunkCount
}
func (i mmapTableIndex) TotalUncompressedData() uint64 {
return i.totalUncompressedData
}
func (i mmapTableIndex) Close() error {
cnt := atomic.AddInt32(i.refCnt, -1)
if cnt == 0 {
return i.data.Unmap()
}
if cnt < 0 {
panic("Close() called and reduced ref count to < 0.")
}
return nil
}
func (i mmapTableIndex) Clone() tableIndex {
cnt := atomic.AddInt32(i.refCnt, 1)
if cnt == 1 {
panic("Clone() called after last Close(). This index is no longer valid.")
}
return i
}
func (i mmapTableIndex) prefixIdx(prefix uint64) (idx uint32) {
// NOTE: The golang impl of sort.Search is basically inlined here. This method can be called in
// an extremely tight loop and inlining the code was a significant perf improvement.
idx, j := 0, i.chunkCount
for idx < j {
h := idx + (j-idx)/2 // avoid overflow when computing h
// i ≤ h < j
if i.prefixes[h] < prefix {
idx = h + 1 // preserves f(i-1) == false
} else {
j = h // preserves f(j) == true
}
}
return
}
func (i mmapTableIndex) Lookup(h *addr) (indexEntry, bool) {
prefix := binary.BigEndian.Uint64(h[:])
for idx := i.prefixIdx(prefix); idx < i.chunkCount && i.prefixes[idx] == prefix; idx++ {
mi := idx * mmapIndexEntrySize
e := mmapIndexEntry(i.data[mi : mi+mmapIndexEntrySize])
if bytes.Equal(e.suffix(), h[addrPrefixSize:]) {
return e, true
}
}
return mmapIndexEntry{}, false
}
func (i mmapTableIndex) EntrySuffixMatches(idx uint32, h *addr) bool {
mi := idx * mmapIndexEntrySize
e := mmapIndexEntry(i.data[mi : mi+mmapIndexEntrySize])
return bytes.Equal(e.suffix(), h[addrPrefixSize:])
}
func (i mmapTableIndex) IndexEntry(idx uint32, a *addr) indexEntry {
mi := idx * mmapIndexEntrySize
e := mmapIndexEntry(i.data[mi : mi+mmapIndexEntrySize])
if a != nil {
binary.BigEndian.PutUint64(a[:], i.prefixes[idx])
copy(a[addrPrefixSize:], e.suffix())
}
return e
}
type mmapIndexEntry []byte
const mmapIndexEntryOffsetStart = addrSuffixSize
const mmapIndexEntryLengthStart = addrSuffixSize + uint64Size
func (e mmapIndexEntry) suffix() []byte {
return e[:addrSuffixSize]
}
func (e mmapIndexEntry) Offset() uint64 {
return binary.BigEndian.Uint64(e[mmapIndexEntryOffsetStart:])
}
func (e mmapIndexEntry) Length() uint32 {
return binary.BigEndian.Uint32(e[mmapIndexEntryLengthStart:])
}
func mmapOffheapSize(chunks int) int {
pageSize := 4096
esz := addrSuffixSize + uint64Size + lengthSize
min := esz * chunks
if min%pageSize == 0 {
return min
} else {
return (min/pageSize + 1) * pageSize
}
}
func newMmapTableIndex(ti onHeapTableIndex, f *os.File) (mmapTableIndex, error) {
flags := 0
if f == nil {
flags = mmap.ANON
}
arr, err := mmap.MapRegion(f, mmapOffheapSize(len(ti.ordinals)), mmap.RDWR, flags, 0)
if err != nil {
return mmapTableIndex{}, err
}
for i := range ti.ordinals {
idx := i * mmapIndexEntrySize
si := addrSuffixSize * ti.ordinals[i]
copy(arr[idx:], ti.suffixes[si:si+addrSuffixSize])
binary.BigEndian.PutUint64(arr[idx+mmapIndexEntryOffsetStart:], ti.offsets[ti.ordinals[i]])
binary.BigEndian.PutUint32(arr[idx+mmapIndexEntryLengthStart:], ti.lengths[ti.ordinals[i]])
}
refCnt := new(int32)
*refCnt = 1
return mmapTableIndex{
ti.chunkCount,
ti.totalUncompressedData,
ti.TableFileSize(),
ti.Prefixes(),
arr,
refCnt,
}, nil
}
type tableReaderAt interface {
ReadAtWithStats(ctx context.Context, p []byte, off int64, stats *Stats) (n int, err error)
}
@@ -120,28 +316,69 @@ type tableReaderAt interface {
// more chunks together into a single read request to backing storage.
type tableReader struct {
tableIndex
r tableReaderAt
blockSize uint64
prefixes []uint64
chunkCount uint32
totalUncompressedData uint64
r tableReaderAt
blockSize uint64
}
type tableIndex interface {
// ChunkCount returns the total number of chunks in the indexed file.
ChunkCount() uint32
// EntrySuffixMatches returns true if the entry at index |idx| matches
// the suffix of the address |h|. Used by |Lookup| after finding
// matching indexes based on |Prefixes|.
EntrySuffixMatches(idx uint32, h *addr) bool
// IndexEntry returns the |indexEntry| at |idx|. Optionally puts the
// full address of that entry in |a| if |a| is not |nil|.
IndexEntry(idx uint32, a *addr) indexEntry
// Lookup returns an |indexEntry| for the chunk corresponding to the
// provided address |h|. Second returns is |true| if an entry exists
// and |false| otherwise.
Lookup(h *addr) (indexEntry, bool)
// Ordinals returns a slice of indexes which maps the |i|th chunk in
// the indexed file to its corresponding entry in index. The |i|th
// entry in the result is the |i|th chunk in the indexed file, and its
// corresponding value in the slice is the index entry that maps to it.
Ordinals() []uint32
// Prefixes returns the sorted slice of |uint64| |addr| prefixes; each
// entry corresponds to an indexed chunk address.
Prefixes() []uint64
// TableFileSize returns the total size of the indexed table file, in bytes.
TableFileSize() uint64
// TotalUncompressedData returns the total uncompressed data size of
// the table file. Used for informational statistics only.
TotalUncompressedData() uint64
// Close releases any resources used by this tableIndex.
Close() error
// Clone returns a |tableIndex| with the same contents which can be
// |Close|d independently.
Clone() tableIndex
}
var _ tableIndex = mmapTableIndex{}
// parses a valid nbs tableIndex from a byte stream. |buff| must end with an NBS index
// and footer, though it may contain an unspecified number of bytes before that data.
// |tableIndex| doesn't keep alive any references to |buff|.
func parseTableIndex(buff []byte) (tableIndex, error) {
func parseTableIndex(buff []byte) (onHeapTableIndex, error) {
pos := int64(len(buff))
// footer
pos -= magicNumberSize
if string(buff[pos:]) != magicNumber {
return tableIndex{}, ErrInvalidTableFile
return onHeapTableIndex{}, ErrInvalidTableFile
}
// total uncompressed chunk data
pos -= uint64Size
if pos < 0 {
return tableIndex{}, ErrInvalidTableFile
return onHeapTableIndex{}, ErrInvalidTableFile
}
totalUncompressedData := binary.BigEndian.Uint64(buff[pos:])
@@ -149,7 +386,7 @@ func parseTableIndex(buff []byte) (tableIndex, error) {
pos -= uint32Size
if pos < 0 {
return tableIndex{}, ErrInvalidTableFile
return onHeapTableIndex{}, ErrInvalidTableFile
}
chunkCount := binary.BigEndian.Uint32(buff[pos:])
@@ -159,7 +396,7 @@ func parseTableIndex(buff []byte) (tableIndex, error) {
pos -= suffixesSize
if pos < 0 {
return tableIndex{}, ErrInvalidTableFile
return onHeapTableIndex{}, ErrInvalidTableFile
}
suffixes := make([]byte, suffixesSize)
@@ -169,7 +406,7 @@ func parseTableIndex(buff []byte) (tableIndex, error) {
pos -= lengthsSize
if pos < 0 {
return tableIndex{}, ErrInvalidTableFile
return onHeapTableIndex{}, ErrInvalidTableFile
}
lengths, offsets := computeOffsets(chunkCount, buff[pos:pos+lengthsSize])
@@ -178,12 +415,12 @@ func parseTableIndex(buff []byte) (tableIndex, error) {
pos -= tuplesSize
if pos < 0 {
return tableIndex{}, ErrInvalidTableFile
return onHeapTableIndex{}, ErrInvalidTableFile
}
prefixes, ordinals := computePrefixes(chunkCount, buff[pos:pos+tuplesSize])
return tableIndex{
return onHeapTableIndex{
chunkCount, totalUncompressedData,
prefixes, offsets,
lengths, ordinals,
@@ -216,28 +453,24 @@ func computePrefixes(count uint32, buff []byte) (prefixes []uint64, ordinals []u
return
}
func (ti tableIndex) prefixIdxToOrdinal(idx uint32) uint32 {
func (ti onHeapTableIndex) prefixIdxToOrdinal(idx uint32) uint32 {
return ti.ordinals[idx]
}
// Returns the size of the table file that this index references.
// This assumes that the index follows immediately after the last
// chunk in the file and that the last chunk in the file is in the
// index.
func (ti tableIndex) tableFileSize() uint64 {
len, offset := uint64(0), uint64(0)
for i := range ti.offsets {
if ti.offsets[i] >= offset {
offset = ti.offsets[i]
len = uint64(ti.lengths[i])
}
// TableFileSize returns the size of the table file that this index references.
// This assumes that the index follows immediately after the last chunk in the
// file and that the last chunk in the file is in the index.
func (ti onHeapTableIndex) TableFileSize() uint64 {
if ti.chunkCount == 0 {
return footerSize
}
len, offset := ti.offsets[ti.chunkCount-1], uint64(ti.lengths[ti.chunkCount-1])
return offset + len + indexSize(ti.chunkCount) + footerSize
}
// returns the first position in |tr.prefixes| whose value == |prefix|. Returns |tr.chunkCount|
// if absent
func (ti tableIndex) prefixIdx(prefix uint64) (idx uint32) {
// prefixIdx returns the first position in |tr.prefixes| whose value ==
// |prefix|. Returns |tr.chunkCount| if absent
func (ti onHeapTableIndex) prefixIdx(prefix uint64) (idx uint32) {
// NOTE: The golang impl of sort.Search is basically inlined here. This method can be called in
// an extremely tight loop and inlining the code was a significant perf improvement.
idx, j := 0, ti.chunkCount
@@ -254,31 +487,81 @@ func (ti tableIndex) prefixIdx(prefix uint64) (idx uint32) {
return
}
// Return true IFF the suffix at insertion order |ordinal| matches the address |a|.
func (ti tableIndex) ordinalSuffixMatches(ordinal uint32, h addr) bool {
li := uint64(ordinal) * addrSuffixSize
// EntrySuffixMatches returns true IFF the suffix for prefix entry |idx|
// matches the address |a|.
func (ti onHeapTableIndex) EntrySuffixMatches(idx uint32, h *addr) bool {
li := uint64(ti.ordinals[idx]) * addrSuffixSize
return bytes.Equal(h[addrPrefixSize:], ti.suffixes[li:li+addrSuffixSize])
}
// returns the ordinal of |h| if present. returns |ti.chunkCount| if absent
func (ti tableIndex) lookupOrdinal(h addr) uint32 {
// lookupOrdinal returns the ordinal of |h| if present. Returns |ti.chunkCount|
// if absent.
func (ti onHeapTableIndex) lookupOrdinal(h *addr) uint32 {
prefix := h.Prefix()
for idx := ti.prefixIdx(prefix); idx < ti.chunkCount && ti.prefixes[idx] == prefix; idx++ {
ordinal := ti.prefixIdxToOrdinal(idx)
if ti.ordinalSuffixMatches(ordinal, h) {
return ordinal
if ti.EntrySuffixMatches(idx, h) {
return ti.ordinals[idx]
}
}
return ti.chunkCount
}
func (ti onHeapTableIndex) IndexEntry(idx uint32, a *addr) indexEntry {
ord := ti.ordinals[idx]
if a != nil {
binary.BigEndian.PutUint64(a[:], ti.prefixes[idx])
li := uint64(ord) * addrSuffixSize
copy(a[addrPrefixSize:], ti.suffixes[li:li+addrSuffixSize])
}
return indexResult{ti.offsets[ord], ti.lengths[ord]}
}
func (ti onHeapTableIndex) Lookup(h *addr) (indexEntry, bool) {
ord := ti.lookupOrdinal(h)
if ord == ti.chunkCount {
return indexResult{}, false
}
return indexResult{ti.offsets[ord], ti.lengths[ord]}, true
}
func (ti onHeapTableIndex) Prefixes() []uint64 {
return ti.prefixes
}
func (ti onHeapTableIndex) Ordinals() []uint32 {
return ti.ordinals
}
func (i onHeapTableIndex) ChunkCount() uint32 {
return i.chunkCount
}
func (i onHeapTableIndex) TotalUncompressedData() uint64 {
return i.totalUncompressedData
}
func (i onHeapTableIndex) Close() error {
return nil
}
func (i onHeapTableIndex) Clone() tableIndex {
return i
}
// newTableReader parses a valid nbs table byte stream and returns a reader. buff must end with an NBS index
// and footer, though it may contain an unspecified number of bytes before that data. r should allow
// retrieving any desired range of bytes from the table.
func newTableReader(index tableIndex, r tableReaderAt, blockSize uint64) tableReader {
return tableReader{index, r, blockSize}
return tableReader{
index,
index.Prefixes(),
index.ChunkCount(),
index.TotalUncompressedData(),
r,
blockSize,
}
}
// Scan across (logically) two ordered slices of address prefixes.
@@ -286,7 +569,7 @@ func (tr tableReader) hasMany(addrs []hasRecord) (bool, error) {
// TODO: Use findInIndex if (tr.chunkCount - len(addrs)*Log2(tr.chunkCount)) > (tr.chunkCount - len(addrs))
filterIdx := uint32(0)
filterLen := uint32(len(tr.prefixes))
filterLen := uint32(tr.chunkCount)
var remaining bool
for i, addr := range addrs {
@@ -309,7 +592,7 @@ func (tr tableReader) hasMany(addrs []hasRecord) (bool, error) {
// prefixes are equal, so locate and compare against the corresponding suffix
for j := filterIdx; j < filterLen && addr.prefix == tr.prefixes[j]; j++ {
if tr.ordinalSuffixMatches(tr.prefixIdxToOrdinal(j), *addr.a) {
if tr.EntrySuffixMatches(j, addr.a) {
addrs[i].has = true
break
}
@@ -337,32 +620,20 @@ func (tr tableReader) index() (tableIndex, error) {
// returns true iff |h| can be found in this table.
func (tr tableReader) has(h addr) (bool, error) {
ordinal := tr.lookupOrdinal(h)
count, err := tr.count()
if err != nil {
return false, err
}
return ordinal < count, nil
_, ok := tr.Lookup(&h)
return ok, nil
}
// returns the storage associated with |h|, iff present. Returns nil if absent. On success,
// the returned byte slice directly references the underlying storage.
func (tr tableReader) get(ctx context.Context, h addr, stats *Stats) ([]byte, error) {
ordinal := tr.lookupOrdinal(h)
cnt, err := tr.count()
if err != nil {
return nil, err
}
if ordinal == cnt {
e, found := tr.Lookup(&h)
if !found {
return nil, nil
}
offset := tr.offsets[ordinal]
length := uint64(tr.lengths[ordinal])
offset := e.Offset()
length := uint64(e.Length())
buff := make([]byte, length) // TODO: Avoid this allocation for every get
n, err := tr.r.ReadAtWithStats(ctx, buff, int64(offset), stats)
@@ -395,9 +666,9 @@ func (tr tableReader) get(ctx context.Context, h addr, stats *Stats) ([]byte, er
}
type offsetRec struct {
a *addr
ordinal uint32
offset uint64
a *addr
offset uint64
length uint32
}
type offsetRecSlice []offsetRec
@@ -467,7 +738,7 @@ func (tr tableReader) readAtOffsetsWithCB(
}
localStart := rec.offset - readStart
localEnd := localStart + uint64(tr.lengths[rec.ordinal])
localEnd := localStart + uint64(rec.length)
if localEnd > readLength {
return errors.New("length goes past the end")
@@ -579,7 +850,7 @@ func (tr tableReader) getManyAtOffsetsWithReadFunc(
}
rec := offsetRecords[i]
length := tr.lengths[rec.ordinal]
length := rec.length
if batch == nil {
batch = make(offsetRecSlice, 1)
@@ -590,7 +861,7 @@ func (tr tableReader) getManyAtOffsetsWithReadFunc(
continue
}
if newReadEnd, canRead := canReadAhead(rec, tr.lengths[rec.ordinal], readStart, readEnd, tr.blockSize); canRead {
if newReadEnd, canRead := canReadAhead(rec, rec.length, readStart, readEnd, tr.blockSize); canRead {
batch = append(batch, rec)
readEnd = newReadEnd
i++
@@ -665,9 +936,10 @@ func (tr tableReader) findOffsets(reqs []getRecord) (ors offsetRecSlice, remaini
// record all offsets within the table which contain the data required.
for j := filterIdx; j < filterLen && req.prefix == tr.prefixes[j]; j++ {
if tr.ordinalSuffixMatches(tr.prefixIdxToOrdinal(j), *req.a) {
if tr.EntrySuffixMatches(j, req.a) {
reqs[i].found = true
ors = append(ors, offsetRec{req.a, tr.ordinals[j], tr.offsets[tr.ordinals[j]]})
entry := tr.IndexEntry(j, nil)
ors = append(ors, offsetRec{req.a, entry.Offset(), entry.Length()})
break
}
}
@@ -709,7 +981,7 @@ func (tr tableReader) calcReads(reqs []getRecord, blockSize uint64) (reads int,
for i := 0; i < len(offsetRecords); {
rec := offsetRecords[i]
length := tr.lengths[rec.ordinal]
length := rec.length
if !readStarted {
readStarted = true
@@ -720,7 +992,7 @@ func (tr tableReader) calcReads(reqs []getRecord, blockSize uint64) (reads int,
continue
}
if newReadEnd, canRead := canReadAhead(rec, tr.lengths[rec.ordinal], readStart, readEnd, tr.blockSize); canRead {
if newReadEnd, canRead := canReadAhead(rec, rec.length, readStart, readEnd, tr.blockSize); canRead {
readEnd = newReadEnd
i++
continue
@@ -733,31 +1005,16 @@ func (tr tableReader) calcReads(reqs []getRecord, blockSize uint64) (reads int,
}
func (tr tableReader) extract(ctx context.Context, chunks chan<- extractRecord) error {
// Build reverse lookup table from ordinal -> chunk hash
hashes := make(addrSlice, len(tr.prefixes))
for idx, prefix := range tr.prefixes {
ordinal := tr.prefixIdxToOrdinal(uint32(idx))
binary.BigEndian.PutUint64(hashes[ordinal][:], prefix)
li := uint64(ordinal) * addrSuffixSize
copy(hashes[ordinal][addrPrefixSize:], tr.suffixes[li:li+addrSuffixSize])
}
chunkLen := tr.offsets[tr.chunkCount-1] + uint64(tr.lengths[tr.chunkCount-1])
buff := make([]byte, chunkLen)
n, err := tr.r.ReadAtWithStats(ctx, buff, int64(tr.offsets[0]), &Stats{})
if err != nil {
return err
}
if uint64(n) != chunkLen {
return errors.New("did not read all data")
}
sendChunk := func(i uint32) error {
localOffset := tr.offsets[i] - tr.offsets[0]
cmp, err := NewCompressedChunk(hash.Hash(hashes[i]), buff[localOffset:localOffset+uint64(tr.lengths[i])])
sendChunk := func(or offsetRec) error {
buff := make([]byte, or.length)
n, err := tr.r.ReadAtWithStats(ctx, buff, int64(or.offset), &Stats{})
if err != nil {
return err
}
if uint32(n) != or.length {
return errors.New("did not read all data")
}
cmp, err := NewCompressedChunk(hash.Hash(*or.a), buff)
if err != nil {
return err
@@ -769,13 +1026,19 @@ func (tr tableReader) extract(ctx context.Context, chunks chan<- extractRecord)
return err
}
chunks <- extractRecord{a: hashes[i], data: chnk.Data()}
chunks <- extractRecord{a: *or.a, data: chnk.Data()}
return nil
}
var ors offsetRecSlice
for i := uint32(0); i < tr.chunkCount; i++ {
err = sendChunk(i)
a := new(addr)
e := tr.IndexEntry(i, a)
ors = append(ors, offsetRec{a, e.Offset(), e.Length()})
}
sort.Sort(ors)
for _, or := range ors {
err := sendChunk(or)
if err != nil {
return err
}
@@ -785,7 +1048,16 @@ func (tr tableReader) extract(ctx context.Context, chunks chan<- extractRecord)
}
func (tr tableReader) reader(ctx context.Context) (io.Reader, error) {
return io.LimitReader(&readerAdapter{tr.r, 0, ctx}, int64(tr.tableIndex.tableFileSize())), nil
i, _ := tr.index()
return io.LimitReader(&readerAdapter{tr.r, 0, ctx}, int64(i.TableFileSize())), nil
}
func (tr tableReader) Close() error {
return tr.tableIndex.Close()
}
func (tr tableReader) Clone() tableReader {
return tableReader{tr.tableIndex.Clone(), tr.prefixes, tr.chunkCount, tr.totalUncompressedData, tr.r, tr.blockSize}
}
type readerAdapter struct {

72
go/store/nbs/table_reader_test.go
View File

@@ -15,7 +15,12 @@
package nbs
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCompressedChunkIsEmpty(t *testing.T) {
@@ -26,3 +31,70 @@ func TestCompressedChunkIsEmpty(t *testing.T) {
t.Fatal("CompressedChunk{}.IsEmpty() should equal true.")
}
}
func TestParseTableIndex(t *testing.T) {
f, err := os.Open("testdata/0oa7mch34jg1rvghrnhr4shrp2fm4ftd.idx")
require.NoError(t, err)
defer f.Close()
bs, err := ioutil.ReadAll(f)
require.NoError(t, err)
idx, err := parseTableIndex(bs)
require.NoError(t, err)
defer idx.Close()
assert.Equal(t, uint32(596), idx.ChunkCount())
seen := make(map[addr]bool)
for i := uint32(0); i < idx.ChunkCount(); i++ {
var onheapaddr addr
e := idx.IndexEntry(i, &onheapaddr)
if _, ok := seen[onheapaddr]; !ok {
seen[onheapaddr] = true
lookupe, ok := idx.Lookup(&onheapaddr)
assert.True(t, ok)
assert.Equal(t, e.Offset(), lookupe.Offset(), "%v does not match %v for address %v", e, lookupe, onheapaddr)
assert.Equal(t, e.Length(), lookupe.Length())
}
}
}
func TestMMapIndex(t *testing.T) {
f, err := os.Open("testdata/0oa7mch34jg1rvghrnhr4shrp2fm4ftd.idx")
require.NoError(t, err)
defer f.Close()
bs, err := ioutil.ReadAll(f)
require.NoError(t, err)
idx, err := parseTableIndex(bs)
require.NoError(t, err)
defer idx.Close()
mmidx, err := newMmapTableIndex(idx, nil)
require.NoError(t, err)
defer mmidx.Close()
assert.Equal(t, idx.ChunkCount(), mmidx.ChunkCount())
seen := make(map[addr]bool)
for i := uint32(0); i < idx.ChunkCount(); i++ {
var onheapaddr addr
onheapentry := idx.IndexEntry(i, &onheapaddr)
var mmaddr addr
mmentry := mmidx.IndexEntry(i, &mmaddr)
assert.Equal(t, onheapaddr, mmaddr)
assert.Equal(t, onheapentry.Offset(), mmentry.Offset())
assert.Equal(t, onheapentry.Length(), mmentry.Length())
if _, ok := seen[onheapaddr]; !ok {
seen[onheapaddr] = true
mmentry, found := mmidx.Lookup(&onheapaddr)
assert.True(t, found)
assert.Equal(t, onheapentry.Offset(), mmentry.Offset(), "%v does not match %v for address %v", onheapentry, mmentry, onheapaddr)
assert.Equal(t, onheapentry.Length(), mmentry.Length())
}
wrongaddr := onheapaddr
if wrongaddr[19] != 0 {
wrongaddr[19] = 0
_, found := mmidx.Lookup(&wrongaddr)
assert.False(t, found)
}
}
assert.Equal(t, idx.Ordinals(), mmidx.Ordinals())
assert.Equal(t, idx.Prefixes(), mmidx.Prefixes())
assert.Equal(t, idx.TableFileSize(), mmidx.TableFileSize())
assert.Equal(t, idx.TotalUncompressedData(), mmidx.TotalUncompressedData())
}

28
go/store/nbs/table_set.go
View File

@@ -298,25 +298,20 @@ func (ts tableSet) physicalLen() (uint64, error) {
f := func(css chunkSources) (data uint64, err error) {
for _, haver := range css {
index, err := haver.index()
if err != nil {
return 0, err
}
data += indexSize(index.chunkCount)
data += index.offsets[index.chunkCount-1] + (uint64(index.lengths[index.chunkCount-1]))
data += index.TableFileSize()
}
return
}
lenNovel, err := f(ts.novel)
if err != nil {
return 0, err
}
lenUp, err := f(ts.upstream)
if err != nil {
return 0, err
}
@@ -324,6 +319,23 @@ func (ts tableSet) physicalLen() (uint64, error) {
return lenNovel + lenUp, nil
}
func (ts tableSet) Close() error {
var firstErr error
for _, t := range ts.novel {
err := t.Close()
if err != nil && firstErr == nil {
firstErr = err
}
}
for _, t := range ts.upstream {
err := t.Close()
if err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
// Size returns the number of tables in this tableSet.
func (ts tableSet) Size() int {
return len(ts.novel) + len(ts.upstream)
@@ -419,7 +431,7 @@ func (ts tableSet) Rebase(ctx context.Context, specs []tableSpec, stats *Stats)
}
if cnt > 0 {
merged.novel = append(merged.novel, t)
merged.novel = append(merged.novel, t.Clone())
}
}
@@ -455,7 +467,7 @@ func (ts tableSet) Rebase(ctx context.Context, specs []tableSpec, stats *Stats)
return
}
if spec.name == h {
merged.upstream[idx] = existing
merged.upstream[idx] = existing.Clone()
return
}
}

BIN
go/store/nbs/testdata/0oa7mch34jg1rvghrnhr4shrp2fm4ftd.idx vendored Normal file
View File

Binary file not shown.

31
go/store/types/map.go
View File

@@ -387,41 +387,12 @@ func (m Map) IteratorFrom(ctx context.Context, key Value) (MapIterator, error) {
}
func (m Map) IteratorBackFrom(ctx context.Context, key Value) (MapIterator, error) {
cur, err := newCursorAtValue(ctx, m.orderedSequence, key, false, false)
cur, err := newCursorBackFromValue(ctx, m.orderedSequence, key)
if err != nil {
return nil, err
}
// kinda hacky, but a lot less work than implementing newCursorFromValueAtEnd which would have to search back
cur.reverse = true
if !cur.valid() {
cur.advance(ctx)
}
if cur.valid() {
item, err := cur.current()
if err != nil {
return nil, err
}
entry := item.(mapEntry)
isLess, err := entry.key.Less(m.Format(), key)
if err != nil {
return nil, err
}
if !isLess && !key.Equals(entry.key) {
_, err := cur.advance(ctx)
if err != nil {
return nil, err
}
}
} // else: we're at the beginning of the map
return &mapIterator{sequenceIter: cur}, nil
}

70
go/store/types/ordered_sequences.go
View File

@@ -89,6 +89,74 @@ func newCursorAt(ctx context.Context, seq orderedSequence, key orderedKey, forIn
return cur, nil
}
// newCursorBackFromValue returns a reverse sequence iterator starting from the greatest value <= the one given
func newCursorBackFromValue(ctx context.Context, seq orderedSequence, val Value) (*sequenceCursor, error) {
var key orderedKey
if val != nil {
var err error
key, err = newOrderedKey(val, seq.format())
if err != nil {
return nil, err
}
}
return newCursorBackFrom(ctx, seq, key)
}
func newCursorBackFrom(ctx context.Context, seq orderedSequence, key orderedKey) (*sequenceCursor, error) {
var cur *sequenceCursor
for {
idx := 0
cur = newReverseSequenceCursor(cur, seq, idx)
if key != emptyKey {
// If we run off the end of the sequence, start the cursor at the last element.
ok, err := seekTo(cur, key, true)
if err != nil {
return nil, err
}
if !ok {
return cur, nil
}
}
cs, err := cur.getChildSequence(ctx)
if err != nil {
return nil, err
}
if cs == nil {
break
}
seq = cs.(orderedSequence)
}
// If we overshot the key, back off by one. We want the greatest element <= key
if cur.valid() {
currKey, err := cur.seq.(orderedSequence).getKey(cur.idx)
if err != nil {
return nil, err
}
isLess, err := key.Less(cur.seq.format(), currKey)
if err != nil {
return nil, err
}
if isLess {
_, err := cur.advance(ctx)
if err != nil {
return nil, err
}
}
}
d.PanicIfFalse(cur != nil)
return cur, nil
}
func seekTo(cur *sequenceCursor, key orderedKey, lastPositionIfNotFound bool) (bool, error) {
seq := cur.seq.(orderedSequence)
@@ -101,7 +169,7 @@ func seekTo(cur *sequenceCursor, key orderedKey, lastPositionIfNotFound bool) (b
}
seqLen := seq.seqLen()
if cur.idx == seqLen && lastPositionIfNotFound {
if cur.idx == seqLen && seqLen > 0 && lastPositionIfNotFound {
d.PanicIfFalse(cur.idx > 0)
cur.idx--
}

456
go/store/types/ordered_sequences_test.go Executable file
View File

@@ -0,0 +1,456 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type testOrderedSequence struct {
testSequence
}
func (t *testOrderedSequence) getKey(idx int) (orderedKey, error) {
cs, err := t.getChildSequence(nil, idx)
if err != nil {
return orderedKey{}, err
}
if cs != nil {
return cs.(*testOrderedSequence).getKey(cs.seqLen() - 1)
}
return newOrderedKey(t.items[idx].(Value), Format_Default)
}
func (t *testOrderedSequence) getChildSequence(_ context.Context, idx int) (sequence, error) {
child := t.items[idx]
switch child := child.(type) {
case *testOrderedSequence:
return child, nil
default:
return nil, nil
}
}
func (t *testOrderedSequence) search(key orderedKey) (int, error) {
idx, err := SearchWithErroringLess(int(t.Len()), func(i int) (bool, error) {
k, err := t.getKey(i)
if err != nil {
return false, err
}
isLess, err := k.Less(t.format(), key)
if err != nil {
return false, nil
}
return !isLess, nil
})
return idx, err
}
// items is a slice of slices of slices... of Values. Each slice that contains non-value children will be treated as the
// parent slice for N additional children, one for each slice.
func newOrderedTestSequence(items []interface{}) *testOrderedSequence {
if len(items) == 0 {
return &testOrderedSequence{
testSequence: testSequence{nil},
}
}
var sequenceItems []interface{}
for _, item := range items {
if slice, ok := item.([]interface{}); ok {
sequenceItems = append(sequenceItems, newOrderedTestSequence(slice))
} else {
sequenceItems = append(sequenceItems, item)
}
}
return &testOrderedSequence{
testSequence: testSequence{sequenceItems},
}
}
type orderedSequenceTestCase struct {
value Int
expectedVals []Int
}
func newOrderedSequenceTestCase(val int, expectedValues ...int) orderedSequenceTestCase {
expected := make([]Int, len(expectedValues))
for i, value := range expectedValues {
expected[i] = Int(value)
}
return orderedSequenceTestCase{
value: Int(val),
expectedVals: expected,
}
}
func TestNewCursorAtValue(t *testing.T) {
t.Run("single level sequence", func(t *testing.T) {
testSequence := newOrderedTestSequence([]interface{}{
Int(1),
Int(2),
Int(4),
Int(5),
Int(7),
Int(10),
Int(11),
Int(20),
})
testCases := []orderedSequenceTestCase{
newOrderedSequenceTestCase(0, 1, 2, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(1, 1, 2, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(4, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(6, 7, 10, 11, 20),
newOrderedSequenceTestCase(7, 7, 10, 11, 20),
newOrderedSequenceTestCase(8, 10, 11, 20),
newOrderedSequenceTestCase(10, 10, 11, 20),
newOrderedSequenceTestCase(11, 11, 20),
newOrderedSequenceTestCase(12, 20),
newOrderedSequenceTestCase(20, 20),
newOrderedSequenceTestCase(21),
}
for _, tt := range testCases {
t.Run(fmt.Sprintf("%d", tt.value), func(t *testing.T) {
cursor, err := newCursorAtValue(context.Background(), testSequence, tt.value, false, false)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
})
t.Run("2 level sequence", func(t *testing.T) {
testSequence := newOrderedTestSequence([]interface{}{
[]interface{}{
Int(1),
Int(2),
},
[]interface{}{
Int(4),
Int(5),
Int(7),
},
[]interface{}{
Int(10),
Int(11),
},
[]interface{}{
Int(20),
},
})
testCases := []orderedSequenceTestCase{
newOrderedSequenceTestCase(0, 1, 2, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(1, 1, 2, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(4, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(6, 7, 10, 11, 20),
newOrderedSequenceTestCase(7, 7, 10, 11, 20),
newOrderedSequenceTestCase(8, 10, 11, 20),
newOrderedSequenceTestCase(10, 10, 11, 20),
newOrderedSequenceTestCase(11, 11, 20),
newOrderedSequenceTestCase(12, 20),
newOrderedSequenceTestCase(20, 20),
newOrderedSequenceTestCase(21),
}
for _, tt := range testCases {
t.Run(fmt.Sprintf("%d", tt.value), func(t *testing.T) {
cursor, err := newCursorAtValue(context.Background(), testSequence, tt.value, false, false)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
})
t.Run("3 level sequence", func(t *testing.T) {
testSequence := newOrderedTestSequence([]interface{}{
[]interface{}{
[]interface{}{
Int(1),
Int(2),
},
[]interface{}{
Int(4),
Int(5),
Int(7),
},
},
[]interface{}{
[]interface{}{
Int(10),
Int(11),
},
[]interface{}{
Int(20),
},
},
})
testCases := []orderedSequenceTestCase{
newOrderedSequenceTestCase(0, 1, 2, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(1, 1, 2, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(4, 4, 5, 7, 10, 11, 20),
newOrderedSequenceTestCase(6, 7, 10, 11, 20),
newOrderedSequenceTestCase(7, 7, 10, 11, 20),
newOrderedSequenceTestCase(8, 10, 11, 20),
newOrderedSequenceTestCase(10, 10, 11, 20),
newOrderedSequenceTestCase(11, 11, 20),
newOrderedSequenceTestCase(12, 20),
newOrderedSequenceTestCase(20, 20),
newOrderedSequenceTestCase(21),
}
for _, tt := range testCases {
t.Run(fmt.Sprintf("%d", tt.value), func(t *testing.T) {
cursor, err := newCursorAtValue(context.Background(), testSequence, tt.value, false, false)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
})
t.Run("unbalanced tree", func(t *testing.T) {
t.Skip("sequence cursors on unbalanced trees break")
// This breaks, because the sequence cursor algorithm only works on balanced trees. When the leaf nodes are at
// different depths, the calls to cursor.sync() error. Here's a call graph of calling advance() on a cursor on the
// test sequence below at the value 1:
// D -> advance()
// C -> advance()
// B -> advance()
// A -> advance(), idx = 1 (E)
// B.sync(): B = A.getChildSeq() = E
// C.sync(): C = E.getChildSeq() = F
// D.sync(): error (F.getChildSeq() == nil, panic)
// The error is that reassigning the cursor's sequence has the side-effect of effectively reassigning the *parent*
// cursor of any child cursors, since sync() overwrites the sequence pointer of the cursor held by child cursors.
testSequence := newOrderedTestSequence([]interface{}{ // Seq A
[]interface{}{ // interior node, level 1, Seq B
[]interface{}{ // interior node, level 2, Seq C
[]interface{}{ // value node, level 3, Seq D
Int(1),
},
},
},
[]interface{}{ // interior node, level 1, Seq E
[]interface{}{ // value node, level 2, Seq F
Int(10),
Int(11),
}, []interface{}{ // value node, level 2, Seq G
Int(20),
},
},
})
testCases := []orderedSequenceTestCase{
newOrderedSequenceTestCase(0, 1, 10, 11, 20),
newOrderedSequenceTestCase(1, 1, 10, 11, 20),
newOrderedSequenceTestCase(9, 10, 11, 20),
newOrderedSequenceTestCase(10, 10, 11, 20),
newOrderedSequenceTestCase(11, 11, 20),
newOrderedSequenceTestCase(12, 20),
newOrderedSequenceTestCase(20, 20),
newOrderedSequenceTestCase(21),
}
for _, tt := range testCases {
t.Run(fmt.Sprintf("%d", tt.value), func(t *testing.T) {
cursor, err := newCursorAtValue(context.Background(), testSequence, tt.value, false, false)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
})
t.Run("empty sequence", func(t *testing.T) {
tt := newOrderedSequenceTestCase(1)
emptySequence := newOrderedTestSequence(nil)
cursor, err := newCursorAtValue(context.Background(), emptySequence, tt.value, false, false)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
func TestNewCursorBackFromValue(t *testing.T) {
t.Run("single level sequence", func(t *testing.T) {
testSequence := newOrderedTestSequence([]interface{}{
Int(1),
Int(2),
Int(4),
Int(5),
Int(7),
Int(10),
Int(11),
Int(20),
})
testCases := []orderedSequenceTestCase{
newOrderedSequenceTestCase(0),
newOrderedSequenceTestCase(1, 1),
newOrderedSequenceTestCase(4, 4, 2, 1),
newOrderedSequenceTestCase(6, 5, 4, 2, 1),
newOrderedSequenceTestCase(7, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(8, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(10, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(11, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(12, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(20, 20, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(21, 20, 11, 10, 7, 5, 4, 2, 1),
}
for _, tt := range testCases {
t.Run(fmt.Sprintf("%d", tt.value), func(t *testing.T) {
cursor, err := newCursorBackFromValue(context.Background(), testSequence, tt.value)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
})
t.Run("two level sequence", func(t *testing.T) {
testSequence := newOrderedTestSequence([]interface{}{
[]interface{}{
Int(1),
Int(2),
},
[]interface{}{
Int(4),
Int(5),
Int(7),
},
[]interface{}{
Int(10),
Int(11),
},
[]interface{}{
Int(20),
},
})
testCases := []orderedSequenceTestCase{
newOrderedSequenceTestCase(0),
newOrderedSequenceTestCase(1, 1),
newOrderedSequenceTestCase(4, 4, 2, 1),
newOrderedSequenceTestCase(6, 5, 4, 2, 1),
newOrderedSequenceTestCase(7, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(8, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(10, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(11, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(12, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(20, 20, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(21, 20, 11, 10, 7, 5, 4, 2, 1),
}
for _, tt := range testCases {
t.Run(fmt.Sprintf("%d", tt.value), func(t *testing.T) {
cursor, err := newCursorBackFromValue(context.Background(), testSequence, tt.value)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
})
t.Run("three level sequence", func(t *testing.T) {
testSequence := newOrderedTestSequence([]interface{}{
[]interface{}{
[]interface{}{
Int(1),
Int(2),
},
[]interface{}{
Int(4),
Int(5),
Int(7),
},
},
[]interface{}{
[]interface{}{
Int(10),
Int(11),
},
[]interface{}{
Int(20),
},
},
})
testCases := []orderedSequenceTestCase{
newOrderedSequenceTestCase(0),
newOrderedSequenceTestCase(1, 1),
newOrderedSequenceTestCase(4, 4, 2, 1),
newOrderedSequenceTestCase(6, 5, 4, 2, 1),
newOrderedSequenceTestCase(7, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(8, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(10, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(11, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(12, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(20, 20, 11, 10, 7, 5, 4, 2, 1),
newOrderedSequenceTestCase(21, 20, 11, 10, 7, 5, 4, 2, 1),
}
for _, tt := range testCases {
t.Run(fmt.Sprintf("%d", tt.value), func(t *testing.T) {
cursor, err := newCursorBackFromValue(context.Background(), testSequence, tt.value)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
})
t.Run("empty sequence", func(t *testing.T) {
// empty sequence
tt := newOrderedSequenceTestCase(1)
emptySequence := newOrderedTestSequence(nil)
cursor, err := newCursorBackFromValue(context.Background(), emptySequence, tt.value)
require.NoError(t, err)
assertCursorContents(t, cursor, tt.expectedVals)
})
}
func assertCursorContents(t *testing.T, cursor *sequenceCursor, expectedVals []Int) {
more := true
vals := make([]Int, 0)
for {
if !cursor.valid() {
break
}
item, err := cursor.current()
require.NoError(t, err)
intVal, ok := item.(Int)
require.True(t, ok)
vals = append(vals, intVal)
more, err = cursor.advance(context.Background())
require.NoError(t, err)
if !more {
break
}
}
assert.Equal(t, expectedVals, vals)
}

10
go/store/types/sequence_cursor_test.go
View File

@@ -82,9 +82,13 @@ func (ts testSequence) format() *NomsBinFormat {
return Format_7_18
}
func (ts testSequence) getChildSequence(ctx context.Context, idx int) (sequence, error) {
func (ts testSequence) getChildSequence(_ context.Context, idx int) (sequence, error) {
child := ts.items[idx]
return testSequence{child.([]interface{})}, nil
childSlice, ok := child.([]interface{})
if !ok {
return nil, nil
}
return testSequence{childSlice}, nil
}
func (ts testSequence) isLeaf() bool {
@@ -120,7 +124,7 @@ func (ts testSequence) typeOf() (*Type, error) {
}
func (ts testSequence) Len() uint64 {
panic("not reached")
return uint64(len(ts.items))
}
func (ts testSequence) Empty() bool {

2
go/store/util/datetime/date_time_test.go
View File

@@ -93,7 +93,7 @@ func TestUnmarshalInvalid(t *testing.T) {
test(types.Float(42))
test(mustStruct(types.NewStruct(types.Format_7_18, "DateTime", types.StructData{})))
test(mustStruct(types.NewStruct(types.Format_7_18, "DateTime", types.StructData{
"secSinceEpoch": types.String(42),
"secSinceEpoch": types.String("42"),
})))
test(mustStruct(types.NewStruct(types.Format_7_18, "DateTime", types.StructData{
"SecSinceEpoch": types.Float(42),

10
mysql-client-tests-entrypoint.sh Executable file
View File

@@ -0,0 +1,10 @@
#!/bin/sh
echo "Updating dolt config for tests:"
dolt config --global --add metrics.disabled true
dolt config --global --add metrics.host localhost
dolt config --global --add user.name mysql-test-runner
dolt config --global --add user.email mysql-test-runner@liquidata.co
echo "Running mysql-client-tests:"
bats /mysql-client-tests/mysql-client-tests.bats

Some files were not shown because too many files have changed in this diff Show More