mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-22 11:29:06 -05:00
Code Issues Packages Projects Releases 156 Wiki Activity

added string escaping from vitess

Browse Source
This commit is contained in:
Andy Arthur
2020-05-13 16:24:31 -07:00
parent d59749a9e2
commit 69138a3d92
6 changed files with 247 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bats/sql-diff.bats
+14 -7
View File
@@ -489,9 +489,6 @@ SQL
}
@test "diff sql recreates tables with all types" {
skip "This test fails due to type incompatibility between SQL and Noms"
dolt checkout -b firstbranch
dolt checkout -b newbranch
dolt sql <<SQL
@@ -526,7 +523,7 @@ SQL
dolt diff --sql newbranch firstbranch
run dolt diff --sql newbranch firstbranch
[ "$status" -eq 0 ]
["$output" = "" ]
[ "$output" = "" ]
}
@test "sql diff supports all types" {
@@ -633,12 +630,22 @@ CREATE TABLE test (
PRIMARY KEY(pk)
);
SQL
dolt add .
dolt commit -m "created table"
dolt branch other
dolt sql -q "insert into test (pk, c1) values (0, '\\\\')";
dolt sql -q "insert into test (pk, c1) values (1, 'this string ends in backslash\\\\')";
dolt diff --sql > $BATS_TMPDIR/input-$$.sql
run dolt sql < $BATS_TMPDIR/input-$$.sql
skip "backslashes at the end of strings not supported correctly by sql diff"
dolt sql -q "insert into test (pk, c1) values (2, 'this string has \\\"double quotes\\\" in it')";
dolt sql -q "insert into test (pk, c1) values (3, 'it\\'s a contraction y\\'all')";
dolt add .
dolt commit -m "added tricky rows"
dolt checkout other
dolt diff --sql master other > patch.sql
run dolt sql < patch.sql
[ "$status" -eq 0 ]
run dolt diff --sql master
[ "$status" -eq 0 ]
[ "$output" = "" ]
}
@test "sql diff ignores dolt docs" {
go/libraries/doltcore/diff/sql_diff_test.go
+2 -2
View File
@@ -91,7 +91,7 @@ func TestSqlTableDiffAddThenInsert(t *testing.T) {
expectedOutput := sqlfmt.SchemaAsCreateStmt("addTable", sch) + "\n"
expectedOutput = expectedOutput +
"INSERT INTO `addTable` (`id`,`name`,`age`,`is_married`,`title`) " +
"VALUES (\"00000000-0000-0000-0000-000000000000\",\"Big Billy\",77,FALSE,\"Doctor\");\n"
"VALUES ('00000000-0000-0000-0000-000000000000','Big Billy',77,FALSE,'Doctor');\n"
assert.Equal(t, expectedOutput, stringWr.String())
}
@@ -151,6 +151,6 @@ func TestSqlTableDiffRenameChangedTable(t *testing.T) {
expectedOutput = expectedOutput +
sqlfmt.SchemaAsCreateStmt("newTableName", sch) + "\n" +
"INSERT INTO `newTableName` (`id`,`name`,`age`,`is_married`,`title`) " +
"VALUES (\"00000000-0000-0000-0000-000000000000\",\"Big Billy\",77,FALSE,\"Doctor\");\n"
"VALUES ('00000000-0000-0000-0000-000000000000','Big Billy',77,FALSE,'Doctor');\n"
assert.Equal(t, expectedOutput, stringWr.String())
}
go/libraries/doltcore/sqle/sqlfmt/sqlstatements.go → go/libraries/doltcore/sqle/sqlfmt/row_fmt.go
+42 -147
View File
@@ -15,10 +15,12 @@
package sqlfmt
import (
"bytes"
"fmt"
"github.com/google/uuid"
"strings"
"github.com/google/uuid"
"vitess.io/vitess/go/sqltypes"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/row"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/schema"
@@ -26,142 +28,18 @@ import (
"github.com/liquidata-inc/dolt/go/store/types"
)
const doubleQuot = `"`
const singleQuote = `'`
// Quotes the identifier given with backticks.
func QuoteIdentifier(s string) string {
return "`" + s + "`"
}
// QuoteString quotes the given string with apostrophes, and escapes any contained within the string.
func QuoteString(s string) string {
// QuoteComment quotes the given string with apostrophes, and escapes any contained within the string.
func QuoteComment(s string) string {
return `'` + strings.ReplaceAll(s, `'`, `\'`) + `'`
}
// SchemaAsCreateStmt takes a Schema and returns a string representing a SQL create table command that could be used to
// create this table
func SchemaAsCreateStmt(tableName string, sch schema.Schema) string {
sb := &strings.Builder{}
fmt.Fprintf(sb, "CREATE TABLE %s (\n", QuoteIdentifier(tableName))
firstLine := true
_ = sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
if firstLine {
firstLine = false
} else {
sb.WriteString(",\n")
}
s := FmtCol(2, 0, 0, col)
sb.WriteString(s)
return false, nil
})
firstPK := true
err := sch.GetPKCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
if firstPK {
sb.WriteString(",\n PRIMARY KEY (")
firstPK = false
} else {
sb.WriteRune(',')
}
sb.WriteString(QuoteIdentifier(col.Name))
return false, nil
})
// TODO: fix panics
if err != nil {
panic(err)
}
sb.WriteRune(')')
for _, index := range sch.Indexes().AllIndexes() {
sb.WriteString(",\n ")
if index.IsUnique() {
sb.WriteString("UNIQUE ")
}
sb.WriteString("INDEX ")
sb.WriteString(QuoteIdentifier(index.Name()))
sb.WriteString(" (")
for i, indexColName := range index.ColumnNames() {
if i != 0 {
sb.WriteRune(',')
}
sb.WriteString(QuoteIdentifier(indexColName))
}
sb.WriteRune(')')
if len(index.Comment()) > 0 {
sb.WriteString(" COMMENT ")
sb.WriteString(QuoteString(index.Comment()))
}
}
sb.WriteString("\n);")
return sb.String()
}
func DropTableStmt(tableName string) string {
var b strings.Builder
b.WriteString("DROP TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(";")
return b.String()
}
func DropTableIfExistsStmt(tableName string) string {
var b strings.Builder
b.WriteString("DROP TABLE IF EXISTS ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(";")
return b.String()
}
func AlterTableAddColStmt(tableName string, newColDef string) string {
var b strings.Builder
b.WriteString("ALTER TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(" ADD ")
b.WriteString(newColDef)
b.WriteRune(';')
return b.String()
}
func AlterTableDropColStmt(tableName string, oldColName string) string {
var b strings.Builder
b.WriteString("ALTER TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(" DROP ")
b.WriteString(QuoteIdentifier(oldColName))
b.WriteRune(';')
return b.String()
}
func AlterTableRenameColStmt(tableName string, oldColName string, newColName string) string {
var b strings.Builder
b.WriteString("ALTER TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(" RENAME COLUMN ")
b.WriteString(QuoteIdentifier(oldColName))
b.WriteString(" TO ")
b.WriteString(QuoteIdentifier(newColName))
b.WriteRune(';')
return b.String()
}
func RenameTableStmt(fromName string, toName string) string {
var b strings.Builder
b.WriteString("RENAME TABLE ")
b.WriteString(QuoteIdentifier(fromName))
b.WriteString(" TO ")
b.WriteString(QuoteIdentifier(toName))
b.WriteString(";")
return b.String()
}
func RowAsInsertStmt(r row.Row, tableName string, tableSch schema.Schema) (string, error) {
var b strings.Builder
b.WriteString("INSERT INTO ")
@@ -191,7 +69,8 @@ func RowAsInsertStmt(r row.Row, tableName string, tableSch schema.Schema) (strin
if seenOne {
b.WriteRune(',')
}
sqlString, err := valueAsSqlString(val)
col, _ := tableSch.GetAllCols().GetByTag(tag)
sqlString, err := valueAsSqlString(col.TypeInfo, val)
if err != nil {
return true, err
}
@@ -217,12 +96,12 @@ func RowAsDeleteStmt(r row.Row, tableName string, tableSch schema.Schema) (strin
b.WriteString(" WHERE (")
seenOne := false
_, err := r.IterSchema(tableSch, func(tag uint64, val types.Value) (stop bool, err error) {
col := tableSch.GetAllCols().TagToCol[tag]
col, _ := tableSch.GetAllCols().GetByTag(tag)
if col.IsPartOfPK {
if seenOne {
b.WriteString(" AND ")
}
sqlString, err := valueAsSqlString(val)
sqlString, err := valueAsSqlString(col.TypeInfo, val)
if err != nil {
return true, err
}
@@ -251,12 +130,12 @@ func RowAsUpdateStmt(r row.Row, tableName string, tableSch schema.Schema) (strin
b.WriteString("SET ")
seenOne := false
_, err := r.IterSchema(tableSch, func(tag uint64, val types.Value) (stop bool, err error) {
col := tableSch.GetAllCols().TagToCol[tag]
col, _ := tableSch.GetAllCols().GetByTag(tag)
if !col.IsPartOfPK {
if seenOne {
b.WriteRune(',')
}
sqlString, err := valueAsSqlString(val)
sqlString, err := valueAsSqlString(col.TypeInfo, val)
if err != nil {
return true, err
}
@@ -275,12 +154,12 @@ func RowAsUpdateStmt(r row.Row, tableName string, tableSch schema.Schema) (strin
b.WriteString(" WHERE (")
seenOne = false
_, err = r.IterSchema(tableSch, func(tag uint64, val types.Value) (stop bool, err error) {
col := tableSch.GetAllCols().TagToCol[tag]
col, _:= tableSch.GetAllCols().GetByTag(tag)
if col.IsPartOfPK {
if seenOne {
b.WriteString(" AND ")
}
sqlString, err := valueAsSqlString(val)
sqlString, err := valueAsSqlString(col.TypeInfo, val)
if err != nil {
return true, err
}
@@ -300,29 +179,45 @@ func RowAsUpdateStmt(r row.Row, tableName string, tableSch schema.Schema) (strin
return b.String(), nil
}
func valueAsSqlString(value types.Value) (string, error) {
func valueAsSqlString(ti typeinfo.TypeInfo, value types.Value) (string, error) {
if types.IsNull(value) {
return "NULL", nil
}
switch value.Kind() {
case types.BoolKind:
switch ti.GetTypeIdentifier() {
case typeinfo.BoolTypeIdentifier:
// todo: unclear if we want this to output with "TRUE/FALSE" or 1/0
if value.(types.Bool) {
return "TRUE", nil
} else {
return "FALSE", nil
}
case types.UUIDKind:
return doubleQuot + uuid.UUID(value.(types.UUID)).String() + doubleQuot, nil
case types.StringKind:
s := string(value.(types.String))
s = strings.ReplaceAll(s, doubleQuot, "\\\"")
return doubleQuot + s + doubleQuot, nil
return "FALSE", nil
case typeinfo.UuidTypeIdentifier:
// todo: typeinfo.UuidTypeIdentifier should handle this
u := uuid.UUID(value.(types.UUID))
return singleQuote + u.String() + singleQuote, nil
case typeinfo.VarStringTypeIdentifier:
s, ok := value.(types.String)
if !ok {
return "", fmt.Errorf("typeinfo.VarStringTypeIdentifier is not types.String")
}
return quoteAndEscapeString(string(s)), nil
default:
str, err := typeinfo.FromKind(value.Kind()).FormatValue(value)
str, err := ti.FormatValue(value)
if err != nil {
return "", err
}
return *str, nil
}
}
// todo: this is a hack, varstring should handle this
func quoteAndEscapeString(s string) string {
buf := &bytes.Buffer{}
v, err := sqltypes.NewValue(sqltypes.VarChar, []byte(s))
if err != nil {
panic(err)
}
v.EncodeSQL(buf)
return buf.String()
}
go/libraries/doltcore/sqle/sqlfmt/sqlstatements_test.go → go/libraries/doltcore/sqle/sqlfmt/row_fmt_test.go
+61 -9
View File
@@ -15,6 +15,8 @@
package sqlfmt
import (
"github.com/liquidata-inc/dolt/go/libraries/doltcore/schema/typeinfo"
"github.com/stretchr/testify/require"
"testing"
"github.com/google/uuid"
@@ -105,22 +107,19 @@ func TestRowAsInsertStmt(t *testing.T) {
name: "simple row",
row: dtestutils.NewTypedRow(id, "some guy", 100, false, strPointer("normie")),
sch: dtestutils.TypedSchema,
expectedOutput: "INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) " +
`VALUES ("00000000-0000-0000-0000-000000000000","some guy",100,FALSE,"normie");`,
expectedOutput: "INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) VALUES ('00000000-0000-0000-0000-000000000000','some guy',100,FALSE,'normie');",
},
{
name: "embedded quotes",
row: dtestutils.NewTypedRow(id, `It's "Mister Perfect" to you`, 100, false, strPointer("normie")),
sch: dtestutils.TypedSchema,
expectedOutput: "INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) " +
`VALUES ("00000000-0000-0000-0000-000000000000","It's \"Mister Perfect\" to you",100,FALSE,"normie");`,
expectedOutput: "INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) VALUES ('00000000-0000-0000-0000-000000000000','It\\'s \\\"Mister Perfect\\\" to you',100,FALSE,'normie');",
},
{
name: "null values",
row: dtestutils.NewTypedRow(id, "some guy", 100, false, nil),
sch: dtestutils.TypedSchema,
expectedOutput: "INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) " +
`VALUES ("00000000-0000-0000-0000-000000000000","some guy",100,FALSE,NULL);`,
expectedOutput: "INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) VALUES ('00000000-0000-0000-0000-000000000000','some guy',100,FALSE,NULL);",
},
}
@@ -179,19 +178,19 @@ func TestRowAsUpdateStmt(t *testing.T) {
name: "simple row",
row: dtestutils.NewTypedRow(id, "some guy", 100, false, strPointer("normie")),
sch: dtestutils.TypedSchema,
expectedOutput: "UPDATE `people` SET `name`=\"some guy\",`age`=100,`is_married`=FALSE,`title`=\"normie\" WHERE (`id`=\"00000000-0000-0000-0000-000000000000\");",
expectedOutput: "UPDATE `people` SET `name`='some guy',`age`=100,`is_married`=FALSE,`title`='normie' WHERE (`id`='00000000-0000-0000-0000-000000000000');",
},
{
name: "embedded quotes",
row: dtestutils.NewTypedRow(id, `It's "Mister Perfect" to you`, 100, false, strPointer("normie")),
sch: dtestutils.TypedSchema,
expectedOutput: "UPDATE `people` SET `name`=\"It's \\\"Mister Perfect\\\" to you\",`age`=100,`is_married`=FALSE,`title`=\"normie\" WHERE (`id`=\"00000000-0000-0000-0000-000000000000\");",
expectedOutput: "UPDATE `people` SET `name`='It\\'s \\\"Mister Perfect\\\" to you',`age`=100,`is_married`=FALSE,`title`='normie' WHERE (`id`='00000000-0000-0000-0000-000000000000');",
},
{
name: "null values",
row: dtestutils.NewTypedRow(id, "some guy", 100, false, nil),
sch: dtestutils.TypedSchema,
expectedOutput: "UPDATE `people` SET `name`=\"some guy\",`age`=100,`is_married`=FALSE,`title`=NULL WHERE (`id`=\"00000000-0000-0000-0000-000000000000\");",
expectedOutput: "UPDATE `people` SET `name`='some guy',`age`=100,`is_married`=FALSE,`title`=NULL WHERE (`id`='00000000-0000-0000-0000-000000000000');",
},
}
@@ -216,6 +215,59 @@ func TestRowAsUpdateStmt(t *testing.T) {
}
}
func TestValueAsSqlString(t *testing.T) {
tu, _ := uuid.Parse("00000000-0000-0000-0000-000000000000")
tests := []struct{
name string
val types.Value
ti typeinfo.TypeInfo
exp string
}{
{
name: "bool(true)",
val: types.Bool(true),
ti: typeinfo.BoolType,
exp: "TRUE",
},
{
name: "bool(false)",
val: types.Bool(false),
ti: typeinfo.BoolType,
exp: "FALSE",
},
{
name: "uuid",
val: types.UUID(tu),
ti: typeinfo.UuidType,
exp: "'00000000-0000-0000-0000-000000000000'",
},
{
name: "string",
val: types.String("leviosa"),
ti: typeinfo.StringDefaultType,
exp: "'leviosa'",
},
{
// borrowed from vitess
name: "escape string",
val: types.String("\x00'\"\b\n\r\t\x1A\\"),
ti: typeinfo.StringDefaultType,
exp: "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
act, err := valueAsSqlString(test.ti, test.val)
require.NoError(t, err)
assert.Equal(t, test.exp, act)
})
}
}
func strPointer(s string) *string {
return &s
}
go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go
+126
View File
@@ -16,6 +16,7 @@ package sqlfmt
import (
"fmt"
"strings"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/schema"
)
@@ -59,3 +60,128 @@ func FmtColPrimaryKey(indent int, colStr string) string {
func FmtColTagComment(tag uint64) string {
return fmt.Sprintf("%s%d", TagCommentPrefix, tag)
}
// SchemaAsCreateStmt takes a Schema and returns a string representing a SQL create table command that could be used to
// create this table
func SchemaAsCreateStmt(tableName string, sch schema.Schema) string {
sb := &strings.Builder{}
fmt.Fprintf(sb, "CREATE TABLE %s (\n", QuoteIdentifier(tableName))
firstLine := true
_ = sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
if firstLine {
firstLine = false
} else {
sb.WriteString(",\n")
}
s := FmtCol(2, 0, 0, col)
sb.WriteString(s)
return false, nil
})
firstPK := true
err := sch.GetPKCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
if firstPK {
sb.WriteString(",\n PRIMARY KEY (")
firstPK = false
} else {
sb.WriteRune(',')
}
sb.WriteString(QuoteIdentifier(col.Name))
return false, nil
})
// TODO: fix panics
if err != nil {
panic(err)
}
sb.WriteRune(')')
for _, index := range sch.Indexes().AllIndexes() {
sb.WriteString(",\n ")
if index.IsUnique() {
sb.WriteString("UNIQUE ")
}
sb.WriteString("INDEX ")
sb.WriteString(QuoteIdentifier(index.Name()))
sb.WriteString(" (")
for i, indexColName := range index.ColumnNames() {
if i != 0 {
sb.WriteRune(',')
}
sb.WriteString(QuoteIdentifier(indexColName))
}
sb.WriteRune(')')
if len(index.Comment()) > 0 {
sb.WriteString(" COMMENT ")
sb.WriteString(QuoteComment(index.Comment()))
}
}
sb.WriteString("\n);")
return sb.String()
}
func DropTableStmt(tableName string) string {
var b strings.Builder
b.WriteString("DROP TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(";")
return b.String()
}
func DropTableIfExistsStmt(tableName string) string {
var b strings.Builder
b.WriteString("DROP TABLE IF EXISTS ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(";")
return b.String()
}
func AlterTableAddColStmt(tableName string, newColDef string) string {
var b strings.Builder
b.WriteString("ALTER TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(" ADD ")
b.WriteString(newColDef)
b.WriteRune(';')
return b.String()
}
func AlterTableDropColStmt(tableName string, oldColName string) string {
var b strings.Builder
b.WriteString("ALTER TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(" DROP ")
b.WriteString(QuoteIdentifier(oldColName))
b.WriteRune(';')
return b.String()
}
func AlterTableRenameColStmt(tableName string, oldColName string, newColName string) string {
var b strings.Builder
b.WriteString("ALTER TABLE ")
b.WriteString(QuoteIdentifier(tableName))
b.WriteString(" RENAME COLUMN ")
b.WriteString(QuoteIdentifier(oldColName))
b.WriteString(" TO ")
b.WriteString(QuoteIdentifier(newColName))
b.WriteRune(';')
return b.String()
}
func RenameTableStmt(fromName string, toName string) string {
var b strings.Builder
b.WriteString("RENAME TABLE ")
b.WriteString(QuoteIdentifier(fromName))
b.WriteString(" TO ")
b.WriteString(QuoteIdentifier(toName))
b.WriteString(";")
return b.String()
}
go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go
+2 -2
View File
@@ -65,9 +65,9 @@ func TestEndToEnd(t *testing.T) {
sch: dtestutils.TypedSchema,
expectedOutput: dropCreateStatement + "\n" +
"INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) " +
`VALUES ("00000000-0000-0000-0000-000000000000","some guy",100,FALSE,"normie");` + "\n" +
`VALUES ('00000000-0000-0000-0000-000000000000','some guy',100,FALSE,'normie');` + "\n" +
"INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) " +
`VALUES ("00000000-0000-0000-0000-000000000000","guy personson",0,TRUE,"officially a person");` + "\n",
`VALUES ('00000000-0000-0000-0000-000000000000','guy personson',0,TRUE,'officially a person');` + "\n",
},
{
name: "no rows",