Mostly done

go/cmd/dolt/commands/ci/run.go

@@ -17,13 +17,14 @@ package ci
 import (
 	"context"
 	"fmt"
-
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/cmd/dolt/commands"
 	"github.com/dolthub/dolt/go/cmd/dolt/errhand"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env/actions/dolt_ci"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
+	"github.com/dolthub/go-mysql-server/sql"
+	"github.com/fatih/color"
 )
 
 var runDocs = cli.CommandDocumentationContent{
@@ -76,7 +77,7 @@ func (cmd RunCmd) Exec(ctx context.Context, commandStr string, args []string, _
 
 	// Check if workflow name provided
 	if len(args) == 0 {
-		return commands.HandleVErrAndExitCode(errhand.VerboseErrorFromError(fmt.Errorf("workflow name is required")),
+		return commands.HandleVErrAndExitCode(errhand.VerboseErrorFromError(fmt.Errorf("must specify workflow name")),
 			usage)
 	}
 	workflowName := args[0]
@@ -102,6 +103,7 @@ func (cmd RunCmd) Exec(ctx context.Context, commandStr string, args []string, _
 	if err != nil {
 		return commands.HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
 	}
+
 	wm := dolt_ci.NewWorkflowManager(name, email, queryist.Query)
 
 	config, err := wm.GetWorkflowConfig(sqlCtx, workflowName)
@@ -109,13 +111,82 @@ func (cmd RunCmd) Exec(ctx context.Context, commandStr string, args []string, _
 		return commands.HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
 	}
 
-	savedQueries, err := 
-
-	for _, job := range config.Jobs {
-		for _, step := range job.Steps {
-			//Do something
-		}
+	savedQueries, err := getSavedQueries(sqlCtx, queryist)
+	if err != nil {
+		return commands.HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
 	}
+	cli.Println(color.CyanString("Running workflow: %s", workflowName))
+	queryAndPrint(sqlCtx, queryist, config, savedQueries)
 
 	return 0
 }
+
+// QueryAndPrint iterates through the jobs and steps for the given config, the runs each saved query and given assertion
+func queryAndPrint(sqlCtx *sql.Context, queryist cli.Queryist, config *dolt_ci.WorkflowConfig, savedQueries map[string]string) {
+	for _, job := range config.Jobs {
+		cli.Println(color.GreenString("Running job: %s", job.Name.Value))
+		for _, step := range job.Steps {
+			cli.Printf("Step: %s - ", step.Name.Value)
+
+			rows, err := runCIQuery(queryist, sqlCtx, step, savedQueries)
+			if err == nil {
+				err = assertQueries(rows, step)
+			}
+
+			if err != nil {
+				cli.Println("FAIL")
+				cli.Println(color.RedString("%s", err))
+			} else {
+				cli.Println("PASS")
+			}
+		}
+	}
+}
+
+func runCIQuery(queryist cli.Queryist, sqlCtx *sql.Context, step dolt_ci.Step, savedQueries map[string]string) ([]sql.Row, error) {
+	query := savedQueries[step.SavedQueryName.Value]
+	if query == "" {
+		return nil, fmt.Errorf("Could not find saved query: %s", step.SavedQueryName.Value)
+	}
+
+	rows, err := commands.GetRowsForSql(queryist, sqlCtx, query)
+	if err != nil {
+		return nil, fmt.Errorf("Query error: %s", err.Error())
+	}
+
+	return rows, nil
+}
+
+func assertQueries(rows []sql.Row, step dolt_ci.Step) error {
+	var colCount int64
+	rowCount := int64(len(rows))
+	if rowCount > 0 {
+		colCount = int64(len(rows[0]))
+	}
+
+	var colAssertError, rowAssertError string
+	rowCompType, expectedRows, err := dolt_ci.ParseSavedQueryExpectedResultString(step.ExpectedRows.Value)
+	if rowCompType != dolt_ci.WorkflowSavedQueryExpectedRowColumnComparisonTypeUnspecified {
+		err = dolt_ci.ValidateQueryExpectedRowOrColumnCount(rowCount, expectedRows, rowCompType, "row")
+		if err != nil {
+			rowAssertError = fmt.Sprintf("Assertion failed: %s", err.Error())
+		}
+	}
+	colCompType, expectedCols, err := dolt_ci.ParseSavedQueryExpectedResultString(step.ExpectedColumns.Value)
+	if colCompType != dolt_ci.WorkflowSavedQueryExpectedRowColumnComparisonTypeUnspecified {
+		err = dolt_ci.ValidateQueryExpectedRowOrColumnCount(colCount, expectedCols, colCompType, "column")
+		if err != nil {
+			colAssertError = fmt.Sprintf("Assertion failed: %s", err.Error())
+		}
+	}
+
+	var errStr, newLine string
+	if colAssertError != "" && rowAssertError != "" {
+		newLine = "\n"
+	}
+	errStr = fmt.Sprintf("%s%s%s", colAssertError, rowAssertError, newLine)
+	if errStr != "" {
+		return fmt.Errorf(errStr)
+	}
+	return nil
+}

go/libraries/doltcore/env/actions/dolt_ci/utils.go

@@ -58,3 +58,36 @@ func ParseSavedQueryExpectedResultString(str string) (WorkflowSavedQueryExpected
 	}
 	return 0, 0, fmt.Errorf("unable to parse comparison string: %s", str)
 }
+
+func ValidateQueryExpectedRowOrColumnCount(countReal int64, countExpected int64, comp WorkflowSavedQueryExpectedRowColumnComparisonType, RowColumnType string) error {
+	switch comp {
+	case WorkflowSavedQueryExpectedRowColumnComparisonTypeEquals:
+		if countReal != countExpected {
+			return fmt.Errorf("expected %s count %d, got %d", RowColumnType, countExpected, countReal)
+		}
+	case WorkflowSavedQueryExpectedRowColumnComparisonTypeNotEquals:
+		if countReal == countExpected {
+			return fmt.Errorf("expected %s count not %d and got %d", RowColumnType, countExpected, countReal)
+		}
+	case WorkflowSavedQueryExpectedRowColumnComparisonTypeGreaterThan:
+		if countReal <= countExpected {
+			return fmt.Errorf("expected %s count greater than %d, got %d", RowColumnType, countExpected, countReal)
+		}
+	case WorkflowSavedQueryExpectedRowColumnComparisonTypeGreaterThanOrEqual:
+		if countReal < countExpected {
+			return fmt.Errorf("expected %s count greater than or equal to %d, got %d", RowColumnType, countExpected, countReal)
+		}
+	case WorkflowSavedQueryExpectedRowColumnComparisonTypeLessThan:
+		if countReal >= countExpected {
+			return fmt.Errorf("expected %s count less than %d, got %d", RowColumnType, countExpected, countReal)
+		}
+	case WorkflowSavedQueryExpectedRowColumnComparisonTypeLessThanOrEqual:
+		if countReal > countExpected {
+			return fmt.Errorf("expected %s count less than or equal to %d, got %d", RowColumnType, countExpected, countReal)
+		}
+	default:
+		return fmt.Errorf("no assertion run")
+	}
+
+	return nil
+}

integration-tests/bats/ci.bats

@@ -459,11 +459,148 @@ EOF
     [[ "$output" =~ "cannot find job with name: invalid job" ]] || false
 }
 
+@test "ci: run with expected rows" {
+    cat > workflow.yaml <<EOF
+name: workflow
+on:
+  push: {}
+jobs:
+  - name: verify initial commit
+    steps:
+      - name: "verify initial commit"
+        saved_query_name: check dolt commit
+        expected_rows: "== 3"
+EOF
+    dolt ci init
+    dolt ci import ./workflow.yaml
+    dolt sql --save "check dolt commit" -q "select * from dolt_commits;"
+    run dolt ci run "workflow"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Running workflow: workflow" ]] || false
+    [[ "$output" =~ "Running job: verify initial commit" ]] || false
+    [[ "$output" =~ "Step: verify initial commit - PASS" ]] || false
+}
 
-# TESTS FOR CI RUN
-# RUN W EXPECTED COLUMNS
-# RUN W EXPECTED ROWS
-# RUN WITH EXPECTED ROWS AND COLUMNS
-# ONLY/ALL ASSERTIONS FAILS
-# SOME ASSERTIONS PASS, SOME ASSERTIONS FAIL
-# RUN ON INVALID WORKFLOW
+@test "ci: ci run with expected columns" {
+    cat > workflow.yaml <<EOF
+name: workflow
+on:
+  push: {}
+jobs:
+  - name: verify initial commit
+    steps:
+      - name: "verify initial commit"
+        saved_query_name: check dolt commit
+        expected_columns: "== 5"
+EOF
+    dolt ci init
+    dolt ci import ./workflow.yaml
+    dolt sql --save "check dolt commit" -q "select * from dolt_commits;"
+    run dolt ci run "workflow"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Running workflow: workflow" ]] || false
+    [[ "$output" =~ "Running job: verify initial commit" ]] || false
+    [[ "$output" =~ "Step: verify initial commit - PASS" ]] || false
+}
+
+@test "ci: each assertion type can be used" {
+    cat > workflow.yaml <<EOF
+name: workflow
+on:
+  push: {}
+jobs:
+  - name: check comparisons
+    steps:
+      - name: equals comp
+        saved_query_name: main
+        expected_columns: "== 5"
+      - name: not equals comp
+        saved_query_name: main
+        expected_columns: "!= 1"
+      - name: greater than comp
+        saved_query_name: main
+        expected_columns: "> 4"
+      - name: greater or equal than comp
+        saved_query_name: main
+        expected_columns: ">= 5"
+      - name: less than comp
+        saved_query_name: main
+        expected_columns: "< 6"
+      - name: less or equal than comp
+        saved_query_name: main
+        expected_columns: "<= 5"
+EOF
+    dolt ci init
+    dolt ci import ./workflow.yaml
+    dolt sql --save "main" -q "select * from dolt_commits;"
+    run dolt ci run "workflow"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Running workflow: workflow" ]] || false
+    [[ "$output" =~ "Step: equals comp - PASS" ]] || false
+    [[ "$output" =~ "Step: not equals comp - PASS" ]] || false
+    [[ "$output" =~ "Step: greater than comp - PASS" ]] || false
+    [[ "$output" =~ "Step: greater or equal than comp - PASS" ]] || false
+    [[ "$output" =~ "Step: less than comp - PASS" ]] || false
+    [[ "$output" =~ "Step: less or equal than comp - PASS" ]] || false
+}
+
+@test "ci: saved queries fail with ci run" {
+    cat > workflow.yaml <<EOF
+name: workflow
+on:
+  push: {}
+jobs:
+  - name: "bad query assertions"
+    steps:
+      - name: expect rows
+        saved_query_name: main
+        expected_rows: "== 2"
+      - name: expect columns
+        saved_query_name: main
+        expected_columns: "< 5"
+EOF
+    dolt ci init
+    dolt ci import ./workflow.yaml
+    dolt sql --save "main" -q "select * from dolt_commits;"
+    run dolt ci run "workflow"
+    echo "$output"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Running workflow: workflow" ]] || false
+    [[ "$output" =~ "Step: expect rows - FAIL" ]] || false
+    [[ "$output" =~ "Assertion failed: expected row count 2, got 3" ]] || false
+    [[ "$output" =~ "Step: expect columns - FAIL" ]] || false
+    [[ "$output" =~ "Assertion failed: expected column count less than 5, got 5" ]] || false
+}
+
+@test "ci: ci run fails on invalid query" {
+    cat > workflow.yaml <<EOF
+name: workflow
+on:
+  push: {}
+jobs:
+  - name: "bad saved queries"
+    steps:
+      - name: should fail, bad table name
+        saved_query_name: invalid table
+EOF
+    dolt ci init
+    dolt ci import ./workflow.yaml
+    dolt sql -q "create table invalid (i int);"
+    dolt sql --save "invalid table" -q "select * from invalid;"
+    dolt sql -q "drop table invalid;"
+    run dolt ci run "workflow"
+    echo "$output"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Running workflow: workflow" ]] || false
+    [[ "$output" =~ "Step: should fail, bad table name - FAIL" ]] || false
+    [[ "$output" =~ "Query error: table not found: invalid" ]] || false
+}
+
+@test "ci: ci run fails on invalid workflow name" {
+    dolt ci init
+    run dolt ci run "invalid"
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "workflow not found" ]] || false
+}
+
+# test ci with workflow with invalid query name