[tests-only][full-ci] Retry ocis startup with ociswrapper (#6768)

* (ociswrapper): retry starting ocis if failed

* (ociswrapper): kill procces and start again

* (ociswrapper): make retry configurable via cli

tests/ociswrapper/cmd/cmd.go

@@ -31,6 +31,7 @@ func serveCmd() *cobra.Command {
 			// set configs
 			ocisConfig.Set("bin", cmd.Flag("bin").Value.String())
 			ocisConfig.Set("url", cmd.Flag("url").Value.String())
+			ocisConfig.Set("retry", cmd.Flag("retry").Value.String())
 		},
 	}
 
@@ -38,6 +39,7 @@ func serveCmd() *cobra.Command {
 	serveCmd.Flags().SortFlags = false
 	serveCmd.Flags().StringP("bin", "", ocisConfig.Get("bin"), "Full oCIS binary path")
 	serveCmd.Flags().StringP("url", "", ocisConfig.Get("url"), "oCIS server url")
+	serveCmd.Flags().StringP("retry", "", ocisConfig.Get("retry"), "Number of retries to start oCIS server")
 	serveCmd.Flags().StringP("port", "p", wrapperConfig.Get("port"), "Wrapper API server port")
 
 	return serveCmd

tests/ociswrapper/ocis/config/config.go

@@ -1,8 +1,9 @@
 package config
 
 var config = map[string]string{
-	"bin": "/usr/bin/ocis",
-	"url": "https://localhost:9200",
+	"bin":   "/usr/bin/ocis",
+	"url":   "https://localhost:9200",
+	"retry": "5",
 }
 
 func Set(key string, value string) {

tests/ociswrapper/ocis/ocis.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"strconv"
 	"time"
 
 	"ociswrapper/common"
@@ -15,9 +16,12 @@ import (
 )
 
 var cmd *exec.Cmd
+var retryCount = 0
 
 func Start(envMap map[string]any) {
-	defer common.Wg.Done()
+	if retryCount == 0 {
+		defer common.Wg.Done()
+	}
 
 	cmd = exec.Command(config.Get("bin"), "server")
 	cmd.Env = os.Environ()
@@ -52,6 +56,15 @@ func Start(envMap map[string]any) {
 	for stdoutScanner.Scan() {
 		m := stdoutScanner.Text()
 		fmt.Println(m)
+		retryCount++
+
+		maxRetry, _ := strconv.Atoi(config.Get("retry"))
+		if retryCount <= maxRetry {
+			fmt.Println(fmt.Sprintf("Retry starting oCIS server... (retry %v)", retryCount))
+			// Stop and start again
+			Stop()
+			Start(envMap)
+		}
 	}
 }
 
@@ -60,13 +73,16 @@ func Stop() {
 	if err != nil {
 		log.Panic("Cannot kill oCIS server")
 	}
+	cmd.Wait()
 }
 
 func WaitForConnection() bool {
 	transport := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
+	// 5 seconds timeout
 	timeoutValue := 5 * time.Second
+
 	client := http.Client{
 		Timeout:   timeoutValue,
 		Transport: transport,
@@ -77,7 +93,7 @@ func WaitForConnection() bool {
 	for {
 		select {
 		case <-timeout:
-			fmt.Println(fmt.Sprintf("Timeout waiting for oCIS server [%f] seconds", timeoutValue.Seconds()))
+			fmt.Println(fmt.Sprintf("%v seconds timeout waiting for oCIS server", int64(timeoutValue.Seconds())))
 			return false
 		default:
 			_, err := client.Get(config.Get("url"))
@@ -87,6 +103,7 @@ func WaitForConnection() bool {
 				fmt.Println(fmt.Sprintf("oCIS server is ready to accept requests"))
 				return true
 			}
+			// 500 milliseconds poll interval
 			time.Sleep(500 * time.Millisecond)
 		}
 	}