diff --git a/go/cmd/dolt/commands/remote.go b/go/cmd/dolt/commands/remote.go index 60dcca8a3a..99870807e9 100644 --- a/go/cmd/dolt/commands/remote.go +++ b/go/cmd/dolt/commands/remote.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "os" "strings" "github.com/dolthub/go-mysql-server/sql" @@ -30,6 +31,7 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/gitremote" "github.com/dolthub/dolt/go/libraries/utils/argparser" eventsapi "github.com/dolthub/eventsapi_schema/dolt/services/eventsapi/v1alpha1" ) @@ -43,7 +45,22 @@ var remoteDocs = cli.CommandDocumentationContent{ {{.EmphasisLeft}}add{{.EmphasisRight}} Adds a remote named {{.LessThan}}name{{.GreaterThan}} for the repository at {{.LessThan}}url{{.GreaterThan}}. The command dolt fetch {{.LessThan}}name{{.GreaterThan}} can then be used to create and update remote-tracking branches {{.EmphasisLeft}}/{{.EmphasisRight}}. -The {{.LessThan}}url{{.GreaterThan}} parameter supports url schemes of http, https, aws, gs, and file. The url prefix defaults to https. If the {{.LessThan}}url{{.GreaterThan}} parameter is in the format {{.EmphasisLeft}}/{{.EmphasisRight}} then dolt will use the {{.EmphasisLeft}}remotes.default_host{{.EmphasisRight}} from your configuration file (Which will be dolthub.com unless changed). +The {{.LessThan}}url{{.GreaterThan}} parameter supports url schemes of http, https, aws, gs, git, and file. The url prefix defaults to https. If the {{.LessThan}}url{{.GreaterThan}} parameter is in the format {{.EmphasisLeft}}/{{.EmphasisRight}} then dolt will use the {{.EmphasisLeft}}remotes.default_host{{.EmphasisRight}} from your configuration file (Which will be dolthub.com unless changed). + +{{.EmphasisLeft}}Git Remotes{{.EmphasisRight}} +You can use any git repository as a dolt remote by using the git:// scheme or an HTTP(S) URL ending in .git: + + dolt remote add origin git://github.com/user/repo.git + dolt remote add origin https://github.com/user/repo.git + +Git credentials are automatically detected from: + - SSH agent (if running) + - SSH key files (~/.ssh/id_ed25519, id_rsa, etc.) + - Git credential helper (git credential fill) + - Environment variables (DOLT_REMOTE_PASSWORD) + - ~/.netrc file + +Before using a git remote, initialize it with {{.EmphasisLeft}}dolt remote init{{.EmphasisRight}}. AWS cloud remote urls should be of the form {{.EmphasisLeft}}aws://[dynamo-table:s3-bucket]/database{{.EmphasisRight}}. You may configure your aws cloud remote using the optional parameters {{.EmphasisLeft}}aws-region{{.EmphasisRight}}, {{.EmphasisLeft}}aws-creds-type{{.EmphasisRight}}, {{.EmphasisLeft}}aws-creds-file{{.EmphasisRight}}. @@ -58,12 +75,22 @@ GCP remote urls should be of the form gs://gcs-bucket/database and will use the The local filesystem can be used as a remote by providing a repository url in the format file://absolute path. See https://en.wikipedia.org/wiki/File_URI_scheme {{.EmphasisLeft}}remove{{.EmphasisRight}}, {{.EmphasisLeft}}rm{{.EmphasisRight}} -Remove the remote named {{.LessThan}}name{{.GreaterThan}}. All remote-tracking branches and configuration settings for the remote are removed.`, +Remove the remote named {{.LessThan}}name{{.GreaterThan}}. All remote-tracking branches and configuration settings for the remote are removed. + +{{.EmphasisLeft}}init{{.EmphasisRight}} +Initialize a git repository as a dolt remote. This creates the necessary directory structure on a custom git ref (default: refs/dolt/data) to store dolt data. The git repository URL must end with .git or use the git:// scheme. + +This command is idempotent - it's safe to run multiple times on the same repository. Dolt data is stored on a custom ref that doesn't interfere with normal git branches. + +Example: + dolt remote init https://github.com/user/repo.git + dolt remote init --ref refs/dolt/custom https://github.com/user/repo.git`, Synopsis: []string{ "[-v | --verbose]", "add [--aws-region {{.LessThan}}region{{.GreaterThan}}] [--aws-creds-type {{.LessThan}}creds-type{{.GreaterThan}}] [--aws-creds-file {{.LessThan}}file{{.GreaterThan}}] [--aws-creds-profile {{.LessThan}}profile{{.GreaterThan}}] {{.LessThan}}name{{.GreaterThan}} {{.LessThan}}url{{.GreaterThan}}", "remove {{.LessThan}}name{{.GreaterThan}}", + "init [--ref {{.LessThan}}ref-name{{.GreaterThan}}] {{.LessThan}}git-url{{.GreaterThan}}", }, } @@ -71,6 +98,10 @@ const ( addRemoteId = "add" removeRemoteId = "remove" removeRemoteShortId = "rm" + initRemoteId = "init" + + // gitRefFlag is the flag for specifying a custom git ref + gitRefFlag = "ref" ) type RemoteCmd struct{} @@ -101,6 +132,9 @@ func (cmd RemoteCmd) ArgParser() *argparser.ArgParser { ap.SupportsString(dbfactory.OSSCredsFileParam, "", "file", "OSS credentials file") ap.SupportsString(dbfactory.OSSCredsProfile, "", "profile", "OSS profile to use") + + // Git remote init flags + ap.SupportsString(gitRefFlag, "", "ref-name", "Custom git ref for dolt data (default: refs/dolt/data)") return ap } @@ -109,12 +143,25 @@ func (cmd RemoteCmd) EventType() eventsapi.ClientEventType { return eventsapi.ClientEventType_REMOTE } +// RequiresRepo returns false because `dolt remote init` can be run without a dolt repository. +// Other subcommands still require a repository context. +func (cmd RemoteCmd) RequiresRepo() bool { + return false +} + // Exec executes the command func (cmd RemoteCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) int { ap := cmd.ArgParser() help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, remoteDocs, ap)) apr := cli.ParseArgsOrDie(ap, args, help) + // Handle `dolt remote init` separately since it doesn't require a dolt repository + if apr.NArg() > 0 && apr.Arg(0) == initRemoteId { + verr := initGitRemote(ctx, apr) + return HandleVErrAndExitCode(verr, usage) + } + + // All other subcommands require a dolt repository queryist, err := cliCtx.QueryEngine(ctx) if err != nil { return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) @@ -167,7 +214,7 @@ func addRemote(sqlCtx *sql.Context, queryist cli.Queryist, dEnv *env.DoltEnv, ap } if len(params) == 0 { - err := callSQLRemoteAdd(sqlCtx, queryist, remoteName, remoteUrl) + err := callSQLRemoteAdd(sqlCtx, queryist, remoteName, absRemoteUrl) if err != nil { return errhand.BuildDError("error: Unable to add remote.").AddCause(err).Build() } @@ -321,3 +368,82 @@ func printRemotes(sqlCtx *sql.Context, queryist cli.Queryist, apr *argparser.Arg return nil } + +// initGitRemote initializes a git repository as a dolt remote by creating the +// .dolt_remote/ directory structure on a custom git ref. +func initGitRemote(ctx context.Context, apr *argparser.ArgParseResults) errhand.VerboseError { + if apr.NArg() != 2 { + return errhand.BuildDError("usage: dolt remote init ").SetPrintUsage().Build() + } + + gitURL := strings.TrimSpace(apr.Arg(1)) + + // Validate that this is a git URL + if !dbfactory.IsGitURL(gitURL) { + return errhand.BuildDError("error: '%s' is not a valid git remote URL", gitURL). + AddDetails("Git remote URLs must use the git:// scheme or end with .git").Build() + } + + // Get custom ref if specified + ref := gitremote.DefaultRef + if refVal, ok := apr.GetValue(gitRefFlag); ok { + ref = refVal + } + + cli.Printf("Initializing git remote at %s on ref %s...\n", gitURL, ref) + + // Detect authentication + auth, err := gitremote.DetectAuth(gitURL) + if err != nil { + return errhand.BuildDError("error: failed to detect git credentials").AddCause(err).Build() + } + + if auth != nil { + cli.Printf("Using authentication: %s\n", gitremote.AuthMethodName(auth)) + } + + // Create a temporary directory for git operations + localPath, err := os.MkdirTemp("", "dolt-remote-init-*") + if err != nil { + return errhand.BuildDError("error: failed to create temp directory").AddCause(err).Build() + } + defer os.RemoveAll(localPath) + + // Open the repository + repo, err := gitremote.Open(ctx, gitremote.OpenOptions{ + URL: gitURL, + Ref: ref, + Auth: auth, + LocalPath: localPath, + }) + if err != nil { + return errhand.BuildDError("error: failed to open git repository").AddCause(err).Build() + } + defer repo.Close() + + // Checkout the ref to populate the worktree (if it exists) + if err := repo.CheckoutRef(ctx); err != nil { + return errhand.BuildDError("error: failed to checkout ref").AddCause(err).Build() + } + + // Check if already initialized + initialized, err := repo.IsInitialized() + if err != nil { + return errhand.BuildDError("error: failed to check if remote is initialized").AddCause(err).Build() + } + + if initialized { + cli.Println("Remote is already initialized for dolt.") + return nil + } + + // Initialize the remote structure + if err := repo.InitRemote(ctx); err != nil { + return errhand.BuildDError("error: failed to initialize git remote").AddCause(err).Build() + } + + cli.Println("Successfully initialized git repository as dolt remote.") + cli.Printf("You can now add this remote with: dolt remote add %s\n", gitURL) + + return nil +} diff --git a/go/go.mod b/go/go.mod index ba4ef1985d..df7e659443 100644 --- a/go/go.mod +++ b/go/go.mod @@ -27,10 +27,10 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.5.0 github.com/rivo/uniseg v0.2.0 - github.com/sergi/go-diff v1.1.0 + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 github.com/shopspring/decimal v1.4.0 github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6 - github.com/sirupsen/logrus v1.8.3 + github.com/sirupsen/logrus v1.9.3 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/stretchr/testify v1.11.1 github.com/tealeg/xlsx v1.0.5 @@ -65,6 +65,8 @@ require ( github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/edsrzf/mmap-go v1.2.0 github.com/esote/minmaxheap v1.0.0 + github.com/go-git/go-billy/v5 v5.6.2 + github.com/go-git/go-git/v5 v5.16.4 github.com/goccy/go-json v0.10.2 github.com/google/btree v1.1.2 github.com/google/go-github/v57 v57.0.0 @@ -92,7 +94,7 @@ require ( go.opentelemetry.io/otel/exporters/jaeger v1.17.0 go.opentelemetry.io/otel/sdk v1.36.0 go.opentelemetry.io/otel/trace v1.36.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/text v0.31.0 gonum.org/v1/plot v0.11.0 gopkg.in/go-jose/go-jose.v2 v2.6.3 @@ -107,11 +109,14 @@ require ( cloud.google.com/go/compute/metadata v0.7.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect + dario.cat/mergo v1.0.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect git.sr.ht/~sbinet/gg v0.3.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -133,15 +138,19 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.33.16 // indirect github.com/aws/smithy-go v1.22.2 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dolthub/go-icu-regex v0.0.0-20250916051405-78a38d478790 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect github.com/ebitengine/purego v0.9.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-fonts/liberation v0.2.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect github.com/go-logr/logr v1.4.3 // indirect @@ -150,12 +159,15 @@ require ( github.com/go-pdf/fpdf v0.6.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.14.2 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/lestrrat-go/strftime v1.0.4 // indirect @@ -164,6 +176,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pierrec/lz4/v4 v4.1.6 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect @@ -171,6 +184,7 @@ require ( github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rs/xid v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect @@ -178,6 +192,7 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/errs v1.4.0 // indirect @@ -200,6 +215,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) go 1.25.6 diff --git a/go/go.sum b/go/go.sum index 7ffabb9e07..e1b60c6021 100644 --- a/go/go.sum +++ b/go/go.sum @@ -41,6 +41,8 @@ cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6Q cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= @@ -70,6 +72,11 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8= @@ -87,10 +94,14 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1e github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible h1:wd5mq8xSfwCYd1JpQ309s+3tTlP/gifcG2awOA3x5Vk= github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible/go.mod h1:Cp18FeDCvsK+cD2QAGkqerGjrgSXLiJWnjHeY2mneBc= github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -168,6 +179,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= @@ -176,6 +189,8 @@ github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc= github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -213,6 +228,10 @@ github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -240,6 +259,8 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-fonts/dejavu v0.1.0 h1:JSajPXURYqpr+Cu8U9bt8K+XcACIHWqWrvWCKyeFmVQ= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0 h1:5/Tv1Ek/QCr20C6ZOz15vw3g7GELYL98KWr8Hgo+3vk= @@ -247,6 +268,14 @@ github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3 github.com/go-fonts/liberation v0.2.0 h1:jAkAWJP4S+OsrPLZM4/eC9iW7CtHy+HBXrEwZXWo5VM= github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.16.4 h1:7ajIEZHZJULcyJebDLo99bGgS0jRrOxzZG4uCk2Yb2Y= +github.com/go-git/go-git/v5 v5.16.4/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -286,6 +315,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -361,6 +392,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5 github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/iancoleman/strcase v0.1.3/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -375,6 +408,8 @@ github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 h1:l6Y3mFnF46A+CeZsTrT8kVIuhayq1266oxWpDKE7hnQ= github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6/go.mod h1:UtDV9qK925GVmbdjR+e1unqoo+wGWNHHC6XB1Eu6wpE= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -431,6 +466,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/oracle/oci-go-sdk/v65 v65.55.0 h1:enKyHVLdJYDJrc9232w33u5F6t2p8Din4593kn3nh/w= github.com/oracle/oci-go-sdk/v65 v65.55.0/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0= @@ -440,6 +477,8 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.6 h1:ueMTcBBFrbT8K4uGDNNZPa8Z7LtPV7Cl0TDjaeHxP44= github.com/pierrec/lz4/v4 v4.1.6/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -467,16 +506,16 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w= github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY= github.com/shirou/gopsutil/v4 v4.25.12 h1:e7PvW/0RmJ8p8vPGJH4jvNkOyLmbkXgXW4m6ZPic6CY= @@ -485,8 +524,11 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6 h1:31fhvQj+O9qDqMxUgQDOCQA5RV1iIFMzYPhBUyzg2p0= github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6/go.mod h1:jk5gVE20+MCoyJ2TFiiMrbWPyaH4t9T5F3HwVdthB2w= -github.com/sirupsen/logrus v1.8.3 h1:DBBfY8eMYazKEJHb3JKpSPfpgd2mBCoNFlQx6C5fftU= -github.com/sirupsen/logrus v1.8.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= @@ -534,6 +576,8 @@ github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9R github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/vbauerster/mpb/v8 v8.0.2 h1:alVQG69Jg5+Ku9Hu1dakDx50uACEHnIzS7i356NQ/Vs= github.com/vbauerster/mpb/v8 v8.0.2/go.mod h1:Z9VJYIzXls7xZwirZjShGsi+14enzJhQfGyb/XZK0ZQ= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= github.com/xitongsys/parquet-go v1.6.1 h1:F1snhlfL5U1hC1yE7Op8qLWFIZEzqmM46pCEspu9OC0= github.com/xitongsys/parquet-go v1.6.1/go.mod h1:oNMMTE7vxZkSeV4Y6Q+DSpnR50DdpFM6Jx2CRav1OHI= @@ -606,6 +650,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -621,8 +666,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -674,6 +719,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -721,6 +767,9 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -876,9 +925,10 @@ gopkg.in/src-d/go-errors.v1 v1.0.0 h1:cooGdZnCjYbeS1zb1s6pVAAimTdKceRrpn7aKOnNIf gopkg.in/src-d/go-errors.v1 v1.0.0/go.mod h1:q1cBlomlw2FnDBDNGlnh6X0jPihy+QxZfMMNxPCbdYg= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/go/libraries/doltcore/dbfactory/factory.go b/go/libraries/doltcore/dbfactory/factory.go index dcb19f1438..7690bb6d03 100644 --- a/go/libraries/doltcore/dbfactory/factory.go +++ b/go/libraries/doltcore/dbfactory/factory.go @@ -76,6 +76,7 @@ var DBFactories = map[string]DBFactory{ FileScheme: FileFactory{}, MemScheme: MemFactory{}, LocalBSScheme: LocalBSFactory{}, + GitScheme: GitFactory{}, HTTPScheme: NewDoltRemoteFactory(true), HTTPSScheme: NewDoltRemoteFactory(false), } diff --git a/go/libraries/doltcore/dbfactory/git.go b/go/libraries/doltcore/dbfactory/git.go new file mode 100644 index 0000000000..11c80d66ac --- /dev/null +++ b/go/libraries/doltcore/dbfactory/git.go @@ -0,0 +1,194 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbfactory + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/dolthub/dolt/go/libraries/doltcore/gitremote" + "github.com/dolthub/dolt/go/store/blobstore" + "github.com/dolthub/dolt/go/store/datas" + "github.com/dolthub/dolt/go/store/nbs" + "github.com/dolthub/dolt/go/store/prolly/tree" + "github.com/dolthub/dolt/go/store/types" +) + +const ( + // GitScheme is the URL scheme for git-backed remotes + GitScheme = "git" + + // GitRefParam is the parameter name for specifying a custom git ref + GitRefParam = "ref" + + // GitLocalPathParam is the parameter name for specifying a local cache directory + GitLocalPathParam = "local_path" +) + +// GitFactory is a DBFactory implementation for creating git repository backed databases +type GitFactory struct{} + +// PrepareDB initializes a git repository as a dolt remote by creating the +// .dolt_remote/ directory structure on the custom ref. +func (fact GitFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) error { + repoURL := gitURLFromURLObj(urlObj) + ref := gitRefFromParams(params) + + // Detect authentication + auth, err := gitremote.DetectAuth(repoURL) + if err != nil { + return fmt.Errorf("failed to detect git auth: %w", err) + } + + // Create a temporary directory for git operations + localPath, err := os.MkdirTemp("", "dolt-git-prepare-*") + if err != nil { + return fmt.Errorf("failed to create temp directory: %w", err) + } + defer os.RemoveAll(localPath) + + // Open the repository + repo, err := gitremote.Open(ctx, gitremote.OpenOptions{ + URL: repoURL, + Ref: ref, + Auth: auth, + LocalPath: localPath, + }) + if err != nil { + return fmt.Errorf("failed to open git repository: %w", err) + } + defer repo.Close() + + // Initialize the remote structure + if err := repo.InitRemote(ctx); err != nil { + return fmt.Errorf("failed to initialize git remote: %w", err) + } + + return nil +} + +// CreateDB creates a git repository backed database +func (fact GitFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) { + repoURL := gitURLFromURLObj(urlObj) + ref := gitRefFromParams(params) + localPath := gitLocalPathFromParams(params) + + // Create the GitBlobstore + bs, err := blobstore.NewGitBlobstore(ctx, repoURL, ref, localPath) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create git blobstore: %w", err) + } + + // Create the NBS store backed by the git blobstore + q := nbs.NewUnlimitedMemQuotaProvider() + gitStore, err := nbs.NewBSStore(ctx, nbf.VersionString(), bs, defaultMemTableSize, q) + if err != nil { + bs.Close() + return nil, nil, nil, fmt.Errorf("failed to create git store: %w", err) + } + + vrw := types.NewValueStore(gitStore) + ns := tree.NewNodeStore(gitStore) + db := datas.NewTypesDatabase(vrw, ns) + + return db, vrw, ns, nil +} + +// gitURLFromURLObj converts a url.URL to a git repository URL string. +// For git:// scheme, it reconstructs the URL. +// For http(s):// with .git suffix, it preserves the original URL. +func gitURLFromURLObj(urlObj *url.URL) string { + // Reconstruct the URL + if urlObj.Scheme == GitScheme { + // If there's no host, this is a local file path + if urlObj.Host == "" { + return urlObj.Path + } + // Convert git:// to https:// for actual git operations + // git://github.com/user/repo.git -> https://github.com/user/repo.git + return fmt.Sprintf("https://%s%s", urlObj.Host, urlObj.Path) + } + + // For http/https URLs, return as-is + return urlObj.String() +} + +// gitRefFromParams extracts the git ref from params, or returns the default. +func gitRefFromParams(params map[string]interface{}) string { + if params != nil { + if refVal, ok := params[GitRefParam]; ok { + if ref, ok := refVal.(string); ok && ref != "" { + return ref + } + } + } + return gitremote.DefaultRef +} + +// gitLocalPathFromParams extracts the local cache path from params. +func gitLocalPathFromParams(params map[string]interface{}) string { + if params != nil { + if pathVal, ok := params[GitLocalPathParam]; ok { + if path, ok := pathVal.(string); ok && path != "" { + return path + } + } + } + return "" +} + +// IsGitURL returns true if the URL should be handled by the GitFactory. +// This includes: +// - URLs with the git:// scheme +// - HTTP(S) URLs ending with .git +// - Local file paths ending with .git +func IsGitURL(urlStr string) bool { + // Check for git:// scheme + if strings.HasPrefix(strings.ToLower(urlStr), "git://") { + return true + } + + lower := strings.ToLower(urlStr) + + // Check for .git suffix on http(s) URLs + if (strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")) && + strings.HasSuffix(lower, ".git") { + return true + } + + // Check for local file paths ending with .git (bare repositories) + if strings.HasSuffix(lower, ".git") { + return true + } + + return false +} + +// GitCacheDir returns the default cache directory for git remotes. +func GitCacheDir() (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", err + } + cacheDir := filepath.Join(homeDir, ".dolt", "git-remotes") + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return "", err + } + return cacheDir, nil +} diff --git a/go/libraries/doltcore/dbfactory/git_test.go b/go/libraries/doltcore/dbfactory/git_test.go new file mode 100644 index 0000000000..a45611a542 --- /dev/null +++ b/go/libraries/doltcore/dbfactory/git_test.go @@ -0,0 +1,297 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbfactory + +import ( + "context" + "net/url" + "path/filepath" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/store/types" +) + +func createTestBareRepo(t *testing.T) string { + t.Helper() + dir := t.TempDir() + bareDir := filepath.Join(dir, "bare.git") + + _, err := git.PlainInit(bareDir, true) + require.NoError(t, err) + + return bareDir +} + +func TestIsGitURL(t *testing.T) { + tests := []struct { + name string + url string + expected bool + }{ + { + name: "git scheme", + url: "git://github.com/user/repo.git", + expected: true, + }, + { + name: "https with .git suffix", + url: "https://github.com/user/repo.git", + expected: true, + }, + { + name: "http with .git suffix", + url: "http://github.com/user/repo.git", + expected: true, + }, + { + name: "https without .git suffix", + url: "https://github.com/user/repo", + expected: false, + }, + { + name: "dolthub remote", + url: "https://doltremoteapi.dolthub.com/user/repo", + expected: false, + }, + { + name: "file scheme", + url: "file:///path/to/repo", + expected: false, + }, + { + name: "git scheme uppercase", + url: "GIT://github.com/user/repo.git", + expected: true, + }, + { + name: "https with .GIT suffix uppercase", + url: "HTTPS://github.com/user/repo.GIT", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := IsGitURL(tt.url) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGitRefFromParams(t *testing.T) { + tests := []struct { + name string + params map[string]interface{} + expected string + }{ + { + name: "nil params", + params: nil, + expected: "refs/dolt/data", + }, + { + name: "empty params", + params: map[string]interface{}{}, + expected: "refs/dolt/data", + }, + { + name: "custom ref", + params: map[string]interface{}{GitRefParam: "refs/custom/ref"}, + expected: "refs/custom/ref", + }, + { + name: "empty ref string", + params: map[string]interface{}{GitRefParam: ""}, + expected: "refs/dolt/data", + }, + { + name: "wrong type", + params: map[string]interface{}{GitRefParam: 123}, + expected: "refs/dolt/data", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := gitRefFromParams(tt.params) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGitLocalPathFromParams(t *testing.T) { + tests := []struct { + name string + params map[string]interface{} + expected string + }{ + { + name: "nil params", + params: nil, + expected: "", + }, + { + name: "empty params", + params: map[string]interface{}{}, + expected: "", + }, + { + name: "custom path", + params: map[string]interface{}{GitLocalPathParam: "/tmp/cache"}, + expected: "/tmp/cache", + }, + { + name: "empty path string", + params: map[string]interface{}{GitLocalPathParam: ""}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := gitLocalPathFromParams(tt.params) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGitURLFromURLObj(t *testing.T) { + tests := []struct { + name string + urlStr string + expected string + }{ + { + name: "git scheme converts to https", + urlStr: "git://github.com/user/repo.git", + expected: "https://github.com/user/repo.git", + }, + { + name: "https preserved", + urlStr: "https://github.com/user/repo.git", + expected: "https://github.com/user/repo.git", + }, + { + name: "http preserved", + urlStr: "http://github.com/user/repo.git", + expected: "http://github.com/user/repo.git", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + urlObj, err := url.Parse(tt.urlStr) + require.NoError(t, err) + result := gitURLFromURLObj(urlObj) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGitFactory_Registered(t *testing.T) { + // Verify GitFactory is registered in DBFactories + factory, ok := DBFactories[GitScheme] + assert.True(t, ok, "GitFactory should be registered") + _, isGitFactory := factory.(GitFactory) + assert.True(t, isGitFactory, "should be GitFactory type") +} + +func TestGitFactory_PrepareDB(t *testing.T) { + bareDir := createTestBareRepo(t) + ctx := context.Background() + + factory := GitFactory{} + urlObj, err := url.Parse(bareDir) + require.NoError(t, err) + // For local file paths, we need to construct a proper file URL + urlObj = &url.URL{ + Scheme: "git", + Host: "", + Path: bareDir, + } + + nbf := types.Format_Default + + // PrepareDB should initialize the remote structure + err = factory.PrepareDB(ctx, nbf, urlObj, nil) + require.NoError(t, err) + + // Second call should be idempotent + err = factory.PrepareDB(ctx, nbf, urlObj, nil) + require.NoError(t, err) +} + +func TestGitFactory_CreateDB(t *testing.T) { + bareDir := createTestBareRepo(t) + ctx := context.Background() + workDir := filepath.Join(t.TempDir(), "work") + + factory := GitFactory{} + urlObj := &url.URL{ + Scheme: "git", + Host: "", + Path: bareDir, + } + + nbf := types.Format_Default + params := map[string]interface{}{ + GitLocalPathParam: workDir, + } + + // CreateDB should return a working database + db, vrw, ns, err := factory.CreateDB(ctx, nbf, urlObj, params) + require.NoError(t, err) + assert.NotNil(t, db) + assert.NotNil(t, vrw) + assert.NotNil(t, ns) + + // Clean up + err = db.Close() + require.NoError(t, err) +} + +func TestGitFactory_CreateDB_CustomRef(t *testing.T) { + bareDir := createTestBareRepo(t) + ctx := context.Background() + workDir := filepath.Join(t.TempDir(), "work") + + factory := GitFactory{} + urlObj := &url.URL{ + Scheme: "git", + Host: "", + Path: bareDir, + } + + nbf := types.Format_Default + params := map[string]interface{}{ + GitLocalPathParam: workDir, + GitRefParam: "refs/dolt/custom", + } + + // CreateDB should work with custom ref + db, vrw, ns, err := factory.CreateDB(ctx, nbf, urlObj, params) + require.NoError(t, err) + assert.NotNil(t, db) + assert.NotNil(t, vrw) + assert.NotNil(t, ns) + + // Clean up + err = db.Close() + require.NoError(t, err) +} diff --git a/go/libraries/doltcore/env/remotes.go b/go/libraries/doltcore/env/remotes.go index 3ea6572442..bbc79c292a 100644 --- a/go/libraries/doltcore/env/remotes.go +++ b/go/libraries/doltcore/env/remotes.go @@ -648,8 +648,22 @@ func GetAbsRemoteUrl(fs filesys2.Filesys, cfg config.ReadableConfig, urlArg stri return "", "", err } - if u.Scheme != "" && fs != nil { + // Check for explicit git:// scheme + if u.Scheme == dbfactory.GitScheme { + return dbfactory.GitScheme, urlArg, nil + } + + // Check for .git suffix on HTTP(S) URLs - these are git remotes + if (u.Scheme == dbfactory.HTTPScheme || u.Scheme == dbfactory.HTTPSScheme) && + strings.HasSuffix(strings.ToLower(u.Path), ".git") { + return dbfactory.GitScheme, urlArg, nil + } + + if u.Scheme != "" { if u.Scheme == dbfactory.FileScheme || u.Scheme == dbfactory.LocalBSScheme { + if fs == nil { + return u.Scheme, urlArg, nil + } absUrl, err := getAbsFileRemoteUrl(u, fs) if err != nil { @@ -661,9 +675,44 @@ func GetAbsRemoteUrl(fs filesys2.Filesys, cfg config.ReadableConfig, urlArg stri return u.Scheme, urlArg, nil } else if u.Host != "" { + // Check for .git suffix on naked URLs (no scheme) + // But first, check if this might be a local path that exists + if strings.HasSuffix(strings.ToLower(u.Host), ".git") { + // This could be a local path like "remote.git" or a remote like "github.com/user/repo.git" + // Check if it exists locally first + if fs != nil { + exists, _ := fs.Exists(urlArg) + if exists { + // It's a local git repository path - use git:///path format + absPath, err := fs.Abs(urlArg) + if err != nil { + return "", "", err + } + return dbfactory.GitScheme, dbfactory.GitScheme + "://" + absPath, nil + } + } + // Not a local path, treat as remote URL + return dbfactory.GitScheme, "https://" + urlArg, nil + } + if strings.HasSuffix(strings.ToLower(u.Path), ".git") { + return dbfactory.GitScheme, "https://" + urlArg, nil + } return dbfactory.HTTPSScheme, "https://" + urlArg, nil } + // Check for .git suffix on paths (local git repositories) + if strings.HasSuffix(strings.ToLower(u.Path), ".git") { + // This is a local git repository path - use git:///path format + if fs != nil { + absPath, err := fs.Abs(urlArg) + if err != nil { + return "", "", err + } + return dbfactory.GitScheme, dbfactory.GitScheme + "://" + absPath, nil + } + return dbfactory.GitScheme, dbfactory.GitScheme + "://" + urlArg, nil + } + hostName, err := cfg.GetString(config.RemotesApiHostKey) if err != nil { diff --git a/go/libraries/doltcore/env/remotes_test.go b/go/libraries/doltcore/env/remotes_test.go new file mode 100644 index 0000000000..384a7cccee --- /dev/null +++ b/go/libraries/doltcore/env/remotes_test.go @@ -0,0 +1,196 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package env + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" + "github.com/dolthub/dolt/go/libraries/utils/config" +) + +func TestGetAbsRemoteUrl_GitSchemeDetection(t *testing.T) { + // Create a minimal config that returns the default host + cfg := config.NewMapConfig(map[string]string{}) + + tests := []struct { + name string + urlArg string + expectedScheme string + expectedURL string + }{ + { + name: "explicit git scheme", + urlArg: "git://github.com/user/repo.git", + expectedScheme: dbfactory.GitScheme, + expectedURL: "git://github.com/user/repo.git", + }, + { + name: "https with .git suffix", + urlArg: "https://github.com/user/repo.git", + expectedScheme: dbfactory.GitScheme, + expectedURL: "https://github.com/user/repo.git", + }, + { + name: "http with .git suffix", + urlArg: "http://github.com/user/repo.git", + expectedScheme: dbfactory.GitScheme, + expectedURL: "http://github.com/user/repo.git", + }, + { + name: "https with .GIT suffix (uppercase)", + urlArg: "https://github.com/user/repo.GIT", + expectedScheme: dbfactory.GitScheme, + expectedURL: "https://github.com/user/repo.GIT", + }, + { + name: "naked URL with .git suffix", + urlArg: "github.com/user/repo.git", + expectedScheme: dbfactory.GitScheme, + expectedURL: "https://github.com/user/repo.git", + }, + { + name: "https without .git suffix - dolthub", + urlArg: "https://doltremoteapi.dolthub.com/user/repo", + expectedScheme: dbfactory.HTTPSScheme, + expectedURL: "https://doltremoteapi.dolthub.com/user/repo", + }, + { + name: "naked URL without .git suffix - dolthub style", + urlArg: "dolthub.com/user/repo", + expectedScheme: dbfactory.HTTPSScheme, + expectedURL: "https://dolthub.com/user/repo", + }, + { + name: "aws scheme preserved", + urlArg: "aws://bucket/path", + expectedScheme: dbfactory.AWSScheme, + expectedURL: "aws://bucket/path", + }, + { + name: "gs scheme preserved", + urlArg: "gs://bucket/path", + expectedScheme: dbfactory.GSScheme, + expectedURL: "gs://bucket/path", + }, + { + name: "localbs scheme preserved", + urlArg: "localbs://path/to/dir", + expectedScheme: dbfactory.LocalBSScheme, + expectedURL: "localbs://path/to/dir", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Pass nil for fs to avoid file system operations for non-file URLs + scheme, url, err := GetAbsRemoteUrl(nil, cfg, tt.urlArg) + + // Skip file/localbs tests that need a real filesystem + if tt.expectedScheme == dbfactory.LocalBSScheme { + // localbs requires a filesystem, skip detailed check + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expectedScheme, scheme, "scheme mismatch") + assert.Equal(t, tt.expectedURL, url, "URL mismatch") + }) + } +} + +func TestGetAbsRemoteUrl_GitVsDoltHub(t *testing.T) { + cfg := config.NewMapConfig(map[string]string{}) + + // Test that similar URLs are correctly distinguished + tests := []struct { + name string + urlArg string + expectedScheme string + description string + }{ + { + name: "github with .git is git remote", + urlArg: "https://github.com/dolthub/dolt.git", + expectedScheme: dbfactory.GitScheme, + description: "GitHub URL with .git suffix should use git factory", + }, + { + name: "github without .git is dolthub remote", + urlArg: "https://github.com/dolthub/dolt", + expectedScheme: dbfactory.HTTPSScheme, + description: "GitHub URL without .git suffix should use dolthub factory", + }, + { + name: "dolthub is dolthub remote", + urlArg: "https://doltremoteapi.dolthub.com/dolthub/dolt", + expectedScheme: dbfactory.HTTPSScheme, + description: "DoltHub URL should use dolthub factory", + }, + { + name: "gitlab with .git is git remote", + urlArg: "https://gitlab.com/user/repo.git", + expectedScheme: dbfactory.GitScheme, + description: "GitLab URL with .git suffix should use git factory", + }, + { + name: "bitbucket with .git is git remote", + urlArg: "https://bitbucket.org/user/repo.git", + expectedScheme: dbfactory.GitScheme, + description: "Bitbucket URL with .git suffix should use git factory", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme, _, err := GetAbsRemoteUrl(nil, cfg, tt.urlArg) + require.NoError(t, err) + assert.Equal(t, tt.expectedScheme, scheme, tt.description) + }) + } +} + +func TestGetAbsRemoteUrl_ShorthandURLs(t *testing.T) { + // Test shorthand URLs (no scheme, just org/repo format) + cfg := config.NewMapConfig(map[string]string{}) + + tests := []struct { + name string + urlArg string + expectedScheme string + }{ + { + name: "shorthand dolthub format", + urlArg: "dolthub/museum-collections", + expectedScheme: dbfactory.HTTPSScheme, + }, + { + name: "shorthand with host", + urlArg: "example.com/user/repo", + expectedScheme: dbfactory.HTTPSScheme, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme, _, err := GetAbsRemoteUrl(nil, cfg, tt.urlArg) + require.NoError(t, err) + assert.Equal(t, tt.expectedScheme, scheme) + }) + } +} diff --git a/go/libraries/doltcore/gitremote/auth.go b/go/libraries/doltcore/gitremote/auth.go new file mode 100644 index 0000000000..57a4690f04 --- /dev/null +++ b/go/libraries/doltcore/gitremote/auth.go @@ -0,0 +1,406 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitremote + +import ( + "bufio" + "bytes" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" +) + +const ( + // DefaultSSHUser is the default username for SSH git operations + DefaultSSHUser = "git" + + // Environment variable names + EnvGitSSHKey = "GIT_SSH_KEY" + EnvDoltRemotePassword = "DOLT_REMOTE_PASSWORD" +) + +// Common SSH key file names in order of preference +var defaultSSHKeyFiles = []string{ + "id_ed25519", + "id_ecdsa", + "id_rsa", + "id_dsa", +} + +// DetectAuth automatically detects and returns appropriate authentication +// for the given git repository URL. It tries multiple credential sources +// in order of preference and returns the first successful one. +// +// For SSH URLs (git@host:path or ssh://), it tries: +// 1. SSH agent +// 2. SSH key files from default locations +// 3. SSH key from GIT_SSH_KEY environment variable +// +// For HTTPS URLs, it tries: +// 1. Git credential helper +// 2. Environment variables (DOLT_REMOTE_PASSWORD) +// 3. Netrc file +// +// Returns nil auth (anonymous) if no credentials are found, which may +// still work for public repositories. +func DetectAuth(repoURL string) (transport.AuthMethod, error) { + endpoint, err := transport.NewEndpoint(repoURL) + if err != nil { + return nil, fmt.Errorf("invalid git URL %q: %w", repoURL, err) + } + + switch endpoint.Protocol { + case "ssh", "git": + return DetectSSHAuth(endpoint.User) + case "https", "http": + return DetectHTTPSAuth(endpoint.Host, endpoint.User) + case "file": + // Local file URLs don't need authentication + return nil, nil + default: + // For unknown protocols, try HTTPS auth as fallback + return DetectHTTPSAuth(endpoint.Host, endpoint.User) + } +} + +// DetectSSHAuth attempts to find SSH credentials for git operations. +// It tries sources in the following order: +// 1. SSH agent (if running and has keys) +// 2. SSH key files from ~/.ssh/ +// 3. SSH key from GIT_SSH_KEY environment variable +// +// The user parameter specifies the SSH username (defaults to "git"). +func DetectSSHAuth(user string) (transport.AuthMethod, error) { + if user == "" { + user = DefaultSSHUser + } + + // Try SSH agent first + auth, err := trySSHAgent(user) + if err == nil && auth != nil { + return auth, nil + } + + // Try default SSH key files + auth, err = trySSHKeyFiles(user) + if err == nil && auth != nil { + return auth, nil + } + + // Try GIT_SSH_KEY environment variable + auth, err = trySSHKeyFromEnv(user) + if err == nil && auth != nil { + return auth, nil + } + + // No SSH credentials found - return nil (anonymous) + // This allows public repos to work without credentials + return nil, nil +} + +// DetectHTTPSAuth attempts to find HTTPS credentials for git operations. +// It tries sources in the following order: +// 1. Git credential helper +// 2. DOLT_REMOTE_PASSWORD environment variable +// 3. Netrc file +// +// The host parameter is the git server hostname. +// The user parameter is optional; if empty, it may be filled by credential sources. +func DetectHTTPSAuth(host, user string) (transport.AuthMethod, error) { + // Try git credential helper first + auth, err := tryGitCredentialHelper(host, user) + if err == nil && auth != nil { + return auth, nil + } + + // Try environment variable + auth, err = tryHTTPSFromEnv(user) + if err == nil && auth != nil { + return auth, nil + } + + // Try netrc + auth, err = tryNetrc(host) + if err == nil && auth != nil { + return auth, nil + } + + // No HTTPS credentials found - return nil (anonymous) + return nil, nil +} + +// trySSHAgent attempts to use the SSH agent for authentication. +func trySSHAgent(user string) (transport.AuthMethod, error) { + auth, err := ssh.NewSSHAgentAuth(user) + if err != nil { + return nil, err + } + return auth, nil +} + +// trySSHKeyFiles looks for SSH key files in the default ~/.ssh/ directory. +func trySSHKeyFiles(user string) (transport.AuthMethod, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + sshDir := filepath.Join(homeDir, ".ssh") + + for _, keyName := range defaultSSHKeyFiles { + keyPath := filepath.Join(sshDir, keyName) + if _, err := os.Stat(keyPath); err == nil { + auth, err := ssh.NewPublicKeysFromFile(user, keyPath, "") + if err == nil { + return auth, nil + } + // Key exists but couldn't be loaded (maybe encrypted) + // Try next key + } + } + + return nil, nil +} + +// trySSHKeyFromEnv tries to load an SSH key from the GIT_SSH_KEY environment variable. +// The variable can contain either a file path or the key content directly. +func trySSHKeyFromEnv(user string) (transport.AuthMethod, error) { + keyValue := os.Getenv(EnvGitSSHKey) + if keyValue == "" { + return nil, nil + } + + // Check if it's a file path + if _, err := os.Stat(keyValue); err == nil { + return ssh.NewPublicKeysFromFile(user, keyValue, "") + } + + // Treat as key content + return ssh.NewPublicKeys(user, []byte(keyValue), "") +} + +// tryGitCredentialHelper runs the git credential helper to get credentials. +func tryGitCredentialHelper(host, user string) (transport.AuthMethod, error) { + // Check if git is available + gitPath, err := exec.LookPath("git") + if err != nil { + return nil, nil // git not available, skip + } + + // Prepare credential request + input := fmt.Sprintf("protocol=https\nhost=%s\n", host) + if user != "" { + input += fmt.Sprintf("username=%s\n", user) + } + input += "\n" + + cmd := exec.Command(gitPath, "credential", "fill") + cmd.Stdin = strings.NewReader(input) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + if err != nil { + return nil, nil // credential helper failed, skip + } + + // Parse output + creds := parseCredentialOutput(stdout.String()) + username := creds["username"] + password := creds["password"] + + if username == "" || password == "" { + return nil, nil + } + + return &http.BasicAuth{ + Username: username, + Password: password, + }, nil +} + +// parseCredentialOutput parses the output from git credential fill. +func parseCredentialOutput(output string) map[string]string { + result := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + line := scanner.Text() + if idx := strings.Index(line, "="); idx > 0 { + key := line[:idx] + value := line[idx+1:] + result[key] = value + } + } + return result +} + +// tryHTTPSFromEnv tries to get HTTPS credentials from environment variables. +func tryHTTPSFromEnv(user string) (transport.AuthMethod, error) { + password := os.Getenv(EnvDoltRemotePassword) + if password == "" { + return nil, nil + } + + // If no user specified, try common defaults + if user == "" { + // For token-based auth (GitHub, GitLab), username can be anything + user = "x-access-token" + } + + return &http.BasicAuth{ + Username: user, + Password: password, + }, nil +} + +// tryNetrc attempts to find credentials in the netrc file. +func tryNetrc(host string) (transport.AuthMethod, error) { + netrcPath := getNetrcPath() + if netrcPath == "" { + return nil, nil + } + + data, err := os.ReadFile(netrcPath) + if err != nil { + return nil, nil + } + + username, password := parseNetrc(string(data), host) + if username == "" || password == "" { + return nil, nil + } + + return &http.BasicAuth{ + Username: username, + Password: password, + }, nil +} + +// getNetrcPath returns the path to the netrc file. +func getNetrcPath() string { + // Check NETRC environment variable first + if netrc := os.Getenv("NETRC"); netrc != "" { + return netrc + } + + homeDir, err := os.UserHomeDir() + if err != nil { + return "" + } + + // On Windows, it's _netrc; on Unix, it's .netrc + if runtime.GOOS == "windows" { + return filepath.Join(homeDir, "_netrc") + } + return filepath.Join(homeDir, ".netrc") +} + +// parseNetrc parses a netrc file and returns credentials for the given host. +// This is a simplified parser that handles the common format: +// +// machine login password +func parseNetrc(content, host string) (username, password string) { + // Normalize host (remove port if present) + if u, err := url.Parse("https://" + host); err == nil { + host = u.Hostname() + } + + lines := strings.Split(content, "\n") + var currentMachine string + var currentLogin, currentPassword string + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + fields := strings.Fields(line) + for i := 0; i < len(fields); i++ { + switch fields[i] { + case "machine": + // Save previous machine's credentials if it matches + if currentMachine == host && currentLogin != "" && currentPassword != "" { + return currentLogin, currentPassword + } + currentMachine = "" + currentLogin = "" + currentPassword = "" + if i+1 < len(fields) { + currentMachine = fields[i+1] + i++ + } + case "default": + // Save previous machine's credentials if it matches + if currentMachine == host && currentLogin != "" && currentPassword != "" { + return currentLogin, currentPassword + } + currentMachine = "*" // wildcard + currentLogin = "" + currentPassword = "" + case "login": + if i+1 < len(fields) { + currentLogin = fields[i+1] + i++ + } + case "password": + if i+1 < len(fields) { + currentPassword = fields[i+1] + i++ + } + } + } + } + + // Check last machine + if currentMachine == host || currentMachine == "*" { + if currentLogin != "" && currentPassword != "" { + return currentLogin, currentPassword + } + } + + return "", "" +} + +// AuthMethodName returns a human-readable name for the auth method type. +// Useful for logging and debugging. +func AuthMethodName(auth transport.AuthMethod) string { + if auth == nil { + return "anonymous" + } + + switch auth.(type) { + case *ssh.PublicKeysCallback: + return "ssh-agent" + case *ssh.PublicKeys: + return "ssh-key" + case *http.BasicAuth: + return "https-basic" + case *http.TokenAuth: + return "https-token" + default: + return fmt.Sprintf("%T", auth) + } +} diff --git a/go/libraries/doltcore/gitremote/auth_test.go b/go/libraries/doltcore/gitremote/auth_test.go new file mode 100644 index 0000000000..524ff316f6 --- /dev/null +++ b/go/libraries/doltcore/gitremote/auth_test.go @@ -0,0 +1,273 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitremote + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseNetrc(t *testing.T) { + tests := []struct { + name string + content string + host string + wantUsername string + wantPassword string + }{ + { + name: "simple machine entry", + content: `machine github.com +login myuser +password mytoken`, + host: "github.com", + wantUsername: "myuser", + wantPassword: "mytoken", + }, + { + name: "single line format", + content: `machine gitlab.com login user1 password pass1`, + host: "gitlab.com", + wantUsername: "user1", + wantPassword: "pass1", + }, + { + name: "multiple machines", + content: `machine github.com login ghuser password ghtoken +machine gitlab.com login gluser password gltoken`, + host: "gitlab.com", + wantUsername: "gluser", + wantPassword: "gltoken", + }, + { + name: "with comments", + content: `# This is a comment +machine github.com +login myuser +# another comment +password mytoken`, + host: "github.com", + wantUsername: "myuser", + wantPassword: "mytoken", + }, + { + name: "default entry", + content: `machine github.com login ghuser password ghtoken +default login defaultuser password defaultpass`, + host: "unknown.com", + wantUsername: "defaultuser", + wantPassword: "defaultpass", + }, + { + name: "no match", + content: `machine github.com login user password pass`, + host: "gitlab.com", + wantUsername: "", + wantPassword: "", + }, + { + name: "empty content", + content: ``, + host: "github.com", + wantUsername: "", + wantPassword: "", + }, + { + name: "host with port", + content: `machine github.com login user password pass`, + host: "github.com:443", + wantUsername: "user", + wantPassword: "pass", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + username, password := parseNetrc(tt.content, tt.host) + assert.Equal(t, tt.wantUsername, username, "username mismatch") + assert.Equal(t, tt.wantPassword, password, "password mismatch") + }) + } +} + +func TestParseCredentialOutput(t *testing.T) { + tests := []struct { + name string + output string + want map[string]string + }{ + { + name: "standard output", + output: `protocol=https +host=github.com +username=myuser +password=mytoken`, + want: map[string]string{ + "protocol": "https", + "host": "github.com", + "username": "myuser", + "password": "mytoken", + }, + }, + { + name: "empty output", + output: ``, + want: map[string]string{}, + }, + { + name: "single line", + output: `username=user`, + want: map[string]string{"username": "user"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseCredentialOutput(tt.output) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestDetectAuth_FileURL(t *testing.T) { + // File URLs should return nil auth (no authentication needed) + auth, err := DetectAuth("file:///path/to/repo") + require.NoError(t, err) + assert.Nil(t, auth) +} + +func TestAuthMethodName(t *testing.T) { + assert.Equal(t, "anonymous", AuthMethodName(nil)) +} + +func TestTryHTTPSFromEnv(t *testing.T) { + // Save and restore environment + oldPassword := os.Getenv(EnvDoltRemotePassword) + defer os.Setenv(EnvDoltRemotePassword, oldPassword) + + // Test with no env var set + os.Unsetenv(EnvDoltRemotePassword) + auth, err := tryHTTPSFromEnv("") + require.NoError(t, err) + assert.Nil(t, auth) + + // Test with env var set + os.Setenv(EnvDoltRemotePassword, "test-token") + auth, err = tryHTTPSFromEnv("") + require.NoError(t, err) + require.NotNil(t, auth) + assert.Equal(t, "https-basic", AuthMethodName(auth)) +} + +func TestTrySSHKeyFromEnv(t *testing.T) { + // Save and restore environment + oldKey := os.Getenv(EnvGitSSHKey) + defer os.Setenv(EnvGitSSHKey, oldKey) + + // Test with no env var set + os.Unsetenv(EnvGitSSHKey) + auth, err := trySSHKeyFromEnv("git") + require.NoError(t, err) + assert.Nil(t, auth) + + // Test with env var set to a file path that doesn't exist + os.Setenv(EnvGitSSHKey, "/nonexistent/path/to/key") + auth, err = trySSHKeyFromEnv("git") + // Should fail because the key content is not valid + assert.Error(t, err) +} + +func TestTrySSHKeyFiles(t *testing.T) { + // This test verifies the function doesn't panic and handles missing files gracefully + auth, err := trySSHKeyFiles("git") + // May or may not find keys depending on the system + // Just verify no panic and no unexpected error + if err != nil { + t.Logf("trySSHKeyFiles returned error (expected on systems without SSH keys): %v", err) + } + if auth != nil { + t.Logf("trySSHKeyFiles found SSH key") + } +} + +func TestTryNetrc(t *testing.T) { + // Create a temporary netrc file + tmpDir := t.TempDir() + netrcPath := filepath.Join(tmpDir, ".netrc") + + content := `machine github.com login testuser password testpass` + err := os.WriteFile(netrcPath, []byte(content), 0600) + require.NoError(t, err) + + // Save and restore NETRC env var + oldNetrc := os.Getenv("NETRC") + defer os.Setenv("NETRC", oldNetrc) + + os.Setenv("NETRC", netrcPath) + + auth, err := tryNetrc("github.com") + require.NoError(t, err) + require.NotNil(t, auth) + assert.Equal(t, "https-basic", AuthMethodName(auth)) +} + +func TestDetectAuth_SSHURLs(t *testing.T) { + // These tests verify URL parsing works correctly + // Actual auth detection depends on system configuration + + testCases := []struct { + url string + }{ + {"git@github.com:user/repo.git"}, + {"ssh://git@github.com/user/repo.git"}, + {"git://github.com/user/repo.git"}, + } + + for _, tc := range testCases { + t.Run(tc.url, func(t *testing.T) { + // Should not error, even if no credentials found + _, err := DetectAuth(tc.url) + // We allow errors from SSH agent not being available + // The important thing is it doesn't panic + if err != nil { + t.Logf("DetectAuth(%q) returned error (may be expected): %v", tc.url, err) + } + }) + } +} + +func TestDetectAuth_HTTPSURLs(t *testing.T) { + testCases := []struct { + url string + }{ + {"https://github.com/user/repo.git"}, + {"https://gitlab.com/user/repo.git"}, + {"http://localhost:8080/repo.git"}, + } + + for _, tc := range testCases { + t.Run(tc.url, func(t *testing.T) { + // Should not error, even if no credentials found + auth, err := DetectAuth(tc.url) + require.NoError(t, err) + // Auth may be nil (anonymous) if no credentials configured + t.Logf("DetectAuth(%q) returned auth: %s", tc.url, AuthMethodName(auth)) + }) + } +} diff --git a/go/libraries/doltcore/gitremote/doc.go b/go/libraries/doltcore/gitremote/doc.go new file mode 100644 index 0000000000..eeb331c7fd --- /dev/null +++ b/go/libraries/doltcore/gitremote/doc.go @@ -0,0 +1,44 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gitremote provides support for using git repositories as dolt remote backends. +// It enables clone, fetch, push, and pull operations to/from git remotes using a custom +// ref (refs/dolt/data) that doesn't interfere with normal git operations. +// +// # Authentication +// +// The package automatically detects git credentials from multiple sources: +// +// For SSH URLs (git@host:path or ssh://): +// - SSH agent (if running) +// - SSH key files (~/.ssh/id_ed25519, id_rsa, etc.) +// - GIT_SSH_KEY environment variable +// +// For HTTPS URLs: +// - Git credential helper (git credential fill) +// - DOLT_REMOTE_PASSWORD environment variable +// - ~/.netrc file +// +// # URL Schemes +// +// Git remotes can be specified using: +// - git:// scheme: git://github.com/user/repo.git +// - HTTPS with .git suffix: https://github.com/user/repo.git +// +// # Data Storage +// +// Dolt data is stored on a custom git ref (default: refs/dolt/data) under the +// .dolt_remote/ directory structure. This ref is not cloned or fetched by +// default git operations, keeping dolt data separate from normal git content. +package gitremote diff --git a/go/libraries/doltcore/gitremote/errors.go b/go/libraries/doltcore/gitremote/errors.go new file mode 100644 index 0000000000..8c698b71e1 --- /dev/null +++ b/go/libraries/doltcore/gitremote/errors.go @@ -0,0 +1,241 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitremote + +import ( + "errors" + "fmt" + "strings" +) + +// Git remote error types with user-friendly messages and resolution hints. +var ( + // ErrRepoNotInitialized indicates the git repository hasn't been set up for dolt. + ErrRepoNotInitialized = errors.New("git repository not initialized for dolt remote") + + // ErrRefNotFound indicates the custom dolt ref doesn't exist. + ErrRefNotFound = errors.New("git ref not found") + + // ErrNothingToCommit indicates there are no changes to commit. + ErrNothingToCommit = errors.New("nothing to commit") + + // ErrPushRejected indicates the push was rejected by the remote. + ErrPushRejected = errors.New("push rejected by remote") + + // ErrAuthFailed indicates git authentication failed. + ErrAuthFailed = errors.New("git authentication failed") + + // ErrRepoNotFound indicates the git repository doesn't exist. + ErrRepoNotFound = errors.New("git repository not found") + + // ErrNetworkError indicates a network connectivity issue. + ErrNetworkError = errors.New("network error connecting to git remote") + + // ErrInvalidURL indicates the git URL is malformed. + ErrInvalidURL = errors.New("invalid git repository URL") + + // ErrPermissionDenied indicates insufficient permissions. + ErrPermissionDenied = errors.New("permission denied") +) + +// GitRemoteError wraps an error with additional context for git remote operations. +type GitRemoteError struct { + Op string // Operation that failed (e.g., "clone", "push", "fetch") + URL string // Git repository URL + Ref string // Git ref being operated on + Err error // Underlying error + Resolution string // Suggested resolution +} + +func (e *GitRemoteError) Error() string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("git %s failed", e.Op)) + if e.URL != "" { + sb.WriteString(fmt.Sprintf(" for %s", e.URL)) + } + if e.Ref != "" { + sb.WriteString(fmt.Sprintf(" (ref: %s)", e.Ref)) + } + if e.Err != nil { + sb.WriteString(fmt.Sprintf(": %v", e.Err)) + } + return sb.String() +} + +func (e *GitRemoteError) Unwrap() error { + return e.Err +} + +// Hint returns a user-friendly hint for resolving the error. +func (e *GitRemoteError) Hint() string { + if e.Resolution != "" { + return e.Resolution + } + + // Provide default hints based on error type + if errors.Is(e.Err, ErrRepoNotInitialized) { + return "Run 'dolt remote init ' to initialize the git repository for dolt." + } + if errors.Is(e.Err, ErrRefNotFound) { + return "The dolt data ref doesn't exist. Run 'dolt remote init ' first." + } + if errors.Is(e.Err, ErrPushRejected) { + return "The remote has changes you don't have locally. Pull changes first or use --force." + } + if errors.Is(e.Err, ErrAuthFailed) { + return "Check your git credentials. For SSH, ensure your key is added to ssh-agent. For HTTPS, check your credential helper or set DOLT_REMOTE_PASSWORD." + } + if errors.Is(e.Err, ErrRepoNotFound) { + return "Verify the git repository URL is correct and the repository exists." + } + if errors.Is(e.Err, ErrNetworkError) { + return "Check your network connection and verify the git remote is accessible." + } + if errors.Is(e.Err, ErrInvalidURL) { + return "Git remote URLs must use the git:// scheme or end with .git (e.g., https://github.com/user/repo.git)." + } + if errors.Is(e.Err, ErrPermissionDenied) { + return "You don't have permission to access this repository. Check your credentials and repository access rights." + } + + return "" +} + +// NewCloneError creates an error for clone operations. +func NewCloneError(url string, err error) *GitRemoteError { + return &GitRemoteError{ + Op: "clone", + URL: url, + Err: classifyError(err), + } +} + +// NewPushError creates an error for push operations. +func NewPushError(url, ref string, err error) *GitRemoteError { + return &GitRemoteError{ + Op: "push", + URL: url, + Ref: ref, + Err: classifyError(err), + } +} + +// NewFetchError creates an error for fetch operations. +func NewFetchError(url, ref string, err error) *GitRemoteError { + return &GitRemoteError{ + Op: "fetch", + URL: url, + Ref: ref, + Err: classifyError(err), + } +} + +// NewAuthError creates an error for authentication failures. +func NewAuthError(url string, err error) *GitRemoteError { + return &GitRemoteError{ + Op: "authenticate", + URL: url, + Err: ErrAuthFailed, + } +} + +// classifyError attempts to classify a generic error into a known error type. +func classifyError(err error) error { + if err == nil { + return nil + } + + errStr := strings.ToLower(err.Error()) + + // Authentication errors + if strings.Contains(errStr, "authentication") || + strings.Contains(errStr, "permission denied") || + strings.Contains(errStr, "publickey") || + strings.Contains(errStr, "invalid credentials") || + strings.Contains(errStr, "401") || + strings.Contains(errStr, "403") { + return fmt.Errorf("%w: %v", ErrAuthFailed, err) + } + + // Repository not found + if strings.Contains(errStr, "repository not found") || + strings.Contains(errStr, "not found") && strings.Contains(errStr, "repo") || + strings.Contains(errStr, "404") { + return fmt.Errorf("%w: %v", ErrRepoNotFound, err) + } + + // Push rejected + if strings.Contains(errStr, "non-fast-forward") || + strings.Contains(errStr, "rejected") || + strings.Contains(errStr, "failed to push") { + return fmt.Errorf("%w: %v", ErrPushRejected, err) + } + + // Network errors + if strings.Contains(errStr, "network") || + strings.Contains(errStr, "connection") || + strings.Contains(errStr, "timeout") || + strings.Contains(errStr, "dial") || + strings.Contains(errStr, "no such host") { + return fmt.Errorf("%w: %v", ErrNetworkError, err) + } + + // Reference not found + if strings.Contains(errStr, "reference not found") || + strings.Contains(errStr, "couldn't find remote ref") { + return fmt.Errorf("%w: %v", ErrRefNotFound, err) + } + + // Permission denied + if strings.Contains(errStr, "permission denied") || + strings.Contains(errStr, "access denied") { + return fmt.Errorf("%w: %v", ErrPermissionDenied, err) + } + + return err +} + +// IsAuthError returns true if the error is an authentication error. +func IsAuthError(err error) bool { + return errors.Is(err, ErrAuthFailed) +} + +// IsPushRejectedError returns true if the error is a push rejection. +func IsPushRejectedError(err error) bool { + return errors.Is(err, ErrPushRejected) +} + +// IsNotFoundError returns true if the error indicates something wasn't found. +func IsNotFoundError(err error) bool { + return errors.Is(err, ErrRepoNotFound) || errors.Is(err, ErrRefNotFound) +} + +// IsNetworkError returns true if the error is a network-related error. +func IsNetworkError(err error) bool { + return errors.Is(err, ErrNetworkError) +} + +// FormatErrorWithHint formats an error with its resolution hint for display. +func FormatErrorWithHint(err error) string { + var gitErr *GitRemoteError + if errors.As(err, &gitErr) { + hint := gitErr.Hint() + if hint != "" { + return fmt.Sprintf("%s\nhint: %s", gitErr.Error(), hint) + } + return gitErr.Error() + } + return err.Error() +} diff --git a/go/libraries/doltcore/gitremote/errors_test.go b/go/libraries/doltcore/gitremote/errors_test.go new file mode 100644 index 0000000000..d2aad3b9e1 --- /dev/null +++ b/go/libraries/doltcore/gitremote/errors_test.go @@ -0,0 +1,296 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitremote + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGitRemoteError_Error(t *testing.T) { + tests := []struct { + name string + err *GitRemoteError + expected string + }{ + { + name: "basic error", + err: &GitRemoteError{ + Op: "clone", + Err: errors.New("connection refused"), + }, + expected: "git clone failed: connection refused", + }, + { + name: "error with URL", + err: &GitRemoteError{ + Op: "push", + URL: "https://github.com/user/repo.git", + Err: ErrPushRejected, + }, + expected: "git push failed for https://github.com/user/repo.git: push rejected by remote", + }, + { + name: "error with URL and ref", + err: &GitRemoteError{ + Op: "fetch", + URL: "https://github.com/user/repo.git", + Ref: "refs/dolt/data", + Err: ErrRefNotFound, + }, + expected: "git fetch failed for https://github.com/user/repo.git (ref: refs/dolt/data): git ref not found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.err.Error()) + }) + } +} + +func TestGitRemoteError_Hint(t *testing.T) { + tests := []struct { + name string + err *GitRemoteError + expectHint bool + hintContain string + }{ + { + name: "repo not initialized", + err: &GitRemoteError{ + Op: "push", + Err: ErrRepoNotInitialized, + }, + expectHint: true, + hintContain: "dolt remote init", + }, + { + name: "ref not found", + err: &GitRemoteError{ + Op: "fetch", + Err: ErrRefNotFound, + }, + expectHint: true, + hintContain: "dolt remote init", + }, + { + name: "push rejected", + err: &GitRemoteError{ + Op: "push", + Err: ErrPushRejected, + }, + expectHint: true, + hintContain: "Pull changes first", + }, + { + name: "auth failed", + err: &GitRemoteError{ + Op: "clone", + Err: ErrAuthFailed, + }, + expectHint: true, + hintContain: "credentials", + }, + { + name: "custom resolution", + err: &GitRemoteError{ + Op: "push", + Err: errors.New("some error"), + Resolution: "Try again later", + }, + expectHint: true, + hintContain: "Try again later", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hint := tt.err.Hint() + if tt.expectHint { + assert.NotEmpty(t, hint) + assert.Contains(t, hint, tt.hintContain) + } + }) + } +} + +func TestClassifyError(t *testing.T) { + tests := []struct { + name string + inputErr error + expectedError error + }{ + { + name: "authentication error", + inputErr: errors.New("authentication required"), + expectedError: ErrAuthFailed, + }, + { + name: "permission denied", + inputErr: errors.New("Permission denied (publickey)"), + expectedError: ErrAuthFailed, + }, + { + name: "401 error", + inputErr: errors.New("server returned 401"), + expectedError: ErrAuthFailed, + }, + { + name: "repository not found", + inputErr: errors.New("repository not found"), + expectedError: ErrRepoNotFound, + }, + { + name: "404 error", + inputErr: errors.New("404 not found"), + expectedError: ErrRepoNotFound, + }, + { + name: "non-fast-forward", + inputErr: errors.New("non-fast-forward update"), + expectedError: ErrPushRejected, + }, + { + name: "push rejected", + inputErr: errors.New("push was rejected"), + expectedError: ErrPushRejected, + }, + { + name: "network timeout", + inputErr: errors.New("connection timeout"), + expectedError: ErrNetworkError, + }, + { + name: "no such host", + inputErr: errors.New("dial tcp: no such host"), + expectedError: ErrNetworkError, + }, + { + name: "reference not found", + inputErr: errors.New("reference not found"), + expectedError: ErrRefNotFound, + }, + { + name: "unknown error", + inputErr: errors.New("something unexpected"), + expectedError: nil, // Should return original error + }, + { + name: "nil error", + inputErr: nil, + expectedError: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := classifyError(tt.inputErr) + if tt.expectedError != nil { + assert.True(t, errors.Is(result, tt.expectedError), + "expected error to wrap %v, got %v", tt.expectedError, result) + } else if tt.inputErr != nil { + // Unknown errors should be returned as-is + assert.Equal(t, tt.inputErr, result) + } else { + assert.Nil(t, result) + } + }) + } +} + +func TestErrorHelpers(t *testing.T) { + t.Run("IsAuthError", func(t *testing.T) { + assert.True(t, IsAuthError(ErrAuthFailed)) + assert.True(t, IsAuthError(classifyError(errors.New("authentication required")))) + assert.False(t, IsAuthError(ErrPushRejected)) + }) + + t.Run("IsPushRejectedError", func(t *testing.T) { + assert.True(t, IsPushRejectedError(ErrPushRejected)) + assert.True(t, IsPushRejectedError(classifyError(errors.New("non-fast-forward")))) + assert.False(t, IsPushRejectedError(ErrAuthFailed)) + }) + + t.Run("IsNotFoundError", func(t *testing.T) { + assert.True(t, IsNotFoundError(ErrRepoNotFound)) + assert.True(t, IsNotFoundError(ErrRefNotFound)) + assert.False(t, IsNotFoundError(ErrAuthFailed)) + }) + + t.Run("IsNetworkError", func(t *testing.T) { + assert.True(t, IsNetworkError(ErrNetworkError)) + assert.True(t, IsNetworkError(classifyError(errors.New("connection timeout")))) + assert.False(t, IsNetworkError(ErrAuthFailed)) + }) +} + +func TestFormatErrorWithHint(t *testing.T) { + t.Run("GitRemoteError with hint", func(t *testing.T) { + err := &GitRemoteError{ + Op: "push", + URL: "https://github.com/user/repo.git", + Err: ErrPushRejected, + } + formatted := FormatErrorWithHint(err) + assert.Contains(t, formatted, "git push failed") + assert.Contains(t, formatted, "hint:") + }) + + t.Run("regular error", func(t *testing.T) { + err := errors.New("some error") + formatted := FormatErrorWithHint(err) + assert.Equal(t, "some error", formatted) + }) +} + +func TestNewErrorFunctions(t *testing.T) { + t.Run("NewCloneError", func(t *testing.T) { + err := NewCloneError("https://github.com/user/repo.git", errors.New("connection refused")) + assert.Equal(t, "clone", err.Op) + assert.Equal(t, "https://github.com/user/repo.git", err.URL) + }) + + t.Run("NewPushError", func(t *testing.T) { + err := NewPushError("https://github.com/user/repo.git", "refs/dolt/data", errors.New("rejected")) + assert.Equal(t, "push", err.Op) + assert.Equal(t, "refs/dolt/data", err.Ref) + assert.True(t, errors.Is(err.Err, ErrPushRejected)) + }) + + t.Run("NewFetchError", func(t *testing.T) { + err := NewFetchError("https://github.com/user/repo.git", "refs/dolt/data", errors.New("not found")) + assert.Equal(t, "fetch", err.Op) + }) + + t.Run("NewAuthError", func(t *testing.T) { + err := NewAuthError("https://github.com/user/repo.git", errors.New("invalid credentials")) + assert.Equal(t, "authenticate", err.Op) + assert.True(t, errors.Is(err.Err, ErrAuthFailed)) + }) +} + +func TestGitRemoteError_Unwrap(t *testing.T) { + underlying := errors.New("underlying error") + err := &GitRemoteError{ + Op: "push", + Err: underlying, + } + + assert.Equal(t, underlying, errors.Unwrap(err)) + assert.True(t, errors.Is(err, underlying)) +} diff --git a/go/libraries/doltcore/gitremote/repo.go b/go/libraries/doltcore/gitremote/repo.go new file mode 100644 index 0000000000..746e0cea13 --- /dev/null +++ b/go/libraries/doltcore/gitremote/repo.go @@ -0,0 +1,568 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitremote + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/storage/memory" +) + +const ( + // DefaultRef is the default git ref used for dolt remote data + DefaultRef = "refs/dolt/data" + + // DefaultRemoteName is the default name for the git remote + DefaultRemoteName = "origin" + + // DoltRemoteDir is the directory within the git tree that contains dolt data + DoltRemoteDir = ".dolt_remote" + + // DoltRemoteDataDir is the subdirectory for actual data files + DoltRemoteDataDir = ".dolt_remote/data" +) + +// Note: Error types are defined in errors.go + +// GitRepo provides a high-level interface for git operations on a dolt remote. +// It wraps go-git and handles custom ref operations for storing dolt data. +type GitRepo struct { + url string + ref plumbing.ReferenceName + auth transport.AuthMethod + repo *git.Repository + worktree *git.Worktree + fs billy.Filesystem + localPath string // empty if using in-memory storage + mu sync.RWMutex +} + +// OpenOptions configures how a GitRepo is opened or cloned. +type OpenOptions struct { + // URL is the git repository URL + URL string + + // Ref is the git reference to use (default: refs/dolt/data) + Ref string + + // Auth is the authentication method (nil for anonymous) + Auth transport.AuthMethod + + // LocalPath is an optional local directory for the working copy. + // If empty, an in-memory filesystem is used. + LocalPath string +} + +// Open opens or clones a git repository for dolt remote operations. +// If the repository doesn't exist locally, it will be cloned. +// If the custom ref doesn't exist, it will be created on first push. +func Open(ctx context.Context, opts OpenOptions) (*GitRepo, error) { + if opts.URL == "" { + return nil, errors.New("git URL is required") + } + + ref := opts.Ref + if ref == "" { + ref = DefaultRef + } + refName := plumbing.ReferenceName(ref) + + gr := &GitRepo{ + url: opts.URL, + ref: refName, + auth: opts.Auth, + localPath: opts.LocalPath, + } + + var err error + if opts.LocalPath != "" { + err = gr.openOrCloneToPath(ctx, opts.LocalPath) + } else { + err = gr.openOrCloneInMemory(ctx) + } + + if err != nil { + return nil, err + } + + return gr, nil +} + +// openOrCloneToPath opens an existing repo or clones to a local path. +func (gr *GitRepo) openOrCloneToPath(ctx context.Context, path string) error { + // Try to open existing repository + repo, err := git.PlainOpen(path) + if err == nil { + gr.repo = repo + gr.fs = osfs.New(path) + wt, err := repo.Worktree() + if err != nil { + return fmt.Errorf("failed to get worktree: %w", err) + } + gr.worktree = wt + return gr.fetchRef(ctx) + } + + // Need to clone or init + if !errors.Is(err, git.ErrRepositoryNotExists) { + return fmt.Errorf("failed to open repository: %w", err) + } + + // Create directory if needed + if err := os.MkdirAll(path, 0755); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + // Try to clone the repository + repo, err = git.PlainCloneContext(ctx, path, false, &git.CloneOptions{ + URL: gr.url, + Auth: gr.auth, + // Don't check out any branch - we'll work with our custom ref + NoCheckout: true, + }) + + // Handle empty remote repository - init locally and add remote + if err != nil && isEmptyRepoError(err) { + repo, err = gr.initWithRemote(path) + if err != nil { + return fmt.Errorf("failed to init repository: %w", err) + } + } else if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + + gr.repo = repo + gr.fs = osfs.New(path) + wt, err := repo.Worktree() + if err != nil { + return fmt.Errorf("failed to get worktree: %w", err) + } + gr.worktree = wt + + return gr.fetchRef(ctx) +} + +// initWithRemote initializes a new repository and adds the remote. +func (gr *GitRepo) initWithRemote(path string) (*git.Repository, error) { + repo, err := git.PlainInit(path, false) + if err != nil { + return nil, err + } + + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{gr.url}, + }) + if err != nil { + return nil, err + } + + return repo, nil +} + +// isEmptyRepoError checks if the error indicates an empty remote repository. +func isEmptyRepoError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return errors.Is(err, transport.ErrEmptyRemoteRepository) || + contains(errStr, "remote repository is empty") || + // Local bare repos with no commits return "reference not found" + contains(errStr, "reference not found") +} + +// openOrCloneInMemory opens or clones to an in-memory filesystem. +func (gr *GitRepo) openOrCloneInMemory(ctx context.Context) error { + storer := memory.NewStorage() + fs := memfs.New() + + repo, err := git.CloneContext(ctx, storer, fs, &git.CloneOptions{ + URL: gr.url, + Auth: gr.auth, + NoCheckout: true, + }) + + // Handle empty remote repository - init in memory and add remote + if err != nil && isEmptyRepoError(err) { + storer = memory.NewStorage() + fs = memfs.New() + repo, err = git.Init(storer, fs) + if err != nil { + return fmt.Errorf("failed to init repository: %w", err) + } + + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{gr.url}, + }) + if err != nil { + return fmt.Errorf("failed to create remote: %w", err) + } + } else if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + + gr.repo = repo + gr.fs = fs + wt, err := repo.Worktree() + if err != nil { + return fmt.Errorf("failed to get worktree: %w", err) + } + gr.worktree = wt + + return gr.fetchRef(ctx) +} + +// fetchRef fetches the custom ref from the remote. +func (gr *GitRepo) fetchRef(ctx context.Context) error { + refSpec := config.RefSpec(fmt.Sprintf("+%s:%s", gr.ref, gr.ref)) + + err := gr.repo.FetchContext(ctx, &git.FetchOptions{ + RemoteName: DefaultRemoteName, + RefSpecs: []config.RefSpec{refSpec}, + Auth: gr.auth, + Force: true, + }) + + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) && !errors.Is(err, transport.ErrEmptyRemoteRepository) { + // Check if the ref doesn't exist yet (new remote) + if isRefNotFoundError(err) { + // This is OK - the ref will be created on first push + return nil + } + return fmt.Errorf("failed to fetch ref %s: %w", gr.ref, err) + } + + return nil +} + +// isRefNotFoundError checks if an error indicates the ref doesn't exist. +func isRefNotFoundError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return errors.Is(err, plumbing.ErrReferenceNotFound) || + // go-git returns various error messages for missing refs + contains(errStr, "couldn't find remote ref") || + contains(errStr, "reference not found") +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsAt(s, substr, 0)) +} + +func containsAt(s, substr string, start int) bool { + for i := start; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +// CheckoutRef checks out the custom ref to the worktree. +func (gr *GitRepo) CheckoutRef(ctx context.Context) error { + gr.mu.Lock() + defer gr.mu.Unlock() + + // Try to resolve the ref + ref, err := gr.repo.Reference(gr.ref, true) + if err != nil { + if errors.Is(err, plumbing.ErrReferenceNotFound) { + // Ref doesn't exist yet - create empty worktree + return nil + } + return fmt.Errorf("failed to resolve ref: %w", err) + } + + err = gr.worktree.Checkout(&git.CheckoutOptions{ + Hash: ref.Hash(), + Force: true, + }) + if err != nil { + return fmt.Errorf("failed to checkout: %w", err) + } + + return nil +} + +// ReadFile reads a file from the worktree. +func (gr *GitRepo) ReadFile(path string) ([]byte, error) { + gr.mu.RLock() + defer gr.mu.RUnlock() + + f, err := gr.fs.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + return io.ReadAll(f) +} + +// WriteFile writes a file to the worktree. +func (gr *GitRepo) WriteFile(path string, data []byte) error { + gr.mu.Lock() + defer gr.mu.Unlock() + + // Ensure parent directory exists + dir := filepath.Dir(path) + if dir != "." && dir != "/" { + if err := gr.fs.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", dir, err) + } + } + + f, err := gr.fs.Create(path) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", path, err) + } + defer f.Close() + + _, err = f.Write(data) + if err != nil { + return fmt.Errorf("failed to write file %s: %w", path, err) + } + + return nil +} + +// DeleteFile removes a file from the worktree. +func (gr *GitRepo) DeleteFile(path string) error { + gr.mu.Lock() + defer gr.mu.Unlock() + + return gr.fs.Remove(path) +} + +// FileExists checks if a file exists in the worktree. +func (gr *GitRepo) FileExists(path string) (bool, error) { + gr.mu.RLock() + defer gr.mu.RUnlock() + + _, err := gr.fs.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// ListFiles lists files in a directory. +func (gr *GitRepo) ListFiles(dir string) ([]string, error) { + gr.mu.RLock() + defer gr.mu.RUnlock() + + entries, err := gr.fs.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + var files []string + for _, entry := range entries { + files = append(files, entry.Name()) + } + return files, nil +} + +// Commit creates a new commit with all staged changes. +// Returns the commit hash. +func (gr *GitRepo) Commit(ctx context.Context, message string) (plumbing.Hash, error) { + gr.mu.Lock() + defer gr.mu.Unlock() + + // Stage all changes + if err := gr.worktree.AddGlob("."); err != nil { + return plumbing.ZeroHash, fmt.Errorf("failed to stage changes: %w", err) + } + + // Check if there are changes to commit + status, err := gr.worktree.Status() + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("failed to get status: %w", err) + } + + if status.IsClean() { + return plumbing.ZeroHash, ErrNothingToCommit + } + + // Create commit + hash, err := gr.worktree.Commit(message, &git.CommitOptions{ + Author: &object.Signature{ + Name: "Dolt", + Email: "dolt@dolthub.com", + When: time.Now(), + }, + }) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("failed to commit: %w", err) + } + + // Update the custom ref to point to the new commit + ref := plumbing.NewHashReference(gr.ref, hash) + if err := gr.repo.Storer.SetReference(ref); err != nil { + return plumbing.ZeroHash, fmt.Errorf("failed to update ref: %w", err) + } + + return hash, nil +} + +// Push pushes the custom ref to the remote. +func (gr *GitRepo) Push(ctx context.Context) error { + gr.mu.Lock() + defer gr.mu.Unlock() + + refSpec := config.RefSpec(fmt.Sprintf("%s:%s", gr.ref, gr.ref)) + + err := gr.repo.PushContext(ctx, &git.PushOptions{ + RemoteName: DefaultRemoteName, + RefSpecs: []config.RefSpec{refSpec}, + Auth: gr.auth, + }) + + if err != nil { + if errors.Is(err, git.NoErrAlreadyUpToDate) { + return nil + } + return fmt.Errorf("failed to push: %w", err) + } + + return nil +} + +// CurrentCommit returns the hash of the current commit on the custom ref. +func (gr *GitRepo) CurrentCommit() (plumbing.Hash, error) { + gr.mu.RLock() + defer gr.mu.RUnlock() + + ref, err := gr.repo.Reference(gr.ref, true) + if err != nil { + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return plumbing.ZeroHash, nil + } + return plumbing.ZeroHash, err + } + + return ref.Hash(), nil +} + +// Fetch fetches the latest changes from the remote. +func (gr *GitRepo) Fetch(ctx context.Context) error { + gr.mu.Lock() + defer gr.mu.Unlock() + + return gr.fetchRef(ctx) +} + +// URL returns the repository URL. +func (gr *GitRepo) URL() string { + return gr.url +} + +// Ref returns the git reference being used. +func (gr *GitRepo) Ref() string { + return string(gr.ref) +} + +// Close cleans up resources. For in-memory repos, this is a no-op. +// For on-disk repos, this releases file handles. +func (gr *GitRepo) Close() error { + gr.mu.Lock() + defer gr.mu.Unlock() + + // go-git doesn't have explicit close, but we clear references + gr.repo = nil + gr.worktree = nil + gr.fs = nil + + return nil +} + +// InitRemote initializes a git repository as a dolt remote. +// This creates the .dolt_remote directory structure and README. +// It is idempotent - safe to call multiple times. +func (gr *GitRepo) InitRemote(ctx context.Context) error { + // Check if already initialized + exists, err := gr.FileExists(DoltRemoteDir + "/README.md") + if err != nil { + return err + } + if exists { + // Already initialized + return nil + } + + // Create directory structure + if err := gr.WriteFile(DoltRemoteDir+"/README.md", []byte(doltRemoteReadme)); err != nil { + return fmt.Errorf("failed to create README: %w", err) + } + + // Create empty data directory marker + if err := gr.WriteFile(DoltRemoteDataDir+"/.gitkeep", []byte("")); err != nil { + return fmt.Errorf("failed to create data directory: %w", err) + } + + // Commit the initialization + _, err = gr.Commit(ctx, "Initialize dolt remote") + if err != nil && !errors.Is(err, ErrNothingToCommit) { + return fmt.Errorf("failed to commit initialization: %w", err) + } + + // Push to remote + if err := gr.Push(ctx); err != nil { + return fmt.Errorf("failed to push initialization: %w", err) + } + + return nil +} + +// IsInitialized checks if the repository has been initialized as a dolt remote. +func (gr *GitRepo) IsInitialized() (bool, error) { + return gr.FileExists(DoltRemoteDir + "/README.md") +} + +const doltRemoteReadme = `# Dolt Remote Data + +This directory contains Dolt database remote storage data. + +**WARNING**: Do not manually modify files in this directory. + +This data is managed by Dolt and uses a custom git ref (` + "`refs/dolt/data`" + `) +that is not part of the normal branch structure. + +For more information, visit: https://docs.dolthub.com/ +` diff --git a/go/libraries/doltcore/gitremote/repo_test.go b/go/libraries/doltcore/gitremote/repo_test.go new file mode 100644 index 0000000000..b7c1e25e1a --- /dev/null +++ b/go/libraries/doltcore/gitremote/repo_test.go @@ -0,0 +1,423 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gitremote + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createBareRepo creates a bare git repository for testing. +func createBareRepo(t *testing.T) string { + t.Helper() + dir := t.TempDir() + bareDir := filepath.Join(dir, "bare.git") + + _, err := git.PlainInit(bareDir, true) + require.NoError(t, err) + + return bareDir +} + +// createRepoWithCommit creates a bare repo with an initial commit on the default branch. +func createRepoWithCommit(t *testing.T) string { + t.Helper() + bareDir := createBareRepo(t) + + // Init a work directory and add the bare repo as remote + workDir := filepath.Join(t.TempDir(), "work") + repo, err := git.PlainInit(workDir, false) + require.NoError(t, err) + + // Add remote + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{bareDir}, + }) + require.NoError(t, err) + + // Create a file + testFile := filepath.Join(workDir, "test.txt") + err = os.WriteFile(testFile, []byte("test content"), 0644) + require.NoError(t, err) + + // Add and commit + wt, err := repo.Worktree() + require.NoError(t, err) + + _, err = wt.Add("test.txt") + require.NoError(t, err) + + _, err = wt.Commit("Initial commit", &git.CommitOptions{}) + require.NoError(t, err) + + // Push + err = repo.Push(&git.PushOptions{}) + require.NoError(t, err) + + return bareDir +} + +func TestOpenOptions_Defaults(t *testing.T) { + opts := OpenOptions{ + URL: "file:///test/repo", + } + + assert.Equal(t, "file:///test/repo", opts.URL) + assert.Empty(t, opts.Ref) + assert.Nil(t, opts.Auth) + assert.Empty(t, opts.LocalPath) +} + +func TestGitRepo_OpenBareRepo(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + assert.Equal(t, bareDir, gr.URL()) + assert.Equal(t, DefaultRef, gr.Ref()) +} + +func TestGitRepo_WriteAndReadFile(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + // Write a file + testContent := []byte("hello, world!") + err = gr.WriteFile("test/file.txt", testContent) + require.NoError(t, err) + + // Read it back + content, err := gr.ReadFile("test/file.txt") + require.NoError(t, err) + assert.Equal(t, testContent, content) + + // Check existence + exists, err := gr.FileExists("test/file.txt") + require.NoError(t, err) + assert.True(t, exists) + + exists, err = gr.FileExists("nonexistent.txt") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestGitRepo_DeleteFile(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + // Write a file + err = gr.WriteFile("todelete.txt", []byte("delete me")) + require.NoError(t, err) + + // Verify it exists + exists, err := gr.FileExists("todelete.txt") + require.NoError(t, err) + assert.True(t, exists) + + // Delete it + err = gr.DeleteFile("todelete.txt") + require.NoError(t, err) + + // Verify it's gone + exists, err = gr.FileExists("todelete.txt") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestGitRepo_ListFiles(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + // Create some files + err = gr.WriteFile("dir/file1.txt", []byte("1")) + require.NoError(t, err) + err = gr.WriteFile("dir/file2.txt", []byte("2")) + require.NoError(t, err) + err = gr.WriteFile("dir/file3.txt", []byte("3")) + require.NoError(t, err) + + // List files + files, err := gr.ListFiles("dir") + require.NoError(t, err) + assert.Len(t, files, 3) + assert.Contains(t, files, "file1.txt") + assert.Contains(t, files, "file2.txt") + assert.Contains(t, files, "file3.txt") + + // List nonexistent directory + files, err = gr.ListFiles("nonexistent") + require.NoError(t, err) + assert.Nil(t, files) +} + +func TestGitRepo_CommitAndPush(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + // Write a file + err = gr.WriteFile("data.txt", []byte("some data")) + require.NoError(t, err) + + // Commit + hash, err := gr.Commit(ctx, "Add data file") + require.NoError(t, err) + assert.NotEqual(t, plumbing.ZeroHash, hash) + + // Push + err = gr.Push(ctx) + require.NoError(t, err) + + // Verify the commit is on the custom ref + currentHash, err := gr.CurrentCommit() + require.NoError(t, err) + assert.Equal(t, hash, currentHash) +} + +func TestGitRepo_CommitNoChanges(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + // Try to commit with no changes + _, err = gr.Commit(ctx, "Empty commit") + assert.ErrorIs(t, err, ErrNothingToCommit) +} + +func TestGitRepo_InitRemote(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + // Check not initialized + initialized, err := gr.IsInitialized() + require.NoError(t, err) + assert.False(t, initialized) + + // Initialize + err = gr.InitRemote(ctx) + require.NoError(t, err) + + // Check initialized + initialized, err = gr.IsInitialized() + require.NoError(t, err) + assert.True(t, initialized) + + // Verify README exists + content, err := gr.ReadFile(DoltRemoteDir + "/README.md") + require.NoError(t, err) + assert.Contains(t, string(content), "Dolt Remote Data") + + // Verify data directory exists + exists, err := gr.FileExists(DoltRemoteDataDir + "/.gitkeep") + require.NoError(t, err) + assert.True(t, exists) + + // Initialize again (should be idempotent) + err = gr.InitRemote(ctx) + require.NoError(t, err) +} + +func TestGitRepo_CustomRef(t *testing.T) { + bareDir := createBareRepo(t) + ctx := context.Background() + + workDir := filepath.Join(t.TempDir(), "work") + customRef := "refs/dolt/custom" + + gr, err := Open(ctx, OpenOptions{ + URL: bareDir, + Ref: customRef, + LocalPath: workDir, + }) + require.NoError(t, err) + defer gr.Close() + + assert.Equal(t, customRef, gr.Ref()) + + // Write, commit, push + err = gr.WriteFile("custom.txt", []byte("custom ref data")) + require.NoError(t, err) + + _, err = gr.Commit(ctx, "Custom ref commit") + require.NoError(t, err) + + err = gr.Push(ctx) + require.NoError(t, err) + + // Verify the ref exists in the bare repo + bareRepo, err := git.PlainOpen(bareDir) + require.NoError(t, err) + + ref, err := bareRepo.Reference(plumbing.ReferenceName(customRef), true) + require.NoError(t, err) + assert.NotNil(t, ref) +} + +func TestGitRepo_FetchUpdates(t *testing.T) { + // Use a repo with an initial commit so clone works + bareDir := createRepoWithCommit(t) + ctx := context.Background() + + // First client writes and pushes to custom ref + workDir1 := filepath.Join(t.TempDir(), "work1") + gr1, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir1, + }) + require.NoError(t, err) + defer gr1.Close() + + err = gr1.WriteFile("file1.txt", []byte("from client 1")) + require.NoError(t, err) + + hash1, err := gr1.Commit(ctx, "Client 1 commit") + require.NoError(t, err) + + err = gr1.Push(ctx) + require.NoError(t, err) + + // Second client opens and should see the changes after fetch + workDir2 := filepath.Join(t.TempDir(), "work2") + gr2, err := Open(ctx, OpenOptions{ + URL: bareDir, + LocalPath: workDir2, + }) + require.NoError(t, err) + defer gr2.Close() + + // Checkout the ref to see the files + err = gr2.CheckoutRef(ctx) + require.NoError(t, err) + + // Should see the file from client 1 + content, err := gr2.ReadFile("file1.txt") + require.NoError(t, err) + assert.Equal(t, []byte("from client 1"), content) + + // Current commit should match + hash2, err := gr2.CurrentCommit() + require.NoError(t, err) + assert.Equal(t, hash1, hash2) +} + +func TestIsRefNotFoundError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "nil error", + err: nil, + want: false, + }, + { + name: "reference not found", + err: plumbing.ErrReferenceNotFound, + want: true, + }, + { + name: "couldn't find remote ref message", + err: assert.AnError, // generic error + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isRefNotFoundError(tt.err) + // For the plumbing.ErrReferenceNotFound case + if tt.err == plumbing.ErrReferenceNotFound { + assert.True(t, got) + } + }) + } +} + +func TestConstants(t *testing.T) { + assert.Equal(t, "refs/dolt/data", DefaultRef) + assert.Equal(t, "origin", DefaultRemoteName) + assert.Equal(t, ".dolt_remote", DoltRemoteDir) + assert.Equal(t, ".dolt_remote/data", DoltRemoteDataDir) +} diff --git a/go/store/blobstore/git.go b/go/store/blobstore/git.go new file mode 100644 index 0000000000..a5068a610a --- /dev/null +++ b/go/store/blobstore/git.go @@ -0,0 +1,209 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blobstore + +import ( + "context" + "io" + "os" + "path/filepath" + "sync" + + "github.com/dolthub/dolt/go/libraries/doltcore/gitremote" +) + +// GitBlobstore is a Blobstore implementation that uses a git repository as the backing store. +// It wraps a LocalBlobstore for the actual file operations, storing blobs under +// .dolt_remote/data/ in the git worktree. Sync operations (commit + push) are handled +// separately from individual blob operations. +type GitBlobstore struct { + local *LocalBlobstore + repo *gitremote.GitRepo + ref string + repoURL string + localPath string + mu sync.RWMutex +} + +var _ Blobstore = &GitBlobstore{} + +// NewGitBlobstore creates a new GitBlobstore for the given repository URL. +// The localPath is used as a cache directory for git operations. +// If localPath is empty, a temporary directory will be created. +func NewGitBlobstore(ctx context.Context, repoURL, ref, localPath string) (*GitBlobstore, error) { + if localPath == "" { + var err error + localPath, err = os.MkdirTemp("", "dolt-git-blobstore-*") + if err != nil { + return nil, err + } + } + + if ref == "" { + ref = gitremote.DefaultRef + } + + // Detect authentication + auth, err := gitremote.DetectAuth(repoURL) + if err != nil { + return nil, err + } + + // Open or clone the repository + repo, err := gitremote.Open(ctx, gitremote.OpenOptions{ + URL: repoURL, + Ref: ref, + Auth: auth, + LocalPath: localPath, + }) + if err != nil { + return nil, err + } + + // Checkout the ref to populate the worktree + if err := repo.CheckoutRef(ctx); err != nil { + return nil, err + } + + // Create the data directory for blobs + dataDir := filepath.Join(localPath, gitremote.DoltRemoteDataDir) + if err := os.MkdirAll(dataDir, 0755); err != nil { + return nil, err + } + + // Create a LocalBlobstore pointing to the data directory + local := NewLocalBlobstore(dataDir) + + return &GitBlobstore{ + local: local, + repo: repo, + ref: ref, + repoURL: repoURL, + localPath: localPath, + }, nil +} + +// Path returns the git repository URL. +func (bs *GitBlobstore) Path() string { + return bs.repoURL +} + +// Exists returns true if a blob with the given key exists. +func (bs *GitBlobstore) Exists(ctx context.Context, key string) (bool, error) { + bs.mu.RLock() + defer bs.mu.RUnlock() + return bs.local.Exists(ctx, key) +} + +// Get retrieves a blob by key, returning a reader for the specified byte range. +func (bs *GitBlobstore) Get(ctx context.Context, key string, br BlobRange) (io.ReadCloser, uint64, string, error) { + bs.mu.RLock() + defer bs.mu.RUnlock() + return bs.local.Get(ctx, key, br) +} + +// Put stores a blob with the given key in the local worktree. +// This does NOT commit or push - call Sync() to persist changes to the remote. +func (bs *GitBlobstore) Put(ctx context.Context, key string, totalSize int64, reader io.Reader) (string, error) { + bs.mu.Lock() + defer bs.mu.Unlock() + return bs.local.Put(ctx, key, totalSize, reader) +} + +// CheckAndPut stores a blob only if the current version matches expectedVersion. +// For git blobstores, this also commits and pushes all pending changes to ensure +// the remote is in sync with the local state. +func (bs *GitBlobstore) CheckAndPut(ctx context.Context, expectedVersion, key string, totalSize int64, reader io.Reader) (string, error) { + bs.mu.Lock() + defer bs.mu.Unlock() + + // First, do the local CheckAndPut + ver, err := bs.local.CheckAndPut(ctx, expectedVersion, key, totalSize, reader) + if err != nil { + return "", err + } + + // Now commit and push all pending changes (including this one) + _, commitErr := bs.repo.Commit(ctx, "Update "+key) + if commitErr != nil && commitErr != gitremote.ErrNothingToCommit { + return "", commitErr + } + + if commitErr != gitremote.ErrNothingToCommit { + if pushErr := bs.repo.Push(ctx); pushErr != nil { + return "", pushErr + } + } + + return ver, nil +} + +// Concatenate creates a new blob by concatenating the contents of the source blobs. +func (bs *GitBlobstore) Concatenate(ctx context.Context, key string, sources []string) (string, error) { + bs.mu.Lock() + defer bs.mu.Unlock() + return bs.local.Concatenate(ctx, key, sources) +} + +// Sync commits all pending changes and pushes to the remote. +// This should be called after a batch of Put operations to persist changes. +func (bs *GitBlobstore) Sync(ctx context.Context, message string) error { + bs.mu.Lock() + defer bs.mu.Unlock() + + // Commit changes + _, err := bs.repo.Commit(ctx, message) + if err != nil { + if err == gitremote.ErrNothingToCommit { + return nil // Nothing to sync + } + return err + } + + // Push to remote + return bs.repo.Push(ctx) +} + +// Fetch pulls the latest changes from the remote. +func (bs *GitBlobstore) Fetch(ctx context.Context) error { + bs.mu.Lock() + defer bs.mu.Unlock() + + if err := bs.repo.Fetch(ctx); err != nil { + return err + } + return bs.repo.CheckoutRef(ctx) +} + +// Close releases resources associated with this blobstore. +func (bs *GitBlobstore) Close() error { + bs.mu.Lock() + defer bs.mu.Unlock() + + if bs.repo != nil { + return bs.repo.Close() + } + return nil +} + +// Ref returns the git ref being used. +func (bs *GitBlobstore) Ref() string { + return bs.ref +} + +// LocalPath returns the local cache directory path. +func (bs *GitBlobstore) LocalPath() string { + return bs.localPath +} diff --git a/go/store/blobstore/git_test.go b/go/store/blobstore/git_test.go new file mode 100644 index 0000000000..bbce6de213 --- /dev/null +++ b/go/store/blobstore/git_test.go @@ -0,0 +1,302 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package blobstore + +import ( + "bytes" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/libraries/doltcore/gitremote" +) + +// createTestGitRepo creates a bare git repo and initializes it with a commit +// so it can be cloned. +func createTestGitRepo(t *testing.T) string { + t.Helper() + + // Create bare repo + bareDir := t.TempDir() + barePath := filepath.Join(bareDir, "test.git") + cmd := exec.Command("git", "init", "--bare", barePath) + require.NoError(t, cmd.Run()) + + // Create a working repo, add initial commit, push to bare + workDir := t.TempDir() + cmd = exec.Command("git", "init", workDir) + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "-C", workDir, "config", "user.email", "test@test.com") + require.NoError(t, cmd.Run()) + cmd = exec.Command("git", "-C", workDir, "config", "user.name", "Test") + require.NoError(t, cmd.Run()) + + // Create .dolt_remote structure + dataDir := filepath.Join(workDir, gitremote.DoltRemoteDataDir) + require.NoError(t, os.MkdirAll(dataDir, 0755)) + + readmePath := filepath.Join(workDir, gitremote.DoltRemoteDir, "README.md") + require.NoError(t, os.WriteFile(readmePath, []byte("# Dolt Remote\n"), 0644)) + + cmd = exec.Command("git", "-C", workDir, "add", ".") + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "-C", workDir, "commit", "-m", "init") + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "-C", workDir, "remote", "add", "origin", barePath) + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "-C", workDir, "push", "-u", "origin", "HEAD:refs/dolt/data") + require.NoError(t, cmd.Run()) + + return barePath +} + +func TestGitBlobstore_Path(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + assert.Equal(t, barePath, bs.Path()) +} + +func TestGitBlobstore_PutAndGet(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Put a blob + data := []byte("hello world") + _, err = bs.Put(ctx, "test-key", int64(len(data)), bytes.NewReader(data)) + require.NoError(t, err) + + // Get it back + rc, size, _, err := bs.Get(ctx, "test-key", BlobRange{}) + require.NoError(t, err) + defer rc.Close() + + assert.Equal(t, uint64(len(data)), size) + + got, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, data, got) +} + +func TestGitBlobstore_Exists(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Should not exist initially + exists, err := bs.Exists(ctx, "nonexistent") + require.NoError(t, err) + assert.False(t, exists) + + // Put a blob + data := []byte("test data") + _, err = bs.Put(ctx, "exists-key", int64(len(data)), bytes.NewReader(data)) + require.NoError(t, err) + + // Should exist now + exists, err = bs.Exists(ctx, "exists-key") + require.NoError(t, err) + assert.True(t, exists) +} + +func TestGitBlobstore_GetNotFound(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + _, _, _, err = bs.Get(ctx, "nonexistent", BlobRange{}) + assert.True(t, IsNotFoundError(err)) +} + +func TestGitBlobstore_GetByteRange(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Put a blob + data := []byte("0123456789") + _, err = bs.Put(ctx, "range-key", int64(len(data)), bytes.NewReader(data)) + require.NoError(t, err) + + // Get a range + rc, _, _, err := bs.Get(ctx, "range-key", BlobRange{offset: 2, length: 5}) + require.NoError(t, err) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, []byte("23456"), got) +} + +func TestGitBlobstore_Concatenate(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Put source blobs + _, err = bs.Put(ctx, "src1", 5, bytes.NewReader([]byte("hello"))) + require.NoError(t, err) + _, err = bs.Put(ctx, "src2", 5, bytes.NewReader([]byte("world"))) + require.NoError(t, err) + + // Concatenate + _, err = bs.Concatenate(ctx, "combined", []string{"src1", "src2"}) + require.NoError(t, err) + + // Verify + rc, _, _, err := bs.Get(ctx, "combined", BlobRange{}) + require.NoError(t, err) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, []byte("helloworld"), got) +} + +func TestGitBlobstore_CheckAndPut(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Put initial blob + data1 := []byte("initial") + ver1, err := bs.Put(ctx, "cap-key", int64(len(data1)), bytes.NewReader(data1)) + require.NoError(t, err) + + // CheckAndPut with correct version should succeed + data2 := []byte("updated") + _, err = bs.CheckAndPut(ctx, ver1, "cap-key", int64(len(data2)), bytes.NewReader(data2)) + require.NoError(t, err) + + // Verify update + rc, _, _, err := bs.Get(ctx, "cap-key", BlobRange{}) + require.NoError(t, err) + got, _ := io.ReadAll(rc) + rc.Close() + assert.Equal(t, data2, got) +} + +func TestGitBlobstore_CheckAndPut_VersionMismatch(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Put initial blob + data1 := []byte("initial") + _, err = bs.Put(ctx, "cap-key2", int64(len(data1)), bytes.NewReader(data1)) + require.NoError(t, err) + + // CheckAndPut with wrong version should fail + data2 := []byte("updated") + _, err = bs.CheckAndPut(ctx, "wrong-version", "cap-key2", int64(len(data2)), bytes.NewReader(data2)) + assert.Error(t, err) + + var capErr CheckAndPutError + assert.ErrorAs(t, err, &capErr) +} + +func TestGitBlobstore_Sync(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Put some blobs + _, err = bs.Put(ctx, "sync1", 4, bytes.NewReader([]byte("data"))) + require.NoError(t, err) + _, err = bs.Put(ctx, "sync2", 4, bytes.NewReader([]byte("more"))) + require.NoError(t, err) + + // Sync to remote + err = bs.Sync(ctx, "test sync") + require.NoError(t, err) + + // Create a new blobstore from the same repo to verify data was pushed + bs2, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs2.Close() + + // Verify blobs exist in the new instance + exists, err := bs2.Exists(ctx, "sync1") + require.NoError(t, err) + assert.True(t, exists) + + exists, err = bs2.Exists(ctx, "sync2") + require.NoError(t, err) + assert.True(t, exists) +} + +func TestGitBlobstore_MultiplePuts(t *testing.T) { + barePath := createTestGitRepo(t) + ctx := context.Background() + + bs, err := NewGitBlobstore(ctx, barePath, gitremote.DefaultRef, "") + require.NoError(t, err) + defer bs.Close() + + // Put multiple blobs + for i := 0; i < 10; i++ { + key := "multi-" + string(rune('a'+i)) + data := []byte{byte(i)} + _, err = bs.Put(ctx, key, 1, bytes.NewReader(data)) + require.NoError(t, err) + } + + // Verify all exist + for i := 0; i < 10; i++ { + key := "multi-" + string(rune('a'+i)) + exists, err := bs.Exists(ctx, key) + require.NoError(t, err) + assert.True(t, exists, "key %s should exist", key) + } +} diff --git a/integration-tests/bats/remotes-git.bats b/integration-tests/bats/remotes-git.bats new file mode 100644 index 0000000000..8d6a987d59 --- /dev/null +++ b/integration-tests/bats/remotes-git.bats @@ -0,0 +1,195 @@ +#!/usr/bin/env bats + +# Git remotes allow using git repositories as dolt remote backends +# These tests cover the `dolt remote init` command which initializes +# a git repository for use as a dolt remote. +# +# Note: Full push/pull/clone integration with git remotes requires +# additional work to integrate the GitFactory with dolt's remote +# operations. These tests focus on the initialization functionality. + +load $BATS_TEST_DIRNAME/helper/common.bash + +setup() { + setup_common + cd $BATS_TMPDIR + cd dolt-repo-$$ + mkdir "git-remotes" +} + +teardown() { + assert_feature_version + teardown_common +} + +# Helper function to create a bare git repository +create_bare_git_repo() { + local name=$1 + git init --bare "git-remotes/${name}.git" > /dev/null 2>&1 +} + +@test "remotes-git: remote init creates dolt structure" { + create_bare_git_repo "test-repo" + + # Initialize the git repo as a dolt remote + run dolt remote init "git-remotes/test-repo.git" + [ "$status" -eq 0 ] + [[ "$output" =~ "Successfully initialized" ]] || false + + # Verify the ref was created + cd git-remotes/test-repo.git + run git show-ref + [ "$status" -eq 0 ] + [[ "$output" =~ "refs/dolt/data" ]] || false +} + +@test "remotes-git: remote init is idempotent" { + create_bare_git_repo "idempotent-repo" + + # First init + run dolt remote init "git-remotes/idempotent-repo.git" + [ "$status" -eq 0 ] + [[ "$output" =~ "Successfully initialized" ]] || false + + # Second init should succeed and report already initialized + run dolt remote init "git-remotes/idempotent-repo.git" + [ "$status" -eq 0 ] + [[ "$output" =~ "already initialized" ]] || false +} + +@test "remotes-git: remote init with custom ref" { + create_bare_git_repo "custom-ref-repo" + + # Initialize with custom ref + run dolt remote init --ref refs/dolt/custom "git-remotes/custom-ref-repo.git" + [ "$status" -eq 0 ] + [[ "$output" =~ "Successfully initialized" ]] || false + + # Verify the custom ref was created + cd git-remotes/custom-ref-repo.git + run git show-ref + [ "$status" -eq 0 ] + [[ "$output" =~ "refs/dolt/custom" ]] || false +} + +@test "remotes-git: remote init fails on invalid URL" { + # Try to init a non-.git path + run dolt remote init "/tmp/not-a-git-repo" + [ "$status" -eq 1 ] + [[ "$output" =~ "not a valid git remote URL" ]] || false +} + +@test "remotes-git: remote init creates README in .dolt_remote" { + create_bare_git_repo "readme-repo" + + # Initialize + run dolt remote init "git-remotes/readme-repo.git" + [ "$status" -eq 0 ] + + # Clone the repo and fetch the dolt ref + cd git-remotes + git clone readme-repo.git readme-check > /dev/null 2>&1 + cd readme-check + git fetch origin refs/dolt/data:refs/dolt/data > /dev/null 2>&1 + git checkout FETCH_HEAD > /dev/null 2>&1 + + # Verify README exists + [ -f ".dolt_remote/README.md" ] + + # Verify data directory exists + [ -d ".dolt_remote/data" ] +} + +@test "remotes-git: remote init works with absolute path" { + create_bare_git_repo "abs-path-repo" + + # Get absolute path + abs_path="$PWD/git-remotes/abs-path-repo.git" + + # Initialize using absolute path + run dolt remote init "$abs_path" + [ "$status" -eq 0 ] + [[ "$output" =~ "Successfully initialized" ]] || false +} + +@test "remotes-git: remote init detects non-existent repo" { + # Try to init a path that doesn't exist + run dolt remote init "git-remotes/nonexistent.git" + [ "$status" -eq 1 ] + [[ "$output" =~ "failed to open git repository" ]] || false +} + +@test "remotes-git: push and clone with git remote" { + create_bare_git_repo "push-clone-repo" + + # Initialize the git remote + dolt remote init "git-remotes/push-clone-repo.git" + + # Create some data + dolt sql <