Use ETag/If-None-Match in url-fetch (#3664)

samples/go/url-fetch/fetch.go

@@ -15,6 +15,7 @@ import (
 	"github.com/attic-labs/noms/go/config"
 	"github.com/attic-labs/noms/go/d"
 	"github.com/attic-labs/noms/go/datas"
+	"github.com/attic-labs/noms/go/marshal"
 	"github.com/attic-labs/noms/go/spec"
 	"github.com/attic-labs/noms/go/types"
 	"github.com/attic-labs/noms/go/util/exit"
@@ -57,18 +58,48 @@ func main() {
 
 	var r io.Reader
 	var contentLength int64
-	var sourceType, sourceVal string
 
+	var root = struct {
+		Meta struct {
+			Etag string `noms:"etag,omitempty"`
+			File string `noms:"file,omitempty"`
+			URL  string `noms:"url,omitempty"`
+		}
+	}{}
+	if ds.HasHead() {
+		err = marshal.Unmarshal(ds.Head(), &root)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Could not unmarshal head: %s\n", err)
+			return
+		}
+	}
+
+	additionalMetaInfo := map[string]string{}
 	if *stdin {
 		r = os.Stdin
 		contentLength = -1
 	} else if url := flag.Arg(0); strings.HasPrefix(url, "http") {
-		resp, err := http.Get(url)
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Could not build http request for url %s, error: %s\n", url, err)
+			return
+		}
+
+		if root.Meta.URL == url && root.Meta.Etag != "" {
+			req.Header.Set("If-None-Match", root.Meta.Etag)
+		}
+
+		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %s\n", url, err)
 			return
 		}
 
+		if resp.StatusCode == http.StatusNotModified {
+			fmt.Fprintf(os.Stdout, "Content unchanged since last fetch, no commit made")
+			return
+		}
+
 		switch resp.StatusCode / 100 {
 		case 4, 5:
 			fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %d (%s)\n", url, resp.StatusCode, resp.Status)
@@ -77,7 +108,10 @@ func main() {
 
 		r = resp.Body
 		contentLength = resp.ContentLength
-		sourceType, sourceVal = "url", url
+		additionalMetaInfo["url"] = url
+		if etag := resp.Header.Get("Etag"); etag != "" {
+			additionalMetaInfo["etag"] = etag
+		}
 	} else {
 		// assume it's a file
 		f, err := os.Open(url)
@@ -94,7 +128,7 @@ func main() {
 
 		r = f
 		contentLength = s.Size()
-		sourceType, sourceVal = "file", url
+		additionalMetaInfo["file"] = url
 	}
 
 	if !*noProgress {
@@ -103,13 +137,9 @@ func main() {
 	b := types.NewBlob(db, r)
 
 	if *performCommit {
-		var additionalMetaInfo map[string]string
-		if sourceType != "" {
-			additionalMetaInfo = map[string]string{sourceType: sourceVal}
-		}
 		meta, err := spec.CreateCommitMetaStruct(db, "", "", additionalMetaInfo, nil)
 		d.CheckErrorNoUsage(err)
-		ds, err = db.Commit(ds, b, datas.CommitOptions{Meta: meta})
+		_, err = db.Commit(ds, b, datas.CommitOptions{Meta: meta})
 		if err != nil {
 			d.Chk.Equal(datas.ErrMergeNeeded, err)
 			fmt.Fprintf(os.Stderr, "Could not commit, optimistic concurrency failed.")

samples/go/url-fetch/fetch_test.go

@@ -6,7 +6,10 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"testing"
 
@@ -90,4 +93,109 @@ func (s *testSuite) TestImportFromFile() {
 	assert.Equal(f.Name(), string(meta.Get("file").(types.String)))
 }
 
-// TODO: TestImportFromURL
+func (s *testSuite) TestImportFromURL() {
+	assert := s.Assert()
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "abcdef")
+	}))
+	defer svr.Close()
+
+	dsName := spec.CreateValueSpecString("nbs", s.DBDir, "ds")
+	s.MustRun(main, []string{svr.URL, dsName})
+
+	sp, err := spec.ForPath(dsName + ".value")
+	assert.NoError(err)
+	defer sp.Close()
+
+	ds := sp.GetDatabase().GetDataset("ds")
+
+	expected := types.NewBlob(ds.Database(), bytes.NewBufferString("abcdef"))
+	assert.True(expected.Equals(sp.GetValue()))
+
+	meta := ds.Head().Get(datas.MetaField).(types.Struct)
+	metaDesc := types.TypeOf(meta).Desc.(types.StructDesc)
+	assert.Equal(2, metaDesc.Len())
+	assert.NotNil(metaDesc.Field("date"))
+	assert.Equal(svr.URL, string(meta.Get("url").(types.String)))
+}
+
+func (s *testSuite) TestImportFromURLStoresEtag() {
+	assert := s.Assert()
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Etag", "xyz123")
+		fmt.Fprint(w, "abcdef")
+	}))
+	defer svr.Close()
+
+	dsName := spec.CreateValueSpecString("nbs", s.DBDir, "ds")
+	s.MustRun(main, []string{svr.URL, dsName})
+
+	sp, err := spec.ForPath(dsName + ".value")
+	assert.NoError(err)
+	defer sp.Close()
+
+	ds := sp.GetDatabase().GetDataset("ds")
+
+	expected := types.NewBlob(ds.Database(), bytes.NewBufferString("abcdef"))
+	assert.True(expected.Equals(sp.GetValue()))
+
+	meta := ds.Head().Get(datas.MetaField).(types.Struct)
+	metaDesc := types.TypeOf(meta).Desc.(types.StructDesc)
+	assert.Equal(3, metaDesc.Len())
+	assert.NotNil(metaDesc.Field("date"))
+	assert.Equal(svr.URL, string(meta.Get("url").(types.String)))
+	assert.Equal("xyz123", string(meta.Get("etag").(types.String)))
+}
+
+func (s *testSuite) TestImportFromURLUsesEtag() {
+	assert := s.Assert()
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("If-None-Match") == "xyz123" {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+		w.Header().Set("Etag", "xyz123")
+		fmt.Fprint(w, "abcdef")
+	}))
+	defer svr.Close()
+
+	dsName := spec.CreateValueSpecString("nbs", s.DBDir, "ds")
+
+	// First fetch commits and stores etag
+	s.MustRun(main, []string{svr.URL, dsName})
+	heightAfterFetch1 := s.commitHeight(dsName)
+
+	// Second fetch should use etag and will not commit
+	s.MustRun(main, []string{svr.URL, dsName})
+	heightAfterFetch2 := s.commitHeight(dsName)
+
+	assert.Equal(heightAfterFetch1, heightAfterFetch2)
+}
+
+func (s *testSuite) TestImportFromURLCommitsMultiple() {
+	assert := s.Assert()
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "abcdef")
+	}))
+	defer svr.Close()
+
+	dsName := spec.CreateValueSpecString("nbs", s.DBDir, "ds")
+
+	// First fetch commits
+	s.MustRun(main, []string{svr.URL, dsName})
+	heightAfterFetch1 := s.commitHeight(dsName)
+
+	// Second fetch also commits since there is no etag
+	s.MustRun(main, []string{svr.URL, dsName})
+	heightAfterFetch2 := s.commitHeight(dsName)
+
+	assert.NotEqual(heightAfterFetch1, heightAfterFetch2)
+}
+
+func (s *testSuite) commitHeight(dsName string) uint64 {
+	sp, err := spec.ForPath(dsName + ".value")
+	s.Assert().NoError(err)
+	ds := sp.GetDatabase().GetDataset("ds")
+	defer sp.Close()
+	return ds.HeadRef().Height()
+}