Introduce tagdex: a program for indexing photo tags.

clients/flickr/gen/types.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/attic-labs/noms/clients/util"
 	"github.com/attic-labs/noms/nomgen"
 	"github.com/attic-labs/noms/types"
 )
@@ -8,29 +9,12 @@ import (
 func main() {
 	ng := nomgen.New("types.go")
 
-	stringSet := ng.AddType(types.NewMap(
-		types.NewString("$type"), types.NewString("noms.SetDef"),
-		types.NewString("elem"), types.NewString("string")))
-
-	photo := ng.AddType(types.NewMap(
-		types.NewString("$type"), types.NewString("noms.StructDef"),
-		types.NewString("$name"), types.NewString("Photo"),
-		types.NewString("id"), types.NewString("string"),
-		types.NewString("title"), types.NewString("string"),
-		types.NewString("url"), types.NewString("string"),
-		types.NewString("image"), types.NewString("blob"),
-		types.NewString("tags"), stringSet))
-
-	photoSet := ng.AddType(types.NewMap(
-		types.NewString("$type"), types.NewString("noms.SetDef"),
-		types.NewString("elem"), photo))
-
 	photoset := ng.AddType(types.NewMap(
 		types.NewString("$type"), types.NewString("noms.StructDef"),
 		types.NewString("$name"), types.NewString("Photoset"),
 		types.NewString("id"), types.NewString("string"),
 		types.NewString("title"), types.NewString("string"),
-		types.NewString("photos"), photoSet))
+		types.NewString("photos"), util.PhotoSetTypeDef))
 
 	photosetSet := ng.AddType(types.NewMap(
 		types.NewString("$type"), types.NewString("noms.SetDef"),

clients/flickr/types.go

@@ -8,88 +8,6 @@ import (
 	"github.com/attic-labs/noms/types"
 )
 
-// SetOfString
-
-type SetOfString struct {
-	s types.Set
-}
-
-type SetOfStringIterCallback (func(p types.String) (stop bool))
-
-func NewSetOfString() SetOfString {
-	return SetOfString{types.NewSet()}
-}
-
-func SetOfStringFromVal(p types.Value) SetOfString {
-	return SetOfString{p.(types.Set)}
-}
-
-func (s SetOfString) NomsValue() types.Set {
-	return s.s
-}
-
-func (s SetOfString) Equals(p SetOfString) bool {
-	return s.s.Equals(p.s)
-}
-
-func (s SetOfString) Ref() ref.Ref {
-	return s.s.Ref()
-}
-
-func (s SetOfString) Empty() bool {
-	return s.s.Empty()
-}
-
-func (s SetOfString) Len() uint64 {
-	return s.s.Len()
-}
-
-func (s SetOfString) Has(p types.String) bool {
-	return s.s.Has(p)
-}
-
-func (s SetOfString) Iter(cb SetOfStringIterCallback) {
-	s.s.Iter(func(v types.Value) bool {
-		return cb(types.StringFromVal(v))
-	})
-}
-
-func (s SetOfString) Insert(p ...types.String) SetOfString {
-	return SetOfString{s.s.Insert(s.fromElemSlice(p)...)}
-}
-
-func (s SetOfString) Remove(p ...types.String) SetOfString {
-	return SetOfString{s.s.Remove(s.fromElemSlice(p)...)}
-}
-
-func (s SetOfString) Union(others ...SetOfString) SetOfString {
-	return SetOfString{s.s.Union(s.fromStructSlice(others)...)}
-}
-
-func (s SetOfString) Subtract(others ...SetOfString) SetOfString {
-	return SetOfString{s.s.Subtract(s.fromStructSlice(others)...)}
-}
-
-func (s SetOfString) Any() types.String {
-	return types.StringFromVal(s.s.Any())
-}
-
-func (s SetOfString) fromStructSlice(p []SetOfString) []types.Set {
-	r := make([]types.Set, len(p))
-	for i, v := range p {
-		r[i] = v.s
-	}
-	return r
-}
-
-func (s SetOfString) fromElemSlice(p []types.String) []types.Value {
-	r := make([]types.Value, len(p))
-	for i, v := range p {
-		r[i] = v
-	}
-	return r
-}
-
 // User
 
 type User struct {
@@ -212,75 +130,6 @@ func (s Photoset) SetPhotos(p SetOfPhoto) Photoset {
 	return PhotosetFromVal(s.m.Set(types.NewString("photos"), p.NomsValue()))
 }
 
-// Photo
-
-type Photo struct {
-	m types.Map
-}
-
-func NewPhoto() Photo {
-	return Photo{
-		types.NewMap(types.NewString("$name"), types.NewString("Photo")),
-	}
-}
-
-func PhotoFromVal(v types.Value) Photo {
-	return Photo{v.(types.Map)}
-}
-
-// TODO: This was going to be called Value() but it collides with root.value. We need some other place to put the built-in fields like Value() and Equals().
-func (s Photo) NomsValue() types.Map {
-	return s.m
-}
-
-func (s Photo) Equals(p Photo) bool {
-	return s.m.Equals(p.m)
-}
-
-func (s Photo) Ref() ref.Ref {
-	return s.m.Ref()
-}
-
-func (s Photo) Title() types.String {
-	return types.StringFromVal(s.m.Get(types.NewString("title")))
-}
-
-func (s Photo) SetTitle(p types.String) Photo {
-	return PhotoFromVal(s.m.Set(types.NewString("title"), p))
-}
-
-func (s Photo) Id() types.String {
-	return types.StringFromVal(s.m.Get(types.NewString("id")))
-}
-
-func (s Photo) SetId(p types.String) Photo {
-	return PhotoFromVal(s.m.Set(types.NewString("id"), p))
-}
-
-func (s Photo) Tags() SetOfString {
-	return SetOfStringFromVal(s.m.Get(types.NewString("tags")))
-}
-
-func (s Photo) SetTags(p SetOfString) Photo {
-	return PhotoFromVal(s.m.Set(types.NewString("tags"), p.NomsValue()))
-}
-
-func (s Photo) Image() types.Blob {
-	return types.BlobFromVal(s.m.Get(types.NewString("image")))
-}
-
-func (s Photo) SetImage(p types.Blob) Photo {
-	return PhotoFromVal(s.m.Set(types.NewString("image"), p))
-}
-
-func (s Photo) Url() types.String {
-	return types.StringFromVal(s.m.Get(types.NewString("url")))
-}
-
-func (s Photo) SetUrl(p types.String) Photo {
-	return PhotoFromVal(s.m.Set(types.NewString("url"), p))
-}
-
 // SetOfPhotoset
 
 type SetOfPhotoset struct {
@@ -445,3 +294,154 @@ func (s SetOfPhoto) fromElemSlice(p []Photo) []types.Value {
 	return r
 }
 
+// Photo
+
+type Photo struct {
+	m types.Map
+}
+
+func NewPhoto() Photo {
+	return Photo{
+		types.NewMap(types.NewString("$name"), types.NewString("Photo")),
+	}
+}
+
+func PhotoFromVal(v types.Value) Photo {
+	return Photo{v.(types.Map)}
+}
+
+// TODO: This was going to be called Value() but it collides with root.value. We need some other place to put the built-in fields like Value() and Equals().
+func (s Photo) NomsValue() types.Map {
+	return s.m
+}
+
+func (s Photo) Equals(p Photo) bool {
+	return s.m.Equals(p.m)
+}
+
+func (s Photo) Ref() ref.Ref {
+	return s.m.Ref()
+}
+
+func (s Photo) Title() types.String {
+	return types.StringFromVal(s.m.Get(types.NewString("title")))
+}
+
+func (s Photo) SetTitle(p types.String) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("title"), p))
+}
+
+func (s Photo) Id() types.String {
+	return types.StringFromVal(s.m.Get(types.NewString("id")))
+}
+
+func (s Photo) SetId(p types.String) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("id"), p))
+}
+
+func (s Photo) Tags() SetOfString {
+	return SetOfStringFromVal(s.m.Get(types.NewString("tags")))
+}
+
+func (s Photo) SetTags(p SetOfString) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("tags"), p.NomsValue()))
+}
+
+func (s Photo) Image() types.Blob {
+	return types.BlobFromVal(s.m.Get(types.NewString("image")))
+}
+
+func (s Photo) SetImage(p types.Blob) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("image"), p))
+}
+
+func (s Photo) Url() types.String {
+	return types.StringFromVal(s.m.Get(types.NewString("url")))
+}
+
+func (s Photo) SetUrl(p types.String) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("url"), p))
+}
+
+// SetOfString
+
+type SetOfString struct {
+	s types.Set
+}
+
+type SetOfStringIterCallback (func(p types.String) (stop bool))
+
+func NewSetOfString() SetOfString {
+	return SetOfString{types.NewSet()}
+}
+
+func SetOfStringFromVal(p types.Value) SetOfString {
+	return SetOfString{p.(types.Set)}
+}
+
+func (s SetOfString) NomsValue() types.Set {
+	return s.s
+}
+
+func (s SetOfString) Equals(p SetOfString) bool {
+	return s.s.Equals(p.s)
+}
+
+func (s SetOfString) Ref() ref.Ref {
+	return s.s.Ref()
+}
+
+func (s SetOfString) Empty() bool {
+	return s.s.Empty()
+}
+
+func (s SetOfString) Len() uint64 {
+	return s.s.Len()
+}
+
+func (s SetOfString) Has(p types.String) bool {
+	return s.s.Has(p)
+}
+
+func (s SetOfString) Iter(cb SetOfStringIterCallback) {
+	s.s.Iter(func(v types.Value) bool {
+		return cb(types.StringFromVal(v))
+	})
+}
+
+func (s SetOfString) Insert(p ...types.String) SetOfString {
+	return SetOfString{s.s.Insert(s.fromElemSlice(p)...)}
+}
+
+func (s SetOfString) Remove(p ...types.String) SetOfString {
+	return SetOfString{s.s.Remove(s.fromElemSlice(p)...)}
+}
+
+func (s SetOfString) Union(others ...SetOfString) SetOfString {
+	return SetOfString{s.s.Union(s.fromStructSlice(others)...)}
+}
+
+func (s SetOfString) Subtract(others ...SetOfString) SetOfString {
+	return SetOfString{s.s.Subtract(s.fromStructSlice(others)...)}
+}
+
+func (s SetOfString) Any() types.String {
+	return types.StringFromVal(s.s.Any())
+}
+
+func (s SetOfString) fromStructSlice(p []SetOfString) []types.Set {
+	r := make([]types.Set, len(p))
+	for i, v := range p {
+		r[i] = v.s
+	}
+	return r
+}
+
+func (s SetOfString) fromElemSlice(p []types.String) []types.Value {
+	r := make([]types.Value, len(p))
+	for i, v := range p {
+		r[i] = v
+	}
+	return r
+}
+

clients/tagdex/gen/types.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"github.com/attic-labs/noms/clients/util"
+	"github.com/attic-labs/noms/nomgen"
+	"github.com/attic-labs/noms/types"
+)
+
+func main() {
+	ng := nomgen.New("types.go")
+	ng.AddType(util.PhotoTypeDef)
+	ng.AddType(util.PhotoSetTypeDef)
+
+	ng.AddType(types.NewMap(
+		types.NewString("$type"), types.NewString("noms.MapDef"),
+		types.NewString("key"), types.NewString("string"),
+		types.NewString("value"), util.PhotoSetTypeDef))
+
+	ng.WriteGo("main")
+}

clients/tagdex/rungen.go

@@ -0,0 +1,3 @@
+package main
+
+//go:generate go run gen/types.go

clients/tagdex/tagdex.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/attic-labs/noms/chunks"
+	"github.com/attic-labs/noms/datas"
+	"github.com/attic-labs/noms/dataset"
+	. "github.com/attic-labs/noms/dbg"
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+)
+
+var (
+	csFlags     = chunks.NewFlags()
+	inputRefStr = flag.String("input-ref", "", "ref to find photos from within input chunkstore")
+	outputID    = flag.String("output-ds", "", "dataset to store data in.")
+)
+
+func main() {
+	flag.Parse()
+
+	cs := csFlags.CreateStore()
+	if cs == nil || *inputRefStr == "" || *outputID == "" {
+		flag.Usage()
+		return
+	}
+
+	inputRef, err := ref.Parse(*inputRefStr)
+	if err != nil {
+		flag.Usage()
+		return
+	}
+
+	ds := dataset.NewDataset(datas.NewDataStore(cs), *outputID)
+	out := NewMapOfStringToSetOfPhoto()
+
+	types.All(inputRef, cs, func(f types.Future) {
+		v, err := f.Deref(cs)
+		Chk.NoError(err)
+		if v, ok := v.(types.Map); ok && types.NewString("Photo").Equals(v.Get(types.NewString("$name"))) {
+			p := PhotoFromVal(v)
+			p.Tags().Iter(func(item types.String) (stop bool) {
+				var s SetOfPhoto
+				if out.Has(item) {
+					s = out.Get(item)
+				} else {
+					s = NewSetOfPhoto()
+				}
+				out = out.Set(item, s.Insert(p))
+				return
+			})
+		}
+	})
+
+	ds = ds.Commit(datas.NewSetOfCommit().Insert(
+		datas.NewCommit().SetParents(ds.Heads().NomsValue()).SetValue(out.NomsValue())))
+
+	fmt.Println(ds.Root().String())
+}

clients/tagdex/types.go

@@ -0,0 +1,303 @@
+// This file was generated by nomgen.
+// To regenerate, run `go generate` in this package.
+
+package main
+
+import (
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+)
+
+// Photo
+
+type Photo struct {
+	m types.Map
+}
+
+func NewPhoto() Photo {
+	return Photo{
+		types.NewMap(types.NewString("$name"), types.NewString("Photo")),
+	}
+}
+
+func PhotoFromVal(v types.Value) Photo {
+	return Photo{v.(types.Map)}
+}
+
+// TODO: This was going to be called Value() but it collides with root.value. We need some other place to put the built-in fields like Value() and Equals().
+func (s Photo) NomsValue() types.Map {
+	return s.m
+}
+
+func (s Photo) Equals(p Photo) bool {
+	return s.m.Equals(p.m)
+}
+
+func (s Photo) Ref() ref.Ref {
+	return s.m.Ref()
+}
+
+func (s Photo) Title() types.String {
+	return types.StringFromVal(s.m.Get(types.NewString("title")))
+}
+
+func (s Photo) SetTitle(p types.String) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("title"), p))
+}
+
+func (s Photo) Id() types.String {
+	return types.StringFromVal(s.m.Get(types.NewString("id")))
+}
+
+func (s Photo) SetId(p types.String) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("id"), p))
+}
+
+func (s Photo) Tags() SetOfString {
+	return SetOfStringFromVal(s.m.Get(types.NewString("tags")))
+}
+
+func (s Photo) SetTags(p SetOfString) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("tags"), p.NomsValue()))
+}
+
+func (s Photo) Image() types.Blob {
+	return types.BlobFromVal(s.m.Get(types.NewString("image")))
+}
+
+func (s Photo) SetImage(p types.Blob) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("image"), p))
+}
+
+func (s Photo) Url() types.String {
+	return types.StringFromVal(s.m.Get(types.NewString("url")))
+}
+
+func (s Photo) SetUrl(p types.String) Photo {
+	return PhotoFromVal(s.m.Set(types.NewString("url"), p))
+}
+
+// SetOfString
+
+type SetOfString struct {
+	s types.Set
+}
+
+type SetOfStringIterCallback (func(p types.String) (stop bool))
+
+func NewSetOfString() SetOfString {
+	return SetOfString{types.NewSet()}
+}
+
+func SetOfStringFromVal(p types.Value) SetOfString {
+	return SetOfString{p.(types.Set)}
+}
+
+func (s SetOfString) NomsValue() types.Set {
+	return s.s
+}
+
+func (s SetOfString) Equals(p SetOfString) bool {
+	return s.s.Equals(p.s)
+}
+
+func (s SetOfString) Ref() ref.Ref {
+	return s.s.Ref()
+}
+
+func (s SetOfString) Empty() bool {
+	return s.s.Empty()
+}
+
+func (s SetOfString) Len() uint64 {
+	return s.s.Len()
+}
+
+func (s SetOfString) Has(p types.String) bool {
+	return s.s.Has(p)
+}
+
+func (s SetOfString) Iter(cb SetOfStringIterCallback) {
+	s.s.Iter(func(v types.Value) bool {
+		return cb(types.StringFromVal(v))
+	})
+}
+
+func (s SetOfString) Insert(p ...types.String) SetOfString {
+	return SetOfString{s.s.Insert(s.fromElemSlice(p)...)}
+}
+
+func (s SetOfString) Remove(p ...types.String) SetOfString {
+	return SetOfString{s.s.Remove(s.fromElemSlice(p)...)}
+}
+
+func (s SetOfString) Union(others ...SetOfString) SetOfString {
+	return SetOfString{s.s.Union(s.fromStructSlice(others)...)}
+}
+
+func (s SetOfString) Subtract(others ...SetOfString) SetOfString {
+	return SetOfString{s.s.Subtract(s.fromStructSlice(others)...)}
+}
+
+func (s SetOfString) Any() types.String {
+	return types.StringFromVal(s.s.Any())
+}
+
+func (s SetOfString) fromStructSlice(p []SetOfString) []types.Set {
+	r := make([]types.Set, len(p))
+	for i, v := range p {
+		r[i] = v.s
+	}
+	return r
+}
+
+func (s SetOfString) fromElemSlice(p []types.String) []types.Value {
+	r := make([]types.Value, len(p))
+	for i, v := range p {
+		r[i] = v
+	}
+	return r
+}
+
+// MapOfStringToSetOfPhoto
+
+type MapOfStringToSetOfPhoto struct {
+	m types.Map
+}
+
+type MapOfStringToSetOfPhotoIterCallback (func(k types.String, v SetOfPhoto) (stop bool))
+
+func NewMapOfStringToSetOfPhoto() MapOfStringToSetOfPhoto {
+	return MapOfStringToSetOfPhoto{types.NewMap()}
+}
+
+func MapOfStringToSetOfPhotoFromVal(p types.Value) MapOfStringToSetOfPhoto {
+	return MapOfStringToSetOfPhoto{p.(types.Map)}
+}
+
+func (m MapOfStringToSetOfPhoto) NomsValue() types.Map {
+	return m.m
+}
+
+func (m MapOfStringToSetOfPhoto) Equals(p MapOfStringToSetOfPhoto) bool {
+	return m.m.Equals(p.m)
+}
+
+func (m MapOfStringToSetOfPhoto) Ref() ref.Ref {
+	return m.m.Ref()
+}
+
+func (m MapOfStringToSetOfPhoto) Empty() bool {
+	return m.m.Empty()
+}
+
+func (m MapOfStringToSetOfPhoto) Len() uint64 {
+	return m.m.Len()
+}
+
+func (m MapOfStringToSetOfPhoto) Has(p types.String) bool {
+	return m.m.Has(p)
+}
+
+func (m MapOfStringToSetOfPhoto) Get(p types.String) SetOfPhoto {
+	return SetOfPhotoFromVal(m.m.Get(p))
+}
+
+func (m MapOfStringToSetOfPhoto) Set(k types.String, v SetOfPhoto) MapOfStringToSetOfPhoto {
+	return MapOfStringToSetOfPhotoFromVal(m.m.Set(k, v.NomsValue()))
+}
+
+// TODO: Implement SetM?
+
+func (m MapOfStringToSetOfPhoto) Remove(p types.String) MapOfStringToSetOfPhoto {
+	return MapOfStringToSetOfPhotoFromVal(m.m.Remove(p))
+}
+
+func (m MapOfStringToSetOfPhoto) Iter(cb MapOfStringToSetOfPhotoIterCallback) {
+	m.m.Iter(func(k, v types.Value) bool {
+		return cb(types.StringFromVal(k), SetOfPhotoFromVal(v))
+	})
+}
+
+// SetOfPhoto
+
+type SetOfPhoto struct {
+	s types.Set
+}
+
+type SetOfPhotoIterCallback (func(p Photo) (stop bool))
+
+func NewSetOfPhoto() SetOfPhoto {
+	return SetOfPhoto{types.NewSet()}
+}
+
+func SetOfPhotoFromVal(p types.Value) SetOfPhoto {
+	return SetOfPhoto{p.(types.Set)}
+}
+
+func (s SetOfPhoto) NomsValue() types.Set {
+	return s.s
+}
+
+func (s SetOfPhoto) Equals(p SetOfPhoto) bool {
+	return s.s.Equals(p.s)
+}
+
+func (s SetOfPhoto) Ref() ref.Ref {
+	return s.s.Ref()
+}
+
+func (s SetOfPhoto) Empty() bool {
+	return s.s.Empty()
+}
+
+func (s SetOfPhoto) Len() uint64 {
+	return s.s.Len()
+}
+
+func (s SetOfPhoto) Has(p Photo) bool {
+	return s.s.Has(p.NomsValue())
+}
+
+func (s SetOfPhoto) Iter(cb SetOfPhotoIterCallback) {
+	s.s.Iter(func(v types.Value) bool {
+		return cb(PhotoFromVal(v))
+	})
+}
+
+func (s SetOfPhoto) Insert(p ...Photo) SetOfPhoto {
+	return SetOfPhoto{s.s.Insert(s.fromElemSlice(p)...)}
+}
+
+func (s SetOfPhoto) Remove(p ...Photo) SetOfPhoto {
+	return SetOfPhoto{s.s.Remove(s.fromElemSlice(p)...)}
+}
+
+func (s SetOfPhoto) Union(others ...SetOfPhoto) SetOfPhoto {
+	return SetOfPhoto{s.s.Union(s.fromStructSlice(others)...)}
+}
+
+func (s SetOfPhoto) Subtract(others ...SetOfPhoto) SetOfPhoto {
+	return SetOfPhoto{s.s.Subtract(s.fromStructSlice(others)...)}
+}
+
+func (s SetOfPhoto) Any() Photo {
+	return PhotoFromVal(s.s.Any())
+}
+
+func (s SetOfPhoto) fromStructSlice(p []SetOfPhoto) []types.Set {
+	r := make([]types.Set, len(p))
+	for i, v := range p {
+		r[i] = v.s
+	}
+	return r
+}
+
+func (s SetOfPhoto) fromElemSlice(p []Photo) []types.Value {
+	r := make([]types.Value, len(p))
+	for i, v := range p {
+		r[i] = v.NomsValue()
+	}
+	return r
+}
+

clients/util/types.go

@@ -0,0 +1,27 @@
+package util
+
+import "github.com/attic-labs/noms/types"
+
+var (
+	PhotoTypeDef    types.Map
+	PhotoSetTypeDef types.Map
+)
+
+func init() {
+	stringSet := types.NewMap(
+		types.NewString("$type"), types.NewString("noms.SetDef"),
+		types.NewString("elem"), types.NewString("string"))
+
+	PhotoTypeDef = types.NewMap(
+		types.NewString("$type"), types.NewString("noms.StructDef"),
+		types.NewString("$name"), types.NewString("Photo"),
+		types.NewString("id"), types.NewString("string"),
+		types.NewString("title"), types.NewString("string"),
+		types.NewString("url"), types.NewString("string"),
+		types.NewString("image"), types.NewString("blob"),
+		types.NewString("tags"), stringSet)
+
+	PhotoSetTypeDef = types.NewMap(
+		types.NewString("$type"), types.NewString("noms.SetDef"),
+		types.NewString("elem"), PhotoTypeDef)
+}