diff --git a/clients/tagdex/gen/types.go b/clients/tagdex/gen/types.go index f9ea658678..d46f2392ef 100644 --- a/clients/tagdex/gen/types.go +++ b/clients/tagdex/gen/types.go @@ -9,12 +9,12 @@ import ( func main() { ng := nomgen.New("types.go") ng.AddType(util.PhotoTypeDef) - ng.AddType(util.PhotoSetTypeDef) + ng.AddType(util.RemotePhotoTypeDef) ng.AddType(types.NewMap( types.NewString("$type"), types.NewString("noms.MapDef"), types.NewString("key"), types.NewString("string"), - types.NewString("value"), util.PhotoSetTypeDef)) + types.NewString("value"), types.NewString("set"))) ng.WriteGo("main") } diff --git a/clients/tagdex/tagdex.go b/clients/tagdex/tagdex.go index 56bf05b396..60a00e4fdd 100644 --- a/clients/tagdex/tagdex.go +++ b/clients/tagdex/tagdex.go @@ -4,61 +4,78 @@ import ( "flag" "fmt" "log" + "time" "github.com/attic-labs/noms/chunks" "github.com/attic-labs/noms/d" "github.com/attic-labs/noms/datas" "github.com/attic-labs/noms/dataset" - "github.com/attic-labs/noms/ref" "github.com/attic-labs/noms/types" ) var ( - csFlags = chunks.NewFlags() - inputRefStr = flag.String("input-ref", "", "ref to find photos from within input chunkstore") - outputID = flag.String("output-ds", "", "dataset to store data in.") + csFlags = chunks.NewFlags() + inputID = flag.String("input-ds", "", "dataset to find photos within") + outputID = flag.String("output-ds", "", "dataset to store index in") ) func main() { flag.Parse() cs := csFlags.CreateStore() - if cs == nil || *inputRefStr == "" || *outputID == "" { + if cs == nil || *inputID == "" || *outputID == "" { flag.Usage() return } defer cs.Close() - var inputRef ref.Ref - err := d.Try(func() { - inputRef = ref.Parse(*inputRefStr) - }) - if err != nil { - log.Fatalf("Invalid ref: %v", *inputRefStr) + store := datas.NewDataStore(cs) + inputDS := dataset.NewDataset(store, *inputID) + if _, ok := inputDS.MaybeHead(); !ok { + log.Fatalf("No dataset named %s", *inputID) } + outputDS := dataset.NewDataset(store, *outputID) - ds := dataset.NewDataset(datas.NewDataStore(cs), *outputID) - out := NewMapOfStringToSetOfPhoto() + out := NewMapOfStringToSet() - types.All(inputRef, cs, func(f types.Future) { + t0 := time.Now() + numRefs := 0 + numPhotos := 0 + + types.Some(inputDS.Head().Value().Ref(), cs, func(f types.Future) (skip bool) { + numRefs++ v := f.Deref(cs) - if v, ok := v.(types.Map); ok && types.NewString("Photo").Equals(v.Get(types.NewString("$name"))) { - p := PhotoFromVal(v) - p.Tags().Iter(func(item types.String) (stop bool) { - var s SetOfPhoto + if v, ok := v.(types.Map); ok { + name := v.Get(types.NewString("$name")) + if name == nil { + return + } + + if !name.Equals(types.NewString("Photo")) && !name.Equals(types.NewString("RemotePhoto")) { + return + } + + skip = true + numPhotos++ + fmt.Println("Indexing", v.Ref()) + + tags := SetOfStringFromVal(v.Get(types.NewString("tags"))) + tags.Iter(func(item types.String) (stop bool) { + var s types.Set if out.Has(item) { s = out.Get(item) } else { - s = NewSetOfPhoto() + s = types.NewSet() } - out = out.Set(item, s.Insert(p)) + out = out.Set(item, s.Insert(v)) return }) } + return }) - _, ok := ds.Commit(out.NomsValue()) + _, ok := outputDS.Commit(out.NomsValue()) d.Exp.True(ok, "Could not commit due to conflicting edit") - fmt.Println(ds.Store().Root().String()) + fmt.Printf("Indexed %v photos from %v refs in %v\n", numPhotos, numRefs, time.Now().Sub(t0)) } diff --git a/clients/tagdex/types.go b/clients/tagdex/types.go index 38faa37465..835ef629b4 100644 --- a/clients/tagdex/types.go +++ b/clients/tagdex/types.go @@ -37,6 +37,14 @@ func (s Photo) Ref() ref.Ref { return s.m.Ref() } +func (s Photo) Width() types.UInt32 { + return types.UInt32FromVal(s.m.Get(types.NewString("width"))) +} + +func (s Photo) SetWidth(p types.UInt32) Photo { + return PhotoFromVal(s.m.Set(types.NewString("width"), p)) +} + func (s Photo) Title() types.String { return types.StringFromVal(s.m.Get(types.NewString("title"))) } @@ -61,6 +69,14 @@ func (s Photo) SetTags(p SetOfString) Photo { return PhotoFromVal(s.m.Set(types.NewString("tags"), p.NomsValue())) } +func (s Photo) Height() types.UInt32 { + return types.UInt32FromVal(s.m.Get(types.NewString("height"))) +} + +func (s Photo) SetHeight(p types.UInt32) Photo { + return PhotoFromVal(s.m.Set(types.NewString("height"), p)) +} + func (s Photo) Image() types.Blob { return types.BlobFromVal(s.m.Get(types.NewString("image"))) } @@ -159,145 +175,237 @@ func (s SetOfString) fromElemSlice(p []types.String) []types.Value { return r } -// MapOfStringToSetOfPhoto +// RemotePhoto -type MapOfStringToSetOfPhoto struct { +type RemotePhoto struct { m types.Map } -type MapOfStringToSetOfPhotoIterCallback (func(k types.String, v SetOfPhoto) (stop bool)) - -func NewMapOfStringToSetOfPhoto() MapOfStringToSetOfPhoto { - return MapOfStringToSetOfPhoto{types.NewMap()} +func NewRemotePhoto() RemotePhoto { + return RemotePhoto{ + types.NewMap(types.NewString("$name"), types.NewString("RemotePhoto")), + } } -func MapOfStringToSetOfPhotoFromVal(p types.Value) MapOfStringToSetOfPhoto { - return MapOfStringToSetOfPhoto{p.(types.Map)} +func RemotePhotoFromVal(v types.Value) RemotePhoto { + return RemotePhoto{v.(types.Map)} } -func (m MapOfStringToSetOfPhoto) NomsValue() types.Map { +// TODO: This was going to be called Value() but it collides with root.value. We need some other place to put the built-in fields like Value() and Equals(). +func (s RemotePhoto) NomsValue() types.Map { + return s.m +} + +func (s RemotePhoto) Equals(p RemotePhoto) bool { + return s.m.Equals(p.m) +} + +func (s RemotePhoto) Ref() ref.Ref { + return s.m.Ref() +} + +func (s RemotePhoto) Sizes() MapOfSizeToString { + return MapOfSizeToStringFromVal(s.m.Get(types.NewString("sizes"))) +} + +func (s RemotePhoto) SetSizes(p MapOfSizeToString) RemotePhoto { + return RemotePhotoFromVal(s.m.Set(types.NewString("sizes"), p.NomsValue())) +} + +func (s RemotePhoto) Title() types.String { + return types.StringFromVal(s.m.Get(types.NewString("title"))) +} + +func (s RemotePhoto) SetTitle(p types.String) RemotePhoto { + return RemotePhotoFromVal(s.m.Set(types.NewString("title"), p)) +} + +func (s RemotePhoto) Id() types.String { + return types.StringFromVal(s.m.Get(types.NewString("id"))) +} + +func (s RemotePhoto) SetId(p types.String) RemotePhoto { + return RemotePhotoFromVal(s.m.Set(types.NewString("id"), p)) +} + +func (s RemotePhoto) Tags() SetOfString { + return SetOfStringFromVal(s.m.Get(types.NewString("tags"))) +} + +func (s RemotePhoto) SetTags(p SetOfString) RemotePhoto { + return RemotePhotoFromVal(s.m.Set(types.NewString("tags"), p.NomsValue())) +} + +func (s RemotePhoto) Url() types.String { + return types.StringFromVal(s.m.Get(types.NewString("url"))) +} + +func (s RemotePhoto) SetUrl(p types.String) RemotePhoto { + return RemotePhotoFromVal(s.m.Set(types.NewString("url"), p)) +} + +// MapOfSizeToString + +type MapOfSizeToString struct { + m types.Map +} + +type MapOfSizeToStringIterCallback (func(k Size, v types.String) (stop bool)) + +func NewMapOfSizeToString() MapOfSizeToString { + return MapOfSizeToString{types.NewMap()} +} + +func MapOfSizeToStringFromVal(p types.Value) MapOfSizeToString { + return MapOfSizeToString{p.(types.Map)} +} + +func (m MapOfSizeToString) NomsValue() types.Map { return m.m } -func (m MapOfStringToSetOfPhoto) Equals(p MapOfStringToSetOfPhoto) bool { +func (m MapOfSizeToString) Equals(p MapOfSizeToString) bool { return m.m.Equals(p.m) } -func (m MapOfStringToSetOfPhoto) Ref() ref.Ref { +func (m MapOfSizeToString) Ref() ref.Ref { return m.m.Ref() } -func (m MapOfStringToSetOfPhoto) Empty() bool { +func (m MapOfSizeToString) Empty() bool { return m.m.Empty() } -func (m MapOfStringToSetOfPhoto) Len() uint64 { +func (m MapOfSizeToString) Len() uint64 { return m.m.Len() } -func (m MapOfStringToSetOfPhoto) Has(p types.String) bool { - return m.m.Has(p) +func (m MapOfSizeToString) Has(p Size) bool { + return m.m.Has(p.NomsValue()) } -func (m MapOfStringToSetOfPhoto) Get(p types.String) SetOfPhoto { - return SetOfPhotoFromVal(m.m.Get(p)) +func (m MapOfSizeToString) Get(p Size) types.String { + return types.StringFromVal(m.m.Get(p.NomsValue())) } -func (m MapOfStringToSetOfPhoto) Set(k types.String, v SetOfPhoto) MapOfStringToSetOfPhoto { - return MapOfStringToSetOfPhotoFromVal(m.m.Set(k, v.NomsValue())) +func (m MapOfSizeToString) Set(k Size, v types.String) MapOfSizeToString { + return MapOfSizeToStringFromVal(m.m.Set(k.NomsValue(), v)) } // TODO: Implement SetM? -func (m MapOfStringToSetOfPhoto) Remove(p types.String) MapOfStringToSetOfPhoto { - return MapOfStringToSetOfPhotoFromVal(m.m.Remove(p)) +func (m MapOfSizeToString) Remove(p Size) MapOfSizeToString { + return MapOfSizeToStringFromVal(m.m.Remove(p.NomsValue())) } -func (m MapOfStringToSetOfPhoto) Iter(cb MapOfStringToSetOfPhotoIterCallback) { +func (m MapOfSizeToString) Iter(cb MapOfSizeToStringIterCallback) { m.m.Iter(func(k, v types.Value) bool { - return cb(types.StringFromVal(k), SetOfPhotoFromVal(v)) + return cb(SizeFromVal(k), types.StringFromVal(v)) }) } -// SetOfPhoto +// Size -type SetOfPhoto struct { - s types.Set +type Size struct { + m types.Map } -type SetOfPhotoIterCallback (func(p Photo) (stop bool)) - -func NewSetOfPhoto() SetOfPhoto { - return SetOfPhoto{types.NewSet()} +func NewSize() Size { + return Size{ + types.NewMap(types.NewString("$name"), types.NewString("Size")), + } } -func SetOfPhotoFromVal(p types.Value) SetOfPhoto { - return SetOfPhoto{p.(types.Set)} +func SizeFromVal(v types.Value) Size { + return Size{v.(types.Map)} } -func (s SetOfPhoto) NomsValue() types.Set { - return s.s +// TODO: This was going to be called Value() but it collides with root.value. We need some other place to put the built-in fields like Value() and Equals(). +func (s Size) NomsValue() types.Map { + return s.m } -func (s SetOfPhoto) Equals(p SetOfPhoto) bool { - return s.s.Equals(p.s) +func (s Size) Equals(p Size) bool { + return s.m.Equals(p.m) } -func (s SetOfPhoto) Ref() ref.Ref { - return s.s.Ref() +func (s Size) Ref() ref.Ref { + return s.m.Ref() } -func (s SetOfPhoto) Empty() bool { - return s.s.Empty() +func (s Size) Width() types.UInt32 { + return types.UInt32FromVal(s.m.Get(types.NewString("width"))) } -func (s SetOfPhoto) Len() uint64 { - return s.s.Len() +func (s Size) SetWidth(p types.UInt32) Size { + return SizeFromVal(s.m.Set(types.NewString("width"), p)) } -func (s SetOfPhoto) Has(p Photo) bool { - return s.s.Has(p.NomsValue()) +func (s Size) Height() types.UInt32 { + return types.UInt32FromVal(s.m.Get(types.NewString("height"))) } -func (s SetOfPhoto) Iter(cb SetOfPhotoIterCallback) { - s.s.Iter(func(v types.Value) bool { - return cb(PhotoFromVal(v)) +func (s Size) SetHeight(p types.UInt32) Size { + return SizeFromVal(s.m.Set(types.NewString("height"), p)) +} + +// MapOfStringToSet + +type MapOfStringToSet struct { + m types.Map +} + +type MapOfStringToSetIterCallback (func(k types.String, v types.Set) (stop bool)) + +func NewMapOfStringToSet() MapOfStringToSet { + return MapOfStringToSet{types.NewMap()} +} + +func MapOfStringToSetFromVal(p types.Value) MapOfStringToSet { + return MapOfStringToSet{p.(types.Map)} +} + +func (m MapOfStringToSet) NomsValue() types.Map { + return m.m +} + +func (m MapOfStringToSet) Equals(p MapOfStringToSet) bool { + return m.m.Equals(p.m) +} + +func (m MapOfStringToSet) Ref() ref.Ref { + return m.m.Ref() +} + +func (m MapOfStringToSet) Empty() bool { + return m.m.Empty() +} + +func (m MapOfStringToSet) Len() uint64 { + return m.m.Len() +} + +func (m MapOfStringToSet) Has(p types.String) bool { + return m.m.Has(p) +} + +func (m MapOfStringToSet) Get(p types.String) types.Set { + return types.SetFromVal(m.m.Get(p)) +} + +func (m MapOfStringToSet) Set(k types.String, v types.Set) MapOfStringToSet { + return MapOfStringToSetFromVal(m.m.Set(k, v)) +} + +// TODO: Implement SetM? + +func (m MapOfStringToSet) Remove(p types.String) MapOfStringToSet { + return MapOfStringToSetFromVal(m.m.Remove(p)) +} + +func (m MapOfStringToSet) Iter(cb MapOfStringToSetIterCallback) { + m.m.Iter(func(k, v types.Value) bool { + return cb(types.StringFromVal(k), types.SetFromVal(v)) }) } -func (s SetOfPhoto) Insert(p ...Photo) SetOfPhoto { - return SetOfPhoto{s.s.Insert(s.fromElemSlice(p)...)} -} - -func (s SetOfPhoto) Remove(p ...Photo) SetOfPhoto { - return SetOfPhoto{s.s.Remove(s.fromElemSlice(p)...)} -} - -func (s SetOfPhoto) Union(others ...SetOfPhoto) SetOfPhoto { - return SetOfPhoto{s.s.Union(s.fromStructSlice(others)...)} -} - -func (s SetOfPhoto) Subtract(others ...SetOfPhoto) SetOfPhoto { - return SetOfPhoto{s.s.Subtract(s.fromStructSlice(others)...)} -} - -func (s SetOfPhoto) Any() Photo { - return PhotoFromVal(s.s.Any()) -} - -func (s SetOfPhoto) fromStructSlice(p []SetOfPhoto) []types.Set { - r := make([]types.Set, len(p)) - for i, v := range p { - r[i] = v.s - } - return r -} - -func (s SetOfPhoto) fromElemSlice(p []Photo) []types.Value { - r := make([]types.Value, len(p)) - for i, v := range p { - r[i] = v.NomsValue() - } - return r -} -