Merge pull request #389 from cmasone-attic/more_imports

Initial support for importing type pacakges by ref

Towards #294

nomdl/codegen/codegen.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"go/build"
 	"go/parser"
 	"go/token"
 	"io"
@@ -65,16 +66,19 @@ func main() {
 	if h, ok := pkgDS.MaybeHead(); ok {
 		// Will panic on failure. Can do better once generated collections implement types.Value.
 		types.SetOfRefOfPackageFromVal(h.Value())
-		log.Fatalf("Dataset %s must contain a SetOfRefOfPackage", *pkgDSFlag)
 	}
 
+	depsDir, err := filepath.Abs(*depsDirFlag)
+	if err != nil {
+		log.Fatalf("Could not canonicalize -deps-dir: %v", err)
+	}
 	packageName := getGoPackageName()
 	if *inFlag != "" {
 		out := *outFlag
 		if out == "" {
 			out = getOutFileName(*inFlag)
 		}
-		generate(packageName, *inFlag, out, *depsDirFlag, pkgDS)
+		generate(packageName, *inFlag, out, depsDir, pkgDS)
 		return
 	}
 
@@ -82,7 +86,7 @@ func main() {
 	nomsFiles, err := filepath.Glob("*" + ext)
 	d.Chk.NoError(err)
 	for _, n := range nomsFiles {
-		pkgDS = generate(packageName, n, getOutFileName(n), *depsDirFlag, pkgDS)
+		pkgDS = generate(packageName, n, getOutFileName(n), depsDir, pkgDS)
 	}
 }
 
@@ -94,11 +98,9 @@ func generate(packageName, in, out, depsDir string, pkgDS dataset.Dataset) datas
 	p := pkg.ParseNomDL(packageName, inFile, pkgDS.Store())
 
 	// Generate code for all p's deps first.
-	pVal := p.New()
-	generateDepCode(depsDir, pVal, pkgDS.Store())
-	deps := derefDeps(pVal, pkgDS.Store()) // Maybe should cache p.New
+	deps := generateDepCode(depsDir, p.New(), pkgDS.Store())
 
-	generateAndEmit(getBareFileName(in), out, p, deps)
+	generateAndEmit(getBareFileName(in), out, importPaths(depsDir, deps), deps, p)
 
 	// Since we're just building up a set of refs to all the packages in pkgDS, simply retrying is the logical response to commit failure.
 	for ok := false; !ok; pkgDS, ok = pkgDS.Commit(buildSetOfRefOfPackage(p, pkgDS).NomsValue()) {
@@ -106,36 +108,29 @@ func generate(packageName, in, out, depsDir string, pkgDS dataset.Dataset) datas
 	return pkgDS
 }
 
-// It seems like there's a bunch of wasteful dereffing between this and derefDeps. Figure out how to do this better.
-func generateDepCode(depsDir string, p types.Package, cs chunks.ChunkSource) {
+type depsMap map[ref.Ref]types.Package
+
+func generateDepCode(depsDir string, p types.Package, cs chunks.ChunkSource) depsMap {
+	deps := depsMap{}
 	p.Dependencies().IterAll(func(r types.RefOfPackage) {
 		p := r.GetValue(cs)
-		deps := derefDeps(p, cs)
+		pDeps := generateDepCode(depsDir, p, cs)
 		tag := toTag(p.Ref().String())
 		parsed := pkg.Parsed{PackageDef: p.Def(), Name: tag}
-		generateAndEmit(tag, filepath.Join(depsDir, tag, tag+".go"), parsed, deps)
-		if !p.Dependencies().Empty() {
-			generateDepCode(depsDir, p, cs)
-		}
-	})
-}
+		generateAndEmit(tag, filepath.Join(depsDir, tag, tag+".go"), importPaths(depsDir, pDeps), pDeps, parsed)
 
-func derefDeps(p types.Package, cs chunks.ChunkSource) map[types.RefOfPackage]types.Package {
-	deps := map[types.RefOfPackage]types.Package{}
-	p.Dependencies().IterAll(func(r types.RefOfPackage) {
-		deps[r] = r.GetValue(cs)
+		for depRef, dep := range pDeps {
+			deps[depRef] = dep
+		}
+		deps[r.Ref()] = p
 	})
 	return deps
 }
 
-func toTag(s string) string {
-	return strings.Replace(s, "-", "_", -1)
-}
-
-func generateAndEmit(tag, out string, p pkg.Parsed, deps map[types.RefOfPackage]types.Package) {
+func generateAndEmit(tag, out string, importPaths []string, deps depsMap, p pkg.Parsed) {
 	var buf bytes.Buffer
-	gen := NewCodeGen(&buf, tag, p, deps)
-	gen.WritePackage(p.Name) // FIXME, no need to take name
+	gen := NewCodeGen(&buf, tag, importPaths, deps, p)
+	gen.WritePackage()
 
 	bs, err := imports.Process(out, buf.Bytes(), nil)
 	if err != nil {
@@ -152,6 +147,20 @@ func generateAndEmit(tag, out string, p pkg.Parsed, deps map[types.RefOfPackage]
 	io.Copy(outFile, bytes.NewBuffer(bs))
 }
 
+func importPaths(depsDir string, deps depsMap) (paths []string) {
+	for depRef := range deps {
+		depDir := filepath.Join(depsDir, toTag(depRef.String()))
+		goPkg, err := build.Default.ImportDir(depDir, build.FindOnly)
+		d.Chk.NoError(err)
+		paths = append(paths, goPkg.ImportPath)
+	}
+	return
+}
+
+func toTag(s string) string {
+	return strings.Replace(s, "-", "_", -1)
+}
+
 func buildSetOfRefOfPackage(pkg pkg.Parsed, ds dataset.Dataset) types.SetOfRefOfPackage {
 	// Can do better once generated collections implement types.Value.
 	s := types.NewSetOfRefOfPackage()
@@ -191,14 +200,15 @@ func getGoPackageName() string {
 type codeGen struct {
 	w         io.Writer
 	pkg       pkg.Parsed
-	deps      map[types.RefOfPackage]types.Package
+	deps      depsMap
 	fileid    string
+	imports   []string
 	written   map[string]bool
 	templates *template.Template
 }
 
-func NewCodeGen(w io.Writer, fileID string, pkg pkg.Parsed, deps map[types.RefOfPackage]types.Package) *codeGen {
-	gen := &codeGen{w, pkg, deps, fileID, map[string]bool{}, nil}
+func NewCodeGen(w io.Writer, fileID string, importPaths []string, deps depsMap, pkg pkg.Parsed) *codeGen {
+	gen := &codeGen{w, pkg, deps, fileID, importPaths, map[string]bool{}, nil}
 	gen.templates = gen.readTemplates()
 	return gen
 }
@@ -217,7 +227,7 @@ func (gen *codeGen) readTemplates() *template.Template {
 			"userZero":       gen.userZero,
 			"valueZero":      gen.valueZero,
 			"title":          strings.Title,
-			"toTypesTypeRef": gen.toTypesTypeRef,
+			"toTypesTypeRef": toTypeRef,
 		}).ParseGlob(glob))
 }
 
@@ -310,12 +320,6 @@ func (gen *codeGen) valueToDef(val string, t types.TypeRef) string {
 	panic("unreachable")
 }
 
-func kindToString(k types.NomsKind) (out string) {
-	out = types.KindToString[k]
-	d.Chk.NotEmpty(out, "Unknown NomsKind %d", k)
-	return
-}
-
 func (gen *codeGen) nativeToValue(val string, t types.TypeRef) string {
 	t = gen.resolve(t)
 	k := t.Desc.Kind()
@@ -394,8 +398,8 @@ func (gen *codeGen) userZero(t types.TypeRef) string {
 		return fmt.Sprintf("%s{ref.Ref{}}", gen.userType(t))
 	case types.StringKind:
 		return `""`
-    case types.StructKind:
-        return fmt.Sprintf("New%s()", gen.userName(t))
+	case types.StructKind:
+		return fmt.Sprintf("New%s()", gen.userName(t))
 	case types.ValueKind:
 		// TODO: This is where a null Value would have been useful.
 		return "types.Bool(false)"
@@ -428,7 +432,11 @@ func (gen *codeGen) valueZero(t types.TypeRef) string {
 	case types.StringKind:
 		return `types.NewString("")`
 	case types.StructKind:
-		return fmt.Sprintf("New%s().NomsValue()", gen.userName(t))
+		userName := gen.userName(t)
+		if period := strings.LastIndex(userName, "."); period != -1 {
+			return fmt.Sprintf("%sNew%s().NomsValue()", userName[:period+1], userName[period+1:])
+		}
+		return fmt.Sprintf("New%s().NomsValue()", userName)
 	case types.ValueKind:
 		// TODO: This is where a null Value would have been useful.
 		return "types.Bool(false)"
@@ -445,15 +453,22 @@ func (gen *codeGen) userName(t types.TypeRef) string {
 	case types.BlobKind, types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind, types.ValueKind, types.TypeRefKind:
 		return kindToString(k)
 	case types.EnumKind:
+		if t.HasPackageRef() {
+			return toTag(t.PackageRef().String()) + "." + t.Name()
+		}
 		return t.Name()
 	case types.ListKind:
+		// TODO: change naming to allow for imported types. Issue #294.
 		return fmt.Sprintf("ListOf%s", gen.userName(t.Desc.(types.CompoundDesc).ElemTypes[0]))
 	case types.MapKind:
+		// TODO: change naming to allow for imported types. Issue #294.
 		elemTypes := t.Desc.(types.CompoundDesc).ElemTypes
 		return fmt.Sprintf("MapOf%sTo%s", gen.userName(elemTypes[0]), gen.userName(elemTypes[1]))
 	case types.RefKind:
+		// TODO: change naming to allow for imported types. Issue #294.
 		return fmt.Sprintf("RefOf%s", gen.userName(t.Desc.(types.CompoundDesc).ElemTypes[0]))
 	case types.SetKind:
+		// TODO: change naming to allow for imported types. Issue #294.
 		return fmt.Sprintf("SetOf%s", gen.userName(t.Desc.(types.CompoundDesc).ElemTypes[0]))
 	case types.StructKind:
 		// We get an empty name when we have a struct that is used as union
@@ -464,25 +479,29 @@ func (gen *codeGen) userName(t types.TypeRef) string {
 				if i > 0 {
 					s += "And"
 				}
-				s += strings.Title(f.Name) + "Of" + gen.userName(f.T)
+				s += strings.Title(f.Name) + "Of" + gen.userName(f.T) // TODO: change naming to allow for imported types. Issue #294.
 			}
 			return s
 		}
+		if t.HasPackageRef() {
+			return toTag(t.PackageRef().String()) + "." + t.Name()
+		}
 		return t.Name()
 	}
 	panic("unreachable")
 }
 
-func (gen *codeGen) toTypesTypeRef(t types.TypeRef, fileID, packageName string) string {
-	if t.IsUnresolved() {
-		refCode := "ref.Ref{}"
-		if t.PackageRef() != (ref.Ref{}) {
-			refCode = fmt.Sprintf(`ref.Parse("%s")`, t.PackageRef().String())
-		} else if fileID != "" {
-			refCode = fmt.Sprintf("__%sPackageInFile_%s_CachedRef", packageName, fileID)
-		}
-		return fmt.Sprintf(`types.MakeTypeRef("%s", %s)`, t.Name(), refCode)
+func toTypeRef(t types.TypeRef, fileID, packageName string) string {
+	if t.HasPackageRef() {
+		return fmt.Sprintf(`types.MakeTypeRef("%s", ref.Parse("%s"))`, t.Name(), t.PackageRef().String())
 	}
+	if t.IsUnresolved() && fileID != "" {
+		return fmt.Sprintf(`types.MakeTypeRef("%s", __%sPackageInFile_%s_CachedRef)`, t.Name(), packageName, fileID)
+	}
+	if t.IsUnresolved() {
+		return fmt.Sprintf(`types.MakeTypeRef("%s", ref.Ref{})`, t.Name())
+	}
+
 	if types.IsPrimitiveKind(t.Desc.Kind()) {
 		return fmt.Sprintf("types.MakePrimitiveTypeRef(types.%sKind)", kindToString(t.Desc.Kind()))
 	}
@@ -490,7 +509,7 @@ func (gen *codeGen) toTypesTypeRef(t types.TypeRef, fileID, packageName string)
 	case types.CompoundDesc:
 		typerefs := make([]string, len(desc.ElemTypes))
 		for i, t := range desc.ElemTypes {
-			typerefs[i] = gen.toTypesTypeRef(t, fileID, packageName)
+			typerefs[i] = toTypeRef(t, fileID, packageName)
 		}
 		return fmt.Sprintf(`types.MakeCompoundTypeRef("%s", types.%sKind, %s)`, t.Name(), kindToString(t.Desc.Kind()), strings.Join(typerefs, ", "))
 	case types.EnumDesc:
@@ -499,7 +518,7 @@ func (gen *codeGen) toTypesTypeRef(t types.TypeRef, fileID, packageName string)
 		flatten := func(f []types.Field) string {
 			out := make([]string, 0, len(f))
 			for _, field := range f {
-				out = append(out, fmt.Sprintf(`types.Field{"%s", %s, %t},`, field.Name, gen.toTypesTypeRef(field.T, fileID, packageName), field.Optional))
+				out = append(out, fmt.Sprintf(`types.Field{"%s", %s, %t},`, field.Name, toTypeRef(field.T, fileID, packageName), field.Optional))
 			}
 			return strings.Join(out, "\n")
 		}
@@ -509,26 +528,43 @@ func (gen *codeGen) toTypesTypeRef(t types.TypeRef, fileID, packageName string)
 	default:
 		d.Chk.Fail("Unknown TypeDesc.", "%#v (%T)", desc, desc)
 	}
-	panic("ain't done")
+	panic("Unreachable")
+}
+
+func kindToString(k types.NomsKind) (out string) {
+	out = types.KindToString[k]
+	d.Chk.NotEmpty(out, "Unknown NomsKind %d", k)
+	return
 }
 
 func (gen *codeGen) resolve(t types.TypeRef) types.TypeRef {
 	if !t.IsUnresolved() {
 		return t
 	}
-	return gen.pkg.NamedTypes[t.Name()]
+	if !t.HasPackageRef() {
+		return gen.pkg.NamedTypes[t.Name()]
+	}
+
+	dep, ok := gen.deps[t.PackageRef()]
+	d.Chk.True(ok, "Package %s is referenced in %+v, but is not a dependency.", t.PackageRef().String(), t)
+	depTypes := dep.NamedTypes()
+	d.Chk.True(depTypes.Has(t.Name()), "Cannot import type %s from package %s.", t.Name(), t.PackageRef().String())
+	return depTypes.Get(t.Name()).MakeImported(t.PackageRef())
 }
 
-func (gen *codeGen) WritePackage(packageName string) {
-	gen.pkg.Name = packageName
+func (gen *codeGen) WritePackage() {
 	data := struct {
+		HasImports bool
 		HasTypes   bool
 		FileID     string
+		Imports    []string
 		Name       string
 		NamedTypes map[string]types.TypeRef
 	}{
+		len(gen.imports) > 0,
 		len(gen.pkg.NamedTypes) > 0,
 		gen.fileid,
+		gen.imports,
 		gen.pkg.Name,
 		gen.pkg.NamedTypes,
 	}
@@ -551,7 +587,8 @@ func (gen *codeGen) WritePackage(packageName string) {
 
 func (gen *codeGen) write(t types.TypeRef) {
 	t = gen.resolve(t)
-	if gen.written[gen.userName(t)] {
+	// If t has a package reference, then it represents an imported type, so we shouldn't generate code for it.
+	if gen.written[gen.userName(t)] || t.HasPackageRef() {
 		return
 	}
 	k := t.Desc.Kind()

nomdl/codegen/codegen_test.go

@@ -3,9 +3,14 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"go/build"
+	"go/parser"
+	"go/token"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -23,6 +28,10 @@ func assertOutput(inPath, goldenPath string, t *testing.T) {
 	assert := assert.New(t)
 	emptyCS := chunks.NewMemoryStore() // Will be ChunkSource containing imports
 
+	depsDir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(depsDir)
+
 	inFile, err := os.Open(inPath)
 	assert.NoError(err)
 	defer inFile.Close()
@@ -34,9 +43,9 @@ func assertOutput(inPath, goldenPath string, t *testing.T) {
 	d.Chk.NoError(err)
 
 	var buf bytes.Buffer
-	pkg := pkg.ParseNomDL("", inFile, emptyCS)
-	gen := NewCodeGen(&buf, getBareFileName(inPath), pkg, map[types.RefOfPackage]types.Package{})
-	gen.WritePackage("test")
+	pkg := pkg.ParseNomDL("test", inFile, emptyCS)
+	gen := NewCodeGen(&buf, getBareFileName(inPath), nil, depsMap{}, pkg)
+	gen.WritePackage()
 
 	bs, err := imports.Process("", buf.Bytes(), nil)
 	d.Chk.NoError(err)
@@ -57,9 +66,13 @@ func TestCanUseDef(t *testing.T) {
 	assert := assert.New(t)
 	emptyCS := chunks.NewMemoryStore()
 
+	depsDir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(depsDir)
+
 	assertCanUseDef := func(s string, using, named bool) {
-		pkg := pkg.ParseNomDL("", bytes.NewBufferString(s), emptyCS)
-		gen := NewCodeGen(nil, "fakefile", pkg, map[types.RefOfPackage]types.Package{})
+		pkg := pkg.ParseNomDL("fakefile", bytes.NewBufferString(s), emptyCS)
+		gen := NewCodeGen(nil, "fakefile", nil, depsMap{}, pkg)
 		for _, t := range pkg.UsingDeclarations {
 			assert.Equal(using, gen.canUseDef(t))
 		}
@@ -138,27 +151,68 @@ func TestCanUseDef(t *testing.T) {
 	}
 }
 
-func TestDerefDeps(t *testing.T) {
+func TestImportedTypes(t *testing.T) {
 	assert := assert.New(t)
-	cs := chunks.NewMemoryStore()
+	ds := datas.NewDataStore(chunks.NewMemoryStore())
+	pkgDS := dataset.NewDataset(ds, "packages")
 
-	leaf1 := types.NewPackage()
-	leaf1Ref := types.WriteValue(leaf1.NomsValue(), cs)
-	leaf2 := types.PackageDef{NamedTypes: types.MapOfStringToTypeRefDef{"foo": types.MakePrimitiveTypeRef(types.BoolKind)}}.New()
-	leaf2Ref := types.WriteValue(leaf2.NomsValue(), cs)
+	dir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
 
-	depender := types.PackageDef{Dependencies: types.SetOfRefOfPackageDef{leaf1Ref: true}}.New()
-	dependerRef := types.WriteValue(depender.NomsValue(), cs)
+	imported := types.PackageDef{
+		NamedTypes: types.MapOfStringToTypeRefDef{
+			"E1": types.MakeEnumTypeRef("E1", "a", "b"),
+			"S1": types.MakeStructTypeRef("S1", []types.Field{
+				types.Field{"f", types.MakePrimitiveTypeRef(types.BoolKind), false},
+			}, types.Choices{})},
+	}.New()
+	importedRef := types.WriteValue(imported.NomsValue(), ds)
+	pkgDS, ok := pkgDS.Commit(types.NewSetOfRefOfPackage().Insert(types.NewRefOfPackage(importedRef)).NomsValue())
+	assert.True(ok)
+	good := fmt.Sprintf(`
+		alias Other = import "%s"
 
-	top := types.PackageDef{Dependencies: types.SetOfRefOfPackageDef{leaf2Ref: true, dependerRef: true}}.New()
-	types.RegisterPackage(&top)
+		struct Simple {
+			E: Other.E1
+			S: Other.S1
+		}
+		`, importedRef)
 
-	immediateDeps := derefDeps(top, cs)
-	assert.Len(immediateDeps, 2)
-	_, ok := immediateDeps[types.NewRefOfPackage(leaf1Ref)]
-	assert.False(ok)
-	assert.True(immediateDeps[types.NewRefOfPackage(leaf2Ref)].Equals(leaf2))
-	assert.True(immediateDeps[types.NewRefOfPackage(dependerRef)].Equals(depender))
+	inFile := filepath.Join(dir, "in.noms")
+	err = ioutil.WriteFile(inFile, []byte(good), 0600)
+	assert.NoError(err)
+
+	depsDir := filepath.Join(thisFileDir(), "deps")
+	defer os.RemoveAll(depsDir)
+
+	outFile := filepath.Join(dir, "out.go")
+	pkgDS = generate("name", inFile, outFile, depsDir, pkgDS)
+
+	// Check that dependency code was generated.
+	expectedDepPkgAbs := filepath.Join(depsDir, toTag(importedRef.String()))
+	_, err = os.Stat(expectedDepPkgAbs)
+	assert.NoError(err)
+
+	// Get the imports from out.go
+	ast, err := parser.ParseFile(token.NewFileSet(), outFile, nil, parser.ImportsOnly)
+	assert.NoError(err)
+	imports := []string{}
+	for _, s := range ast.Imports {
+		//Strip enclosing quotes from s.Path.Value
+		imports = append(imports, s.Path.Value[1:len(s.Path.Value)-1])
+	}
+	// Get the canonical import path for the generated dependency code.
+	expectedDepPkg, err := build.ImportDir(expectedDepPkgAbs, build.FindOnly)
+	assert.NoError(err)
+
+	// Make sure that out.go imported the dependency code.
+	assert.Contains(imports, expectedDepPkg.ImportPath)
+}
+
+func thisFileDir() string {
+	_, filename, _, _ := runtime.Caller(1)
+	return path.Dir(filename)
 }
 
 func TestGenerateDeps(t *testing.T) {

nomdl/codegen/header.tmpl

@@ -2,6 +2,13 @@
 
 package {{.Name}}
 
+{{if .HasImports}}
+import (
+	{{range $imp := .Imports}}"{{$imp}}"
+	{{end}}
+)
+{{end}}
+
 {{if .HasTypes}}
 var __{{.Name}}PackageInFile_{{.FileID}}_CachedRef = __{{.Name}}PackageInFile_{{.FileID}}_Ref()
 

nomdl/codegen/test/rungen.go

@@ -1,3 +1,9 @@
 package test
 
-//go:generate go run ../codegen.go
+//go:generate go run ../codegen.go -deps-dir=gen -ldb=/tmp/depGenTest -package-ds=testDeps -in=../testDeps/leafDep/leafDep.noms -out=../testDeps/leafDep/leafDep.go
+
+//go:generate go run ../codegen.go -deps-dir=gen -ldb=/tmp/depGenTest -package-ds=testDeps -in=../testDeps/dep.noms -out=../testDeps/dep.go
+
+//go:generate go run ../codegen.go -deps-dir=gen -ldb=/tmp/depGenTest -package-ds=testDeps
+
+//go:generate rm -rf /tmp/depGenTest

nomdl/codegen/test/struct_with_imports.go

@@ -0,0 +1,128 @@
+// This file was generated by nomdl/codegen.
+
+package test
+
+import (
+	"github.com/attic-labs/noms/nomdl/codegen/test/gen/sha1_f9397427926127f67d8f3edb21c92bf642262e9b"
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+)
+
+var __testPackageInFile_struct_with_imports_CachedRef = __testPackageInFile_struct_with_imports_Ref()
+
+// This function builds up a Noms value that describes the type
+// package implemented by this file and registers it with the global
+// type package definition cache.
+func __testPackageInFile_struct_with_imports_Ref() ref.Ref {
+	p := types.PackageDef{
+		NamedTypes: types.MapOfStringToTypeRefDef{
+
+			"E": types.MakeEnumTypeRef("E", "E1", "Ignored"),
+			"ImportUser": types.MakeStructTypeRef("ImportUser",
+				[]types.Field{
+					types.Field{"importedStruct", types.MakeTypeRef("D", ref.Parse("sha1-f9397427926127f67d8f3edb21c92bf642262e9b")), false},
+					types.Field{"enum", types.MakeTypeRef("E", ref.Ref{}), false},
+				},
+				types.Choices{},
+			),
+		},
+	}.New()
+	return types.RegisterPackage(&p)
+}
+
+// E
+
+type E uint32
+
+const (
+	E1 E = iota
+	Ignored
+)
+
+// ImportUser
+
+type ImportUser struct {
+	m types.Map
+}
+
+func NewImportUser() ImportUser {
+	return ImportUser{types.NewMap(
+		types.NewString("$name"), types.NewString("ImportUser"),
+		types.NewString("$type"), types.MakeTypeRef("ImportUser", __testPackageInFile_struct_with_imports_CachedRef),
+		types.NewString("importedStruct"), sha1_f9397427926127f67d8f3edb21c92bf642262e9b.NewD().NomsValue(),
+		types.NewString("enum"), types.UInt32(0),
+	)}
+}
+
+type ImportUserDef struct {
+	ImportedStruct sha1_f9397427926127f67d8f3edb21c92bf642262e9b.DDef
+	Enum           E
+}
+
+func (def ImportUserDef) New() ImportUser {
+	return ImportUser{
+		types.NewMap(
+			types.NewString("$name"), types.NewString("ImportUser"),
+			types.NewString("$type"), types.MakeTypeRef("ImportUser", __testPackageInFile_struct_with_imports_CachedRef),
+			types.NewString("importedStruct"), def.ImportedStruct.New().NomsValue(),
+			types.NewString("enum"), types.UInt32(def.Enum),
+		)}
+}
+
+func (s ImportUser) Def() (d ImportUserDef) {
+	d.ImportedStruct = sha1_f9397427926127f67d8f3edb21c92bf642262e9b.DFromVal(s.m.Get(types.NewString("importedStruct"))).Def()
+	d.Enum = E(s.m.Get(types.NewString("enum")).(types.UInt32))
+	return
+}
+
+var __typeRefForImportUser = types.MakeTypeRef("ImportUser", __testPackageInFile_struct_with_imports_CachedRef)
+
+func (m ImportUser) TypeRef() types.TypeRef {
+	return __typeRefForImportUser
+}
+
+func init() {
+	types.RegisterFromValFunction(__typeRefForImportUser, func(v types.Value) types.NomsValue {
+		return ImportUserFromVal(v)
+	})
+}
+
+func ImportUserFromVal(val types.Value) ImportUser {
+	// TODO: Validate here
+	return ImportUser{val.(types.Map)}
+}
+
+func (s ImportUser) NomsValue() types.Value {
+	return s.m
+}
+
+func (s ImportUser) Equals(other types.Value) bool {
+	if other, ok := other.(ImportUser); ok {
+		return s.m.Equals(other.m)
+	}
+	return false
+}
+
+func (s ImportUser) Ref() ref.Ref {
+	return s.m.Ref()
+}
+
+func (s ImportUser) Chunks() []types.Future {
+	return s.m.Chunks()
+}
+
+func (s ImportUser) ImportedStruct() sha1_f9397427926127f67d8f3edb21c92bf642262e9b.D {
+	return sha1_f9397427926127f67d8f3edb21c92bf642262e9b.DFromVal(s.m.Get(types.NewString("importedStruct")))
+}
+
+func (s ImportUser) SetImportedStruct(val sha1_f9397427926127f67d8f3edb21c92bf642262e9b.D) ImportUser {
+	return ImportUser{s.m.Set(types.NewString("importedStruct"), val.NomsValue())}
+}
+
+func (s ImportUser) Enum() E {
+	return E(s.m.Get(types.NewString("enum")).(types.UInt32))
+}
+
+func (s ImportUser) SetEnum(val E) ImportUser {
+	return ImportUser{s.m.Set(types.NewString("enum"), types.UInt32(val))}
+}

nomdl/codegen/test/struct_with_imports.noms

@@ -0,0 +1,11 @@
+alias dep = import "sha1-f9397427926127f67d8f3edb21c92bf642262e9b"
+
+enum E {
+	E1
+	Ignored
+}
+
+struct ImportUser {
+	importedStruct :dep.D
+	enum :E
+}

nomdl/codegen/test/struct_with_imports_test.go

@@ -0,0 +1,37 @@
+package test
+
+import (
+	"testing"
+
+	"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert"
+	leaf "github.com/attic-labs/noms/nomdl/codegen/test/gen/sha1_f1a192312c01fb47e8e329471242e475eb7001a4"
+	dep "github.com/attic-labs/noms/nomdl/codegen/test/gen/sha1_f9397427926127f67d8f3edb21c92bf642262e9b"
+)
+
+func TestWithImportsDef(t *testing.T) {
+	assert := assert.New(t)
+
+	def := ImportUserDef{
+		ImportedStruct: dep.DDef{
+			StructField: leaf.SDef{S: "hi", B: true},
+			EnumField:   leaf.E2,
+		},
+		Enum: E1,
+	}
+	st := def.New()
+
+	def2 := st.Def()
+	st2 := def.New()
+
+	assert.Equal(def, def2)
+	assert.True(st.Equals(st2))
+
+	ds := dep.NewD().SetStructField(leaf.NewS().SetS("hi").SetB(true)).SetEnumField(leaf.E2)
+	st3 := NewImportUser()
+	st3 = st3.SetImportedStruct(ds).SetEnum(E1)
+	assert.True(st.Equals(st3))
+	ddef := st3.ImportedStruct().Def()
+	assert.Equal("hi", ddef.StructField.S)
+	assert.Equal(true, ddef.StructField.B)
+	assert.Equal(leaf.E2, ddef.EnumField)
+}

nomdl/codegen/testDeps/dep.go

@@ -0,0 +1,118 @@
+// This file was generated by nomdl/codegen.
+
+package test
+
+import (
+	"github.com/attic-labs/noms/nomdl/codegen/test/gen/sha1_f1a192312c01fb47e8e329471242e475eb7001a4"
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+)
+
+var __testPackageInFile_dep_CachedRef = __testPackageInFile_dep_Ref()
+
+// This function builds up a Noms value that describes the type
+// package implemented by this file and registers it with the global
+// type package definition cache.
+func __testPackageInFile_dep_Ref() ref.Ref {
+	p := types.PackageDef{
+		NamedTypes: types.MapOfStringToTypeRefDef{
+
+			"D": types.MakeStructTypeRef("D",
+				[]types.Field{
+					types.Field{"structField", types.MakeTypeRef("S", ref.Parse("sha1-f1a192312c01fb47e8e329471242e475eb7001a4")), false},
+					types.Field{"enumField", types.MakeTypeRef("E", ref.Parse("sha1-f1a192312c01fb47e8e329471242e475eb7001a4")), false},
+				},
+				types.Choices{},
+			),
+		},
+	}.New()
+	return types.RegisterPackage(&p)
+}
+
+// D
+
+type D struct {
+	m types.Map
+}
+
+func NewD() D {
+	return D{types.NewMap(
+		types.NewString("$name"), types.NewString("D"),
+		types.NewString("$type"), types.MakeTypeRef("D", __testPackageInFile_dep_CachedRef),
+		types.NewString("structField"), sha1_f1a192312c01fb47e8e329471242e475eb7001a4.NewS().NomsValue(),
+		types.NewString("enumField"), types.UInt32(0),
+	)}
+}
+
+type DDef struct {
+	StructField sha1_f1a192312c01fb47e8e329471242e475eb7001a4.SDef
+	EnumField   sha1_f1a192312c01fb47e8e329471242e475eb7001a4.E
+}
+
+func (def DDef) New() D {
+	return D{
+		types.NewMap(
+			types.NewString("$name"), types.NewString("D"),
+			types.NewString("$type"), types.MakeTypeRef("D", __testPackageInFile_dep_CachedRef),
+			types.NewString("structField"), def.StructField.New().NomsValue(),
+			types.NewString("enumField"), types.UInt32(def.EnumField),
+		)}
+}
+
+func (s D) Def() (d DDef) {
+	d.StructField = sha1_f1a192312c01fb47e8e329471242e475eb7001a4.SFromVal(s.m.Get(types.NewString("structField"))).Def()
+	d.EnumField = sha1_f1a192312c01fb47e8e329471242e475eb7001a4.E(s.m.Get(types.NewString("enumField")).(types.UInt32))
+	return
+}
+
+var __typeRefForD = types.MakeTypeRef("D", __testPackageInFile_dep_CachedRef)
+
+func (m D) TypeRef() types.TypeRef {
+	return __typeRefForD
+}
+
+func init() {
+	types.RegisterFromValFunction(__typeRefForD, func(v types.Value) types.NomsValue {
+		return DFromVal(v)
+	})
+}
+
+func DFromVal(val types.Value) D {
+	// TODO: Validate here
+	return D{val.(types.Map)}
+}
+
+func (s D) NomsValue() types.Value {
+	return s.m
+}
+
+func (s D) Equals(other types.Value) bool {
+	if other, ok := other.(D); ok {
+		return s.m.Equals(other.m)
+	}
+	return false
+}
+
+func (s D) Ref() ref.Ref {
+	return s.m.Ref()
+}
+
+func (s D) Chunks() []types.Future {
+	return s.m.Chunks()
+}
+
+func (s D) StructField() sha1_f1a192312c01fb47e8e329471242e475eb7001a4.S {
+	return sha1_f1a192312c01fb47e8e329471242e475eb7001a4.SFromVal(s.m.Get(types.NewString("structField")))
+}
+
+func (s D) SetStructField(val sha1_f1a192312c01fb47e8e329471242e475eb7001a4.S) D {
+	return D{s.m.Set(types.NewString("structField"), val.NomsValue())}
+}
+
+func (s D) EnumField() sha1_f1a192312c01fb47e8e329471242e475eb7001a4.E {
+	return sha1_f1a192312c01fb47e8e329471242e475eb7001a4.E(s.m.Get(types.NewString("enumField")).(types.UInt32))
+}
+
+func (s D) SetEnumField(val sha1_f1a192312c01fb47e8e329471242e475eb7001a4.E) D {
+	return D{s.m.Set(types.NewString("enumField"), types.UInt32(val))}
+}

nomdl/codegen/testDeps/dep.noms

@@ -0,0 +1,6 @@
+alias leaf = import "sha1-f1a192312c01fb47e8e329471242e475eb7001a4"
+
+struct D {
+  structField: leaf.S
+  enumField: leaf.E
+}

nomdl/codegen/testDeps/leafDep/leafDep.go

@@ -0,0 +1,128 @@
+// This file was generated by nomdl/codegen.
+
+package test
+
+import (
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+)
+
+var __testPackageInFile_leafDep_CachedRef = __testPackageInFile_leafDep_Ref()
+
+// This function builds up a Noms value that describes the type
+// package implemented by this file and registers it with the global
+// type package definition cache.
+func __testPackageInFile_leafDep_Ref() ref.Ref {
+	p := types.PackageDef{
+		NamedTypes: types.MapOfStringToTypeRefDef{
+
+			"E": types.MakeEnumTypeRef("E", "e1", "e2", "e3"),
+			"S": types.MakeStructTypeRef("S",
+				[]types.Field{
+					types.Field{"s", types.MakePrimitiveTypeRef(types.StringKind), false},
+					types.Field{"b", types.MakePrimitiveTypeRef(types.BoolKind), false},
+				},
+				types.Choices{},
+			),
+		},
+	}.New()
+	return types.RegisterPackage(&p)
+}
+
+// E
+
+type E uint32
+
+const (
+	E1 E = iota
+	E2
+	E3
+)
+
+// S
+
+type S struct {
+	m types.Map
+}
+
+func NewS() S {
+	return S{types.NewMap(
+		types.NewString("$name"), types.NewString("S"),
+		types.NewString("$type"), types.MakeTypeRef("S", __testPackageInFile_leafDep_CachedRef),
+		types.NewString("s"), types.NewString(""),
+		types.NewString("b"), types.Bool(false),
+	)}
+}
+
+type SDef struct {
+	S string
+	B bool
+}
+
+func (def SDef) New() S {
+	return S{
+		types.NewMap(
+			types.NewString("$name"), types.NewString("S"),
+			types.NewString("$type"), types.MakeTypeRef("S", __testPackageInFile_leafDep_CachedRef),
+			types.NewString("s"), types.NewString(def.S),
+			types.NewString("b"), types.Bool(def.B),
+		)}
+}
+
+func (s S) Def() (d SDef) {
+	d.S = s.m.Get(types.NewString("s")).(types.String).String()
+	d.B = bool(s.m.Get(types.NewString("b")).(types.Bool))
+	return
+}
+
+var __typeRefForS = types.MakeTypeRef("S", __testPackageInFile_leafDep_CachedRef)
+
+func (m S) TypeRef() types.TypeRef {
+	return __typeRefForS
+}
+
+func init() {
+	types.RegisterFromValFunction(__typeRefForS, func(v types.Value) types.NomsValue {
+		return SFromVal(v)
+	})
+}
+
+func SFromVal(val types.Value) S {
+	// TODO: Validate here
+	return S{val.(types.Map)}
+}
+
+func (s S) NomsValue() types.Value {
+	return s.m
+}
+
+func (s S) Equals(other types.Value) bool {
+	if other, ok := other.(S); ok {
+		return s.m.Equals(other.m)
+	}
+	return false
+}
+
+func (s S) Ref() ref.Ref {
+	return s.m.Ref()
+}
+
+func (s S) Chunks() []types.Future {
+	return s.m.Chunks()
+}
+
+func (s S) S() string {
+	return s.m.Get(types.NewString("s")).(types.String).String()
+}
+
+func (s S) SetS(val string) S {
+	return S{s.m.Set(types.NewString("s"), types.NewString(val))}
+}
+
+func (s S) B() bool {
+	return bool(s.m.Get(types.NewString("b")).(types.Bool))
+}
+
+func (s S) SetB(val bool) S {
+	return S{s.m.Set(types.NewString("b"), types.Bool(val))}
+}

nomdl/codegen/testDeps/leafDep/leafDep.noms

@@ -0,0 +1,10 @@
+struct S {
+  s: String
+  b: Bool
+}
+
+enum E {
+  e1
+  e2
+  e3
+}

nomdl/pkg/import_test.go

@@ -18,7 +18,9 @@ func TestImportSuite(t *testing.T) {
 type ImportTestSuite struct {
 	suite.Suite
 	cs        chunks.ChunkStore
+	imported  types.Package
 	importRef ref.Ref
+	nested    types.Package
 	nestedRef ref.Ref
 }
 
@@ -29,9 +31,10 @@ func (suite *ImportTestSuite) SetupTest() {
 		types.Field{"b", types.MakePrimitiveTypeRef(types.BoolKind), false},
 		types.Field{"i", types.MakePrimitiveTypeRef(types.Int8Kind), false},
 	})
-	suite.nestedRef = types.WriteValue(types.PackageDef{
+	suite.nested = types.PackageDef{
 		NamedTypes: types.MapOfStringToTypeRefDef{"NestedDepStruct": ns},
-	}.New().NomsValue(), suite.cs)
+	}.New()
+	suite.nestedRef = types.WriteValue(suite.nested.NomsValue(), suite.cs)
 
 	fs := types.MakeStructTypeRef("ForeignStruct", []types.Field{
 		types.Field{"b", types.MakePrimitiveTypeRef(types.BoolKind), false},
@@ -39,11 +42,11 @@ func (suite *ImportTestSuite) SetupTest() {
 	},
 		types.Choices{})
 	fe := types.MakeEnumTypeRef("ForeignEnum", "uno", "dos")
-
-	suite.importRef = types.WriteValue(types.PackageDef{
+	suite.imported = types.PackageDef{
 		Dependencies: types.SetOfRefOfPackageDef{suite.nestedRef: true},
 		NamedTypes:   types.MapOfStringToTypeRefDef{"ForeignStruct": fs, "ForeignEnum": fe},
-	}.New().NomsValue(), suite.cs)
+	}.New()
+	suite.importRef = types.WriteValue(suite.imported.NomsValue(), suite.cs)
 }
 
 func (suite *ImportTestSuite) TestGetDeps() {
@@ -52,7 +55,7 @@ func (suite *ImportTestSuite) TestGetDeps() {
 	imported, ok := deps[suite.importRef]
 	suite.True(ok, "%s is a dep; should have been found.", suite.importRef.String())
 
-	deps = GetDeps(imported.Dependencies, suite.cs)
+	deps = GetDeps(imported.Dependencies().Def(), suite.cs)
 	suite.Len(deps, 1)
 	imported, ok = deps[suite.nestedRef]
 	suite.True(ok, "%s is a dep; should have been found.", suite.nestedRef.String())
@@ -85,7 +88,7 @@ func (suite *ImportTestSuite) TestDetectFreeVariable() {
 	},
 		types.Choices{})
 	suite.Panics(func() {
-		resolveNamespaces(map[string]types.TypeRef{"Local": ls}, map[string]ref.Ref{}, map[ref.Ref]types.PackageDef{})
+		resolveNamespaces(map[string]types.TypeRef{"Local": ls}, map[string]ref.Ref{}, map[ref.Ref]types.Package{})
 	})
 }
 

nomdl/pkg/parse.go

@@ -27,12 +27,12 @@ func ParseNomDL(packageName string, r io.Reader, cs chunks.ChunkSource) Parsed {
 }
 
 // GetDeps reads the types.Package objects referred to by depRefs out of cs and returns a map of ref: PackageDef.
-func GetDeps(depRefs types.SetOfRefOfPackageDef, cs chunks.ChunkSource) map[ref.Ref]types.PackageDef {
-	deps := map[ref.Ref]types.PackageDef{}
+func GetDeps(depRefs types.SetOfRefOfPackageDef, cs chunks.ChunkSource) map[ref.Ref]types.Package {
+	deps := map[ref.Ref]types.Package{}
 	for depRef := range depRefs {
 		v := types.ReadValue(depRef, cs)
 		d.Chk.NotNil(v, "Importing package by ref %s failed.", depRef.String())
-		deps[depRef] = types.PackageFromVal(v).Def()
+		deps[depRef] = types.PackageFromVal(v)
 	}
 	return deps
 }
@@ -71,7 +71,7 @@ func resolveImports(pkg intermediate) map[string]ref.Ref {
 	return aliases
 }
 
-func resolveNamespaces(namedTypes map[string]types.TypeRef, aliases map[string]ref.Ref, deps map[ref.Ref]types.PackageDef) {
+func resolveNamespaces(namedTypes map[string]types.TypeRef, aliases map[string]ref.Ref, deps map[ref.Ref]types.Package) {
 	var rec func(t types.TypeRef) types.TypeRef
 	resolveFields := func(fields []types.Field) {
 		for idx, f := range fields {
@@ -107,10 +107,9 @@ func resolveNamespaces(namedTypes map[string]types.TypeRef, aliases map[string]r
 
 }
 
-func resolveNamespace(t types.TypeRef, aliases map[string]ref.Ref, deps map[ref.Ref]types.PackageDef) types.TypeRef {
+func resolveNamespace(t types.TypeRef, aliases map[string]ref.Ref, deps map[ref.Ref]types.Package) types.TypeRef {
 	target, ok := aliases[t.Namespace()]
 	d.Exp.True(ok, "Could not find import aliased to %s", t.Namespace())
-	_, ok = deps[target].NamedTypes[t.Name()]
-	d.Exp.True(ok, "Could not find type %s in package %s (aliased to %s).", t.Name(), target.String(), t.Namespace())
+	d.Exp.True(deps[target].NamedTypes().Has(t.Name()), "Could not find type %s in package %s (aliased to %s).", t.Name(), target.String(), t.Namespace())
 	return types.MakeTypeRef(t.Name(), target)
 }

types/type_ref.go

@@ -43,6 +43,10 @@ func (t TypeRef) IsUnresolved() bool {
 	return ok
 }
 
+func (t TypeRef) HasPackageRef() bool {
+	return t.pkgRef != nil
+}
+
 // Describe() methods generate text that should parse into the struct being described.
 // TODO: Figure out a way that they can exist only in the test file.
 func (t TypeRef) Describe() (out string) {
@@ -78,6 +82,10 @@ func (t TypeRef) Namespace() string {
 	return t.name.namespace
 }
 
+func (t TypeRef) MakeImported(pkg ref.Ref) TypeRef {
+	return TypeRef{name: name{name: t.Name()}, pkgRef: &Ref{R: pkg}, Desc: t.Desc, ref: &ref.Ref{}}
+}
+
 func (t TypeRef) Ref() ref.Ref {
 	return ensureRef(t.ref, t)
 }
@@ -132,19 +140,6 @@ func MakeStructTypeRef(name string, fields []Field, choices Choices) TypeRef {
 	return buildType(name, StructDesc{fields, choices})
 }
 
-func MakeImportedTypeRef(k NomsKind, n string, pkg ref.Ref) TypeRef {
-	var desc TypeDesc
-	switch k {
-	default:
-		d.Chk.Fail("Must be StructKind or EnumKind")
-	case StructKind:
-		desc = StructDesc{}
-	case EnumKind:
-		desc = EnumDesc{}
-	}
-	return TypeRef{name: name{name: n}, pkgRef: &Ref{R: pkg}, Desc: desc, ref: &ref.Ref{}}
-}
-
 func MakeTypeRef(n string, pkg ref.Ref) TypeRef {
 	return TypeRef{name: name{name: n}, pkgRef: &Ref{R: pkg}, Desc: UnresolvedDesc{}, ref: &ref.Ref{}}
 }