Files
dolt/nomdl/codegen/codegen.go
Erik Arvidsson b67ab2a9e1 Convert datas/ to nomdl/codegen
This make Commit a typed struct with a Set(Commit).

This also fixes a case where the recursive detection for determining
if a Def can be created was not working.
2015-09-24 12:56:58 -04:00

679 lines
19 KiB
Go

package main
import (
"bytes"
"flag"
"fmt"
"go/parser"
"go/token"
"io"
"os"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
"text/template"
"github.com/attic-labs/noms/Godeps/_workspace/src/golang.org/x/tools/imports"
"github.com/attic-labs/noms/types"
"github.com/attic-labs/noms/d"
"github.com/attic-labs/noms/nomdl/parse"
)
var (
inFlag = flag.String("in", "", "The name of the noms file to read")
outFlag = flag.String("out", "", "The name of the go file to write")
packageFlag = flag.String("package", "", "The name of the go package to write")
)
const ext = ".noms"
func main() {
flag.Parse()
packageName := getGoPackageName()
if *inFlag != "" {
out := *outFlag
if out == "" {
out = getOutFileName(*inFlag)
}
generate(packageName, *inFlag, out)
return
}
// Generate code from all .noms file in the current directory
nomsFiles, err := filepath.Glob("*" + ext)
d.Chk.NoError(err)
for _, n := range nomsFiles {
generate(packageName, n, getOutFileName(n))
}
}
func getOutFileName(in string) string {
return in[:len(in)-len(ext)] + ".go"
}
func getBareFileName(in string) string {
base := filepath.Base(in)
return base[:len(base)-len(filepath.Ext(base))]
}
func generate(packageName, in, out string) {
inFile, err := os.Open(in)
d.Chk.NoError(err)
defer inFile.Close()
var buf bytes.Buffer
pkg := parse.ParsePackage("", inFile)
gen := NewCodeGen(&buf, getBareFileName(in), pkg)
gen.WritePackage(packageName)
bs, err := imports.Process(out, buf.Bytes(), nil)
d.Chk.NoError(err)
outFile, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
d.Chk.NoError(err)
defer outFile.Close()
io.Copy(outFile, bytes.NewBuffer(bs))
}
func getGoPackageName() string {
if *packageFlag != "" {
return *packageFlag
}
// It is illegal to have multiple go files in the same directory with different package names.
// We can therefore just pick the first one and get the package name from there.
goFiles, err := filepath.Glob("*.go")
d.Chk.NoError(err)
d.Chk.True(len(goFiles) > 0)
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, goFiles[0], nil, parser.PackageClauseOnly)
d.Chk.NoError(err)
return f.Name.String()
}
type codeGen struct {
w io.Writer
pkg parse.Package
fileid string
written map[string]bool
templates *template.Template
}
func NewCodeGen(w io.Writer, fileID string, pkg parse.Package) *codeGen {
gen := &codeGen{w, pkg, fileID, map[string]bool{}, nil}
gen.templates = gen.readTemplates()
return gen
}
func (gen *codeGen) readTemplates() *template.Template {
_, thisfile, _, _ := runtime.Caller(1)
glob := path.Join(path.Dir(thisfile), "*.tmpl")
return template.Must(template.New("").Funcs(
template.FuncMap{
"defType": gen.defType,
"defToValue": gen.defToValue,
"valueToDef": gen.valueToDef,
"userType": gen.userType,
"userToValue": gen.userToValue,
"valueToUser": gen.valueToUser,
"userZero": gen.userZero,
"valueZero": gen.valueZero,
"title": strings.Title,
"toTypesTypeRef": gen.toTypesTypeRef,
}).ParseGlob(glob))
}
// Conceptually there are few type spaces here:
//
// - Def - MyStructDef, ListOfBoolDef
// - Native - such as string, uint32
// - Value - the generic types.Value
// - Nom - types.String, types.UInt32, MyStruct, ListOfBool
// - User - User defined structs, enums etc as well as native primitves. This uses Native when possible or Nom if not
//
// These naming conventions are used for the conversion functions available
// in the templates.
func (gen *codeGen) defType(t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BlobKind:
return "types.Blob"
case types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return strings.ToLower(kindToString(k))
case types.EnumKind:
return gen.userName(t)
case types.ListKind, types.MapKind, types.SetKind, types.StructKind:
return gen.userName(t) + "Def"
case types.RefKind:
return "ref.Ref"
case types.ValueKind:
return "types.Value"
case types.TypeRefKind:
return "types.TypeRef"
}
panic("unreachable")
}
func (gen *codeGen) userType(t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BlobKind:
return "types.Blob"
case types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return strings.ToLower(kindToString(k))
case types.EnumKind, types.ListKind, types.MapKind, types.RefKind, types.SetKind, types.StructKind:
return gen.userName(t)
case types.ValueKind:
return "types.Value"
case types.TypeRefKind:
return "types.TypeRef"
}
panic("unreachable")
}
func (gen *codeGen) defToValue(val string, t parse.TypeRef) string {
t = gen.resolve(t)
switch t.Desc.Kind() {
case types.BlobKind, types.ValueKind, types.TypeRefKind:
return val // Blob & Value type has no Def
case types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return gen.nativeToValue(val, t)
case types.EnumKind:
return fmt.Sprintf("types.Int32(%s)", val)
case types.ListKind, types.MapKind, types.SetKind, types.StructKind:
return fmt.Sprintf("%s.New().NomsValue()", val)
case types.RefKind:
return fmt.Sprintf("types.Ref{R: %s}", val)
}
panic("unreachable")
}
func (gen *codeGen) valueToDef(val string, t parse.TypeRef) string {
t = gen.resolve(t)
switch t.Desc.Kind() {
case types.BlobKind:
return gen.valueToUser(val, t)
case types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return gen.valueToNative(val, t)
case types.EnumKind:
return fmt.Sprintf("%s(%s.(types.Int32))", gen.userName(t), val)
case types.ListKind, types.MapKind, types.SetKind, types.StructKind:
return fmt.Sprintf("%s.Def()", gen.valueToUser(val, t))
case types.RefKind:
return fmt.Sprintf("%s.Ref()", val)
case types.ValueKind:
return val // Value type has no Def
case types.TypeRefKind:
return gen.valueToUser(val, t)
}
panic("unreachable")
}
func kindToString(k types.NomsKind) string {
switch k {
case types.BlobKind:
return "Blob"
case types.BoolKind:
return "Bool"
case types.Float32Kind:
return "Float32"
case types.Float64Kind:
return "Float64"
case types.Int16Kind:
return "Int16"
case types.Int32Kind:
return "Int32"
case types.Int64Kind:
return "Int64"
case types.Int8Kind:
return "Int8"
case types.StringKind:
return "String"
case types.UInt16Kind:
return "UInt16"
case types.UInt32Kind:
return "UInt32"
case types.UInt64Kind:
return "UInt64"
case types.UInt8Kind:
return "UInt8"
case types.ValueKind:
return "Value"
case types.TypeRefKind:
return "TypeRef"
case types.ListKind:
return "List"
case types.MapKind:
return "Map"
case types.RefKind:
return "Ref"
case types.SetKind:
return "Set"
}
panic("unreachable")
}
func (gen *codeGen) nativeToValue(val string, t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return fmt.Sprintf("types.%s(%s)", kindToString(k), val)
case types.EnumKind:
return fmt.Sprintf("types.Int32(%s)", val)
case types.StringKind:
return "types.NewString(" + val + ")"
}
panic("unreachable")
}
func (gen *codeGen) valueToNative(val string, t parse.TypeRef) string {
k := t.Desc.Kind()
switch k {
case types.EnumKind:
return fmt.Sprintf("%s(%s.(types.Int32))", gen.userType(t), val)
case types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
n := kindToString(k)
return fmt.Sprintf("%s(%s.(types.%s))", strings.ToLower(n), val, n)
case types.StringKind:
return val + ".(types.String).String()"
}
panic("unreachable")
}
func (gen *codeGen) userToValue(val string, t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BlobKind, types.ValueKind, types.TypeRefKind:
return val
case types.BoolKind, types.EnumKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return gen.nativeToValue(val, t)
case types.ListKind, types.MapKind, types.RefKind, types.SetKind, types.StructKind:
return fmt.Sprintf("%s.NomsValue()", val)
}
panic("unreachable")
}
func (gen *codeGen) valueToUser(val string, t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BlobKind:
return fmt.Sprintf("%s.(types.Blob)", val)
case types.BoolKind, types.EnumKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return gen.valueToNative(val, t)
case types.ListKind, types.MapKind, types.RefKind, types.SetKind, types.StructKind:
return fmt.Sprintf("%sFromVal(%s)", gen.userName(t), val)
case types.ValueKind:
return val
case types.TypeRefKind:
return fmt.Sprintf("%s.(types.TypeRef)", val)
}
panic("unreachable")
}
func (gen *codeGen) userZero(t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BlobKind:
return "types.NewEmptyBlob()"
case types.BoolKind:
return "false"
case types.EnumKind:
return fmt.Sprintf("%s(0)", gen.userType(t))
case types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return fmt.Sprintf("%s(0)", strings.ToLower(kindToString(k)))
case types.ListKind, types.MapKind, types.SetKind:
return fmt.Sprintf("New%s()", gen.userType(t))
case types.RefKind:
return fmt.Sprintf("%s{ref.Ref{}}", gen.userType(t))
case types.StringKind:
return `""`
case types.ValueKind:
// TODO: This is where a null Value would have been useful.
return "types.Bool(false)"
case types.TypeRefKind:
return "types.TypeRef{}"
}
panic("unreachable")
}
func (gen *codeGen) valueZero(t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BlobKind:
return "types.NewEmptyBlob()"
case types.BoolKind:
return "types.Bool(false)"
case types.EnumKind:
return "types.Int32(0)"
case types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind:
return fmt.Sprintf("types.%s(0)", kindToString(k))
case types.ListKind:
return "types.NewList()"
case types.MapKind:
return "types.NewMap()"
case types.RefKind:
return "types.Ref{R: ref.Ref{}}"
case types.SetKind:
return "types.NewSet()"
case types.StringKind:
return `types.NewString("")`
case types.StructKind:
return fmt.Sprintf("New%s().NomsValue()", gen.userName(t))
case types.ValueKind:
// TODO: This is where a null Value would have been useful.
return "types.Bool(false)"
case types.TypeRefKind:
return "types.TypeRef{}"
}
panic("unreachable")
}
func (gen *codeGen) userName(t parse.TypeRef) string {
t = gen.resolve(t)
k := t.Desc.Kind()
switch k {
case types.BlobKind, types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind, types.ValueKind, types.TypeRefKind:
return kindToString(k)
case types.EnumKind:
return t.Name
case types.ListKind:
return fmt.Sprintf("ListOf%s", gen.userName(t.Desc.(parse.CompoundDesc).ElemTypes[0]))
case types.MapKind:
elemTypes := t.Desc.(parse.CompoundDesc).ElemTypes
return fmt.Sprintf("MapOf%sTo%s", gen.userName(elemTypes[0]), gen.userName(elemTypes[1]))
case types.RefKind:
return fmt.Sprintf("RefOf%s", gen.userName(t.Desc.(parse.CompoundDesc).ElemTypes[0]))
case types.SetKind:
return fmt.Sprintf("SetOf%s", gen.userName(t.Desc.(parse.CompoundDesc).ElemTypes[0]))
case types.StructKind:
// We get an empty name when we have a struct that is used as union
if t.Name == "" {
union := t.Desc.(parse.StructDesc).Union
s := "__unionOf"
for i, f := range union.Choices {
if i > 0 {
s += "And"
}
s += strings.Title(f.Name) + "Of" + gen.userName(f.T)
}
return s
}
return t.Name
}
panic("unreachable")
}
func (gen *codeGen) toTypesTypeRef(t parse.TypeRef) string {
if t.IsUnresolved() {
// needs to be pkgRef
return fmt.Sprintf(`types.MakeTypeRef(types.NewString("%s"), types.Ref{})`, t.Name)
}
if types.IsPrimitiveKind(t.Desc.Kind()) {
return fmt.Sprintf("types.MakePrimitiveTypeRef(types.%sKind)", kindToString(t.Desc.Kind()))
}
switch desc := t.Desc.(type) {
case parse.CompoundDesc:
typerefs := make([]string, len(desc.ElemTypes))
for i, t := range desc.ElemTypes {
typerefs[i] = gen.toTypesTypeRef(t)
}
return fmt.Sprintf(`types.MakeCompoundTypeRef(types.NewString("%s"), types.%sKind, %s)`, t.Name, kindToString(t.Desc.Kind()), strings.Join(typerefs, ", "))
case parse.EnumDesc:
ids := ""
for _, id := range desc.IDs {
ids += fmt.Sprintf(`types.NewString("%s"),`, id) + "\n"
}
return fmt.Sprintf(`types.MakeEnumTypeRef(types.NewString("%s"), []types.String{%s})`, t.Name, ids)
case parse.StructDesc:
flatten := func(f []parse.Field) string {
out := make([]string, 0, len(f))
for _, field := range f {
out = append(out, fmt.Sprintf(`types.NewString("%s"), %s,`, field.Name, gen.toTypesTypeRef(field.T)))
}
return strings.Join(out, "\n")
}
fields := "nil"
choices := "nil"
if len(desc.Fields) != 0 {
fields = fmt.Sprintf("types.NewList(%s)", flatten(desc.Fields))
}
if desc.Union != nil {
choices = fmt.Sprintf("types.NewList(%s)", flatten(desc.Union.Choices))
}
return fmt.Sprintf(`types.MakeStructTypeRef(types.NewString("%s"), %s, %s)`, t.Name, fields, choices)
default:
d.Chk.Fail("Unknown TypeDesc.", "%#v (%T)", desc, desc)
}
panic("ain't done")
}
func (gen *codeGen) resolve(t parse.TypeRef) parse.TypeRef {
if !t.IsUnresolved() {
return t
}
return gen.pkg.NamedTypes[t.Name]
}
func (gen *codeGen) WritePackage(packageName string) {
gen.pkg.Name = packageName
data := struct {
HasTypes bool
FileID string
Name string
NamedTypes map[string]parse.TypeRef
}{
len(gen.pkg.NamedTypes) > 0,
gen.fileid,
gen.pkg.Name,
gen.pkg.NamedTypes,
}
err := gen.templates.ExecuteTemplate(gen.w, "header.tmpl", data)
d.Exp.NoError(err)
for _, t := range gen.pkg.UsingDeclarations {
gen.write(t)
}
names := make([]string, 0, len(gen.pkg.NamedTypes))
for n, _ := range gen.pkg.NamedTypes {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
gen.write(gen.pkg.NamedTypes[n])
}
}
func (gen *codeGen) write(t parse.TypeRef) {
t = gen.resolve(t)
if gen.written[gen.userName(t)] {
return
}
k := t.Desc.Kind()
switch k {
case types.BlobKind, types.BoolKind, types.Float32Kind, types.Float64Kind, types.Int16Kind, types.Int32Kind, types.Int64Kind, types.Int8Kind, types.StringKind, types.UInt16Kind, types.UInt32Kind, types.UInt64Kind, types.UInt8Kind, types.ValueKind, types.TypeRefKind:
return
case types.EnumKind:
gen.writeEnum(t)
case types.ListKind:
gen.writeList(t)
case types.MapKind:
gen.writeMap(t)
case types.RefKind:
gen.writeRef(t)
case types.SetKind:
gen.writeSet(t)
case types.StructKind:
gen.writeStruct(t)
default:
panic("unreachable")
}
}
func (gen *codeGen) writeTemplate(tmpl string, t parse.TypeRef, data interface{}) {
err := gen.templates.ExecuteTemplate(gen.w, tmpl, data)
d.Exp.NoError(err)
gen.written[gen.userName(t)] = true
}
func (gen *codeGen) writeStruct(t parse.TypeRef) {
desc := t.Desc.(parse.StructDesc)
data := struct {
FileID string
PackageName string
Name string
Fields []parse.Field
Choices []parse.Field
HasUnion bool
UnionZeroType parse.TypeRef
CanUseDef bool
}{
gen.fileid,
gen.pkg.Name,
gen.userName(t),
desc.Fields,
nil,
desc.Union != nil,
parse.TypeRef{Desc: parse.PrimitiveDesc(types.UInt32Kind)},
gen.canUseDef(t),
}
if data.HasUnion {
data.Choices = desc.Union.Choices
data.UnionZeroType = data.Choices[0].T
}
gen.writeTemplate("struct.tmpl", t, data)
for _, f := range desc.Fields {
gen.write(f.T)
}
if data.HasUnion {
for _, f := range desc.Union.Choices {
gen.write(f.T)
}
}
}
func (gen *codeGen) writeList(t parse.TypeRef) {
elemTypes := t.Desc.(parse.CompoundDesc).ElemTypes
data := struct {
Name string
ElemType parse.TypeRef
CanUseDef bool
}{
gen.userName(t),
elemTypes[0],
gen.canUseDef(t),
}
gen.writeTemplate("list.tmpl", t, data)
gen.write(elemTypes[0])
}
func (gen *codeGen) writeMap(t parse.TypeRef) {
elemTypes := t.Desc.(parse.CompoundDesc).ElemTypes
data := struct {
Name string
KeyType parse.TypeRef
ValueType parse.TypeRef
CanUseDef bool
}{
gen.userName(t),
elemTypes[0],
elemTypes[1],
gen.canUseDef(t),
}
gen.writeTemplate("map.tmpl", t, data)
gen.write(elemTypes[0])
gen.write(elemTypes[1])
}
func (gen *codeGen) writeRef(t parse.TypeRef) {
elemTypes := t.Desc.(parse.CompoundDesc).ElemTypes
data := struct {
Name string
ElemType parse.TypeRef
}{
gen.userName(t),
elemTypes[0],
}
gen.writeTemplate("ref.tmpl", t, data)
gen.write(elemTypes[0])
}
func (gen *codeGen) writeSet(t parse.TypeRef) {
elemTypes := t.Desc.(parse.CompoundDesc).ElemTypes
data := struct {
Name string
ElemType parse.TypeRef
CanUseDef bool
}{
gen.userName(t),
elemTypes[0],
gen.canUseDef(t),
}
gen.writeTemplate("set.tmpl", t, data)
gen.write(elemTypes[0])
}
func (gen *codeGen) writeEnum(t parse.TypeRef) {
data := struct {
Name string
Ids []string
}{
t.Name,
t.Desc.(parse.EnumDesc).IDs,
}
gen.writeTemplate("enum.tmpl", t, data)
}
// We use a go map as the def for Set and Map. These cannot have a key that is a
// Set, Map or a List because slices and maps are not comparable in go.
func (gen *codeGen) canUseDef(t parse.TypeRef) bool {
cache := map[string]bool{}
var rec func(t parse.TypeRef, inKey bool) bool
rec = func(t parse.TypeRef, inKey bool) bool {
t = gen.resolve(t)
switch t.Desc.Kind() {
case types.ListKind:
return !inKey && rec(t.Desc.(parse.CompoundDesc).ElemTypes[0], inKey)
case types.SetKind:
return !inKey && rec(t.Desc.(parse.CompoundDesc).ElemTypes[0], true)
case types.MapKind:
elemTypes := t.Desc.(parse.CompoundDesc).ElemTypes
return !inKey && rec(elemTypes[0], true) && rec(elemTypes[1], false)
case types.StructKind:
// Only structs can be recursive
userName := gen.userName(t)
if b, ok := cache[userName]; ok {
return b
}
cache[userName] = false
for _, f := range t.Desc.(parse.StructDesc).Fields {
if !rec(f.T, inKey) {
cache[userName] = false
return false
}
}
cache[userName] = true
return true
default:
return true
}
}
return rec(t, false)
}