Reimplement fitask generator using gengo

This commit is contained in:
John Gardiner Myers 2020-07-26 18:05:47 -07:00
parent b23f0ebb26
commit 9596ed8f37
11 changed files with 129 additions and 355 deletions

View File

@ -162,14 +162,8 @@ upup/models/bindata.go: ${UPUP_MODELS_BINDATA_SOURCES}
.PHONY: codegen
codegen: kops-gobindata
go install k8s.io/kops/upup/tools/generators/...
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/cloudup/awstasks
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/cloudup/gcetasks
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/cloudup/dotasks
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/cloudup/alitasks
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/cloudup/spotinsttasks
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/assettasks
PATH="${GOPATH_1ST}/bin:${PATH}" go generate k8s.io/kops/upup/pkg/fi/fitasks
${GOPATH_1ST}/bin/fitask --input-dirs k8s.io/kops/upup/pkg/fi/... \
--go-header-file "hack/boilerplate/boilerplate.go.txt"
.PHONY: protobuf
protobuf:

1
go.mod
View File

@ -114,6 +114,7 @@ require (
k8s.io/client-go v0.18.1
k8s.io/cloud-provider-openstack v1.17.0
k8s.io/component-base v0.18.1
k8s.io/gengo v0.0.0-20200710205751-c0d492a0f3ca
k8s.io/helm v2.9.0+incompatible
k8s.io/klog v1.0.0
k8s.io/kubectl v0.0.0

7
go.sum
View File

@ -370,6 +370,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@ -1017,6 +1019,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371 h1:Cjq6sG3gnKDchzWy7ouGQklhxMtWvh4AhSNJ0qGIeo4=
golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f h1:JcoF/bowzCDI+MXu1yLqQGNO3ibqWsWq+Sk7pOT218w=
golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1142,6 +1145,8 @@ k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKO
k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4=
k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200710205751-c0d492a0f3ca h1:/o8XeHsWWmi4lTKp3uxWAZY7Eq/v1HelCDmrKZM4SVQ=
k8s.io/gengo v0.0.0-20200710205751-c0d492a0f3ca/go.mod h1:aG2eeomYfcUw8sE3fa7YdkjgnGtyY56TjZlaJJ0ZoWo=
k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
k8s.io/helm v2.9.0+incompatible h1:3EFDJoqKSUe1BpC9qP+YaHi2Oua9hFT+C24/LhX2G1g=
k8s.io/helm v2.9.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI=
@ -1151,6 +1156,8 @@ k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68=
k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/kube-aggregator v0.18.1/go.mod h1:cXwR5+w/IZ/6tbDGFz3aEYrZctFN9R3X6u0gUcWwVzA=
k8s.io/kube-controller-manager v0.18.1/go.mod h1:HFp15+aGPbGns4K9jD9TxJVuc9eeiylCtjgCunRV3B4=
k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=

View File

@ -184,7 +184,6 @@ k8s.io/kops/upup/pkg/fi/secrets
k8s.io/kops/upup/pkg/fi/utils
k8s.io/kops/upup/pkg/kutil
k8s.io/kops/upup/tools/generators/fitask
k8s.io/kops/upup/tools/generators/pkg/codegen
k8s.io/kops/util/pkg/architectures
k8s.io/kops/util/pkg/env
k8s.io/kops/util/pkg/exec

View File

@ -8,7 +8,14 @@ go_library(
],
importpath = "k8s.io/kops/upup/tools/generators/fitask",
visibility = ["//visibility:private"],
deps = ["//upup/tools/generators/pkg/codegen:go_default_library"],
deps = [
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/gengo/args:go_default_library",
"//vendor/k8s.io/gengo/generator:go_default_library",
"//vendor/k8s.io/gengo/namer:go_default_library",
"//vendor/k8s.io/gengo/types:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
],
)
go_binary(

View File

@ -17,46 +17,27 @@ limitations under the License.
package main
import (
"fmt"
"io"
"path/filepath"
"strings"
"text/template"
"k8s.io/kops/upup/tools/generators/pkg/codegen"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/gengo/args"
"k8s.io/gengo/generator"
"k8s.io/gengo/namer"
"k8s.io/gengo/types"
"k8s.io/klog/v2"
)
type FitaskGenerator struct {
Package string
// These are the comment tags that carry parameters for fitask generation.
const tagName = "kops:fitask"
func extractTag(comments []string) []string {
return types.ExtractCommentTags("+", comments)[tagName]
}
var _ codegen.Generator = &FitaskGenerator{}
const fileHeaderDef = `/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
`
var preambleDef = `
package {{.Package}}
import (
"encoding/json"
"k8s.io/kops/upup/pkg/fi"
)
`
const perTypeDef = `
// {{.Name}}
@ -109,33 +90,99 @@ func (o *{{.Name}}) String() string {
}
`
func (g *FitaskGenerator) Init(parser *codegen.GoParser) error {
g.Package = parser.Package.Name
return nil
// NameSystems returns the name system used by the generators in this package.
func NameSystems() namer.NameSystems {
return namer.NameSystems{
"public": namer.NewPublicNamer(0),
"private": namer.NewPrivateNamer(0),
"raw": namer.NewRawNamer("", nil),
}
}
func (g *FitaskGenerator) WriteFileHeader(w io.Writer) error {
t := template.Must(template.New("FileHeader").Parse(fileHeaderDef))
return t.Execute(w, g)
// DefaultNameSystem returns the default name system for ordering the types to be
// processed by the generators in this package.
func DefaultNameSystem() string {
return "public"
}
func (g *FitaskGenerator) WritePreamble(w io.Writer) error {
t := template.Must(template.New("Preamble").Parse(preambleDef))
// Packages makes the sets package definition.
func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages {
boilerplate, err := arguments.LoadGoBoilerplate()
if err != nil {
klog.Fatalf("Failed loading boilerplate: %v", err)
}
return t.Execute(w, g)
inputs := sets.NewString(context.Inputs...)
packages := generator.Packages{}
header := append([]byte(fmt.Sprintf("// +build !%s\n\n", arguments.GeneratedBuildTag)), boilerplate...)
for i := range inputs {
klog.V(5).Infof("considering pkg %q", i)
pkg := context.Universe[i]
if pkg == nil {
// If the input had no Go files, for example.
continue
}
fitasks := map[*types.Type]bool{}
for _, t := range pkg.Types {
if t.Kind == types.Struct && len(extractTag(t.CommentLines)) > 0 {
fitasks[t] = true
}
}
packages = append(packages, &generator.DefaultPackage{
PackageName: filepath.Base(pkg.Path),
PackagePath: strings.TrimPrefix(pkg.Path, "k8s.io/kops/"),
HeaderText: header,
GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
for t := range fitasks {
generators = append(generators, NewGenFitask(t))
}
return generators
},
FilterFunc: func(c *generator.Context, t *types.Type) bool {
return fitasks[t]
},
})
}
return packages
}
type genFitask struct {
generator.DefaultGen
typeToMatch *types.Type
}
func NewGenFitask(t *types.Type) generator.Generator {
return &genFitask{
DefaultGen: generator.DefaultGen{
OptionalName: strings.ToLower(t.Name.Name) + "_fitask",
},
typeToMatch: t,
}
}
// Filter ignores all but one type because we're making a single file per type.
func (g *genFitask) Filter(c *generator.Context, t *types.Type) bool { return t == g.typeToMatch }
func (g *genFitask) Imports(c *generator.Context) (imports []string) {
return []string{
"encoding/json",
"k8s.io/kops/upup/pkg/fi",
}
}
type TypeData struct {
Name string
}
func (g *FitaskGenerator) WriteType(w io.Writer, typeName string) error {
t := template.Must(template.New("PerType").Parse(perTypeDef))
func (g *genFitask) GenerateType(_ *generator.Context, t *types.Type, w io.Writer) error {
tmpl := template.Must(template.New("PerType").Parse(perTypeDef))
d := &TypeData{}
d.Name = typeName
d.Name = t.Name.Name
return t.Execute(w, d)
return tmpl.Execute(w, d)
}

View File

@ -1,5 +1,5 @@
/*
Copyright 2019 The Kubernetes Authors.
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -17,10 +17,22 @@ limitations under the License.
package main
import (
"k8s.io/kops/upup/tools/generators/pkg/codegen"
"os"
"k8s.io/gengo/args"
"k8s.io/klog/v2"
)
func main() {
generator := &FitaskGenerator{}
codegen.RunGenerator("fitask", generator)
klog.InitFlags(nil)
arguments := args.Default()
if err := arguments.Execute(
NameSystems(),
DefaultNameSystem(),
Packages,
); err != nil {
klog.Errorf("Error: %v", err)
os.Exit(1)
}
klog.V(2).Info("Completed successfully.")
}

View File

@ -1,12 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"generator.go",
"main_helper.go",
"parse.go",
],
importpath = "k8s.io/kops/upup/tools/generators/pkg/codegen",
visibility = ["//visibility:public"],
)

View File

@ -1,26 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package codegen
import "io"
type Generator interface {
Init(parser *GoParser) error
WriteFileHeader(w io.Writer) error
WritePreamble(w io.Writer) error
WriteType(w io.Writer, typeName string) error
}

View File

@ -1,136 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package codegen
import (
"bytes"
"flag"
"fmt"
"go/format"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
)
var (
flagTypes = flag.String("type", "", "comma-separated list of type names; must be set")
output = flag.String("output", "", "output file name; default srcdir/<type>_fitask.go")
)
// Usage is a replacement usage function for the flags package.
func Usage() {
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\tfitask [flags] -type T [directory]\n")
fmt.Fprintf(os.Stderr, "\tfitask [flags] -type T files... # Must be a single package\n")
fmt.Fprintf(os.Stderr, "Flags:\n")
flag.PrintDefaults()
}
// isDirectory reports whether the named file is a directory.
func isDirectory(name string) bool {
info, err := os.Stat(name)
if err != nil {
log.Fatal(err)
}
return info.IsDir()
}
func RunGenerator(key string, generator Generator) {
flag.Usage = Usage
flag.Parse()
if len(*flagTypes) == 0 {
flag.Usage()
os.Exit(2)
}
typeNames := strings.Split(*flagTypes, ",")
// We accept either one directory or a list of files. Which do we have?
args := flag.Args()
if len(args) == 0 {
// Default: process whole package in current directory.
args = []string{"."}
}
// Parse the package once.
var dir string
var parser GoParser
if len(args) == 1 && isDirectory(args[0]) {
dir = args[0]
parser.parsePackageDir(args[0])
//} else {
// dir = filepath.Dir(args[0])
// g.parsePackageFiles(args)
}
err := generator.Init(&parser)
if err != nil {
log.Fatalf("error initializing generator: %v", err)
}
var b bytes.Buffer
err = generator.WriteFileHeader(&b)
if err != nil {
log.Fatalf("error generating header: %v", err)
}
fmt.Fprintf(&b, "// Code generated by \"%q %s\"; DO NOT EDIT\n\n", key, strings.Join(os.Args[1:], " "))
err = generator.WritePreamble(&b)
if err != nil {
log.Fatalf("error generating header: %v", err)
}
// Run generate for each type.
for _, typeName := range typeNames {
err := generator.WriteType(&b, typeName)
if err != nil {
log.Fatalf("error generating header: %v", err)
}
}
// gofmt the output.
src := gofmt(b.Bytes())
// Write to file.
outputName := *output
if outputName == "" {
baseName := fmt.Sprintf("%s_%s.go", typeNames[0], key)
outputName = filepath.Join(dir, strings.ToLower(baseName))
}
err = ioutil.WriteFile(outputName, src, 0644)
if err != nil {
log.Fatalf("writing output: %s", err)
}
}
// format returns the gofmt-ed contents of the Generator's buffer.
func gofmt(src []byte) []byte {
formatted, err := format.Source(src)
if err != nil {
// Should never happen, but can arise when developing this code.
// The user can compile the output to see the error.
log.Printf("warning: internal error: invalid Go generated: %s", err)
log.Printf("warning: compile the package to analyze the error")
return src
}
return formatted
}

View File

@ -1,119 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package codegen
import (
"go/ast"
"go/build"
"go/importer"
"go/parser"
"go/token"
"go/types"
"log"
"path/filepath"
"strings"
)
type GoParser struct {
Package *Package // Package we are scanning.
}
// File holds a single parsed file and associated data.
type File struct {
pkg *Package // Package to which this file belongs.
file *ast.File // Parsed AST.
}
type Package struct {
dir string
Name string
defs map[*ast.Ident]types.Object
files []*File
typesPkg *types.Package
}
// prefixDirectory prepends the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) []string {
if directory == "." {
return names
}
ret := make([]string, len(names))
for i, name := range names {
ret[i] = filepath.Join(directory, name)
}
return ret
}
// parsePackageDir parses the package residing in the directory.
func (g *GoParser) parsePackageDir(directory string) {
pkg, err := build.Default.ImportDir(directory, 0)
if err != nil {
log.Fatalf("cannot process directory %s: %s", directory, err)
}
var names []string
names = append(names, pkg.GoFiles...)
//names = append(names, pkg.CgoFiles...)
//names = append(names, pkg.SFiles...)
names = prefixDirectory(directory, names)
g.parsePackage(directory, names, nil)
}
// parsePackage analyzes the single package constructed from the named files.
// If text is non-nil, it is a string to be used instead of the content of the file,
// to be used for testing. parsePackage exits if there is an error.
func (g *GoParser) parsePackage(directory string, names []string, text interface{}) {
var files []*File
var astFiles []*ast.File
g.Package = new(Package)
fs := token.NewFileSet()
for _, name := range names {
if !strings.HasSuffix(name, ".go") {
continue
}
parsedFile, err := parser.ParseFile(fs, name, text, 0)
if err != nil {
log.Fatalf("parsing package: %s: %s", name, err)
}
astFiles = append(astFiles, parsedFile)
files = append(files, &File{
file: parsedFile,
pkg: g.Package,
})
}
if len(astFiles) == 0 {
log.Fatalf("%s: no buildable Go files", directory)
}
g.Package.Name = astFiles[0].Name.Name
g.Package.files = files
g.Package.dir = directory
// Type check the package.
g.Package.check(fs, astFiles)
}
// check type-checks the package. The package must be OK to proceed.
func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) {
pkg.defs = make(map[*ast.Ident]types.Object)
config := types.Config{Importer: importer.ForCompiler(token.NewFileSet(), "source", nil), FakeImportC: true}
info := &types.Info{
Defs: pkg.defs,
}
typesPkg, err := config.Check(pkg.dir, fs, astFiles, info)
if err != nil {
log.Fatalf("checking package: %s", err)
}
pkg.typesPkg = typesPkg
}