/* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package markers import ( "bytes" "fmt" "reflect" "strconv" "strings" sc "text/scanner" "unicode" "sigs.k8s.io/controller-tools/pkg/loader" ) // expect checks that the next token of the scanner is the given token, adding an error // to the scanner if not. It returns whether the token was as expected. func expect(scanner *sc.Scanner, expected rune, errDesc string) bool { tok := scanner.Scan() if tok != expected { scanner.Error(scanner, fmt.Sprintf("expected %s, got %q", errDesc, scanner.TokenText())) return false } return true } // peekNoSpace is equivalent to scanner.Peek, except that it will consume intervening whitespace. func peekNoSpace(scanner *sc.Scanner) rune { hint := scanner.Peek() for ; hint <= rune(' ') && ((1<") case IntType: out.WriteString("int") case StringType: out.WriteString("string") case BoolType: out.WriteString("bool") case AnyType: out.WriteString("") case SliceType: out.WriteString("[]") // arguments can't be non-pointer optional, so just call into typeString again. a.ItemType.typeString(out) case MapType: out.WriteString("map[string]") a.ItemType.typeString(out) case RawType: out.WriteString("") } } // TypeString returns a string roughly equivalent // (but not identical) to the underlying Go type that // this argument would parse to. It's mainly useful // for user-friendly formatting of this argument (e.g. // help strings). func (a Argument) TypeString() string { out := &strings.Builder{} a.typeString(out) return out.String() } func (a Argument) String() string { if a.Optional { return fmt.Sprintf("", a.TypeString()) } return fmt.Sprintf("", a.TypeString()) } // castAndSet casts val to out's type if needed, // then sets out to val. func castAndSet(out, val reflect.Value) { outType := out.Type() if outType != val.Type() { val = val.Convert(outType) } out.Set(val) } // makeSliceType makes a reflect.Type for a slice of the given type. // Useful for constructing the out value for when AnyType's guess returns a slice. func makeSliceType(itemType Argument) (reflect.Type, error) { var itemReflectedType reflect.Type switch itemType.Type { case IntType: itemReflectedType = reflect.TypeOf(int(0)) case StringType: itemReflectedType = reflect.TypeOf("") case BoolType: itemReflectedType = reflect.TypeOf(false) case SliceType: subItemType, err := makeSliceType(*itemType.ItemType) if err != nil { return nil, err } itemReflectedType = subItemType case MapType: subItemType, err := makeMapType(*itemType.ItemType) if err != nil { return nil, err } itemReflectedType = subItemType // TODO(directxman12): support non-uniform slices? (probably not) default: return nil, fmt.Errorf("invalid type when constructing guessed slice out: %v", itemType.Type) } if itemType.Pointer { itemReflectedType = reflect.PtrTo(itemReflectedType) } return reflect.SliceOf(itemReflectedType), nil } // makeMapType makes a reflect.Type for a map of the given item type. // Useful for constructing the out value for when AnyType's guess returns a map. func makeMapType(itemType Argument) (reflect.Type, error) { var itemReflectedType reflect.Type switch itemType.Type { case IntType: itemReflectedType = reflect.TypeOf(int(0)) case StringType: itemReflectedType = reflect.TypeOf("") case BoolType: itemReflectedType = reflect.TypeOf(false) case SliceType: subItemType, err := makeSliceType(*itemType.ItemType) if err != nil { return nil, err } itemReflectedType = subItemType // TODO(directxman12): support non-uniform slices? (probably not) case MapType: subItemType, err := makeMapType(*itemType.ItemType) if err != nil { return nil, err } itemReflectedType = subItemType case AnyType: // NB(directxman12): maps explicitly allow non-uniform item types, unlike slices at the moment itemReflectedType = interfaceType default: return nil, fmt.Errorf("invalid type when constructing guessed slice out: %v", itemType.Type) } if itemType.Pointer { itemReflectedType = reflect.PtrTo(itemReflectedType) } return reflect.MapOf(reflect.TypeOf(""), itemReflectedType), nil } // guessType takes an educated guess about the type of the next field. If allowSlice // is false, it will not guess slices. It's less efficient than parsing with actual // type information, since we need to allocate to peek ahead full tokens, and the scanner // only allows peeking ahead one character. // Maps are *always* non-uniform (i.e. type the AnyType item type), since they're frequently // used to represent things like defaults for an object in JSON. func guessType(scanner *sc.Scanner, raw string, allowSlice bool) *Argument { if allowSlice { maybeItem := guessType(scanner, raw, false) subRaw := raw[scanner.Pos().Offset:] subScanner := parserScanner(subRaw, scanner.Error) var tok rune for tok = subScanner.Scan(); tok != ',' && tok != sc.EOF && tok != ';'; tok = subScanner.Scan() { // wait till we get something interesting } // semicolon means it's a legacy slice if tok == ';' { return &Argument{ Type: SliceType, ItemType: maybeItem, } } return maybeItem } // everything else needs a duplicate scanner to scan properly // (so we don't consume our scanner tokens until we actually // go to use this -- Go doesn't like scanners that can be rewound). subRaw := raw[scanner.Pos().Offset:] subScanner := parserScanner(subRaw, scanner.Error) // skip whitespace hint := peekNoSpace(subScanner) // first, try the easy case -- quoted strings strings switch hint { case '"', '\'', '`': return &Argument{Type: StringType} } // next, check for slices or maps if hint == '{' { subScanner.Scan() // TODO(directxman12): this can't guess at empty objects, but that's generally ok. // We'll cross that bridge when we get there. // look ahead till we can figure out if this is a map or a slice firstElemType := guessType(subScanner, subRaw, false) if firstElemType.Type == StringType { // might be a map or slice, parse the string and check for colon // (blech, basically arbitrary look-ahead due to raw strings). var keyVal string // just ignore this (&Argument{Type: StringType}).parseString(subScanner, raw, reflect.Indirect(reflect.ValueOf(&keyVal))) if subScanner.Scan() == ':' { // it's got a string followed by a colon -- it's a map return &Argument{ Type: MapType, ItemType: &Argument{Type: AnyType}, } } } // definitely a slice -- maps have to have string keys and have a value followed by a colon return &Argument{ Type: SliceType, ItemType: firstElemType, } } // then, bools... probablyString := false if hint == 't' || hint == 'f' { // maybe a bool if nextTok := subScanner.Scan(); nextTok == sc.Ident { switch subScanner.TokenText() { case "true", "false": // definitely a bool return &Argument{Type: BoolType} } // probably a string probablyString = true } else { // we shouldn't ever get here scanner.Error(scanner, fmt.Sprintf("got a token (%q) that looked like an ident, but was not", scanner.TokenText())) return &Argument{Type: InvalidType} } } if !probablyString { if nextTok := subScanner.Scan(); nextTok == sc.Int { return &Argument{Type: IntType} } } // otherwise assume bare strings return &Argument{Type: StringType} } // parseString parses either of the two accepted string forms (quoted, or bare tokens). func (a *Argument) parseString(scanner *sc.Scanner, raw string, out reflect.Value) { // strings are a bit weird -- the "easy" case is quoted strings (tokenized as strings), // the "hard" case (present for backwards compat) is a bare sequence of tokens that aren't // a comma. tok := scanner.Scan() if tok == sc.String || tok == sc.RawString { // the easy case val, err := strconv.Unquote(scanner.TokenText()) if err != nil { scanner.Error(scanner, fmt.Sprintf("unable to parse string: %v", err)) return } castAndSet(out, reflect.ValueOf(val)) return } // the "hard" case -- bare tokens not including ',' (the argument // separator), ';' (the slice separator), ':' (the map separator), or '}' // (delimitted slice ender) startPos := scanner.Position.Offset for hint := peekNoSpace(scanner); hint != ',' && hint != ';' && hint != ':' && hint != '}' && hint != sc.EOF; hint = peekNoSpace(scanner) { // skip this token scanner.Scan() } endPos := scanner.Position.Offset + len(scanner.TokenText()) castAndSet(out, reflect.ValueOf(raw[startPos:endPos])) } // parseSlice parses either of the two slice forms (curly-brace-delimitted and semicolon-separated). func (a *Argument) parseSlice(scanner *sc.Scanner, raw string, out reflect.Value) { // slices have two supported formats, like string: // - `{val, val, val}` (preferred) // - `val;val;val` (legacy) resSlice := reflect.Zero(out.Type()) elem := reflect.Indirect(reflect.New(out.Type().Elem())) // preferred case if peekNoSpace(scanner) == '{' { // NB(directxman12): supporting delimitted slices in bare slices // would require an extra look-ahead here :-/ scanner.Scan() // skip '{' for hint := peekNoSpace(scanner); hint != '}' && hint != sc.EOF; hint = peekNoSpace(scanner) { a.ItemType.parse(scanner, raw, elem, true /* parsing a slice */) resSlice = reflect.Append(resSlice, elem) tok := peekNoSpace(scanner) if tok == '}' { break } if !expect(scanner, ',', "comma") { return } } if !expect(scanner, '}', "close curly brace") { return } castAndSet(out, resSlice) return } // legacy case for hint := peekNoSpace(scanner); hint != ',' && hint != '}' && hint != sc.EOF; hint = peekNoSpace(scanner) { a.ItemType.parse(scanner, raw, elem, true /* parsing a slice */) resSlice = reflect.Append(resSlice, elem) tok := peekNoSpace(scanner) if tok == ',' || tok == '}' || tok == sc.EOF { break } scanner.Scan() if tok != ';' { scanner.Error(scanner, fmt.Sprintf("expected comma, got %q", scanner.TokenText())) return } } castAndSet(out, resSlice) } // parseMap parses a map of the form {string: val, string: val, string: val} func (a *Argument) parseMap(scanner *sc.Scanner, raw string, out reflect.Value) { resMap := reflect.MakeMap(out.Type()) elem := reflect.Indirect(reflect.New(out.Type().Elem())) key := reflect.Indirect(reflect.New(out.Type().Key())) if !expect(scanner, '{', "open curly brace") { return } for hint := peekNoSpace(scanner); hint != '}' && hint != sc.EOF; hint = peekNoSpace(scanner) { a.parseString(scanner, raw, key) if !expect(scanner, ':', "colon") { return } a.ItemType.parse(scanner, raw, elem, false /* not in a slice */) resMap.SetMapIndex(key, elem) if peekNoSpace(scanner) == '}' { break } if !expect(scanner, ',', "comma") { return } } if !expect(scanner, '}', "close curly brace") { return } castAndSet(out, resMap) } // parse functions like Parse, except that it allows passing down whether or not we're // already in a slice, to avoid duplicate legacy slice detection for AnyType func (a *Argument) parse(scanner *sc.Scanner, raw string, out reflect.Value, inSlice bool) { // nolint:gocyclo if a.Type == InvalidType { scanner.Error(scanner, fmt.Sprintf("cannot parse invalid type")) return } if a.Pointer { out.Set(reflect.New(out.Type().Elem())) out = reflect.Indirect(out) } switch a.Type { case RawType: // raw consumes everything else castAndSet(out, reflect.ValueOf(raw[scanner.Pos().Offset:])) // consume everything else for tok := scanner.Scan(); tok != sc.EOF; tok = scanner.Scan() { } case IntType: if !expect(scanner, sc.Int, "integer") { return } // TODO(directxman12): respect the size when parsing val, err := strconv.Atoi(scanner.TokenText()) if err != nil { scanner.Error(scanner, fmt.Sprintf("unable to parse integer: %v", err)) return } castAndSet(out, reflect.ValueOf(val)) case StringType: // strings are a bit weird -- the "easy" case is quoted strings (tokenized as strings), // the "hard" case (present for backwards compat) is a bare sequence of tokens that aren't // a comma. a.parseString(scanner, raw, out) case BoolType: if !expect(scanner, sc.Ident, "true or false") { return } switch scanner.TokenText() { case "true": castAndSet(out, reflect.ValueOf(true)) case "false": castAndSet(out, reflect.ValueOf(false)) default: scanner.Error(scanner, fmt.Sprintf("expected true or false, got %q", scanner.TokenText())) return } case AnyType: guessedType := guessType(scanner, raw, !inSlice) newOut := out // we need to be able to construct the right element types, below // in parse, so construct a concretely-typed value to use as "out" switch guessedType.Type { case SliceType: newType, err := makeSliceType(*guessedType.ItemType) if err != nil { scanner.Error(scanner, err.Error()) return } newOut = reflect.Indirect(reflect.New(newType)) case MapType: newType, err := makeMapType(*guessedType.ItemType) if err != nil { scanner.Error(scanner, err.Error()) return } newOut = reflect.Indirect(reflect.New(newType)) } if !newOut.CanSet() { panic("at the disco") // TODO(directxman12): this is left over from debugging -- it might need to be an error } guessedType.Parse(scanner, raw, newOut) castAndSet(out, newOut) case SliceType: // slices have two supported formats, like string: // - `{val, val, val}` (preferred) // - `val;val;val` (legacy) a.parseSlice(scanner, raw, out) case MapType: // maps are {string: val, string: val, string: val} a.parseMap(scanner, raw, out) } } // Parse attempts to consume the argument from the given scanner (based on the given // raw input as well for collecting ranges of content), and places the output value // in the given reflect.Value. Errors are reported via the given scanner. func (a *Argument) Parse(scanner *sc.Scanner, raw string, out reflect.Value) { a.parse(scanner, raw, out, false) } // ArgumentFromType constructs an Argument by examining the given // raw reflect.Type. It can construct arguments from the Go types // corresponding to any of the types listed in ArgumentType. func ArgumentFromType(rawType reflect.Type) (Argument, error) { if rawType == rawArgsType { return Argument{ Type: RawType, }, nil } if rawType == interfaceType { return Argument{ Type: AnyType, }, nil } arg := Argument{} if rawType.Kind() == reflect.Ptr { rawType = rawType.Elem() arg.Pointer = true arg.Optional = true } switch rawType.Kind() { case reflect.String: arg.Type = StringType case reflect.Int, reflect.Int32: // NB(directxman12): all ints in kubernetes are int32, so explicitly support that arg.Type = IntType case reflect.Bool: arg.Type = BoolType case reflect.Slice: arg.Type = SliceType itemType, err := ArgumentFromType(rawType.Elem()) if err != nil { return Argument{}, fmt.Errorf("bad slice item type: %w", err) } arg.ItemType = &itemType case reflect.Map: arg.Type = MapType if rawType.Key().Kind() != reflect.String { return Argument{}, fmt.Errorf("bad map key type: map keys must be strings") } itemType, err := ArgumentFromType(rawType.Elem()) if err != nil { return Argument{}, fmt.Errorf("bad slice item type: %w", err) } arg.ItemType = &itemType default: return Argument{}, fmt.Errorf("type has unsupported kind %s", rawType.Kind()) } return arg, nil } // TargetType describes which kind of node a given marker is associated with. type TargetType int const ( // DescribesPackage indicates that a marker is associated with a package. DescribesPackage TargetType = iota // DescribesType indicates that a marker is associated with a type declaration. DescribesType // DescribesField indicates that a marker is associated with a struct field. DescribesField ) func (t TargetType) String() string { switch t { case DescribesPackage: return "package" case DescribesType: return "type" case DescribesField: return "field" default: return "(unknown)" } } // Definition is a parsed definition of a marker. type Definition struct { // Output is the deserialized Go type of the marker. Output reflect.Type // Name is the marker's name. Name string // Target indicates which kind of node this marker can be associated with. Target TargetType // Fields lists out the types of each field that this marker has, by // argument name as used in the marker (if the output type isn't a struct, // it'll have a single, blank field name). This only lists exported fields, // (as per reflection rules). Fields map[string]Argument // FieldNames maps argument names (as used in the marker) to struct field name // in the output type. FieldNames map[string]string // Strict indicates that this definition should error out when parsing if // not all non-optional fields were seen. Strict bool } // AnonymousField indicates that the definition has one field, // (actually the original object), and thus the field // doesn't get named as part of the name. func (d *Definition) AnonymousField() bool { if len(d.Fields) != 1 { return false } _, hasAnonField := d.Fields[""] return hasAnonField } // Empty indicates that this definition has no fields. func (d *Definition) Empty() bool { return len(d.Fields) == 0 } // argumentInfo returns information about an argument field as the marker parser's field loader // would see it. This can be useful if you have to interact with marker definition structs // externally (e.g. at compile time). func argumentInfo(fieldName string, tag reflect.StructTag) (argName string, optionalOpt bool) { argName = lowerCamelCase(fieldName) markerTag, tagSpecified := tag.Lookup("marker") markerTagParts := strings.Split(markerTag, ",") if tagSpecified && markerTagParts[0] != "" { // allow overriding to support legacy cases where we don't follow camelCase conventions argName = markerTagParts[0] } optionalOpt = false for _, tagOption := range markerTagParts[1:] { switch tagOption { case "optional": optionalOpt = true } } return argName, optionalOpt } // loadFields uses reflection to populate argument information from the Output type. func (d *Definition) loadFields() error { if d.Fields == nil { d.Fields = make(map[string]Argument) d.FieldNames = make(map[string]string) } if d.Output.Kind() != reflect.Struct { // anonymous field type argType, err := ArgumentFromType(d.Output) if err != nil { return err } d.Fields[""] = argType d.FieldNames[""] = "" return nil } for i := 0; i < d.Output.NumField(); i++ { field := d.Output.Field(i) if field.PkgPath != "" { // as per the reflect package docs, pkgpath is empty for exported fields, // so non-empty package path means a private field, which we should skip continue } argName, optionalOpt := argumentInfo(field.Name, field.Tag) argType, err := ArgumentFromType(field.Type) if err != nil { return fmt.Errorf("unable to extract type information for field %q: %w", field.Name, err) } if argType.Type == RawType { return fmt.Errorf("RawArguments must be the direct type of a marker, and not a field") } argType.Optional = optionalOpt || argType.Optional d.Fields[argName] = argType d.FieldNames[argName] = field.Name } return nil } // parserScanner makes a new scanner appropriate for use in parsing definitions and arguments. func parserScanner(raw string, err func(*sc.Scanner, string)) *sc.Scanner { scanner := &sc.Scanner{} scanner.Init(bytes.NewBufferString(raw)) scanner.Mode = sc.ScanIdents | sc.ScanInts | sc.ScanStrings | sc.ScanRawStrings | sc.SkipComments scanner.Error = err return scanner } // Parse uses the type information in this Definition to parse the given // raw marker in the form `+a:b:c=arg,d=arg` into an output object of the // type specified in the definition. func (d *Definition) Parse(rawMarker string) (interface{}, error) { name, anonName, fields := splitMarker(rawMarker) out := reflect.Indirect(reflect.New(d.Output)) // if we're a not a struct or have no arguments, treat the full `a:b:c` as the name, // otherwise, treat `c` as a field name, and `a:b` as the marker name. if !d.AnonymousField() && !d.Empty() && len(anonName) >= len(name)+1 { fields = anonName[len(name)+1:] + "=" + fields } var errs []error scanner := parserScanner(fields, func(scanner *sc.Scanner, msg string) { errs = append(errs, &ScannerError{Msg: msg, Pos: scanner.Position}) }) // TODO(directxman12): strict parsing where we error out if certain fields aren't optional seen := make(map[string]struct{}, len(d.Fields)) if d.AnonymousField() && scanner.Peek() != sc.EOF { // might still be a struct that something fiddled with, so double check structFieldName := d.FieldNames[""] outTarget := out if structFieldName != "" { // it's a struct field mapped to an anonymous marker outTarget = out.FieldByName(structFieldName) if !outTarget.CanSet() { scanner.Error(scanner, fmt.Sprintf("cannot set field %q (might not exist)", structFieldName)) return out.Interface(), loader.MaybeErrList(errs) } } // no need for trying to parse field names if we're not a struct field := d.Fields[""] field.Parse(scanner, fields, outTarget) seen[""] = struct{}{} // mark as seen for strict definitions } else if !d.Empty() && scanner.Peek() != sc.EOF { // if we expect *and* actually have arguments passed for { // parse the argument name if !expect(scanner, sc.Ident, "argument name") { break } argName := scanner.TokenText() if !expect(scanner, '=', "equals") { break } // make sure we know the field fieldName, known := d.FieldNames[argName] if !known { scanner.Error(scanner, fmt.Sprintf("unknown argument %q", argName)) break } fieldType, known := d.Fields[argName] if !known { scanner.Error(scanner, fmt.Sprintf("unknown argument %q", argName)) break } seen[argName] = struct{}{} // mark as seen for strict definitions // parse the field value fieldVal := out.FieldByName(fieldName) if !fieldVal.CanSet() { scanner.Error(scanner, fmt.Sprintf("cannot set field %q (might not exist)", fieldName)) break } fieldType.Parse(scanner, fields, fieldVal) if len(errs) > 0 { break } if scanner.Peek() == sc.EOF { break } if !expect(scanner, ',', "comma") { break } } } if tok := scanner.Scan(); tok != sc.EOF { scanner.Error(scanner, fmt.Sprintf("extra arguments provided: %q", fields[scanner.Position.Offset:])) } if d.Strict { for argName, arg := range d.Fields { if _, wasSeen := seen[argName]; !wasSeen && !arg.Optional { scanner.Error(scanner, fmt.Sprintf("missing argument %q", argName)) } } } return out.Interface(), loader.MaybeErrList(errs) } // MakeDefinition constructs a definition from a name, type, and the output type. // All such definitions are strict by default. If a struct is passed as the output // type, its public fields will automatically be populated into Fields (and similar // fields in Definition). Other values will have a single, empty-string-named Fields // entry. func MakeDefinition(name string, target TargetType, output interface{}) (*Definition, error) { def := &Definition{ Name: name, Target: target, Output: reflect.TypeOf(output), Strict: true, } if err := def.loadFields(); err != nil { return nil, err } return def, nil } // MakeAnyTypeDefinition constructs a definition for an output struct with a // field named `Value` of type `interface{}`. The argument to the marker will // be parsed as AnyType and assigned to the field named `Value`. func MakeAnyTypeDefinition(name string, target TargetType, output interface{}) (*Definition, error) { defn, err := MakeDefinition(name, target, output) if err != nil { return nil, err } defn.FieldNames = map[string]string{"": "Value"} defn.Fields = map[string]Argument{"": defn.Fields["value"]} return defn, nil } // splitMarker takes a marker in the form of `+a:b:c=arg,d=arg` and splits it // into the name (`a:b`), the name if it's not a struct (`a:b:c`), and the parts // that are definitely fields (`arg,d=arg`). func splitMarker(raw string) (name string, anonymousName string, restFields string) { raw = raw[1:] // get rid of the leading '+' nameFieldParts := strings.SplitN(raw, "=", 2) if len(nameFieldParts) == 1 { return nameFieldParts[0], nameFieldParts[0], "" } anonymousName = nameFieldParts[0] name = anonymousName restFields = nameFieldParts[1] nameParts := strings.Split(name, ":") if len(nameParts) > 1 { name = strings.Join(nameParts[:len(nameParts)-1], ":") } return name, anonymousName, restFields } type ScannerError struct { Msg string Pos sc.Position } func (e *ScannerError) Error() string { return fmt.Sprintf("%s (at %s)", e.Msg, e.Pos) }