decoder.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660
  1. package hcl
  2. import (
  3. "errors"
  4. "fmt"
  5. "reflect"
  6. "sort"
  7. "strconv"
  8. "strings"
  9. "github.com/hashicorp/hcl/hcl/ast"
  10. "github.com/hashicorp/hcl/hcl/parser"
  11. "github.com/hashicorp/hcl/hcl/token"
  12. )
  13. // This is the tag to use with structures to have settings for HCL
  14. const tagName = "hcl"
  15. var (
  16. // nodeType holds a reference to the type of ast.Node
  17. nodeType reflect.Type = findNodeType()
  18. )
  19. // Unmarshal accepts a byte slice as input and writes the
  20. // data to the value pointed to by v.
  21. func Unmarshal(bs []byte, v interface{}) error {
  22. root, err := parse(bs)
  23. if err != nil {
  24. return err
  25. }
  26. return DecodeObject(v, root)
  27. }
  28. // Decode reads the given input and decodes it into the structure
  29. // given by `out`.
  30. func Decode(out interface{}, in string) error {
  31. obj, err := Parse(in)
  32. if err != nil {
  33. return err
  34. }
  35. return DecodeObject(out, obj)
  36. }
  37. // DecodeObject is a lower-level version of Decode. It decodes a
  38. // raw Object into the given output.
  39. func DecodeObject(out interface{}, n ast.Node) error {
  40. val := reflect.ValueOf(out)
  41. if val.Kind() != reflect.Ptr {
  42. return errors.New("result must be a pointer")
  43. }
  44. // If we have the file, we really decode the root node
  45. if f, ok := n.(*ast.File); ok {
  46. n = f.Node
  47. }
  48. var d decoder
  49. return d.decode("root", n, val.Elem())
  50. }
  51. type decoder struct {
  52. stack []reflect.Kind
  53. }
  54. func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error {
  55. k := result
  56. // If we have an interface with a valid value, we use that
  57. // for the check.
  58. if result.Kind() == reflect.Interface {
  59. elem := result.Elem()
  60. if elem.IsValid() {
  61. k = elem
  62. }
  63. }
  64. // Push current onto stack unless it is an interface.
  65. if k.Kind() != reflect.Interface {
  66. d.stack = append(d.stack, k.Kind())
  67. // Schedule a pop
  68. defer func() {
  69. d.stack = d.stack[:len(d.stack)-1]
  70. }()
  71. }
  72. switch k.Kind() {
  73. case reflect.Bool:
  74. return d.decodeBool(name, node, result)
  75. case reflect.Float64:
  76. return d.decodeFloat(name, node, result)
  77. case reflect.Int:
  78. return d.decodeInt(name, node, result)
  79. case reflect.Interface:
  80. // When we see an interface, we make our own thing
  81. return d.decodeInterface(name, node, result)
  82. case reflect.Map:
  83. return d.decodeMap(name, node, result)
  84. case reflect.Ptr:
  85. return d.decodePtr(name, node, result)
  86. case reflect.Slice:
  87. return d.decodeSlice(name, node, result)
  88. case reflect.String:
  89. return d.decodeString(name, node, result)
  90. case reflect.Struct:
  91. return d.decodeStruct(name, node, result)
  92. default:
  93. return &parser.PosError{
  94. Pos: node.Pos(),
  95. Err: fmt.Errorf("%s: unknown kind to decode into: %s", name, k.Kind()),
  96. }
  97. }
  98. }
  99. func (d *decoder) decodeBool(name string, node ast.Node, result reflect.Value) error {
  100. switch n := node.(type) {
  101. case *ast.LiteralType:
  102. if n.Token.Type == token.BOOL {
  103. v, err := strconv.ParseBool(n.Token.Text)
  104. if err != nil {
  105. return err
  106. }
  107. result.Set(reflect.ValueOf(v))
  108. return nil
  109. }
  110. }
  111. return &parser.PosError{
  112. Pos: node.Pos(),
  113. Err: fmt.Errorf("%s: unknown type %T", name, node),
  114. }
  115. }
  116. func (d *decoder) decodeFloat(name string, node ast.Node, result reflect.Value) error {
  117. switch n := node.(type) {
  118. case *ast.LiteralType:
  119. if n.Token.Type == token.FLOAT {
  120. v, err := strconv.ParseFloat(n.Token.Text, 64)
  121. if err != nil {
  122. return err
  123. }
  124. result.Set(reflect.ValueOf(v))
  125. return nil
  126. }
  127. }
  128. return &parser.PosError{
  129. Pos: node.Pos(),
  130. Err: fmt.Errorf("%s: unknown type %T", name, node),
  131. }
  132. }
  133. func (d *decoder) decodeInt(name string, node ast.Node, result reflect.Value) error {
  134. switch n := node.(type) {
  135. case *ast.LiteralType:
  136. switch n.Token.Type {
  137. case token.NUMBER:
  138. v, err := strconv.ParseInt(n.Token.Text, 0, 0)
  139. if err != nil {
  140. return err
  141. }
  142. result.Set(reflect.ValueOf(int(v)))
  143. return nil
  144. case token.STRING:
  145. v, err := strconv.ParseInt(n.Token.Value().(string), 0, 0)
  146. if err != nil {
  147. return err
  148. }
  149. result.Set(reflect.ValueOf(int(v)))
  150. return nil
  151. }
  152. }
  153. return &parser.PosError{
  154. Pos: node.Pos(),
  155. Err: fmt.Errorf("%s: unknown type %T", name, node),
  156. }
  157. }
  158. func (d *decoder) decodeInterface(name string, node ast.Node, result reflect.Value) error {
  159. // When we see an ast.Node, we retain the value to enable deferred decoding.
  160. // Very useful in situations where we want to preserve ast.Node information
  161. // like Pos
  162. if result.Type() == nodeType && result.CanSet() {
  163. result.Set(reflect.ValueOf(node))
  164. return nil
  165. }
  166. var set reflect.Value
  167. redecode := true
  168. // For testing types, ObjectType should just be treated as a list. We
  169. // set this to a temporary var because we want to pass in the real node.
  170. testNode := node
  171. if ot, ok := node.(*ast.ObjectType); ok {
  172. testNode = ot.List
  173. }
  174. switch n := testNode.(type) {
  175. case *ast.ObjectList:
  176. // If we're at the root or we're directly within a slice, then we
  177. // decode objects into map[string]interface{}, otherwise we decode
  178. // them into lists.
  179. if len(d.stack) == 0 || d.stack[len(d.stack)-1] == reflect.Slice {
  180. var temp map[string]interface{}
  181. tempVal := reflect.ValueOf(temp)
  182. result := reflect.MakeMap(
  183. reflect.MapOf(
  184. reflect.TypeOf(""),
  185. tempVal.Type().Elem()))
  186. set = result
  187. } else {
  188. var temp []map[string]interface{}
  189. tempVal := reflect.ValueOf(temp)
  190. result := reflect.MakeSlice(
  191. reflect.SliceOf(tempVal.Type().Elem()), 0, len(n.Items))
  192. set = result
  193. }
  194. case *ast.ObjectType:
  195. // If we're at the root or we're directly within a slice, then we
  196. // decode objects into map[string]interface{}, otherwise we decode
  197. // them into lists.
  198. if len(d.stack) == 0 || d.stack[len(d.stack)-1] == reflect.Slice {
  199. var temp map[string]interface{}
  200. tempVal := reflect.ValueOf(temp)
  201. result := reflect.MakeMap(
  202. reflect.MapOf(
  203. reflect.TypeOf(""),
  204. tempVal.Type().Elem()))
  205. set = result
  206. } else {
  207. var temp []map[string]interface{}
  208. tempVal := reflect.ValueOf(temp)
  209. result := reflect.MakeSlice(
  210. reflect.SliceOf(tempVal.Type().Elem()), 0, 1)
  211. set = result
  212. }
  213. case *ast.ListType:
  214. var temp []interface{}
  215. tempVal := reflect.ValueOf(temp)
  216. result := reflect.MakeSlice(
  217. reflect.SliceOf(tempVal.Type().Elem()), 0, 0)
  218. set = result
  219. case *ast.LiteralType:
  220. switch n.Token.Type {
  221. case token.BOOL:
  222. var result bool
  223. set = reflect.Indirect(reflect.New(reflect.TypeOf(result)))
  224. case token.FLOAT:
  225. var result float64
  226. set = reflect.Indirect(reflect.New(reflect.TypeOf(result)))
  227. case token.NUMBER:
  228. var result int
  229. set = reflect.Indirect(reflect.New(reflect.TypeOf(result)))
  230. case token.STRING, token.HEREDOC:
  231. set = reflect.Indirect(reflect.New(reflect.TypeOf("")))
  232. default:
  233. return &parser.PosError{
  234. Pos: node.Pos(),
  235. Err: fmt.Errorf("%s: cannot decode into interface: %T", name, node),
  236. }
  237. }
  238. default:
  239. return fmt.Errorf(
  240. "%s: cannot decode into interface: %T",
  241. name, node)
  242. }
  243. // Set the result to what its supposed to be, then reset
  244. // result so we don't reflect into this method anymore.
  245. result.Set(set)
  246. if redecode {
  247. // Revisit the node so that we can use the newly instantiated
  248. // thing and populate it.
  249. if err := d.decode(name, node, result); err != nil {
  250. return err
  251. }
  252. }
  253. return nil
  254. }
  255. func (d *decoder) decodeMap(name string, node ast.Node, result reflect.Value) error {
  256. if item, ok := node.(*ast.ObjectItem); ok {
  257. node = &ast.ObjectList{Items: []*ast.ObjectItem{item}}
  258. }
  259. if ot, ok := node.(*ast.ObjectType); ok {
  260. node = ot.List
  261. }
  262. n, ok := node.(*ast.ObjectList)
  263. if !ok {
  264. return &parser.PosError{
  265. Pos: node.Pos(),
  266. Err: fmt.Errorf("%s: not an object type for map (%T)", name, node),
  267. }
  268. }
  269. // If we have an interface, then we can address the interface,
  270. // but not the slice itself, so get the element but set the interface
  271. set := result
  272. if result.Kind() == reflect.Interface {
  273. result = result.Elem()
  274. }
  275. resultType := result.Type()
  276. resultElemType := resultType.Elem()
  277. resultKeyType := resultType.Key()
  278. if resultKeyType.Kind() != reflect.String {
  279. return &parser.PosError{
  280. Pos: node.Pos(),
  281. Err: fmt.Errorf("%s: map must have string keys", name),
  282. }
  283. }
  284. // Make a map if it is nil
  285. resultMap := result
  286. if result.IsNil() {
  287. resultMap = reflect.MakeMap(
  288. reflect.MapOf(resultKeyType, resultElemType))
  289. }
  290. // Go through each element and decode it.
  291. done := make(map[string]struct{})
  292. for _, item := range n.Items {
  293. if item.Val == nil {
  294. continue
  295. }
  296. // github.com/hashicorp/terraform/issue/5740
  297. if len(item.Keys) == 0 {
  298. return &parser.PosError{
  299. Pos: node.Pos(),
  300. Err: fmt.Errorf("%s: map must have string keys", name),
  301. }
  302. }
  303. // Get the key we're dealing with, which is the first item
  304. keyStr := item.Keys[0].Token.Value().(string)
  305. // If we've already processed this key, then ignore it
  306. if _, ok := done[keyStr]; ok {
  307. continue
  308. }
  309. // Determine the value. If we have more than one key, then we
  310. // get the objectlist of only these keys.
  311. itemVal := item.Val
  312. if len(item.Keys) > 1 {
  313. itemVal = n.Filter(keyStr)
  314. done[keyStr] = struct{}{}
  315. }
  316. // Make the field name
  317. fieldName := fmt.Sprintf("%s.%s", name, keyStr)
  318. // Get the key/value as reflection values
  319. key := reflect.ValueOf(keyStr)
  320. val := reflect.Indirect(reflect.New(resultElemType))
  321. // If we have a pre-existing value in the map, use that
  322. oldVal := resultMap.MapIndex(key)
  323. if oldVal.IsValid() {
  324. val.Set(oldVal)
  325. }
  326. // Decode!
  327. if err := d.decode(fieldName, itemVal, val); err != nil {
  328. return err
  329. }
  330. // Set the value on the map
  331. resultMap.SetMapIndex(key, val)
  332. }
  333. // Set the final map if we can
  334. set.Set(resultMap)
  335. return nil
  336. }
  337. func (d *decoder) decodePtr(name string, node ast.Node, result reflect.Value) error {
  338. // Create an element of the concrete (non pointer) type and decode
  339. // into that. Then set the value of the pointer to this type.
  340. resultType := result.Type()
  341. resultElemType := resultType.Elem()
  342. val := reflect.New(resultElemType)
  343. if err := d.decode(name, node, reflect.Indirect(val)); err != nil {
  344. return err
  345. }
  346. result.Set(val)
  347. return nil
  348. }
  349. func (d *decoder) decodeSlice(name string, node ast.Node, result reflect.Value) error {
  350. // If we have an interface, then we can address the interface,
  351. // but not the slice itself, so get the element but set the interface
  352. set := result
  353. if result.Kind() == reflect.Interface {
  354. result = result.Elem()
  355. }
  356. // Create the slice if it isn't nil
  357. resultType := result.Type()
  358. resultElemType := resultType.Elem()
  359. if result.IsNil() {
  360. resultSliceType := reflect.SliceOf(resultElemType)
  361. result = reflect.MakeSlice(
  362. resultSliceType, 0, 0)
  363. }
  364. // Figure out the items we'll be copying into the slice
  365. var items []ast.Node
  366. switch n := node.(type) {
  367. case *ast.ObjectList:
  368. items = make([]ast.Node, len(n.Items))
  369. for i, item := range n.Items {
  370. items[i] = item
  371. }
  372. case *ast.ObjectType:
  373. items = []ast.Node{n}
  374. case *ast.ListType:
  375. items = n.List
  376. default:
  377. return &parser.PosError{
  378. Pos: node.Pos(),
  379. Err: fmt.Errorf("unknown slice type: %T", node),
  380. }
  381. }
  382. for i, item := range items {
  383. fieldName := fmt.Sprintf("%s[%d]", name, i)
  384. // Decode
  385. val := reflect.Indirect(reflect.New(resultElemType))
  386. if err := d.decode(fieldName, item, val); err != nil {
  387. return err
  388. }
  389. // Append it onto the slice
  390. result = reflect.Append(result, val)
  391. }
  392. set.Set(result)
  393. return nil
  394. }
  395. func (d *decoder) decodeString(name string, node ast.Node, result reflect.Value) error {
  396. switch n := node.(type) {
  397. case *ast.LiteralType:
  398. switch n.Token.Type {
  399. case token.NUMBER:
  400. result.Set(reflect.ValueOf(n.Token.Text).Convert(result.Type()))
  401. return nil
  402. case token.STRING, token.HEREDOC:
  403. result.Set(reflect.ValueOf(n.Token.Value()).Convert(result.Type()))
  404. return nil
  405. }
  406. }
  407. return &parser.PosError{
  408. Pos: node.Pos(),
  409. Err: fmt.Errorf("%s: unknown type for string %T", name, node),
  410. }
  411. }
  412. func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) error {
  413. var item *ast.ObjectItem
  414. if it, ok := node.(*ast.ObjectItem); ok {
  415. item = it
  416. node = it.Val
  417. }
  418. if ot, ok := node.(*ast.ObjectType); ok {
  419. node = ot.List
  420. }
  421. // Handle the special case where the object itself is a literal. Previously
  422. // the yacc parser would always ensure top-level elements were arrays. The new
  423. // parser does not make the same guarantees, thus we need to convert any
  424. // top-level literal elements into a list.
  425. if _, ok := node.(*ast.LiteralType); ok {
  426. node = &ast.ObjectList{Items: []*ast.ObjectItem{item}}
  427. }
  428. list, ok := node.(*ast.ObjectList)
  429. if !ok {
  430. return &parser.PosError{
  431. Pos: node.Pos(),
  432. Err: fmt.Errorf("%s: not an object type for struct (%T)", name, node),
  433. }
  434. }
  435. // This slice will keep track of all the structs we'll be decoding.
  436. // There can be more than one struct if there are embedded structs
  437. // that are squashed.
  438. structs := make([]reflect.Value, 1, 5)
  439. structs[0] = result
  440. // Compile the list of all the fields that we're going to be decoding
  441. // from all the structs.
  442. fields := make(map[*reflect.StructField]reflect.Value)
  443. for len(structs) > 0 {
  444. structVal := structs[0]
  445. structs = structs[1:]
  446. structType := structVal.Type()
  447. for i := 0; i < structType.NumField(); i++ {
  448. fieldType := structType.Field(i)
  449. tagParts := strings.Split(fieldType.Tag.Get(tagName), ",")
  450. // Ignore fields with tag name "-"
  451. if tagParts[0] == "-" {
  452. continue
  453. }
  454. if fieldType.Anonymous {
  455. fieldKind := fieldType.Type.Kind()
  456. if fieldKind != reflect.Struct {
  457. return &parser.PosError{
  458. Pos: node.Pos(),
  459. Err: fmt.Errorf("%s: unsupported type to struct: %s",
  460. fieldType.Name, fieldKind),
  461. }
  462. }
  463. // We have an embedded field. We "squash" the fields down
  464. // if specified in the tag.
  465. squash := false
  466. for _, tag := range tagParts[1:] {
  467. if tag == "squash" {
  468. squash = true
  469. break
  470. }
  471. }
  472. if squash {
  473. structs = append(
  474. structs, result.FieldByName(fieldType.Name))
  475. continue
  476. }
  477. }
  478. // Normal struct field, store it away
  479. fields[&fieldType] = structVal.Field(i)
  480. }
  481. }
  482. usedKeys := make(map[string]struct{})
  483. decodedFields := make([]string, 0, len(fields))
  484. decodedFieldsVal := make([]reflect.Value, 0)
  485. unusedKeysVal := make([]reflect.Value, 0)
  486. for fieldType, field := range fields {
  487. if !field.IsValid() {
  488. // This should never happen
  489. panic("field is not valid")
  490. }
  491. // If we can't set the field, then it is unexported or something,
  492. // and we just continue onwards.
  493. if !field.CanSet() {
  494. continue
  495. }
  496. fieldName := fieldType.Name
  497. tagValue := fieldType.Tag.Get(tagName)
  498. tagParts := strings.SplitN(tagValue, ",", 2)
  499. if len(tagParts) >= 2 {
  500. switch tagParts[1] {
  501. case "decodedFields":
  502. decodedFieldsVal = append(decodedFieldsVal, field)
  503. continue
  504. case "key":
  505. if item == nil {
  506. return &parser.PosError{
  507. Pos: node.Pos(),
  508. Err: fmt.Errorf("%s: %s asked for 'key', impossible",
  509. name, fieldName),
  510. }
  511. }
  512. field.SetString(item.Keys[0].Token.Value().(string))
  513. continue
  514. case "unusedKeys":
  515. unusedKeysVal = append(unusedKeysVal, field)
  516. continue
  517. }
  518. }
  519. if tagParts[0] != "" {
  520. fieldName = tagParts[0]
  521. }
  522. // Determine the element we'll use to decode. If it is a single
  523. // match (only object with the field), then we decode it exactly.
  524. // If it is a prefix match, then we decode the matches.
  525. filter := list.Filter(fieldName)
  526. prefixMatches := filter.Children()
  527. matches := filter.Elem()
  528. if len(matches.Items) == 0 && len(prefixMatches.Items) == 0 {
  529. continue
  530. }
  531. // Track the used key
  532. usedKeys[fieldName] = struct{}{}
  533. // Create the field name and decode. We range over the elements
  534. // because we actually want the value.
  535. fieldName = fmt.Sprintf("%s.%s", name, fieldName)
  536. if len(prefixMatches.Items) > 0 {
  537. if err := d.decode(fieldName, prefixMatches, field); err != nil {
  538. return err
  539. }
  540. }
  541. for _, match := range matches.Items {
  542. var decodeNode ast.Node = match.Val
  543. if ot, ok := decodeNode.(*ast.ObjectType); ok {
  544. decodeNode = &ast.ObjectList{Items: ot.List.Items}
  545. }
  546. if err := d.decode(fieldName, decodeNode, field); err != nil {
  547. return err
  548. }
  549. }
  550. decodedFields = append(decodedFields, fieldType.Name)
  551. }
  552. if len(decodedFieldsVal) > 0 {
  553. // Sort it so that it is deterministic
  554. sort.Strings(decodedFields)
  555. for _, v := range decodedFieldsVal {
  556. v.Set(reflect.ValueOf(decodedFields))
  557. }
  558. }
  559. return nil
  560. }
  561. // findNodeType returns the type of ast.Node
  562. func findNodeType() reflect.Type {
  563. var nodeContainer struct {
  564. Node ast.Node
  565. }
  566. value := reflect.ValueOf(nodeContainer).FieldByName("Node")
  567. return value.Type()
  568. }