You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

yaml.go 19 kB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662
  1. //
  2. // Copyright (c) 2011-2019 Canonical Ltd
  3. //
  4. // Licensed under the Apache License, Version 2.0 (the "License");
  5. // you may not use this file except in compliance with the License.
  6. // You may obtain a copy of the License at
  7. //
  8. // http://www.apache.org/licenses/LICENSE-2.0
  9. //
  10. // Unless required by applicable law or agreed to in writing, software
  11. // distributed under the License is distributed on an "AS IS" BASIS,
  12. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. // See the License for the specific language governing permissions and
  14. // limitations under the License.
  15. // Package yaml implements YAML support for the Go language.
  16. //
  17. // Source code and other details for the project are available at GitHub:
  18. //
  19. // https://github.com/go-yaml/yaml
  20. //
  21. package yaml
  22. import (
  23. "errors"
  24. "fmt"
  25. "io"
  26. "reflect"
  27. "strings"
  28. "sync"
  29. "unicode/utf8"
  30. )
  31. // The Unmarshaler interface may be implemented by types to customize their
  32. // behavior when being unmarshaled from a YAML document.
  33. type Unmarshaler interface {
  34. UnmarshalYAML(value *Node) error
  35. }
  36. type obsoleteUnmarshaler interface {
  37. UnmarshalYAML(unmarshal func(interface{}) error) error
  38. }
  39. // The Marshaler interface may be implemented by types to customize their
  40. // behavior when being marshaled into a YAML document. The returned value
  41. // is marshaled in place of the original value implementing Marshaler.
  42. //
  43. // If an error is returned by MarshalYAML, the marshaling procedure stops
  44. // and returns with the provided error.
  45. type Marshaler interface {
  46. MarshalYAML() (interface{}, error)
  47. }
  48. // Unmarshal decodes the first document found within the in byte slice
  49. // and assigns decoded values into the out value.
  50. //
  51. // Maps and pointers (to a struct, string, int, etc) are accepted as out
  52. // values. If an internal pointer within a struct is not initialized,
  53. // the yaml package will initialize it if necessary for unmarshalling
  54. // the provided data. The out parameter must not be nil.
  55. //
  56. // The type of the decoded values should be compatible with the respective
  57. // values in out. If one or more values cannot be decoded due to a type
  58. // mismatches, decoding continues partially until the end of the YAML
  59. // content, and a *yaml.TypeError is returned with details for all
  60. // missed values.
  61. //
  62. // Struct fields are only unmarshalled if they are exported (have an
  63. // upper case first letter), and are unmarshalled using the field name
  64. // lowercased as the default key. Custom keys may be defined via the
  65. // "yaml" name in the field tag: the content preceding the first comma
  66. // is used as the key, and the following comma-separated options are
  67. // used to tweak the marshalling process (see Marshal).
  68. // Conflicting names result in a runtime error.
  69. //
  70. // For example:
  71. //
  72. // type T struct {
  73. // F int `yaml:"a,omitempty"`
  74. // B int
  75. // }
  76. // var t T
  77. // yaml.Unmarshal([]byte("a: 1\nb: 2"), &t)
  78. //
  79. // See the documentation of Marshal for the format of tags and a list of
  80. // supported tag options.
  81. //
  82. func Unmarshal(in []byte, out interface{}) (err error) {
  83. return unmarshal(in, out, false)
  84. }
  85. // A Decorder reads and decodes YAML values from an input stream.
  86. type Decoder struct {
  87. parser *parser
  88. knownFields bool
  89. }
  90. // NewDecoder returns a new decoder that reads from r.
  91. //
  92. // The decoder introduces its own buffering and may read
  93. // data from r beyond the YAML values requested.
  94. func NewDecoder(r io.Reader) *Decoder {
  95. return &Decoder{
  96. parser: newParserFromReader(r),
  97. }
  98. }
  99. // KnownFields ensures that the keys in decoded mappings to
  100. // exist as fields in the struct being decoded into.
  101. func (dec *Decoder) KnownFields(enable bool) {
  102. dec.knownFields = enable
  103. }
  104. // Decode reads the next YAML-encoded value from its input
  105. // and stores it in the value pointed to by v.
  106. //
  107. // See the documentation for Unmarshal for details about the
  108. // conversion of YAML into a Go value.
  109. func (dec *Decoder) Decode(v interface{}) (err error) {
  110. d := newDecoder()
  111. d.knownFields = dec.knownFields
  112. defer handleErr(&err)
  113. node := dec.parser.parse()
  114. if node == nil {
  115. return io.EOF
  116. }
  117. out := reflect.ValueOf(v)
  118. if out.Kind() == reflect.Ptr && !out.IsNil() {
  119. out = out.Elem()
  120. }
  121. d.unmarshal(node, out)
  122. if len(d.terrors) > 0 {
  123. return &TypeError{d.terrors}
  124. }
  125. return nil
  126. }
  127. // Decode decodes the node and stores its data into the value pointed to by v.
  128. //
  129. // See the documentation for Unmarshal for details about the
  130. // conversion of YAML into a Go value.
  131. func (n *Node) Decode(v interface{}) (err error) {
  132. d := newDecoder()
  133. defer handleErr(&err)
  134. out := reflect.ValueOf(v)
  135. if out.Kind() == reflect.Ptr && !out.IsNil() {
  136. out = out.Elem()
  137. }
  138. d.unmarshal(n, out)
  139. if len(d.terrors) > 0 {
  140. return &TypeError{d.terrors}
  141. }
  142. return nil
  143. }
  144. func unmarshal(in []byte, out interface{}, strict bool) (err error) {
  145. defer handleErr(&err)
  146. d := newDecoder()
  147. p := newParser(in)
  148. defer p.destroy()
  149. node := p.parse()
  150. if node != nil {
  151. v := reflect.ValueOf(out)
  152. if v.Kind() == reflect.Ptr && !v.IsNil() {
  153. v = v.Elem()
  154. }
  155. d.unmarshal(node, v)
  156. }
  157. if len(d.terrors) > 0 {
  158. return &TypeError{d.terrors}
  159. }
  160. return nil
  161. }
  162. // Marshal serializes the value provided into a YAML document. The structure
  163. // of the generated document will reflect the structure of the value itself.
  164. // Maps and pointers (to struct, string, int, etc) are accepted as the in value.
  165. //
  166. // Struct fields are only marshalled if they are exported (have an upper case
  167. // first letter), and are marshalled using the field name lowercased as the
  168. // default key. Custom keys may be defined via the "yaml" name in the field
  169. // tag: the content preceding the first comma is used as the key, and the
  170. // following comma-separated options are used to tweak the marshalling process.
  171. // Conflicting names result in a runtime error.
  172. //
  173. // The field tag format accepted is:
  174. //
  175. // `(...) yaml:"[<key>][,<flag1>[,<flag2>]]" (...)`
  176. //
  177. // The following flags are currently supported:
  178. //
  179. // omitempty Only include the field if it's not set to the zero
  180. // value for the type or to empty slices or maps.
  181. // Zero valued structs will be omitted if all their public
  182. // fields are zero, unless they implement an IsZero
  183. // method (see the IsZeroer interface type), in which
  184. // case the field will be included if that method returns true.
  185. //
  186. // flow Marshal using a flow style (useful for structs,
  187. // sequences and maps).
  188. //
  189. // inline Inline the field, which must be a struct or a map,
  190. // causing all of its fields or keys to be processed as if
  191. // they were part of the outer struct. For maps, keys must
  192. // not conflict with the yaml keys of other struct fields.
  193. //
  194. // In addition, if the key is "-", the field is ignored.
  195. //
  196. // For example:
  197. //
  198. // type T struct {
  199. // F int `yaml:"a,omitempty"`
  200. // B int
  201. // }
  202. // yaml.Marshal(&T{B: 2}) // Returns "b: 2\n"
  203. // yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n"
  204. //
  205. func Marshal(in interface{}) (out []byte, err error) {
  206. defer handleErr(&err)
  207. e := newEncoder()
  208. defer e.destroy()
  209. e.marshalDoc("", reflect.ValueOf(in))
  210. e.finish()
  211. out = e.out
  212. return
  213. }
  214. // An Encoder writes YAML values to an output stream.
  215. type Encoder struct {
  216. encoder *encoder
  217. }
  218. // NewEncoder returns a new encoder that writes to w.
  219. // The Encoder should be closed after use to flush all data
  220. // to w.
  221. func NewEncoder(w io.Writer) *Encoder {
  222. return &Encoder{
  223. encoder: newEncoderWithWriter(w),
  224. }
  225. }
  226. // Encode writes the YAML encoding of v to the stream.
  227. // If multiple items are encoded to the stream, the
  228. // second and subsequent document will be preceded
  229. // with a "---" document separator, but the first will not.
  230. //
  231. // See the documentation for Marshal for details about the conversion of Go
  232. // values to YAML.
  233. func (e *Encoder) Encode(v interface{}) (err error) {
  234. defer handleErr(&err)
  235. e.encoder.marshalDoc("", reflect.ValueOf(v))
  236. return nil
  237. }
  238. // SetIndent changes the used indentation used when encoding.
  239. func (e *Encoder) SetIndent(spaces int) {
  240. if spaces < 0 {
  241. panic("yaml: cannot indent to a negative number of spaces")
  242. }
  243. e.encoder.indent = spaces
  244. }
  245. // Close closes the encoder by writing any remaining data.
  246. // It does not write a stream terminating string "...".
  247. func (e *Encoder) Close() (err error) {
  248. defer handleErr(&err)
  249. e.encoder.finish()
  250. return nil
  251. }
  252. func handleErr(err *error) {
  253. if v := recover(); v != nil {
  254. if e, ok := v.(yamlError); ok {
  255. *err = e.err
  256. } else {
  257. panic(v)
  258. }
  259. }
  260. }
  261. type yamlError struct {
  262. err error
  263. }
  264. func fail(err error) {
  265. panic(yamlError{err})
  266. }
  267. func failf(format string, args ...interface{}) {
  268. panic(yamlError{fmt.Errorf("yaml: "+format, args...)})
  269. }
  270. // A TypeError is returned by Unmarshal when one or more fields in
  271. // the YAML document cannot be properly decoded into the requested
  272. // types. When this error is returned, the value is still
  273. // unmarshaled partially.
  274. type TypeError struct {
  275. Errors []string
  276. }
  277. func (e *TypeError) Error() string {
  278. return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n "))
  279. }
  280. type Kind uint32
  281. const (
  282. DocumentNode Kind = 1 << iota
  283. SequenceNode
  284. MappingNode
  285. ScalarNode
  286. AliasNode
  287. )
  288. type Style uint32
  289. const (
  290. TaggedStyle Style = 1 << iota
  291. DoubleQuotedStyle
  292. SingleQuotedStyle
  293. LiteralStyle
  294. FoldedStyle
  295. FlowStyle
  296. )
  297. // Node represents an element in the YAML document hierarchy. While documents
  298. // are typically encoded and decoded into higher level types, such as structs
  299. // and maps, Node is an intermediate representation that allows detailed
  300. // control over the content being decoded or encoded.
  301. //
  302. // Values that make use of the Node type interact with the yaml package in the
  303. // same way any other type would do, by encoding and decoding yaml data
  304. // directly or indirectly into them.
  305. //
  306. // For example:
  307. //
  308. // var person struct {
  309. // Name string
  310. // Address yaml.Node
  311. // }
  312. // err := yaml.Unmarshal(data, &person)
  313. //
  314. // Or by itself:
  315. //
  316. // var person Node
  317. // err := yaml.Unmarshal(data, &person)
  318. //
  319. type Node struct {
  320. // Kind defines whether the node is a document, a mapping, a sequence,
  321. // a scalar value, or an alias to another node. The specific data type of
  322. // scalar nodes may be obtained via the ShortTag and LongTag methods.
  323. Kind Kind
  324. // Style allows customizing the apperance of the node in the tree.
  325. Style Style
  326. // Tag holds the YAML tag defining the data type for the value.
  327. // When decoding, this field will always be set to the resolved tag,
  328. // even when it wasn't explicitly provided in the YAML content.
  329. // When encoding, if this field is unset the value type will be
  330. // implied from the node properties, and if it is set, it will only
  331. // be serialized into the representation if TaggedStyle is used or
  332. // the implicit tag diverges from the provided one.
  333. Tag string
  334. // Value holds the unescaped and unquoted represenation of the value.
  335. Value string
  336. // Anchor holds the anchor name for this node, which allows aliases to point to it.
  337. Anchor string
  338. // Alias holds the node that this alias points to. Only valid when Kind is AliasNode.
  339. Alias *Node
  340. // Content holds contained nodes for documents, mappings, and sequences.
  341. Content []*Node
  342. // HeadComment holds any comments in the lines preceding the node and
  343. // not separated by an empty line.
  344. HeadComment string
  345. // LineComment holds any comments at the end of the line where the node is in.
  346. LineComment string
  347. // FootComment holds any comments following the node and before empty lines.
  348. FootComment string
  349. // Line and Column hold the node position in the decoded YAML text.
  350. // These fields are not respected when encoding the node.
  351. Line int
  352. Column int
  353. }
  354. // LongTag returns the long form of the tag that indicates the data type for
  355. // the node. If the Tag field isn't explicitly defined, one will be computed
  356. // based on the node properties.
  357. func (n *Node) LongTag() string {
  358. return longTag(n.ShortTag())
  359. }
  360. // ShortTag returns the short form of the YAML tag that indicates data type for
  361. // the node. If the Tag field isn't explicitly defined, one will be computed
  362. // based on the node properties.
  363. func (n *Node) ShortTag() string {
  364. if n.indicatedString() {
  365. return strTag
  366. }
  367. if n.Tag == "" || n.Tag == "!" {
  368. switch n.Kind {
  369. case MappingNode:
  370. return mapTag
  371. case SequenceNode:
  372. return seqTag
  373. case AliasNode:
  374. if n.Alias != nil {
  375. return n.Alias.ShortTag()
  376. }
  377. case ScalarNode:
  378. tag, _ := resolve("", n.Value)
  379. return tag
  380. }
  381. return ""
  382. }
  383. return shortTag(n.Tag)
  384. }
  385. func (n *Node) indicatedString() bool {
  386. return n.Kind == ScalarNode &&
  387. (shortTag(n.Tag) == strTag ||
  388. (n.Tag == "" || n.Tag == "!") && n.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0)
  389. }
  390. // SetString is a convenience function that sets the node to a string value
  391. // and defines its style in a pleasant way depending on its content.
  392. func (n *Node) SetString(s string) {
  393. n.Kind = ScalarNode
  394. if utf8.ValidString(s) {
  395. n.Value = s
  396. n.Tag = strTag
  397. } else {
  398. n.Value = encodeBase64(s)
  399. n.Tag = binaryTag
  400. }
  401. if strings.Contains(n.Value, "\n") {
  402. n.Style = LiteralStyle
  403. }
  404. }
  405. // --------------------------------------------------------------------------
  406. // Maintain a mapping of keys to structure field indexes
  407. // The code in this section was copied from mgo/bson.
  408. // structInfo holds details for the serialization of fields of
  409. // a given struct.
  410. type structInfo struct {
  411. FieldsMap map[string]fieldInfo
  412. FieldsList []fieldInfo
  413. // InlineMap is the number of the field in the struct that
  414. // contains an ,inline map, or -1 if there's none.
  415. InlineMap int
  416. // InlineUnmarshalers holds indexes to inlined fields that
  417. // contain unmarshaler values.
  418. InlineUnmarshalers [][]int
  419. }
  420. type fieldInfo struct {
  421. Key string
  422. Num int
  423. OmitEmpty bool
  424. Flow bool
  425. // Id holds the unique field identifier, so we can cheaply
  426. // check for field duplicates without maintaining an extra map.
  427. Id int
  428. // Inline holds the field index if the field is part of an inlined struct.
  429. Inline []int
  430. }
  431. var structMap = make(map[reflect.Type]*structInfo)
  432. var fieldMapMutex sync.RWMutex
  433. var unmarshalerType reflect.Type
  434. func init() {
  435. var v Unmarshaler
  436. unmarshalerType = reflect.ValueOf(&v).Elem().Type()
  437. }
  438. func getStructInfo(st reflect.Type) (*structInfo, error) {
  439. fieldMapMutex.RLock()
  440. sinfo, found := structMap[st]
  441. fieldMapMutex.RUnlock()
  442. if found {
  443. return sinfo, nil
  444. }
  445. n := st.NumField()
  446. fieldsMap := make(map[string]fieldInfo)
  447. fieldsList := make([]fieldInfo, 0, n)
  448. inlineMap := -1
  449. inlineUnmarshalers := [][]int(nil)
  450. for i := 0; i != n; i++ {
  451. field := st.Field(i)
  452. if field.PkgPath != "" && !field.Anonymous {
  453. continue // Private field
  454. }
  455. info := fieldInfo{Num: i}
  456. tag := field.Tag.Get("yaml")
  457. if tag == "" && strings.Index(string(field.Tag), ":") < 0 {
  458. tag = string(field.Tag)
  459. }
  460. if tag == "-" {
  461. continue
  462. }
  463. inline := false
  464. fields := strings.Split(tag, ",")
  465. if len(fields) > 1 {
  466. for _, flag := range fields[1:] {
  467. switch flag {
  468. case "omitempty":
  469. info.OmitEmpty = true
  470. case "flow":
  471. info.Flow = true
  472. case "inline":
  473. inline = true
  474. default:
  475. return nil, errors.New(fmt.Sprintf("unsupported flag %q in tag %q of type %s", flag, tag, st))
  476. }
  477. }
  478. tag = fields[0]
  479. }
  480. if inline {
  481. switch field.Type.Kind() {
  482. case reflect.Map:
  483. if inlineMap >= 0 {
  484. return nil, errors.New("multiple ,inline maps in struct " + st.String())
  485. }
  486. if field.Type.Key() != reflect.TypeOf("") {
  487. return nil, errors.New("option ,inline needs a map with string keys in struct " + st.String())
  488. }
  489. inlineMap = info.Num
  490. case reflect.Struct, reflect.Ptr:
  491. ftype := field.Type
  492. for ftype.Kind() == reflect.Ptr {
  493. ftype = ftype.Elem()
  494. }
  495. if ftype.Kind() != reflect.Struct {
  496. return nil, errors.New("option ,inline may only be used on a struct or map field")
  497. }
  498. if reflect.PtrTo(ftype).Implements(unmarshalerType) {
  499. inlineUnmarshalers = append(inlineUnmarshalers, []int{i})
  500. } else {
  501. sinfo, err := getStructInfo(ftype)
  502. if err != nil {
  503. return nil, err
  504. }
  505. for _, index := range sinfo.InlineUnmarshalers {
  506. inlineUnmarshalers = append(inlineUnmarshalers, append([]int{i}, index...))
  507. }
  508. for _, finfo := range sinfo.FieldsList {
  509. if _, found := fieldsMap[finfo.Key]; found {
  510. msg := "duplicated key '" + finfo.Key + "' in struct " + st.String()
  511. return nil, errors.New(msg)
  512. }
  513. if finfo.Inline == nil {
  514. finfo.Inline = []int{i, finfo.Num}
  515. } else {
  516. finfo.Inline = append([]int{i}, finfo.Inline...)
  517. }
  518. finfo.Id = len(fieldsList)
  519. fieldsMap[finfo.Key] = finfo
  520. fieldsList = append(fieldsList, finfo)
  521. }
  522. }
  523. default:
  524. return nil, errors.New("option ,inline may only be used on a struct or map field")
  525. }
  526. continue
  527. }
  528. if tag != "" {
  529. info.Key = tag
  530. } else {
  531. info.Key = strings.ToLower(field.Name)
  532. }
  533. if _, found = fieldsMap[info.Key]; found {
  534. msg := "duplicated key '" + info.Key + "' in struct " + st.String()
  535. return nil, errors.New(msg)
  536. }
  537. info.Id = len(fieldsList)
  538. fieldsList = append(fieldsList, info)
  539. fieldsMap[info.Key] = info
  540. }
  541. sinfo = &structInfo{
  542. FieldsMap: fieldsMap,
  543. FieldsList: fieldsList,
  544. InlineMap: inlineMap,
  545. InlineUnmarshalers: inlineUnmarshalers,
  546. }
  547. fieldMapMutex.Lock()
  548. structMap[st] = sinfo
  549. fieldMapMutex.Unlock()
  550. return sinfo, nil
  551. }
  552. // IsZeroer is used to check whether an object is zero to
  553. // determine whether it should be omitted when marshaling
  554. // with the omitempty flag. One notable implementation
  555. // is time.Time.
  556. type IsZeroer interface {
  557. IsZero() bool
  558. }
  559. func isZero(v reflect.Value) bool {
  560. kind := v.Kind()
  561. if z, ok := v.Interface().(IsZeroer); ok {
  562. if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() {
  563. return true
  564. }
  565. return z.IsZero()
  566. }
  567. switch kind {
  568. case reflect.String:
  569. return len(v.String()) == 0
  570. case reflect.Interface, reflect.Ptr:
  571. return v.IsNil()
  572. case reflect.Slice:
  573. return v.Len() == 0
  574. case reflect.Map:
  575. return v.Len() == 0
  576. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
  577. return v.Int() == 0
  578. case reflect.Float32, reflect.Float64:
  579. return v.Float() == 0
  580. case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
  581. return v.Uint() == 0
  582. case reflect.Bool:
  583. return !v.Bool()
  584. case reflect.Struct:
  585. vt := v.Type()
  586. for i := v.NumField() - 1; i >= 0; i-- {
  587. if vt.Field(i).PkgPath != "" {
  588. continue // Private field
  589. }
  590. if !isZero(v.Field(i)) {
  591. return false
  592. }
  593. }
  594. return true
  595. }
  596. return false
  597. }