dyaml.token source code

1 
2 //          Copyright Ferdinand Majerech 2011-2014.
3 // Distributed under the Boost Software License, Version 1.0.
4 //    (See accompanying file LICENSE_1_0.txt or copy at
5 //          http://www.boost.org/LICENSE_1_0.txt)
6 
7 /// YAML tokens.
8 /// Code based on PyYAML: http://www.pyyaml.org
9 module dyaml.token;
10 
11 
12 import std.conv;
13 
14 import dyaml.encoding;
15 import dyaml.exception;
16 import dyaml.reader;
17 import dyaml.style;
18 
19 
20 package:
21 
22 /// Token types.
23 enum TokenID : ubyte
24 {
25     // Invalid (uninitialized) token
26     invalid = 0,
27     directive,
28     documentStart,
29     documentEnd,
30     streamStart,
31     streamEnd,
32     blockSequenceStart,
33     blockMappingStart,
34     blockEnd,
35     flowSequenceStart,
36     flowMappingStart,
37     flowSequenceEnd,
38     flowMappingEnd,
39     key,
40     value,
41     blockEntry,
42     flowEntry,
43     alias_,
44     anchor,
45     tag,
46     scalar
47 }
48 
49 /// Specifies the type of a tag directive token.
50 enum DirectiveType : ubyte
51 {
52     // YAML version directive.
53     yaml,
54     // Tag directive.
55     tag,
56     // Any other directive is "reserved" for future YAML versions.
57     reserved
58 }
59 
60 /// Token produced by scanner.
61 ///
62 /// 32 bytes on 64-bit.
63 struct Token
64 {
65     @disable int opCmp(ref Token);
66 
67     // 16B
68     /// Value of the token, if any.
69     ///
70     /// Values are char[] instead of string, as Parser may still change them in a few
71     /// cases. Parser casts values to strings when producing Events.
72     char[] value;
73     // 4B
74     /// Start position of the token in file/stream.
75     Mark startMark;
76     // 4B
77     /// End position of the token in file/stream.
78     Mark endMark;
79     // 1B
80     /// Token type.
81     TokenID id;
82     // 1B
83     /// Style of scalar token, if this is a scalar token.
84     ScalarStyle style;
85     // 1B
86     /// Encoding, if this is a stream start token.
87     Encoding encoding;
88     // 1B
89     /// Type of directive for directiveToken.
90     DirectiveType directive;
91     // 4B
92     /// Used to split value into 2 substrings for tokens that need 2 values (tagToken)
93     uint valueDivider;
94 
95     /// Get string representation of the token ID.
96     @property string idString() @safe pure const {return id.to!string;}
97 }
98 
99 /// Construct a directive token.
100 ///
101 /// Params:  start     = Start position of the token.
102 ///          end       = End position of the token.
103 ///          value     = Value of the token.
104 ///          directive = Directive type (YAML or TAG in YAML 1.1).
105 ///          nameEnd = Position of the end of the name
106 Token directiveToken(const Mark start, const Mark end, char[] value,
107                      DirectiveType directive, const uint nameEnd) @safe pure nothrow @nogc
108 {
109     return Token(value, start, end, TokenID.directive, ScalarStyle.init, Encoding.init,
110                  directive, nameEnd);
111 }
112 
113 /// Construct a simple (no value) token with specified type.
114 ///
115 /// Params:  id    = Type of the token.
116 ///          start = Start position of the token.
117 ///          end   = End position of the token.
118 Token simpleToken(TokenID id)(const Mark start, const Mark end)
119 {
120     return Token(null, start, end, id);
121 }
122 
123 /// Construct a stream start token.
124 ///
125 /// Params:  start    = Start position of the token.
126 ///          end      = End position of the token.
127 ///          encoding = Encoding of the stream.
128 Token streamStartToken(const Mark start, const Mark end, const Encoding encoding) @safe pure nothrow @nogc
129 {
130     return Token(null, start, end, TokenID.streamStart, ScalarStyle.invalid, encoding);
131 }
132 
133 /// Aliases for construction of simple token types.
134 alias streamEndToken = simpleToken!(TokenID.streamEnd);
135 alias blockSequenceStartToken = simpleToken!(TokenID.blockSequenceStart);
136 alias blockMappingStartToken = simpleToken!(TokenID.blockMappingStart);
137 alias blockEndToken = simpleToken!(TokenID.blockEnd);
138 alias keyToken = simpleToken!(TokenID.key);
139 alias valueToken = simpleToken!(TokenID.value);
140 alias blockEntryToken = simpleToken!(TokenID.blockEntry);
141 alias flowEntryToken = simpleToken!(TokenID.flowEntry);
142 
143 /// Construct a simple token with value with specified type.
144 ///
145 /// Params:  id           = Type of the token.
146 ///          start        = Start position of the token.
147 ///          end          = End position of the token.
148 ///          value        = Value of the token.
149 ///          valueDivider = A hack for TagToken to store 2 values in value; the first
150 ///                         value goes up to valueDivider, the second after it.
151 Token simpleValueToken(TokenID id)(const Mark start, const Mark end, char[] value,
152                                    const uint valueDivider = uint.max)
153 {
154     return Token(value, start, end, id, ScalarStyle.invalid, Encoding.init,
155                  DirectiveType.init, valueDivider);
156 }
157 
158 /// Alias for construction of tag token.
159 alias tagToken = simpleValueToken!(TokenID.tag);
160 alias aliasToken = simpleValueToken!(TokenID.alias_);
161 alias anchorToken = simpleValueToken!(TokenID.anchor);
162 
163 /// Construct a scalar token.
164 ///
165 /// Params:  start = Start position of the token.
166 ///          end   = End position of the token.
167 ///          value = Value of the token.
168 ///          style = Style of the token.
169 Token scalarToken(const Mark start, const Mark end, char[] value, const ScalarStyle style) @safe pure nothrow @nogc
170 {
171     return Token(value, start, end, TokenID.scalar, style);
172 }