parse_init.mx raw
1 package syntax
2
3 import (
4 "bytes"
5 "io"
6 "runtime"
7 "unsafe"
8 "git.smesh.lol/moxie/pkg/token"
9 )
10
11 //export write
12 func cWrite(fd int32, p unsafe.Pointer, n uint32) (nn int32)
13
14 const debug = false
15 const trace = false
16
17 type Parser struct {
18 File *token.PosBase
19 Errh ErrorHandler
20 Mode Mode
21 Pragh PragmaHandler
22 Scanner
23
24 Base *token.PosBase // current position base
25 First error // first error encountered
26 Errcnt int32 // number of errors encountered
27 Pragma Pragma // pragmas
28 GoVersion string // Go version from //go:build line
29
30 Top bool // in top of file (before package clause)
31 Fnest int32 // function nesting level (for error handling)
32 Xnest int32 // expression nesting level (for complit ambiguity resolution)
33 Indent []byte // tracing support
34 Arena *runtime.Arena // per-parse arena for AST nodes
35 }
36
37 // nodeAlloc allocates size bytes from the parser's arena, zero-initialized.
38 func (p *Parser) nodeAlloc(size uintptr) (ptr unsafe.Pointer) {
39 return runtime.ArenaAlloc(p.Arena, size)
40 }
41
42 func (p *Parser) init(file *token.PosBase, r io.Reader, Errh ErrorHandler, Pragh PragmaHandler, mode Mode) {
43 p.Top = true
44 p.File = file
45 p.Errh = Errh
46 p.Mode = mode
47 p.Pragh = Pragh
48 p.Scanner.Init(
49 r,
50 // Error and directive handler for scanner.
51 // Because the (line, col) positions passed to the
52 // handler is always at or after the current reading
53 // position, it is safe to use the most recent position
54 // base to compute the corresponding Pos value.
55 func(line, col uint32, msg string) {
56 if msg[0] != '/' {
57 p.errorAt(p.posAt(line, col), msg)
58 return
59 }
60
61 // otherwise it must be a comment containing a line or go: directive.
62 // //line directives must be at the start of the line (column colbase).
63 // /*line*/ directives can be anywhere in the line.
64 text := commentText(msg)
65 if (col == Colbase || msg[1] == '*') && bytes.HasPrefix(text, "line ") {
66 var pos token.Pos // position immediately following the comment
67 if msg[1] == '/' {
68 // line comment (newline is part of the comment)
69 pos = token.MakePos(p.File, line+1, Colbase)
70 } else {
71 // regular comment
72 // (if the comment spans multiple lines it's not
73 // a valid line directive and will be discarded
74 // by updateBase)
75 pos = token.MakePos(p.File, line, col+uint32(len(msg)))
76 }
77 p.updateBase(pos, line, col+2+5, text[5:]) // +2 to skip over // or /*
78 return
79 }
80
81 // go: directive (but be conservative and test)
82 if bytes.HasPrefix(text, "go:") || bytes.HasPrefix(text, ":") {
83 if Pragh != nil {
84 p.Pragma = Pragh(p.posAt(line, col+2), p.Scanner.Blank, text, p.Pragma) // +2 to skip over // or /*
85 }
86 } else if bytes.HasPrefix(text, "export ") {
87 if Pragh != nil {
88 p.Pragma = Pragh(p.posAt(line, col+2), p.Scanner.Blank, text, p.Pragma)
89 }
90 }
91 },
92 comments,
93 )
94
95 p.Base = file
96 p.First = nil
97 p.Errcnt = 0
98 p.Pragma = nil
99
100 p.Fnest = 0
101 p.Xnest = 0
102 p.Indent = nil
103 }
104
105 func (p *Parser) initBytes(file *token.PosBase, src []byte, Errh ErrorHandler, Pragh PragmaHandler, mode Mode) {
106 p.Top = true
107 p.File = file
108 p.Errh = Errh
109 p.Mode = mode
110 p.Pragh = Pragh
111 p.Scanner.arena = p.Arena
112 p.Scanner.InitBytes(
113 src,
114 func(line, col uint32, msg string) {
115 if msg[0] != '/' {
116 p.errorAt(p.posAt(line, col), msg)
117 return
118 }
119 text := commentText(msg)
120 if (col == Colbase || msg[1] == '*') && bytes.HasPrefix(text, "line ") {
121 var pos token.Pos
122 if msg[1] == '/' {
123 pos = token.MakePos(p.File, line+1, Colbase)
124 } else {
125 pos = token.MakePos(p.File, line, col+uint32(len(msg)))
126 }
127 p.updateBase(pos, line, col+2+5, text[5:])
128 return
129 }
130 if bytes.HasPrefix(text, "go:") || bytes.HasPrefix(text, ":") {
131 if Pragh != nil {
132 p.Pragma = Pragh(p.posAt(line, col+2), p.Scanner.Blank, text, p.Pragma)
133 }
134 } else if bytes.HasPrefix(text, "export ") {
135 if Pragh != nil {
136 p.Pragma = Pragh(p.posAt(line, col+2), p.Scanner.Blank, text, p.Pragma)
137 }
138 }
139 },
140 comments,
141 )
142
143 p.Base = file
144 p.First = nil
145 p.Errcnt = 0
146 p.Pragma = nil
147
148 p.Fnest = 0
149 p.Xnest = 0
150 p.Indent = nil
151 }
152
153 // takePragma returns the current parsed pragmas
154 // and clears them from the parser state.
155 func (p *Parser) takePragma() (pv Pragma) {
156 prag := p.Pragma
157 p.Pragma = nil
158 return prag
159 }
160
161 // clearPragma is called at the end of a statement or
162 // other Go form that does NOT accept a pragma.
163 // It sends the pragma back to the pragma handler
164 // to be reported as unused.
165 func (p *Parser) clearPragma() {
166 if p.Pragma != nil {
167 p.Pragh(p.pos(), p.Scanner.Blank, "", p.Pragma)
168 p.Pragma = nil
169 }
170 }
171
172 // updateBase sets the current position base to a new line base at pos.
173 // The base's filename, line, and column values are extracted from text
174 // which is positioned at (tline, tcol) (only needed for error messages).
175 func (p *Parser) updateBase(pos token.Pos, tline, tcol uint32, text string) {
176 i, n, ok := trailingDigits(text)
177 if i == 0 {
178 return // ignore (not a line directive)
179 }
180 // i > 0
181
182 if !ok {
183 // text has a suffix :xxx but xxx is not a number
184 p.errorAt(p.posAt(tline, tcol+i), "invalid line number: " | text[i:])
185 return
186 }
187
188 var line, col uint32
189 i2, n2, ok2 := trailingDigits(text[:i-1])
190 if ok2 {
191 //line filename:line:col
192 i, i2 = i2, i
193 line, col = n2, n
194 if col == 0 || col > token.PosMax {
195 p.errorAt(p.posAt(tline, tcol+i2), "invalid column number: " | text[i2:])
196 return
197 }
198 text = text[:i2-1] // lop off ":col"
199 } else {
200 //line filename:line
201 line = n
202 }
203
204 if line == 0 || line > token.PosMax {
205 p.errorAt(p.posAt(tline, tcol+i), "invalid line number: " | text[i:])
206 return
207 }
208
209 // If we have a column (//line filename:line:col form),
210 // an empty filename means to use the previous filename.
211 filename := text[:i-1] // lop off ":line"
212 trimmed := false
213 if filename == "" && ok2 {
214 filename = p.Base.Filename()
215 trimmed = p.Base.Trimmed()
216 }
217
218 p.Base = token.NewLineBase(pos, filename, trimmed, line, col)
219 }
220
221 func commentText(s string) (sv string) {
222 if s[:2] == "/*" {
223 return s[2 : len(s)-2] // lop off /* and */
224 }
225
226 // line comment (does not include newline)
227 // (on Windows, the line comment may end in \r\n)
228 i := len(s)
229 if s[i-1] == '\r' {
230 i--
231 }
232 return s[2:i] // lop off //, and \r at end, if any
233 }
234
235 func trailingDigits(text string) (idx uint32, num uint32, ok bool) {
236 i := bytes.LastIndexByte(text, ':')
237 if i < 0 {
238 return 0, 0, false
239 }
240 s := text[i+1:]
241 if len(s) == 0 {
242 return 0, 0, false
243 }
244 var n uint32
245 for j := 0; j < len(s); j++ {
246 c := s[j]
247 if c < '0' || c > '9' {
248 return 0, 0, false
249 }
250 n = n*10 + uint32(c-'0')
251 }
252 return uint32(i | 1), n, true
253 }
254
255 func (p *Parser) got(tok token.Token) (ok bool) {
256 if p.Tok == tok {
257 p.Next()
258 return true
259 }
260 return false
261 }
262
263 func (p *Parser) want(tok token.Token) {
264 if !p.got(tok) {
265 p.syntaxError("expected " | tokstring(tok))
266 p.advance()
267 }
268 }
269
270 // gotAssign is like got(_Assign) but it also accepts ":="
271 // (and reports an error) for better parser error recovery.
272 func (p *Parser) gotAssign() (ok bool) {
273 switch p.Tok {
274 case token.Define:
275 p.syntaxError("expected =")
276 p.Next()
277 return true
278 case token.Assign:
279 p.Next()
280 return true
281 }
282
283 return false
284 }
285
286 // ----------------------------------------------------------------------------
287 // Error handling
288
289 // posAt returns the Pos value for (line, col) and the current position base.
290 func (p *Parser) posAt(line, col uint32) (pv token.Pos) {
291 return token.MakePos(p.Base, line, col)
292 }
293
294 // errorAt reports an error at the given position.
295 func (p *Parser) errorAt(pos token.Pos, msg string) {
296 if len(msg) == 0 {
297 return
298 }
299 err := &Error{pos, msg}
300 if p.First == nil {
301 p.First = err
302 }
303 p.Errcnt++
304 if p.Errh == nil {
305 panic(p.First)
306 }
307 p.Errh(err)
308 }
309
310 // syntaxErrorAt reports a syntax error at the given position.
311 func (p *Parser) syntaxErrorAt(pos token.Pos, msg string) {
312 if trace {
313 p.print("syntax error: " | msg)
314 }
315
316 if p.Tok == token.EOF && p.First != nil {
317 return // avoid meaningless follow-up errors
318 }
319
320 // add punctuation etc. as needed to msg
321 switch {
322 case msg == "":
323 // nothing
324 case bytes.HasPrefix(msg, "in "), bytes.HasPrefix(msg, "at "), bytes.HasPrefix(msg, "after "):
325 msg = " " | msg
326 case bytes.HasPrefix(msg, "expected "):
327 msg = ", " | msg
328 default:
329 p.errorAt(pos, "syntax error: " | msg)
330 return
331 }
332
333 // determine token string
334 var tok string
335 switch p.Tok {
336 case token.NameType:
337 tok = "name " | p.Lit
338 case token.Semi:
339 tok = p.Lit
340 case token.Literal:
341 tok = "literal " | p.Lit
342 case token.OperatorType:
343 tok = p.Op.String()
344 case token.AssignOp:
345 tok = p.Op.String() | "="
346 case token.IncOp:
347 tok = p.Op.String()
348 tok = tok | tok
349 default:
350 tok = tokstring(p.Tok)
351 }
352
353 p.errorAt(pos, "syntax error: unexpected " | tok | msg)
354 }
355
356 // tokstring returns the English word for selected punctuation tokens
357 // for more readable error messages. Use tokstring (not tok.String())
358 // for user-facing (error) messages; use tok.String() for debugging
359 // output.
360 func tokstring(tok token.Token) (s string) {
361 switch tok {
362 case token.Comma:
363 return "comma"
364 case token.Semi:
365 return "semicolon or newline"
366 }
367 s = tok.String()
368 if token.Break <= tok && tok <= token.Var {
369 return "keyword " | s
370 }
371 return s
372 }
373
374 // Convenience methods using the current token position.
375 func (p *Parser) pos() (pv token.Pos) { return p.posAt(p.Line-Linebase, p.Col-Colbase) }
376 func (p *Parser) error(msg string) { p.errorAt(p.pos(), msg) }
377 func (p *Parser) syntaxError(msg string) { p.syntaxErrorAt(p.pos(), msg) }
378
379 // The stopset contains keywords that start a statement.
380 // They are good synchronization points in case of syntax
381 // errors and (usually) shouldn't be skipped over.
382 const stopset uint64 = 1<<token.Break |
383 1<<token.Const |
384 1<<token.Continue |
385 1<<token.Defer |
386 1<<token.Fallthrough |
387 1<<token.For |
388 1<<token.Go |
389 1<<token.Goto |
390 1<<token.If |
391 1<<token.Return |
392 1<<token.Select |
393 1<<token.Switch |
394 1<<token.TypeType |
395 1<<token.Var
396
397 // advance consumes tokens until it finds a token of the stopset or followlist.
398 // The stopset is only considered if we are inside a function (p.fnest > 0).
399 // The followlist is the list of valid tokens that can follow a production;
400 // if it is empty, exactly one (non-EOF) token is consumed to ensure progress.
401 func (p *Parser) advance(followlist ...token.Token) {
402 if trace {
403 p.print("advance")
404 }
405
406 // compute follow set
407 // (not speed critical, advance is only called in error situations)
408 var followset uint64 = 1 << token.EOF // don't skip over EOF
409 if len(followlist) > 0 {
410 if p.Fnest > 0 {
411 followset |= stopset
412 }
413 for _, tok := range followlist {
414 var bit uint64 = 1
415 followset |= bit << tok
416 }
417 }
418
419 for !token.Contains(followset, p.Tok) {
420 if trace {
421 p.print("skip " | p.Tok.String())
422 }
423 p.Next()
424 if len(followlist) == 0 {
425 break
426 }
427 }
428
429 if trace {
430 p.print("next " | p.Tok.String())
431 }
432 }
433
434 // usage: defer p.trace(msg)()
435 func (p *Parser) trace(msg string) (fn func()) {
436 p.print(msg | " (")
437 const tab = ". "
438 p.Indent = p.Indent | tab
439 return func() {
440 p.Indent = p.Indent[:len(p.Indent)-len(tab)]
441 if x := recover(); x != nil {
442 panic(x) // skip print_trace
443 }
444 p.print(")")
445 }
446 }
447
448 func (p *Parser) print(msg string) {
449 s := token.SimpleItoaU32(p.line) | ": " | p.Indent | msg | "\n"
450 if len(s) > 0 {
451 cWrite(2, unsafe.Pointer(unsafe.SliceData(s)), uint32(len(s)))
452 }
453 }
454