1 /**
2  * Lexical analyzer
3  *
4  * Copyright: © 2017 Andrey Kabylin
5  * License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6  */
7 
8 module rpdl.lexer;
9 
10 import std.stdio;
11 import std.format : formattedWrite;
12 import std.array : appender;
13 import std.ascii;
14 
15 import rpdl.token;
16 import rpdl.stream;
17 
18 class LexerError : Exception {
19     this(in uint line, in uint pos, in string details) {
20         auto writer = appender!string();
21         formattedWrite(writer, "line %d, pos %d: %s", line, pos, details);
22         super(writer.data);
23     }
24 }
25 
26 /// Lexical analyzer - convert steam of symbols to stream of tokens
27 class Lexer {
28     /// Create lexer with `rpdl.stream.IStream`
29     this(SymbolStream stream) {
30         this.stream = stream;
31         stream.read();
32     }
33 
34     /// Get next token in the stream and store it to `currentToken`
35     Token nextToken() {
36         if (stackCursor < tokenStack.length) {
37             p_currentToken = tokenStack[stackCursor++];
38         } else {
39             p_currentToken = lexToken();
40             tokenStack ~= p_currentToken;
41             stackCursor = tokenStack.length;
42         }
43 
44         return p_currentToken;
45     }
46 
47     /// Get previous token in the stream and store it to `currentToken`
48     Token prevToken() {
49         --stackCursor;
50         p_currentToken = tokenStack[stackCursor-1];
51         return p_currentToken;
52     }
53 
54     @property Token currentToken() { return p_currentToken; }
55 
56 private:
57     SymbolStream stream;
58     bool negative = false;  /// If true, then number will be negative
59     Token p_currentToken;
60 
61     Token[] tokenStack;  /// Save tokens in stack
62     size_t stackCursor = 0;
63 
64     /// Determines which token to create
65     Token lexToken() {
66         switch (stream.lastChar) {
67             case ' ', '\n', '\r':
68                 stream.read();
69                 return lexToken();
70 
71             case '-', '+':
72                 negative = stream.lastChar == '-';
73                 stream.read();
74 
75                 if (!isDigit(stream.lastChar)) {
76                     negative = false;
77                     goto default;
78                 }
79 
80             case '0': .. case '9':
81                 auto token = new NumberToken(stream, negative);
82                 negative = false;
83                 return token;
84 
85             case 'A': .. case 'Z': case 'a': .. case 'z': case '_':
86                 return new IdToken(stream);
87 
88             case '\"':
89                 return new StringToken(stream);
90 
91             case '#':
92                 skipComment();
93                 return lexToken();
94 
95             default:
96                 auto token = new SymbolToken(stream, stream.lastChar);
97                 stream.read();
98                 return token;
99         }
100     }
101 
102     /// Skip symbol in whole line
103     void skipComment() {
104         while (!stream.eof && stream.lastChar != '\n' && stream.lastChar != '\r')
105             stream.read();
106     }
107 }