1 /** 2 * Lexical analyzer 3 * 4 * Copyright: © 2017 Andrey Kabylin 5 * License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 */ 7 8 module rpdl.lexer; 9 10 import std.stdio; 11 import std.format : formattedWrite; 12 import std.array : appender; 13 import std.ascii; 14 15 import rpdl.token; 16 import rpdl.stream; 17 18 class LexerError : Exception { 19 this(in uint line, in uint pos, in string details) { 20 auto writer = appender!string(); 21 formattedWrite(writer, "line %d, pos %d: %s", line, pos, details); 22 super(writer.data); 23 } 24 } 25 26 /// Lexical analyzer - convert steam of symbols to stream of tokens 27 class Lexer { 28 /// Create lexer with `rpdl.stream.IStream` 29 this(SymbolStream stream) { 30 this.stream = stream; 31 stream.read(); 32 } 33 34 /// Get next token in the stream and store it to `currentToken` 35 Token nextToken() { 36 if (stackCursor < tokenStack.length) { 37 p_currentToken = tokenStack[stackCursor++]; 38 } else { 39 p_currentToken = lexToken(); 40 tokenStack ~= p_currentToken; 41 stackCursor = tokenStack.length; 42 } 43 44 return p_currentToken; 45 } 46 47 /// Get previous token in the stream and store it to `currentToken` 48 Token prevToken() { 49 --stackCursor; 50 p_currentToken = tokenStack[stackCursor-1]; 51 return p_currentToken; 52 } 53 54 @property Token currentToken() { return p_currentToken; } 55 56 private: 57 SymbolStream stream; 58 bool negative = false; /// If true, then number will be negative 59 Token p_currentToken; 60 61 Token[] tokenStack; /// Save tokens in stack 62 size_t stackCursor = 0; 63 64 /// Determines which token to create 65 Token lexToken() { 66 switch (stream.lastChar) { 67 case ' ', '\n', '\r': 68 stream.read(); 69 return lexToken(); 70 71 case '-', '+': 72 negative = stream.lastChar == '-'; 73 stream.read(); 74 75 if (!isDigit(stream.lastChar)) { 76 negative = false; 77 goto default; 78 } 79 80 case '0': .. case '9': 81 auto token = new NumberToken(stream, negative); 82 negative = false; 83 return token; 84 85 case 'A': .. case 'Z': case 'a': .. case 'z': case '_': 86 return new IdToken(stream); 87 88 case '\"': 89 return new StringToken(stream); 90 91 case '#': 92 skipComment(); 93 return lexToken(); 94 95 default: 96 auto token = new SymbolToken(stream, stream.lastChar); 97 stream.read(); 98 return token; 99 } 100 } 101 102 /// Skip symbol in whole line 103 void skipComment() { 104 while (!stream.eof && stream.lastChar != '\n' && stream.lastChar != '\r') 105 stream.read(); 106 } 107 }