701 lines
14 KiB
C++
701 lines
14 KiB
C++
#include "src/util/c99_stdint.h"
|
|
#include <stddef.h>
|
|
#include <string.h>
|
|
#include <algorithm>
|
|
#include <limits>
|
|
#include <string>
|
|
|
|
#include "src/codegen/output.h"
|
|
#include "src/conf/opt.h"
|
|
#include "src/conf/warn.h"
|
|
#include "src/globals.h"
|
|
#include "src/ir/regexp/encoding/enc.h"
|
|
#include "src/ir/regexp/regexp.h"
|
|
#include "src/ir/regexp/regexp_null.h"
|
|
#include "src/parse/code.h"
|
|
#include "src/parse/extop.h"
|
|
#include "src/parse/input.h"
|
|
#include "src/parse/scanner.h"
|
|
#include "src/parse/parser.h" // needed by "y.tab.h"
|
|
#include "src/parse/unescape.h"
|
|
#include "src/util/range.h"
|
|
#include "src/util/s_to_n32_unsafe.h"
|
|
#include "y.tab.h"
|
|
|
|
extern YYSTYPE yylval;
|
|
|
|
#define YYCTYPE unsigned char
|
|
#define YYCURSOR cur
|
|
#define YYLIMIT lim
|
|
#define YYMARKER ptr
|
|
#define YYCTXMARKER ctx
|
|
#define YYFILL(n) { fill (n); }
|
|
|
|
namespace re2c
|
|
{
|
|
|
|
// source code is in ASCII: pointers have type 'char *'
|
|
// but re2c makes an implicit assumption that YYCTYPE is unsigned
|
|
// when it generates comparisons
|
|
/*!re2c
|
|
re2c:yych:conversion = 1;
|
|
*/
|
|
|
|
/*!re2c
|
|
zero = "\000";
|
|
dstring = "\"" ((. \ [\\"] ) | "\\" .)* "\"";
|
|
sstring = "'" ((. \ [\\'] ) | "\\" .)* "'" ;
|
|
letter = [a-zA-Z];
|
|
digit = [0-9];
|
|
lineno = [1-9] digit*;
|
|
name = (letter|digit|"_")+;
|
|
space = [ \t];
|
|
ws = (space | [\r\n]);
|
|
eol = ("\r\n" | "\n");
|
|
lineinf = lineno (space+ dstring)? eol;
|
|
|
|
esc = "\\";
|
|
hex_digit = [0-9a-fA-F];
|
|
esc_hex = esc ("x" hex_digit{2} | [uX] hex_digit{4} | "U" hex_digit{8});
|
|
esc_oct = esc [0-3] [0-7]{2}; // max 1-byte octal value is '\377'
|
|
esc_simple = esc [abfnrtv\\];
|
|
*/
|
|
|
|
Scanner::ParseMode Scanner::echo()
|
|
{
|
|
bool ignore_eoc = false;
|
|
int ignore_cnt = 0;
|
|
|
|
if (eof && cur == eof) // Catch EOF
|
|
{
|
|
return Stop;
|
|
}
|
|
|
|
tok = cur;
|
|
echo:
|
|
/*!re2c
|
|
beginRE = "%{" | "/*!re2c";
|
|
beginRE {
|
|
if (opts->rFlag)
|
|
{
|
|
fatal("found standard 're2c' block while using -r flag");
|
|
}
|
|
if (opts->target == opt_t::CODE)
|
|
{
|
|
const size_t lexeme_len = cur[-1] == '{'
|
|
? sizeof ("%{") - 1
|
|
: sizeof ("/*!re2c") - 1;
|
|
out.wraw(tok, tok_len () - lexeme_len);
|
|
}
|
|
tok = cur;
|
|
return Parse;
|
|
}
|
|
"/*!rules:re2c" {
|
|
if (opts->rFlag)
|
|
{
|
|
opts.reset_mapCodeName ();
|
|
}
|
|
else
|
|
{
|
|
fatal("found 'rules:re2c' block without -r flag");
|
|
}
|
|
tok = cur;
|
|
return Rules;
|
|
}
|
|
"/*!use:re2c" {
|
|
if (!opts->rFlag)
|
|
{
|
|
fatal("found 'use:re2c' block without -r flag");
|
|
}
|
|
reuse();
|
|
if (opts->target == opt_t::CODE)
|
|
{
|
|
const size_t lexeme_len = sizeof ("/*!use:re2c") - 1;
|
|
out.wraw(tok, tok_len () - lexeme_len);
|
|
}
|
|
tok = cur;
|
|
return Reuse;
|
|
}
|
|
"/*!max:re2c" {
|
|
if (opts->target != opt_t::DOT)
|
|
{
|
|
out.wdelay_yymaxfill ();
|
|
}
|
|
tok = pos = cur;
|
|
ignore_eoc = true;
|
|
goto echo;
|
|
}
|
|
"/*!getstate:re2c" {
|
|
tok = pos = cur;
|
|
out.wdelay_state_goto (opts->topIndent);
|
|
ignore_eoc = true;
|
|
goto echo;
|
|
}
|
|
"/*!ignore:re2c" {
|
|
tok = pos = cur;
|
|
ignore_eoc = true;
|
|
goto echo;
|
|
}
|
|
"/*!types:re2c" {
|
|
tok = pos = cur;
|
|
ignore_eoc = true;
|
|
if (opts->target != opt_t::DOT)
|
|
{
|
|
out.wdelay_line_info ().ws("\n")
|
|
.wdelay_types ().ws("\n")
|
|
.wline_info (cline, get_fname ().c_str ());
|
|
}
|
|
goto echo;
|
|
}
|
|
"*" "/" "\r"? "\n" {
|
|
cline++;
|
|
if (ignore_eoc)
|
|
{
|
|
if (ignore_cnt)
|
|
{
|
|
out.wline_info (cline, get_fname ().c_str ());
|
|
}
|
|
ignore_eoc = false;
|
|
ignore_cnt = 0;
|
|
}
|
|
else if (opts->target == opt_t::CODE)
|
|
{
|
|
out.wraw(tok, tok_len ());
|
|
}
|
|
tok = pos = cur;
|
|
goto echo;
|
|
}
|
|
"*" "/" {
|
|
if (ignore_eoc)
|
|
{
|
|
if (ignore_cnt)
|
|
{
|
|
out.ws("\n").wline_info (cline, get_fname ().c_str ());
|
|
}
|
|
ignore_eoc = false;
|
|
ignore_cnt = 0;
|
|
}
|
|
else if (opts->target == opt_t::CODE)
|
|
{
|
|
out.wraw(tok, tok_len ());
|
|
}
|
|
tok = pos = cur;
|
|
goto echo;
|
|
}
|
|
"\n" space* "#" space* "line" space+ / lineinf {
|
|
set_sourceline ();
|
|
goto echo;
|
|
}
|
|
"\n" {
|
|
if (ignore_eoc)
|
|
{
|
|
ignore_cnt++;
|
|
}
|
|
else if (opts->target == opt_t::CODE)
|
|
{
|
|
out.wraw(tok, tok_len ());
|
|
}
|
|
tok = pos = cur;
|
|
cline++;
|
|
goto echo;
|
|
}
|
|
zero {
|
|
if (!ignore_eoc && opts->target == opt_t::CODE)
|
|
{
|
|
out.wraw(tok, tok_len () - 1);
|
|
// -1 so we don't write out the \0
|
|
}
|
|
if(cur == eof)
|
|
{
|
|
return Stop;
|
|
}
|
|
}
|
|
* {
|
|
goto echo;
|
|
}
|
|
*/
|
|
}
|
|
|
|
int Scanner::scan()
|
|
{
|
|
uint32_t depth;
|
|
|
|
scan:
|
|
tchar = cur - pos;
|
|
tline = cline;
|
|
tok = cur;
|
|
switch (lexer_state)
|
|
{
|
|
case LEX_NORMAL: goto start;
|
|
case LEX_FLEX_NAME: goto flex_name;
|
|
}
|
|
|
|
start:
|
|
/*!re2c
|
|
"{" {
|
|
depth = 1;
|
|
goto code;
|
|
}
|
|
|
|
":" / "=>" {
|
|
return *tok;
|
|
}
|
|
|
|
":=" {
|
|
tok += 2; /* skip ":=" */
|
|
depth = 0;
|
|
goto code;
|
|
}
|
|
|
|
"//" {
|
|
goto nextLine;
|
|
}
|
|
"/*" {
|
|
depth = 1;
|
|
goto comment;
|
|
}
|
|
|
|
endRE = "%}" | "*/";
|
|
endRE {
|
|
tok = cur;
|
|
return 0;
|
|
}
|
|
|
|
"'" { yylval.regexp = lex_str('\'', opts->bCaseInsensitive || !opts->bCaseInverted); return TOKEN_REGEXP; }
|
|
"\"" { yylval.regexp = lex_str('"', opts->bCaseInsensitive || opts->bCaseInverted); return TOKEN_REGEXP; }
|
|
"[" { yylval.regexp = lex_cls(false); return TOKEN_REGEXP; }
|
|
"[^" { yylval.regexp = lex_cls(true); return TOKEN_REGEXP; }
|
|
|
|
"<>" / (space* ("{" | "=>" | ":=")) {
|
|
return TOKEN_NOCOND;
|
|
}
|
|
"<!" {
|
|
return TOKEN_SETUP;
|
|
}
|
|
[<>,()|=;/\\] {
|
|
return *tok;
|
|
}
|
|
|
|
"*" {
|
|
yylval.op = *tok;
|
|
return TOKEN_STAR;
|
|
}
|
|
[+?] {
|
|
yylval.op = *tok;
|
|
return TOKEN_CLOSE;
|
|
}
|
|
|
|
"{" [0-9]+ "}" {
|
|
if (!s_to_u32_unsafe (tok + 1, cur - 1, yylval.extop.min))
|
|
{
|
|
fatal ("repetition count overflow");
|
|
}
|
|
yylval.extop.max = yylval.extop.min;
|
|
return TOKEN_CLOSESIZE;
|
|
}
|
|
|
|
"{" [0-9]+ "," [0-9]+ "}" {
|
|
const char * p = strchr (tok, ',');
|
|
if (!s_to_u32_unsafe (tok + 1, p, yylval.extop.min))
|
|
{
|
|
fatal ("repetition lower bound overflow");
|
|
}
|
|
if (!s_to_u32_unsafe (p + 1, cur - 1, yylval.extop.max))
|
|
{
|
|
fatal ("repetition upper bound overflow");
|
|
}
|
|
return TOKEN_CLOSESIZE;
|
|
}
|
|
|
|
"{" [0-9]+ ",}" {
|
|
if (!s_to_u32_unsafe (tok + 1, cur - 2, yylval.extop.min))
|
|
{
|
|
fatal ("repetition lower bound overflow");
|
|
}
|
|
yylval.extop.max = std::numeric_limits<uint32_t>::max();
|
|
return TOKEN_CLOSESIZE;
|
|
}
|
|
|
|
"{" [0-9]* "," {
|
|
fatal("illegal closure form, use '{n}', '{n,}', '{n,m}' where n and m are numbers");
|
|
}
|
|
|
|
"{" name "}" {
|
|
if (!opts->FFlag) {
|
|
fatal("curly braces for names only allowed with -F switch");
|
|
}
|
|
yylval.str = new std::string (tok + 1, tok_len () - 2); // -2 to omit braces
|
|
return TOKEN_ID;
|
|
}
|
|
|
|
"re2c:" { lex_conf (); return TOKEN_CONF; }
|
|
|
|
name / (space+ [^=>,]) {
|
|
yylval.str = new std::string (tok, tok_len ());
|
|
if (opts->FFlag)
|
|
{
|
|
lexer_state = LEX_FLEX_NAME;
|
|
return TOKEN_FID;
|
|
}
|
|
else
|
|
{
|
|
return TOKEN_ID;
|
|
}
|
|
}
|
|
|
|
name / (space* [=>,]) {
|
|
yylval.str = new std::string (tok, tok_len ());
|
|
return TOKEN_ID;
|
|
}
|
|
|
|
name / [^] {
|
|
if (!opts->FFlag) {
|
|
yylval.str = new std::string (tok, tok_len());
|
|
return TOKEN_ID;
|
|
} else {
|
|
RegExp *r = NULL;
|
|
const bool casing = opts->bCaseInsensitive || opts->bCaseInverted;
|
|
for (char *s = tok; s < cur; ++s) {
|
|
const uint32_t c = static_cast<uint8_t>(*s);
|
|
r = doCat(r, casing ? ichr(c) : schr(c));
|
|
}
|
|
yylval.regexp = r ? r : new NullOp;
|
|
return TOKEN_REGEXP;
|
|
}
|
|
}
|
|
|
|
"." {
|
|
yylval.regexp = mkDot();
|
|
return TOKEN_REGEXP;
|
|
}
|
|
|
|
space+ {
|
|
goto scan;
|
|
}
|
|
|
|
eol space* "#" space* "line" space+ / lineinf {
|
|
set_sourceline ();
|
|
goto scan;
|
|
}
|
|
|
|
eol {
|
|
if (cur == eof) return 0;
|
|
pos = cur;
|
|
cline++;
|
|
goto scan;
|
|
}
|
|
|
|
* {
|
|
fatalf("unexpected character: '%c'", *tok);
|
|
goto scan;
|
|
}
|
|
*/
|
|
|
|
flex_name:
|
|
/*!re2c
|
|
eol
|
|
{
|
|
YYCURSOR = tok;
|
|
lexer_state = LEX_NORMAL;
|
|
return TOKEN_FID_END;
|
|
}
|
|
*
|
|
{
|
|
YYCURSOR = tok;
|
|
goto start;
|
|
}
|
|
*/
|
|
|
|
code:
|
|
/*!re2c
|
|
"}" {
|
|
if (depth == 0)
|
|
{
|
|
fatal("Curly braces are not allowed after ':='");
|
|
}
|
|
else if (--depth == 0)
|
|
{
|
|
yylval.code = new Code (tok, tok_len (), get_fname (), tline);
|
|
return TOKEN_CODE;
|
|
}
|
|
goto code;
|
|
}
|
|
"{" {
|
|
if (depth == 0)
|
|
{
|
|
fatal("Curly braces are not allowed after ':='");
|
|
}
|
|
else
|
|
{
|
|
++depth;
|
|
}
|
|
goto code;
|
|
}
|
|
"\n" space* "#" space* "line" space+ / lineinf {
|
|
set_sourceline ();
|
|
goto code;
|
|
}
|
|
"\n" / ws {
|
|
if (depth == 0)
|
|
{
|
|
goto code;
|
|
}
|
|
else if (cur == eof)
|
|
{
|
|
fatal("missing '}'");
|
|
}
|
|
pos = cur;
|
|
cline++;
|
|
goto code;
|
|
}
|
|
"\n" {
|
|
if (depth == 0)
|
|
{
|
|
tok += strspn(tok, " \t\r\n");
|
|
while (cur > tok && strchr(" \t\r\n", cur[-1]))
|
|
{
|
|
--cur;
|
|
}
|
|
yylval.code = new Code (tok, tok_len (), get_fname (), tline);
|
|
return TOKEN_CODE;
|
|
}
|
|
else if (cur == eof)
|
|
{
|
|
fatal("missing '}'");
|
|
}
|
|
pos = cur;
|
|
cline++;
|
|
goto code;
|
|
}
|
|
zero {
|
|
if (cur == eof)
|
|
{
|
|
if (depth)
|
|
{
|
|
fatal("missing '}'");
|
|
}
|
|
return 0;
|
|
}
|
|
goto code;
|
|
}
|
|
dstring | sstring {
|
|
goto code;
|
|
}
|
|
* {
|
|
goto code;
|
|
}
|
|
*/
|
|
|
|
comment:
|
|
/*!re2c
|
|
"*/" {
|
|
if (--depth == 0)
|
|
{
|
|
goto scan;
|
|
}
|
|
else
|
|
{
|
|
goto comment;
|
|
}
|
|
}
|
|
"/*" {
|
|
++depth;
|
|
fatal("ambiguous /* found");
|
|
goto comment;
|
|
}
|
|
"\n" space* "#" space* "line" space+ / lineinf {
|
|
set_sourceline ();
|
|
goto comment;
|
|
}
|
|
"\n" {
|
|
if (cur == eof)
|
|
{
|
|
return 0;
|
|
}
|
|
tok = pos = cur;
|
|
cline++;
|
|
goto comment;
|
|
}
|
|
* {
|
|
if (cur == eof)
|
|
{
|
|
return 0;
|
|
}
|
|
goto comment;
|
|
}
|
|
*/
|
|
|
|
nextLine:
|
|
/*!re2c /* resync emacs */
|
|
"\n" { if(cur == eof) {
|
|
return 0;
|
|
}
|
|
tok = pos = cur;
|
|
cline++;
|
|
goto scan;
|
|
}
|
|
* { if(cur == eof) {
|
|
return 0;
|
|
}
|
|
goto nextLine;
|
|
}
|
|
*/
|
|
}
|
|
|
|
static void escape (std::string & dest, const std::string & src)
|
|
{
|
|
dest = src;
|
|
size_t l = dest.length();
|
|
for (size_t p = 0; p < l; ++p)
|
|
{
|
|
if (dest[p] == '\\')
|
|
{
|
|
dest.insert(++p, "\\");
|
|
++l;
|
|
}
|
|
}
|
|
}
|
|
|
|
RegExp *Scanner::lex_cls(bool neg)
|
|
{
|
|
Range *r = NULL, *s;
|
|
uint32_t u, l;
|
|
fst:
|
|
/*!re2c
|
|
"]" { goto end; }
|
|
"" { l = lex_cls_chr(); goto snd; }
|
|
*/
|
|
snd:
|
|
/*!re2c
|
|
"" { u = l; goto add; }
|
|
"-" / [^\]] {
|
|
u = lex_cls_chr();
|
|
if (l > u) {
|
|
warn.swapped_range(get_line(), l, u);
|
|
std::swap(l, u);
|
|
}
|
|
goto add;
|
|
}
|
|
*/
|
|
add:
|
|
if (!(s = opts->encoding.encodeRange(l, u))) {
|
|
fatalf ("Bad code point range: '0x%X - 0x%X'", l, u);
|
|
}
|
|
r = Range::add(r, s);
|
|
goto fst;
|
|
end:
|
|
if (neg) {
|
|
r = Range::sub(opts->encoding.fullRange(), r);
|
|
}
|
|
return cls(r);
|
|
}
|
|
|
|
uint32_t Scanner::lex_cls_chr()
|
|
{
|
|
tok = cur;
|
|
/*!re2c
|
|
* { fatal ((tok - pos) - tchar, "syntax error"); }
|
|
esc [xXuU] { fatal ((tok - pos) - tchar, "syntax error in hexadecimal escape sequence"); }
|
|
esc [0-7] { fatal ((tok - pos) - tchar, "syntax error in octal escape sequence"); }
|
|
esc { fatal ((tok - pos) - tchar, "syntax error in escape sequence"); }
|
|
|
|
. \ esc { return static_cast<uint8_t>(tok[0]); }
|
|
esc_hex { return unesc_hex(tok, cur); }
|
|
esc_oct { return unesc_oct(tok, cur); }
|
|
esc "a" { return static_cast<uint8_t>('\a'); }
|
|
esc "b" { return static_cast<uint8_t>('\b'); }
|
|
esc "f" { return static_cast<uint8_t>('\f'); }
|
|
esc "n" { return static_cast<uint8_t>('\n'); }
|
|
esc "r" { return static_cast<uint8_t>('\r'); }
|
|
esc "t" { return static_cast<uint8_t>('\t'); }
|
|
esc "v" { return static_cast<uint8_t>('\v'); }
|
|
esc "\\" { return static_cast<uint8_t>('\\'); }
|
|
esc "-" { return static_cast<uint8_t>('-'); }
|
|
esc "]" { return static_cast<uint8_t>(']'); }
|
|
esc . {
|
|
warn.useless_escape(tline, tok - pos, tok[1]);
|
|
return static_cast<uint8_t>(tok[1]);
|
|
}
|
|
*/
|
|
}
|
|
|
|
uint32_t Scanner::lex_str_chr(char quote, bool &end)
|
|
{
|
|
end = false;
|
|
tok = cur;
|
|
/*!re2c
|
|
* { fatal ((tok - pos) - tchar, "syntax error"); }
|
|
esc [xXuU] { fatal ((tok - pos) - tchar, "syntax error in hexadecimal escape sequence"); }
|
|
esc [0-7] { fatal ((tok - pos) - tchar, "syntax error in octal escape sequence"); }
|
|
esc { fatal ((tok - pos) - tchar, "syntax error in escape sequence"); }
|
|
|
|
. \ esc {
|
|
end = tok[0] == quote;
|
|
return static_cast<uint8_t>(tok[0]);
|
|
}
|
|
esc_hex { return unesc_hex(tok, cur); }
|
|
esc_oct { return unesc_oct(tok, cur); }
|
|
esc "a" { return static_cast<uint8_t>('\a'); }
|
|
esc "b" { return static_cast<uint8_t>('\b'); }
|
|
esc "f" { return static_cast<uint8_t>('\f'); }
|
|
esc "n" { return static_cast<uint8_t>('\n'); }
|
|
esc "r" { return static_cast<uint8_t>('\r'); }
|
|
esc "t" { return static_cast<uint8_t>('\t'); }
|
|
esc "v" { return static_cast<uint8_t>('\v'); }
|
|
esc "\\" { return static_cast<uint8_t>('\\'); }
|
|
esc . {
|
|
if (tok[1] != quote) {
|
|
warn.useless_escape(tline, tok - pos, tok[1]);
|
|
}
|
|
return static_cast<uint8_t>(tok[1]);
|
|
}
|
|
*/
|
|
}
|
|
|
|
RegExp *Scanner::lex_str(char quote, bool casing)
|
|
{
|
|
RegExp *r = NULL;
|
|
for (bool end;;) {
|
|
const uint32_t c = lex_str_chr(quote, end);
|
|
if (end) {
|
|
return r ? r : new NullOp;
|
|
}
|
|
r = doCat(r, casing ? ichr(c) : schr(c));
|
|
}
|
|
}
|
|
|
|
void Scanner::set_sourceline ()
|
|
{
|
|
sourceline:
|
|
tok = cur;
|
|
/*!re2c
|
|
lineno {
|
|
if (!s_to_u32_unsafe (tok, cur, cline))
|
|
{
|
|
fatal ("line number overflow");
|
|
}
|
|
goto sourceline;
|
|
}
|
|
dstring {
|
|
escape (in.file_name, std::string (tok + 1, tok_len () - 2)); // -2 to omit quotes
|
|
goto sourceline;
|
|
}
|
|
"\n" {
|
|
if (cur == eof)
|
|
{
|
|
--cur;
|
|
}
|
|
else
|
|
{
|
|
pos = cur;
|
|
}
|
|
tok = cur;
|
|
return;
|
|
}
|
|
* {
|
|
goto sourceline;
|
|
}
|
|
*/
|
|
}
|
|
|
|
} // end namespace re2c
|