From ec364c53c10e44065e7d89773a6183d343665670 Mon Sep 17 00:00:00 2001 From: Randy Heit Date: Sun, 21 Feb 2010 04:33:08 +0000 Subject: [PATCH] - Update Lemon to the latest SQLite CVS versions: * lemon.c @ 1.69 ; lempar.c @ 1.45 SVN r2172 (scripting) --- tools/lemon/lemon.c | 290 ++++++++++++++++++++++++++----------------- tools/lemon/lempar.c | 127 ++++++++++++++----- 2 files changed, 276 insertions(+), 141 deletions(-) diff --git a/tools/lemon/lemon.c b/tools/lemon/lemon.c index f4a08f789..7fd95eb2d 100644 --- a/tools/lemon/lemon.c +++ b/tools/lemon/lemon.c @@ -6,9 +6,9 @@ ** ** The author of this program disclaims copyright. ** -** This file is based on version 1.43 of lemon.c from the SQLite -** source tree, with modifications to make it work nicer when run -** by Developer Studio. +** This file is based on version 1.69 of lemon.c from the SQLite +** CVS, with modifications to make it work nicer when run +** from Developer Studio. */ #include #include @@ -40,6 +40,13 @@ extern int access(); static void *msort(void *list, void *next, int (*cmp)()); +/* +** Compilers are getting increasingly pedantic about type conversions +** as C evolves ever closer to Ada.... To work around the latest problems +** we have to define the following variant of strlen(). +*/ +#define lemonStrlen(X) ((int)strlen(X)) + /******** From the file "action.h" *************************************/ static struct action *Action_new(void); static struct action *Action_sort(struct action *); @@ -133,13 +140,13 @@ struct symbol { RIGHT, NONE, UNK - } assoc; /* Associativity if predecence is defined */ + } assoc; /* Associativity if precedence is defined */ char *firstset; /* First-set for all rules of this symbol */ Boolean lambda; /* True if NT and can generate an empty string */ int useCnt; /* Number of times used */ char *destructor; /* Code which executes whenever this symbol is ** popped from the stack during error processing */ - int destructorln; /* Line number of destructor code */ + int destLineno; /* Line number for start of destructor */ char *datatype; /* The data type of information held by this ** object. Only used if type==NONTERMINAL */ int dtnum; /* The data type number. In the parser, the value @@ -217,7 +224,7 @@ struct action { struct state { struct config *bp; /* The basis configurations for this state */ struct config *cfp; /* All configurations in this set */ - int statenum; /* Sequencial number for this state */ + int statenum; /* Sequential number for this state */ struct action *ap; /* Array of actions for this state */ int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ @@ -255,28 +262,21 @@ struct lemon { char *start; /* Name of the start symbol for the grammar */ char *stacksize; /* Size of the parser stack */ char *include; /* Code to put at the start of the C file */ - int includeln; /* Line number for start of include code */ char *error; /* Code to execute when an error is seen */ - int errorln; /* Line number for start of error code */ char *overflow; /* Code to execute on a stack overflow */ - int overflowln; /* Line number for start of overflow code */ char *failure; /* Code to execute on parser failure */ - int failureln; /* Line number for start of failure code */ char *accept; /* Code to execute when the parser excepts */ - int acceptln; /* Line number for the start of accept code */ char *extracode; /* Code appended to the generated file */ - int extracodeln; /* Line number for the start of the extra code */ char *tokendest; /* Code to execute to destroy token data */ - int tokendestln; /* Line number for token destroyer code */ char *vardest; /* Code for the default non-terminal destructor */ - int vardestln; /* Line number for default non-term destructor code*/ char *filename; /* Name of the input file */ char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ int nconflict; /* Number of parsing conflicts */ int tablesize; /* Size of the parse tables */ int basisflag; /* Print only basis configurations */ - int has_fallback; /* True if any %fallback is seen in the grammer */ + int has_fallback; /* True if any %fallback is seen in the grammar */ + int nolinenosflag; /* True if #line statements should not be printed */ char *argv0; /* Name of the program */ }; @@ -997,7 +997,7 @@ struct lemon *lemp; } /* Resolve a conflict between the two given actions. If the -** conflict can't be resolve, return non-zero. +** conflict can't be resolved, return non-zero. ** ** NO LONGER TRUE: ** To resolve a conflict, first look to see if either action @@ -1344,13 +1344,13 @@ void ErrorMsg(const char *filename, int lineno, const char *format, ...){ sprintf(prefix,"%.*s: ",PREFIXLIMIT-10,filename); } #endif - prefixsize = strlen(prefix); + prefixsize = lemonStrlen(prefix); availablewidth = LINEWIDTH - prefixsize; /* Generate the error message */ vsprintf(errmsg,format,ap); va_end(ap); - errmsgsize = strlen(errmsg); + errmsgsize = lemonStrlen(errmsg); /* Remove trailing '\n's from the error message. */ while( errmsgsize>0 && errmsg[errmsgsize-1]=='\n' ){ errmsg[--errmsgsize] = 0; @@ -1394,7 +1394,7 @@ static void handle_D_option(char *z){ exit(1); } paz = &azDefine[nDefine-1]; - *paz = malloc( strlen(z)+1 ); + *paz = malloc( lemonStrlen(z)+1 ); if( *paz==0 ){ fprintf(stderr,"out of memory\n"); exit(1); @@ -1417,12 +1417,14 @@ char **argv; static int quiet = 0; static int statistics = 0; static int mhflag = 0; + static int nolinenosflag = 0; static struct s_options options[] = { {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."}, {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."}, {OPT_FSTR, "D", (char*)handle_D_option, "Define an %ifdef macro."}, {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."}, - {OPT_FLAG, "m", (char*)&mhflag, "Output a makeheaders compatible file"}, + {OPT_FLAG, "m", (char*)&mhflag, "Output a makeheaders compatible file."}, + {OPT_FLAG, "l", (char*)&nolinenosflag, "Do not print #line statements."}, {OPT_FLAG, "q", (char*)&quiet, "(Quiet) Don't print the report file."}, {OPT_FLAG, "s", (char*)&statistics, "Print parser stats to standard output."}, @@ -1451,6 +1453,7 @@ char **argv; lem.argv0 = argv[0]; lem.filename = OptArg(0); lem.basisflag = basisflag; + lem.nolinenosflag = nolinenosflag; Symbol_new("$"); lem.errsym = Symbol_new("error"); lem.errsym->useCnt = 0; @@ -1666,10 +1669,10 @@ FILE *err; int i; size_t spcnt; if( argv[0] ) fprintf(err,"%s",argv[0]); - spcnt = strlen(argv[0]) + 1; + spcnt = lemonStrlen(argv[0]) + 1; for(i=1; i%*s %s\n",op[i].label, - (int)(max-strlen(op[i].label)-9),"",op[i].message); + (int)(max-lemonStrlen(op[i].label)-9),"",op[i].message); break; case OPT_DBL: case OPT_FDBL: fprintf(errstream," %s=%*s %s\n",op[i].label, - (int)(max-strlen(op[i].label)-6),"",op[i].message); + (int)(max-lemonStrlen(op[i].label)-6),"",op[i].message); break; case OPT_STR: case OPT_FSTR: fprintf(errstream," %s=%*s %s\n",op[i].label, - (int)(max-strlen(op[i].label)-8),"",op[i].message); + (int)(max-lemonStrlen(op[i].label)-8),"",op[i].message); break; } } @@ -1978,7 +1981,8 @@ struct pstate { struct rule *prevrule; /* Previous rule parsed */ char *declkeyword; /* Keyword of a declaration */ char **declargslot; /* Where the declaration argument should be put */ - int *decllnslot; /* Where the declaration linenumber is put */ + int insertLineMacro; /* Add #line before declaration insert */ + int *decllinenoslot; /* Where to write declaration line number */ enum e_assoc declassoc; /* Assign this association to decl arguments */ int preccounter; /* Assign this precedence to decl arguments */ struct rule *firstrule; /* Pointer to first rule in the grammar */ @@ -2013,7 +2017,7 @@ struct pstate *psp; }else if( x[0]=='{' ){ if( psp->prevrule==0 ){ ErrorMsg(psp->filename,psp->tokenlineno, -"There is not prior rule opon which to attach the code \ +"There is no prior rule upon which to attach the code \ fragment which begins on this line."); psp->errorcnt++; }else if( psp->prevrule->code!=0 ){ @@ -2211,46 +2215,46 @@ to follow the previous rule."); if( isalpha(x[0]) ){ psp->declkeyword = x; psp->declargslot = 0; - psp->decllnslot = 0; + psp->decllinenoslot = 0; + psp->insertLineMacro = 1; psp->state = WAITING_FOR_DECL_ARG; if( strcmp(x,"name")==0 ){ psp->declargslot = &(psp->gp->name); + psp->insertLineMacro = 0; }else if( strcmp(x,"include")==0 ){ psp->declargslot = &(psp->gp->include); - psp->decllnslot = &psp->gp->includeln; }else if( strcmp(x,"code")==0 ){ psp->declargslot = &(psp->gp->extracode); - psp->decllnslot = &psp->gp->extracodeln; }else if( strcmp(x,"token_destructor")==0 ){ psp->declargslot = &psp->gp->tokendest; - psp->decllnslot = &psp->gp->tokendestln; }else if( strcmp(x,"default_destructor")==0 ){ psp->declargslot = &psp->gp->vardest; - psp->decllnslot = &psp->gp->vardestln; }else if( strcmp(x,"token_prefix")==0 ){ psp->declargslot = &psp->gp->tokenprefix; + psp->insertLineMacro = 0; }else if( strcmp(x,"syntax_error")==0 ){ psp->declargslot = &(psp->gp->error); - psp->decllnslot = &psp->gp->errorln; }else if( strcmp(x,"parse_accept")==0 ){ psp->declargslot = &(psp->gp->accept); - psp->decllnslot = &psp->gp->acceptln; }else if( strcmp(x,"parse_failure")==0 ){ psp->declargslot = &(psp->gp->failure); - psp->decllnslot = &psp->gp->failureln; }else if( strcmp(x,"stack_overflow")==0 ){ psp->declargslot = &(psp->gp->overflow); - psp->decllnslot = &psp->gp->overflowln; }else if( strcmp(x,"extra_argument")==0 ){ psp->declargslot = &(psp->gp->arg); + psp->insertLineMacro = 0; }else if( strcmp(x,"token_type")==0 ){ psp->declargslot = &(psp->gp->tokentype); + psp->insertLineMacro = 0; }else if( strcmp(x,"default_type")==0 ){ psp->declargslot = &(psp->gp->vartype); + psp->insertLineMacro = 0; }else if( strcmp(x,"stack_size")==0 ){ psp->declargslot = &(psp->gp->stacksize); + psp->insertLineMacro = 0; }else if( strcmp(x,"start_symbol")==0 ){ psp->declargslot = &(psp->gp->start); + psp->insertLineMacro = 0; }else if( strcmp(x,"left")==0 ){ psp->preccounter++; psp->declassoc = LEFT; @@ -2294,7 +2298,8 @@ to follow the previous rule."); }else{ struct symbol *sp = Symbol_new(x); psp->declargslot = &sp->destructor; - psp->decllnslot = &sp->destructorln; + psp->decllinenoslot = &sp->destLineno; + psp->insertLineMacro = 1; psp->state = WAITING_FOR_DECL_ARG; } break; @@ -2307,7 +2312,7 @@ to follow the previous rule."); }else{ struct symbol *sp = Symbol_new(x); psp->declargslot = &sp->datatype; - psp->decllnslot = 0; + psp->insertLineMacro = 0; psp->state = WAITING_FOR_DECL_ARG; } break; @@ -2332,18 +2337,56 @@ to follow the previous rule."); } break; case WAITING_FOR_DECL_ARG: - if( (x[0]=='{' || x[0]=='\"' || isalnum(x[0])) ){ - if( *(psp->declargslot)!=0 ){ - ErrorMsg(psp->filename,psp->tokenlineno, - "The argument \"%s\" to declaration \"%%%s\" is not the first.", - x[0]=='\"' ? &x[1] : x,psp->declkeyword); - psp->errorcnt++; - psp->state = RESYNC_AFTER_DECL_ERROR; - }else{ - *(psp->declargslot) = (x[0]=='\"' || x[0]=='{') ? &x[1] : x; - if( psp->decllnslot ) *psp->decllnslot = psp->tokenlineno; - psp->state = WAITING_FOR_DECL_OR_RULE; - } + if( x[0]=='{' || x[0]=='\"' || isalnum(x[0]) ){ + char *zOld, *zNew, *zBuf, *z; + int nOld, n, nLine, nNew, nBack; + int addLineMacro; + char zLine[50]; + zNew = x; + if( zNew[0]=='"' || zNew[0]=='{' ) zNew++; + nNew = lemonStrlen(zNew); + if( *psp->declargslot ){ + zOld = *psp->declargslot; + }else{ + zOld = ""; + } + nOld = lemonStrlen(zOld); + n = nOld + nNew + 20; + addLineMacro = !psp->gp->nolinenosflag && psp->insertLineMacro && + (psp->decllinenoslot==0 || psp->decllinenoslot[0]!=0); + if( addLineMacro ){ + for(z=psp->filename, nBack=0; *z; z++){ + if( *z=='\\' ) nBack++; + } + sprintf(zLine, "#line %d ", psp->tokenlineno); + nLine = lemonStrlen(zLine); + n += nLine + lemonStrlen(psp->filename) + nBack; + } + *psp->declargslot = zBuf = realloc(*psp->declargslot, n); + zBuf += nOld; + if( addLineMacro ){ + if( nOld && zBuf[-1]!='\n' ){ + *(zBuf++) = '\n'; + } + memcpy(zBuf, zLine, nLine); + zBuf += nLine; + *(zBuf++) = '"'; + for(z=psp->filename; *z; z++){ + if( *z=='\\' ){ + *(zBuf++) = '\\'; + } + *(zBuf++) = *z; + } + *(zBuf++) = '"'; + *(zBuf++) = '\n'; + } + if( psp->decllinenoslot && psp->decllinenoslot[0]==0 ){ + psp->decllinenoslot[0] = psp->tokenlineno; + } + memcpy(zBuf, zNew, nNew); + zBuf += nNew; + *zBuf = 0; + psp->state = WAITING_FOR_DECL_OR_RULE; }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Illegal argument to %%%s: %s",psp->declkeyword,x); @@ -2400,7 +2443,7 @@ to follow the previous rule."); } } -/* Run the proprocessor over the input file text. The global variables +/* Run the preprocessor over the input file text. The global variables ** azDefine[0] through azDefine[nDefine-1] contains the names of all defined ** macros. This routine looks for "%ifdef" and "%ifndef" and "%endif" and ** comments them out. Text in between is also commented out as appropriate. @@ -2431,7 +2474,7 @@ static void preprocess_input(char *z){ for(n=0; z[j+n] && !isspace(z[j+n]); n++){} exclude = 1; for(k=0; kfilename) + strlen(suffix) + 5 ); + name = malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5 ); if( name==0 ){ fprintf(stderr,"Can't allocate space for a filename.\n"); exit(1); @@ -2748,7 +2791,7 @@ struct lemon *lemp; maxlen = 10; for(i=0; insymbol; i++){ sp = lemp->symbols[i]; - len = (int)strlen(sp->name); + len = lemonStrlen(sp->name); if( len>maxlen ) maxlen = len; } ncolumns = 76/(maxlen+5); @@ -2956,7 +2999,7 @@ int modemask; char c; #ifdef __WIN32__ - for (cp = argv0 + strlen(argv0); cp-- > argv0; ) + for (cp = argv0 + lemonStrlen(argv0); cp-- > argv0; ) { if( *cp == '\\' || *cp == '/' ) break; @@ -2967,18 +3010,18 @@ int modemask; if( cp ){ c = *cp; *cp = 0; - path = (char *)malloc( strlen(argv0) + strlen(name) + 2 ); + path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 ); if( path ) sprintf(path,"%s/%s",argv0,name); *cp = c; }else{ extern char *getenv(); pathlist = getenv("PATH"); if( pathlist==0 ) pathlist = ".:/bin:/usr/bin"; - path = (char *)malloc( strlen(pathlist)+strlen(name)+2 ); + path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); if( path!=0 ){ while( *pathlist ){ cp = strchr(pathlist,':'); - if( cp==0 ) cp = &pathlist[strlen(pathlist)]; + if( cp==0 ) cp = &pathlist[lemonStrlen(pathlist)]; c = *cp; *cp = 0; sprintf(path,"%s/%s",pathlist,name); @@ -3103,16 +3146,13 @@ char *filename; } /* Print a string to the file and keep the linenumber up to date */ -PRIVATE void tplt_print(out,lemp,str,strln,lineno) +PRIVATE void tplt_print(out,lemp,str,lineno) FILE *out; struct lemon *lemp; char *str; -int strln; int *lineno; { if( str==0 ) return; - tplt_linedir(out,strln,lemp->filename); - (*lineno)++; while( *str ){ if( *str=='\n' ) (*lineno)++; putc(*str,out); @@ -3122,8 +3162,10 @@ int *lineno; putc('\n',out); (*lineno)++; } - tplt_linedir(out,*lineno+1,lemp->outname); - (*lineno)+=1; + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + return; } @@ -3139,21 +3181,18 @@ int *lineno; { char *cp = 0; - int linecnt = 0; if( sp->type==TERMINAL ){ cp = lemp->tokendest; if( cp==0 ) return; - tplt_linedir(out,lemp->tokendestln,lemp->filename); - fprintf(out,"{"); + fprintf(out,"{\n"); (*lineno)++; }else if( sp->destructor ){ cp = sp->destructor; - tplt_linedir(out,sp->destructorln,lemp->filename); - fprintf(out,"{"); + fprintf(out,"{\n"); (*lineno)++; + if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,sp->destLineno,lemp->filename); } }else if( lemp->vardest ){ cp = lemp->vardest; if( cp==0 ) return; - tplt_linedir(out,lemp->vardestln,lemp->filename); - fprintf(out,"{"); + fprintf(out,"{\n"); (*lineno)++; }else{ assert( 0 ); /* Cannot happen */ } @@ -3163,12 +3202,14 @@ int *lineno; cp++; continue; } - if( *cp=='\n' ) linecnt++; + if( *cp=='\n' ) (*lineno)++; fputc(*cp,out); } - (*lineno) += 3 + linecnt; - fprintf(out,"}\n"); - tplt_linedir(out,*lineno,lemp->outname); + fprintf(out,"\n"); (*lineno)++; + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + fprintf(out,"}\n"); (*lineno)++; return; } @@ -3216,7 +3257,7 @@ PRIVATE char *append_str(char *zText, int n, int p1, int p2, int bNoSubst){ used += n; assert( used>=0 ); } - n = (int)strlen(zText); + n = lemonStrlen(zText); } if( n+sizeof(zInt)*2+used >= (size_t)alloced ){ alloced = n + sizeof(zInt)*2 + used + 200; @@ -3229,7 +3270,7 @@ PRIVATE char *append_str(char *zText, int n, int p1, int p2, int bNoSubst){ sprintf(zInt, "%d", p1); p1 = p2; strcpy(&z[used], zInt); - used += (int)strlen(&z[used]); + used += lemonStrlen(&z[used]); zText++; n--; }else{ @@ -3316,7 +3357,7 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){ lemp->errorcnt++; }else if( rp->rhsalias[i]==0 ){ if( has_destructor(rp->rhs[i],lemp) ){ - append_str(" yy_destructor(%d,&yymsp[%d].minor);\n", 0, + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, rp->rhs[i]->index,i-rp->nrhs+1,0); }else{ /* No destructor defined for this term */ @@ -3340,18 +3381,16 @@ struct lemon *lemp; int *lineno; { char *cp; - int linecnt = 0; /* Generate code to do the reduce action */ if( rp->code ){ - tplt_linedir(out,rp->line,lemp->filename); + if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,rp->line,lemp->filename); } fprintf(out,"{%s",rp->code); for(cp=rp->code; *cp; cp++){ - if( *cp=='\n' ) linecnt++; + if( *cp=='\n' ) (*lineno)++; } /* End loop */ - (*lineno) += 3 + linecnt; - fprintf(out,"}\n"); - tplt_linedir(out,*lineno,lemp->outname); + fprintf(out,"}\n"); (*lineno)++; + if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } } /* End if( rp->code ) */ return; @@ -3384,13 +3423,13 @@ int mhflag; /* True if generating makeheaders output */ types = (char**)calloc( arraysize, sizeof(char*) ); maxdtlength = 0; if( lemp->vartype ){ - maxdtlength = (int)strlen(lemp->vartype); + maxdtlength = lemonStrlen(lemp->vartype); } for(i=0; insymbol; i++){ int len; struct symbol *sp = lemp->symbols[i]; if( sp->datatype==0 ) continue; - len = (int)strlen(sp->datatype); + len = lemonStrlen(sp->datatype); if( len>maxdtlength ) maxdtlength = len; } stddt = (char*)malloc( maxdtlength*2 + 1 ); @@ -3423,6 +3462,10 @@ int mhflag; /* True if generating makeheaders output */ while( *cp ) stddt[j++] = *cp++; while( j>0 && isspace(stddt[j-1]) ) j--; stddt[j] = 0; + if( lemp->tokentype && strcmp(stddt, lemp->tokentype)==0 ){ + sp->dtnum = 0; + continue; + } hash = 0; for(j=0; stddt[j]; j++){ hash = hash*53 + stddt[j]; @@ -3438,7 +3481,7 @@ int mhflag; /* True if generating makeheaders output */ } if( types[hash]==0 ){ sp->dtnum = hash + 1; - types[hash] = (char*)malloc( strlen(stddt)+1 ); + types[hash] = (char*)malloc( lemonStrlen(stddt)+1 ); if( types[hash]==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); @@ -3455,6 +3498,7 @@ int mhflag; /* True if generating makeheaders output */ lemp->tokentype?lemp->tokentype:"void*"); lineno++; if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } fprintf(out,"typedef union {\n"); lineno++; + fprintf(out," int yyinit;\n"); lineno++; fprintf(out," %sTOKENTYPE yy0;\n",name); lineno++; for(i=0; iname,in,out,&lineno); /* Generate the include code, if any */ - tplt_print(out,lemp,lemp->include,lemp->includeln,&lineno); + tplt_print(out,lemp,lemp->include,&lineno); if( mhflag ){ char *name = file_makename(lemp, ".h"); fprintf(out,"#include \"%s\"\n", name); lineno++; @@ -3585,7 +3629,7 @@ int mhflag; /* Output in makeheaders format if true */ /* Generate the defines */ fprintf(out,"#define YYCODETYPE %s\n", - minimum_size_type(0, lemp->nsymbol+5)); lineno++; + minimum_size_type(0, lemp->nsymbol+1)); lineno++; fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1); lineno++; fprintf(out,"#define YYACTIONTYPE %s\n", minimum_size_type(0, lemp->nstate+lemp->nrule+5)); lineno++; @@ -3607,7 +3651,7 @@ int mhflag; /* Output in makeheaders format if true */ name = lemp->name ? lemp->name : "Parse"; if( lemp->arg && lemp->arg[0] ){ size_t i; - i = strlen(lemp->arg); + i = lemonStrlen(lemp->arg); while( i>=1 && isspace(lemp->arg[i-1]) ) i--; while( i>=1 && (isalnum(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--; fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++; @@ -3800,7 +3844,9 @@ int mhflag; /* Output in makeheaders format if true */ /* Generate the table of fallback tokens. */ if( lemp->has_fallback ){ - for(i=0; interminal; i++){ + int mx = lemp->nterminal - 1; + while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } + for(i=0; i<=mx; i++){ struct symbol *p = lemp->symbols[i]; if( p->fallback==0 ){ fprintf(out, " 0, /* %10s => nothing */\n", p->name); @@ -3824,7 +3870,7 @@ int mhflag; /* Output in makeheaders format if true */ tplt_xfer(lemp->name,in,out,&lineno); /* Generate a table containing a text string that describes every - ** rule in the rule set of the grammer. This information is used + ** rule in the rule set of the grammar. This information is used ** when tracing REDUCE actions. */ for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ @@ -3840,11 +3886,15 @@ int mhflag; /* Output in makeheaders format if true */ ** (In other words, generate the %destructor actions) */ if( lemp->tokendest ){ + int once = 1; for(i=0; insymbol; i++){ struct symbol *sp = lemp->symbols[i]; if( sp==0 || sp->type!=TERMINAL ) continue; - fprintf(out," case %d: /* %s */\n", - sp->index, sp->name); lineno++; + if( once ){ + fprintf(out, " /* TERMINAL Destructor */\n"); lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; } for(i=0; insymbol && lemp->symbols[i]->type!=TERMINAL; i++); if( insymbol ){ @@ -3854,24 +3904,27 @@ int mhflag; /* Output in makeheaders format if true */ } if( lemp->vardest ){ struct symbol *dflt_sp = 0; + int once = 1; for(i=0; insymbol; i++){ struct symbol *sp = lemp->symbols[i]; if( sp==0 || sp->type==TERMINAL || sp->index<=0 || sp->destructor!=0 ) continue; - fprintf(out," case %d: /* %s */\n", - sp->index, sp->name); lineno++; + if( once ){ + fprintf(out, " /* Default NON-TERMINAL Destructor */\n"); lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; dflt_sp = sp; } if( dflt_sp!=0 ){ emit_destructor_code(out,dflt_sp,lemp,&lineno); - fprintf(out," break;\n"); lineno++; } + fprintf(out," break;\n"); lineno++; } for(i=0; insymbol; i++){ struct symbol *sp = lemp->symbols[i]; if( sp==0 || sp->type==TERMINAL || sp->destructor==0 ) continue; - fprintf(out," case %d: /* %s */\n", - sp->index, sp->name); lineno++; + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; /* Combine duplicate destructors into a single case */ for(j=i+1; jnsymbol; j++){ @@ -3891,7 +3944,7 @@ int mhflag; /* Output in makeheaders format if true */ tplt_xfer(lemp->name,in,out,&lineno); /* Generate code which executes whenever the parser stack overflows */ - tplt_print(out,lemp,lemp->overflow,lemp->overflowln,&lineno); + tplt_print(out,lemp,lemp->overflow,&lineno); tplt_xfer(lemp->name,in,out,&lineno); /* Generate the table of rule information @@ -3908,39 +3961,53 @@ int mhflag; /* Output in makeheaders format if true */ for(rp=lemp->rule; rp; rp=rp->next){ translate_code(lemp, rp); } + /* First output rules other than the default: rule */ for(rp=lemp->rule; rp; rp=rp->next){ - struct rule *rp2; + struct rule *rp2; /* Other rules with the same action */ if( rp->code==0 ) continue; + if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */ fprintf(out," case %d: /* ",rp->index); writeRuleText(out, rp); - fprintf(out, " */\n"); lineno++; + fprintf(out," */\n"); lineno++; for(rp2=rp->next; rp2; rp2=rp2->next){ if( rp2->code==rp->code ){ fprintf(out," case %d: /*",rp2->index); writeRuleText(out, rp2); - fprintf(out," */\n"); lineno++; + fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp2->index); lineno++; rp2->code = 0; } } emit_code(out,rp,lemp,&lineno); fprintf(out," break;\n"); lineno++; + rp->code = 0; } + /* Finally, output the default: rule. We choose as the default: all + ** empty actions. */ + fprintf(out," default:\n"); lineno++; + for(rp=lemp->rule; rp; rp=rp->next){ + if( rp->code==0 ) continue; + assert( rp->code[0]=='\n' && rp->code[1]==0 ); + fprintf(out," /* (%d) ", rp->index); + writeRuleText(out, rp); + fprintf(out," */ yytestcase(yyruleno==%d);\n", rp->index); lineno++; + } + fprintf(out," break;\n"); lineno++; tplt_xfer(lemp->name,in,out,&lineno); /* Generate code which executes if a parse fails */ - tplt_print(out,lemp,lemp->failure,lemp->failureln,&lineno); + tplt_print(out,lemp,lemp->failure,&lineno); tplt_xfer(lemp->name,in,out,&lineno); /* Generate code which executes when a syntax error occurs */ - tplt_print(out,lemp,lemp->error,lemp->errorln,&lineno); + tplt_print(out,lemp,lemp->error,&lineno); tplt_xfer(lemp->name,in,out,&lineno); /* Generate code which executes when the parser accepts its input */ - tplt_print(out,lemp,lemp->accept,lemp->acceptln,&lineno); + tplt_print(out,lemp,lemp->accept,&lineno); tplt_xfer(lemp->name,in,out,&lineno); /* Append any addition code the user desires */ - tplt_print(out,lemp,lemp->extracode,lemp->extracodeln,&lineno); + tplt_print(out,lemp,lemp->extracode,&lineno); fclose(in); fclose(out); @@ -4028,7 +4095,7 @@ struct lemon *lemp; /* Do not make a default if the number of rules to default ** is not at least 1 or if the wildcard token is a possbile - ** lookahed. + ** lookahead. */ if( nbest<1 || usesWildcard ) continue; @@ -4196,7 +4263,7 @@ char *y; if( y==0 ) return 0; z = Strsafe_find(y); - if( z==0 && (z=malloc( strlen(y)+1 ))!=0 ){ + if( z==0 && (z=malloc( lemonStrlen(y)+1 ))!=0 ){ strcpy(z,y); Strsafe_insert(z); } @@ -4343,6 +4410,7 @@ char *x; sp->firstset = 0; sp->lambda = LEMON_FALSE; sp->destructor = 0; + sp->destLineno = 0; sp->datatype = 0; sp->useCnt = 0; Symbol_insert(sp,sp->name); diff --git a/tools/lemon/lempar.c b/tools/lemon/lempar.c index c6c032ca9..6e9bf92c2 100644 --- a/tools/lemon/lempar.c +++ b/tools/lemon/lempar.c @@ -1,8 +1,8 @@ /* Driver template for the LEMON parser generator. ** The author disclaims copyright to this source code. */ -/* First off, code is included which follows the "include" declaration -** in the input file. */ +/* First off, code is included that follows the "include" declaration +** in the input grammar file. */ #include #include #include @@ -69,7 +69,24 @@ #define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) #define YY_ERROR_ACTION (YYNSTATE+YYNRULE) -/* Next are that tables used to determine what action to take based on the +/* The yyzerominor constant is used to initialize instances of +** YYMINORTYPE objects to zero. */ +static const YYMINORTYPE yyzerominor = { 0 }; + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement ** functions that take a state number and lookahead value and return an ** action integer. @@ -124,7 +141,7 @@ ** ** %fallback ID X Y Z. ** -** appears in the grammer, then ID becomes a fallback token for X, Y, +** appears in the grammar, then ID becomes a fallback token for X, Y, ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. @@ -148,11 +165,11 @@ static const YYCODETYPE yyFallback[] = { ** It is sometimes called the "minor" token. */ struct yyStackEntry { - int stateno; /* The state-number */ - int major; /* The major token value. This is the code - ** number for the token at this stack level */ - YYMINORTYPE minor; /* The user-supplied minor token value. This - ** is the value of the token */ + YYACTIONTYPE stateno; /* The state-number */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ }; typedef struct yyStackEntry yyStackEntry; @@ -160,6 +177,9 @@ typedef struct yyStackEntry yyStackEntry; ** the following structure */ struct yyParser { int yyidx; /* Index of top element in stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyidxMax; /* Maximum value of yyidx */ +#endif int yyerrcnt; /* Shifts left before out of the error */ ParseARG_SDECL /* A place to hold %extra_argument */ #if YYSTACKDEPTH<=0 @@ -259,7 +279,12 @@ void *ParseAlloc(void *(CDECL *mallocProc)(size_t)){ pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyidxMax = 0; +#endif #if YYSTACKDEPTH<=0 + pParser->yystack = NULL; + pParser->yystksz = 0; yyGrowStack(pParser); #endif } @@ -271,7 +296,12 @@ void *ParseAlloc(void *(CDECL *mallocProc)(size_t)){ ** "yymajor" is the symbol code, and "yypminor" is a pointer to ** the value. */ -static void yy_destructor(YYCODETYPE yymajor, YYMINORTYPE *yypminor){ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + ParseARG_FETCH; switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -309,7 +339,7 @@ static int yy_pop_parser_stack(yyParser *pParser){ } #endif yymajor = yytos->major; - yy_destructor( yymajor, &yytos->minor); + yy_destructor(pParser, yymajor, &yytos->minor); pParser->yyidx--; return yymajor; } @@ -339,6 +369,16 @@ void ParseFree( (*freeProc)((void*)pParser); } +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int ParseStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyidxMax; +} +#endif + /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. @@ -362,7 +402,7 @@ static int yy_find_shift_action( if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){ if( iLookAhead>0 ){ #ifdef YYFALLBACK - int iFallback; /* Fallback token */ + YYCODETYPE iFallback; /* Fallback token */ if( iLookAheadYY_REDUCE_MAX || - (i = yy_reduce_ofst[stateno])==YY_REDUCE_USE_DFLT ){ +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_MAX ){ return yy_default[stateno]; } +#else + assert( stateno<=YY_REDUCE_MAX ); +#endif + i = yy_reduce_ofst[stateno]; assert( i!=YY_REDUCE_USE_DFLT ); assert( iLookAhead!=YYNOCODE ); i += iLookAhead; +#ifdef YYERRORSYMBOL if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){ return yy_default[stateno]; - }else{ - return yy_action[i]; } +#else + assert( i>=0 && iyyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser - ** stack every overflows */ + ** stack ever overflows */ %% ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } @@ -448,10 +495,15 @@ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ - YYMINORTYPE *yypMinor /* Pointer ot the minor token to shift in */ + YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ ){ yyStackEntry *yytos; yypParser->yyidx++; +#ifdef YYTRACKMAXSTACKDEPTH + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; + } +#endif #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); @@ -467,8 +519,8 @@ static void yy_shift( } #endif yytos = &yypParser->yystack[yypParser->yyidx]; - yytos->stateno = yyNewState; - yytos->major = yyMajor; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; #ifndef NDEBUG if( yyTraceFILE && yypParser->yyidx>0 ){ @@ -531,8 +583,8 @@ static void yy_reduce( ** from wireshark this week. Clearly they are stressing Lemon in ways ** that it has not been previously stressed... (SQLite ticket #2172) */ - memset(&yygotominor, 0, sizeof(yygotominor)); - + /*memset(&yygotominor, 0, sizeof(yygotominor));*/ + yygotominor = yyzerominor; switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -548,7 +600,7 @@ static void yy_reduce( yygoto = yyRuleInfo[yyruleno].lhs; yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; - yyact = yy_find_reduce_action(yymsp[-yysize].stateno,yygoto); + yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); if( yyact < YYNSTATE ){ #ifdef NDEBUG /* If we are not debugging and the reduce action popped at least @@ -558,8 +610,8 @@ static void yy_reduce( if( yysize ){ yypParser->yyidx++; yymsp -= yysize-1; - yymsp->stateno = yyact; - yymsp->major = yygoto; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; }else #endif @@ -575,6 +627,7 @@ static void yy_reduce( /* ** The following code executes when the parse fails */ +#ifndef YYNOERRORRECOVERY static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ @@ -590,6 +643,7 @@ static void yy_parse_failed( %% ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } +#endif /* YYNOERRORRECOVERY */ /* ** The following code executes when a syntax error first occurs. @@ -662,7 +716,8 @@ void Parse( if( yypParser->yyidx<0 ){ #if YYSTACKDEPTH<=0 if( yypParser->yystksz <=0 ){ - memset(&yyminorunion, 0, sizeof(yyminorunion)); + /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ + yyminorunion = yyzerominor; yyStackOverflow(yypParser, &yyminorunion); return; } @@ -683,7 +738,7 @@ void Parse( #endif do{ - yyact = yy_find_shift_action(yypParser,yymajor); + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyactyyidx < 0 || yymajor==0 ){ - yy_destructor(yymajor,&yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ @@ -756,6 +811,18 @@ void Parse( } yypParser->yyerrcnt = 3; yyerrorhit = 1; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -770,7 +837,7 @@ void Parse( yy_syntax_error(yypParser,yymajor,yyminorunion); } yypParser->yyerrcnt = 3; - yy_destructor(yymajor,&yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); if( yyendofinput ){ yy_parse_failed(yypParser); }