diff --git a/TODO b/TODO index 0f06e37e..2799eca4 100644 --- a/TODO +++ b/TODO @@ -2,4 +2,8 @@ Things to be worked on: * Tests for %option user-init, %option pre-action, %option post-action. +* integrate examples directory into tests so that normal testing + proves the examples are up to date. +* Do away with any need for -lfl by generating a default yywrap + if the user doesn't specify one. diff --git a/doc/flex.texi b/doc/flex.texi index 66790f89..8d3d5734 100644 --- a/doc/flex.texi +++ b/doc/flex.texi @@ -354,28 +354,7 @@ Here's another simple example: @cindex counting characters and lines; reentrant @example -@verbatim - %{ - int num_lines = 0, num_chars = 0; - %} - %option reentrant - %% - \n ++num_lines; ++num_chars; - . ++num_chars; - - %% - - int main() { - yyscan_t scanner; - - yylex_init ( &scanner ); - yylex ( scanner ); - yylex_destroy ( scanner ); - - printf( "# of lines = %d, # of chars = %d\n", - num_lines, num_chars ); - } -@end verbatim +@verbatiminclude ../examples/manual/example_r.lex @end example If you have looked at older versions of the Flex nanual, you might @@ -383,20 +362,7 @@ have seen a version of the above example that looked more like this: @cindex counting characters and lines; non-reentrant @example -@verbatim - int num_lines = 0, num_chars = 0; - %% - \n ++num_lines; ++num_chars; - . ++num_chars; - - %% - - int main() { - yylex(); - printf( "# of lines = %d, # of chars = %d\n", - num_lines, num_chars ); - } -@end verbatim +@verbatiminclude ../examples/manual/example_nr.lex @end example Both versions count the number of characters and the number of lines in @@ -422,13 +388,24 @@ language other than the original C/C++ non-reentrancy is not even an option. This, it's a good idea to get used to using the reentrant interface -from the beginning of your Flex prgramming. This is so even though the +from the beginning of your Flex programming. This is so even though the reentrant example above is a rather poor one; it avoids exposing the scanner state in globals but creates globals of its own. There is a mechanism for including user-defined fields in the scanner structure which will be explained in detail at @xref{Extra Data}. For now, consider this: +@example +@verbatiminclude ../examples/manual/example_er.lex +@end example + +While it requires a bit more ceremony, several instances of this +scanner can be run concurrently without stepping on each others' +storage. + +(The @code{%option noyywrap} in these examples is helpful in +making them run standalone, but does not change the behavior of the scsnner.) + A somewhat more complicated example: @cindex Pascal-like language @@ -596,7 +573,8 @@ themselves. A @code{%top} block is similar to a @samp{%@{} ... @samp{%@}} block, except that the code in a @code{%top} block is relocated to the @emph{top} of the -generated file, before any flex definitions @footnote{Actually, +generated file, before any flex definitions @footnote{Actually, in the +C/C++ back end, @code{yyIN_HEADER} is defined before the @samp{%top} block.}. The @code{%top} block is useful when you want definitions to be evaluated or certain files to be included before the generated code. @@ -1642,7 +1620,7 @@ condition remains unchanged; it does @emph{not} revert to @cindex yywrap, default for @cindex noyywrap, %option -@cindex %option noyywrapp +@cindex %option noyywrap If you do not supply your own version of @code{yywrap()}, then you must either use @code{%option noyywrap} (in which case the scanner behaves as though @code{yywrap()} returned 1), or you must link with @samp{-lfl} to diff --git a/examples/manual/Makefile.am b/examples/manual/Makefile.am index 37d91c83..24affe1b 100644 --- a/examples/manual/Makefile.am +++ b/examples/manual/Makefile.am @@ -30,6 +30,7 @@ EXTRA_DIST = \ eof_test01.txt \ eof_test02.txt \ eof_test03.txt \ + example_er.lex \ example_r.lex \ example_nr.lex \ expr.lex \ diff --git a/examples/manual/Makefile.examples b/examples/manual/Makefile.examples index 57a17c8b..472653e0 100644 --- a/examples/manual/Makefile.examples +++ b/examples/manual/Makefile.examples @@ -18,7 +18,7 @@ ALLOCA = # DO NOT CHANGE ANYTHING FROM HERE ON !!!!!!!!! # ############################################################ -PATH = ${PATH}:/usr/local/bin +PATH := /usr/local/bin:${PATH} all: expr front myname eof wc replace user_act string1\ string2 yymore numbers dates cat @@ -31,6 +31,10 @@ example_nr: example_nr.lex $(LEX) example_nr.lex $(CC) lex.yy.c -o example_nr +example_er: example_er.lex + $(LEX) example_er.lex + $(CC) lex.yy.c -o example_er + expr: expr.y expr.lex $(YACC) expr.y $(LEX) expr.lex diff --git a/examples/manual/dates.lex b/examples/manual/dates.lex index 9429e1db..f9c648d0 100644 --- a/examples/manual/dates.lex +++ b/examples/manual/dates.lex @@ -54,13 +54,13 @@ day_ext (st|nd|rd|th)? /* the default is month-day-year */ {day_of_the_week} strcpy(dow,yytext); -{month} strcpy(month,yytext); BEGIN(DAY); +{month} strcpy(month,yytext); yybegin(DAY); /* handle the form: day-month-year */ -{nday}{day_ext} strcpy(day,yytext); BEGIN(DAY_FIRST); -{month} strcpy(month,yytext); BEGIN(LONG); -{nday}{day_ext} strcpy(day,yytext); BEGIN(LONG); +{nday}{day_ext} strcpy(day,yytext); yybegin(DAY_FIRST); +{month} strcpy(month,yytext); yybegin(LONG); +{nday}{day_ext} strcpy(day,yytext); yybegin(LONG); {nyear}{year_ext} { printf("Long:\n"); @@ -75,15 +75,15 @@ day_ext (st|nd|rd|th)? /* handle dates of the form: day-month-year */ -{nday} strcpy(day,yytext); BEGIN(YEAR_LAST); -{nmonth} strcpy(month,yytext);BEGIN(YLMONTH); -{nyear} strcpy(year,yytext); BEGIN(SHORT); +{nday} strcpy(day,yytext); yybegin(YEAR_LAST); +{nmonth} strcpy(month,yytext);yybegin(YLMONTH); +{nyear} strcpy(year,yytext); yybegin(SHORT); /* handle dates of the form: year-month-day */ -{nyear} strcpy(year,yytext); BEGIN(YEAR_FIRST); -{nmonth} strcpy(month,yytext);BEGIN(YFMONTH); -{nday} strcpy(day,yytext); BEGIN(SHORT); +{nyear} strcpy(year,yytext); yybegin(YEAR_FIRST); +{nmonth} strcpy(month,yytext);yybegin(YFMONTH); +{nday} strcpy(day,yytext); yybegin(SHORT); \n { @@ -96,8 +96,8 @@ day_ext (st|nd|rd|th)? strcpy(month,""); } -long\n BEGIN(LONG); -short\n BEGIN(SHORT); +long\n yybegin(LONG); +short\n yybegin(SHORT); {skip}* \n diff --git a/examples/manual/eof_rules.lex b/examples/manual/eof_rules.lex index b575f2c0..e42fda71 100644 --- a/examples/manual/eof_rules.lex +++ b/examples/manual/eof_rules.lex @@ -17,8 +17,8 @@ int include_count = -1; %% -^"#include"[ \t]*\" BEGIN(INCLUDE); -\" BEGIN(INITIAL); +^"#include"[ \t]*\" yybegin(INCLUDE); +\" yybegin(INITIAL); [^\"]+ { /* get the include file name */ if ( include_count >= MAX_NEST){ fprintf( stderr, "Too many include files" ); @@ -35,7 +35,7 @@ int include_count = -1; yy_switch_to_buffer(yy_create_buffer(yyin,YY_BUF_SIZE)); - BEGIN(INITIAL); + yybegin(INITIAL); } <> { @@ -48,7 +48,7 @@ int include_count = -1; } else { yy_delete_buffer(include_stack[include_count--] ); yy_switch_to_buffer(include_stack[include_count] ); - BEGIN(INCLUDE); + yybegin(INCLUDE); } } [a-z]+ ECHO; diff --git a/examples/manual/example_er.lex b/examples/manual/example_er.lex new file mode 100644 index 00000000..284031c9 --- /dev/null +++ b/examples/manual/example_er.lex @@ -0,0 +1,35 @@ +/* basic example, fully reentrant thread-safe version */ +%{ + struct stats { + int num_lines; + int num_chars; + }; +%} +%option reentrant noyywrap +%option extra-type="struct stats" +%% +\n { + struct stats ns = yyget_extra(yyscanner); + ++ns.num_lines; ++ns.num_chars; + yyset_extra(ns, yyscanner); + } +. { + struct stats ns = yyget_extra(yyscanner); + ++ns.num_chars; + yyset_extra(ns, yyscanner); + } + +%% + +int main() { + yyscan_t scanner; + struct stats ns; + + yylex_init ( &scanner ); + yylex ( scanner ); + + ns = yyget_extra(scanner); + printf( "# of lines = %d, # of chars = %d\n", + ns.num_lines, ns.num_chars); + yylex_destroy ( scanner ); +} diff --git a/examples/manual/example_nr.lex b/examples/manual/example_nr.lex index ae9e83d4..8722b145 100644 --- a/examples/manual/example_nr.lex +++ b/examples/manual/example_nr.lex @@ -1,4 +1,8 @@ - int num_lines = 0, num_chars = 0; +/* basic example - non-reentrant version */ +%{ + int num_lines = 0, num_chars = 0; +%} +%option noyywrap %% \n ++num_lines; ++num_chars; . ++num_chars; diff --git a/examples/manual/example_r.lex b/examples/manual/example_r.lex index 54c7f981..54c94d94 100644 --- a/examples/manual/example_r.lex +++ b/examples/manual/example_r.lex @@ -1,7 +1,8 @@ +/* basic example - flawed reentrant version with global */ %{ int num_lines = 0, num_chars = 0; %} -%option reentrant +%option reentrant noyywrap %% \n ++num_lines; ++num_chars; . ++num_chars; diff --git a/examples/manual/front.lex b/examples/manual/front.lex index 449cb007..0737a374 100644 --- a/examples/manual/front.lex +++ b/examples/manual/front.lex @@ -3,9 +3,6 @@ #include #include "y.tab.h" /* this comes from bison */ -#define TRUE 1 -#define FALSE 0 - #define copy_and_return(token_type) { strcpy(yylval.name,yytext); \ return(token_type); } diff --git a/examples/manual/j2t.lex b/examples/manual/j2t.lex index 08fbd21d..eb09eedc 100644 --- a/examples/manual/j2t.lex +++ b/examples/manual/j2t.lex @@ -158,7 +158,7 @@ void write_block_header(char *type) printf("\n\n@table @b\n"); } -"Examples:"[^\.]+ ECHO; +"Examples:"[^\.]+ yyecho(); "*"[^*\n]+"*" { /* @emph{}(emphasized) text */ yytext[yyleng-1] = '\0'; @@ -205,12 +205,12 @@ void write_block_header(char *type) yyless(loop+1); statep++; states[statep] = EXAMPLE2; - BEGIN(EXAMPLE2); + yybegin(EXAMPLE2); } ^\n { printf("@end example\n\n"); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } /* @@ -231,7 +231,7 @@ void write_block_header(char *type) yyless(loop); statep++; states[statep] = ENUM; - BEGIN(ENUM); + yybegin(ENUM); } "@" printf("@@"); @@ -239,7 +239,7 @@ void write_block_header(char *type) printf(":\n\n@example\n"); statep++; states[statep] = EXAMPLE; - BEGIN(EXAMPLE); + yybegin(EXAMPLE); } @@ -250,7 +250,7 @@ void write_block_header(char *type) \n\n\n[ \t]+[^0-9] { printf("\n\n@end enumerate\n\n"); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } /* @@ -265,7 +265,7 @@ void write_block_header(char *type) yyless(2); statep++; states[statep] = LITEM2; - BEGIN(LITEM2); + yybegin(LITEM2); } ^":".+":" { (void)check_and_convert(&yytext[1]); @@ -275,9 +275,9 @@ void write_block_header(char *type) \n\n\n+[^:\n] { printf("\n\n@end itemize\n\n"); - ECHO; + yyecho(); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } /* @@ -300,7 +300,7 @@ void write_block_header(char *type) yyless(loop); statep++; states[statep] = LITEM; - BEGIN(LITEM); + yybegin(LITEM); } ^.+":" { (void)check_and_convert(yytext); @@ -318,7 +318,7 @@ void write_block_header(char *type) printf("@end itemize\n\n"); printf("%s",&buffer[loop+1]); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } /* @@ -338,27 +338,27 @@ void write_block_header(char *type) yyless((len-loop)+2); statep++; states[statep] = BITEM; - BEGIN(BITEM); + yybegin(BITEM); } ^" "*"*" { printf("@item"); statep++; states[statep] = BITEM_ITEM; - BEGIN(BITEM_ITEM); + yybegin(BITEM_ITEM); } "@" printf("@@"); ^\n { printf("@end itemize\n\n"); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } [^\:]* { printf(" @b{%s}\n\n",check_and_convert(yytext)); } ":" { statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } /* @@ -369,13 +369,13 @@ void write_block_header(char *type) (void)check_and_convert(&yytext[1]); statep++; states[statep] = HEADING; - BEGIN(HEADING); + yybegin(HEADING); } :[^\n] { printf("@item @b{%s}\n",buffer); write_underline(strlen(buffer),6,'~'); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } :\n"*"* { if(need_closing == TRUE){ @@ -385,7 +385,7 @@ void write_block_header(char *type) printf("@chapter %s\n",buffer); write_underline(strlen(buffer),9,'*'); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } :\n"="* { if(need_closing == TRUE){ @@ -395,7 +395,7 @@ void write_block_header(char *type) printf("@section %s\n",buffer); write_underline(strlen(buffer),9,'='); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } "@" printf("@@"); :\n"-"* { @@ -406,7 +406,7 @@ void write_block_header(char *type) printf("@subsection %s\n",buffer); write_underline(strlen(buffer),12,'-'); statep--; - BEGIN(states[statep]); + yybegin(states[statep]); } /* @@ -417,10 +417,10 @@ void write_block_header(char *type) printf("@example\n"); statep++; states[statep] = EXAMPLE; - BEGIN(EXAMPLE); + yybegin(EXAMPLE); } ^" " -. ECHO; +. yyecho(); %% diff --git a/examples/manual/pas_include.lex b/examples/manual/pas_include.lex index 58cf5903..117ad564 100644 --- a/examples/manual/pas_include.lex +++ b/examples/manual/pas_include.lex @@ -19,13 +19,13 @@ int include_count = -1; %% -"{" BEGIN(COMMENT); +"{" yybegin(COMMENT); -"}" BEGIN(INITIAL); -"$include"[ \t]*"(" BEGIN(INCLUDE); +"}" yybegin(INITIAL); +"$include"[ \t]*"(" yybegin(INCLUDE); [ \t]* /* skip whitespace */ -")" BEGIN(COMMENT); +")" yybegin(COMMENT); [ \t]* /* skip whitespace */ [^ \t\n() ]+ { /* get the include file name */ if ( include_count >= MAX_NEST){ @@ -43,7 +43,7 @@ int include_count = -1; yy_switch_to_buffer(yy_create_buffer(yyin,YY_BUF_SIZE)); - BEGIN(INITIAL); + yybegin(INITIAL); } <> { @@ -61,11 +61,11 @@ int include_count = -1; } else { yy_delete_buffer(include_stack[include_count--] ); yy_switch_to_buffer(include_stack[include_count] ); - BEGIN(INCLUDE); + yybegin(INCLUDE); } } -[a-z]+ ECHO; -.|\n ECHO; +[a-z]+ yyecho(); +.|\n yyecho(); diff --git a/examples/manual/pascal.lex b/examples/manual/pascal.lex index d406bbe0..df94313c 100644 --- a/examples/manual/pascal.lex +++ b/examples/manual/pascal.lex @@ -31,17 +31,17 @@ bad_string \'([^'\n]|\'\')+ %% -"{" BEGIN(COMMENT1); +"{" yybegin(COMMENT1); [^}\n]+ \n ++line_number; <> yyerror("EOF in comment"); -"}" BEGIN(INITIAL); +"}" yybegin(INITIAL); -"(*" BEGIN(COMMENT2); +"(*" yybegin(COMMENT2); [^)*\n]+ \n ++line_number; <> yyerror("EOF in comment"); -"*)" BEGIN(INITIAL); +"*)" yybegin(INITIAL); [*)] /* note that FILE and BEGIN are already diff --git a/examples/manual/reject.lex b/examples/manual/reject.lex index a7b817f2..9e0bbd8a 100644 --- a/examples/manual/reject.lex +++ b/examples/manual/reject.lex @@ -1,12 +1,12 @@ /* - * reject.lex: An example of REJECT and unput() + * reject.lex: An example of yyreject() and yyunput() * misuse. */ %% UNIX { - unput('U'); unput('N'); unput('G'); unput('\0'); - REJECT; + yyunput('U'); yyunput('N'); yyunput('G'); yyunput('\0'); + yyreject(); } GNU printf("GNU is Not Unix!\n"); %% diff --git a/examples/manual/replace.lex b/examples/manual/replace.lex index c5c8d877..a7062bc1 100644 --- a/examples/manual/replace.lex +++ b/examples/manual/replace.lex @@ -16,7 +16,7 @@ char upper_replace[1024]; "yy" printf("%s",lower_replace); "YY" printf("%s",upper_replace); -, ECHO; +, yyecho(); %% diff --git a/examples/manual/string1.lex b/examples/manual/string1.lex index b62ed88b..3a3276d1 100644 --- a/examples/manual/string1.lex +++ b/examples/manual/string1.lex @@ -1,5 +1,5 @@ /* - * string1.lex: Handling strings by using input() + * string1.lex: Handling strings by using yyinput() */ %{ @@ -27,13 +27,13 @@ void yyerror(char *message) buffer = malloc(ALLOC_SIZE); max_size = ALLOC_SIZE; - inch = input(); + inch = yyinput(); count = 0; while(inch != EOF && inch != '"' && inch != '\n'){ if(inch == '\\'){ - inch = input(); + inch = yyinput(); switch(inch){ - case '\n': inch = input(); break; + case '\n': inch = yyinput(); break; case 'b' : inch = '\b'; break; case 't' : inch = '\t'; break; case 'n' : inch = '\n'; break; @@ -41,10 +41,10 @@ void yyerror(char *message) case 'f' : inch = '\f'; break; case 'r' : inch = '\r'; break; case 'X' : - case 'x' : inch = input(); + case 'x' : inch = yyinput(); if(isxdigit(inch)){ temp = hextoint(toupper(inch)); - inch = input(); + inch = yyinput(); if(isxdigit(inch)){ temp = (temp << 4) + hextoint(toupper(inch)); } else { @@ -59,14 +59,14 @@ void yyerror(char *message) default: if(isodigit(inch)){ temp = inch - '0'; - inch = input(); + inch = yyinput(); if(isodigit(inch)){ temp = (temp << 3) + (inch - '0'); } else { unput(inch); goto done; } - inch = input(); + inch = yyinput(); if(isodigit(inch)){ temp = (temp << 3) + (inch - '0'); } else { @@ -82,7 +82,7 @@ void yyerror(char *message) buffer = realloc(buffer,max_size + ALLOC_SIZE); max_size += ALLOC_SIZE; } - inch = input(); + inch = yyinput(); } if(inch == EOF || inch == '\n'){ yyerror("Unterminated string."); diff --git a/examples/manual/string2.lex b/examples/manual/string2.lex index 2c9d35fb..f6d0c171 100644 --- a/examples/manual/string2.lex +++ b/examples/manual/string2.lex @@ -30,17 +30,17 @@ oct [0-7]{1,3} \" { buffer = malloc(1); buffer_size = 1; strcpy(buffer,""); - BEGIN(STRING); + yybegin(STRING); } \n { yyerror("Unterminated string"); free(buffer); - BEGIN(INITIAL); + yybegin(INITIAL); } <> { yyerror("EOF in string"); free(buffer); - BEGIN(INITIAL); + yybegin(INITIAL); } [^\\\n"] { buffer = realloc(buffer,buffer_size+yyleng+1); @@ -87,7 +87,7 @@ oct [0-7]{1,3} \" { printf("string = \"%s\"",buffer); free(buffer); - BEGIN(INITIAL); + yybegin(INITIAL); } %% diff --git a/examples/manual/unput.lex b/examples/manual/unput.lex index 161471a8..21b42dc4 100644 --- a/examples/manual/unput.lex +++ b/examples/manual/unput.lex @@ -1,6 +1,6 @@ /* * unput.l : An example of what *not* - * to do with unput(). + * to do with yyunput(). */ @@ -24,7 +24,7 @@ void putback_yytext(void) strcpy(buffer,yytext); printf("Got: %s\n",yytext); for(i=0; i[^\\\n"]* yymore(); -<> yyerror("EOF in string."); BEGIN(INITIAL); -\n yyerror("Unterminated string."); BEGIN(INITIAL); +<> yyerror("EOF in string."); yybegin(INITIAL); +\n yyerror("Unterminated string."); yybegin(INITIAL); \\\n yymore(); \" { yytext[yyleng-1] = '\0'; - printf("string = \"%s\"",yytext); BEGIN(INITIAL); + printf("string = \"%s\"",yytext); yybegin(INITIAL); } %% diff --git a/examples/manual/yymore2.lex b/examples/manual/yymore2.lex index f49ea237..dbe49153 100644 --- a/examples/manual/yymore2.lex +++ b/examples/manual/yymore2.lex @@ -16,11 +16,11 @@ void yyerror(char *message) %x STRING %% -\" BEGIN(STRING); +\" yybegin(STRING); [^\\\n"]* yymore(); -<> yyerror("EOF in string."); BEGIN(INITIAL); -\n yyerror("Unterminated string."); BEGIN(INITIAL); +<> yyerror("EOF in string."); yybegin(INITIAL); +\n yyerror("Unterminated string."); yybegin(INITIAL); \\\n { bcopy(yytext,yytext+2,yyleng-2); yytext += 2; yyleng -= 2; @@ -28,6 +28,6 @@ void yyerror(char *message) } \" { yyleng -= 1; yytext[yyleng] = '\0'; - printf("string = \"%s\"",yytext); BEGIN(INITIAL); + printf("string = \"%s\"",yytext); yybegin(INITIAL); } %%