mirror/flex
1
0
Fork 0
mirror of https://github.com/westes/flex.git synced 2026-01-26 07:37:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update all the examples to use the new API elements.

Browse Source 
Add a fully reentrant example.  And update to TODO file.
This commit is contained in:
Eric S. Raymond 2020-10-12 21:07:14 -04:00
parent 68711fc4a7
commit 8d0162b80a
22 changed files with 150 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
TODO
View File

@ -2,4 +2,8 @@ Things to be worked on:
* Tests for %option user-init, %option pre-action, %option post-action.
* integrate examples directory into tests so that normal testing
proves the examples are up to date.
* Do away with any need for -lfl by generating a default yywrap
if the user doesn't specify one.

56
doc/flex.texi
View File

@ -354,28 +354,7 @@ Here's another simple example:
@cindex counting characters and lines; reentrant
@example
@verbatim
%{
int num_lines = 0, num_chars = 0;
%}
%option reentrant
%%
\n ++num_lines; ++num_chars;
. ++num_chars;
%%
int main() {
yyscan_t scanner;
yylex_init ( &scanner );
yylex ( scanner );
yylex_destroy ( scanner );
printf( "# of lines = %d, # of chars = %d\n",
num_lines, num_chars );
}
@end verbatim
@verbatiminclude ../examples/manual/example_r.lex
@end example
If you have looked at older versions of the Flex nanual, you might
@ -383,20 +362,7 @@ have seen a version of the above example that looked more like this:
@cindex counting characters and lines; non-reentrant
@example
@verbatim
int num_lines = 0, num_chars = 0;
%%
\n ++num_lines; ++num_chars;
. ++num_chars;
%%
int main() {
yylex();
printf( "# of lines = %d, # of chars = %d\n",
num_lines, num_chars );
}
@end verbatim
@verbatiminclude ../examples/manual/example_nr.lex
@end example
Both versions count the number of characters and the number of lines in
@ -422,13 +388,24 @@ language other than the original C/C++ non-reentrancy is not even an
option.
This, it's a good idea to get used to using the reentrant interface
from the beginning of your Flex prgramming. This is so even though the
from the beginning of your Flex programming. This is so even though the
reentrant example above is a rather poor one; it avoids exposing the
scanner state in globals but creates globals of its own. There is a
mechanism for including user-defined fields in the scanner structure
which will be explained in detail at @xref{Extra Data}. For now,
consider this:
@example
@verbatiminclude ../examples/manual/example_er.lex
@end example
While it requires a bit more ceremony, several instances of this
scanner can be run concurrently without stepping on each others'
storage.
(The @code{%option noyywrap} in these examples is helpful in
making them run standalone, but does not change the behavior of the scsnner.)
A somewhat more complicated example:
@cindex Pascal-like language
@ -596,7 +573,8 @@ themselves.
A @code{%top} block is similar to a @samp{%@{} ... @samp{%@}} block, except
that the code in a @code{%top} block is relocated to the @emph{top} of the
generated file, before any flex definitions @footnote{Actually,
generated file, before any flex definitions @footnote{Actually, in the
C/C++ back end,
@code{yyIN_HEADER} is defined before the @samp{%top} block.}.
The @code{%top} block is useful when you want definitions to be
evaluated or certain files to be included before the generated code.
@ -1642,7 +1620,7 @@ condition remains unchanged; it does @emph{not} revert to
@cindex yywrap, default for
@cindex noyywrap, %option
@cindex %option noyywrapp
@cindex %option noyywrap
If you do not supply your own version of @code{yywrap()}, then you must
either use @code{%option noyywrap} (in which case the scanner behaves as
though @code{yywrap()} returned 1), or you must link with @samp{-lfl} to

1
examples/manual/Makefile.am
View File

@ -30,6 +30,7 @@ EXTRA_DIST = \
eof_test01.txt \
eof_test02.txt \
eof_test03.txt \
example_er.lex \
example_r.lex \
example_nr.lex \
expr.lex \

6
examples/manual/Makefile.examples
View File

@ -18,7 +18,7 @@ ALLOCA =
# DO NOT CHANGE ANYTHING FROM HERE ON !!!!!!!!!
#
############################################################
PATH = ${PATH}:/usr/local/bin
PATH := /usr/local/bin:${PATH}
all: expr front myname eof wc replace user_act string1\
string2 yymore numbers dates cat
@ -31,6 +31,10 @@ example_nr: example_nr.lex
$(LEX) example_nr.lex
$(CC) lex.yy.c -o example_nr
example_er: example_er.lex
$(LEX) example_er.lex
$(CC) lex.yy.c -o example_er
expr: expr.y expr.lex
$(YACC) expr.y
$(LEX) expr.lex

24
examples/manual/dates.lex
View File

@ -54,13 +54,13 @@ day_ext (st|nd|rd|th)?
/* the default is month-day-year */
<LONG>{day_of_the_week} strcpy(dow,yytext);
<LONG>{month} strcpy(month,yytext); BEGIN(DAY);
<LONG>{month} strcpy(month,yytext); yybegin(DAY);
/* handle the form: day-month-year */
<LONG>{nday}{day_ext} strcpy(day,yytext); BEGIN(DAY_FIRST);
<DAY_FIRST>{month} strcpy(month,yytext); BEGIN(LONG);
<DAY>{nday}{day_ext} strcpy(day,yytext); BEGIN(LONG);
<LONG>{nday}{day_ext} strcpy(day,yytext); yybegin(DAY_FIRST);
<DAY_FIRST>{month} strcpy(month,yytext); yybegin(LONG);
<DAY>{nday}{day_ext} strcpy(day,yytext); yybegin(LONG);
<LONG>{nyear}{year_ext} {
printf("Long:\n");
@ -75,15 +75,15 @@ day_ext (st|nd|rd|th)?
/* handle dates of the form: day-month-year */
<SHORT>{nday} strcpy(day,yytext); BEGIN(YEAR_LAST);
<YEAR_LAST>{nmonth} strcpy(month,yytext);BEGIN(YLMONTH);
<YLMONTH>{nyear} strcpy(year,yytext); BEGIN(SHORT);
<SHORT>{nday} strcpy(day,yytext); yybegin(YEAR_LAST);
<YEAR_LAST>{nmonth} strcpy(month,yytext);yybegin(YLMONTH);
<YLMONTH>{nyear} strcpy(year,yytext); yybegin(SHORT);
/* handle dates of the form: year-month-day */
<SHORT>{nyear} strcpy(year,yytext); BEGIN(YEAR_FIRST);
<YEAR_FIRST>{nmonth} strcpy(month,yytext);BEGIN(YFMONTH);
<YFMONTH>{nday} strcpy(day,yytext); BEGIN(SHORT);
<SHORT>{nyear} strcpy(year,yytext); yybegin(YEAR_FIRST);
<YEAR_FIRST>{nmonth} strcpy(month,yytext);yybegin(YFMONTH);
<YFMONTH>{nday} strcpy(day,yytext); yybegin(SHORT);
<SHORT>\n {
@ -96,8 +96,8 @@ day_ext (st|nd|rd|th)?
strcpy(month,"");
}
long\n BEGIN(LONG);
short\n BEGIN(SHORT);
long\n yybegin(LONG);
short\n yybegin(SHORT);
{skip}*
\n

8
examples/manual/eof_rules.lex
View File

@ -17,8 +17,8 @@ int include_count = -1;
%%
^"#include"[ \t]*\" BEGIN(INCLUDE);
<INCLUDE>\" BEGIN(INITIAL);
^"#include"[ \t]*\" yybegin(INCLUDE);
<INCLUDE>\" yybegin(INITIAL);
<INCLUDE>[^\"]+ { /* get the include file name */
if ( include_count >= MAX_NEST){
fprintf( stderr, "Too many include files" );
@ -35,7 +35,7 @@ int include_count = -1;
yy_switch_to_buffer(yy_create_buffer(yyin,YY_BUF_SIZE));
BEGIN(INITIAL);
yybegin(INITIAL);
}
<INCLUDE><<EOF>>
{
@ -48,7 +48,7 @@ int include_count = -1;
} else {
yy_delete_buffer(include_stack[include_count--] );
yy_switch_to_buffer(include_stack[include_count] );
BEGIN(INCLUDE);
yybegin(INCLUDE);
}
}
[a-z]+ ECHO;

35
examples/manual/example_er.lex Normal file
View File

@ -0,0 +1,35 @@
/* basic example, fully reentrant thread-safe version */
%{
struct stats {
int num_lines;
int num_chars;
};
%}
%option reentrant noyywrap
%option extra-type="struct stats"
%%
\n {
struct stats ns = yyget_extra(yyscanner);
++ns.num_lines; ++ns.num_chars;
yyset_extra(ns, yyscanner);
}
. {
struct stats ns = yyget_extra(yyscanner);
++ns.num_chars;
yyset_extra(ns, yyscanner);
}
%%
int main() {
yyscan_t scanner;
struct stats ns;
yylex_init ( &scanner );
yylex ( scanner );
ns = yyget_extra(scanner);
printf( "# of lines = %d, # of chars = %d\n",
ns.num_lines, ns.num_chars);
yylex_destroy ( scanner );
}

6
examples/manual/example_nr.lex
View File

@ -1,4 +1,8 @@
int num_lines = 0, num_chars = 0;
/* basic example - non-reentrant version */
%{
int num_lines = 0, num_chars = 0;
%}
%option noyywrap
%%
\n ++num_lines; ++num_chars;
. ++num_chars;

3
examples/manual/example_r.lex
View File

@ -1,7 +1,8 @@
/* basic example - flawed reentrant version with global */
%{
int num_lines = 0, num_chars = 0;
%}
%option reentrant
%option reentrant noyywrap
%%
\n ++num_lines; ++num_chars;
. ++num_chars;

3
examples/manual/front.lex
View File

@ -3,9 +3,6 @@
#include <string.h>
#include "y.tab.h" /* this comes from bison */
#define TRUE 1
#define FALSE 0
#define copy_and_return(token_type) { strcpy(yylval.name,yytext); \
return(token_type); }

44
examples/manual/j2t.lex
View File

@ -158,7 +158,7 @@ void write_block_header(char *type)
printf("\n\n@table @b\n");
}
"Examples:"[^\.]+ ECHO;
"Examples:"[^\.]+ yyecho();
"*"[^*\n]+"*" { /* @emph{}(emphasized) text */
yytext[yyleng-1] = '\0';
@ -205,12 +205,12 @@ void write_block_header(char *type)
yyless(loop+1);
statep++;
states[statep] = EXAMPLE2;
BEGIN(EXAMPLE2);
yybegin(EXAMPLE2);
}
<EXAMPLE,EXAMPLE2>^\n {
printf("@end example\n\n");
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
/*
@ -231,7 +231,7 @@ void write_block_header(char *type)
yyless(loop);
statep++;
states[statep] = ENUM;
BEGIN(ENUM);
yybegin(ENUM);
}
<ENUM>"@" printf("@@");
@ -239,7 +239,7 @@ void write_block_header(char *type)
printf(":\n\n@example\n");
statep++;
states[statep] = EXAMPLE;
BEGIN(EXAMPLE);
yybegin(EXAMPLE);
}
@ -250,7 +250,7 @@ void write_block_header(char *type)
<ENUM>\n\n\n[ \t]+[^0-9] {
printf("\n\n@end enumerate\n\n");
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
/*
@ -265,7 +265,7 @@ void write_block_header(char *type)
yyless(2);
statep++;
states[statep] = LITEM2;
BEGIN(LITEM2);
yybegin(LITEM2);
}
<LITEM2>^":".+":" {
(void)check_and_convert(&yytext[1]);
@ -275,9 +275,9 @@ void write_block_header(char *type)
<LITEM2>\n\n\n+[^:\n] {
printf("\n\n@end itemize\n\n");
ECHO;
yyecho();
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
/*
@ -300,7 +300,7 @@ void write_block_header(char *type)
yyless(loop);
statep++;
states[statep] = LITEM;
BEGIN(LITEM);
yybegin(LITEM);
}
<LITEM>^.+":" {
(void)check_and_convert(yytext);
@ -318,7 +318,7 @@ void write_block_header(char *type)
printf("@end itemize\n\n");
printf("%s",&buffer[loop+1]);
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
/*
@ -338,27 +338,27 @@ void write_block_header(char *type)
yyless((len-loop)+2);
statep++;
states[statep] = BITEM;
BEGIN(BITEM);
yybegin(BITEM);
}
<BITEM>^" "*"*" {
printf("@item");
statep++;
states[statep] = BITEM_ITEM;
BEGIN(BITEM_ITEM);
yybegin(BITEM_ITEM);
}
<BITEM>"@" printf("@@");
<BITEM>^\n {
printf("@end itemize\n\n");
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
<BITEM_ITEM>[^\:]* {
printf(" @b{%s}\n\n",check_and_convert(yytext));
}
<BITEM_ITEM>":" {
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
/*
@ -369,13 +369,13 @@ void write_block_header(char *type)
(void)check_and_convert(&yytext[1]);
statep++;
states[statep] = HEADING;
BEGIN(HEADING);
yybegin(HEADING);
}
<HEADING>:[^\n] {
printf("@item @b{%s}\n",buffer);
write_underline(strlen(buffer),6,'~');
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
<HEADING>:\n"*"* {
if(need_closing == TRUE){
@ -385,7 +385,7 @@ void write_block_header(char *type)
printf("@chapter %s\n",buffer);
write_underline(strlen(buffer),9,'*');
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
<HEADING>:\n"="* {
if(need_closing == TRUE){
@ -395,7 +395,7 @@ void write_block_header(char *type)
printf("@section %s\n",buffer);
write_underline(strlen(buffer),9,'=');
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
<HEADING>"@" printf("@@");
<HEADING>:\n"-"* {
@ -406,7 +406,7 @@ void write_block_header(char *type)
printf("@subsection %s\n",buffer);
write_underline(strlen(buffer),12,'-');
statep--;
BEGIN(states[statep]);
yybegin(states[statep]);
}
/*
@ -417,10 +417,10 @@ void write_block_header(char *type)
printf("@example\n");
statep++;
states[statep] = EXAMPLE;
BEGIN(EXAMPLE);
yybegin(EXAMPLE);
}
<EXAMPLE>^" "
. ECHO;
. yyecho();
%%

16
examples/manual/pas_include.lex
View File

@ -19,13 +19,13 @@ int include_count = -1;
%%
"{" BEGIN(COMMENT);
"{" yybegin(COMMENT);
<COMMENT>"}" BEGIN(INITIAL);
<COMMENT>"$include"[ \t]*"(" BEGIN(INCLUDE);
<COMMENT>"}" yybegin(INITIAL);
<COMMENT>"$include"[ \t]*"(" yybegin(INCLUDE);
<COMMENT>[ \t]* /* skip whitespace */
<INCLUDE>")" BEGIN(COMMENT);
<INCLUDE>")" yybegin(COMMENT);
<INCLUDE>[ \t]* /* skip whitespace */
<INCLUDE>[^ \t\n() ]+ { /* get the include file name */
if ( include_count >= MAX_NEST){
@ -43,7 +43,7 @@ int include_count = -1;
yy_switch_to_buffer(yy_create_buffer(yyin,YY_BUF_SIZE));
BEGIN(INITIAL);
yybegin(INITIAL);
}
<INCLUDE><<EOF>>
{
@ -61,11 +61,11 @@ int include_count = -1;
} else {
yy_delete_buffer(include_stack[include_count--] );
yy_switch_to_buffer(include_stack[include_count] );
BEGIN(INCLUDE);
yybegin(INCLUDE);
}
}
[a-z]+ ECHO;
.|\n ECHO;
[a-z]+ yyecho();
.|\n yyecho();

8
examples/manual/pascal.lex
View File

@ -31,17 +31,17 @@ bad_string \'([^'\n]|\'\')+
%%
"{" BEGIN(COMMENT1);
"{" yybegin(COMMENT1);
<COMMENT1>[^}\n]+
<COMMENT1>\n ++line_number;
<COMMENT1><<EOF>> yyerror("EOF in comment");
<COMMENT1>"}" BEGIN(INITIAL);
<COMMENT1>"}" yybegin(INITIAL);
"(*" BEGIN(COMMENT2);
"(*" yybegin(COMMENT2);
<COMMENT2>[^)*\n]+
<COMMENT2>\n ++line_number;
<COMMENT2><<EOF>> yyerror("EOF in comment");
<COMMENT2>"*)" BEGIN(INITIAL);
<COMMENT2>"*)" yybegin(INITIAL);
<COMMENT2>[*)]
/* note that FILE and BEGIN are already

6
examples/manual/reject.lex
View File

@ -1,12 +1,12 @@
/*
* reject.lex: An example of REJECT and unput()
* reject.lex: An example of yyreject() and yyunput()
* misuse.
*/
%%
UNIX {
unput('U'); unput('N'); unput('G'); unput('\0');
REJECT;
yyunput('U'); yyunput('N'); yyunput('G'); yyunput('\0');
yyreject();
}
GNU printf("GNU is Not Unix!\n");
%%

2
examples/manual/replace.lex
View File

@ -16,7 +16,7 @@ char upper_replace[1024];
"yy" printf("%s",lower_replace);
"YY" printf("%s",upper_replace);
, ECHO;
, yyecho();
%%

18
examples/manual/string1.lex
View File

@ -1,5 +1,5 @@
/*
* string1.lex: Handling strings by using input()
* string1.lex: Handling strings by using yyinput()
*/
%{
@ -27,13 +27,13 @@ void yyerror(char *message)
buffer = malloc(ALLOC_SIZE);
max_size = ALLOC_SIZE;
inch = input();
inch = yyinput();
count = 0;
while(inch != EOF && inch != '"' && inch != '\n'){
if(inch == '\\'){
inch = input();
inch = yyinput();
switch(inch){
case '\n': inch = input(); break;
case '\n': inch = yyinput(); break;
case 'b' : inch = '\b'; break;
case 't' : inch = '\t'; break;
case 'n' : inch = '\n'; break;
@ -41,10 +41,10 @@ void yyerror(char *message)
case 'f' : inch = '\f'; break;
case 'r' : inch = '\r'; break;
case 'X' :
case 'x' : inch = input();
case 'x' : inch = yyinput();
if(isxdigit(inch)){
temp = hextoint(toupper(inch));
inch = input();
inch = yyinput();
if(isxdigit(inch)){
temp = (temp << 4) + hextoint(toupper(inch));
} else {
@ -59,14 +59,14 @@ void yyerror(char *message)
default:
if(isodigit(inch)){
temp = inch - '0';
inch = input();
inch = yyinput();
if(isodigit(inch)){
temp = (temp << 3) + (inch - '0');
} else {
unput(inch);
goto done;
}
inch = input();
inch = yyinput();
if(isodigit(inch)){
temp = (temp << 3) + (inch - '0');
} else {
@ -82,7 +82,7 @@ void yyerror(char *message)
buffer = realloc(buffer,max_size + ALLOC_SIZE);
max_size += ALLOC_SIZE;
}
inch = input();
inch = yyinput();
}
if(inch == EOF || inch == '\n'){
yyerror("Unterminated string.");

8
examples/manual/string2.lex
View File

@ -30,17 +30,17 @@ oct [0-7]{1,3}
\" {
buffer = malloc(1);
buffer_size = 1; strcpy(buffer,"");
BEGIN(STRING);
yybegin(STRING);
}
<STRING>\n {
yyerror("Unterminated string");
free(buffer);
BEGIN(INITIAL);
yybegin(INITIAL);
}
<STRING><<EOF>> {
yyerror("EOF in string");
free(buffer);
BEGIN(INITIAL);
yybegin(INITIAL);
}
<STRING>[^\\\n"] {
buffer = realloc(buffer,buffer_size+yyleng+1);
@ -87,7 +87,7 @@ oct [0-7]{1,3}
<STRING>\" {
printf("string = \"%s\"",buffer);
free(buffer);
BEGIN(INITIAL);
yybegin(INITIAL);
}
%%

4
examples/manual/unput.lex
View File

@ -1,6 +1,6 @@
/*
* unput.l : An example of what *not*
* to do with unput().
* to do with yyunput().
*/
@ -24,7 +24,7 @@ void putback_yytext(void)
strcpy(buffer,yytext);
printf("Got: %s\n",yytext);
for(i=0; i<l; i++){
unput(buffer[i]);
yyunput(buffer[i]);
}
}

7
examples/manual/user_act.lex
View File

@ -4,14 +4,13 @@
void user_action(void);
#define YY_USER_ACTION user_action();
%}
%option pre-action = "user_action();"
%%
.* ECHO;
\n ECHO;
.* yyecho();
\n yyecho();
%%

3
examples/manual/userinit.lex
View File

@ -1,6 +1,4 @@
%{
#define YY_USER_INIT open_input_file()
extern FILE *yyin;
void open_input_file(void)
@ -27,4 +25,5 @@ void open_input_file(void)
}
%}
%option user-init = "open_input_file();"
%%

8
examples/manual/yymore.lex
View File

@ -16,14 +16,14 @@ void yyerror(char *message)
%x STRING
%%
\" BEGIN(STRING);
\" yybegin(STRING);
<STRING>[^\\\n"]* yymore();
<STRING><<EOF>> yyerror("EOF in string."); BEGIN(INITIAL);
<STRING>\n yyerror("Unterminated string."); BEGIN(INITIAL);
<STRING><<EOF>> yyerror("EOF in string."); yybegin(INITIAL);
<STRING>\n yyerror("Unterminated string."); yybegin(INITIAL);
<STRING>\\\n yymore();
<STRING>\" {
yytext[yyleng-1] = '\0';
printf("string = \"%s\"",yytext); BEGIN(INITIAL);
printf("string = \"%s\"",yytext); yybegin(INITIAL);
}
%%

8
examples/manual/yymore2.lex
View File

@ -16,11 +16,11 @@ void yyerror(char *message)
%x STRING
%%
\" BEGIN(STRING);
\" yybegin(STRING);
<STRING>[^\\\n"]* yymore();
<STRING><<EOF>> yyerror("EOF in string."); BEGIN(INITIAL);
<STRING>\n yyerror("Unterminated string."); BEGIN(INITIAL);
<STRING><<EOF>> yyerror("EOF in string."); yybegin(INITIAL);
<STRING>\n yyerror("Unterminated string."); yybegin(INITIAL);
<STRING>\\\n {
bcopy(yytext,yytext+2,yyleng-2);
yytext += 2; yyleng -= 2;
@ -28,6 +28,6 @@ void yyerror(char *message)
}
<STRING>\" {
yyleng -= 1; yytext[yyleng] = '\0';
printf("string = \"%s\"",yytext); BEGIN(INITIAL);
printf("string = \"%s\"",yytext); yybegin(INITIAL);
}
%%