diff --git a/harbour/ChangeLog b/harbour/ChangeLog index 5d31c90c92..cdfc1f54a9 100644 --- a/harbour/ChangeLog +++ b/harbour/ChangeLog @@ -1,9 +1,16 @@ +2000-07-08 15:10 UTC+0800 Ron Pinkas + * source\compiler\simplex.c + * Improved handling of WhiteSpace, NewLine, and Eof. + + * source\compiler\harbour.slx + + Added rule to return only last CR in sequence of empty lines. + 2000-07-08 10:40 UTC+0100 Ryszard Glab *source/vm/garbage.c * fixed possible endless loop during the garbage collecting (this time for real :-) - + 2000-07-07 17:30 UTC+0800 Ron Pinkas - include\harbour.slx - source\compiler\harbourl.c diff --git a/harbour/source/compiler/harbour.slx b/harbour/source/compiler/harbour.slx index 0897b8fa57..cb16f29193 100644 --- a/harbour/source/compiler/harbour.slx +++ b/harbour/source/compiler/harbour.slx @@ -29,9 +29,7 @@ */ //#define SHOW_LEX_TOKENS -/* -#define DEBUG_LEX -*/ +//#define DEBUG_LEX #ifdef DEBUG_LEX #undef DEBUG_INFO(x) @@ -281,6 +279,9 @@ LANGUAGE_RULES_ARE { IF_SEQUENCE_IS( '^' , 0 , 0 , 0 ) REDUCE_TO( POWER , 0 ), IF_SEQUENCE_IS( '!' , 0 , 0 , 0 ) REDUCE_TO( NOT , 0 ), + /* Return only last CR in a sequence. */ + IF_SEQUENCE_IS( '\n' , '\n' , 0 , 0 ) REDUCE_TO( '\n' , 0 ), + /* Strong Types */ IF_SEQUENCE_IS( _AS_ , _ARRAY_ , 0 , 0 ) REDUCE_TO( AS_ARRAY , 0 ), IF_SEQUENCE_IS( _AS_ , _BLOCK_ , 0 , 0 ) REDUCE_TO( AS_BLOCK , 0 ), @@ -509,8 +510,8 @@ LANGUAGE_RULES_ARE { IF_SEQUENCE_IS( RETURN , INASSIGN , 0 , 0 ) REDUCE_TO( HB_RETURN_ID , INASSIGN ), IF_SEQUENCE_IS( RETURN , INC , '\n' , 0 ) REDUCE_TO( HB_RETURN_ID , _INC_CR ), IF_SEQUENCE_IS( RETURN , DEC , '\n' , 0 ) REDUCE_TO( HB_RETURN_ID , _DEC_CR ), - IF_SEQUENCE_IS( RETURN , INC , ';' , 0 ) REDUCE_TO( HB_RETURN_ID , _INC_CR ), - IF_SEQUENCE_IS( RETURN , DEC , ';' , 0 ) REDUCE_TO( HB_RETURN_ID , _DEC_CR ), + IF_SEQUENCE_IS( RETURN , INC , ';' , 0 ) REDUCE_TO( HB_RETURN_ID , _INC_CR ), + IF_SEQUENCE_IS( RETURN , DEC , ';' , 0 ) REDUCE_TO( HB_RETURN_ID , _DEC_CR ), IF_SEQUENCE_IS( RETURN , ALIASOP , 0 , 0 ) REDUCE_TO( HB_RETURN_ID , ALIASOP ), IF_SEQUENCE_IS( RETURN , PLUSEQ , 0 , 0 ) REDUCE_TO( HB_RETURN_ID , PLUSEQ ), IF_SEQUENCE_IS( RETURN , MINUSEQ , 0 , 0 ) REDUCE_TO( HB_RETURN_ID , MINUSEQ ), @@ -960,7 +961,7 @@ LANGUAGE_RULES_ARE { iTexts++;\ CONVERT_NUMBER();\ \ - DEBUG_INFO( printf( "Element \"%s\" is %i yylval.string = >%s<\n", sToken, iRet, yylval.string ) );\ + DEBUG_INFO( printf( "Element \"%s\" is %i\n", sToken, iRet ) );\ }\ } diff --git a/harbour/source/compiler/simplex.c b/harbour/source/compiler/simplex.c index dc4cfeb309..611b765187 100644 --- a/harbour/source/compiler/simplex.c +++ b/harbour/source/compiler/simplex.c @@ -84,7 +84,7 @@ typedef struct _LEX_PAIR int iToken; } LEX_PAIR; /* support structure for KEYS and WORDS */ -/* Abouve are NOT overidable !!! Need to precede the Language Definitions. */ +/* Above are NOT overidable !!! Need to precede the Language Definitions. */ /* --------------------------------------------------------------------------------- */ @@ -100,7 +100,7 @@ typedef struct _LEX_PAIR #define LEX_CASE(x) /* - * Can be used to indicate hoe many chars needed to qualify as an abbreviation for Key Words. + * Can be used to indicate how many chars needed to qualify as an abbreviation for Key Words. * #define LEX_ABBREVIATE_KEYS #define LEX_ABBREVIATE_WORDS @@ -172,7 +172,7 @@ static int iReduce = 0; /* yylex */ static char * tmpPtr; static char sToken[TOKEN_SIZE]; -static int iLen; +static int iLen = 0; static char chr, cPrev = 0; static int iKey, iWord, iMatch, iRemove, iScan, iWordLen, iPush, iTexts = 0; static char szLexBuffer[ YY_BUF_SIZE ]; @@ -191,7 +191,7 @@ char * yytext; int yyleng; /* NewLine Support. */ -static BOOL bNewLine = TRUE; +static BOOL bNewLine = TRUE, bStart = TRUE; static int iSelfs = (int) ( sizeof( aSelfs ) / LEX_WORD_SIZE ); static int iKeys = (int) ( sizeof( aKeys ) / LEX_WORD_SIZE ); @@ -212,19 +212,9 @@ int Reduce( int iToken, BOOL bReal ); #define IF_TOKEN_READY() if( iReturn ) #define IF_TOKEN_ON_HOLD() if( iHold ) #define REDUCE( x ) Reduce( (x), TRUE ) -#define RESET_LEX() { iLen = 0; iMatched = 0; iHold = 0; iReturn = 0; } +#define RESET_LEX() { iLen = 0; iMatched = 0; iHold = 0; iReturn = 0; bNewLine = TRUE; bStart = TRUE; } #define FORCE_REDUCE() Reduce( 0, TRUE ) - -#define CHECK_NEW_LINE() \ - if( bNewLine )\ - {\ - NEW_LINE_ACTION();\ - \ - iTexts = 0;\ - } - - #define HOLD_TOKEN(x, y) \ /* Last In, First Out */ \ { iRet = x ; aiHold[ iHold++ ] = iRet; \ @@ -259,10 +249,9 @@ int Reduce( int iToken, BOOL bReal ); tmpPtr = (char*) szNewLine; \ while ( *tmpPtr && chr != *tmpPtr ) \ { \ - tmpPtr++; \ + tmpPtr++; \ } \ - \ - /* NewLine Found. */\ + /* NewLine Found?. */\ if( *tmpPtr ) #define IF_BEGIN_PAIR(chr) \ @@ -329,13 +318,15 @@ int Reduce( int iToken, BOOL bReal ); /* Terminate current token and check it. */ \ sToken[ iLen ] = '\0'; \ \ + /* Last charcter read. */\ + chr = chrSelf;\ + \ goto CheckToken; \ } \ else \ { \ DEBUG_INFO( printf( "Reducing Self >%s<\n", sSelf ) ); \ \ - CHECK_NEW_LINE();\ bNewLine = FALSE;\ \ RETURN_TOKEN( REDUCE( aSelfs[iSelf].iToken ), sSelf ); \ @@ -401,7 +392,7 @@ int Reduce( int iToken, BOOL bReal ); iReturn--; \ iRet = aiReturn[iReturn]; \ \ - DEBUG_INFO( printf( "Returning Ready: %i yytext = \"%s\"\n", iRet, yytext ) ); \ + DEBUG_INFO( printf( "Returning Ready: %i\n", iRet ) ); \ \ LEX_RETURN(iRet, NULL); @@ -439,6 +430,17 @@ int Reduce( int iToken, BOOL bReal ); RESET_LEX();\ return -1; \ }\ + else if( iRet < 256 )\ + {\ + IF_NEWLINE( (char) iRet )\ + {\ + bNewLine = TRUE;\ + }\ + else\ + {\ + bNewLine = FALSE;\ + }\ + }\ \ DEBUG_INFO( printf( "Reducing Held: %i Pos: %i\n", iRet, iHold ) ); \ \ @@ -462,18 +464,10 @@ int Reduce( int iToken, BOOL bReal ); } \ else \ { \ - if( x < 256 ) \ - { \ - IF_NEWLINE( iRet ) \ - { \ - /* Signify NewLine. */ \ - bNewLine = TRUE; \ - } \ - } \ - \ - DEBUG_INFO( printf( "Returning: %i yytext = \"%s\"\n", x, yytext ) ); \ + DEBUG_INFO( printf( "Returning: %i\n", x ) ); \ \ INTERCEPT_ACTION(x)\ + \ return x; \ } @@ -825,6 +819,21 @@ int yylex( void /*YYSTYPE * yylval*/ ) } else { + if( iMatched ) + { + /* Returning Pending Rule Match Tokens without further tests. */ \ + iMatch = 0; \ + while( iMatch < iMatched ) \ + { \ + DEBUG_INFO( printf( "Returning Pending Match Tokens at: \n" ) ); \ + \ + aiReturn[ iReturn++ ] = aiMatched[ iMatch ]; \ + iMatch++; \ + } \ + iMatched = 0;\ + goto Start;\ + } + DEBUG_INFO( printf( "Returning: \n" ) ); return -1; @@ -848,13 +857,23 @@ int yylex( void /*YYSTYPE * yylval*/ ) IF_OMMIT(chr) { - /* No need to check whit space. */ - } - else - { - CHECK_SELF_CONTAINED(chr); + if ( iLen ) + { + /* Terminate current token and check it. */ + sToken[ iLen ] = '\0'; + + DEBUG_INFO( printf( "Token: \"%s\" Ommited: \'%c\'\n", sToken, chr ) ); + + goto CheckToken; + } + else + { + continue; + } } + CHECK_SELF_CONTAINED(chr); + /* Soft Pair Terminator ? */ if ( cTerm && chr == cTerm ) { @@ -927,6 +946,9 @@ int yylex( void /*YYSTYPE * yylval*/ ) /* Resetting. */ cTerm = 0; + /* Last charcter read. */ + chr = chrPair; + goto on_error; } /* Terminator found. */ @@ -942,6 +964,11 @@ int yylex( void /*YYSTYPE * yylval*/ ) /* TODO: Move Action to Pair Definition! */ yylval.string = hb_strdup( sPair ); + /* Last charcter read. */ + chr = chrPair; + + bNewLine = FALSE; + /* LITERAL */ RETURN_TOKEN( REDUCE( aPairs[ iPair ].iToken ), NULL ); } @@ -993,27 +1020,14 @@ int yylex( void /*YYSTYPE * yylval*/ ) { DEBUG_INFO( printf( "Reducing NewLine '%c'\n", chr ) ); + bNewLine = TRUE; + iTexts = 0; + NEW_LINE_ACTION(); + RETURN_TOKEN( REDUCE( (int) chr ), NULL ); } } - IF_OMMIT(chr) - { - if ( iLen ) - { - /* Terminate current token and check it. */ - sToken[ iLen ] = '\0'; - - DEBUG_INFO( printf( "Token: \"%s\" Ommited: \'%c\'\n", sToken, chr ) ); - - goto CheckToken; - } - else - { - continue; - } - } - IF_APPEND_DELIMITER( chr ) { /* Append and Terminate current token and check it. */ @@ -1043,7 +1057,6 @@ int yylex( void /*YYSTYPE * yylval*/ ) { DEBUG_INFO( printf( "Reducing Delimiter: '%c'\n", chr ) ); - CHECK_NEW_LINE(); bNewLine = FALSE; RETURN_TOKEN( REDUCE( (int) chr ), NULL ); @@ -1085,8 +1098,6 @@ int yylex( void /*YYSTYPE * yylval*/ ) CheckToken: { - CHECK_NEW_LINE() - #ifdef LEX_ABBREVIATE_KEYS iWordLen = strlen( sToken ); @@ -1107,14 +1118,16 @@ int yylex( void /*YYSTYPE * yylval*/ ) if( strcmp( sToken, aKeys[ iKey++ ].sWord ) == 0 ) #endif { - bNewLine = FALSE; - DEBUG_INFO( printf( "Reducing Key Word: %s\n", sToken ) ); + bNewLine = FALSE; + RETURN_TOKEN( REDUCE( aKeys[ iKey - 1 ].iToken ), sToken ); } } + bNewLine = FALSE; + #ifdef LEX_ABBREVIATE_WORDS iWordLen = strlen( sToken ); @@ -1133,17 +1146,13 @@ int yylex( void /*YYSTYPE * yylval*/ ) if( strcmp( sToken, aWords[ iWord++ ].sWord ) == 0 ) #endif { - bNewLine = FALSE; - DEBUG_INFO( printf( "Reducing Word: %s\n", sToken ) ); RETURN_TOKEN( REDUCE( aWords[ iWord - 1 ].iToken ), sToken ); } } - bNewLine = FALSE; - - DEBUG_INFO( printf( "Reducing Element: \"%s\" yytext = \"%s\"\n", sToken, yytext ) ); + DEBUG_INFO( printf( "Reducing Element: \"%s\"\n", sToken ) ); /* "Returns" the Token as iRet. */ ELEMENT_TOKEN( sToken )